arch/mips/kvm/emulate.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * KVM/MIPS: Instruction/Exception emulation
   7  *
   8  * Copyright (C) 2012  MIPS Technologies, Inc.  All rights reserved.
   9  * Authors: Sanjay Lal <sanjayl@kymasys.com>
  10  */
  11
  12 #include <linux/errno.h>
  13 #include <linux/err.h>
  14 #include <linux/ktime.h>
  15 #include <linux/kvm_host.h>
  16 #include <linux/vmalloc.h>
  17 #include <linux/fs.h>
  18 #include <linux/bootmem.h>
  19 #include <linux/random.h>
  20 #include <asm/page.h>
  21 #include <asm/cacheflush.h>
  22 #include <asm/cacheops.h>
  23 #include <asm/cpu-info.h>
  24 #include <asm/mmu_context.h>
  25 #include <asm/tlbflush.h>
  26 #include <asm/inst.h>
  27
  28 #undef CONFIG_MIPS_MT
  29 #include <asm/r4kcache.h>
  30 #define CONFIG_MIPS_MT
  31
  32 #include "interrupt.h"
  33 #include "commpage.h"
  34
  35 #include "trace.h"
  36
  37 /*
  38  * Compute the return address and do emulate branch simulation, if required.
  39  * This function should be called only in branch delay slot active.
  40  */
  41 static int kvm_compute_return_epc(struct kvm_vcpu *vcpu, unsigned long instpc,
  42                                   unsigned long *out)
  43 {
  44         unsigned int dspcontrol;
  45         union mips_instruction insn;
  46         struct kvm_vcpu_arch *arch = &vcpu->arch;
  47         long epc = instpc;
  48         long nextpc;
  49         int err;
  50
  51         if (epc & 3) {
  52                 kvm_err("%s: unaligned epc\n", __func__);
  53                 return -EINVAL;
  54         }
  55
  56         /* Read the instruction */
  57         err = kvm_get_badinstrp((u32 *)epc, vcpu, &insn.word);
  58         if (err)
  59                 return err;
  60
  61         switch (insn.i_format.opcode) {
  62                 /* jr and jalr are in r_format format. */
  63         case spec_op:
  64                 switch (insn.r_format.func) {
  65                 case jalr_op:
  66                         arch->gprs[insn.r_format.rd] = epc + 8;
  67                         /* Fall through */
  68                 case jr_op:
  69                         nextpc = arch->gprs[insn.r_format.rs];
  70                         break;
  71                 default:
  72                         return -EINVAL;
  73                 }
  74                 break;
  75
  76                 /*
  77                  * This group contains:
  78                  * bltz_op, bgez_op, bltzl_op, bgezl_op,
  79                  * bltzal_op, bgezal_op, bltzall_op, bgezall_op.
  80                  */
  81         case bcond_op:
  82                 switch (insn.i_format.rt) {
  83                 case bltz_op:
  84                 case bltzl_op:
  85                         if ((long)arch->gprs[insn.i_format.rs] < 0)
  86                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  87                         else
  88                                 epc += 8;
  89                         nextpc = epc;
  90                         break;
  91
  92                 case bgez_op:
  93                 case bgezl_op:
  94                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
  95                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
  96                         else
  97                                 epc += 8;
  98                         nextpc = epc;
  99                         break;
 100
 101                 case bltzal_op:
 102                 case bltzall_op:
 103                         arch->gprs[31] = epc + 8;
 104                         if ((long)arch->gprs[insn.i_format.rs] < 0)
 105                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 106                         else
 107                                 epc += 8;
 108                         nextpc = epc;
 109                         break;
 110
 111                 case bgezal_op:
 112                 case bgezall_op:
 113                         arch->gprs[31] = epc + 8;
 114                         if ((long)arch->gprs[insn.i_format.rs] >= 0)
 115                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 116                         else
 117                                 epc += 8;
 118                         nextpc = epc;
 119                         break;
 120                 case bposge32_op:
 121                         if (!cpu_has_dsp) {
 122                                 kvm_err("%s: DSP branch but not DSP ASE\n",
 123                                         __func__);
 124                                 return -EINVAL;
 125                         }
 126
 127                         dspcontrol = rddsp(0x01);
 128
 129                         if (dspcontrol >= 32)
 130                                 epc = epc + 4 + (insn.i_format.simmediate << 2);
 131                         else
 132                                 epc += 8;
 133                         nextpc = epc;
 134                         break;
 135                 default:
 136                         return -EINVAL;
 137                 }
 138                 break;
 139
 140                 /* These are unconditional and in j_format. */
 141         case jal_op:
 142                 arch->gprs[31] = instpc + 8;
 143         case j_op:
 144                 epc += 4;
 145                 epc >>= 28;
 146                 epc <<= 28;
 147                 epc |= (insn.j_format.target << 2);
 148                 nextpc = epc;
 149                 break;
 150
 151                 /* These are conditional and in i_format. */
 152         case beq_op:
 153         case beql_op:
 154                 if (arch->gprs[insn.i_format.rs] ==
 155                     arch->gprs[insn.i_format.rt])
 156                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 157                 else
 158                         epc += 8;
 159                 nextpc = epc;
 160                 break;
 161
 162         case bne_op:
 163         case bnel_op:
 164                 if (arch->gprs[insn.i_format.rs] !=
 165                     arch->gprs[insn.i_format.rt])
 166                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 167                 else
 168                         epc += 8;
 169                 nextpc = epc;
 170                 break;
 171
 172         case blez_op:   /* POP06 */
 173 #ifndef CONFIG_CPU_MIPSR6
 174         case blezl_op:  /* removed in R6 */
 175 #endif
 176                 if (insn.i_format.rt != 0)
 177                         goto compact_branch;
 178                 if ((long)arch->gprs[insn.i_format.rs] <= 0)
 179                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 180                 else
 181                         epc += 8;
 182                 nextpc = epc;
 183                 break;
 184
 185         case bgtz_op:   /* POP07 */
 186 #ifndef CONFIG_CPU_MIPSR6
 187         case bgtzl_op:  /* removed in R6 */
 188 #endif
 189                 if (insn.i_format.rt != 0)
 190                         goto compact_branch;
 191                 if ((long)arch->gprs[insn.i_format.rs] > 0)
 192                         epc = epc + 4 + (insn.i_format.simmediate << 2);
 193                 else
 194                         epc += 8;
 195                 nextpc = epc;
 196                 break;
 197
 198                 /* And now the FPA/cp1 branch instructions. */
 199         case cop1_op:
 200                 kvm_err("%s: unsupported cop1_op\n", __func__);
 201                 return -EINVAL;
 202
 203 #ifdef CONFIG_CPU_MIPSR6
 204         /* R6 added the following compact branches with forbidden slots */
 205         case blezl_op:  /* POP26 */
 206         case bgtzl_op:  /* POP27 */
 207                 /* only rt == 0 isn't compact branch */
 208                 if (insn.i_format.rt != 0)
 209                         goto compact_branch;
 210                 return -EINVAL;
 211         case pop10_op:
 212         case pop30_op:
 213                 /* only rs == rt == 0 is reserved, rest are compact branches */
 214                 if (insn.i_format.rs != 0 || insn.i_format.rt != 0)
 215                         goto compact_branch;
 216                 return -EINVAL;
 217         case pop66_op:
 218         case pop76_op:
 219                 /* only rs == 0 isn't compact branch */
 220                 if (insn.i_format.rs != 0)
 221                         goto compact_branch;
 222                 return -EINVAL;
 223 compact_branch:
 224                 /*
 225                  * If we've hit an exception on the forbidden slot, then
 226                  * the branch must not have been taken.
 227                  */
 228                 epc += 8;
 229                 nextpc = epc;
 230                 break;
 231 #else
 232 compact_branch:
 233                 /* Fall through - Compact branches not supported before R6 */
 234 #endif
 235         default:
 236                 return -EINVAL;
 237         }
 238
 239         *out = nextpc;
 240         return 0;
 241 }
 242
 243 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause)
 244 {
 245         int err;
 246
 247         if (cause & CAUSEF_BD) {
 248                 err = kvm_compute_return_epc(vcpu, vcpu->arch.pc,
 249                                              &vcpu->arch.pc);
 250                 if (err)
 251                         return EMULATE_FAIL;
 252         } else {
 253                 vcpu->arch.pc += 4;
 254         }
 255
 256         kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc);
 257
 258         return EMULATE_DONE;
 259 }
 260
 261 /**
 262  * kvm_get_badinstr() - Get bad instruction encoding.
 263  * @opc:        Guest pointer to faulting instruction.
 264  * @vcpu:       KVM VCPU information.
 265  *
 266  * Gets the instruction encoding of the faulting instruction, using the saved
 267  * BadInstr register value if it exists, otherwise falling back to reading guest
 268  * memory at @opc.
 269  *
 270  * Returns:     The instruction encoding of the faulting instruction.
 271  */
 272 int kvm_get_badinstr(u32 *opc, struct kvm_vcpu *vcpu, u32 *out)
 273 {
 274         if (cpu_has_badinstr) {
 275                 *out = vcpu->arch.host_cp0_badinstr;
 276                 return 0;
 277         } else {
 278                 return kvm_get_inst(opc, vcpu, out);
 279         }
 280 }
 281
 282 /**
 283  * kvm_get_badinstrp() - Get bad prior instruction encoding.
 284  * @opc:        Guest pointer to prior faulting instruction.
 285  * @vcpu:       KVM VCPU information.
 286  *
 287  * Gets the instruction encoding of the prior faulting instruction (the branch
 288  * containing the delay slot which faulted), using the saved BadInstrP register
 289  * value if it exists, otherwise falling back to reading guest memory at @opc.
 290  *
 291  * Returns:     The instruction encoding of the prior faulting instruction.
 292  */
 293 int kvm_get_badinstrp(u32 *opc, struct kvm_vcpu *vcpu, u32 *out)
 294 {
 295         if (cpu_has_badinstrp) {
 296                 *out = vcpu->arch.host_cp0_badinstrp;
 297                 return 0;
 298         } else {
 299                 return kvm_get_inst(opc, vcpu, out);
 300         }
 301 }
 302
 303 /**
 304  * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled.
 305  * @vcpu:       Virtual CPU.
 306  *
 307  * Returns:     1 if the CP0_Count timer is disabled by either the guest
 308  *              CP0_Cause.DC bit or the count_ctl.DC bit.
 309  *              0 otherwise (in which case CP0_Count timer is running).
 310  */
 311 static inline int kvm_mips_count_disabled(struct kvm_vcpu *vcpu)
 312 {
 313         struct mips_coproc *cop0 = vcpu->arch.cop0;
 314
 315         return  (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) ||
 316                 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC);
 317 }
 318
 319 /**
 320  * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count.
 321  *
 322  * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias.
 323  *
 324  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 325  */
 326 static u32 kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now)
 327 {
 328         s64 now_ns, periods;
 329         u64 delta;
 330
 331         now_ns = ktime_to_ns(now);
 332         delta = now_ns + vcpu->arch.count_dyn_bias;
 333
 334         if (delta >= vcpu->arch.count_period) {
 335                 /* If delta is out of safe range the bias needs adjusting */
 336                 periods = div64_s64(now_ns, vcpu->arch.count_period);
 337                 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period;
 338                 /* Recalculate delta with new bias */
 339                 delta = now_ns + vcpu->arch.count_dyn_bias;
 340         }
 341
 342         /*
 343          * We've ensured that:
 344          *   delta < count_period
 345          *
 346          * Therefore the intermediate delta*count_hz will never overflow since
 347          * at the boundary condition:
 348          *   delta = count_period
 349          *   delta = NSEC_PER_SEC * 2^32 / count_hz
 350          *   delta * count_hz = NSEC_PER_SEC * 2^32
 351          */
 352         return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC);
 353 }
 354
 355 /**
 356  * kvm_mips_count_time() - Get effective current time.
 357  * @vcpu:       Virtual CPU.
 358  *
 359  * Get effective monotonic ktime. This is usually a straightforward ktime_get(),
 360  * except when the master disable bit is set in count_ctl, in which case it is
 361  * count_resume, i.e. the time that the count was disabled.
 362  *
 363  * Returns:     Effective monotonic ktime for CP0_Count.
 364  */
 365 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu)
 366 {
 367         if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 368                 return vcpu->arch.count_resume;
 369
 370         return ktime_get();
 371 }
 372
 373 /**
 374  * kvm_mips_read_count_running() - Read the current count value as if running.
 375  * @vcpu:       Virtual CPU.
 376  * @now:        Kernel time to read CP0_Count at.
 377  *
 378  * Returns the current guest CP0_Count register at time @now and handles if the
 379  * timer interrupt is pending and hasn't been handled yet.
 380  *
 381  * Returns:     The current value of the guest CP0_Count register.
 382  */
 383 static u32 kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now)
 384 {
 385         struct mips_coproc *cop0 = vcpu->arch.cop0;
 386         ktime_t expires, threshold;
 387         u32 count, compare;
 388         int running;
 389
 390         /* Calculate the biased and scaled guest CP0_Count */
 391         count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now);
 392         compare = kvm_read_c0_guest_compare(cop0);
 393
 394         /*
 395          * Find whether CP0_Count has reached the closest timer interrupt. If
 396          * not, we shouldn't inject it.
 397          */
 398         if ((s32)(count - compare) < 0)
 399                 return count;
 400
 401         /*
 402          * The CP0_Count we're going to return has already reached the closest
 403          * timer interrupt. Quickly check if it really is a new interrupt by
 404          * looking at whether the interval until the hrtimer expiry time is
 405          * less than 1/4 of the timer period.
 406          */
 407         expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer);
 408         threshold = ktime_add_ns(now, vcpu->arch.count_period / 4);
 409         if (ktime_before(expires, threshold)) {
 410                 /*
 411                  * Cancel it while we handle it so there's no chance of
 412                  * interference with the timeout handler.
 413                  */
 414                 running = hrtimer_cancel(&vcpu->arch.comparecount_timer);
 415
 416                 /* Nothing should be waiting on the timeout */
 417                 kvm_mips_callbacks->queue_timer_int(vcpu);
 418
 419                 /*
 420                  * Restart the timer if it was running based on the expiry time
 421                  * we read, so that we don't push it back 2 periods.
 422                  */
 423                 if (running) {
 424                         expires = ktime_add_ns(expires,
 425                                                vcpu->arch.count_period);
 426                         hrtimer_start(&vcpu->arch.comparecount_timer, expires,
 427                                       HRTIMER_MODE_ABS);
 428                 }
 429         }
 430
 431         return count;
 432 }
 433
 434 /**
 435  * kvm_mips_read_count() - Read the current count value.
 436  * @vcpu:       Virtual CPU.
 437  *
 438  * Read the current guest CP0_Count value, taking into account whether the timer
 439  * is stopped.
 440  *
 441  * Returns:     The current guest CP0_Count value.
 442  */
 443 u32 kvm_mips_read_count(struct kvm_vcpu *vcpu)
 444 {
 445         struct mips_coproc *cop0 = vcpu->arch.cop0;
 446
 447         /* If count disabled just read static copy of count */
 448         if (kvm_mips_count_disabled(vcpu))
 449                 return kvm_read_c0_guest_count(cop0);
 450
 451         return kvm_mips_read_count_running(vcpu, ktime_get());
 452 }
 453
 454 /**
 455  * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer.
 456  * @vcpu:       Virtual CPU.
 457  * @count:      Output pointer for CP0_Count value at point of freeze.
 458  *
 459  * Freeze the hrtimer safely and return both the ktime and the CP0_Count value
 460  * at the point it was frozen. It is guaranteed that any pending interrupts at
 461  * the point it was frozen are handled, and none after that point.
 462  *
 463  * This is useful where the time/CP0_Count is needed in the calculation of the
 464  * new parameters.
 465  *
 466  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 467  *
 468  * Returns:     The ktime at the point of freeze.
 469  */
 470 static ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count)
 471 {
 472         ktime_t now;
 473
 474         /* stop hrtimer before finding time */
 475         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 476         now = ktime_get();
 477
 478         /* find count at this point and handle pending hrtimer */
 479         *count = kvm_mips_read_count_running(vcpu, now);
 480
 481         return now;
 482 }
 483
 484 /**
 485  * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry.
 486  * @vcpu:       Virtual CPU.
 487  * @now:        ktime at point of resume.
 488  * @count:      CP0_Count at point of resume.
 489  *
 490  * Resumes the timer and updates the timer expiry based on @now and @count.
 491  * This can be used in conjunction with kvm_mips_freeze_timer() when timer
 492  * parameters need to be changed.
 493  *
 494  * It is guaranteed that a timer interrupt immediately after resume will be
 495  * handled, but not if CP_Compare is exactly at @count. That case is already
 496  * handled by kvm_mips_freeze_timer().
 497  *
 498  * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running).
 499  */
 500 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu,
 501                                     ktime_t now, u32 count)
 502 {
 503         struct mips_coproc *cop0 = vcpu->arch.cop0;
 504         u32 compare;
 505         u64 delta;
 506         ktime_t expire;
 507
 508         /* Calculate timeout (wrap 0 to 2^32) */
 509         compare = kvm_read_c0_guest_compare(cop0);
 510         delta = (u64)(u32)(compare - count - 1) + 1;
 511         delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz);
 512         expire = ktime_add_ns(now, delta);
 513
 514         /* Update hrtimer to use new timeout */
 515         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 516         hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS);
 517 }
 518
 519 /**
 520  * kvm_mips_write_count() - Modify the count and update timer.
 521  * @vcpu:       Virtual CPU.
 522  * @count:      Guest CP0_Count value to set.
 523  *
 524  * Sets the CP0_Count value and updates the timer accordingly.
 525  */
 526 void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count)
 527 {
 528         struct mips_coproc *cop0 = vcpu->arch.cop0;
 529         ktime_t now;
 530
 531         /* Calculate bias */
 532         now = kvm_mips_count_time(vcpu);
 533         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 534
 535         if (kvm_mips_count_disabled(vcpu))
 536                 /* The timer's disabled, adjust the static count */
 537                 kvm_write_c0_guest_count(cop0, count);
 538         else
 539                 /* Update timeout */
 540                 kvm_mips_resume_hrtimer(vcpu, now, count);
 541 }
 542
 543 /**
 544  * kvm_mips_init_count() - Initialise timer.
 545  * @vcpu:       Virtual CPU.
 546  *
 547  * Initialise the timer to a sensible frequency, namely 100MHz, zero it, and set
 548  * it going if it's enabled.
 549  */
 550 void kvm_mips_init_count(struct kvm_vcpu *vcpu)
 551 {
 552         /* 100 MHz */
 553         vcpu->arch.count_hz = 100*1000*1000;
 554         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32,
 555                                           vcpu->arch.count_hz);
 556         vcpu->arch.count_dyn_bias = 0;
 557
 558         /* Starting at 0 */
 559         kvm_mips_write_count(vcpu, 0);
 560 }
 561
 562 /**
 563  * kvm_mips_set_count_hz() - Update the frequency of the timer.
 564  * @vcpu:       Virtual CPU.
 565  * @count_hz:   Frequency of CP0_Count timer in Hz.
 566  *
 567  * Change the frequency of the CP0_Count timer. This is done atomically so that
 568  * CP0_Count is continuous and no timer interrupt is lost.
 569  *
 570  * Returns:     -EINVAL if @count_hz is out of range.
 571  *              0 on success.
 572  */
 573 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz)
 574 {
 575         struct mips_coproc *cop0 = vcpu->arch.cop0;
 576         int dc;
 577         ktime_t now;
 578         u32 count;
 579
 580         /* ensure the frequency is in a sensible range... */
 581         if (count_hz <= 0 || count_hz > NSEC_PER_SEC)
 582                 return -EINVAL;
 583         /* ... and has actually changed */
 584         if (vcpu->arch.count_hz == count_hz)
 585                 return 0;
 586
 587         /* Safely freeze timer so we can keep it continuous */
 588         dc = kvm_mips_count_disabled(vcpu);
 589         if (dc) {
 590                 now = kvm_mips_count_time(vcpu);
 591                 count = kvm_read_c0_guest_count(cop0);
 592         } else {
 593                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 594         }
 595
 596         /* Update the frequency */
 597         vcpu->arch.count_hz = count_hz;
 598         vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz);
 599         vcpu->arch.count_dyn_bias = 0;
 600
 601         /* Calculate adjusted bias so dynamic count is unchanged */
 602         vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now);
 603
 604         /* Update and resume hrtimer */
 605         if (!dc)
 606                 kvm_mips_resume_hrtimer(vcpu, now, count);
 607         return 0;
 608 }
 609
 610 /**
 611  * kvm_mips_write_compare() - Modify compare and update timer.
 612  * @vcpu:       Virtual CPU.
 613  * @compare:    New CP0_Compare value.
 614  * @ack:        Whether to acknowledge timer interrupt.
 615  *
 616  * Update CP0_Compare to a new value and update the timeout.
 617  * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure
 618  * any pending timer interrupt is preserved.
 619  */
 620 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack)
 621 {
 622         struct mips_coproc *cop0 = vcpu->arch.cop0;
 623         int dc;
 624         u32 old_compare = kvm_read_c0_guest_compare(cop0);
 625         ktime_t now;
 626         u32 count;
 627
 628         /* if unchanged, must just be an ack */
 629         if (old_compare == compare) {
 630                 if (!ack)
 631                         return;
 632                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 633                 kvm_write_c0_guest_compare(cop0, compare);
 634                 return;
 635         }
 636
 637         /* freeze_hrtimer() takes care of timer interrupts <= count */
 638         dc = kvm_mips_count_disabled(vcpu);
 639         if (!dc)
 640                 now = kvm_mips_freeze_hrtimer(vcpu, &count);
 641
 642         if (ack)
 643                 kvm_mips_callbacks->dequeue_timer_int(vcpu);
 644
 645         kvm_write_c0_guest_compare(cop0, compare);
 646
 647         /* resume_hrtimer() takes care of timer interrupts > count */
 648         if (!dc)
 649                 kvm_mips_resume_hrtimer(vcpu, now, count);
 650 }
 651
 652 /**
 653  * kvm_mips_count_disable() - Disable count.
 654  * @vcpu:       Virtual CPU.
 655  *
 656  * Disable the CP0_Count timer. A timer interrupt on or before the final stop
 657  * time will be handled but not after.
 658  *
 659  * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or
 660  * count_ctl.DC has been set (count disabled).
 661  *
 662  * Returns:     The time that the timer was stopped.
 663  */
 664 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu)
 665 {
 666         struct mips_coproc *cop0 = vcpu->arch.cop0;
 667         u32 count;
 668         ktime_t now;
 669
 670         /* Stop hrtimer */
 671         hrtimer_cancel(&vcpu->arch.comparecount_timer);
 672
 673         /* Set the static count from the dynamic count, handling pending TI */
 674         now = ktime_get();
 675         count = kvm_mips_read_count_running(vcpu, now);
 676         kvm_write_c0_guest_count(cop0, count);
 677
 678         return now;
 679 }
 680
 681 /**
 682  * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC.
 683  * @vcpu:       Virtual CPU.
 684  *
 685  * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or
 686  * before the final stop time will be handled if the timer isn't disabled by
 687  * count_ctl.DC, but not after.
 688  *
 689  * Assumes CP0_Cause.DC is clear (count enabled).
 690  */
 691 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu)
 692 {
 693         struct mips_coproc *cop0 = vcpu->arch.cop0;
 694
 695         kvm_set_c0_guest_cause(cop0, CAUSEF_DC);
 696         if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC))
 697                 kvm_mips_count_disable(vcpu);
 698 }
 699
 700 /**
 701  * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC.
 702  * @vcpu:       Virtual CPU.
 703  *
 704  * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after
 705  * the start time will be handled if the timer isn't disabled by count_ctl.DC,
 706  * potentially before even returning, so the caller should be careful with
 707  * ordering of CP0_Cause modifications so as not to lose it.
 708  *
 709  * Assumes CP0_Cause.DC is set (count disabled).
 710  */
 711 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu)
 712 {
 713         struct mips_coproc *cop0 = vcpu->arch.cop0;
 714         u32 count;
 715
 716         kvm_clear_c0_guest_cause(cop0, CAUSEF_DC);
 717
 718         /*
 719          * Set the dynamic count to match the static count.
 720          * This starts the hrtimer if count_ctl.DC allows it.
 721          * Otherwise it conveniently updates the biases.
 722          */
 723         count = kvm_read_c0_guest_count(cop0);
 724         kvm_mips_write_count(vcpu, count);
 725 }
 726
 727 /**
 728  * kvm_mips_set_count_ctl() - Update the count control KVM register.
 729  * @vcpu:       Virtual CPU.
 730  * @count_ctl:  Count control register new value.
 731  *
 732  * Set the count control KVM register. The timer is updated accordingly.
 733  *
 734  * Returns:     -EINVAL if reserved bits are set.
 735  *              0 on success.
 736  */
 737 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl)
 738 {
 739         struct mips_coproc *cop0 = vcpu->arch.cop0;
 740         s64 changed = count_ctl ^ vcpu->arch.count_ctl;
 741         s64 delta;
 742         ktime_t expire, now;
 743         u32 count, compare;
 744
 745         /* Only allow defined bits to be changed */
 746         if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC))
 747                 return -EINVAL;
 748
 749         /* Apply new value */
 750         vcpu->arch.count_ctl = count_ctl;
 751
 752         /* Master CP0_Count disable */
 753         if (changed & KVM_REG_MIPS_COUNT_CTL_DC) {
 754                 /* Is CP0_Cause.DC already disabling CP0_Count? */
 755                 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) {
 756                         if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)
 757                                 /* Just record the current time */
 758                                 vcpu->arch.count_resume = ktime_get();
 759                 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) {
 760                         /* disable timer and record current time */
 761                         vcpu->arch.count_resume = kvm_mips_count_disable(vcpu);
 762                 } else {
 763                         /*
 764                          * Calculate timeout relative to static count at resume
 765                          * time (wrap 0 to 2^32).
 766                          */
 767                         count = kvm_read_c0_guest_count(cop0);
 768                         compare = kvm_read_c0_guest_compare(cop0);
 769                         delta = (u64)(u32)(compare - count - 1) + 1;
 770                         delta = div_u64(delta * NSEC_PER_SEC,
 771                                         vcpu->arch.count_hz);
 772                         expire = ktime_add_ns(vcpu->arch.count_resume, delta);
 773
 774                         /* Handle pending interrupt */
 775                         now = ktime_get();
 776                         if (ktime_compare(now, expire) >= 0)
 777                                 /* Nothing should be waiting on the timeout */
 778                                 kvm_mips_callbacks->queue_timer_int(vcpu);
 779
 780                         /* Resume hrtimer without changing bias */
 781                         count = kvm_mips_read_count_running(vcpu, now);
 782                         kvm_mips_resume_hrtimer(vcpu, now, count);
 783                 }
 784         }
 785
 786         return 0;
 787 }
 788
 789 /**
 790  * kvm_mips_set_count_resume() - Update the count resume KVM register.
 791  * @vcpu:               Virtual CPU.
 792  * @count_resume:       Count resume register new value.
 793  *
 794  * Set the count resume KVM register.
 795  *
 796  * Returns:     -EINVAL if out of valid range (0..now).
 797  *              0 on success.
 798  */
 799 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume)
 800 {
 801         /*
 802          * It doesn't make sense for the resume time to be in the future, as it
 803          * would be possible for the next interrupt to be more than a full
 804          * period in the future.
 805          */
 806         if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get()))
 807                 return -EINVAL;
 808
 809         vcpu->arch.count_resume = ns_to_ktime(count_resume);
 810         return 0;
 811 }
 812
 813 /**
 814  * kvm_mips_count_timeout() - Push timer forward on timeout.
 815  * @vcpu:       Virtual CPU.
 816  *
 817  * Handle an hrtimer event by push the hrtimer forward a period.
 818  *
 819  * Returns:     The hrtimer_restart value to return to the hrtimer subsystem.
 820  */
 821 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu)
 822 {
 823         /* Add the Count period to the current expiry time */
 824         hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer,
 825                                vcpu->arch.count_period);
 826         return HRTIMER_RESTART;
 827 }
 828
 829 enum emulation_result kvm_mips_emul_eret(struct kvm_vcpu *vcpu)
 830 {
 831         struct mips_coproc *cop0 = vcpu->arch.cop0;
 832         enum emulation_result er = EMULATE_DONE;
 833
 834         if (kvm_read_c0_guest_status(cop0) & ST0_ERL) {
 835                 kvm_clear_c0_guest_status(cop0, ST0_ERL);
 836                 vcpu->arch.pc = kvm_read_c0_guest_errorepc(cop0);
 837         } else if (kvm_read_c0_guest_status(cop0) & ST0_EXL) {
 838                 kvm_debug("[%#lx] ERET to %#lx\n", vcpu->arch.pc,
 839                           kvm_read_c0_guest_epc(cop0));
 840                 kvm_clear_c0_guest_status(cop0, ST0_EXL);
 841                 vcpu->arch.pc = kvm_read_c0_guest_epc(cop0);
 842
 843         } else {
 844                 kvm_err("[%#lx] ERET when MIPS_SR_EXL|MIPS_SR_ERL == 0\n",
 845                         vcpu->arch.pc);
 846                 er = EMULATE_FAIL;
 847         }
 848
 849         return er;
 850 }
 851
 852 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu)
 853 {
 854         kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc,
 855                   vcpu->arch.pending_exceptions);
 856
 857         ++vcpu->stat.wait_exits;
 858         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_WAIT);
 859         if (!vcpu->arch.pending_exceptions) {
 860                 vcpu->arch.wait = 1;
 861                 kvm_vcpu_block(vcpu);
 862
 863                 /*
 864                  * We we are runnable, then definitely go off to user space to
 865                  * check if any I/O interrupts are pending.
 866                  */
 867                 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) {
 868                         clear_bit(KVM_REQ_UNHALT, &vcpu->requests);
 869                         vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN;
 870                 }
 871         }
 872
 873         return EMULATE_DONE;
 874 }
 875
 876 /*
 877  * XXXKYMA: Linux doesn't seem to use TLBR, return EMULATE_FAIL for now so that
 878  * we can catch this, if things ever change
 879  */
 880 enum emulation_result kvm_mips_emul_tlbr(struct kvm_vcpu *vcpu)
 881 {
 882         struct mips_coproc *cop0 = vcpu->arch.cop0;
 883         unsigned long pc = vcpu->arch.pc;
 884
 885         kvm_err("[%#lx] COP0_TLBR [%ld]\n", pc, kvm_read_c0_guest_index(cop0));
 886         return EMULATE_FAIL;
 887 }
 888
 889 /**
 890  * kvm_mips_invalidate_guest_tlb() - Indicates a change in guest MMU map.
 891  * @vcpu:       VCPU with changed mappings.
 892  * @tlb:        TLB entry being removed.
 893  *
 894  * This is called to indicate a single change in guest MMU mappings, so that we
 895  * can arrange TLB flushes on this and other CPUs.
 896  */
 897 static void kvm_mips_invalidate_guest_tlb(struct kvm_vcpu *vcpu,
 898                                           struct kvm_mips_tlb *tlb)
 899 {
 900         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
 901         struct mm_struct *user_mm = &vcpu->arch.guest_user_mm;
 902         int cpu, i;
 903         bool user;
 904
 905         /* No need to flush for entries which are already invalid */
 906         if (!((tlb->tlb_lo[0] | tlb->tlb_lo[1]) & ENTRYLO_V))
 907                 return;
 908         /* Don't touch host kernel page tables or TLB mappings */
 909         if ((unsigned long)tlb->tlb_hi > 0x7fffffff)
 910                 return;
 911         /* User address space doesn't need flushing for KSeg2/3 changes */
 912         user = tlb->tlb_hi < KVM_GUEST_KSEG0;
 913
 914         preempt_disable();
 915
 916         /* Invalidate page table entries */
 917         kvm_trap_emul_invalidate_gva(vcpu, tlb->tlb_hi & VPN2_MASK, user);
 918
 919         /*
 920          * Probe the shadow host TLB for the entry being overwritten, if one
 921          * matches, invalidate it
 922          */
 923         kvm_mips_host_tlb_inv(vcpu, tlb->tlb_hi, user, true);
 924
 925         /* Invalidate the whole ASID on other CPUs */
 926         cpu = smp_processor_id();
 927         for_each_possible_cpu(i) {
 928                 if (i == cpu)
 929                         continue;
 930                 if (user)
 931                         cpu_context(i, user_mm) = 0;
 932                 cpu_context(i, kern_mm) = 0;
 933         }
 934
 935         preempt_enable();
 936 }
 937
 938 /* Write Guest TLB Entry @ Index */
 939 enum emulation_result kvm_mips_emul_tlbwi(struct kvm_vcpu *vcpu)
 940 {
 941         struct mips_coproc *cop0 = vcpu->arch.cop0;
 942         int index = kvm_read_c0_guest_index(cop0);
 943         struct kvm_mips_tlb *tlb = NULL;
 944         unsigned long pc = vcpu->arch.pc;
 945
 946         if (index < 0 || index >= KVM_MIPS_GUEST_TLB_SIZE) {
 947                 kvm_debug("%s: illegal index: %d\n", __func__, index);
 948                 kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 949                           pc, index, kvm_read_c0_guest_entryhi(cop0),
 950                           kvm_read_c0_guest_entrylo0(cop0),
 951                           kvm_read_c0_guest_entrylo1(cop0),
 952                           kvm_read_c0_guest_pagemask(cop0));
 953                 index = (index & ~0x80000000) % KVM_MIPS_GUEST_TLB_SIZE;
 954         }
 955
 956         tlb = &vcpu->arch.guest_tlb[index];
 957
 958         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
 959
 960         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 961         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 962         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 963         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 964
 965         kvm_debug("[%#lx] COP0_TLBWI [%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx, mask: %#lx)\n",
 966                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 967                   kvm_read_c0_guest_entrylo0(cop0),
 968                   kvm_read_c0_guest_entrylo1(cop0),
 969                   kvm_read_c0_guest_pagemask(cop0));
 970
 971         return EMULATE_DONE;
 972 }
 973
 974 /* Write Guest TLB Entry @ Random Index */
 975 enum emulation_result kvm_mips_emul_tlbwr(struct kvm_vcpu *vcpu)
 976 {
 977         struct mips_coproc *cop0 = vcpu->arch.cop0;
 978         struct kvm_mips_tlb *tlb = NULL;
 979         unsigned long pc = vcpu->arch.pc;
 980         int index;
 981
 982         get_random_bytes(&index, sizeof(index));
 983         index &= (KVM_MIPS_GUEST_TLB_SIZE - 1);
 984
 985         tlb = &vcpu->arch.guest_tlb[index];
 986
 987         kvm_mips_invalidate_guest_tlb(vcpu, tlb);
 988
 989         tlb->tlb_mask = kvm_read_c0_guest_pagemask(cop0);
 990         tlb->tlb_hi = kvm_read_c0_guest_entryhi(cop0);
 991         tlb->tlb_lo[0] = kvm_read_c0_guest_entrylo0(cop0);
 992         tlb->tlb_lo[1] = kvm_read_c0_guest_entrylo1(cop0);
 993
 994         kvm_debug("[%#lx] COP0_TLBWR[%d] (entryhi: %#lx, entrylo0: %#lx entrylo1: %#lx)\n",
 995                   pc, index, kvm_read_c0_guest_entryhi(cop0),
 996                   kvm_read_c0_guest_entrylo0(cop0),
 997                   kvm_read_c0_guest_entrylo1(cop0));
 998
 999         return EMULATE_DONE;
1000 }
1001
1002 enum emulation_result kvm_mips_emul_tlbp(struct kvm_vcpu *vcpu)
1003 {
1004         struct mips_coproc *cop0 = vcpu->arch.cop0;
1005         long entryhi = kvm_read_c0_guest_entryhi(cop0);
1006         unsigned long pc = vcpu->arch.pc;
1007         int index = -1;
1008
1009         index = kvm_mips_guest_tlb_lookup(vcpu, entryhi);
1010
1011         kvm_write_c0_guest_index(cop0, index);
1012
1013         kvm_debug("[%#lx] COP0_TLBP (entryhi: %#lx), index: %d\n", pc, entryhi,
1014                   index);
1015
1016         return EMULATE_DONE;
1017 }
1018
1019 /**
1020  * kvm_mips_config1_wrmask() - Find mask of writable bits in guest Config1
1021  * @vcpu:       Virtual CPU.
1022  *
1023  * Finds the mask of bits which are writable in the guest's Config1 CP0
1024  * register, by userland (currently read-only to the guest).
1025  */
1026 unsigned int kvm_mips_config1_wrmask(struct kvm_vcpu *vcpu)
1027 {
1028         unsigned int mask = 0;
1029
1030         /* Permit FPU to be present if FPU is supported */
1031         if (kvm_mips_guest_can_have_fpu(&vcpu->arch))
1032                 mask |= MIPS_CONF1_FP;
1033
1034         return mask;
1035 }
1036
1037 /**
1038  * kvm_mips_config3_wrmask() - Find mask of writable bits in guest Config3
1039  * @vcpu:       Virtual CPU.
1040  *
1041  * Finds the mask of bits which are writable in the guest's Config3 CP0
1042  * register, by userland (currently read-only to the guest).
1043  */
1044 unsigned int kvm_mips_config3_wrmask(struct kvm_vcpu *vcpu)
1045 {
1046         /* Config4 and ULRI are optional */
1047         unsigned int mask = MIPS_CONF_M | MIPS_CONF3_ULRI;
1048
1049         /* Permit MSA to be present if MSA is supported */
1050         if (kvm_mips_guest_can_have_msa(&vcpu->arch))
1051                 mask |= MIPS_CONF3_MSA;
1052
1053         return mask;
1054 }
1055
1056 /**
1057  * kvm_mips_config4_wrmask() - Find mask of writable bits in guest Config4
1058  * @vcpu:       Virtual CPU.
1059  *
1060  * Finds the mask of bits which are writable in the guest's Config4 CP0
1061  * register, by userland (currently read-only to the guest).
1062  */
1063 unsigned int kvm_mips_config4_wrmask(struct kvm_vcpu *vcpu)
1064 {
1065         /* Config5 is optional */
1066         unsigned int mask = MIPS_CONF_M;
1067
1068         /* KScrExist */
1069         mask |= 0xfc << MIPS_CONF4_KSCREXIST_SHIFT;
1070
1071         return mask;
1072 }
1073
1074 /**
1075  * kvm_mips_config5_wrmask() - Find mask of writable bits in guest Config5
1076  * @vcpu:       Virtual CPU.
1077  *
1078  * Finds the mask of bits which are writable in the guest's Config5 CP0
1079  * register, by the guest itself.
1080  */
1081 unsigned int kvm_mips_config5_wrmask(struct kvm_vcpu *vcpu)
1082 {
1083         unsigned int mask = 0;
1084
1085         /* Permit MSAEn changes if MSA supported and enabled */
1086         if (kvm_mips_guest_has_msa(&vcpu->arch))
1087                 mask |= MIPS_CONF5_MSAEN;
1088
1089         /*
1090          * Permit guest FPU mode changes if FPU is enabled and the relevant
1091          * feature exists according to FIR register.
1092          */
1093         if (kvm_mips_guest_has_fpu(&vcpu->arch)) {
1094                 if (cpu_has_fre)
1095                         mask |= MIPS_CONF5_FRE;
1096                 /* We don't support UFR or UFE */
1097         }
1098
1099         return mask;
1100 }
1101
1102 enum emulation_result kvm_mips_emulate_CP0(union mips_instruction inst,
1103                                            u32 *opc, u32 cause,
1104                                            struct kvm_run *run,
1105                                            struct kvm_vcpu *vcpu)
1106 {
1107         struct mips_coproc *cop0 = vcpu->arch.cop0;
1108         struct mm_struct *kern_mm = &vcpu->arch.guest_kernel_mm;
1109         enum emulation_result er = EMULATE_DONE;
1110         u32 rt, rd, sel;
1111         unsigned long curr_pc;
1112         int cpu, i;
1113
1114         /*
1115          * Update PC and hold onto current PC in case there is
1116          * an error and we want to rollback the PC
1117          */
1118         curr_pc = vcpu->arch.pc;
1119         er = update_pc(vcpu, cause);
1120         if (er == EMULATE_FAIL)
1121                 return er;
1122
1123         if (inst.co_format.co) {
1124                 switch (inst.co_format.func) {
1125                 case tlbr_op:   /*  Read indexed TLB entry  */
1126                         er = kvm_mips_emul_tlbr(vcpu);
1127                         break;
1128                 case tlbwi_op:  /*  Write indexed  */
1129                         er = kvm_mips_emul_tlbwi(vcpu);
1130                         break;
1131                 case tlbwr_op:  /*  Write random  */
1132                         er = kvm_mips_emul_tlbwr(vcpu);
1133                         break;
1134                 case tlbp_op:   /* TLB Probe */
1135                         er = kvm_mips_emul_tlbp(vcpu);
1136                         break;
1137                 case rfe_op:
1138                         kvm_err("!!!COP0_RFE!!!\n");
1139                         break;
1140                 case eret_op:
1141                         er = kvm_mips_emul_eret(vcpu);
1142                         goto dont_update_pc;
1143                 case wait_op:
1144                         er = kvm_mips_emul_wait(vcpu);
1145                         break;
1146                 case hypcall_op:
1147                         er = kvm_mips_emul_hypcall(vcpu, inst);
1148                         break;
1149                 }
1150         } else {
1151                 rt = inst.c0r_format.rt;
1152                 rd = inst.c0r_format.rd;
1153                 sel = inst.c0r_format.sel;
1154
1155                 switch (inst.c0r_format.rs) {
1156                 case mfc_op:
1157 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1158                         cop0->stat[rd][sel]++;
1159 #endif
1160                         /* Get reg */
1161                         if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1162                                 vcpu->arch.gprs[rt] =
1163                                     (s32)kvm_mips_read_count(vcpu);
1164                         } else if ((rd == MIPS_CP0_ERRCTL) && (sel == 0)) {
1165                                 vcpu->arch.gprs[rt] = 0x0;
1166 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1167                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1168 #endif
1169                         } else {
1170                                 vcpu->arch.gprs[rt] = (s32)cop0->reg[rd][sel];
1171
1172 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1173                                 kvm_mips_trans_mfc0(inst, opc, vcpu);
1174 #endif
1175                         }
1176
1177                         trace_kvm_hwr(vcpu, KVM_TRACE_MFC0,
1178                                       KVM_TRACE_COP0(rd, sel),
1179                                       vcpu->arch.gprs[rt]);
1180                         break;
1181
1182                 case dmfc_op:
1183                         vcpu->arch.gprs[rt] = cop0->reg[rd][sel];
1184
1185                         trace_kvm_hwr(vcpu, KVM_TRACE_DMFC0,
1186                                       KVM_TRACE_COP0(rd, sel),
1187                                       vcpu->arch.gprs[rt]);
1188                         break;
1189
1190                 case mtc_op:
1191 #ifdef CONFIG_KVM_MIPS_DEBUG_COP0_COUNTERS
1192                         cop0->stat[rd][sel]++;
1193 #endif
1194                         trace_kvm_hwr(vcpu, KVM_TRACE_MTC0,
1195                                       KVM_TRACE_COP0(rd, sel),
1196                                       vcpu->arch.gprs[rt]);
1197
1198                         if ((rd == MIPS_CP0_TLB_INDEX)
1199                             && (vcpu->arch.gprs[rt] >=
1200                                 KVM_MIPS_GUEST_TLB_SIZE)) {
1201                                 kvm_err("Invalid TLB Index: %ld",
1202                                         vcpu->arch.gprs[rt]);
1203                                 er = EMULATE_FAIL;
1204                                 break;
1205                         }
1206                         if ((rd == MIPS_CP0_PRID) && (sel == 1)) {
1207                                 /*
1208                                  * Preserve core number, and keep the exception
1209                                  * base in guest KSeg0.
1210                                  */
1211                                 kvm_change_c0_guest_ebase(cop0, 0x1ffff000,
1212                                                           vcpu->arch.gprs[rt]);
1213                         } else if (rd == MIPS_CP0_TLB_HI && sel == 0) {
1214                                 u32 nasid =
1215                                         vcpu->arch.gprs[rt] & KVM_ENTRYHI_ASID;
1216                                 if (((kvm_read_c0_guest_entryhi(cop0) &
1217                                       KVM_ENTRYHI_ASID) != nasid)) {
1218                                         trace_kvm_asid_change(vcpu,
1219                                                 kvm_read_c0_guest_entryhi(cop0)
1220                                                         & KVM_ENTRYHI_ASID,
1221                                                 nasid);
1222
1223                                         /*
1224                                          * Flush entries from the GVA page
1225                                          * tables.
1226                                          * Guest user page table will get
1227                                          * flushed lazily on re-entry to guest
1228                                          * user if the guest ASID actually
1229                                          * changes.
1230                                          */
1231                                         kvm_mips_flush_gva_pt(kern_mm->pgd,
1232                                                               KMF_KERN);
1233
1234                                         /*
1235                                          * Regenerate/invalidate kernel MMU
1236                                          * context.
1237                                          * The user MMU context will be
1238                                          * regenerated lazily on re-entry to
1239                                          * guest user if the guest ASID actually
1240                                          * changes.
1241                                          */
1242                                         preempt_disable();
1243                                         cpu = smp_processor_id();
1244                                         get_new_mmu_context(kern_mm, cpu);
1245                                         for_each_possible_cpu(i)
1246                                                 if (i != cpu)
1247                                                         cpu_context(i, kern_mm) = 0;
1248                                         preempt_enable();
1249                                 }
1250                                 kvm_write_c0_guest_entryhi(cop0,
1251                                                            vcpu->arch.gprs[rt]);
1252                         }
1253                         /* Are we writing to COUNT */
1254                         else if ((rd == MIPS_CP0_COUNT) && (sel == 0)) {
1255                                 kvm_mips_write_count(vcpu, vcpu->arch.gprs[rt]);
1256                                 goto done;
1257                         } else if ((rd == MIPS_CP0_COMPARE) && (sel == 0)) {
1258                                 /* If we are writing to COMPARE */
1259                                 /* Clear pending timer interrupt, if any */
1260                                 kvm_mips_write_compare(vcpu,
1261                                                        vcpu->arch.gprs[rt],
1262                                                        true);
1263                         } else if ((rd == MIPS_CP0_STATUS) && (sel == 0)) {
1264                                 unsigned int old_val, val, change;
1265
1266                                 old_val = kvm_read_c0_guest_status(cop0);
1267                                 val = vcpu->arch.gprs[rt];
1268                                 change = val ^ old_val;
1269
1270                                 /* Make sure that the NMI bit is never set */
1271                                 val &= ~ST0_NMI;
1272
1273                                 /*
1274                                  * Don't allow CU1 or FR to be set unless FPU
1275                                  * capability enabled and exists in guest
1276                                  * configuration.
1277                                  */
1278                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1279                                         val &= ~(ST0_CU1 | ST0_FR);
1280
1281                                 /*
1282                                  * Also don't allow FR to be set if host doesn't
1283                                  * support it.
1284                                  */
1285                                 if (!(current_cpu_data.fpu_id & MIPS_FPIR_F64))
1286                                         val &= ~ST0_FR;
1287
1288
1289                                 /* Handle changes in FPU mode */
1290                                 preempt_disable();
1291
1292                                 /*
1293                                  * FPU and Vector register state is made
1294                                  * UNPREDICTABLE by a change of FR, so don't
1295                                  * even bother saving it.
1296                                  */
1297                                 if (change & ST0_FR)
1298                                         kvm_drop_fpu(vcpu);
1299
1300                                 /*
1301                                  * If MSA state is already live, it is undefined
1302                                  * how it interacts with FR=0 FPU state, and we
1303                                  * don't want to hit reserved instruction
1304                                  * exceptions trying to save the MSA state later
1305                                  * when CU=1 && FR=1, so play it safe and save
1306                                  * it first.
1307                                  */
1308                                 if (change & ST0_CU1 && !(val & ST0_FR) &&
1309                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1310                                         kvm_lose_fpu(vcpu);
1311
1312                                 /*
1313                                  * Propagate CU1 (FPU enable) changes
1314                                  * immediately if the FPU context is already
1315                                  * loaded. When disabling we leave the context
1316                                  * loaded so it can be quickly enabled again in
1317                                  * the near future.
1318                                  */
1319                                 if (change & ST0_CU1 &&
1320                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1321                                         change_c0_status(ST0_CU1, val);
1322
1323                                 preempt_enable();
1324
1325                                 kvm_write_c0_guest_status(cop0, val);
1326
1327 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1328                                 /*
1329                                  * If FPU present, we need CU1/FR bits to take
1330                                  * effect fairly soon.
1331                                  */
1332                                 if (!kvm_mips_guest_has_fpu(&vcpu->arch))
1333                                         kvm_mips_trans_mtc0(inst, opc, vcpu);
1334 #endif
1335                         } else if ((rd == MIPS_CP0_CONFIG) && (sel == 5)) {
1336                                 unsigned int old_val, val, change, wrmask;
1337
1338                                 old_val = kvm_read_c0_guest_config5(cop0);
1339                                 val = vcpu->arch.gprs[rt];
1340
1341                                 /* Only a few bits are writable in Config5 */
1342                                 wrmask = kvm_mips_config5_wrmask(vcpu);
1343                                 change = (val ^ old_val) & wrmask;
1344                                 val = old_val ^ change;
1345
1346
1347                                 /* Handle changes in FPU/MSA modes */
1348                                 preempt_disable();
1349
1350                                 /*
1351                                  * Propagate FRE changes immediately if the FPU
1352                                  * context is already loaded.
1353                                  */
1354                                 if (change & MIPS_CONF5_FRE &&
1355                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_FPU)
1356                                         change_c0_config5(MIPS_CONF5_FRE, val);
1357
1358                                 /*
1359                                  * Propagate MSAEn changes immediately if the
1360                                  * MSA context is already loaded. When disabling
1361                                  * we leave the context loaded so it can be
1362                                  * quickly enabled again in the near future.
1363                                  */
1364                                 if (change & MIPS_CONF5_MSAEN &&
1365                                     vcpu->arch.aux_inuse & KVM_MIPS_AUX_MSA)
1366                                         change_c0_config5(MIPS_CONF5_MSAEN,
1367                                                           val);
1368
1369                                 preempt_enable();
1370
1371                                 kvm_write_c0_guest_config5(cop0, val);
1372                         } else if ((rd == MIPS_CP0_CAUSE) && (sel == 0)) {
1373                                 u32 old_cause, new_cause;
1374
1375                                 old_cause = kvm_read_c0_guest_cause(cop0);
1376                                 new_cause = vcpu->arch.gprs[rt];
1377                                 /* Update R/W bits */
1378                                 kvm_change_c0_guest_cause(cop0, 0x08800300,
1379                                                           new_cause);
1380                                 /* DC bit enabling/disabling timer? */
1381                                 if ((old_cause ^ new_cause) & CAUSEF_DC) {
1382                                         if (new_cause & CAUSEF_DC)
1383                                                 kvm_mips_count_disable_cause(vcpu);
1384                                         else
1385                                                 kvm_mips_count_enable_cause(vcpu);
1386                                 }
1387                         } else if ((rd == MIPS_CP0_HWRENA) && (sel == 0)) {
1388                                 u32 mask = MIPS_HWRENA_CPUNUM |
1389                                            MIPS_HWRENA_SYNCISTEP |
1390                                            MIPS_HWRENA_CC |
1391                                            MIPS_HWRENA_CCRES;
1392
1393                                 if (kvm_read_c0_guest_config3(cop0) &
1394                                     MIPS_CONF3_ULRI)
1395                                         mask |= MIPS_HWRENA_ULR;
1396                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt] & mask;
1397                         } else {
1398                                 cop0->reg[rd][sel] = vcpu->arch.gprs[rt];
1399 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1400                                 kvm_mips_trans_mtc0(inst, opc, vcpu);
1401 #endif
1402                         }
1403                         break;
1404
1405                 case dmtc_op:
1406                         kvm_err("!!!!!!![%#lx]dmtc_op: rt: %d, rd: %d, sel: %d!!!!!!\n",
1407                                 vcpu->arch.pc, rt, rd, sel);
1408                         trace_kvm_hwr(vcpu, KVM_TRACE_DMTC0,
1409                                       KVM_TRACE_COP0(rd, sel),
1410                                       vcpu->arch.gprs[rt]);
1411                         er = EMULATE_FAIL;
1412                         break;
1413
1414                 case mfmc0_op:
1415 #ifdef KVM_MIPS_DEBUG_COP0_COUNTERS
1416                         cop0->stat[MIPS_CP0_STATUS][0]++;
1417 #endif
1418                         if (rt != 0)
1419                                 vcpu->arch.gprs[rt] =
1420                                     kvm_read_c0_guest_status(cop0);
1421                         /* EI */
1422                         if (inst.mfmc0_format.sc) {
1423                                 kvm_debug("[%#lx] mfmc0_op: EI\n",
1424                                           vcpu->arch.pc);
1425                                 kvm_set_c0_guest_status(cop0, ST0_IE);
1426                         } else {
1427                                 kvm_debug("[%#lx] mfmc0_op: DI\n",
1428                                           vcpu->arch.pc);
1429                                 kvm_clear_c0_guest_status(cop0, ST0_IE);
1430                         }
1431
1432                         break;
1433
1434                 case wrpgpr_op:
1435                         {
1436                                 u32 css = cop0->reg[MIPS_CP0_STATUS][2] & 0xf;
1437                                 u32 pss =
1438                                     (cop0->reg[MIPS_CP0_STATUS][2] >> 6) & 0xf;
1439                                 /*
1440                                  * We don't support any shadow register sets, so
1441                                  * SRSCtl[PSS] == SRSCtl[CSS] = 0
1442                                  */
1443                                 if (css || pss) {
1444                                         er = EMULATE_FAIL;
1445                                         break;
1446                                 }
1447                                 kvm_debug("WRPGPR[%d][%d] = %#lx\n", pss, rd,
1448                                           vcpu->arch.gprs[rt]);
1449                                 vcpu->arch.gprs[rd] = vcpu->arch.gprs[rt];
1450                         }
1451                         break;
1452                 default:
1453                         kvm_err("[%#lx]MachEmulateCP0: unsupported COP0, copz: 0x%x\n",
1454                                 vcpu->arch.pc, inst.c0r_format.rs);
1455                         er = EMULATE_FAIL;
1456                         break;
1457                 }
1458         }
1459
1460 done:
1461         /* Rollback PC only if emulation was unsuccessful */
1462         if (er == EMULATE_FAIL)
1463                 vcpu->arch.pc = curr_pc;
1464
1465 dont_update_pc:
1466         /*
1467          * This is for special instructions whose emulation
1468          * updates the PC, so do not overwrite the PC under
1469          * any circumstances
1470          */
1471
1472         return er;
1473 }
1474
1475 enum emulation_result kvm_mips_emulate_store(union mips_instruction inst,
1476                                              u32 cause,
1477                                              struct kvm_run *run,
1478                                              struct kvm_vcpu *vcpu)
1479 {
1480         enum emulation_result er = EMULATE_DO_MMIO;
1481         u32 rt;
1482         u32 bytes;
1483         void *data = run->mmio.data;
1484         unsigned long curr_pc;
1485
1486         /*
1487          * Update PC and hold onto current PC in case there is
1488          * an error and we want to rollback the PC
1489          */
1490         curr_pc = vcpu->arch.pc;
1491         er = update_pc(vcpu, cause);
1492         if (er == EMULATE_FAIL)
1493                 return er;
1494
1495         rt = inst.i_format.rt;
1496
1497         switch (inst.i_format.opcode) {
1498         case sb_op:
1499                 bytes = 1;
1500                 if (bytes > sizeof(run->mmio.data)) {
1501                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1502                                run->mmio.len);
1503                 }
1504                 run->mmio.phys_addr =
1505                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1506                                                    host_cp0_badvaddr);
1507                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1508                         er = EMULATE_FAIL;
1509                         break;
1510                 }
1511                 run->mmio.len = bytes;
1512                 run->mmio.is_write = 1;
1513                 vcpu->mmio_needed = 1;
1514                 vcpu->mmio_is_write = 1;
1515                 *(u8 *) data = vcpu->arch.gprs[rt];
1516                 kvm_debug("OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1517                           vcpu->arch.host_cp0_badvaddr, vcpu->arch.gprs[rt],
1518                           *(u8 *) data);
1519
1520                 break;
1521
1522         case sw_op:
1523                 bytes = 4;
1524                 if (bytes > sizeof(run->mmio.data)) {
1525                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1526                                run->mmio.len);
1527                 }
1528                 run->mmio.phys_addr =
1529                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1530                                                    host_cp0_badvaddr);
1531                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1532                         er = EMULATE_FAIL;
1533                         break;
1534                 }
1535
1536                 run->mmio.len = bytes;
1537                 run->mmio.is_write = 1;
1538                 vcpu->mmio_needed = 1;
1539                 vcpu->mmio_is_write = 1;
1540                 *(u32 *) data = vcpu->arch.gprs[rt];
1541
1542                 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1543                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1544                           vcpu->arch.gprs[rt], *(u32 *) data);
1545                 break;
1546
1547         case sh_op:
1548                 bytes = 2;
1549                 if (bytes > sizeof(run->mmio.data)) {
1550                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1551                                run->mmio.len);
1552                 }
1553                 run->mmio.phys_addr =
1554                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1555                                                    host_cp0_badvaddr);
1556                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1557                         er = EMULATE_FAIL;
1558                         break;
1559                 }
1560
1561                 run->mmio.len = bytes;
1562                 run->mmio.is_write = 1;
1563                 vcpu->mmio_needed = 1;
1564                 vcpu->mmio_is_write = 1;
1565                 *(u16 *) data = vcpu->arch.gprs[rt];
1566
1567                 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n",
1568                           vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr,
1569                           vcpu->arch.gprs[rt], *(u32 *) data);
1570                 break;
1571
1572         default:
1573                 kvm_err("Store not yet supported (inst=0x%08x)\n",
1574                         inst.word);
1575                 er = EMULATE_FAIL;
1576                 break;
1577         }
1578
1579         /* Rollback PC if emulation was unsuccessful */
1580         if (er == EMULATE_FAIL)
1581                 vcpu->arch.pc = curr_pc;
1582
1583         return er;
1584 }
1585
1586 enum emulation_result kvm_mips_emulate_load(union mips_instruction inst,
1587                                             u32 cause, struct kvm_run *run,
1588                                             struct kvm_vcpu *vcpu)
1589 {
1590         enum emulation_result er = EMULATE_DO_MMIO;
1591         unsigned long curr_pc;
1592         u32 op, rt;
1593         u32 bytes;
1594
1595         rt = inst.i_format.rt;
1596         op = inst.i_format.opcode;
1597
1598         /*
1599          * Find the resume PC now while we have safe and easy access to the
1600          * prior branch instruction, and save it for
1601          * kvm_mips_complete_mmio_load() to restore later.
1602          */
1603         curr_pc = vcpu->arch.pc;
1604         er = update_pc(vcpu, cause);
1605         if (er == EMULATE_FAIL)
1606                 return er;
1607         vcpu->arch.io_pc = vcpu->arch.pc;
1608         vcpu->arch.pc = curr_pc;
1609
1610         vcpu->arch.io_gpr = rt;
1611
1612         switch (op) {
1613         case lw_op:
1614                 bytes = 4;
1615                 if (bytes > sizeof(run->mmio.data)) {
1616                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1617                                run->mmio.len);
1618                         er = EMULATE_FAIL;
1619                         break;
1620                 }
1621                 run->mmio.phys_addr =
1622                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1623                                                    host_cp0_badvaddr);
1624                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1625                         er = EMULATE_FAIL;
1626                         break;
1627                 }
1628
1629                 run->mmio.len = bytes;
1630                 run->mmio.is_write = 0;
1631                 vcpu->mmio_needed = 1;
1632                 vcpu->mmio_is_write = 0;
1633                 break;
1634
1635         case lh_op:
1636         case lhu_op:
1637                 bytes = 2;
1638                 if (bytes > sizeof(run->mmio.data)) {
1639                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1640                                run->mmio.len);
1641                         er = EMULATE_FAIL;
1642                         break;
1643                 }
1644                 run->mmio.phys_addr =
1645                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1646                                                    host_cp0_badvaddr);
1647                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1648                         er = EMULATE_FAIL;
1649                         break;
1650                 }
1651
1652                 run->mmio.len = bytes;
1653                 run->mmio.is_write = 0;
1654                 vcpu->mmio_needed = 1;
1655                 vcpu->mmio_is_write = 0;
1656
1657                 if (op == lh_op)
1658                         vcpu->mmio_needed = 2;
1659                 else
1660                         vcpu->mmio_needed = 1;
1661
1662                 break;
1663
1664         case lbu_op:
1665         case lb_op:
1666                 bytes = 1;
1667                 if (bytes > sizeof(run->mmio.data)) {
1668                         kvm_err("%s: bad MMIO length: %d\n", __func__,
1669                                run->mmio.len);
1670                         er = EMULATE_FAIL;
1671                         break;
1672                 }
1673                 run->mmio.phys_addr =
1674                     kvm_mips_callbacks->gva_to_gpa(vcpu->arch.
1675                                                    host_cp0_badvaddr);
1676                 if (run->mmio.phys_addr == KVM_INVALID_ADDR) {
1677                         er = EMULATE_FAIL;
1678                         break;
1679                 }
1680
1681                 run->mmio.len = bytes;
1682                 run->mmio.is_write = 0;
1683                 vcpu->mmio_is_write = 0;
1684
1685                 if (op == lb_op)
1686                         vcpu->mmio_needed = 2;
1687                 else
1688                         vcpu->mmio_needed = 1;
1689
1690                 break;
1691
1692         default:
1693                 kvm_err("Load not yet supported (inst=0x%08x)\n",
1694                         inst.word);
1695                 er = EMULATE_FAIL;
1696                 break;
1697         }
1698
1699         return er;
1700 }
1701
1702 static enum emulation_result kvm_mips_guest_cache_op(int (*fn)(unsigned long),
1703                                                      unsigned long curr_pc,
1704                                                      unsigned long addr,
1705                                                      struct kvm_run *run,
1706                                                      struct kvm_vcpu *vcpu,
1707                                                      u32 cause)
1708 {
1709         int err;
1710
1711         for (;;) {
1712                 /* Carefully attempt the cache operation */
1713                 kvm_trap_emul_gva_lockless_begin(vcpu);
1714                 err = fn(addr);
1715                 kvm_trap_emul_gva_lockless_end(vcpu);
1716
1717                 if (likely(!err))
1718                         return EMULATE_DONE;
1719
1720                 /*
1721                  * Try to handle the fault and retry, maybe we just raced with a
1722                  * GVA invalidation.
1723                  */
1724                 switch (kvm_trap_emul_gva_fault(vcpu, addr, false)) {
1725                 case KVM_MIPS_GVA:
1726                 case KVM_MIPS_GPA:
1727                         /* bad virtual or physical address */
1728                         return EMULATE_FAIL;
1729                 case KVM_MIPS_TLB:
1730                         /* no matching guest TLB */
1731                         vcpu->arch.host_cp0_badvaddr = addr;
1732                         vcpu->arch.pc = curr_pc;
1733                         kvm_mips_emulate_tlbmiss_ld(cause, NULL, run, vcpu);
1734                         return EMULATE_EXCEPT;
1735                 case KVM_MIPS_TLBINV:
1736                         /* invalid matching guest TLB */
1737                         vcpu->arch.host_cp0_badvaddr = addr;
1738                         vcpu->arch.pc = curr_pc;
1739                         kvm_mips_emulate_tlbinv_ld(cause, NULL, run, vcpu);
1740                         return EMULATE_EXCEPT;
1741                 default:
1742                         break;
1743                 };
1744         }
1745 }
1746
1747 enum emulation_result kvm_mips_emulate_cache(union mips_instruction inst,
1748                                              u32 *opc, u32 cause,
1749                                              struct kvm_run *run,
1750                                              struct kvm_vcpu *vcpu)
1751 {
1752         enum emulation_result er = EMULATE_DONE;
1753         u32 cache, op_inst, op, base;
1754         s16 offset;
1755         struct kvm_vcpu_arch *arch = &vcpu->arch;
1756         unsigned long va;
1757         unsigned long curr_pc;
1758
1759         /*
1760          * Update PC and hold onto current PC in case there is
1761          * an error and we want to rollback the PC
1762          */
1763         curr_pc = vcpu->arch.pc;
1764         er = update_pc(vcpu, cause);
1765         if (er == EMULATE_FAIL)
1766                 return er;
1767
1768         base = inst.i_format.rs;
1769         op_inst = inst.i_format.rt;
1770         if (cpu_has_mips_r6)
1771                 offset = inst.spec3_format.simmediate;
1772         else
1773                 offset = inst.i_format.simmediate;
1774         cache = op_inst & CacheOp_Cache;
1775         op = op_inst & CacheOp_Op;
1776
1777         va = arch->gprs[base] + offset;
1778
1779         kvm_debug("CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1780                   cache, op, base, arch->gprs[base], offset);
1781
1782         /*
1783          * Treat INDEX_INV as a nop, basically issued by Linux on startup to
1784          * invalidate the caches entirely by stepping through all the
1785          * ways/indexes
1786          */
1787         if (op == Index_Writeback_Inv) {
1788                 kvm_debug("@ %#lx/%#lx CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1789                           vcpu->arch.pc, vcpu->arch.gprs[31], cache, op, base,
1790                           arch->gprs[base], offset);
1791
1792                 if (cache == Cache_D)
1793                         r4k_blast_dcache();
1794                 else if (cache == Cache_I)
1795                         r4k_blast_icache();
1796                 else {
1797                         kvm_err("%s: unsupported CACHE INDEX operation\n",
1798                                 __func__);
1799                         return EMULATE_FAIL;
1800                 }
1801
1802 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1803                 kvm_mips_trans_cache_index(inst, opc, vcpu);
1804 #endif
1805                 goto done;
1806         }
1807
1808         /* XXXKYMA: Only a subset of cache ops are supported, used by Linux */
1809         if (op_inst == Hit_Writeback_Inv_D || op_inst == Hit_Invalidate_D) {
1810                 /*
1811                  * Perform the dcache part of icache synchronisation on the
1812                  * guest's behalf.
1813                  */
1814                 er = kvm_mips_guest_cache_op(protected_writeback_dcache_line,
1815                                              curr_pc, va, run, vcpu, cause);
1816                 if (er != EMULATE_DONE)
1817                         goto done;
1818 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1819                 /*
1820                  * Replace the CACHE instruction, with a SYNCI, not the same,
1821                  * but avoids a trap
1822                  */
1823                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1824 #endif
1825         } else if (op_inst == Hit_Invalidate_I) {
1826                 /* Perform the icache synchronisation on the guest's behalf */
1827                 er = kvm_mips_guest_cache_op(protected_writeback_dcache_line,
1828                                              curr_pc, va, run, vcpu, cause);
1829                 if (er != EMULATE_DONE)
1830                         goto done;
1831                 er = kvm_mips_guest_cache_op(protected_flush_icache_line,
1832                                              curr_pc, va, run, vcpu, cause);
1833                 if (er != EMULATE_DONE)
1834                         goto done;
1835
1836 #ifdef CONFIG_KVM_MIPS_DYN_TRANS
1837                 /* Replace the CACHE instruction, with a SYNCI */
1838                 kvm_mips_trans_cache_va(inst, opc, vcpu);
1839 #endif
1840         } else {
1841                 kvm_err("NO-OP CACHE (cache: %#x, op: %#x, base[%d]: %#lx, offset: %#x\n",
1842                         cache, op, base, arch->gprs[base], offset);
1843                 er = EMULATE_FAIL;
1844         }
1845
1846 done:
1847         /* Rollback PC only if emulation was unsuccessful */
1848         if (er == EMULATE_FAIL)
1849                 vcpu->arch.pc = curr_pc;
1850         /* Guest exception needs guest to resume */
1851         if (er == EMULATE_EXCEPT)
1852                 er = EMULATE_DONE;
1853
1854         return er;
1855 }
1856
1857 enum emulation_result kvm_mips_emulate_inst(u32 cause, u32 *opc,
1858                                             struct kvm_run *run,
1859                                             struct kvm_vcpu *vcpu)
1860 {
1861         union mips_instruction inst;
1862         enum emulation_result er = EMULATE_DONE;
1863         int err;
1864
1865         /* Fetch the instruction. */
1866         if (cause & CAUSEF_BD)
1867                 opc += 1;
1868         err = kvm_get_badinstr(opc, vcpu, &inst.word);
1869         if (err)
1870                 return EMULATE_FAIL;
1871
1872         switch (inst.r_format.opcode) {
1873         case cop0_op:
1874                 er = kvm_mips_emulate_CP0(inst, opc, cause, run, vcpu);
1875                 break;
1876         case sb_op:
1877         case sh_op:
1878         case sw_op:
1879                 er = kvm_mips_emulate_store(inst, cause, run, vcpu);
1880                 break;
1881         case lb_op:
1882         case lbu_op:
1883         case lhu_op:
1884         case lh_op:
1885         case lw_op:
1886                 er = kvm_mips_emulate_load(inst, cause, run, vcpu);
1887                 break;
1888
1889 #ifndef CONFIG_CPU_MIPSR6
1890         case cache_op:
1891                 ++vcpu->stat.cache_exits;
1892                 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1893                 er = kvm_mips_emulate_cache(inst, opc, cause, run, vcpu);
1894                 break;
1895 #else
1896         case spec3_op:
1897                 switch (inst.spec3_format.func) {
1898                 case cache6_op:
1899                         ++vcpu->stat.cache_exits;
1900                         trace_kvm_exit(vcpu, KVM_TRACE_EXIT_CACHE);
1901                         er = kvm_mips_emulate_cache(inst, opc, cause, run,
1902                                                     vcpu);
1903                         break;
1904                 default:
1905                         goto unknown;
1906                 };
1907                 break;
1908 unknown:
1909 #endif
1910
1911         default:
1912                 kvm_err("Instruction emulation not supported (%p/%#x)\n", opc,
1913                         inst.word);
1914                 kvm_arch_vcpu_dump_regs(vcpu);
1915                 er = EMULATE_FAIL;
1916                 break;
1917         }
1918
1919         return er;
1920 }
1921
1922 /**
1923  * kvm_mips_guest_exception_base() - Find guest exception vector base address.
1924  *
1925  * Returns:     The base address of the current guest exception vector, taking
1926  *              both Guest.CP0_Status.BEV and Guest.CP0_EBase into account.
1927  */
1928 long kvm_mips_guest_exception_base(struct kvm_vcpu *vcpu)
1929 {
1930         struct mips_coproc *cop0 = vcpu->arch.cop0;
1931
1932         if (kvm_read_c0_guest_status(cop0) & ST0_BEV)
1933                 return KVM_GUEST_CKSEG1ADDR(0x1fc00200);
1934         else
1935                 return kvm_read_c0_guest_ebase(cop0) & MIPS_EBASE_BASE;
1936 }
1937
1938 enum emulation_result kvm_mips_emulate_syscall(u32 cause,
1939                                                u32 *opc,
1940                                                struct kvm_run *run,
1941                                                struct kvm_vcpu *vcpu)
1942 {
1943         struct mips_coproc *cop0 = vcpu->arch.cop0;
1944         struct kvm_vcpu_arch *arch = &vcpu->arch;
1945         enum emulation_result er = EMULATE_DONE;
1946
1947         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1948                 /* save old pc */
1949                 kvm_write_c0_guest_epc(cop0, arch->pc);
1950                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1951
1952                 if (cause & CAUSEF_BD)
1953                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1954                 else
1955                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1956
1957                 kvm_debug("Delivering SYSCALL @ pc %#lx\n", arch->pc);
1958
1959                 kvm_change_c0_guest_cause(cop0, (0xff),
1960                                           (EXCCODE_SYS << CAUSEB_EXCCODE));
1961
1962                 /* Set PC to the exception entry point */
1963                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
1964
1965         } else {
1966                 kvm_err("Trying to deliver SYSCALL when EXL is already set\n");
1967                 er = EMULATE_FAIL;
1968         }
1969
1970         return er;
1971 }
1972
1973 enum emulation_result kvm_mips_emulate_tlbmiss_ld(u32 cause,
1974                                                   u32 *opc,
1975                                                   struct kvm_run *run,
1976                                                   struct kvm_vcpu *vcpu)
1977 {
1978         struct mips_coproc *cop0 = vcpu->arch.cop0;
1979         struct kvm_vcpu_arch *arch = &vcpu->arch;
1980         unsigned long entryhi = (vcpu->arch.  host_cp0_badvaddr & VPN2_MASK) |
1981                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
1982
1983         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
1984                 /* save old pc */
1985                 kvm_write_c0_guest_epc(cop0, arch->pc);
1986                 kvm_set_c0_guest_status(cop0, ST0_EXL);
1987
1988                 if (cause & CAUSEF_BD)
1989                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
1990                 else
1991                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
1992
1993                 kvm_debug("[EXL == 0] delivering TLB MISS @ pc %#lx\n",
1994                           arch->pc);
1995
1996                 /* set pc to the exception entry point */
1997                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x0;
1998
1999         } else {
2000                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
2001                           arch->pc);
2002
2003                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2004         }
2005
2006         kvm_change_c0_guest_cause(cop0, (0xff),
2007                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
2008
2009         /* setup badvaddr, context and entryhi registers for the guest */
2010         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2011         /* XXXKYMA: is the context register used by linux??? */
2012         kvm_write_c0_guest_entryhi(cop0, entryhi);
2013
2014         return EMULATE_DONE;
2015 }
2016
2017 enum emulation_result kvm_mips_emulate_tlbinv_ld(u32 cause,
2018                                                  u32 *opc,
2019                                                  struct kvm_run *run,
2020                                                  struct kvm_vcpu *vcpu)
2021 {
2022         struct mips_coproc *cop0 = vcpu->arch.cop0;
2023         struct kvm_vcpu_arch *arch = &vcpu->arch;
2024         unsigned long entryhi =
2025                 (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2026                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2027
2028         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2029                 /* save old pc */
2030                 kvm_write_c0_guest_epc(cop0, arch->pc);
2031                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2032
2033                 if (cause & CAUSEF_BD)
2034                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2035                 else
2036                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2037
2038                 kvm_debug("[EXL == 0] delivering TLB INV @ pc %#lx\n",
2039                           arch->pc);
2040         } else {
2041                 kvm_debug("[EXL == 1] delivering TLB MISS @ pc %#lx\n",
2042                           arch->pc);
2043         }
2044
2045         /* set pc to the exception entry point */
2046         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2047
2048         kvm_change_c0_guest_cause(cop0, (0xff),
2049                                   (EXCCODE_TLBL << CAUSEB_EXCCODE));
2050
2051         /* setup badvaddr, context and entryhi registers for the guest */
2052         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2053         /* XXXKYMA: is the context register used by linux??? */
2054         kvm_write_c0_guest_entryhi(cop0, entryhi);
2055
2056         return EMULATE_DONE;
2057 }
2058
2059 enum emulation_result kvm_mips_emulate_tlbmiss_st(u32 cause,
2060                                                   u32 *opc,
2061                                                   struct kvm_run *run,
2062                                                   struct kvm_vcpu *vcpu)
2063 {
2064         struct mips_coproc *cop0 = vcpu->arch.cop0;
2065         struct kvm_vcpu_arch *arch = &vcpu->arch;
2066         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2067                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2068
2069         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2070                 /* save old pc */
2071                 kvm_write_c0_guest_epc(cop0, arch->pc);
2072                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2073
2074                 if (cause & CAUSEF_BD)
2075                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2076                 else
2077                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2078
2079                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
2080                           arch->pc);
2081
2082                 /* Set PC to the exception entry point */
2083                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x0;
2084         } else {
2085                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
2086                           arch->pc);
2087                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2088         }
2089
2090         kvm_change_c0_guest_cause(cop0, (0xff),
2091                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
2092
2093         /* setup badvaddr, context and entryhi registers for the guest */
2094         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2095         /* XXXKYMA: is the context register used by linux??? */
2096         kvm_write_c0_guest_entryhi(cop0, entryhi);
2097
2098         return EMULATE_DONE;
2099 }
2100
2101 enum emulation_result kvm_mips_emulate_tlbinv_st(u32 cause,
2102                                                  u32 *opc,
2103                                                  struct kvm_run *run,
2104                                                  struct kvm_vcpu *vcpu)
2105 {
2106         struct mips_coproc *cop0 = vcpu->arch.cop0;
2107         struct kvm_vcpu_arch *arch = &vcpu->arch;
2108         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2109                 (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2110
2111         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2112                 /* save old pc */
2113                 kvm_write_c0_guest_epc(cop0, arch->pc);
2114                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2115
2116                 if (cause & CAUSEF_BD)
2117                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2118                 else
2119                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2120
2121                 kvm_debug("[EXL == 0] Delivering TLB MISS @ pc %#lx\n",
2122                           arch->pc);
2123         } else {
2124                 kvm_debug("[EXL == 1] Delivering TLB MISS @ pc %#lx\n",
2125                           arch->pc);
2126         }
2127
2128         /* Set PC to the exception entry point */
2129         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2130
2131         kvm_change_c0_guest_cause(cop0, (0xff),
2132                                   (EXCCODE_TLBS << CAUSEB_EXCCODE));
2133
2134         /* setup badvaddr, context and entryhi registers for the guest */
2135         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2136         /* XXXKYMA: is the context register used by linux??? */
2137         kvm_write_c0_guest_entryhi(cop0, entryhi);
2138
2139         return EMULATE_DONE;
2140 }
2141
2142 enum emulation_result kvm_mips_emulate_tlbmod(u32 cause,
2143                                               u32 *opc,
2144                                               struct kvm_run *run,
2145                                               struct kvm_vcpu *vcpu)
2146 {
2147         struct mips_coproc *cop0 = vcpu->arch.cop0;
2148         unsigned long entryhi = (vcpu->arch.host_cp0_badvaddr & VPN2_MASK) |
2149                         (kvm_read_c0_guest_entryhi(cop0) & KVM_ENTRYHI_ASID);
2150         struct kvm_vcpu_arch *arch = &vcpu->arch;
2151
2152         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2153                 /* save old pc */
2154                 kvm_write_c0_guest_epc(cop0, arch->pc);
2155                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2156
2157                 if (cause & CAUSEF_BD)
2158                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2159                 else
2160                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2161
2162                 kvm_debug("[EXL == 0] Delivering TLB MOD @ pc %#lx\n",
2163                           arch->pc);
2164         } else {
2165                 kvm_debug("[EXL == 1] Delivering TLB MOD @ pc %#lx\n",
2166                           arch->pc);
2167         }
2168
2169         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2170
2171         kvm_change_c0_guest_cause(cop0, (0xff),
2172                                   (EXCCODE_MOD << CAUSEB_EXCCODE));
2173
2174         /* setup badvaddr, context and entryhi registers for the guest */
2175         kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2176         /* XXXKYMA: is the context register used by linux??? */
2177         kvm_write_c0_guest_entryhi(cop0, entryhi);
2178
2179         return EMULATE_DONE;
2180 }
2181
2182 enum emulation_result kvm_mips_emulate_fpu_exc(u32 cause,
2183                                                u32 *opc,
2184                                                struct kvm_run *run,
2185                                                struct kvm_vcpu *vcpu)
2186 {
2187         struct mips_coproc *cop0 = vcpu->arch.cop0;
2188         struct kvm_vcpu_arch *arch = &vcpu->arch;
2189
2190         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2191                 /* save old pc */
2192                 kvm_write_c0_guest_epc(cop0, arch->pc);
2193                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2194
2195                 if (cause & CAUSEF_BD)
2196                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2197                 else
2198                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2199
2200         }
2201
2202         arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2203
2204         kvm_change_c0_guest_cause(cop0, (0xff),
2205                                   (EXCCODE_CPU << CAUSEB_EXCCODE));
2206         kvm_change_c0_guest_cause(cop0, (CAUSEF_CE), (0x1 << CAUSEB_CE));
2207
2208         return EMULATE_DONE;
2209 }
2210
2211 enum emulation_result kvm_mips_emulate_ri_exc(u32 cause,
2212                                               u32 *opc,
2213                                               struct kvm_run *run,
2214                                               struct kvm_vcpu *vcpu)
2215 {
2216         struct mips_coproc *cop0 = vcpu->arch.cop0;
2217         struct kvm_vcpu_arch *arch = &vcpu->arch;
2218         enum emulation_result er = EMULATE_DONE;
2219
2220         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2221                 /* save old pc */
2222                 kvm_write_c0_guest_epc(cop0, arch->pc);
2223                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2224
2225                 if (cause & CAUSEF_BD)
2226                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2227                 else
2228                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2229
2230                 kvm_debug("Delivering RI @ pc %#lx\n", arch->pc);
2231
2232                 kvm_change_c0_guest_cause(cop0, (0xff),
2233                                           (EXCCODE_RI << CAUSEB_EXCCODE));
2234
2235                 /* Set PC to the exception entry point */
2236                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2237
2238         } else {
2239                 kvm_err("Trying to deliver RI when EXL is already set\n");
2240                 er = EMULATE_FAIL;
2241         }
2242
2243         return er;
2244 }
2245
2246 enum emulation_result kvm_mips_emulate_bp_exc(u32 cause,
2247                                               u32 *opc,
2248                                               struct kvm_run *run,
2249                                               struct kvm_vcpu *vcpu)
2250 {
2251         struct mips_coproc *cop0 = vcpu->arch.cop0;
2252         struct kvm_vcpu_arch *arch = &vcpu->arch;
2253         enum emulation_result er = EMULATE_DONE;
2254
2255         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2256                 /* save old pc */
2257                 kvm_write_c0_guest_epc(cop0, arch->pc);
2258                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2259
2260                 if (cause & CAUSEF_BD)
2261                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2262                 else
2263                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2264
2265                 kvm_debug("Delivering BP @ pc %#lx\n", arch->pc);
2266
2267                 kvm_change_c0_guest_cause(cop0, (0xff),
2268                                           (EXCCODE_BP << CAUSEB_EXCCODE));
2269
2270                 /* Set PC to the exception entry point */
2271                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2272
2273         } else {
2274                 kvm_err("Trying to deliver BP when EXL is already set\n");
2275                 er = EMULATE_FAIL;
2276         }
2277
2278         return er;
2279 }
2280
2281 enum emulation_result kvm_mips_emulate_trap_exc(u32 cause,
2282                                                 u32 *opc,
2283                                                 struct kvm_run *run,
2284                                                 struct kvm_vcpu *vcpu)
2285 {
2286         struct mips_coproc *cop0 = vcpu->arch.cop0;
2287         struct kvm_vcpu_arch *arch = &vcpu->arch;
2288         enum emulation_result er = EMULATE_DONE;
2289
2290         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2291                 /* save old pc */
2292                 kvm_write_c0_guest_epc(cop0, arch->pc);
2293                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2294
2295                 if (cause & CAUSEF_BD)
2296                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2297                 else
2298                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2299
2300                 kvm_debug("Delivering TRAP @ pc %#lx\n", arch->pc);
2301
2302                 kvm_change_c0_guest_cause(cop0, (0xff),
2303                                           (EXCCODE_TR << CAUSEB_EXCCODE));
2304
2305                 /* Set PC to the exception entry point */
2306                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2307
2308         } else {
2309                 kvm_err("Trying to deliver TRAP when EXL is already set\n");
2310                 er = EMULATE_FAIL;
2311         }
2312
2313         return er;
2314 }
2315
2316 enum emulation_result kvm_mips_emulate_msafpe_exc(u32 cause,
2317                                                   u32 *opc,
2318                                                   struct kvm_run *run,
2319                                                   struct kvm_vcpu *vcpu)
2320 {
2321         struct mips_coproc *cop0 = vcpu->arch.cop0;
2322         struct kvm_vcpu_arch *arch = &vcpu->arch;
2323         enum emulation_result er = EMULATE_DONE;
2324
2325         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2326                 /* save old pc */
2327                 kvm_write_c0_guest_epc(cop0, arch->pc);
2328                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2329
2330                 if (cause & CAUSEF_BD)
2331                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2332                 else
2333                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2334
2335                 kvm_debug("Delivering MSAFPE @ pc %#lx\n", arch->pc);
2336
2337                 kvm_change_c0_guest_cause(cop0, (0xff),
2338                                           (EXCCODE_MSAFPE << CAUSEB_EXCCODE));
2339
2340                 /* Set PC to the exception entry point */
2341                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2342
2343         } else {
2344                 kvm_err("Trying to deliver MSAFPE when EXL is already set\n");
2345                 er = EMULATE_FAIL;
2346         }
2347
2348         return er;
2349 }
2350
2351 enum emulation_result kvm_mips_emulate_fpe_exc(u32 cause,
2352                                                u32 *opc,
2353                                                struct kvm_run *run,
2354                                                struct kvm_vcpu *vcpu)
2355 {
2356         struct mips_coproc *cop0 = vcpu->arch.cop0;
2357         struct kvm_vcpu_arch *arch = &vcpu->arch;
2358         enum emulation_result er = EMULATE_DONE;
2359
2360         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2361                 /* save old pc */
2362                 kvm_write_c0_guest_epc(cop0, arch->pc);
2363                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2364
2365                 if (cause & CAUSEF_BD)
2366                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2367                 else
2368                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2369
2370                 kvm_debug("Delivering FPE @ pc %#lx\n", arch->pc);
2371
2372                 kvm_change_c0_guest_cause(cop0, (0xff),
2373                                           (EXCCODE_FPE << CAUSEB_EXCCODE));
2374
2375                 /* Set PC to the exception entry point */
2376                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2377
2378         } else {
2379                 kvm_err("Trying to deliver FPE when EXL is already set\n");
2380                 er = EMULATE_FAIL;
2381         }
2382
2383         return er;
2384 }
2385
2386 enum emulation_result kvm_mips_emulate_msadis_exc(u32 cause,
2387                                                   u32 *opc,
2388                                                   struct kvm_run *run,
2389                                                   struct kvm_vcpu *vcpu)
2390 {
2391         struct mips_coproc *cop0 = vcpu->arch.cop0;
2392         struct kvm_vcpu_arch *arch = &vcpu->arch;
2393         enum emulation_result er = EMULATE_DONE;
2394
2395         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2396                 /* save old pc */
2397                 kvm_write_c0_guest_epc(cop0, arch->pc);
2398                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2399
2400                 if (cause & CAUSEF_BD)
2401                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2402                 else
2403                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2404
2405                 kvm_debug("Delivering MSADIS @ pc %#lx\n", arch->pc);
2406
2407                 kvm_change_c0_guest_cause(cop0, (0xff),
2408                                           (EXCCODE_MSADIS << CAUSEB_EXCCODE));
2409
2410                 /* Set PC to the exception entry point */
2411                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2412
2413         } else {
2414                 kvm_err("Trying to deliver MSADIS when EXL is already set\n");
2415                 er = EMULATE_FAIL;
2416         }
2417
2418         return er;
2419 }
2420
2421 enum emulation_result kvm_mips_handle_ri(u32 cause, u32 *opc,
2422                                          struct kvm_run *run,
2423                                          struct kvm_vcpu *vcpu)
2424 {
2425         struct mips_coproc *cop0 = vcpu->arch.cop0;
2426         struct kvm_vcpu_arch *arch = &vcpu->arch;
2427         enum emulation_result er = EMULATE_DONE;
2428         unsigned long curr_pc;
2429         union mips_instruction inst;
2430         int err;
2431
2432         /*
2433          * Update PC and hold onto current PC in case there is
2434          * an error and we want to rollback the PC
2435          */
2436         curr_pc = vcpu->arch.pc;
2437         er = update_pc(vcpu, cause);
2438         if (er == EMULATE_FAIL)
2439                 return er;
2440
2441         /* Fetch the instruction. */
2442         if (cause & CAUSEF_BD)
2443                 opc += 1;
2444         err = kvm_get_badinstr(opc, vcpu, &inst.word);
2445         if (err) {
2446                 kvm_err("%s: Cannot get inst @ %p (%d)\n", __func__, opc, err);
2447                 return EMULATE_FAIL;
2448         }
2449
2450         if (inst.r_format.opcode == spec3_op &&
2451             inst.r_format.func == rdhwr_op &&
2452             inst.r_format.rs == 0 &&
2453             (inst.r_format.re >> 3) == 0) {
2454                 int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2455                 int rd = inst.r_format.rd;
2456                 int rt = inst.r_format.rt;
2457                 int sel = inst.r_format.re & 0x7;
2458
2459                 /* If usermode, check RDHWR rd is allowed by guest HWREna */
2460                 if (usermode && !(kvm_read_c0_guest_hwrena(cop0) & BIT(rd))) {
2461                         kvm_debug("RDHWR %#x disallowed by HWREna @ %p\n",
2462                                   rd, opc);
2463                         goto emulate_ri;
2464                 }
2465                 switch (rd) {
2466                 case MIPS_HWR_CPUNUM:           /* CPU number */
2467                         arch->gprs[rt] = vcpu->vcpu_id;
2468                         break;
2469                 case MIPS_HWR_SYNCISTEP:        /* SYNCI length */
2470                         arch->gprs[rt] = min(current_cpu_data.dcache.linesz,
2471                                              current_cpu_data.icache.linesz);
2472                         break;
2473                 case MIPS_HWR_CC:               /* Read count register */
2474                         arch->gprs[rt] = (s32)kvm_mips_read_count(vcpu);
2475                         break;
2476                 case MIPS_HWR_CCRES:            /* Count register resolution */
2477                         switch (current_cpu_data.cputype) {
2478                         case CPU_20KC:
2479                         case CPU_25KF:
2480                                 arch->gprs[rt] = 1;
2481                                 break;
2482                         default:
2483                                 arch->gprs[rt] = 2;
2484                         }
2485                         break;
2486                 case MIPS_HWR_ULR:              /* Read UserLocal register */
2487                         arch->gprs[rt] = kvm_read_c0_guest_userlocal(cop0);
2488                         break;
2489
2490                 default:
2491                         kvm_debug("RDHWR %#x not supported @ %p\n", rd, opc);
2492                         goto emulate_ri;
2493                 }
2494
2495                 trace_kvm_hwr(vcpu, KVM_TRACE_RDHWR, KVM_TRACE_HWR(rd, sel),
2496                               vcpu->arch.gprs[rt]);
2497         } else {
2498                 kvm_debug("Emulate RI not supported @ %p: %#x\n",
2499                           opc, inst.word);
2500                 goto emulate_ri;
2501         }
2502
2503         return EMULATE_DONE;
2504
2505 emulate_ri:
2506         /*
2507          * Rollback PC (if in branch delay slot then the PC already points to
2508          * branch target), and pass the RI exception to the guest OS.
2509          */
2510         vcpu->arch.pc = curr_pc;
2511         return kvm_mips_emulate_ri_exc(cause, opc, run, vcpu);
2512 }
2513
2514 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu,
2515                                                   struct kvm_run *run)
2516 {
2517         unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr];
2518         enum emulation_result er = EMULATE_DONE;
2519
2520         if (run->mmio.len > sizeof(*gpr)) {
2521                 kvm_err("Bad MMIO length: %d", run->mmio.len);
2522                 er = EMULATE_FAIL;
2523                 goto done;
2524         }
2525
2526         /* Restore saved resume PC */
2527         vcpu->arch.pc = vcpu->arch.io_pc;
2528
2529         switch (run->mmio.len) {
2530         case 4:
2531                 *gpr = *(s32 *) run->mmio.data;
2532                 break;
2533
2534         case 2:
2535                 if (vcpu->mmio_needed == 2)
2536                         *gpr = *(s16 *) run->mmio.data;
2537                 else
2538                         *gpr = *(u16 *)run->mmio.data;
2539
2540                 break;
2541         case 1:
2542                 if (vcpu->mmio_needed == 2)
2543                         *gpr = *(s8 *) run->mmio.data;
2544                 else
2545                         *gpr = *(u8 *) run->mmio.data;
2546                 break;
2547         }
2548
2549 done:
2550         return er;
2551 }
2552
2553 static enum emulation_result kvm_mips_emulate_exc(u32 cause,
2554                                                   u32 *opc,
2555                                                   struct kvm_run *run,
2556                                                   struct kvm_vcpu *vcpu)
2557 {
2558         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2559         struct mips_coproc *cop0 = vcpu->arch.cop0;
2560         struct kvm_vcpu_arch *arch = &vcpu->arch;
2561         enum emulation_result er = EMULATE_DONE;
2562
2563         if ((kvm_read_c0_guest_status(cop0) & ST0_EXL) == 0) {
2564                 /* save old pc */
2565                 kvm_write_c0_guest_epc(cop0, arch->pc);
2566                 kvm_set_c0_guest_status(cop0, ST0_EXL);
2567
2568                 if (cause & CAUSEF_BD)
2569                         kvm_set_c0_guest_cause(cop0, CAUSEF_BD);
2570                 else
2571                         kvm_clear_c0_guest_cause(cop0, CAUSEF_BD);
2572
2573                 kvm_change_c0_guest_cause(cop0, (0xff),
2574                                           (exccode << CAUSEB_EXCCODE));
2575
2576                 /* Set PC to the exception entry point */
2577                 arch->pc = kvm_mips_guest_exception_base(vcpu) + 0x180;
2578                 kvm_write_c0_guest_badvaddr(cop0, vcpu->arch.host_cp0_badvaddr);
2579
2580                 kvm_debug("Delivering EXC %d @ pc %#lx, badVaddr: %#lx\n",
2581                           exccode, kvm_read_c0_guest_epc(cop0),
2582                           kvm_read_c0_guest_badvaddr(cop0));
2583         } else {
2584                 kvm_err("Trying to deliver EXC when EXL is already set\n");
2585                 er = EMULATE_FAIL;
2586         }
2587
2588         return er;
2589 }
2590
2591 enum emulation_result kvm_mips_check_privilege(u32 cause,
2592                                                u32 *opc,
2593                                                struct kvm_run *run,
2594                                                struct kvm_vcpu *vcpu)
2595 {
2596         enum emulation_result er = EMULATE_DONE;
2597         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2598         unsigned long badvaddr = vcpu->arch.host_cp0_badvaddr;
2599
2600         int usermode = !KVM_GUEST_KERNEL_MODE(vcpu);
2601
2602         if (usermode) {
2603                 switch (exccode) {
2604                 case EXCCODE_INT:
2605                 case EXCCODE_SYS:
2606                 case EXCCODE_BP:
2607                 case EXCCODE_RI:
2608                 case EXCCODE_TR:
2609                 case EXCCODE_MSAFPE:
2610                 case EXCCODE_FPE:
2611                 case EXCCODE_MSADIS:
2612                         break;
2613
2614                 case EXCCODE_CPU:
2615                         if (((cause & CAUSEF_CE) >> CAUSEB_CE) == 0)
2616                                 er = EMULATE_PRIV_FAIL;
2617                         break;
2618
2619                 case EXCCODE_MOD:
2620                         break;
2621
2622                 case EXCCODE_TLBL:
2623                         /*
2624                          * We we are accessing Guest kernel space, then send an
2625                          * address error exception to the guest
2626                          */
2627                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2628                                 kvm_debug("%s: LD MISS @ %#lx\n", __func__,
2629                                           badvaddr);
2630                                 cause &= ~0xff;
2631                                 cause |= (EXCCODE_ADEL << CAUSEB_EXCCODE);
2632                                 er = EMULATE_PRIV_FAIL;
2633                         }
2634                         break;
2635
2636                 case EXCCODE_TLBS:
2637                         /*
2638                          * We we are accessing Guest kernel space, then send an
2639                          * address error exception to the guest
2640                          */
2641                         if (badvaddr >= (unsigned long) KVM_GUEST_KSEG0) {
2642                                 kvm_debug("%s: ST MISS @ %#lx\n", __func__,
2643                                           badvaddr);
2644                                 cause &= ~0xff;
2645                                 cause |= (EXCCODE_ADES << CAUSEB_EXCCODE);
2646                                 er = EMULATE_PRIV_FAIL;
2647                         }
2648                         break;
2649
2650                 case EXCCODE_ADES:
2651                         kvm_debug("%s: address error ST @ %#lx\n", __func__,
2652                                   badvaddr);
2653                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2654                                 cause &= ~0xff;
2655                                 cause |= (EXCCODE_TLBS << CAUSEB_EXCCODE);
2656                         }
2657                         er = EMULATE_PRIV_FAIL;
2658                         break;
2659                 case EXCCODE_ADEL:
2660                         kvm_debug("%s: address error LD @ %#lx\n", __func__,
2661                                   badvaddr);
2662                         if ((badvaddr & PAGE_MASK) == KVM_GUEST_COMMPAGE_ADDR) {
2663                                 cause &= ~0xff;
2664                                 cause |= (EXCCODE_TLBL << CAUSEB_EXCCODE);
2665                         }
2666                         er = EMULATE_PRIV_FAIL;
2667                         break;
2668                 default:
2669                         er = EMULATE_PRIV_FAIL;
2670                         break;
2671                 }
2672         }
2673
2674         if (er == EMULATE_PRIV_FAIL)
2675                 kvm_mips_emulate_exc(cause, opc, run, vcpu);
2676
2677         return er;
2678 }
2679
2680 /*
2681  * User Address (UA) fault, this could happen if
2682  * (1) TLB entry not present/valid in both Guest and shadow host TLBs, in this
2683  *     case we pass on the fault to the guest kernel and let it handle it.
2684  * (2) TLB entry is present in the Guest TLB but not in the shadow, in this
2685  *     case we inject the TLB from the Guest TLB into the shadow host TLB
2686  */
2687 enum emulation_result kvm_mips_handle_tlbmiss(u32 cause,
2688                                               u32 *opc,
2689                                               struct kvm_run *run,
2690                                               struct kvm_vcpu *vcpu,
2691                                               bool write_fault)
2692 {
2693         enum emulation_result er = EMULATE_DONE;
2694         u32 exccode = (cause >> CAUSEB_EXCCODE) & 0x1f;
2695         unsigned long va = vcpu->arch.host_cp0_badvaddr;
2696         int index;
2697
2698         kvm_debug("kvm_mips_handle_tlbmiss: badvaddr: %#lx\n",
2699                   vcpu->arch.host_cp0_badvaddr);
2700
2701         /*
2702          * KVM would not have got the exception if this entry was valid in the
2703          * shadow host TLB. Check the Guest TLB, if the entry is not there then
2704          * send the guest an exception. The guest exc handler should then inject
2705          * an entry into the guest TLB.
2706          */
2707         index = kvm_mips_guest_tlb_lookup(vcpu,
2708                       (va & VPN2_MASK) |
2709                       (kvm_read_c0_guest_entryhi(vcpu->arch.cop0) &
2710                        KVM_ENTRYHI_ASID));
2711         if (index < 0) {
2712                 if (exccode == EXCCODE_TLBL) {
2713                         er = kvm_mips_emulate_tlbmiss_ld(cause, opc, run, vcpu);
2714                 } else if (exccode == EXCCODE_TLBS) {
2715                         er = kvm_mips_emulate_tlbmiss_st(cause, opc, run, vcpu);
2716                 } else {
2717                         kvm_err("%s: invalid exc code: %d\n", __func__,
2718                                 exccode);
2719                         er = EMULATE_FAIL;
2720                 }
2721         } else {
2722                 struct kvm_mips_tlb *tlb = &vcpu->arch.guest_tlb[index];
2723
2724                 /*
2725                  * Check if the entry is valid, if not then setup a TLB invalid
2726                  * exception to the guest
2727                  */
2728                 if (!TLB_IS_VALID(*tlb, va)) {
2729                         if (exccode == EXCCODE_TLBL) {
2730                                 er = kvm_mips_emulate_tlbinv_ld(cause, opc, run,
2731                                                                 vcpu);
2732                         } else if (exccode == EXCCODE_TLBS) {
2733                                 er = kvm_mips_emulate_tlbinv_st(cause, opc, run,
2734                                                                 vcpu);
2735                         } else {
2736                                 kvm_err("%s: invalid exc code: %d\n", __func__,
2737                                         exccode);
2738                                 er = EMULATE_FAIL;
2739                         }
2740                 } else {
2741                         kvm_debug("Injecting hi: %#lx, lo0: %#lx, lo1: %#lx into shadow host TLB\n",
2742                                   tlb->tlb_hi, tlb->tlb_lo[0], tlb->tlb_lo[1]);
2743                         /*
2744                          * OK we have a Guest TLB entry, now inject it into the
2745                          * shadow host TLB
2746                          */
2747                         if (kvm_mips_handle_mapped_seg_tlb_fault(vcpu, tlb, va,
2748                                                                  write_fault)) {
2749                                 kvm_err("%s: handling mapped seg tlb fault for %lx, index: %u, vcpu: %p, ASID: %#lx\n",
2750                                         __func__, va, index, vcpu,
2751                                         read_c0_entryhi());
2752                                 er = EMULATE_FAIL;
2753                         }
2754                 }
2755         }
2756
2757         return er;
2758 }