arch/x86/kvm/svm.c

   1 /*
   2  * Kernel-based Virtual Machine driver for Linux
   3  *
   4  * AMD SVM support
   5  *
   6  * Copyright (C) 2006 Qumranet, Inc.
   7  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   8  *
   9  * Authors:
  10  *   Yaniv Kamay  <yaniv@qumranet.com>
  11  *   Avi Kivity   <avi@qumranet.com>
  12  *
  13  * This work is licensed under the terms of the GNU GPL, version 2.  See
  14  * the COPYING file in the top-level directory.
  15  *
  16  */
  17
  18 #define pr_fmt(fmt) "SVM: " fmt
  19
  20 #include <linux/kvm_host.h>
  21
  22 #include "irq.h"
  23 #include "mmu.h"
  24 #include "kvm_cache_regs.h"
  25 #include "x86.h"
  26 #include "cpuid.h"
  27 #include "pmu.h"
  28
  29 #include <linux/module.h>
  30 #include <linux/mod_devicetable.h>
  31 #include <linux/kernel.h>
  32 #include <linux/vmalloc.h>
  33 #include <linux/highmem.h>
  34 #include <linux/sched.h>
  35 #include <linux/trace_events.h>
  36 #include <linux/slab.h>
  37
  38 #include <asm/perf_event.h>
  39 #include <asm/tlbflush.h>
  40 #include <asm/desc.h>
  41 #include <asm/debugreg.h>
  42 #include <asm/kvm_para.h>
  43
  44 #include <asm/virtext.h>
  45 #include "trace.h"
  46
  47 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  48
  49 MODULE_AUTHOR("Qumranet");
  50 MODULE_LICENSE("GPL");
  51
  52 static const struct x86_cpu_id svm_cpu_id[] = {
  53         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  54         {}
  55 };
  56 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  57
  58 #define IOPM_ALLOC_ORDER 2
  59 #define MSRPM_ALLOC_ORDER 1
  60
  61 #define SEG_TYPE_LDT 2
  62 #define SEG_TYPE_BUSY_TSS16 3
  63
  64 #define SVM_FEATURE_NPT            (1 <<  0)
  65 #define SVM_FEATURE_LBRV           (1 <<  1)
  66 #define SVM_FEATURE_SVML           (1 <<  2)
  67 #define SVM_FEATURE_NRIP           (1 <<  3)
  68 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  69 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  70 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  71 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  72 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  73
  74 #define SVM_AVIC_DOORBELL       0xc001011b
  75
  76 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  77 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  78 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  79
  80 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  81
  82 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  83 #define TSC_RATIO_MIN           0x0000000000000001ULL
  84 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  85
  86 #define AVIC_HPA_MASK   ~((0xFFFULL << 52) || 0xFFF)
  87
  88 /*
  89  * 0xff is broadcast, so the max index allowed for physical APIC ID
  90  * table is 0xfe.  APIC IDs above 0xff are reserved.
  91  */
  92 #define AVIC_MAX_PHYSICAL_ID_COUNT      255
  93
  94 static bool erratum_383_found __read_mostly;
  95
  96 static const u32 host_save_user_msrs[] = {
  97 #ifdef CONFIG_X86_64
  98         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
  99         MSR_FS_BASE,
 100 #endif
 101         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 102         MSR_TSC_AUX,
 103 };
 104
 105 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 106
 107 struct kvm_vcpu;
 108
 109 struct nested_state {
 110         struct vmcb *hsave;
 111         u64 hsave_msr;
 112         u64 vm_cr_msr;
 113         u64 vmcb;
 114
 115         /* These are the merged vectors */
 116         u32 *msrpm;
 117
 118         /* gpa pointers to the real vectors */
 119         u64 vmcb_msrpm;
 120         u64 vmcb_iopm;
 121
 122         /* A VMEXIT is required but not yet emulated */
 123         bool exit_required;
 124
 125         /* cache for intercepts of the guest */
 126         u32 intercept_cr;
 127         u32 intercept_dr;
 128         u32 intercept_exceptions;
 129         u64 intercept;
 130
 131         /* Nested Paging related state */
 132         u64 nested_cr3;
 133 };
 134
 135 #define MSRPM_OFFSETS   16
 136 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 137
 138 /*
 139  * Set osvw_len to higher value when updated Revision Guides
 140  * are published and we know what the new status bits are
 141  */
 142 static uint64_t osvw_len = 4, osvw_status;
 143
 144 struct vcpu_svm {
 145         struct kvm_vcpu vcpu;
 146         struct vmcb *vmcb;
 147         unsigned long vmcb_pa;
 148         struct svm_cpu_data *svm_data;
 149         uint64_t asid_generation;
 150         uint64_t sysenter_esp;
 151         uint64_t sysenter_eip;
 152         uint64_t tsc_aux;
 153
 154         u64 next_rip;
 155
 156         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 157         struct {
 158                 u16 fs;
 159                 u16 gs;
 160                 u16 ldt;
 161                 u64 gs_base;
 162         } host;
 163
 164         u32 *msrpm;
 165
 166         ulong nmi_iret_rip;
 167
 168         struct nested_state nested;
 169
 170         bool nmi_singlestep;
 171
 172         unsigned int3_injected;
 173         unsigned long int3_rip;
 174         u32 apf_reason;
 175
 176         /* cached guest cpuid flags for faster access */
 177         bool nrips_enabled      : 1;
 178
 179         struct page *avic_backing_page;
 180         u64 *avic_physical_id_cache;
 181 };
 182
 183 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK    (0xFF)
 184 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK                (1 << 31)
 185
 186 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK    (0xFFULL)
 187 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK        (0xFFFFFFFFFFULL << 12)
 188 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK          (1ULL << 62)
 189 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK               (1ULL << 63)
 190
 191 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 192 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 193
 194 #define MSR_INVALID                     0xffffffffU
 195
 196 static const struct svm_direct_access_msrs {
 197         u32 index;   /* Index of the MSR */
 198         bool always; /* True if intercept is always on */
 199 } direct_access_msrs[] = {
 200         { .index = MSR_STAR,                            .always = true  },
 201         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 202 #ifdef CONFIG_X86_64
 203         { .index = MSR_GS_BASE,                         .always = true  },
 204         { .index = MSR_FS_BASE,                         .always = true  },
 205         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 206         { .index = MSR_LSTAR,                           .always = true  },
 207         { .index = MSR_CSTAR,                           .always = true  },
 208         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 209 #endif
 210         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 211         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 212         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 213         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 214         { .index = MSR_INVALID,                         .always = false },
 215 };
 216
 217 /* enable NPT for AMD64 and X86 with PAE */
 218 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 219 static bool npt_enabled = true;
 220 #else
 221 static bool npt_enabled;
 222 #endif
 223
 224 /* allow nested paging (virtualized MMU) for all guests */
 225 static int npt = true;
 226 module_param(npt, int, S_IRUGO);
 227
 228 /* allow nested virtualization in KVM/SVM */
 229 static int nested = true;
 230 module_param(nested, int, S_IRUGO);
 231
 232 /* enable / disable AVIC */
 233 static int avic;
 234 module_param(avic, int, S_IRUGO);
 235
 236 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 237 static void svm_flush_tlb(struct kvm_vcpu *vcpu);
 238 static void svm_complete_interrupts(struct vcpu_svm *svm);
 239
 240 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 241 static int nested_svm_intercept(struct vcpu_svm *svm);
 242 static int nested_svm_vmexit(struct vcpu_svm *svm);
 243 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 244                                       bool has_error_code, u32 error_code);
 245
 246 enum {
 247         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 248                             pause filter count */
 249         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 250         VMCB_ASID,       /* ASID */
 251         VMCB_INTR,       /* int_ctl, int_vector */
 252         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 253         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 254         VMCB_DR,         /* DR6, DR7 */
 255         VMCB_DT,         /* GDT, IDT */
 256         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 257         VMCB_CR2,        /* CR2 only */
 258         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 259         VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
 260                           * AVIC PHYSICAL_TABLE pointer,
 261                           * AVIC LOGICAL_TABLE pointer
 262                           */
 263         VMCB_DIRTY_MAX,
 264 };
 265
 266 /* TPR and CR2 are always written before VMRUN */
 267 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 268
 269 #define VMCB_AVIC_APIC_BAR_MASK         0xFFFFFFFFFF000ULL
 270
 271 static inline void mark_all_dirty(struct vmcb *vmcb)
 272 {
 273         vmcb->control.clean = 0;
 274 }
 275
 276 static inline void mark_all_clean(struct vmcb *vmcb)
 277 {
 278         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 279                                & ~VMCB_ALWAYS_DIRTY_MASK;
 280 }
 281
 282 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 283 {
 284         vmcb->control.clean &= ~(1 << bit);
 285 }
 286
 287 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 288 {
 289         return container_of(vcpu, struct vcpu_svm, vcpu);
 290 }
 291
 292 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
 293 {
 294         svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
 295         mark_dirty(svm->vmcb, VMCB_AVIC);
 296 }
 297
 298 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
 299 {
 300         struct vcpu_svm *svm = to_svm(vcpu);
 301         u64 *entry = svm->avic_physical_id_cache;
 302
 303         if (!entry)
 304                 return false;
 305
 306         return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 307 }
 308
 309 static void recalc_intercepts(struct vcpu_svm *svm)
 310 {
 311         struct vmcb_control_area *c, *h;
 312         struct nested_state *g;
 313
 314         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 315
 316         if (!is_guest_mode(&svm->vcpu))
 317                 return;
 318
 319         c = &svm->vmcb->control;
 320         h = &svm->nested.hsave->control;
 321         g = &svm->nested;
 322
 323         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 324         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 325         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 326         c->intercept = h->intercept | g->intercept;
 327 }
 328
 329 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 330 {
 331         if (is_guest_mode(&svm->vcpu))
 332                 return svm->nested.hsave;
 333         else
 334                 return svm->vmcb;
 335 }
 336
 337 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 338 {
 339         struct vmcb *vmcb = get_host_vmcb(svm);
 340
 341         vmcb->control.intercept_cr |= (1U << bit);
 342
 343         recalc_intercepts(svm);
 344 }
 345
 346 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 347 {
 348         struct vmcb *vmcb = get_host_vmcb(svm);
 349
 350         vmcb->control.intercept_cr &= ~(1U << bit);
 351
 352         recalc_intercepts(svm);
 353 }
 354
 355 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 356 {
 357         struct vmcb *vmcb = get_host_vmcb(svm);
 358
 359         return vmcb->control.intercept_cr & (1U << bit);
 360 }
 361
 362 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 363 {
 364         struct vmcb *vmcb = get_host_vmcb(svm);
 365
 366         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 367                 | (1 << INTERCEPT_DR1_READ)
 368                 | (1 << INTERCEPT_DR2_READ)
 369                 | (1 << INTERCEPT_DR3_READ)
 370                 | (1 << INTERCEPT_DR4_READ)
 371                 | (1 << INTERCEPT_DR5_READ)
 372                 | (1 << INTERCEPT_DR6_READ)
 373                 | (1 << INTERCEPT_DR7_READ)
 374                 | (1 << INTERCEPT_DR0_WRITE)
 375                 | (1 << INTERCEPT_DR1_WRITE)
 376                 | (1 << INTERCEPT_DR2_WRITE)
 377                 | (1 << INTERCEPT_DR3_WRITE)
 378                 | (1 << INTERCEPT_DR4_WRITE)
 379                 | (1 << INTERCEPT_DR5_WRITE)
 380                 | (1 << INTERCEPT_DR6_WRITE)
 381                 | (1 << INTERCEPT_DR7_WRITE);
 382
 383         recalc_intercepts(svm);
 384 }
 385
 386 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 387 {
 388         struct vmcb *vmcb = get_host_vmcb(svm);
 389
 390         vmcb->control.intercept_dr = 0;
 391
 392         recalc_intercepts(svm);
 393 }
 394
 395 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 396 {
 397         struct vmcb *vmcb = get_host_vmcb(svm);
 398
 399         vmcb->control.intercept_exceptions |= (1U << bit);
 400
 401         recalc_intercepts(svm);
 402 }
 403
 404 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 405 {
 406         struct vmcb *vmcb = get_host_vmcb(svm);
 407
 408         vmcb->control.intercept_exceptions &= ~(1U << bit);
 409
 410         recalc_intercepts(svm);
 411 }
 412
 413 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 414 {
 415         struct vmcb *vmcb = get_host_vmcb(svm);
 416
 417         vmcb->control.intercept |= (1ULL << bit);
 418
 419         recalc_intercepts(svm);
 420 }
 421
 422 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 423 {
 424         struct vmcb *vmcb = get_host_vmcb(svm);
 425
 426         vmcb->control.intercept &= ~(1ULL << bit);
 427
 428         recalc_intercepts(svm);
 429 }
 430
 431 static inline void enable_gif(struct vcpu_svm *svm)
 432 {
 433         svm->vcpu.arch.hflags |= HF_GIF_MASK;
 434 }
 435
 436 static inline void disable_gif(struct vcpu_svm *svm)
 437 {
 438         svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 439 }
 440
 441 static inline bool gif_set(struct vcpu_svm *svm)
 442 {
 443         return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 444 }
 445
 446 static unsigned long iopm_base;
 447
 448 struct kvm_ldttss_desc {
 449         u16 limit0;
 450         u16 base0;
 451         unsigned base1:8, type:5, dpl:2, p:1;
 452         unsigned limit1:4, zero0:3, g:1, base2:8;
 453         u32 base3;
 454         u32 zero1;
 455 } __attribute__((packed));
 456
 457 struct svm_cpu_data {
 458         int cpu;
 459
 460         u64 asid_generation;
 461         u32 max_asid;
 462         u32 next_asid;
 463         struct kvm_ldttss_desc *tss_desc;
 464
 465         struct page *save_area;
 466 };
 467
 468 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 469
 470 struct svm_init_data {
 471         int cpu;
 472         int r;
 473 };
 474
 475 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 476
 477 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 478 #define MSRS_RANGE_SIZE 2048
 479 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 480
 481 static u32 svm_msrpm_offset(u32 msr)
 482 {
 483         u32 offset;
 484         int i;
 485
 486         for (i = 0; i < NUM_MSR_MAPS; i++) {
 487                 if (msr < msrpm_ranges[i] ||
 488                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 489                         continue;
 490
 491                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 492                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 493
 494                 /* Now we have the u8 offset - but need the u32 offset */
 495                 return offset / 4;
 496         }
 497
 498         /* MSR not in any range */
 499         return MSR_INVALID;
 500 }
 501
 502 #define MAX_INST_SIZE 15
 503
 504 static inline void clgi(void)
 505 {
 506         asm volatile (__ex(SVM_CLGI));
 507 }
 508
 509 static inline void stgi(void)
 510 {
 511         asm volatile (__ex(SVM_STGI));
 512 }
 513
 514 static inline void invlpga(unsigned long addr, u32 asid)
 515 {
 516         asm volatile (__ex(SVM_INVLPGA) : : "a"(addr), "c"(asid));
 517 }
 518
 519 static int get_npt_level(void)
 520 {
 521 #ifdef CONFIG_X86_64
 522         return PT64_ROOT_LEVEL;
 523 #else
 524         return PT32E_ROOT_LEVEL;
 525 #endif
 526 }
 527
 528 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 529 {
 530         vcpu->arch.efer = efer;
 531         if (!npt_enabled && !(efer & EFER_LMA))
 532                 efer &= ~EFER_LME;
 533
 534         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 535         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 536 }
 537
 538 static int is_external_interrupt(u32 info)
 539 {
 540         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 541         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 542 }
 543
 544 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 545 {
 546         struct vcpu_svm *svm = to_svm(vcpu);
 547         u32 ret = 0;
 548
 549         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 550                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 551         return ret;
 552 }
 553
 554 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 555 {
 556         struct vcpu_svm *svm = to_svm(vcpu);
 557
 558         if (mask == 0)
 559                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 560         else
 561                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 562
 563 }
 564
 565 static void skip_emulated_instruction(struct kvm_vcpu *vcpu)
 566 {
 567         struct vcpu_svm *svm = to_svm(vcpu);
 568
 569         if (svm->vmcb->control.next_rip != 0) {
 570                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 571                 svm->next_rip = svm->vmcb->control.next_rip;
 572         }
 573
 574         if (!svm->next_rip) {
 575                 if (emulate_instruction(vcpu, EMULTYPE_SKIP) !=
 576                                 EMULATE_DONE)
 577                         printk(KERN_DEBUG "%s: NOP\n", __func__);
 578                 return;
 579         }
 580         if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 581                 printk(KERN_ERR "%s: ip 0x%lx next 0x%llx\n",
 582                        __func__, kvm_rip_read(vcpu), svm->next_rip);
 583
 584         kvm_rip_write(vcpu, svm->next_rip);
 585         svm_set_interrupt_shadow(vcpu, 0);
 586 }
 587
 588 static void svm_queue_exception(struct kvm_vcpu *vcpu, unsigned nr,
 589                                 bool has_error_code, u32 error_code,
 590                                 bool reinject)
 591 {
 592         struct vcpu_svm *svm = to_svm(vcpu);
 593
 594         /*
 595          * If we are within a nested VM we'd better #VMEXIT and let the guest
 596          * handle the exception
 597          */
 598         if (!reinject &&
 599             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 600                 return;
 601
 602         if (nr == BP_VECTOR && !static_cpu_has(X86_FEATURE_NRIPS)) {
 603                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 604
 605                 /*
 606                  * For guest debugging where we have to reinject #BP if some
 607                  * INT3 is guest-owned:
 608                  * Emulate nRIP by moving RIP forward. Will fail if injection
 609                  * raises a fault that is not intercepted. Still better than
 610                  * failing in all cases.
 611                  */
 612                 skip_emulated_instruction(&svm->vcpu);
 613                 rip = kvm_rip_read(&svm->vcpu);
 614                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 615                 svm->int3_injected = rip - old_rip;
 616         }
 617
 618         svm->vmcb->control.event_inj = nr
 619                 | SVM_EVTINJ_VALID
 620                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 621                 | SVM_EVTINJ_TYPE_EXEPT;
 622         svm->vmcb->control.event_inj_err = error_code;
 623 }
 624
 625 static void svm_init_erratum_383(void)
 626 {
 627         u32 low, high;
 628         int err;
 629         u64 val;
 630
 631         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 632                 return;
 633
 634         /* Use _safe variants to not break nested virtualization */
 635         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 636         if (err)
 637                 return;
 638
 639         val |= (1ULL << 47);
 640
 641         low  = lower_32_bits(val);
 642         high = upper_32_bits(val);
 643
 644         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 645
 646         erratum_383_found = true;
 647 }
 648
 649 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 650 {
 651         /*
 652          * Guests should see errata 400 and 415 as fixed (assuming that
 653          * HLT and IO instructions are intercepted).
 654          */
 655         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 656         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 657
 658         /*
 659          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 660          * all osvw.status bits inside that length, including bit 0 (which is
 661          * reserved for erratum 298), are valid. However, if host processor's
 662          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 663          * be conservative here and therefore we tell the guest that erratum 298
 664          * is present (because we really don't know).
 665          */
 666         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 667                 vcpu->arch.osvw.status |= 1;
 668 }
 669
 670 static int has_svm(void)
 671 {
 672         const char *msg;
 673
 674         if (!cpu_has_svm(&msg)) {
 675                 printk(KERN_INFO "has_svm: %s\n", msg);
 676                 return 0;
 677         }
 678
 679         return 1;
 680 }
 681
 682 static void svm_hardware_disable(void)
 683 {
 684         /* Make sure we clean up behind us */
 685         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 686                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 687
 688         cpu_svm_disable();
 689
 690         amd_pmu_disable_virt();
 691 }
 692
 693 static int svm_hardware_enable(void)
 694 {
 695
 696         struct svm_cpu_data *sd;
 697         uint64_t efer;
 698         struct desc_ptr gdt_descr;
 699         struct desc_struct *gdt;
 700         int me = raw_smp_processor_id();
 701
 702         rdmsrl(MSR_EFER, efer);
 703         if (efer & EFER_SVME)
 704                 return -EBUSY;
 705
 706         if (!has_svm()) {
 707                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 708                 return -EINVAL;
 709         }
 710         sd = per_cpu(svm_data, me);
 711         if (!sd) {
 712                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 713                 return -EINVAL;
 714         }
 715
 716         sd->asid_generation = 1;
 717         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 718         sd->next_asid = sd->max_asid + 1;
 719
 720         native_store_gdt(&gdt_descr);
 721         gdt = (struct desc_struct *)gdt_descr.address;
 722         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 723
 724         wrmsrl(MSR_EFER, efer | EFER_SVME);
 725
 726         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 727
 728         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 729                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 730                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 731         }
 732
 733
 734         /*
 735          * Get OSVW bits.
 736          *
 737          * Note that it is possible to have a system with mixed processor
 738          * revisions and therefore different OSVW bits. If bits are not the same
 739          * on different processors then choose the worst case (i.e. if erratum
 740          * is present on one processor and not on another then assume that the
 741          * erratum is present everywhere).
 742          */
 743         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 744                 uint64_t len, status = 0;
 745                 int err;
 746
 747                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 748                 if (!err)
 749                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 750                                                       &err);
 751
 752                 if (err)
 753                         osvw_status = osvw_len = 0;
 754                 else {
 755                         if (len < osvw_len)
 756                                 osvw_len = len;
 757                         osvw_status |= status;
 758                         osvw_status &= (1ULL << osvw_len) - 1;
 759                 }
 760         } else
 761                 osvw_status = osvw_len = 0;
 762
 763         svm_init_erratum_383();
 764
 765         amd_pmu_enable_virt();
 766
 767         return 0;
 768 }
 769
 770 static void svm_cpu_uninit(int cpu)
 771 {
 772         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 773
 774         if (!sd)
 775                 return;
 776
 777         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 778         __free_page(sd->save_area);
 779         kfree(sd);
 780 }
 781
 782 static int svm_cpu_init(int cpu)
 783 {
 784         struct svm_cpu_data *sd;
 785         int r;
 786
 787         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
 788         if (!sd)
 789                 return -ENOMEM;
 790         sd->cpu = cpu;
 791         sd->save_area = alloc_page(GFP_KERNEL);
 792         r = -ENOMEM;
 793         if (!sd->save_area)
 794                 goto err_1;
 795
 796         per_cpu(svm_data, cpu) = sd;
 797
 798         return 0;
 799
 800 err_1:
 801         kfree(sd);
 802         return r;
 803
 804 }
 805
 806 static bool valid_msr_intercept(u32 index)
 807 {
 808         int i;
 809
 810         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
 811                 if (direct_access_msrs[i].index == index)
 812                         return true;
 813
 814         return false;
 815 }
 816
 817 static void set_msr_interception(u32 *msrpm, unsigned msr,
 818                                  int read, int write)
 819 {
 820         u8 bit_read, bit_write;
 821         unsigned long tmp;
 822         u32 offset;
 823
 824         /*
 825          * If this warning triggers extend the direct_access_msrs list at the
 826          * beginning of the file
 827          */
 828         WARN_ON(!valid_msr_intercept(msr));
 829
 830         offset    = svm_msrpm_offset(msr);
 831         bit_read  = 2 * (msr & 0x0f);
 832         bit_write = 2 * (msr & 0x0f) + 1;
 833         tmp       = msrpm[offset];
 834
 835         BUG_ON(offset == MSR_INVALID);
 836
 837         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
 838         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
 839
 840         msrpm[offset] = tmp;
 841 }
 842
 843 static void svm_vcpu_init_msrpm(u32 *msrpm)
 844 {
 845         int i;
 846
 847         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
 848
 849         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 850                 if (!direct_access_msrs[i].always)
 851                         continue;
 852
 853                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
 854         }
 855 }
 856
 857 static void add_msr_offset(u32 offset)
 858 {
 859         int i;
 860
 861         for (i = 0; i < MSRPM_OFFSETS; ++i) {
 862
 863                 /* Offset already in list? */
 864                 if (msrpm_offsets[i] == offset)
 865                         return;
 866
 867                 /* Slot used by another offset? */
 868                 if (msrpm_offsets[i] != MSR_INVALID)
 869                         continue;
 870
 871                 /* Add offset to list */
 872                 msrpm_offsets[i] = offset;
 873
 874                 return;
 875         }
 876
 877         /*
 878          * If this BUG triggers the msrpm_offsets table has an overflow. Just
 879          * increase MSRPM_OFFSETS in this case.
 880          */
 881         BUG();
 882 }
 883
 884 static void init_msrpm_offsets(void)
 885 {
 886         int i;
 887
 888         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
 889
 890         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
 891                 u32 offset;
 892
 893                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
 894                 BUG_ON(offset == MSR_INVALID);
 895
 896                 add_msr_offset(offset);
 897         }
 898 }
 899
 900 static void svm_enable_lbrv(struct vcpu_svm *svm)
 901 {
 902         u32 *msrpm = svm->msrpm;
 903
 904         svm->vmcb->control.lbr_ctl = 1;
 905         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
 906         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
 907         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
 908         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
 909 }
 910
 911 static void svm_disable_lbrv(struct vcpu_svm *svm)
 912 {
 913         u32 *msrpm = svm->msrpm;
 914
 915         svm->vmcb->control.lbr_ctl = 0;
 916         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
 917         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
 918         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
 919         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
 920 }
 921
 922 static __init int svm_hardware_setup(void)
 923 {
 924         int cpu;
 925         struct page *iopm_pages;
 926         void *iopm_va;
 927         int r;
 928
 929         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
 930
 931         if (!iopm_pages)
 932                 return -ENOMEM;
 933
 934         iopm_va = page_address(iopm_pages);
 935         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
 936         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
 937
 938         init_msrpm_offsets();
 939
 940         if (boot_cpu_has(X86_FEATURE_NX))
 941                 kvm_enable_efer_bits(EFER_NX);
 942
 943         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
 944                 kvm_enable_efer_bits(EFER_FFXSR);
 945
 946         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 947                 kvm_has_tsc_control = true;
 948                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
 949                 kvm_tsc_scaling_ratio_frac_bits = 32;
 950         }
 951
 952         if (nested) {
 953                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
 954                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
 955         }
 956
 957         for_each_possible_cpu(cpu) {
 958                 r = svm_cpu_init(cpu);
 959                 if (r)
 960                         goto err;
 961         }
 962
 963         if (!boot_cpu_has(X86_FEATURE_NPT))
 964                 npt_enabled = false;
 965
 966         if (npt_enabled && !npt) {
 967                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
 968                 npt_enabled = false;
 969         }
 970
 971         if (npt_enabled) {
 972                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
 973                 kvm_enable_tdp();
 974         } else
 975                 kvm_disable_tdp();
 976
 977         if (avic && (!npt_enabled || !boot_cpu_has(X86_FEATURE_AVIC)))
 978                 avic = false;
 979
 980         if (avic)
 981                 pr_info("AVIC enabled\n");
 982
 983         return 0;
 984
 985 err:
 986         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
 987         iopm_base = 0;
 988         return r;
 989 }
 990
 991 static __exit void svm_hardware_unsetup(void)
 992 {
 993         int cpu;
 994
 995         for_each_possible_cpu(cpu)
 996                 svm_cpu_uninit(cpu);
 997
 998         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
 999         iopm_base = 0;
1000 }
1001
1002 static void init_seg(struct vmcb_seg *seg)
1003 {
1004         seg->selector = 0;
1005         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1006                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1007         seg->limit = 0xffff;
1008         seg->base = 0;
1009 }
1010
1011 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1012 {
1013         seg->selector = 0;
1014         seg->attrib = SVM_SELECTOR_P_MASK | type;
1015         seg->limit = 0xffff;
1016         seg->base = 0;
1017 }
1018
1019 static u64 svm_read_tsc_offset(struct kvm_vcpu *vcpu)
1020 {
1021         struct vcpu_svm *svm = to_svm(vcpu);
1022
1023         return svm->vmcb->control.tsc_offset;
1024 }
1025
1026 static void svm_write_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1027 {
1028         struct vcpu_svm *svm = to_svm(vcpu);
1029         u64 g_tsc_offset = 0;
1030
1031         if (is_guest_mode(vcpu)) {
1032                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1033                                svm->nested.hsave->control.tsc_offset;
1034                 svm->nested.hsave->control.tsc_offset = offset;
1035         } else
1036                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1037                                            svm->vmcb->control.tsc_offset,
1038                                            offset);
1039
1040         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1041
1042         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1043 }
1044
1045 static void svm_adjust_tsc_offset_guest(struct kvm_vcpu *vcpu, s64 adjustment)
1046 {
1047         struct vcpu_svm *svm = to_svm(vcpu);
1048
1049         svm->vmcb->control.tsc_offset += adjustment;
1050         if (is_guest_mode(vcpu))
1051                 svm->nested.hsave->control.tsc_offset += adjustment;
1052         else
1053                 trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1054                                      svm->vmcb->control.tsc_offset - adjustment,
1055                                      svm->vmcb->control.tsc_offset);
1056
1057         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1058 }
1059
1060 static void avic_init_vmcb(struct vcpu_svm *svm)
1061 {
1062         struct vmcb *vmcb = svm->vmcb;
1063         struct kvm_arch *vm_data = &svm->vcpu.kvm->arch;
1064         phys_addr_t bpa = page_to_phys(svm->avic_backing_page);
1065         phys_addr_t lpa = page_to_phys(vm_data->avic_logical_id_table_page);
1066         phys_addr_t ppa = page_to_phys(vm_data->avic_physical_id_table_page);
1067
1068         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1069         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1070         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1071         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1072         vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1073         svm->vcpu.arch.apicv_active = true;
1074 }
1075
1076 static void init_vmcb(struct vcpu_svm *svm)
1077 {
1078         struct vmcb_control_area *control = &svm->vmcb->control;
1079         struct vmcb_save_area *save = &svm->vmcb->save;
1080
1081         svm->vcpu.fpu_active = 1;
1082         svm->vcpu.arch.hflags = 0;
1083
1084         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1085         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1086         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1087         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1088         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1089         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1090         set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1091
1092         set_dr_intercepts(svm);
1093
1094         set_exception_intercept(svm, PF_VECTOR);
1095         set_exception_intercept(svm, UD_VECTOR);
1096         set_exception_intercept(svm, MC_VECTOR);
1097         set_exception_intercept(svm, AC_VECTOR);
1098         set_exception_intercept(svm, DB_VECTOR);
1099
1100         set_intercept(svm, INTERCEPT_INTR);
1101         set_intercept(svm, INTERCEPT_NMI);
1102         set_intercept(svm, INTERCEPT_SMI);
1103         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1104         set_intercept(svm, INTERCEPT_RDPMC);
1105         set_intercept(svm, INTERCEPT_CPUID);
1106         set_intercept(svm, INTERCEPT_INVD);
1107         set_intercept(svm, INTERCEPT_HLT);
1108         set_intercept(svm, INTERCEPT_INVLPG);
1109         set_intercept(svm, INTERCEPT_INVLPGA);
1110         set_intercept(svm, INTERCEPT_IOIO_PROT);
1111         set_intercept(svm, INTERCEPT_MSR_PROT);
1112         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1113         set_intercept(svm, INTERCEPT_SHUTDOWN);
1114         set_intercept(svm, INTERCEPT_VMRUN);
1115         set_intercept(svm, INTERCEPT_VMMCALL);
1116         set_intercept(svm, INTERCEPT_VMLOAD);
1117         set_intercept(svm, INTERCEPT_VMSAVE);
1118         set_intercept(svm, INTERCEPT_STGI);
1119         set_intercept(svm, INTERCEPT_CLGI);
1120         set_intercept(svm, INTERCEPT_SKINIT);
1121         set_intercept(svm, INTERCEPT_WBINVD);
1122         set_intercept(svm, INTERCEPT_MONITOR);
1123         set_intercept(svm, INTERCEPT_MWAIT);
1124         set_intercept(svm, INTERCEPT_XSETBV);
1125
1126         control->iopm_base_pa = iopm_base;
1127         control->msrpm_base_pa = __pa(svm->msrpm);
1128         control->int_ctl = V_INTR_MASKING_MASK;
1129
1130         init_seg(&save->es);
1131         init_seg(&save->ss);
1132         init_seg(&save->ds);
1133         init_seg(&save->fs);
1134         init_seg(&save->gs);
1135
1136         save->cs.selector = 0xf000;
1137         save->cs.base = 0xffff0000;
1138         /* Executable/Readable Code Segment */
1139         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1140                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1141         save->cs.limit = 0xffff;
1142
1143         save->gdtr.limit = 0xffff;
1144         save->idtr.limit = 0xffff;
1145
1146         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1147         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1148
1149         svm_set_efer(&svm->vcpu, 0);
1150         save->dr6 = 0xffff0ff0;
1151         kvm_set_rflags(&svm->vcpu, 2);
1152         save->rip = 0x0000fff0;
1153         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1154
1155         /*
1156          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1157          * It also updates the guest-visible cr0 value.
1158          */
1159         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1160         kvm_mmu_reset_context(&svm->vcpu);
1161
1162         save->cr4 = X86_CR4_PAE;
1163         /* rdx = ?? */
1164
1165         if (npt_enabled) {
1166                 /* Setup VMCB for Nested Paging */
1167                 control->nested_ctl = 1;
1168                 clr_intercept(svm, INTERCEPT_INVLPG);
1169                 clr_exception_intercept(svm, PF_VECTOR);
1170                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1171                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1172                 save->g_pat = svm->vcpu.arch.pat;
1173                 save->cr3 = 0;
1174                 save->cr4 = 0;
1175         }
1176         svm->asid_generation = 0;
1177
1178         svm->nested.vmcb = 0;
1179         svm->vcpu.arch.hflags = 0;
1180
1181         if (boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1182                 control->pause_filter_count = 3000;
1183                 set_intercept(svm, INTERCEPT_PAUSE);
1184         }
1185
1186         if (avic)
1187                 avic_init_vmcb(svm);
1188
1189         mark_all_dirty(svm->vmcb);
1190
1191         enable_gif(svm);
1192
1193 }
1194
1195 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu, int index)
1196 {
1197         u64 *avic_physical_id_table;
1198         struct kvm_arch *vm_data = &vcpu->kvm->arch;
1199
1200         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1201                 return NULL;
1202
1203         avic_physical_id_table = page_address(vm_data->avic_physical_id_table_page);
1204
1205         return &avic_physical_id_table[index];
1206 }
1207
1208 /**
1209  * Note:
1210  * AVIC hardware walks the nested page table to check permissions,
1211  * but does not use the SPA address specified in the leaf page
1212  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
1213  * field of the VMCB. Therefore, we set up the
1214  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1215  */
1216 static int avic_init_access_page(struct kvm_vcpu *vcpu)
1217 {
1218         struct kvm *kvm = vcpu->kvm;
1219         int ret;
1220
1221         if (kvm->arch.apic_access_page_done)
1222                 return 0;
1223
1224         ret = x86_set_memory_region(kvm,
1225                                     APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1226                                     APIC_DEFAULT_PHYS_BASE,
1227                                     PAGE_SIZE);
1228         if (ret)
1229                 return ret;
1230
1231         kvm->arch.apic_access_page_done = true;
1232         return 0;
1233 }
1234
1235 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1236 {
1237         int ret;
1238         u64 *entry, new_entry;
1239         int id = vcpu->vcpu_id;
1240         struct vcpu_svm *svm = to_svm(vcpu);
1241
1242         ret = avic_init_access_page(vcpu);
1243         if (ret)
1244                 return ret;
1245
1246         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1247                 return -EINVAL;
1248
1249         if (!svm->vcpu.arch.apic->regs)
1250                 return -EINVAL;
1251
1252         svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1253
1254         /* Setting AVIC backing page address in the phy APIC ID table */
1255         entry = avic_get_physical_id_entry(vcpu, id);
1256         if (!entry)
1257                 return -EINVAL;
1258
1259         new_entry = READ_ONCE(*entry);
1260         new_entry = (page_to_phys(svm->avic_backing_page) &
1261                      AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1262                      AVIC_PHYSICAL_ID_ENTRY_VALID_MASK;
1263         WRITE_ONCE(*entry, new_entry);
1264
1265         svm->avic_physical_id_cache = entry;
1266
1267         return 0;
1268 }
1269
1270 static void avic_vm_destroy(struct kvm *kvm)
1271 {
1272         struct kvm_arch *vm_data = &kvm->arch;
1273
1274         if (vm_data->avic_logical_id_table_page)
1275                 __free_page(vm_data->avic_logical_id_table_page);
1276         if (vm_data->avic_physical_id_table_page)
1277                 __free_page(vm_data->avic_physical_id_table_page);
1278 }
1279
1280 static int avic_vm_init(struct kvm *kvm)
1281 {
1282         int err = -ENOMEM;
1283         struct kvm_arch *vm_data = &kvm->arch;
1284         struct page *p_page;
1285         struct page *l_page;
1286
1287         if (!avic)
1288                 return 0;
1289
1290         /* Allocating physical APIC ID table (4KB) */
1291         p_page = alloc_page(GFP_KERNEL);
1292         if (!p_page)
1293                 goto free_avic;
1294
1295         vm_data->avic_physical_id_table_page = p_page;
1296         clear_page(page_address(p_page));
1297
1298         /* Allocating logical APIC ID table (4KB) */
1299         l_page = alloc_page(GFP_KERNEL);
1300         if (!l_page)
1301                 goto free_avic;
1302
1303         vm_data->avic_logical_id_table_page = l_page;
1304         clear_page(page_address(l_page));
1305
1306         return 0;
1307
1308 free_avic:
1309         avic_vm_destroy(kvm);
1310         return err;
1311 }
1312
1313 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
1314 {
1315         struct vcpu_svm *svm = to_svm(vcpu);
1316         u32 dummy;
1317         u32 eax = 1;
1318
1319         if (!init_event) {
1320                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
1321                                            MSR_IA32_APICBASE_ENABLE;
1322                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
1323                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
1324         }
1325         init_vmcb(svm);
1326
1327         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy);
1328         kvm_register_write(vcpu, VCPU_REGS_RDX, eax);
1329
1330         if (kvm_vcpu_apicv_active(vcpu) && !init_event)
1331                 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
1332 }
1333
1334 static struct kvm_vcpu *svm_create_vcpu(struct kvm *kvm, unsigned int id)
1335 {
1336         struct vcpu_svm *svm;
1337         struct page *page;
1338         struct page *msrpm_pages;
1339         struct page *hsave_page;
1340         struct page *nested_msrpm_pages;
1341         int err;
1342
1343         svm = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL);
1344         if (!svm) {
1345                 err = -ENOMEM;
1346                 goto out;
1347         }
1348
1349         err = kvm_vcpu_init(&svm->vcpu, kvm, id);
1350         if (err)
1351                 goto free_svm;
1352
1353         err = -ENOMEM;
1354         page = alloc_page(GFP_KERNEL);
1355         if (!page)
1356                 goto uninit;
1357
1358         msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1359         if (!msrpm_pages)
1360                 goto free_page1;
1361
1362         nested_msrpm_pages = alloc_pages(GFP_KERNEL, MSRPM_ALLOC_ORDER);
1363         if (!nested_msrpm_pages)
1364                 goto free_page2;
1365
1366         hsave_page = alloc_page(GFP_KERNEL);
1367         if (!hsave_page)
1368                 goto free_page3;
1369
1370         if (avic) {
1371                 err = avic_init_backing_page(&svm->vcpu);
1372                 if (err)
1373                         goto free_page4;
1374         }
1375
1376         svm->nested.hsave = page_address(hsave_page);
1377
1378         svm->msrpm = page_address(msrpm_pages);
1379         svm_vcpu_init_msrpm(svm->msrpm);
1380
1381         svm->nested.msrpm = page_address(nested_msrpm_pages);
1382         svm_vcpu_init_msrpm(svm->nested.msrpm);
1383
1384         svm->vmcb = page_address(page);
1385         clear_page(svm->vmcb);
1386         svm->vmcb_pa = page_to_pfn(page) << PAGE_SHIFT;
1387         svm->asid_generation = 0;
1388         init_vmcb(svm);
1389
1390         svm_init_osvw(&svm->vcpu);
1391
1392         return &svm->vcpu;
1393
1394 free_page4:
1395         __free_page(hsave_page);
1396 free_page3:
1397         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
1398 free_page2:
1399         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
1400 free_page1:
1401         __free_page(page);
1402 uninit:
1403         kvm_vcpu_uninit(&svm->vcpu);
1404 free_svm:
1405         kmem_cache_free(kvm_vcpu_cache, svm);
1406 out:
1407         return ERR_PTR(err);
1408 }
1409
1410 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
1411 {
1412         struct vcpu_svm *svm = to_svm(vcpu);
1413
1414         __free_page(pfn_to_page(svm->vmcb_pa >> PAGE_SHIFT));
1415         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
1416         __free_page(virt_to_page(svm->nested.hsave));
1417         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
1418         kvm_vcpu_uninit(vcpu);
1419         kmem_cache_free(kvm_vcpu_cache, svm);
1420 }
1421
1422 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
1423 {
1424         struct vcpu_svm *svm = to_svm(vcpu);
1425         int i;
1426
1427         if (unlikely(cpu != vcpu->cpu)) {
1428                 svm->asid_generation = 0;
1429                 mark_all_dirty(svm->vmcb);
1430         }
1431
1432 #ifdef CONFIG_X86_64
1433         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
1434 #endif
1435         savesegment(fs, svm->host.fs);
1436         savesegment(gs, svm->host.gs);
1437         svm->host.ldt = kvm_read_ldt();
1438
1439         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1440                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1441
1442         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1443                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
1444                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
1445                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
1446                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
1447                 }
1448         }
1449         /* This assumes that the kernel never uses MSR_TSC_AUX */
1450         if (static_cpu_has(X86_FEATURE_RDTSCP))
1451                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
1452 }
1453
1454 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
1455 {
1456         struct vcpu_svm *svm = to_svm(vcpu);
1457         int i;
1458
1459         ++vcpu->stat.host_state_reload;
1460         kvm_load_ldt(svm->host.ldt);
1461 #ifdef CONFIG_X86_64
1462         loadsegment(fs, svm->host.fs);
1463         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gs);
1464         load_gs_index(svm->host.gs);
1465 #else
1466 #ifdef CONFIG_X86_32_LAZY_GS
1467         loadsegment(gs, svm->host.gs);
1468 #endif
1469 #endif
1470         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
1471                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
1472 }
1473
1474 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
1475 {
1476         return to_svm(vcpu)->vmcb->save.rflags;
1477 }
1478
1479 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
1480 {
1481        /*
1482         * Any change of EFLAGS.VM is accompained by a reload of SS
1483         * (caused by either a task switch or an inter-privilege IRET),
1484         * so we do not need to update the CPL here.
1485         */
1486         to_svm(vcpu)->vmcb->save.rflags = rflags;
1487 }
1488
1489 static u32 svm_get_pkru(struct kvm_vcpu *vcpu)
1490 {
1491         return 0;
1492 }
1493
1494 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
1495 {
1496         switch (reg) {
1497         case VCPU_EXREG_PDPTR:
1498                 BUG_ON(!npt_enabled);
1499                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
1500                 break;
1501         default:
1502                 BUG();
1503         }
1504 }
1505
1506 static void svm_set_vintr(struct vcpu_svm *svm)
1507 {
1508         set_intercept(svm, INTERCEPT_VINTR);
1509 }
1510
1511 static void svm_clear_vintr(struct vcpu_svm *svm)
1512 {
1513         clr_intercept(svm, INTERCEPT_VINTR);
1514 }
1515
1516 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
1517 {
1518         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1519
1520         switch (seg) {
1521         case VCPU_SREG_CS: return &save->cs;
1522         case VCPU_SREG_DS: return &save->ds;
1523         case VCPU_SREG_ES: return &save->es;
1524         case VCPU_SREG_FS: return &save->fs;
1525         case VCPU_SREG_GS: return &save->gs;
1526         case VCPU_SREG_SS: return &save->ss;
1527         case VCPU_SREG_TR: return &save->tr;
1528         case VCPU_SREG_LDTR: return &save->ldtr;
1529         }
1530         BUG();
1531         return NULL;
1532 }
1533
1534 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
1535 {
1536         struct vmcb_seg *s = svm_seg(vcpu, seg);
1537
1538         return s->base;
1539 }
1540
1541 static void svm_get_segment(struct kvm_vcpu *vcpu,
1542                             struct kvm_segment *var, int seg)
1543 {
1544         struct vmcb_seg *s = svm_seg(vcpu, seg);
1545
1546         var->base = s->base;
1547         var->limit = s->limit;
1548         var->selector = s->selector;
1549         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
1550         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
1551         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1552         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
1553         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
1554         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
1555         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
1556
1557         /*
1558          * AMD CPUs circa 2014 track the G bit for all segments except CS.
1559          * However, the SVM spec states that the G bit is not observed by the
1560          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
1561          * So let's synthesize a legal G bit for all segments, this helps
1562          * running KVM nested. It also helps cross-vendor migration, because
1563          * Intel's vmentry has a check on the 'G' bit.
1564          */
1565         var->g = s->limit > 0xfffff;
1566
1567         /*
1568          * AMD's VMCB does not have an explicit unusable field, so emulate it
1569          * for cross vendor migration purposes by "not present"
1570          */
1571         var->unusable = !var->present || (var->type == 0);
1572
1573         switch (seg) {
1574         case VCPU_SREG_TR:
1575                 /*
1576                  * Work around a bug where the busy flag in the tr selector
1577                  * isn't exposed
1578                  */
1579                 var->type |= 0x2;
1580                 break;
1581         case VCPU_SREG_DS:
1582         case VCPU_SREG_ES:
1583         case VCPU_SREG_FS:
1584         case VCPU_SREG_GS:
1585                 /*
1586                  * The accessed bit must always be set in the segment
1587                  * descriptor cache, although it can be cleared in the
1588                  * descriptor, the cached bit always remains at 1. Since
1589                  * Intel has a check on this, set it here to support
1590                  * cross-vendor migration.
1591                  */
1592                 if (!var->unusable)
1593                         var->type |= 0x1;
1594                 break;
1595         case VCPU_SREG_SS:
1596                 /*
1597                  * On AMD CPUs sometimes the DB bit in the segment
1598                  * descriptor is left as 1, although the whole segment has
1599                  * been made unusable. Clear it here to pass an Intel VMX
1600                  * entry check when cross vendor migrating.
1601                  */
1602                 if (var->unusable)
1603                         var->db = 0;
1604                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
1605                 break;
1606         }
1607 }
1608
1609 static int svm_get_cpl(struct kvm_vcpu *vcpu)
1610 {
1611         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
1612
1613         return save->cpl;
1614 }
1615
1616 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1617 {
1618         struct vcpu_svm *svm = to_svm(vcpu);
1619
1620         dt->size = svm->vmcb->save.idtr.limit;
1621         dt->address = svm->vmcb->save.idtr.base;
1622 }
1623
1624 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1625 {
1626         struct vcpu_svm *svm = to_svm(vcpu);
1627
1628         svm->vmcb->save.idtr.limit = dt->size;
1629         svm->vmcb->save.idtr.base = dt->address ;
1630         mark_dirty(svm->vmcb, VMCB_DT);
1631 }
1632
1633 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1634 {
1635         struct vcpu_svm *svm = to_svm(vcpu);
1636
1637         dt->size = svm->vmcb->save.gdtr.limit;
1638         dt->address = svm->vmcb->save.gdtr.base;
1639 }
1640
1641 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
1642 {
1643         struct vcpu_svm *svm = to_svm(vcpu);
1644
1645         svm->vmcb->save.gdtr.limit = dt->size;
1646         svm->vmcb->save.gdtr.base = dt->address ;
1647         mark_dirty(svm->vmcb, VMCB_DT);
1648 }
1649
1650 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
1651 {
1652 }
1653
1654 static void svm_decache_cr3(struct kvm_vcpu *vcpu)
1655 {
1656 }
1657
1658 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
1659 {
1660 }
1661
1662 static void update_cr0_intercept(struct vcpu_svm *svm)
1663 {
1664         ulong gcr0 = svm->vcpu.arch.cr0;
1665         u64 *hcr0 = &svm->vmcb->save.cr0;
1666
1667         if (!svm->vcpu.fpu_active)
1668                 *hcr0 |= SVM_CR0_SELECTIVE_MASK;
1669         else
1670                 *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
1671                         | (gcr0 & SVM_CR0_SELECTIVE_MASK);
1672
1673         mark_dirty(svm->vmcb, VMCB_CR);
1674
1675         if (gcr0 == *hcr0 && svm->vcpu.fpu_active) {
1676                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
1677                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1678         } else {
1679                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
1680                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1681         }
1682 }
1683
1684 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
1685 {
1686         struct vcpu_svm *svm = to_svm(vcpu);
1687
1688 #ifdef CONFIG_X86_64
1689         if (vcpu->arch.efer & EFER_LME) {
1690                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
1691                         vcpu->arch.efer |= EFER_LMA;
1692                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
1693                 }
1694
1695                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
1696                         vcpu->arch.efer &= ~EFER_LMA;
1697                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
1698                 }
1699         }
1700 #endif
1701         vcpu->arch.cr0 = cr0;
1702
1703         if (!npt_enabled)
1704                 cr0 |= X86_CR0_PG | X86_CR0_WP;
1705
1706         if (!vcpu->fpu_active)
1707                 cr0 |= X86_CR0_TS;
1708         /*
1709          * re-enable caching here because the QEMU bios
1710          * does not do it - this results in some delay at
1711          * reboot
1712          */
1713         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
1714                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
1715         svm->vmcb->save.cr0 = cr0;
1716         mark_dirty(svm->vmcb, VMCB_CR);
1717         update_cr0_intercept(svm);
1718 }
1719
1720 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
1721 {
1722         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
1723         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
1724
1725         if (cr4 & X86_CR4_VMXE)
1726                 return 1;
1727
1728         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
1729                 svm_flush_tlb(vcpu);
1730
1731         vcpu->arch.cr4 = cr4;
1732         if (!npt_enabled)
1733                 cr4 |= X86_CR4_PAE;
1734         cr4 |= host_cr4_mce;
1735         to_svm(vcpu)->vmcb->save.cr4 = cr4;
1736         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
1737         return 0;
1738 }
1739
1740 static void svm_set_segment(struct kvm_vcpu *vcpu,
1741                             struct kvm_segment *var, int seg)
1742 {
1743         struct vcpu_svm *svm = to_svm(vcpu);
1744         struct vmcb_seg *s = svm_seg(vcpu, seg);
1745
1746         s->base = var->base;
1747         s->limit = var->limit;
1748         s->selector = var->selector;
1749         if (var->unusable)
1750                 s->attrib = 0;
1751         else {
1752                 s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
1753                 s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
1754                 s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
1755                 s->attrib |= (var->present & 1) << SVM_SELECTOR_P_SHIFT;
1756                 s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
1757                 s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
1758                 s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
1759                 s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
1760         }
1761
1762         /*
1763          * This is always accurate, except if SYSRET returned to a segment
1764          * with SS.DPL != 3.  Intel does not have this quirk, and always
1765          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
1766          * would entail passing the CPL to userspace and back.
1767          */
1768         if (seg == VCPU_SREG_SS)
1769                 svm->vmcb->save.cpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
1770
1771         mark_dirty(svm->vmcb, VMCB_SEG);
1772 }
1773
1774 static void update_bp_intercept(struct kvm_vcpu *vcpu)
1775 {
1776         struct vcpu_svm *svm = to_svm(vcpu);
1777
1778         clr_exception_intercept(svm, BP_VECTOR);
1779
1780         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
1781                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
1782                         set_exception_intercept(svm, BP_VECTOR);
1783         } else
1784                 vcpu->guest_debug = 0;
1785 }
1786
1787 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
1788 {
1789         if (sd->next_asid > sd->max_asid) {
1790                 ++sd->asid_generation;
1791                 sd->next_asid = 1;
1792                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
1793         }
1794
1795         svm->asid_generation = sd->asid_generation;
1796         svm->vmcb->control.asid = sd->next_asid++;
1797
1798         mark_dirty(svm->vmcb, VMCB_ASID);
1799 }
1800
1801 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
1802 {
1803         return to_svm(vcpu)->vmcb->save.dr6;
1804 }
1805
1806 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
1807 {
1808         struct vcpu_svm *svm = to_svm(vcpu);
1809
1810         svm->vmcb->save.dr6 = value;
1811         mark_dirty(svm->vmcb, VMCB_DR);
1812 }
1813
1814 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
1815 {
1816         struct vcpu_svm *svm = to_svm(vcpu);
1817
1818         get_debugreg(vcpu->arch.db[0], 0);
1819         get_debugreg(vcpu->arch.db[1], 1);
1820         get_debugreg(vcpu->arch.db[2], 2);
1821         get_debugreg(vcpu->arch.db[3], 3);
1822         vcpu->arch.dr6 = svm_get_dr6(vcpu);
1823         vcpu->arch.dr7 = svm->vmcb->save.dr7;
1824
1825         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
1826         set_dr_intercepts(svm);
1827 }
1828
1829 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
1830 {
1831         struct vcpu_svm *svm = to_svm(vcpu);
1832
1833         svm->vmcb->save.dr7 = value;
1834         mark_dirty(svm->vmcb, VMCB_DR);
1835 }
1836
1837 static int pf_interception(struct vcpu_svm *svm)
1838 {
1839         u64 fault_address = svm->vmcb->control.exit_info_2;
1840         u32 error_code;
1841         int r = 1;
1842
1843         switch (svm->apf_reason) {
1844         default:
1845                 error_code = svm->vmcb->control.exit_info_1;
1846
1847                 trace_kvm_page_fault(fault_address, error_code);
1848                 if (!npt_enabled && kvm_event_needs_reinjection(&svm->vcpu))
1849                         kvm_mmu_unprotect_page_virt(&svm->vcpu, fault_address);
1850                 r = kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
1851                         svm->vmcb->control.insn_bytes,
1852                         svm->vmcb->control.insn_len);
1853                 break;
1854         case KVM_PV_REASON_PAGE_NOT_PRESENT:
1855                 svm->apf_reason = 0;
1856                 local_irq_disable();
1857                 kvm_async_pf_task_wait(fault_address);
1858                 local_irq_enable();
1859                 break;
1860         case KVM_PV_REASON_PAGE_READY:
1861                 svm->apf_reason = 0;
1862                 local_irq_disable();
1863                 kvm_async_pf_task_wake(fault_address);
1864                 local_irq_enable();
1865                 break;
1866         }
1867         return r;
1868 }
1869
1870 static int db_interception(struct vcpu_svm *svm)
1871 {
1872         struct kvm_run *kvm_run = svm->vcpu.run;
1873
1874         if (!(svm->vcpu.guest_debug &
1875               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
1876                 !svm->nmi_singlestep) {
1877                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
1878                 return 1;
1879         }
1880
1881         if (svm->nmi_singlestep) {
1882                 svm->nmi_singlestep = false;
1883                 if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP))
1884                         svm->vmcb->save.rflags &=
1885                                 ~(X86_EFLAGS_TF | X86_EFLAGS_RF);
1886         }
1887
1888         if (svm->vcpu.guest_debug &
1889             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
1890                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
1891                 kvm_run->debug.arch.pc =
1892                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1893                 kvm_run->debug.arch.exception = DB_VECTOR;
1894                 return 0;
1895         }
1896
1897         return 1;
1898 }
1899
1900 static int bp_interception(struct vcpu_svm *svm)
1901 {
1902         struct kvm_run *kvm_run = svm->vcpu.run;
1903
1904         kvm_run->exit_reason = KVM_EXIT_DEBUG;
1905         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
1906         kvm_run->debug.arch.exception = BP_VECTOR;
1907         return 0;
1908 }
1909
1910 static int ud_interception(struct vcpu_svm *svm)
1911 {
1912         int er;
1913
1914         er = emulate_instruction(&svm->vcpu, EMULTYPE_TRAP_UD);
1915         if (er != EMULATE_DONE)
1916                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
1917         return 1;
1918 }
1919
1920 static int ac_interception(struct vcpu_svm *svm)
1921 {
1922         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
1923         return 1;
1924 }
1925
1926 static void svm_fpu_activate(struct kvm_vcpu *vcpu)
1927 {
1928         struct vcpu_svm *svm = to_svm(vcpu);
1929
1930         clr_exception_intercept(svm, NM_VECTOR);
1931
1932         svm->vcpu.fpu_active = 1;
1933         update_cr0_intercept(svm);
1934 }
1935
1936 static int nm_interception(struct vcpu_svm *svm)
1937 {
1938         svm_fpu_activate(&svm->vcpu);
1939         return 1;
1940 }
1941
1942 static bool is_erratum_383(void)
1943 {
1944         int err, i;
1945         u64 value;
1946
1947         if (!erratum_383_found)
1948                 return false;
1949
1950         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
1951         if (err)
1952                 return false;
1953
1954         /* Bit 62 may or may not be set for this mce */
1955         value &= ~(1ULL << 62);
1956
1957         if (value != 0xb600000000010015ULL)
1958                 return false;
1959
1960         /* Clear MCi_STATUS registers */
1961         for (i = 0; i < 6; ++i)
1962                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
1963
1964         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
1965         if (!err) {
1966                 u32 low, high;
1967
1968                 value &= ~(1ULL << 2);
1969                 low    = lower_32_bits(value);
1970                 high   = upper_32_bits(value);
1971
1972                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
1973         }
1974
1975         /* Flush tlb to evict multi-match entries */
1976         __flush_tlb_all();
1977
1978         return true;
1979 }
1980
1981 static void svm_handle_mce(struct vcpu_svm *svm)
1982 {
1983         if (is_erratum_383()) {
1984                 /*
1985                  * Erratum 383 triggered. Guest state is corrupt so kill the
1986                  * guest.
1987                  */
1988                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
1989
1990                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
1991
1992                 return;
1993         }
1994
1995         /*
1996          * On an #MC intercept the MCE handler is not called automatically in
1997          * the host. So do it by hand here.
1998          */
1999         asm volatile (
2000                 "int $0x12\n");
2001         /* not sure if we ever come back to this point */
2002
2003         return;
2004 }
2005
2006 static int mc_interception(struct vcpu_svm *svm)
2007 {
2008         return 1;
2009 }
2010
2011 static int shutdown_interception(struct vcpu_svm *svm)
2012 {
2013         struct kvm_run *kvm_run = svm->vcpu.run;
2014
2015         /*
2016          * VMCB is undefined after a SHUTDOWN intercept
2017          * so reinitialize it.
2018          */
2019         clear_page(svm->vmcb);
2020         init_vmcb(svm);
2021
2022         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2023         return 0;
2024 }
2025
2026 static int io_interception(struct vcpu_svm *svm)
2027 {
2028         struct kvm_vcpu *vcpu = &svm->vcpu;
2029         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2030         int size, in, string;
2031         unsigned port;
2032
2033         ++svm->vcpu.stat.io_exits;
2034         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2035         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2036         if (string || in)
2037                 return emulate_instruction(vcpu, 0) == EMULATE_DONE;
2038
2039         port = io_info >> 16;
2040         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2041         svm->next_rip = svm->vmcb->control.exit_info_2;
2042         skip_emulated_instruction(&svm->vcpu);
2043
2044         return kvm_fast_pio_out(vcpu, size, port);
2045 }
2046
2047 static int nmi_interception(struct vcpu_svm *svm)
2048 {
2049         return 1;
2050 }
2051
2052 static int intr_interception(struct vcpu_svm *svm)
2053 {
2054         ++svm->vcpu.stat.irq_exits;
2055         return 1;
2056 }
2057
2058 static int nop_on_interception(struct vcpu_svm *svm)
2059 {
2060         return 1;
2061 }
2062
2063 static int halt_interception(struct vcpu_svm *svm)
2064 {
2065         svm->next_rip = kvm_rip_read(&svm->vcpu) + 1;
2066         return kvm_emulate_halt(&svm->vcpu);
2067 }
2068
2069 static int vmmcall_interception(struct vcpu_svm *svm)
2070 {
2071         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2072         return kvm_emulate_hypercall(&svm->vcpu);
2073 }
2074
2075 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2076 {
2077         struct vcpu_svm *svm = to_svm(vcpu);
2078
2079         return svm->nested.nested_cr3;
2080 }
2081
2082 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2083 {
2084         struct vcpu_svm *svm = to_svm(vcpu);
2085         u64 cr3 = svm->nested.nested_cr3;
2086         u64 pdpte;
2087         int ret;
2088
2089         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(cr3), &pdpte,
2090                                        offset_in_page(cr3) + index * 8, 8);
2091         if (ret)
2092                 return 0;
2093         return pdpte;
2094 }
2095
2096 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2097                                    unsigned long root)
2098 {
2099         struct vcpu_svm *svm = to_svm(vcpu);
2100
2101         svm->vmcb->control.nested_cr3 = root;
2102         mark_dirty(svm->vmcb, VMCB_NPT);
2103         svm_flush_tlb(vcpu);
2104 }
2105
2106 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2107                                        struct x86_exception *fault)
2108 {
2109         struct vcpu_svm *svm = to_svm(vcpu);
2110
2111         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2112                 /*
2113                  * TODO: track the cause of the nested page fault, and
2114                  * correctly fill in the high bits of exit_info_1.
2115                  */
2116                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2117                 svm->vmcb->control.exit_code_hi = 0;
2118                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2119                 svm->vmcb->control.exit_info_2 = fault->address;
2120         }
2121
2122         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2123         svm->vmcb->control.exit_info_1 |= fault->error_code;
2124
2125         /*
2126          * The present bit is always zero for page structure faults on real
2127          * hardware.
2128          */
2129         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2130                 svm->vmcb->control.exit_info_1 &= ~1;
2131
2132         nested_svm_vmexit(svm);
2133 }
2134
2135 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2136 {
2137         WARN_ON(mmu_is_nested(vcpu));
2138         kvm_init_shadow_mmu(vcpu);
2139         vcpu->arch.mmu.set_cr3           = nested_svm_set_tdp_cr3;
2140         vcpu->arch.mmu.get_cr3           = nested_svm_get_tdp_cr3;
2141         vcpu->arch.mmu.get_pdptr         = nested_svm_get_tdp_pdptr;
2142         vcpu->arch.mmu.inject_page_fault = nested_svm_inject_npf_exit;
2143         vcpu->arch.mmu.shadow_root_level = get_npt_level();
2144         reset_shadow_zero_bits_mask(vcpu, &vcpu->arch.mmu);
2145         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
2146 }
2147
2148 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
2149 {
2150         vcpu->arch.walk_mmu = &vcpu->arch.mmu;
2151 }
2152
2153 static int nested_svm_check_permissions(struct vcpu_svm *svm)
2154 {
2155         if (!(svm->vcpu.arch.efer & EFER_SVME)
2156             || !is_paging(&svm->vcpu)) {
2157                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2158                 return 1;
2159         }
2160
2161         if (svm->vmcb->save.cpl) {
2162                 kvm_inject_gp(&svm->vcpu, 0);
2163                 return 1;
2164         }
2165
2166        return 0;
2167 }
2168
2169 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
2170                                       bool has_error_code, u32 error_code)
2171 {
2172         int vmexit;
2173
2174         if (!is_guest_mode(&svm->vcpu))
2175                 return 0;
2176
2177         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
2178         svm->vmcb->control.exit_code_hi = 0;
2179         svm->vmcb->control.exit_info_1 = error_code;
2180         svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
2181
2182         vmexit = nested_svm_intercept(svm);
2183         if (vmexit == NESTED_EXIT_DONE)
2184                 svm->nested.exit_required = true;
2185
2186         return vmexit;
2187 }
2188
2189 /* This function returns true if it is save to enable the irq window */
2190 static inline bool nested_svm_intr(struct vcpu_svm *svm)
2191 {
2192         if (!is_guest_mode(&svm->vcpu))
2193                 return true;
2194
2195         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2196                 return true;
2197
2198         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
2199                 return false;
2200
2201         /*
2202          * if vmexit was already requested (by intercepted exception
2203          * for instance) do not overwrite it with "external interrupt"
2204          * vmexit.
2205          */
2206         if (svm->nested.exit_required)
2207                 return false;
2208
2209         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
2210         svm->vmcb->control.exit_info_1 = 0;
2211         svm->vmcb->control.exit_info_2 = 0;
2212
2213         if (svm->nested.intercept & 1ULL) {
2214                 /*
2215                  * The #vmexit can't be emulated here directly because this
2216                  * code path runs with irqs and preemption disabled. A
2217                  * #vmexit emulation might sleep. Only signal request for
2218                  * the #vmexit here.
2219                  */
2220                 svm->nested.exit_required = true;
2221                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
2222                 return false;
2223         }
2224
2225         return true;
2226 }
2227
2228 /* This function returns true if it is save to enable the nmi window */
2229 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
2230 {
2231         if (!is_guest_mode(&svm->vcpu))
2232                 return true;
2233
2234         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
2235                 return true;
2236
2237         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
2238         svm->nested.exit_required = true;
2239
2240         return false;
2241 }
2242
2243 static void *nested_svm_map(struct vcpu_svm *svm, u64 gpa, struct page **_page)
2244 {
2245         struct page *page;
2246
2247         might_sleep();
2248
2249         page = kvm_vcpu_gfn_to_page(&svm->vcpu, gpa >> PAGE_SHIFT);
2250         if (is_error_page(page))
2251                 goto error;
2252
2253         *_page = page;
2254
2255         return kmap(page);
2256
2257 error:
2258         kvm_inject_gp(&svm->vcpu, 0);
2259
2260         return NULL;
2261 }
2262
2263 static void nested_svm_unmap(struct page *page)
2264 {
2265         kunmap(page);
2266         kvm_release_page_dirty(page);
2267 }
2268
2269 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
2270 {
2271         unsigned port, size, iopm_len;
2272         u16 val, mask;
2273         u8 start_bit;
2274         u64 gpa;
2275
2276         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
2277                 return NESTED_EXIT_HOST;
2278
2279         port = svm->vmcb->control.exit_info_1 >> 16;
2280         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
2281                 SVM_IOIO_SIZE_SHIFT;
2282         gpa  = svm->nested.vmcb_iopm + (port / 8);
2283         start_bit = port % 8;
2284         iopm_len = (start_bit + size > 8) ? 2 : 1;
2285         mask = (0xf >> (4 - size)) << start_bit;
2286         val = 0;
2287
2288         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
2289                 return NESTED_EXIT_DONE;
2290
2291         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2292 }
2293
2294 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
2295 {
2296         u32 offset, msr, value;
2297         int write, mask;
2298
2299         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2300                 return NESTED_EXIT_HOST;
2301
2302         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
2303         offset = svm_msrpm_offset(msr);
2304         write  = svm->vmcb->control.exit_info_1 & 1;
2305         mask   = 1 << ((2 * (msr & 0xf)) + write);
2306
2307         if (offset == MSR_INVALID)
2308                 return NESTED_EXIT_DONE;
2309
2310         /* Offset is in 32 bit units but need in 8 bit units */
2311         offset *= 4;
2312
2313         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
2314                 return NESTED_EXIT_DONE;
2315
2316         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
2317 }
2318
2319 static int nested_svm_exit_special(struct vcpu_svm *svm)
2320 {
2321         u32 exit_code = svm->vmcb->control.exit_code;
2322
2323         switch (exit_code) {
2324         case SVM_EXIT_INTR:
2325         case SVM_EXIT_NMI:
2326         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
2327                 return NESTED_EXIT_HOST;
2328         case SVM_EXIT_NPF:
2329                 /* For now we are always handling NPFs when using them */
2330                 if (npt_enabled)
2331                         return NESTED_EXIT_HOST;
2332                 break;
2333         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
2334                 /* When we're shadowing, trap PFs, but not async PF */
2335                 if (!npt_enabled && svm->apf_reason == 0)
2336                         return NESTED_EXIT_HOST;
2337                 break;
2338         case SVM_EXIT_EXCP_BASE + NM_VECTOR:
2339                 nm_interception(svm);
2340                 break;
2341         default:
2342                 break;
2343         }
2344
2345         return NESTED_EXIT_CONTINUE;
2346 }
2347
2348 /*
2349  * If this function returns true, this #vmexit was already handled
2350  */
2351 static int nested_svm_intercept(struct vcpu_svm *svm)
2352 {
2353         u32 exit_code = svm->vmcb->control.exit_code;
2354         int vmexit = NESTED_EXIT_HOST;
2355
2356         switch (exit_code) {
2357         case SVM_EXIT_MSR:
2358                 vmexit = nested_svm_exit_handled_msr(svm);
2359                 break;
2360         case SVM_EXIT_IOIO:
2361                 vmexit = nested_svm_intercept_ioio(svm);
2362                 break;
2363         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
2364                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
2365                 if (svm->nested.intercept_cr & bit)
2366                         vmexit = NESTED_EXIT_DONE;
2367                 break;
2368         }
2369         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
2370                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
2371                 if (svm->nested.intercept_dr & bit)
2372                         vmexit = NESTED_EXIT_DONE;
2373                 break;
2374         }
2375         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
2376                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
2377                 if (svm->nested.intercept_exceptions & excp_bits)
2378                         vmexit = NESTED_EXIT_DONE;
2379                 /* async page fault always cause vmexit */
2380                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
2381                          svm->apf_reason != 0)
2382                         vmexit = NESTED_EXIT_DONE;
2383                 break;
2384         }
2385         case SVM_EXIT_ERR: {
2386                 vmexit = NESTED_EXIT_DONE;
2387                 break;
2388         }
2389         default: {
2390                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
2391                 if (svm->nested.intercept & exit_bits)
2392                         vmexit = NESTED_EXIT_DONE;
2393         }
2394         }
2395
2396         return vmexit;
2397 }
2398
2399 static int nested_svm_exit_handled(struct vcpu_svm *svm)
2400 {
2401         int vmexit;
2402
2403         vmexit = nested_svm_intercept(svm);
2404
2405         if (vmexit == NESTED_EXIT_DONE)
2406                 nested_svm_vmexit(svm);
2407
2408         return vmexit;
2409 }
2410
2411 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
2412 {
2413         struct vmcb_control_area *dst  = &dst_vmcb->control;
2414         struct vmcb_control_area *from = &from_vmcb->control;
2415
2416         dst->intercept_cr         = from->intercept_cr;
2417         dst->intercept_dr         = from->intercept_dr;
2418         dst->intercept_exceptions = from->intercept_exceptions;
2419         dst->intercept            = from->intercept;
2420         dst->iopm_base_pa         = from->iopm_base_pa;
2421         dst->msrpm_base_pa        = from->msrpm_base_pa;
2422         dst->tsc_offset           = from->tsc_offset;
2423         dst->asid                 = from->asid;
2424         dst->tlb_ctl              = from->tlb_ctl;
2425         dst->int_ctl              = from->int_ctl;
2426         dst->int_vector           = from->int_vector;
2427         dst->int_state            = from->int_state;
2428         dst->exit_code            = from->exit_code;
2429         dst->exit_code_hi         = from->exit_code_hi;
2430         dst->exit_info_1          = from->exit_info_1;
2431         dst->exit_info_2          = from->exit_info_2;
2432         dst->exit_int_info        = from->exit_int_info;
2433         dst->exit_int_info_err    = from->exit_int_info_err;
2434         dst->nested_ctl           = from->nested_ctl;
2435         dst->event_inj            = from->event_inj;
2436         dst->event_inj_err        = from->event_inj_err;
2437         dst->nested_cr3           = from->nested_cr3;
2438         dst->lbr_ctl              = from->lbr_ctl;
2439 }
2440
2441 static int nested_svm_vmexit(struct vcpu_svm *svm)
2442 {
2443         struct vmcb *nested_vmcb;
2444         struct vmcb *hsave = svm->nested.hsave;
2445         struct vmcb *vmcb = svm->vmcb;
2446         struct page *page;
2447
2448         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
2449                                        vmcb->control.exit_info_1,
2450                                        vmcb->control.exit_info_2,
2451                                        vmcb->control.exit_int_info,
2452                                        vmcb->control.exit_int_info_err,
2453                                        KVM_ISA_SVM);
2454
2455         nested_vmcb = nested_svm_map(svm, svm->nested.vmcb, &page);
2456         if (!nested_vmcb)
2457                 return 1;
2458
2459         /* Exit Guest-Mode */
2460         leave_guest_mode(&svm->vcpu);
2461         svm->nested.vmcb = 0;
2462
2463         /* Give the current vmcb to the guest */
2464         disable_gif(svm);
2465
2466         nested_vmcb->save.es     = vmcb->save.es;
2467         nested_vmcb->save.cs     = vmcb->save.cs;
2468         nested_vmcb->save.ss     = vmcb->save.ss;
2469         nested_vmcb->save.ds     = vmcb->save.ds;
2470         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
2471         nested_vmcb->save.idtr   = vmcb->save.idtr;
2472         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
2473         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
2474         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
2475         nested_vmcb->save.cr2    = vmcb->save.cr2;
2476         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
2477         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
2478         nested_vmcb->save.rip    = vmcb->save.rip;
2479         nested_vmcb->save.rsp    = vmcb->save.rsp;
2480         nested_vmcb->save.rax    = vmcb->save.rax;
2481         nested_vmcb->save.dr7    = vmcb->save.dr7;
2482         nested_vmcb->save.dr6    = vmcb->save.dr6;
2483         nested_vmcb->save.cpl    = vmcb->save.cpl;
2484
2485         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
2486         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
2487         nested_vmcb->control.int_state         = vmcb->control.int_state;
2488         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
2489         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
2490         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
2491         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
2492         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
2493         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
2494
2495         if (svm->nrips_enabled)
2496                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
2497
2498         /*
2499          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
2500          * to make sure that we do not lose injected events. So check event_inj
2501          * here and copy it to exit_int_info if it is valid.
2502          * Exit_int_info and event_inj can't be both valid because the case
2503          * below only happens on a VMRUN instruction intercept which has
2504          * no valid exit_int_info set.
2505          */
2506         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
2507                 struct vmcb_control_area *nc = &nested_vmcb->control;
2508
2509                 nc->exit_int_info     = vmcb->control.event_inj;
2510                 nc->exit_int_info_err = vmcb->control.event_inj_err;
2511         }
2512
2513         nested_vmcb->control.tlb_ctl           = 0;
2514         nested_vmcb->control.event_inj         = 0;
2515         nested_vmcb->control.event_inj_err     = 0;
2516
2517         /* We always set V_INTR_MASKING and remember the old value in hflags */
2518         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
2519                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
2520
2521         /* Restore the original control entries */
2522         copy_vmcb_control_area(vmcb, hsave);
2523
2524         kvm_clear_exception_queue(&svm->vcpu);
2525         kvm_clear_interrupt_queue(&svm->vcpu);
2526
2527         svm->nested.nested_cr3 = 0;
2528
2529         /* Restore selected save entries */
2530         svm->vmcb->save.es = hsave->save.es;
2531         svm->vmcb->save.cs = hsave->save.cs;
2532         svm->vmcb->save.ss = hsave->save.ss;
2533         svm->vmcb->save.ds = hsave->save.ds;
2534         svm->vmcb->save.gdtr = hsave->save.gdtr;
2535         svm->vmcb->save.idtr = hsave->save.idtr;
2536         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
2537         svm_set_efer(&svm->vcpu, hsave->save.efer);
2538         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
2539         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
2540         if (npt_enabled) {
2541                 svm->vmcb->save.cr3 = hsave->save.cr3;
2542                 svm->vcpu.arch.cr3 = hsave->save.cr3;
2543         } else {
2544                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
2545         }
2546         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, hsave->save.rax);
2547         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, hsave->save.rsp);
2548         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, hsave->save.rip);
2549         svm->vmcb->save.dr7 = 0;
2550         svm->vmcb->save.cpl = 0;
2551         svm->vmcb->control.exit_int_info = 0;
2552
2553         mark_all_dirty(svm->vmcb);
2554
2555         nested_svm_unmap(page);
2556
2557         nested_svm_uninit_mmu_context(&svm->vcpu);
2558         kvm_mmu_reset_context(&svm->vcpu);
2559         kvm_mmu_load(&svm->vcpu);
2560
2561         return 0;
2562 }
2563
2564 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
2565 {
2566         /*
2567          * This function merges the msr permission bitmaps of kvm and the
2568          * nested vmcb. It is optimized in that it only merges the parts where
2569          * the kvm msr permission bitmap may contain zero bits
2570          */
2571         int i;
2572
2573         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
2574                 return true;
2575
2576         for (i = 0; i < MSRPM_OFFSETS; i++) {
2577                 u32 value, p;
2578                 u64 offset;
2579
2580                 if (msrpm_offsets[i] == 0xffffffff)
2581                         break;
2582
2583                 p      = msrpm_offsets[i];
2584                 offset = svm->nested.vmcb_msrpm + (p * 4);
2585
2586                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
2587                         return false;
2588
2589                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
2590         }
2591
2592         svm->vmcb->control.msrpm_base_pa = __pa(svm->nested.msrpm);
2593
2594         return true;
2595 }
2596
2597 static bool nested_vmcb_checks(struct vmcb *vmcb)
2598 {
2599         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
2600                 return false;
2601
2602         if (vmcb->control.asid == 0)
2603                 return false;
2604
2605         if (vmcb->control.nested_ctl && !npt_enabled)
2606                 return false;
2607
2608         return true;
2609 }
2610
2611 static bool nested_svm_vmrun(struct vcpu_svm *svm)
2612 {
2613         struct vmcb *nested_vmcb;
2614         struct vmcb *hsave = svm->nested.hsave;
2615         struct vmcb *vmcb = svm->vmcb;
2616         struct page *page;
2617         u64 vmcb_gpa;
2618
2619         vmcb_gpa = svm->vmcb->save.rax;
2620
2621         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2622         if (!nested_vmcb)
2623                 return false;
2624
2625         if (!nested_vmcb_checks(nested_vmcb)) {
2626                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
2627                 nested_vmcb->control.exit_code_hi = 0;
2628                 nested_vmcb->control.exit_info_1  = 0;
2629                 nested_vmcb->control.exit_info_2  = 0;
2630
2631                 nested_svm_unmap(page);
2632
2633                 return false;
2634         }
2635
2636         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
2637                                nested_vmcb->save.rip,
2638                                nested_vmcb->control.int_ctl,
2639                                nested_vmcb->control.event_inj,
2640                                nested_vmcb->control.nested_ctl);
2641
2642         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
2643                                     nested_vmcb->control.intercept_cr >> 16,
2644                                     nested_vmcb->control.intercept_exceptions,
2645                                     nested_vmcb->control.intercept);
2646
2647         /* Clear internal status */
2648         kvm_clear_exception_queue(&svm->vcpu);
2649         kvm_clear_interrupt_queue(&svm->vcpu);
2650
2651         /*
2652          * Save the old vmcb, so we don't need to pick what we save, but can
2653          * restore everything when a VMEXIT occurs
2654          */
2655         hsave->save.es     = vmcb->save.es;
2656         hsave->save.cs     = vmcb->save.cs;
2657         hsave->save.ss     = vmcb->save.ss;
2658         hsave->save.ds     = vmcb->save.ds;
2659         hsave->save.gdtr   = vmcb->save.gdtr;
2660         hsave->save.idtr   = vmcb->save.idtr;
2661         hsave->save.efer   = svm->vcpu.arch.efer;
2662         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
2663         hsave->save.cr4    = svm->vcpu.arch.cr4;
2664         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
2665         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
2666         hsave->save.rsp    = vmcb->save.rsp;
2667         hsave->save.rax    = vmcb->save.rax;
2668         if (npt_enabled)
2669                 hsave->save.cr3    = vmcb->save.cr3;
2670         else
2671                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
2672
2673         copy_vmcb_control_area(hsave, vmcb);
2674
2675         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
2676                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
2677         else
2678                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
2679
2680         if (nested_vmcb->control.nested_ctl) {
2681                 kvm_mmu_unload(&svm->vcpu);
2682                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
2683                 nested_svm_init_mmu_context(&svm->vcpu);
2684         }
2685
2686         /* Load the nested guest state */
2687         svm->vmcb->save.es = nested_vmcb->save.es;
2688         svm->vmcb->save.cs = nested_vmcb->save.cs;
2689         svm->vmcb->save.ss = nested_vmcb->save.ss;
2690         svm->vmcb->save.ds = nested_vmcb->save.ds;
2691         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
2692         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
2693         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
2694         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
2695         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
2696         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
2697         if (npt_enabled) {
2698                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
2699                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
2700         } else
2701                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
2702
2703         /* Guest paging mode is active - reset mmu */
2704         kvm_mmu_reset_context(&svm->vcpu);
2705
2706         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
2707         kvm_register_write(&svm->vcpu, VCPU_REGS_RAX, nested_vmcb->save.rax);
2708         kvm_register_write(&svm->vcpu, VCPU_REGS_RSP, nested_vmcb->save.rsp);
2709         kvm_register_write(&svm->vcpu, VCPU_REGS_RIP, nested_vmcb->save.rip);
2710
2711         /* In case we don't even reach vcpu_run, the fields are not updated */
2712         svm->vmcb->save.rax = nested_vmcb->save.rax;
2713         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
2714         svm->vmcb->save.rip = nested_vmcb->save.rip;
2715         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
2716         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
2717         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
2718
2719         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
2720         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
2721
2722         /* cache intercepts */
2723         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
2724         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
2725         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
2726         svm->nested.intercept            = nested_vmcb->control.intercept;
2727
2728         svm_flush_tlb(&svm->vcpu);
2729         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
2730         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
2731                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
2732         else
2733                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
2734
2735         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
2736                 /* We only want the cr8 intercept bits of the guest */
2737                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
2738                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
2739         }
2740
2741         /* We don't want to see VMMCALLs from a nested guest */
2742         clr_intercept(svm, INTERCEPT_VMMCALL);
2743
2744         svm->vmcb->control.lbr_ctl = nested_vmcb->control.lbr_ctl;
2745         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
2746         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
2747         svm->vmcb->control.tsc_offset += nested_vmcb->control.tsc_offset;
2748         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
2749         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
2750
2751         nested_svm_unmap(page);
2752
2753         /* Enter Guest-Mode */
2754         enter_guest_mode(&svm->vcpu);
2755
2756         /*
2757          * Merge guest and host intercepts - must be called  with vcpu in
2758          * guest-mode to take affect here
2759          */
2760         recalc_intercepts(svm);
2761
2762         svm->nested.vmcb = vmcb_gpa;
2763
2764         enable_gif(svm);
2765
2766         mark_all_dirty(svm->vmcb);
2767
2768         return true;
2769 }
2770
2771 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
2772 {
2773         to_vmcb->save.fs = from_vmcb->save.fs;
2774         to_vmcb->save.gs = from_vmcb->save.gs;
2775         to_vmcb->save.tr = from_vmcb->save.tr;
2776         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
2777         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
2778         to_vmcb->save.star = from_vmcb->save.star;
2779         to_vmcb->save.lstar = from_vmcb->save.lstar;
2780         to_vmcb->save.cstar = from_vmcb->save.cstar;
2781         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
2782         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
2783         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
2784         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
2785 }
2786
2787 static int vmload_interception(struct vcpu_svm *svm)
2788 {
2789         struct vmcb *nested_vmcb;
2790         struct page *page;
2791
2792         if (nested_svm_check_permissions(svm))
2793                 return 1;
2794
2795         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2796         if (!nested_vmcb)
2797                 return 1;
2798
2799         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2800         skip_emulated_instruction(&svm->vcpu);
2801
2802         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
2803         nested_svm_unmap(page);
2804
2805         return 1;
2806 }
2807
2808 static int vmsave_interception(struct vcpu_svm *svm)
2809 {
2810         struct vmcb *nested_vmcb;
2811         struct page *page;
2812
2813         if (nested_svm_check_permissions(svm))
2814                 return 1;
2815
2816         nested_vmcb = nested_svm_map(svm, svm->vmcb->save.rax, &page);
2817         if (!nested_vmcb)
2818                 return 1;
2819
2820         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2821         skip_emulated_instruction(&svm->vcpu);
2822
2823         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
2824         nested_svm_unmap(page);
2825
2826         return 1;
2827 }
2828
2829 static int vmrun_interception(struct vcpu_svm *svm)
2830 {
2831         if (nested_svm_check_permissions(svm))
2832                 return 1;
2833
2834         /* Save rip after vmrun instruction */
2835         kvm_rip_write(&svm->vcpu, kvm_rip_read(&svm->vcpu) + 3);
2836
2837         if (!nested_svm_vmrun(svm))
2838                 return 1;
2839
2840         if (!nested_svm_vmrun_msrpm(svm))
2841                 goto failed;
2842
2843         return 1;
2844
2845 failed:
2846
2847         svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
2848         svm->vmcb->control.exit_code_hi = 0;
2849         svm->vmcb->control.exit_info_1  = 0;
2850         svm->vmcb->control.exit_info_2  = 0;
2851
2852         nested_svm_vmexit(svm);
2853
2854         return 1;
2855 }
2856
2857 static int stgi_interception(struct vcpu_svm *svm)
2858 {
2859         if (nested_svm_check_permissions(svm))
2860                 return 1;
2861
2862         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2863         skip_emulated_instruction(&svm->vcpu);
2864         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
2865
2866         enable_gif(svm);
2867
2868         return 1;
2869 }
2870
2871 static int clgi_interception(struct vcpu_svm *svm)
2872 {
2873         if (nested_svm_check_permissions(svm))
2874                 return 1;
2875
2876         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2877         skip_emulated_instruction(&svm->vcpu);
2878
2879         disable_gif(svm);
2880
2881         /* After a CLGI no interrupts should come */
2882         if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
2883                 svm_clear_vintr(svm);
2884                 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
2885                 mark_dirty(svm->vmcb, VMCB_INTR);
2886         }
2887
2888         return 1;
2889 }
2890
2891 static int invlpga_interception(struct vcpu_svm *svm)
2892 {
2893         struct kvm_vcpu *vcpu = &svm->vcpu;
2894
2895         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RCX),
2896                           kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2897
2898         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
2899         kvm_mmu_invlpg(vcpu, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2900
2901         svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2902         skip_emulated_instruction(&svm->vcpu);
2903         return 1;
2904 }
2905
2906 static int skinit_interception(struct vcpu_svm *svm)
2907 {
2908         trace_kvm_skinit(svm->vmcb->save.rip, kvm_register_read(&svm->vcpu, VCPU_REGS_RAX));
2909
2910         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
2911         return 1;
2912 }
2913
2914 static int wbinvd_interception(struct vcpu_svm *svm)
2915 {
2916         kvm_emulate_wbinvd(&svm->vcpu);
2917         return 1;
2918 }
2919
2920 static int xsetbv_interception(struct vcpu_svm *svm)
2921 {
2922         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
2923         u32 index = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
2924
2925         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
2926                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 3;
2927                 skip_emulated_instruction(&svm->vcpu);
2928         }
2929
2930         return 1;
2931 }
2932
2933 static int task_switch_interception(struct vcpu_svm *svm)
2934 {
2935         u16 tss_selector;
2936         int reason;
2937         int int_type = svm->vmcb->control.exit_int_info &
2938                 SVM_EXITINTINFO_TYPE_MASK;
2939         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
2940         uint32_t type =
2941                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
2942         uint32_t idt_v =
2943                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
2944         bool has_error_code = false;
2945         u32 error_code = 0;
2946
2947         tss_selector = (u16)svm->vmcb->control.exit_info_1;
2948
2949         if (svm->vmcb->control.exit_info_2 &
2950             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
2951                 reason = TASK_SWITCH_IRET;
2952         else if (svm->vmcb->control.exit_info_2 &
2953                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
2954                 reason = TASK_SWITCH_JMP;
2955         else if (idt_v)
2956                 reason = TASK_SWITCH_GATE;
2957         else
2958                 reason = TASK_SWITCH_CALL;
2959
2960         if (reason == TASK_SWITCH_GATE) {
2961                 switch (type) {
2962                 case SVM_EXITINTINFO_TYPE_NMI:
2963                         svm->vcpu.arch.nmi_injected = false;
2964                         break;
2965                 case SVM_EXITINTINFO_TYPE_EXEPT:
2966                         if (svm->vmcb->control.exit_info_2 &
2967                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
2968                                 has_error_code = true;
2969                                 error_code =
2970                                         (u32)svm->vmcb->control.exit_info_2;
2971                         }
2972                         kvm_clear_exception_queue(&svm->vcpu);
2973                         break;
2974                 case SVM_EXITINTINFO_TYPE_INTR:
2975                         kvm_clear_interrupt_queue(&svm->vcpu);
2976                         break;
2977                 default:
2978                         break;
2979                 }
2980         }
2981
2982         if (reason != TASK_SWITCH_GATE ||
2983             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
2984             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
2985              (int_vec == OF_VECTOR || int_vec == BP_VECTOR)))
2986                 skip_emulated_instruction(&svm->vcpu);
2987
2988         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
2989                 int_vec = -1;
2990
2991         if (kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
2992                                 has_error_code, error_code) == EMULATE_FAIL) {
2993                 svm->vcpu.run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
2994                 svm->vcpu.run->internal.suberror = KVM_INTERNAL_ERROR_EMULATION;
2995                 svm->vcpu.run->internal.ndata = 0;
2996                 return 0;
2997         }
2998         return 1;
2999 }
3000
3001 static int cpuid_interception(struct vcpu_svm *svm)
3002 {
3003         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3004         kvm_emulate_cpuid(&svm->vcpu);
3005         return 1;
3006 }
3007
3008 static int iret_interception(struct vcpu_svm *svm)
3009 {
3010         ++svm->vcpu.stat.nmi_window_exits;
3011         clr_intercept(svm, INTERCEPT_IRET);
3012         svm->vcpu.arch.hflags |= HF_IRET_MASK;
3013         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3014         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3015         return 1;
3016 }
3017
3018 static int invlpg_interception(struct vcpu_svm *svm)
3019 {
3020         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3021                 return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3022
3023         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3024         skip_emulated_instruction(&svm->vcpu);
3025         return 1;
3026 }
3027
3028 static int emulate_on_interception(struct vcpu_svm *svm)
3029 {
3030         return emulate_instruction(&svm->vcpu, 0) == EMULATE_DONE;
3031 }
3032
3033 static int rdpmc_interception(struct vcpu_svm *svm)
3034 {
3035         int err;
3036
3037         if (!static_cpu_has(X86_FEATURE_NRIPS))
3038                 return emulate_on_interception(svm);
3039
3040         err = kvm_rdpmc(&svm->vcpu);
3041         kvm_complete_insn_gp(&svm->vcpu, err);
3042
3043         return 1;
3044 }
3045
3046 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3047                                             unsigned long val)
3048 {
3049         unsigned long cr0 = svm->vcpu.arch.cr0;
3050         bool ret = false;
3051         u64 intercept;
3052
3053         intercept = svm->nested.intercept;
3054
3055         if (!is_guest_mode(&svm->vcpu) ||
3056             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3057                 return false;
3058
3059         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3060         val &= ~SVM_CR0_SELECTIVE_MASK;
3061
3062         if (cr0 ^ val) {
3063                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3064                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3065         }
3066
3067         return ret;
3068 }
3069
3070 #define CR_VALID (1ULL << 63)
3071
3072 static int cr_interception(struct vcpu_svm *svm)
3073 {
3074         int reg, cr;
3075         unsigned long val;
3076         int err;
3077
3078         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3079                 return emulate_on_interception(svm);
3080
3081         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
3082                 return emulate_on_interception(svm);
3083
3084         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3085         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
3086                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
3087         else
3088                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
3089
3090         err = 0;
3091         if (cr >= 16) { /* mov to cr */
3092                 cr -= 16;
3093                 val = kvm_register_read(&svm->vcpu, reg);
3094                 switch (cr) {
3095                 case 0:
3096                         if (!check_selective_cr0_intercepted(svm, val))
3097                                 err = kvm_set_cr0(&svm->vcpu, val);
3098                         else
3099                                 return 1;
3100
3101                         break;
3102                 case 3:
3103                         err = kvm_set_cr3(&svm->vcpu, val);
3104                         break;
3105                 case 4:
3106                         err = kvm_set_cr4(&svm->vcpu, val);
3107                         break;
3108                 case 8:
3109                         err = kvm_set_cr8(&svm->vcpu, val);
3110                         break;
3111                 default:
3112                         WARN(1, "unhandled write to CR%d", cr);
3113                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3114                         return 1;
3115                 }
3116         } else { /* mov from cr */
3117                 switch (cr) {
3118                 case 0:
3119                         val = kvm_read_cr0(&svm->vcpu);
3120                         break;
3121                 case 2:
3122                         val = svm->vcpu.arch.cr2;
3123                         break;
3124                 case 3:
3125                         val = kvm_read_cr3(&svm->vcpu);
3126                         break;
3127                 case 4:
3128                         val = kvm_read_cr4(&svm->vcpu);
3129                         break;
3130                 case 8:
3131                         val = kvm_get_cr8(&svm->vcpu);
3132                         break;
3133                 default:
3134                         WARN(1, "unhandled read from CR%d", cr);
3135                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3136                         return 1;
3137                 }
3138                 kvm_register_write(&svm->vcpu, reg, val);
3139         }
3140         kvm_complete_insn_gp(&svm->vcpu, err);
3141
3142         return 1;
3143 }
3144
3145 static int dr_interception(struct vcpu_svm *svm)
3146 {
3147         int reg, dr;
3148         unsigned long val;
3149
3150         if (svm->vcpu.guest_debug == 0) {
3151                 /*
3152                  * No more DR vmexits; force a reload of the debug registers
3153                  * and reenter on this instruction.  The next vmexit will
3154                  * retrieve the full state of the debug registers.
3155                  */
3156                 clr_dr_intercepts(svm);
3157                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
3158                 return 1;
3159         }
3160
3161         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
3162                 return emulate_on_interception(svm);
3163
3164         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
3165         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
3166
3167         if (dr >= 16) { /* mov to DRn */
3168                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
3169                         return 1;
3170                 val = kvm_register_read(&svm->vcpu, reg);
3171                 kvm_set_dr(&svm->vcpu, dr - 16, val);
3172         } else {
3173                 if (!kvm_require_dr(&svm->vcpu, dr))
3174                         return 1;
3175                 kvm_get_dr(&svm->vcpu, dr, &val);
3176                 kvm_register_write(&svm->vcpu, reg, val);
3177         }
3178
3179         skip_emulated_instruction(&svm->vcpu);
3180
3181         return 1;
3182 }
3183
3184 static int cr8_write_interception(struct vcpu_svm *svm)
3185 {
3186         struct kvm_run *kvm_run = svm->vcpu.run;
3187         int r;
3188
3189         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
3190         /* instruction emulation calls kvm_set_cr8() */
3191         r = cr_interception(svm);
3192         if (lapic_in_kernel(&svm->vcpu))
3193                 return r;
3194         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
3195                 return r;
3196         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
3197         return 0;
3198 }
3199
3200 static u64 svm_read_l1_tsc(struct kvm_vcpu *vcpu, u64 host_tsc)
3201 {
3202         struct vmcb *vmcb = get_host_vmcb(to_svm(vcpu));
3203         return vmcb->control.tsc_offset + host_tsc;
3204 }
3205
3206 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
3207 {
3208         struct vcpu_svm *svm = to_svm(vcpu);
3209
3210         switch (msr_info->index) {
3211         case MSR_IA32_TSC: {
3212                 msr_info->data = svm->vmcb->control.tsc_offset +
3213                         kvm_scale_tsc(vcpu, rdtsc());
3214
3215                 break;
3216         }
3217         case MSR_STAR:
3218                 msr_info->data = svm->vmcb->save.star;
3219                 break;
3220 #ifdef CONFIG_X86_64
3221         case MSR_LSTAR:
3222                 msr_info->data = svm->vmcb->save.lstar;
3223                 break;
3224         case MSR_CSTAR:
3225                 msr_info->data = svm->vmcb->save.cstar;
3226                 break;
3227         case MSR_KERNEL_GS_BASE:
3228                 msr_info->data = svm->vmcb->save.kernel_gs_base;
3229                 break;
3230         case MSR_SYSCALL_MASK:
3231                 msr_info->data = svm->vmcb->save.sfmask;
3232                 break;
3233 #endif
3234         case MSR_IA32_SYSENTER_CS:
3235                 msr_info->data = svm->vmcb->save.sysenter_cs;
3236                 break;
3237         case MSR_IA32_SYSENTER_EIP:
3238                 msr_info->data = svm->sysenter_eip;
3239                 break;
3240         case MSR_IA32_SYSENTER_ESP:
3241                 msr_info->data = svm->sysenter_esp;
3242                 break;
3243         case MSR_TSC_AUX:
3244                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3245                         return 1;
3246                 msr_info->data = svm->tsc_aux;
3247                 break;
3248         /*
3249          * Nobody will change the following 5 values in the VMCB so we can
3250          * safely return them on rdmsr. They will always be 0 until LBRV is
3251          * implemented.
3252          */
3253         case MSR_IA32_DEBUGCTLMSR:
3254                 msr_info->data = svm->vmcb->save.dbgctl;
3255                 break;
3256         case MSR_IA32_LASTBRANCHFROMIP:
3257                 msr_info->data = svm->vmcb->save.br_from;
3258                 break;
3259         case MSR_IA32_LASTBRANCHTOIP:
3260                 msr_info->data = svm->vmcb->save.br_to;
3261                 break;
3262         case MSR_IA32_LASTINTFROMIP:
3263                 msr_info->data = svm->vmcb->save.last_excp_from;
3264                 break;
3265         case MSR_IA32_LASTINTTOIP:
3266                 msr_info->data = svm->vmcb->save.last_excp_to;
3267                 break;
3268         case MSR_VM_HSAVE_PA:
3269                 msr_info->data = svm->nested.hsave_msr;
3270                 break;
3271         case MSR_VM_CR:
3272                 msr_info->data = svm->nested.vm_cr_msr;
3273                 break;
3274         case MSR_IA32_UCODE_REV:
3275                 msr_info->data = 0x01000065;
3276                 break;
3277         case MSR_F15H_IC_CFG: {
3278
3279                 int family, model;
3280
3281                 family = guest_cpuid_family(vcpu);
3282                 model  = guest_cpuid_model(vcpu);
3283
3284                 if (family < 0 || model < 0)
3285                         return kvm_get_msr_common(vcpu, msr_info);
3286
3287                 msr_info->data = 0;
3288
3289                 if (family == 0x15 &&
3290                     (model >= 0x2 && model < 0x20))
3291                         msr_info->data = 0x1E;
3292                 }
3293                 break;
3294         default:
3295                 return kvm_get_msr_common(vcpu, msr_info);
3296         }
3297         return 0;
3298 }
3299
3300 static int rdmsr_interception(struct vcpu_svm *svm)
3301 {
3302         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3303         struct msr_data msr_info;
3304
3305         msr_info.index = ecx;
3306         msr_info.host_initiated = false;
3307         if (svm_get_msr(&svm->vcpu, &msr_info)) {
3308                 trace_kvm_msr_read_ex(ecx);
3309                 kvm_inject_gp(&svm->vcpu, 0);
3310         } else {
3311                 trace_kvm_msr_read(ecx, msr_info.data);
3312
3313                 kvm_register_write(&svm->vcpu, VCPU_REGS_RAX,
3314                                    msr_info.data & 0xffffffff);
3315                 kvm_register_write(&svm->vcpu, VCPU_REGS_RDX,
3316                                    msr_info.data >> 32);
3317                 svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3318                 skip_emulated_instruction(&svm->vcpu);
3319         }
3320         return 1;
3321 }
3322
3323 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
3324 {
3325         struct vcpu_svm *svm = to_svm(vcpu);
3326         int svm_dis, chg_mask;
3327
3328         if (data & ~SVM_VM_CR_VALID_MASK)
3329                 return 1;
3330
3331         chg_mask = SVM_VM_CR_VALID_MASK;
3332
3333         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
3334                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
3335
3336         svm->nested.vm_cr_msr &= ~chg_mask;
3337         svm->nested.vm_cr_msr |= (data & chg_mask);
3338
3339         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
3340
3341         /* check for svm_disable while efer.svme is set */
3342         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
3343                 return 1;
3344
3345         return 0;
3346 }
3347
3348 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
3349 {
3350         struct vcpu_svm *svm = to_svm(vcpu);
3351
3352         u32 ecx = msr->index;
3353         u64 data = msr->data;
3354         switch (ecx) {
3355         case MSR_IA32_TSC:
3356                 kvm_write_tsc(vcpu, msr);
3357                 break;
3358         case MSR_STAR:
3359                 svm->vmcb->save.star = data;
3360                 break;
3361 #ifdef CONFIG_X86_64
3362         case MSR_LSTAR:
3363                 svm->vmcb->save.lstar = data;
3364                 break;
3365         case MSR_CSTAR:
3366                 svm->vmcb->save.cstar = data;
3367                 break;
3368         case MSR_KERNEL_GS_BASE:
3369                 svm->vmcb->save.kernel_gs_base = data;
3370                 break;
3371         case MSR_SYSCALL_MASK:
3372                 svm->vmcb->save.sfmask = data;
3373                 break;
3374 #endif
3375         case MSR_IA32_SYSENTER_CS:
3376                 svm->vmcb->save.sysenter_cs = data;
3377                 break;
3378         case MSR_IA32_SYSENTER_EIP:
3379                 svm->sysenter_eip = data;
3380                 svm->vmcb->save.sysenter_eip = data;
3381                 break;
3382         case MSR_IA32_SYSENTER_ESP:
3383                 svm->sysenter_esp = data;
3384                 svm->vmcb->save.sysenter_esp = data;
3385                 break;
3386         case MSR_TSC_AUX:
3387                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
3388                         return 1;
3389
3390                 /*
3391                  * This is rare, so we update the MSR here instead of using
3392                  * direct_access_msrs.  Doing that would require a rdmsr in
3393                  * svm_vcpu_put.
3394                  */
3395                 svm->tsc_aux = data;
3396                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
3397                 break;
3398         case MSR_IA32_DEBUGCTLMSR:
3399                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
3400                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
3401                                     __func__, data);
3402                         break;
3403                 }
3404                 if (data & DEBUGCTL_RESERVED_BITS)
3405                         return 1;
3406
3407                 svm->vmcb->save.dbgctl = data;
3408                 mark_dirty(svm->vmcb, VMCB_LBR);
3409                 if (data & (1ULL<<0))
3410                         svm_enable_lbrv(svm);
3411                 else
3412                         svm_disable_lbrv(svm);
3413                 break;
3414         case MSR_VM_HSAVE_PA:
3415                 svm->nested.hsave_msr = data;
3416                 break;
3417         case MSR_VM_CR:
3418                 return svm_set_vm_cr(vcpu, data);
3419         case MSR_VM_IGNNE:
3420                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
3421                 break;
3422         case MSR_IA32_APICBASE:
3423                 if (kvm_vcpu_apicv_active(vcpu))
3424                         avic_update_vapic_bar(to_svm(vcpu), data);
3425                 /* Follow through */
3426         default:
3427                 return kvm_set_msr_common(vcpu, msr);
3428         }
3429         return 0;
3430 }
3431
3432 static int wrmsr_interception(struct vcpu_svm *svm)
3433 {
3434         struct msr_data msr;
3435         u32 ecx = kvm_register_read(&svm->vcpu, VCPU_REGS_RCX);
3436         u64 data = kvm_read_edx_eax(&svm->vcpu);
3437
3438         msr.data = data;
3439         msr.index = ecx;
3440         msr.host_initiated = false;
3441
3442         svm->next_rip = kvm_rip_read(&svm->vcpu) + 2;
3443         if (kvm_set_msr(&svm->vcpu, &msr)) {
3444                 trace_kvm_msr_write_ex(ecx, data);
3445                 kvm_inject_gp(&svm->vcpu, 0);
3446         } else {
3447                 trace_kvm_msr_write(ecx, data);
3448                 skip_emulated_instruction(&svm->vcpu);
3449         }
3450         return 1;
3451 }
3452
3453 static int msr_interception(struct vcpu_svm *svm)
3454 {
3455         if (svm->vmcb->control.exit_info_1)
3456                 return wrmsr_interception(svm);
3457         else
3458                 return rdmsr_interception(svm);
3459 }
3460
3461 static int interrupt_window_interception(struct vcpu_svm *svm)
3462 {
3463         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3464         svm_clear_vintr(svm);
3465         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3466         mark_dirty(svm->vmcb, VMCB_INTR);
3467         ++svm->vcpu.stat.irq_window_exits;
3468         return 1;
3469 }
3470
3471 static int pause_interception(struct vcpu_svm *svm)
3472 {
3473         kvm_vcpu_on_spin(&(svm->vcpu));
3474         return 1;
3475 }
3476
3477 static int nop_interception(struct vcpu_svm *svm)
3478 {
3479         skip_emulated_instruction(&(svm->vcpu));
3480         return 1;
3481 }
3482
3483 static int monitor_interception(struct vcpu_svm *svm)
3484 {
3485         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
3486         return nop_interception(svm);
3487 }
3488
3489 static int mwait_interception(struct vcpu_svm *svm)
3490 {
3491         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
3492         return nop_interception(svm);
3493 }
3494
3495 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
3496         [SVM_EXIT_READ_CR0]                     = cr_interception,
3497         [SVM_EXIT_READ_CR3]                     = cr_interception,
3498         [SVM_EXIT_READ_CR4]                     = cr_interception,
3499         [SVM_EXIT_READ_CR8]                     = cr_interception,
3500         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
3501         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
3502         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
3503         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
3504         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
3505         [SVM_EXIT_READ_DR0]                     = dr_interception,
3506         [SVM_EXIT_READ_DR1]                     = dr_interception,
3507         [SVM_EXIT_READ_DR2]                     = dr_interception,
3508         [SVM_EXIT_READ_DR3]                     = dr_interception,
3509         [SVM_EXIT_READ_DR4]                     = dr_interception,
3510         [SVM_EXIT_READ_DR5]                     = dr_interception,
3511         [SVM_EXIT_READ_DR6]                     = dr_interception,
3512         [SVM_EXIT_READ_DR7]                     = dr_interception,
3513         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
3514         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
3515         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
3516         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
3517         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
3518         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
3519         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
3520         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
3521         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
3522         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
3523         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
3524         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
3525         [SVM_EXIT_EXCP_BASE + NM_VECTOR]        = nm_interception,
3526         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
3527         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
3528         [SVM_EXIT_INTR]                         = intr_interception,
3529         [SVM_EXIT_NMI]                          = nmi_interception,
3530         [SVM_EXIT_SMI]                          = nop_on_interception,
3531         [SVM_EXIT_INIT]                         = nop_on_interception,
3532         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
3533         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
3534         [SVM_EXIT_CPUID]                        = cpuid_interception,
3535         [SVM_EXIT_IRET]                         = iret_interception,
3536         [SVM_EXIT_INVD]                         = emulate_on_interception,
3537         [SVM_EXIT_PAUSE]                        = pause_interception,
3538         [SVM_EXIT_HLT]                          = halt_interception,
3539         [SVM_EXIT_INVLPG]                       = invlpg_interception,
3540         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
3541         [SVM_EXIT_IOIO]                         = io_interception,
3542         [SVM_EXIT_MSR]                          = msr_interception,
3543         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
3544         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
3545         [SVM_EXIT_VMRUN]                        = vmrun_interception,
3546         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
3547         [SVM_EXIT_VMLOAD]                       = vmload_interception,
3548         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
3549         [SVM_EXIT_STGI]                         = stgi_interception,
3550         [SVM_EXIT_CLGI]                         = clgi_interception,
3551         [SVM_EXIT_SKINIT]                       = skinit_interception,
3552         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
3553         [SVM_EXIT_MONITOR]                      = monitor_interception,
3554         [SVM_EXIT_MWAIT]                        = mwait_interception,
3555         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
3556         [SVM_EXIT_NPF]                          = pf_interception,
3557         [SVM_EXIT_RSM]                          = emulate_on_interception,
3558 };
3559
3560 static void dump_vmcb(struct kvm_vcpu *vcpu)
3561 {
3562         struct vcpu_svm *svm = to_svm(vcpu);
3563         struct vmcb_control_area *control = &svm->vmcb->control;
3564         struct vmcb_save_area *save = &svm->vmcb->save;
3565
3566         pr_err("VMCB Control Area:\n");
3567         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
3568         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
3569         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
3570         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
3571         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
3572         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
3573         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
3574         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
3575         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
3576         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
3577         pr_err("%-20s%d\n", "asid:", control->asid);
3578         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
3579         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
3580         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
3581         pr_err("%-20s%08x\n", "int_state:", control->int_state);
3582         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
3583         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
3584         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
3585         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
3586         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
3587         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
3588         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
3589         pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
3590         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
3591         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
3592         pr_err("%-20s%lld\n", "lbr_ctl:", control->lbr_ctl);
3593         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
3594         pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
3595         pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
3596         pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
3597         pr_err("VMCB State Save Area:\n");
3598         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3599                "es:",
3600                save->es.selector, save->es.attrib,
3601                save->es.limit, save->es.base);
3602         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3603                "cs:",
3604                save->cs.selector, save->cs.attrib,
3605                save->cs.limit, save->cs.base);
3606         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3607                "ss:",
3608                save->ss.selector, save->ss.attrib,
3609                save->ss.limit, save->ss.base);
3610         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3611                "ds:",
3612                save->ds.selector, save->ds.attrib,
3613                save->ds.limit, save->ds.base);
3614         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3615                "fs:",
3616                save->fs.selector, save->fs.attrib,
3617                save->fs.limit, save->fs.base);
3618         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3619                "gs:",
3620                save->gs.selector, save->gs.attrib,
3621                save->gs.limit, save->gs.base);
3622         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3623                "gdtr:",
3624                save->gdtr.selector, save->gdtr.attrib,
3625                save->gdtr.limit, save->gdtr.base);
3626         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3627                "ldtr:",
3628                save->ldtr.selector, save->ldtr.attrib,
3629                save->ldtr.limit, save->ldtr.base);
3630         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3631                "idtr:",
3632                save->idtr.selector, save->idtr.attrib,
3633                save->idtr.limit, save->idtr.base);
3634         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
3635                "tr:",
3636                save->tr.selector, save->tr.attrib,
3637                save->tr.limit, save->tr.base);
3638         pr_err("cpl:            %d                efer:         %016llx\n",
3639                 save->cpl, save->efer);
3640         pr_err("%-15s %016llx %-13s %016llx\n",
3641                "cr0:", save->cr0, "cr2:", save->cr2);
3642         pr_err("%-15s %016llx %-13s %016llx\n",
3643                "cr3:", save->cr3, "cr4:", save->cr4);
3644         pr_err("%-15s %016llx %-13s %016llx\n",
3645                "dr6:", save->dr6, "dr7:", save->dr7);
3646         pr_err("%-15s %016llx %-13s %016llx\n",
3647                "rip:", save->rip, "rflags:", save->rflags);
3648         pr_err("%-15s %016llx %-13s %016llx\n",
3649                "rsp:", save->rsp, "rax:", save->rax);
3650         pr_err("%-15s %016llx %-13s %016llx\n",
3651                "star:", save->star, "lstar:", save->lstar);
3652         pr_err("%-15s %016llx %-13s %016llx\n",
3653                "cstar:", save->cstar, "sfmask:", save->sfmask);
3654         pr_err("%-15s %016llx %-13s %016llx\n",
3655                "kernel_gs_base:", save->kernel_gs_base,
3656                "sysenter_cs:", save->sysenter_cs);
3657         pr_err("%-15s %016llx %-13s %016llx\n",
3658                "sysenter_esp:", save->sysenter_esp,
3659                "sysenter_eip:", save->sysenter_eip);
3660         pr_err("%-15s %016llx %-13s %016llx\n",
3661                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
3662         pr_err("%-15s %016llx %-13s %016llx\n",
3663                "br_from:", save->br_from, "br_to:", save->br_to);
3664         pr_err("%-15s %016llx %-13s %016llx\n",
3665                "excp_from:", save->last_excp_from,
3666                "excp_to:", save->last_excp_to);
3667 }
3668
3669 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
3670 {
3671         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
3672
3673         *info1 = control->exit_info_1;
3674         *info2 = control->exit_info_2;
3675 }
3676
3677 static int handle_exit(struct kvm_vcpu *vcpu)
3678 {
3679         struct vcpu_svm *svm = to_svm(vcpu);
3680         struct kvm_run *kvm_run = vcpu->run;
3681         u32 exit_code = svm->vmcb->control.exit_code;
3682
3683         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
3684
3685         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
3686                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
3687         if (npt_enabled)
3688                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
3689
3690         if (unlikely(svm->nested.exit_required)) {
3691                 nested_svm_vmexit(svm);
3692                 svm->nested.exit_required = false;
3693
3694                 return 1;
3695         }
3696
3697         if (is_guest_mode(vcpu)) {
3698                 int vmexit;
3699
3700                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
3701                                         svm->vmcb->control.exit_info_1,
3702                                         svm->vmcb->control.exit_info_2,
3703                                         svm->vmcb->control.exit_int_info,
3704                                         svm->vmcb->control.exit_int_info_err,
3705                                         KVM_ISA_SVM);
3706
3707                 vmexit = nested_svm_exit_special(svm);
3708
3709                 if (vmexit == NESTED_EXIT_CONTINUE)
3710                         vmexit = nested_svm_exit_handled(svm);
3711
3712                 if (vmexit == NESTED_EXIT_DONE)
3713                         return 1;
3714         }
3715
3716         svm_complete_interrupts(svm);
3717
3718         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
3719                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
3720                 kvm_run->fail_entry.hardware_entry_failure_reason
3721                         = svm->vmcb->control.exit_code;
3722                 pr_err("KVM: FAILED VMRUN WITH VMCB:\n");
3723                 dump_vmcb(vcpu);
3724                 return 0;
3725         }
3726
3727         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
3728             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
3729             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
3730             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
3731                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
3732                        "exit_code 0x%x\n",
3733                        __func__, svm->vmcb->control.exit_int_info,
3734                        exit_code);
3735
3736         if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
3737             || !svm_exit_handlers[exit_code]) {
3738                 WARN_ONCE(1, "svm: unexpected exit reason 0x%x\n", exit_code);
3739                 kvm_queue_exception(vcpu, UD_VECTOR);
3740                 return 1;
3741         }
3742
3743         return svm_exit_handlers[exit_code](svm);
3744 }
3745
3746 static void reload_tss(struct kvm_vcpu *vcpu)
3747 {
3748         int cpu = raw_smp_processor_id();
3749
3750         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3751         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
3752         load_TR_desc();
3753 }
3754
3755 static void pre_svm_run(struct vcpu_svm *svm)
3756 {
3757         int cpu = raw_smp_processor_id();
3758
3759         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
3760
3761         /* FIXME: handle wraparound of asid_generation */
3762         if (svm->asid_generation != sd->asid_generation)
3763                 new_asid(svm, sd);
3764 }
3765
3766 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
3767 {
3768         struct vcpu_svm *svm = to_svm(vcpu);
3769
3770         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
3771         vcpu->arch.hflags |= HF_NMI_MASK;
3772         set_intercept(svm, INTERCEPT_IRET);
3773         ++vcpu->stat.nmi_injections;
3774 }
3775
3776 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
3777 {
3778         struct vmcb_control_area *control;
3779
3780         /* The following fields are ignored when AVIC is enabled */
3781         control = &svm->vmcb->control;
3782         control->int_vector = irq;
3783         control->int_ctl &= ~V_INTR_PRIO_MASK;
3784         control->int_ctl |= V_IRQ_MASK |
3785                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
3786         mark_dirty(svm->vmcb, VMCB_INTR);
3787 }
3788
3789 static void svm_set_irq(struct kvm_vcpu *vcpu)
3790 {
3791         struct vcpu_svm *svm = to_svm(vcpu);
3792
3793         BUG_ON(!(gif_set(svm)));
3794
3795         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
3796         ++vcpu->stat.irq_injections;
3797
3798         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
3799                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
3800 }
3801
3802 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
3803 {
3804         struct vcpu_svm *svm = to_svm(vcpu);
3805
3806         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3807                 return;
3808
3809         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3810
3811         if (irr == -1)
3812                 return;
3813
3814         if (tpr >= irr)
3815                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3816 }
3817
3818 static void svm_set_virtual_x2apic_mode(struct kvm_vcpu *vcpu, bool set)
3819 {
3820         return;
3821 }
3822
3823 static bool svm_get_enable_apicv(void)
3824 {
3825         return avic;
3826 }
3827
3828 static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
3829 {
3830 }
3831
3832 static void svm_hwapic_isr_update(struct kvm *kvm, int isr)
3833 {
3834 }
3835
3836 /* Note: Currently only used by Hyper-V. */
3837 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
3838 {
3839         struct vcpu_svm *svm = to_svm(vcpu);
3840         struct vmcb *vmcb = svm->vmcb;
3841
3842         if (!avic)
3843                 return;
3844
3845         vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
3846         mark_dirty(vmcb, VMCB_INTR);
3847 }
3848
3849 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
3850 {
3851         return;
3852 }
3853
3854 static void svm_sync_pir_to_irr(struct kvm_vcpu *vcpu)
3855 {
3856         return;
3857 }
3858
3859 static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
3860 {
3861         kvm_lapic_set_irr(vec, vcpu->arch.apic);
3862         smp_mb__after_atomic();
3863
3864         if (avic_vcpu_is_running(vcpu))
3865                 wrmsrl(SVM_AVIC_DOORBELL,
3866                        __default_cpu_present_to_apicid(vcpu->cpu));
3867         else
3868                 kvm_vcpu_wake_up(vcpu);
3869 }
3870
3871 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
3872 {
3873         struct vcpu_svm *svm = to_svm(vcpu);
3874         struct vmcb *vmcb = svm->vmcb;
3875         int ret;
3876         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
3877               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
3878         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
3879
3880         return ret;
3881 }
3882
3883 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
3884 {
3885         struct vcpu_svm *svm = to_svm(vcpu);
3886
3887         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
3888 }
3889
3890 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
3891 {
3892         struct vcpu_svm *svm = to_svm(vcpu);
3893
3894         if (masked) {
3895                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
3896                 set_intercept(svm, INTERCEPT_IRET);
3897         } else {
3898                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
3899                 clr_intercept(svm, INTERCEPT_IRET);
3900         }
3901 }
3902
3903 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
3904 {
3905         struct vcpu_svm *svm = to_svm(vcpu);
3906         struct vmcb *vmcb = svm->vmcb;
3907         int ret;
3908
3909         if (!gif_set(svm) ||
3910              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
3911                 return 0;
3912
3913         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
3914
3915         if (is_guest_mode(vcpu))
3916                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
3917
3918         return ret;
3919 }
3920
3921 static void enable_irq_window(struct kvm_vcpu *vcpu)
3922 {
3923         struct vcpu_svm *svm = to_svm(vcpu);
3924
3925         if (kvm_vcpu_apicv_active(vcpu))
3926                 return;
3927
3928         /*
3929          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
3930          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
3931          * get that intercept, this function will be called again though and
3932          * we'll get the vintr intercept.
3933          */
3934         if (gif_set(svm) && nested_svm_intr(svm)) {
3935                 svm_set_vintr(svm);
3936                 svm_inject_irq(svm, 0x0);
3937         }
3938 }
3939
3940 static void enable_nmi_window(struct kvm_vcpu *vcpu)
3941 {
3942         struct vcpu_svm *svm = to_svm(vcpu);
3943
3944         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
3945             == HF_NMI_MASK)
3946                 return; /* IRET will cause a vm exit */
3947
3948         /*
3949          * Something prevents NMI from been injected. Single step over possible
3950          * problem (IRET or exception injection or interrupt shadow)
3951          */
3952         svm->nmi_singlestep = true;
3953         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
3954 }
3955
3956 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
3957 {
3958         return 0;
3959 }
3960
3961 static void svm_flush_tlb(struct kvm_vcpu *vcpu)
3962 {
3963         struct vcpu_svm *svm = to_svm(vcpu);
3964
3965         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
3966                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
3967         else
3968                 svm->asid_generation--;
3969 }
3970
3971 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
3972 {
3973 }
3974
3975 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
3976 {
3977         struct vcpu_svm *svm = to_svm(vcpu);
3978
3979         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3980                 return;
3981
3982         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
3983                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
3984                 kvm_set_cr8(vcpu, cr8);
3985         }
3986 }
3987
3988 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
3989 {
3990         struct vcpu_svm *svm = to_svm(vcpu);
3991         u64 cr8;
3992
3993         if (is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK))
3994                 return;
3995
3996         cr8 = kvm_get_cr8(vcpu);
3997         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
3998         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
3999 }
4000
4001 static void svm_complete_interrupts(struct vcpu_svm *svm)
4002 {
4003         u8 vector;
4004         int type;
4005         u32 exitintinfo = svm->vmcb->control.exit_int_info;
4006         unsigned int3_injected = svm->int3_injected;
4007
4008         svm->int3_injected = 0;
4009
4010         /*
4011          * If we've made progress since setting HF_IRET_MASK, we've
4012          * executed an IRET and can allow NMI injection.
4013          */
4014         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
4015             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
4016                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
4017                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4018         }
4019
4020         svm->vcpu.arch.nmi_injected = false;
4021         kvm_clear_exception_queue(&svm->vcpu);
4022         kvm_clear_interrupt_queue(&svm->vcpu);
4023
4024         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
4025                 return;
4026
4027         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4028
4029         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
4030         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
4031
4032         switch (type) {
4033         case SVM_EXITINTINFO_TYPE_NMI:
4034                 svm->vcpu.arch.nmi_injected = true;
4035                 break;
4036         case SVM_EXITINTINFO_TYPE_EXEPT:
4037                 /*
4038                  * In case of software exceptions, do not reinject the vector,
4039                  * but re-execute the instruction instead. Rewind RIP first
4040                  * if we emulated INT3 before.
4041                  */
4042                 if (kvm_exception_is_soft(vector)) {
4043                         if (vector == BP_VECTOR && int3_injected &&
4044                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
4045                                 kvm_rip_write(&svm->vcpu,
4046                                               kvm_rip_read(&svm->vcpu) -
4047                                               int3_injected);
4048                         break;
4049                 }
4050                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
4051                         u32 err = svm->vmcb->control.exit_int_info_err;
4052                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
4053
4054                 } else
4055                         kvm_requeue_exception(&svm->vcpu, vector);
4056                 break;
4057         case SVM_EXITINTINFO_TYPE_INTR:
4058                 kvm_queue_interrupt(&svm->vcpu, vector, false);
4059                 break;
4060         default:
4061                 break;
4062         }
4063 }
4064
4065 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
4066 {
4067         struct vcpu_svm *svm = to_svm(vcpu);
4068         struct vmcb_control_area *control = &svm->vmcb->control;
4069
4070         control->exit_int_info = control->event_inj;
4071         control->exit_int_info_err = control->event_inj_err;
4072         control->event_inj = 0;
4073         svm_complete_interrupts(svm);
4074 }
4075
4076 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
4077 {
4078         struct vcpu_svm *svm = to_svm(vcpu);
4079
4080         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
4081         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
4082         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
4083
4084         /*
4085          * A vmexit emulation is required before the vcpu can be executed
4086          * again.
4087          */
4088         if (unlikely(svm->nested.exit_required))
4089                 return;
4090
4091         pre_svm_run(svm);
4092
4093         sync_lapic_to_cr8(vcpu);
4094
4095         svm->vmcb->save.cr2 = vcpu->arch.cr2;
4096
4097         clgi();
4098
4099         local_irq_enable();
4100
4101         asm volatile (
4102                 "push %%" _ASM_BP "; \n\t"
4103                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
4104                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
4105                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
4106                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
4107                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
4108                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
4109 #ifdef CONFIG_X86_64
4110                 "mov %c[r8](%[svm]),  %%r8  \n\t"
4111                 "mov %c[r9](%[svm]),  %%r9  \n\t"
4112                 "mov %c[r10](%[svm]), %%r10 \n\t"
4113                 "mov %c[r11](%[svm]), %%r11 \n\t"
4114                 "mov %c[r12](%[svm]), %%r12 \n\t"
4115                 "mov %c[r13](%[svm]), %%r13 \n\t"
4116                 "mov %c[r14](%[svm]), %%r14 \n\t"
4117                 "mov %c[r15](%[svm]), %%r15 \n\t"
4118 #endif
4119
4120                 /* Enter guest mode */
4121                 "push %%" _ASM_AX " \n\t"
4122                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
4123                 __ex(SVM_VMLOAD) "\n\t"
4124                 __ex(SVM_VMRUN) "\n\t"
4125                 __ex(SVM_VMSAVE) "\n\t"
4126                 "pop %%" _ASM_AX " \n\t"
4127
4128                 /* Save guest registers, load host registers */
4129                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
4130                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
4131                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
4132                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
4133                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
4134                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
4135 #ifdef CONFIG_X86_64
4136                 "mov %%r8,  %c[r8](%[svm]) \n\t"
4137                 "mov %%r9,  %c[r9](%[svm]) \n\t"
4138                 "mov %%r10, %c[r10](%[svm]) \n\t"
4139                 "mov %%r11, %c[r11](%[svm]) \n\t"
4140                 "mov %%r12, %c[r12](%[svm]) \n\t"
4141                 "mov %%r13, %c[r13](%[svm]) \n\t"
4142                 "mov %%r14, %c[r14](%[svm]) \n\t"
4143                 "mov %%r15, %c[r15](%[svm]) \n\t"
4144 #endif
4145                 "pop %%" _ASM_BP
4146                 :
4147                 : [svm]"a"(svm),
4148                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
4149                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
4150                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
4151                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
4152                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
4153                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
4154                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
4155 #ifdef CONFIG_X86_64
4156                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
4157                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
4158                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
4159                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
4160                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
4161                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
4162                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
4163                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
4164 #endif
4165                 : "cc", "memory"
4166 #ifdef CONFIG_X86_64
4167                 , "rbx", "rcx", "rdx", "rsi", "rdi"
4168                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
4169 #else
4170                 , "ebx", "ecx", "edx", "esi", "edi"
4171 #endif
4172                 );
4173
4174 #ifdef CONFIG_X86_64
4175         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
4176 #else
4177         loadsegment(fs, svm->host.fs);
4178 #ifndef CONFIG_X86_32_LAZY_GS
4179         loadsegment(gs, svm->host.gs);
4180 #endif
4181 #endif
4182
4183         reload_tss(vcpu);
4184
4185         local_irq_disable();
4186
4187         vcpu->arch.cr2 = svm->vmcb->save.cr2;
4188         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
4189         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
4190         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
4191
4192         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4193                 kvm_before_handle_nmi(&svm->vcpu);
4194
4195         stgi();
4196
4197         /* Any pending NMI will happen here */
4198
4199         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
4200                 kvm_after_handle_nmi(&svm->vcpu);
4201
4202         sync_cr8_to_lapic(vcpu);
4203
4204         svm->next_rip = 0;
4205
4206         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
4207
4208         /* if exit due to PF check for async PF */
4209         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
4210                 svm->apf_reason = kvm_read_and_reset_pf_reason();
4211
4212         if (npt_enabled) {
4213                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
4214                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
4215         }
4216
4217         /*
4218          * We need to handle MC intercepts here before the vcpu has a chance to
4219          * change the physical cpu
4220          */
4221         if (unlikely(svm->vmcb->control.exit_code ==
4222                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
4223                 svm_handle_mce(svm);
4224
4225         mark_all_clean(svm->vmcb);
4226 }
4227
4228 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
4229 {
4230         struct vcpu_svm *svm = to_svm(vcpu);
4231
4232         svm->vmcb->save.cr3 = root;
4233         mark_dirty(svm->vmcb, VMCB_CR);
4234         svm_flush_tlb(vcpu);
4235 }
4236
4237 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
4238 {
4239         struct vcpu_svm *svm = to_svm(vcpu);
4240
4241         svm->vmcb->control.nested_cr3 = root;
4242         mark_dirty(svm->vmcb, VMCB_NPT);
4243
4244         /* Also sync guest cr3 here in case we live migrate */
4245         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
4246         mark_dirty(svm->vmcb, VMCB_CR);
4247
4248         svm_flush_tlb(vcpu);
4249 }
4250
4251 static int is_disabled(void)
4252 {
4253         u64 vm_cr;
4254
4255         rdmsrl(MSR_VM_CR, vm_cr);
4256         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
4257                 return 1;
4258
4259         return 0;
4260 }
4261
4262 static void
4263 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
4264 {
4265         /*
4266          * Patch in the VMMCALL instruction:
4267          */
4268         hypercall[0] = 0x0f;
4269         hypercall[1] = 0x01;
4270         hypercall[2] = 0xd9;
4271 }
4272
4273 static void svm_check_processor_compat(void *rtn)
4274 {
4275         *(int *)rtn = 0;
4276 }
4277
4278 static bool svm_cpu_has_accelerated_tpr(void)
4279 {
4280         return false;
4281 }
4282
4283 static bool svm_has_high_real_mode_segbase(void)
4284 {
4285         return true;
4286 }
4287
4288 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
4289 {
4290         return 0;
4291 }
4292
4293 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
4294 {
4295         struct vcpu_svm *svm = to_svm(vcpu);
4296
4297         /* Update nrips enabled cache */
4298         svm->nrips_enabled = !!guest_cpuid_has_nrips(&svm->vcpu);
4299 }
4300
4301 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
4302 {
4303         switch (func) {
4304         case 0x80000001:
4305                 if (nested)
4306                         entry->ecx |= (1 << 2); /* Set SVM bit */
4307                 break;
4308         case 0x8000000A:
4309                 entry->eax = 1; /* SVM revision 1 */
4310                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
4311                                    ASID emulation to nested SVM */
4312                 entry->ecx = 0; /* Reserved */
4313                 entry->edx = 0; /* Per default do not support any
4314                                    additional features */
4315
4316                 /* Support next_rip if host supports it */
4317                 if (boot_cpu_has(X86_FEATURE_NRIPS))
4318                         entry->edx |= SVM_FEATURE_NRIP;
4319
4320                 /* Support NPT for the guest if enabled */
4321                 if (npt_enabled)
4322                         entry->edx |= SVM_FEATURE_NPT;
4323
4324                 break;
4325         }
4326 }
4327
4328 static int svm_get_lpage_level(void)
4329 {
4330         return PT_PDPE_LEVEL;
4331 }
4332
4333 static bool svm_rdtscp_supported(void)
4334 {
4335         return boot_cpu_has(X86_FEATURE_RDTSCP);
4336 }
4337
4338 static bool svm_invpcid_supported(void)
4339 {
4340         return false;
4341 }
4342
4343 static bool svm_mpx_supported(void)
4344 {
4345         return false;
4346 }
4347
4348 static bool svm_xsaves_supported(void)
4349 {
4350         return false;
4351 }
4352
4353 static bool svm_has_wbinvd_exit(void)
4354 {
4355         return true;
4356 }
4357
4358 static void svm_fpu_deactivate(struct kvm_vcpu *vcpu)
4359 {
4360         struct vcpu_svm *svm = to_svm(vcpu);
4361
4362         set_exception_intercept(svm, NM_VECTOR);
4363         update_cr0_intercept(svm);
4364 }
4365
4366 #define PRE_EX(exit)  { .exit_code = (exit), \
4367                         .stage = X86_ICPT_PRE_EXCEPT, }
4368 #define POST_EX(exit) { .exit_code = (exit), \
4369                         .stage = X86_ICPT_POST_EXCEPT, }
4370 #define POST_MEM(exit) { .exit_code = (exit), \
4371                         .stage = X86_ICPT_POST_MEMACCESS, }
4372
4373 static const struct __x86_intercept {
4374         u32 exit_code;
4375         enum x86_intercept_stage stage;
4376 } x86_intercept_map[] = {
4377         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
4378         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
4379         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
4380         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
4381         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
4382         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
4383         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
4384         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
4385         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
4386         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
4387         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
4388         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
4389         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
4390         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
4391         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
4392         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
4393         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
4394         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
4395         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
4396         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
4397         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
4398         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
4399         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
4400         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
4401         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
4402         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
4403         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
4404         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
4405         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
4406         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
4407         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
4408         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
4409         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
4410         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
4411         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
4412         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
4413         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
4414         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
4415         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
4416         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
4417         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
4418         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
4419         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
4420         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
4421         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
4422         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
4423 };
4424
4425 #undef PRE_EX
4426 #undef POST_EX
4427 #undef POST_MEM
4428
4429 static int svm_check_intercept(struct kvm_vcpu *vcpu,
4430                                struct x86_instruction_info *info,
4431                                enum x86_intercept_stage stage)
4432 {
4433         struct vcpu_svm *svm = to_svm(vcpu);
4434         int vmexit, ret = X86EMUL_CONTINUE;
4435         struct __x86_intercept icpt_info;
4436         struct vmcb *vmcb = svm->vmcb;
4437
4438         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
4439                 goto out;
4440
4441         icpt_info = x86_intercept_map[info->intercept];
4442
4443         if (stage != icpt_info.stage)
4444                 goto out;
4445
4446         switch (icpt_info.exit_code) {
4447         case SVM_EXIT_READ_CR0:
4448                 if (info->intercept == x86_intercept_cr_read)
4449                         icpt_info.exit_code += info->modrm_reg;
4450                 break;
4451         case SVM_EXIT_WRITE_CR0: {
4452                 unsigned long cr0, val;
4453                 u64 intercept;
4454
4455                 if (info->intercept == x86_intercept_cr_write)
4456                         icpt_info.exit_code += info->modrm_reg;
4457
4458                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
4459                     info->intercept == x86_intercept_clts)
4460                         break;
4461
4462                 intercept = svm->nested.intercept;
4463
4464                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
4465                         break;
4466
4467                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
4468                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
4469
4470                 if (info->intercept == x86_intercept_lmsw) {
4471                         cr0 &= 0xfUL;
4472                         val &= 0xfUL;
4473                         /* lmsw can't clear PE - catch this here */
4474                         if (cr0 & X86_CR0_PE)
4475                                 val |= X86_CR0_PE;
4476                 }
4477
4478                 if (cr0 ^ val)
4479                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
4480
4481                 break;
4482         }
4483         case SVM_EXIT_READ_DR0:
4484         case SVM_EXIT_WRITE_DR0:
4485                 icpt_info.exit_code += info->modrm_reg;
4486                 break;
4487         case SVM_EXIT_MSR:
4488                 if (info->intercept == x86_intercept_wrmsr)
4489                         vmcb->control.exit_info_1 = 1;
4490                 else
4491                         vmcb->control.exit_info_1 = 0;
4492                 break;
4493         case SVM_EXIT_PAUSE:
4494                 /*
4495                  * We get this for NOP only, but pause
4496                  * is rep not, check this here
4497                  */
4498                 if (info->rep_prefix != REPE_PREFIX)
4499                         goto out;
4500         case SVM_EXIT_IOIO: {
4501                 u64 exit_info;
4502                 u32 bytes;
4503
4504                 if (info->intercept == x86_intercept_in ||
4505                     info->intercept == x86_intercept_ins) {
4506                         exit_info = ((info->src_val & 0xffff) << 16) |
4507                                 SVM_IOIO_TYPE_MASK;
4508                         bytes = info->dst_bytes;
4509                 } else {
4510                         exit_info = (info->dst_val & 0xffff) << 16;
4511                         bytes = info->src_bytes;
4512                 }
4513
4514                 if (info->intercept == x86_intercept_outs ||
4515                     info->intercept == x86_intercept_ins)
4516                         exit_info |= SVM_IOIO_STR_MASK;
4517
4518                 if (info->rep_prefix)
4519                         exit_info |= SVM_IOIO_REP_MASK;
4520
4521                 bytes = min(bytes, 4u);
4522
4523                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
4524
4525                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
4526
4527                 vmcb->control.exit_info_1 = exit_info;
4528                 vmcb->control.exit_info_2 = info->next_rip;
4529
4530                 break;
4531         }
4532         default:
4533                 break;
4534         }
4535
4536         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
4537         if (static_cpu_has(X86_FEATURE_NRIPS))
4538                 vmcb->control.next_rip  = info->next_rip;
4539         vmcb->control.exit_code = icpt_info.exit_code;
4540         vmexit = nested_svm_exit_handled(svm);
4541
4542         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
4543                                            : X86EMUL_CONTINUE;
4544
4545 out:
4546         return ret;
4547 }
4548
4549 static void svm_handle_external_intr(struct kvm_vcpu *vcpu)
4550 {
4551         local_irq_enable();
4552 }
4553
4554 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
4555 {
4556 }
4557
4558 static struct kvm_x86_ops svm_x86_ops = {
4559         .cpu_has_kvm_support = has_svm,
4560         .disabled_by_bios = is_disabled,
4561         .hardware_setup = svm_hardware_setup,
4562         .hardware_unsetup = svm_hardware_unsetup,
4563         .check_processor_compatibility = svm_check_processor_compat,
4564         .hardware_enable = svm_hardware_enable,
4565         .hardware_disable = svm_hardware_disable,
4566         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
4567         .cpu_has_high_real_mode_segbase = svm_has_high_real_mode_segbase,
4568
4569         .vcpu_create = svm_create_vcpu,
4570         .vcpu_free = svm_free_vcpu,
4571         .vcpu_reset = svm_vcpu_reset,
4572
4573         .vm_init = avic_vm_init,
4574         .vm_destroy = avic_vm_destroy,
4575
4576         .prepare_guest_switch = svm_prepare_guest_switch,
4577         .vcpu_load = svm_vcpu_load,
4578         .vcpu_put = svm_vcpu_put,
4579
4580         .update_bp_intercept = update_bp_intercept,
4581         .get_msr = svm_get_msr,
4582         .set_msr = svm_set_msr,
4583         .get_segment_base = svm_get_segment_base,
4584         .get_segment = svm_get_segment,
4585         .set_segment = svm_set_segment,
4586         .get_cpl = svm_get_cpl,
4587         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
4588         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
4589         .decache_cr3 = svm_decache_cr3,
4590         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
4591         .set_cr0 = svm_set_cr0,
4592         .set_cr3 = svm_set_cr3,
4593         .set_cr4 = svm_set_cr4,
4594         .set_efer = svm_set_efer,
4595         .get_idt = svm_get_idt,
4596         .set_idt = svm_set_idt,
4597         .get_gdt = svm_get_gdt,
4598         .set_gdt = svm_set_gdt,
4599         .get_dr6 = svm_get_dr6,
4600         .set_dr6 = svm_set_dr6,
4601         .set_dr7 = svm_set_dr7,
4602         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
4603         .cache_reg = svm_cache_reg,
4604         .get_rflags = svm_get_rflags,
4605         .set_rflags = svm_set_rflags,
4606
4607         .get_pkru = svm_get_pkru,
4608
4609         .fpu_activate = svm_fpu_activate,
4610         .fpu_deactivate = svm_fpu_deactivate,
4611
4612         .tlb_flush = svm_flush_tlb,
4613
4614         .run = svm_vcpu_run,
4615         .handle_exit = handle_exit,
4616         .skip_emulated_instruction = skip_emulated_instruction,
4617         .set_interrupt_shadow = svm_set_interrupt_shadow,
4618         .get_interrupt_shadow = svm_get_interrupt_shadow,
4619         .patch_hypercall = svm_patch_hypercall,
4620         .set_irq = svm_set_irq,
4621         .set_nmi = svm_inject_nmi,
4622         .queue_exception = svm_queue_exception,
4623         .cancel_injection = svm_cancel_injection,
4624         .interrupt_allowed = svm_interrupt_allowed,
4625         .nmi_allowed = svm_nmi_allowed,
4626         .get_nmi_mask = svm_get_nmi_mask,
4627         .set_nmi_mask = svm_set_nmi_mask,
4628         .enable_nmi_window = enable_nmi_window,
4629         .enable_irq_window = enable_irq_window,
4630         .update_cr8_intercept = update_cr8_intercept,
4631         .set_virtual_x2apic_mode = svm_set_virtual_x2apic_mode,
4632         .get_enable_apicv = svm_get_enable_apicv,
4633         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
4634         .load_eoi_exitmap = svm_load_eoi_exitmap,
4635         .sync_pir_to_irr = svm_sync_pir_to_irr,
4636         .hwapic_irr_update = svm_hwapic_irr_update,
4637         .hwapic_isr_update = svm_hwapic_isr_update,
4638
4639         .set_tss_addr = svm_set_tss_addr,
4640         .get_tdp_level = get_npt_level,
4641         .get_mt_mask = svm_get_mt_mask,
4642
4643         .get_exit_info = svm_get_exit_info,
4644
4645         .get_lpage_level = svm_get_lpage_level,
4646
4647         .cpuid_update = svm_cpuid_update,
4648
4649         .rdtscp_supported = svm_rdtscp_supported,
4650         .invpcid_supported = svm_invpcid_supported,
4651         .mpx_supported = svm_mpx_supported,
4652         .xsaves_supported = svm_xsaves_supported,
4653
4654         .set_supported_cpuid = svm_set_supported_cpuid,
4655
4656         .has_wbinvd_exit = svm_has_wbinvd_exit,
4657
4658         .read_tsc_offset = svm_read_tsc_offset,
4659         .write_tsc_offset = svm_write_tsc_offset,
4660         .adjust_tsc_offset_guest = svm_adjust_tsc_offset_guest,
4661         .read_l1_tsc = svm_read_l1_tsc,
4662
4663         .set_tdp_cr3 = set_tdp_cr3,
4664
4665         .check_intercept = svm_check_intercept,
4666         .handle_external_intr = svm_handle_external_intr,
4667
4668         .sched_in = svm_sched_in,
4669
4670         .pmu_ops = &amd_pmu_ops,
4671         .deliver_posted_interrupt = svm_deliver_avic_intr,
4672 };
4673
4674 static int __init svm_init(void)
4675 {
4676         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
4677                         __alignof__(struct vcpu_svm), THIS_MODULE);
4678 }
4679
4680 static void __exit svm_exit(void)
4681 {
4682         kvm_exit();
4683 }
4684
4685 module_init(svm_init)
4686 module_exit(svm_exit)