arch/x86/kvm/svm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Kernel-based Virtual Machine driver for Linux
   4  *
   5  * AMD SVM support
   6  *
   7  * Copyright (C) 2006 Qumranet, Inc.
   8  * Copyright 2010 Red Hat, Inc. and/or its affiliates.
   9  *
  10  * Authors:
  11  *   Yaniv Kamay  <yaniv@qumranet.com>
  12  *   Avi Kivity   <avi@qumranet.com>
  13  */
  14
  15 #define pr_fmt(fmt) "SVM: " fmt
  16
  17 #include <linux/kvm_host.h>
  18
  19 #include "irq.h"
  20 #include "mmu.h"
  21 #include "kvm_cache_regs.h"
  22 #include "x86.h"
  23 #include "cpuid.h"
  24 #include "pmu.h"
  25
  26 #include <linux/module.h>
  27 #include <linux/mod_devicetable.h>
  28 #include <linux/kernel.h>
  29 #include <linux/vmalloc.h>
  30 #include <linux/highmem.h>
  31 #include <linux/sched.h>
  32 #include <linux/trace_events.h>
  33 #include <linux/slab.h>
  34 #include <linux/amd-iommu.h>
  35 #include <linux/hashtable.h>
  36 #include <linux/frame.h>
  37 #include <linux/psp-sev.h>
  38 #include <linux/file.h>
  39 #include <linux/pagemap.h>
  40 #include <linux/swap.h>
  41 #include <linux/rwsem.h>
  42
  43 #include <asm/apic.h>
  44 #include <asm/perf_event.h>
  45 #include <asm/tlbflush.h>
  46 #include <asm/desc.h>
  47 #include <asm/debugreg.h>
  48 #include <asm/kvm_para.h>
  49 #include <asm/irq_remapping.h>
  50 #include <asm/spec-ctrl.h>
  51
  52 #include <asm/virtext.h>
  53 #include "trace.h"
  54
  55 #define __ex(x) __kvm_handle_fault_on_reboot(x)
  56
  57 MODULE_AUTHOR("Qumranet");
  58 MODULE_LICENSE("GPL");
  59
  60 static const struct x86_cpu_id svm_cpu_id[] = {
  61         X86_FEATURE_MATCH(X86_FEATURE_SVM),
  62         {}
  63 };
  64 MODULE_DEVICE_TABLE(x86cpu, svm_cpu_id);
  65
  66 #define IOPM_ALLOC_ORDER 2
  67 #define MSRPM_ALLOC_ORDER 1
  68
  69 #define SEG_TYPE_LDT 2
  70 #define SEG_TYPE_BUSY_TSS16 3
  71
  72 #define SVM_FEATURE_LBRV           (1 <<  1)
  73 #define SVM_FEATURE_SVML           (1 <<  2)
  74 #define SVM_FEATURE_TSC_RATE       (1 <<  4)
  75 #define SVM_FEATURE_VMCB_CLEAN     (1 <<  5)
  76 #define SVM_FEATURE_FLUSH_ASID     (1 <<  6)
  77 #define SVM_FEATURE_DECODE_ASSIST  (1 <<  7)
  78 #define SVM_FEATURE_PAUSE_FILTER   (1 << 10)
  79
  80 #define SVM_AVIC_DOORBELL       0xc001011b
  81
  82 #define NESTED_EXIT_HOST        0       /* Exit handled on host level */
  83 #define NESTED_EXIT_DONE        1       /* Exit caused nested vmexit  */
  84 #define NESTED_EXIT_CONTINUE    2       /* Further checks needed      */
  85
  86 #define DEBUGCTL_RESERVED_BITS (~(0x3fULL))
  87
  88 #define TSC_RATIO_RSVD          0xffffff0000000000ULL
  89 #define TSC_RATIO_MIN           0x0000000000000001ULL
  90 #define TSC_RATIO_MAX           0x000000ffffffffffULL
  91
  92 #define AVIC_HPA_MASK   ~((0xFFFULL << 52) | 0xFFF)
  93
  94 /*
  95  * 0xff is broadcast, so the max index allowed for physical APIC ID
  96  * table is 0xfe.  APIC IDs above 0xff are reserved.
  97  */
  98 #define AVIC_MAX_PHYSICAL_ID_COUNT      255
  99
 100 #define AVIC_UNACCEL_ACCESS_WRITE_MASK          1
 101 #define AVIC_UNACCEL_ACCESS_OFFSET_MASK         0xFF0
 102 #define AVIC_UNACCEL_ACCESS_VECTOR_MASK         0xFFFFFFFF
 103
 104 /* AVIC GATAG is encoded using VM and VCPU IDs */
 105 #define AVIC_VCPU_ID_BITS               8
 106 #define AVIC_VCPU_ID_MASK               ((1 << AVIC_VCPU_ID_BITS) - 1)
 107
 108 #define AVIC_VM_ID_BITS                 24
 109 #define AVIC_VM_ID_NR                   (1 << AVIC_VM_ID_BITS)
 110 #define AVIC_VM_ID_MASK                 ((1 << AVIC_VM_ID_BITS) - 1)
 111
 112 #define AVIC_GATAG(x, y)                (((x & AVIC_VM_ID_MASK) << AVIC_VCPU_ID_BITS) | \
 113                                                 (y & AVIC_VCPU_ID_MASK))
 114 #define AVIC_GATAG_TO_VMID(x)           ((x >> AVIC_VCPU_ID_BITS) & AVIC_VM_ID_MASK)
 115 #define AVIC_GATAG_TO_VCPUID(x)         (x & AVIC_VCPU_ID_MASK)
 116
 117 static bool erratum_383_found __read_mostly;
 118
 119 static const u32 host_save_user_msrs[] = {
 120 #ifdef CONFIG_X86_64
 121         MSR_STAR, MSR_LSTAR, MSR_CSTAR, MSR_SYSCALL_MASK, MSR_KERNEL_GS_BASE,
 122         MSR_FS_BASE,
 123 #endif
 124         MSR_IA32_SYSENTER_CS, MSR_IA32_SYSENTER_ESP, MSR_IA32_SYSENTER_EIP,
 125         MSR_TSC_AUX,
 126 };
 127
 128 #define NR_HOST_SAVE_USER_MSRS ARRAY_SIZE(host_save_user_msrs)
 129
 130 struct kvm_sev_info {
 131         bool active;            /* SEV enabled guest */
 132         unsigned int asid;      /* ASID used for this guest */
 133         unsigned int handle;    /* SEV firmware handle */
 134         int fd;                 /* SEV device fd */
 135         unsigned long pages_locked; /* Number of pages locked */
 136         struct list_head regions_list;  /* List of registered regions */
 137 };
 138
 139 struct kvm_svm {
 140         struct kvm kvm;
 141
 142         /* Struct members for AVIC */
 143         u32 avic_vm_id;
 144         struct page *avic_logical_id_table_page;
 145         struct page *avic_physical_id_table_page;
 146         struct hlist_node hnode;
 147
 148         struct kvm_sev_info sev_info;
 149 };
 150
 151 struct kvm_vcpu;
 152
 153 struct nested_state {
 154         struct vmcb *hsave;
 155         u64 hsave_msr;
 156         u64 vm_cr_msr;
 157         u64 vmcb;
 158
 159         /* These are the merged vectors */
 160         u32 *msrpm;
 161
 162         /* gpa pointers to the real vectors */
 163         u64 vmcb_msrpm;
 164         u64 vmcb_iopm;
 165
 166         /* A VMEXIT is required but not yet emulated */
 167         bool exit_required;
 168
 169         /* cache for intercepts of the guest */
 170         u32 intercept_cr;
 171         u32 intercept_dr;
 172         u32 intercept_exceptions;
 173         u64 intercept;
 174
 175         /* Nested Paging related state */
 176         u64 nested_cr3;
 177 };
 178
 179 #define MSRPM_OFFSETS   16
 180 static u32 msrpm_offsets[MSRPM_OFFSETS] __read_mostly;
 181
 182 /*
 183  * Set osvw_len to higher value when updated Revision Guides
 184  * are published and we know what the new status bits are
 185  */
 186 static uint64_t osvw_len = 4, osvw_status;
 187
 188 struct vcpu_svm {
 189         struct kvm_vcpu vcpu;
 190         struct vmcb *vmcb;
 191         unsigned long vmcb_pa;
 192         struct svm_cpu_data *svm_data;
 193         uint64_t asid_generation;
 194         uint64_t sysenter_esp;
 195         uint64_t sysenter_eip;
 196         uint64_t tsc_aux;
 197
 198         u64 msr_decfg;
 199
 200         u64 next_rip;
 201
 202         u64 host_user_msrs[NR_HOST_SAVE_USER_MSRS];
 203         struct {
 204                 u16 fs;
 205                 u16 gs;
 206                 u16 ldt;
 207                 u64 gs_base;
 208         } host;
 209
 210         u64 spec_ctrl;
 211         /*
 212          * Contains guest-controlled bits of VIRT_SPEC_CTRL, which will be
 213          * translated into the appropriate L2_CFG bits on the host to
 214          * perform speculative control.
 215          */
 216         u64 virt_spec_ctrl;
 217
 218         u32 *msrpm;
 219
 220         ulong nmi_iret_rip;
 221
 222         struct nested_state nested;
 223
 224         bool nmi_singlestep;
 225         u64 nmi_singlestep_guest_rflags;
 226
 227         unsigned int3_injected;
 228         unsigned long int3_rip;
 229
 230         /* cached guest cpuid flags for faster access */
 231         bool nrips_enabled      : 1;
 232
 233         u32 ldr_reg;
 234         u32 dfr_reg;
 235         struct page *avic_backing_page;
 236         u64 *avic_physical_id_cache;
 237         bool avic_is_running;
 238
 239         /*
 240          * Per-vcpu list of struct amd_svm_iommu_ir:
 241          * This is used mainly to store interrupt remapping information used
 242          * when update the vcpu affinity. This avoids the need to scan for
 243          * IRTE and try to match ga_tag in the IOMMU driver.
 244          */
 245         struct list_head ir_list;
 246         spinlock_t ir_list_lock;
 247
 248         /* which host CPU was used for running this vcpu */
 249         unsigned int last_cpu;
 250 };
 251
 252 /*
 253  * This is a wrapper of struct amd_iommu_ir_data.
 254  */
 255 struct amd_svm_iommu_ir {
 256         struct list_head node;  /* Used by SVM for per-vcpu ir_list */
 257         void *data;             /* Storing pointer to struct amd_ir_data */
 258 };
 259
 260 #define AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK    (0xFF)
 261 #define AVIC_LOGICAL_ID_ENTRY_VALID_BIT                 31
 262 #define AVIC_LOGICAL_ID_ENTRY_VALID_MASK                (1 << 31)
 263
 264 #define AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK    (0xFFULL)
 265 #define AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK        (0xFFFFFFFFFFULL << 12)
 266 #define AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK          (1ULL << 62)
 267 #define AVIC_PHYSICAL_ID_ENTRY_VALID_MASK               (1ULL << 63)
 268
 269 static DEFINE_PER_CPU(u64, current_tsc_ratio);
 270 #define TSC_RATIO_DEFAULT       0x0100000000ULL
 271
 272 #define MSR_INVALID                     0xffffffffU
 273
 274 static const struct svm_direct_access_msrs {
 275         u32 index;   /* Index of the MSR */
 276         bool always; /* True if intercept is always on */
 277 } direct_access_msrs[] = {
 278         { .index = MSR_STAR,                            .always = true  },
 279         { .index = MSR_IA32_SYSENTER_CS,                .always = true  },
 280 #ifdef CONFIG_X86_64
 281         { .index = MSR_GS_BASE,                         .always = true  },
 282         { .index = MSR_FS_BASE,                         .always = true  },
 283         { .index = MSR_KERNEL_GS_BASE,                  .always = true  },
 284         { .index = MSR_LSTAR,                           .always = true  },
 285         { .index = MSR_CSTAR,                           .always = true  },
 286         { .index = MSR_SYSCALL_MASK,                    .always = true  },
 287 #endif
 288         { .index = MSR_IA32_SPEC_CTRL,                  .always = false },
 289         { .index = MSR_IA32_PRED_CMD,                   .always = false },
 290         { .index = MSR_IA32_LASTBRANCHFROMIP,           .always = false },
 291         { .index = MSR_IA32_LASTBRANCHTOIP,             .always = false },
 292         { .index = MSR_IA32_LASTINTFROMIP,              .always = false },
 293         { .index = MSR_IA32_LASTINTTOIP,                .always = false },
 294         { .index = MSR_INVALID,                         .always = false },
 295 };
 296
 297 /* enable NPT for AMD64 and X86 with PAE */
 298 #if defined(CONFIG_X86_64) || defined(CONFIG_X86_PAE)
 299 static bool npt_enabled = true;
 300 #else
 301 static bool npt_enabled;
 302 #endif
 303
 304 /*
 305  * These 2 parameters are used to config the controls for Pause-Loop Exiting:
 306  * pause_filter_count: On processors that support Pause filtering(indicated
 307  *      by CPUID Fn8000_000A_EDX), the VMCB provides a 16 bit pause filter
 308  *      count value. On VMRUN this value is loaded into an internal counter.
 309  *      Each time a pause instruction is executed, this counter is decremented
 310  *      until it reaches zero at which time a #VMEXIT is generated if pause
 311  *      intercept is enabled. Refer to  AMD APM Vol 2 Section 15.14.4 Pause
 312  *      Intercept Filtering for more details.
 313  *      This also indicate if ple logic enabled.
 314  *
 315  * pause_filter_thresh: In addition, some processor families support advanced
 316  *      pause filtering (indicated by CPUID Fn8000_000A_EDX) upper bound on
 317  *      the amount of time a guest is allowed to execute in a pause loop.
 318  *      In this mode, a 16-bit pause filter threshold field is added in the
 319  *      VMCB. The threshold value is a cycle count that is used to reset the
 320  *      pause counter. As with simple pause filtering, VMRUN loads the pause
 321  *      count value from VMCB into an internal counter. Then, on each pause
 322  *      instruction the hardware checks the elapsed number of cycles since
 323  *      the most recent pause instruction against the pause filter threshold.
 324  *      If the elapsed cycle count is greater than the pause filter threshold,
 325  *      then the internal pause count is reloaded from the VMCB and execution
 326  *      continues. If the elapsed cycle count is less than the pause filter
 327  *      threshold, then the internal pause count is decremented. If the count
 328  *      value is less than zero and PAUSE intercept is enabled, a #VMEXIT is
 329  *      triggered. If advanced pause filtering is supported and pause filter
 330  *      threshold field is set to zero, the filter will operate in the simpler,
 331  *      count only mode.
 332  */
 333
 334 static unsigned short pause_filter_thresh = KVM_DEFAULT_PLE_GAP;
 335 module_param(pause_filter_thresh, ushort, 0444);
 336
 337 static unsigned short pause_filter_count = KVM_SVM_DEFAULT_PLE_WINDOW;
 338 module_param(pause_filter_count, ushort, 0444);
 339
 340 /* Default doubles per-vcpu window every exit. */
 341 static unsigned short pause_filter_count_grow = KVM_DEFAULT_PLE_WINDOW_GROW;
 342 module_param(pause_filter_count_grow, ushort, 0444);
 343
 344 /* Default resets per-vcpu window every exit to pause_filter_count. */
 345 static unsigned short pause_filter_count_shrink = KVM_DEFAULT_PLE_WINDOW_SHRINK;
 346 module_param(pause_filter_count_shrink, ushort, 0444);
 347
 348 /* Default is to compute the maximum so we can never overflow. */
 349 static unsigned short pause_filter_count_max = KVM_SVM_DEFAULT_PLE_WINDOW_MAX;
 350 module_param(pause_filter_count_max, ushort, 0444);
 351
 352 /* allow nested paging (virtualized MMU) for all guests */
 353 static int npt = true;
 354 module_param(npt, int, S_IRUGO);
 355
 356 /* allow nested virtualization in KVM/SVM */
 357 static int nested = true;
 358 module_param(nested, int, S_IRUGO);
 359
 360 /* enable / disable AVIC */
 361 static int avic;
 362 #ifdef CONFIG_X86_LOCAL_APIC
 363 module_param(avic, int, S_IRUGO);
 364 #endif
 365
 366 /* enable/disable Next RIP Save */
 367 static int nrips = true;
 368 module_param(nrips, int, 0444);
 369
 370 /* enable/disable Virtual VMLOAD VMSAVE */
 371 static int vls = true;
 372 module_param(vls, int, 0444);
 373
 374 /* enable/disable Virtual GIF */
 375 static int vgif = true;
 376 module_param(vgif, int, 0444);
 377
 378 /* enable/disable SEV support */
 379 static int sev = IS_ENABLED(CONFIG_AMD_MEM_ENCRYPT_ACTIVE_BY_DEFAULT);
 380 module_param(sev, int, 0444);
 381
 382 static bool __read_mostly dump_invalid_vmcb = 0;
 383 module_param(dump_invalid_vmcb, bool, 0644);
 384
 385 static u8 rsm_ins_bytes[] = "\x0f\xaa";
 386
 387 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0);
 388 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa);
 389 static void svm_complete_interrupts(struct vcpu_svm *svm);
 390
 391 static int nested_svm_exit_handled(struct vcpu_svm *svm);
 392 static int nested_svm_intercept(struct vcpu_svm *svm);
 393 static int nested_svm_vmexit(struct vcpu_svm *svm);
 394 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
 395                                       bool has_error_code, u32 error_code);
 396
 397 enum {
 398         VMCB_INTERCEPTS, /* Intercept vectors, TSC offset,
 399                             pause filter count */
 400         VMCB_PERM_MAP,   /* IOPM Base and MSRPM Base */
 401         VMCB_ASID,       /* ASID */
 402         VMCB_INTR,       /* int_ctl, int_vector */
 403         VMCB_NPT,        /* npt_en, nCR3, gPAT */
 404         VMCB_CR,         /* CR0, CR3, CR4, EFER */
 405         VMCB_DR,         /* DR6, DR7 */
 406         VMCB_DT,         /* GDT, IDT */
 407         VMCB_SEG,        /* CS, DS, SS, ES, CPL */
 408         VMCB_CR2,        /* CR2 only */
 409         VMCB_LBR,        /* DBGCTL, BR_FROM, BR_TO, LAST_EX_FROM, LAST_EX_TO */
 410         VMCB_AVIC,       /* AVIC APIC_BAR, AVIC APIC_BACKING_PAGE,
 411                           * AVIC PHYSICAL_TABLE pointer,
 412                           * AVIC LOGICAL_TABLE pointer
 413                           */
 414         VMCB_DIRTY_MAX,
 415 };
 416
 417 /* TPR and CR2 are always written before VMRUN */
 418 #define VMCB_ALWAYS_DIRTY_MASK  ((1U << VMCB_INTR) | (1U << VMCB_CR2))
 419
 420 #define VMCB_AVIC_APIC_BAR_MASK         0xFFFFFFFFFF000ULL
 421
 422 static int sev_flush_asids(void);
 423 static DECLARE_RWSEM(sev_deactivate_lock);
 424 static DEFINE_MUTEX(sev_bitmap_lock);
 425 static unsigned int max_sev_asid;
 426 static unsigned int min_sev_asid;
 427 static unsigned long *sev_asid_bitmap;
 428 static unsigned long *sev_reclaim_asid_bitmap;
 429 #define __sme_page_pa(x) __sme_set(page_to_pfn(x) << PAGE_SHIFT)
 430
 431 struct enc_region {
 432         struct list_head list;
 433         unsigned long npages;
 434         struct page **pages;
 435         unsigned long uaddr;
 436         unsigned long size;
 437 };
 438
 439
 440 static inline struct kvm_svm *to_kvm_svm(struct kvm *kvm)
 441 {
 442         return container_of(kvm, struct kvm_svm, kvm);
 443 }
 444
 445 static inline bool svm_sev_enabled(void)
 446 {
 447         return IS_ENABLED(CONFIG_KVM_AMD_SEV) ? max_sev_asid : 0;
 448 }
 449
 450 static inline bool sev_guest(struct kvm *kvm)
 451 {
 452 #ifdef CONFIG_KVM_AMD_SEV
 453         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 454
 455         return sev->active;
 456 #else
 457         return false;
 458 #endif
 459 }
 460
 461 static inline int sev_get_asid(struct kvm *kvm)
 462 {
 463         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
 464
 465         return sev->asid;
 466 }
 467
 468 static inline void mark_all_dirty(struct vmcb *vmcb)
 469 {
 470         vmcb->control.clean = 0;
 471 }
 472
 473 static inline void mark_all_clean(struct vmcb *vmcb)
 474 {
 475         vmcb->control.clean = ((1 << VMCB_DIRTY_MAX) - 1)
 476                                & ~VMCB_ALWAYS_DIRTY_MASK;
 477 }
 478
 479 static inline void mark_dirty(struct vmcb *vmcb, int bit)
 480 {
 481         vmcb->control.clean &= ~(1 << bit);
 482 }
 483
 484 static inline struct vcpu_svm *to_svm(struct kvm_vcpu *vcpu)
 485 {
 486         return container_of(vcpu, struct vcpu_svm, vcpu);
 487 }
 488
 489 static inline void avic_update_vapic_bar(struct vcpu_svm *svm, u64 data)
 490 {
 491         svm->vmcb->control.avic_vapic_bar = data & VMCB_AVIC_APIC_BAR_MASK;
 492         mark_dirty(svm->vmcb, VMCB_AVIC);
 493 }
 494
 495 static inline bool avic_vcpu_is_running(struct kvm_vcpu *vcpu)
 496 {
 497         struct vcpu_svm *svm = to_svm(vcpu);
 498         u64 *entry = svm->avic_physical_id_cache;
 499
 500         if (!entry)
 501                 return false;
 502
 503         return (READ_ONCE(*entry) & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
 504 }
 505
 506 static void recalc_intercepts(struct vcpu_svm *svm)
 507 {
 508         struct vmcb_control_area *c, *h;
 509         struct nested_state *g;
 510
 511         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
 512
 513         if (!is_guest_mode(&svm->vcpu))
 514                 return;
 515
 516         c = &svm->vmcb->control;
 517         h = &svm->nested.hsave->control;
 518         g = &svm->nested;
 519
 520         c->intercept_cr = h->intercept_cr | g->intercept_cr;
 521         c->intercept_dr = h->intercept_dr | g->intercept_dr;
 522         c->intercept_exceptions = h->intercept_exceptions | g->intercept_exceptions;
 523         c->intercept = h->intercept | g->intercept;
 524 }
 525
 526 static inline struct vmcb *get_host_vmcb(struct vcpu_svm *svm)
 527 {
 528         if (is_guest_mode(&svm->vcpu))
 529                 return svm->nested.hsave;
 530         else
 531                 return svm->vmcb;
 532 }
 533
 534 static inline void set_cr_intercept(struct vcpu_svm *svm, int bit)
 535 {
 536         struct vmcb *vmcb = get_host_vmcb(svm);
 537
 538         vmcb->control.intercept_cr |= (1U << bit);
 539
 540         recalc_intercepts(svm);
 541 }
 542
 543 static inline void clr_cr_intercept(struct vcpu_svm *svm, int bit)
 544 {
 545         struct vmcb *vmcb = get_host_vmcb(svm);
 546
 547         vmcb->control.intercept_cr &= ~(1U << bit);
 548
 549         recalc_intercepts(svm);
 550 }
 551
 552 static inline bool is_cr_intercept(struct vcpu_svm *svm, int bit)
 553 {
 554         struct vmcb *vmcb = get_host_vmcb(svm);
 555
 556         return vmcb->control.intercept_cr & (1U << bit);
 557 }
 558
 559 static inline void set_dr_intercepts(struct vcpu_svm *svm)
 560 {
 561         struct vmcb *vmcb = get_host_vmcb(svm);
 562
 563         vmcb->control.intercept_dr = (1 << INTERCEPT_DR0_READ)
 564                 | (1 << INTERCEPT_DR1_READ)
 565                 | (1 << INTERCEPT_DR2_READ)
 566                 | (1 << INTERCEPT_DR3_READ)
 567                 | (1 << INTERCEPT_DR4_READ)
 568                 | (1 << INTERCEPT_DR5_READ)
 569                 | (1 << INTERCEPT_DR6_READ)
 570                 | (1 << INTERCEPT_DR7_READ)
 571                 | (1 << INTERCEPT_DR0_WRITE)
 572                 | (1 << INTERCEPT_DR1_WRITE)
 573                 | (1 << INTERCEPT_DR2_WRITE)
 574                 | (1 << INTERCEPT_DR3_WRITE)
 575                 | (1 << INTERCEPT_DR4_WRITE)
 576                 | (1 << INTERCEPT_DR5_WRITE)
 577                 | (1 << INTERCEPT_DR6_WRITE)
 578                 | (1 << INTERCEPT_DR7_WRITE);
 579
 580         recalc_intercepts(svm);
 581 }
 582
 583 static inline void clr_dr_intercepts(struct vcpu_svm *svm)
 584 {
 585         struct vmcb *vmcb = get_host_vmcb(svm);
 586
 587         vmcb->control.intercept_dr = 0;
 588
 589         recalc_intercepts(svm);
 590 }
 591
 592 static inline void set_exception_intercept(struct vcpu_svm *svm, int bit)
 593 {
 594         struct vmcb *vmcb = get_host_vmcb(svm);
 595
 596         vmcb->control.intercept_exceptions |= (1U << bit);
 597
 598         recalc_intercepts(svm);
 599 }
 600
 601 static inline void clr_exception_intercept(struct vcpu_svm *svm, int bit)
 602 {
 603         struct vmcb *vmcb = get_host_vmcb(svm);
 604
 605         vmcb->control.intercept_exceptions &= ~(1U << bit);
 606
 607         recalc_intercepts(svm);
 608 }
 609
 610 static inline void set_intercept(struct vcpu_svm *svm, int bit)
 611 {
 612         struct vmcb *vmcb = get_host_vmcb(svm);
 613
 614         vmcb->control.intercept |= (1ULL << bit);
 615
 616         recalc_intercepts(svm);
 617 }
 618
 619 static inline void clr_intercept(struct vcpu_svm *svm, int bit)
 620 {
 621         struct vmcb *vmcb = get_host_vmcb(svm);
 622
 623         vmcb->control.intercept &= ~(1ULL << bit);
 624
 625         recalc_intercepts(svm);
 626 }
 627
 628 static inline bool vgif_enabled(struct vcpu_svm *svm)
 629 {
 630         return !!(svm->vmcb->control.int_ctl & V_GIF_ENABLE_MASK);
 631 }
 632
 633 static inline void enable_gif(struct vcpu_svm *svm)
 634 {
 635         if (vgif_enabled(svm))
 636                 svm->vmcb->control.int_ctl |= V_GIF_MASK;
 637         else
 638                 svm->vcpu.arch.hflags |= HF_GIF_MASK;
 639 }
 640
 641 static inline void disable_gif(struct vcpu_svm *svm)
 642 {
 643         if (vgif_enabled(svm))
 644                 svm->vmcb->control.int_ctl &= ~V_GIF_MASK;
 645         else
 646                 svm->vcpu.arch.hflags &= ~HF_GIF_MASK;
 647 }
 648
 649 static inline bool gif_set(struct vcpu_svm *svm)
 650 {
 651         if (vgif_enabled(svm))
 652                 return !!(svm->vmcb->control.int_ctl & V_GIF_MASK);
 653         else
 654                 return !!(svm->vcpu.arch.hflags & HF_GIF_MASK);
 655 }
 656
 657 static unsigned long iopm_base;
 658
 659 struct kvm_ldttss_desc {
 660         u16 limit0;
 661         u16 base0;
 662         unsigned base1:8, type:5, dpl:2, p:1;
 663         unsigned limit1:4, zero0:3, g:1, base2:8;
 664         u32 base3;
 665         u32 zero1;
 666 } __attribute__((packed));
 667
 668 struct svm_cpu_data {
 669         int cpu;
 670
 671         u64 asid_generation;
 672         u32 max_asid;
 673         u32 next_asid;
 674         u32 min_asid;
 675         struct kvm_ldttss_desc *tss_desc;
 676
 677         struct page *save_area;
 678         struct vmcb *current_vmcb;
 679
 680         /* index = sev_asid, value = vmcb pointer */
 681         struct vmcb **sev_vmcbs;
 682 };
 683
 684 static DEFINE_PER_CPU(struct svm_cpu_data *, svm_data);
 685
 686 static const u32 msrpm_ranges[] = {0, 0xc0000000, 0xc0010000};
 687
 688 #define NUM_MSR_MAPS ARRAY_SIZE(msrpm_ranges)
 689 #define MSRS_RANGE_SIZE 2048
 690 #define MSRS_IN_RANGE (MSRS_RANGE_SIZE * 8 / 2)
 691
 692 static u32 svm_msrpm_offset(u32 msr)
 693 {
 694         u32 offset;
 695         int i;
 696
 697         for (i = 0; i < NUM_MSR_MAPS; i++) {
 698                 if (msr < msrpm_ranges[i] ||
 699                     msr >= msrpm_ranges[i] + MSRS_IN_RANGE)
 700                         continue;
 701
 702                 offset  = (msr - msrpm_ranges[i]) / 4; /* 4 msrs per u8 */
 703                 offset += (i * MSRS_RANGE_SIZE);       /* add range offset */
 704
 705                 /* Now we have the u8 offset - but need the u32 offset */
 706                 return offset / 4;
 707         }
 708
 709         /* MSR not in any range */
 710         return MSR_INVALID;
 711 }
 712
 713 #define MAX_INST_SIZE 15
 714
 715 static inline void clgi(void)
 716 {
 717         asm volatile (__ex("clgi"));
 718 }
 719
 720 static inline void stgi(void)
 721 {
 722         asm volatile (__ex("stgi"));
 723 }
 724
 725 static inline void invlpga(unsigned long addr, u32 asid)
 726 {
 727         asm volatile (__ex("invlpga %1, %0") : : "c"(asid), "a"(addr));
 728 }
 729
 730 static int get_npt_level(struct kvm_vcpu *vcpu)
 731 {
 732 #ifdef CONFIG_X86_64
 733         return PT64_ROOT_4LEVEL;
 734 #else
 735         return PT32E_ROOT_LEVEL;
 736 #endif
 737 }
 738
 739 static void svm_set_efer(struct kvm_vcpu *vcpu, u64 efer)
 740 {
 741         vcpu->arch.efer = efer;
 742
 743         if (!npt_enabled) {
 744                 /* Shadow paging assumes NX to be available.  */
 745                 efer |= EFER_NX;
 746
 747                 if (!(efer & EFER_LMA))
 748                         efer &= ~EFER_LME;
 749         }
 750
 751         to_svm(vcpu)->vmcb->save.efer = efer | EFER_SVME;
 752         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
 753 }
 754
 755 static int is_external_interrupt(u32 info)
 756 {
 757         info &= SVM_EVTINJ_TYPE_MASK | SVM_EVTINJ_VALID;
 758         return info == (SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR);
 759 }
 760
 761 static u32 svm_get_interrupt_shadow(struct kvm_vcpu *vcpu)
 762 {
 763         struct vcpu_svm *svm = to_svm(vcpu);
 764         u32 ret = 0;
 765
 766         if (svm->vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK)
 767                 ret = KVM_X86_SHADOW_INT_STI | KVM_X86_SHADOW_INT_MOV_SS;
 768         return ret;
 769 }
 770
 771 static void svm_set_interrupt_shadow(struct kvm_vcpu *vcpu, int mask)
 772 {
 773         struct vcpu_svm *svm = to_svm(vcpu);
 774
 775         if (mask == 0)
 776                 svm->vmcb->control.int_state &= ~SVM_INTERRUPT_SHADOW_MASK;
 777         else
 778                 svm->vmcb->control.int_state |= SVM_INTERRUPT_SHADOW_MASK;
 779
 780 }
 781
 782 static int skip_emulated_instruction(struct kvm_vcpu *vcpu)
 783 {
 784         struct vcpu_svm *svm = to_svm(vcpu);
 785
 786         if (nrips && svm->vmcb->control.next_rip != 0) {
 787                 WARN_ON_ONCE(!static_cpu_has(X86_FEATURE_NRIPS));
 788                 svm->next_rip = svm->vmcb->control.next_rip;
 789         }
 790
 791         if (!svm->next_rip) {
 792                 if (!kvm_emulate_instruction(vcpu, EMULTYPE_SKIP))
 793                         return 0;
 794         } else {
 795                 if (svm->next_rip - kvm_rip_read(vcpu) > MAX_INST_SIZE)
 796                         pr_err("%s: ip 0x%lx next 0x%llx\n",
 797                                __func__, kvm_rip_read(vcpu), svm->next_rip);
 798                 kvm_rip_write(vcpu, svm->next_rip);
 799         }
 800         svm_set_interrupt_shadow(vcpu, 0);
 801
 802         return 1;
 803 }
 804
 805 static void svm_queue_exception(struct kvm_vcpu *vcpu)
 806 {
 807         struct vcpu_svm *svm = to_svm(vcpu);
 808         unsigned nr = vcpu->arch.exception.nr;
 809         bool has_error_code = vcpu->arch.exception.has_error_code;
 810         bool reinject = vcpu->arch.exception.injected;
 811         u32 error_code = vcpu->arch.exception.error_code;
 812
 813         /*
 814          * If we are within a nested VM we'd better #VMEXIT and let the guest
 815          * handle the exception
 816          */
 817         if (!reinject &&
 818             nested_svm_check_exception(svm, nr, has_error_code, error_code))
 819                 return;
 820
 821         kvm_deliver_exception_payload(&svm->vcpu);
 822
 823         if (nr == BP_VECTOR && !nrips) {
 824                 unsigned long rip, old_rip = kvm_rip_read(&svm->vcpu);
 825
 826                 /*
 827                  * For guest debugging where we have to reinject #BP if some
 828                  * INT3 is guest-owned:
 829                  * Emulate nRIP by moving RIP forward. Will fail if injection
 830                  * raises a fault that is not intercepted. Still better than
 831                  * failing in all cases.
 832                  */
 833                 (void)skip_emulated_instruction(&svm->vcpu);
 834                 rip = kvm_rip_read(&svm->vcpu);
 835                 svm->int3_rip = rip + svm->vmcb->save.cs.base;
 836                 svm->int3_injected = rip - old_rip;
 837         }
 838
 839         svm->vmcb->control.event_inj = nr
 840                 | SVM_EVTINJ_VALID
 841                 | (has_error_code ? SVM_EVTINJ_VALID_ERR : 0)
 842                 | SVM_EVTINJ_TYPE_EXEPT;
 843         svm->vmcb->control.event_inj_err = error_code;
 844 }
 845
 846 static void svm_init_erratum_383(void)
 847 {
 848         u32 low, high;
 849         int err;
 850         u64 val;
 851
 852         if (!static_cpu_has_bug(X86_BUG_AMD_TLB_MMATCH))
 853                 return;
 854
 855         /* Use _safe variants to not break nested virtualization */
 856         val = native_read_msr_safe(MSR_AMD64_DC_CFG, &err);
 857         if (err)
 858                 return;
 859
 860         val |= (1ULL << 47);
 861
 862         low  = lower_32_bits(val);
 863         high = upper_32_bits(val);
 864
 865         native_write_msr_safe(MSR_AMD64_DC_CFG, low, high);
 866
 867         erratum_383_found = true;
 868 }
 869
 870 static void svm_init_osvw(struct kvm_vcpu *vcpu)
 871 {
 872         /*
 873          * Guests should see errata 400 and 415 as fixed (assuming that
 874          * HLT and IO instructions are intercepted).
 875          */
 876         vcpu->arch.osvw.length = (osvw_len >= 3) ? (osvw_len) : 3;
 877         vcpu->arch.osvw.status = osvw_status & ~(6ULL);
 878
 879         /*
 880          * By increasing VCPU's osvw.length to 3 we are telling the guest that
 881          * all osvw.status bits inside that length, including bit 0 (which is
 882          * reserved for erratum 298), are valid. However, if host processor's
 883          * osvw_len is 0 then osvw_status[0] carries no information. We need to
 884          * be conservative here and therefore we tell the guest that erratum 298
 885          * is present (because we really don't know).
 886          */
 887         if (osvw_len == 0 && boot_cpu_data.x86 == 0x10)
 888                 vcpu->arch.osvw.status |= 1;
 889 }
 890
 891 static int has_svm(void)
 892 {
 893         const char *msg;
 894
 895         if (!cpu_has_svm(&msg)) {
 896                 printk(KERN_INFO "has_svm: %s\n", msg);
 897                 return 0;
 898         }
 899
 900         return 1;
 901 }
 902
 903 static void svm_hardware_disable(void)
 904 {
 905         /* Make sure we clean up behind us */
 906         if (static_cpu_has(X86_FEATURE_TSCRATEMSR))
 907                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 908
 909         cpu_svm_disable();
 910
 911         amd_pmu_disable_virt();
 912 }
 913
 914 static int svm_hardware_enable(void)
 915 {
 916
 917         struct svm_cpu_data *sd;
 918         uint64_t efer;
 919         struct desc_struct *gdt;
 920         int me = raw_smp_processor_id();
 921
 922         rdmsrl(MSR_EFER, efer);
 923         if (efer & EFER_SVME)
 924                 return -EBUSY;
 925
 926         if (!has_svm()) {
 927                 pr_err("%s: err EOPNOTSUPP on %d\n", __func__, me);
 928                 return -EINVAL;
 929         }
 930         sd = per_cpu(svm_data, me);
 931         if (!sd) {
 932                 pr_err("%s: svm_data is NULL on %d\n", __func__, me);
 933                 return -EINVAL;
 934         }
 935
 936         sd->asid_generation = 1;
 937         sd->max_asid = cpuid_ebx(SVM_CPUID_FUNC) - 1;
 938         sd->next_asid = sd->max_asid + 1;
 939         sd->min_asid = max_sev_asid + 1;
 940
 941         gdt = get_current_gdt_rw();
 942         sd->tss_desc = (struct kvm_ldttss_desc *)(gdt + GDT_ENTRY_TSS);
 943
 944         wrmsrl(MSR_EFER, efer | EFER_SVME);
 945
 946         wrmsrl(MSR_VM_HSAVE_PA, page_to_pfn(sd->save_area) << PAGE_SHIFT);
 947
 948         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
 949                 wrmsrl(MSR_AMD64_TSC_RATIO, TSC_RATIO_DEFAULT);
 950                 __this_cpu_write(current_tsc_ratio, TSC_RATIO_DEFAULT);
 951         }
 952
 953
 954         /*
 955          * Get OSVW bits.
 956          *
 957          * Note that it is possible to have a system with mixed processor
 958          * revisions and therefore different OSVW bits. If bits are not the same
 959          * on different processors then choose the worst case (i.e. if erratum
 960          * is present on one processor and not on another then assume that the
 961          * erratum is present everywhere).
 962          */
 963         if (cpu_has(&boot_cpu_data, X86_FEATURE_OSVW)) {
 964                 uint64_t len, status = 0;
 965                 int err;
 966
 967                 len = native_read_msr_safe(MSR_AMD64_OSVW_ID_LENGTH, &err);
 968                 if (!err)
 969                         status = native_read_msr_safe(MSR_AMD64_OSVW_STATUS,
 970                                                       &err);
 971
 972                 if (err)
 973                         osvw_status = osvw_len = 0;
 974                 else {
 975                         if (len < osvw_len)
 976                                 osvw_len = len;
 977                         osvw_status |= status;
 978                         osvw_status &= (1ULL << osvw_len) - 1;
 979                 }
 980         } else
 981                 osvw_status = osvw_len = 0;
 982
 983         svm_init_erratum_383();
 984
 985         amd_pmu_enable_virt();
 986
 987         return 0;
 988 }
 989
 990 static void svm_cpu_uninit(int cpu)
 991 {
 992         struct svm_cpu_data *sd = per_cpu(svm_data, raw_smp_processor_id());
 993
 994         if (!sd)
 995                 return;
 996
 997         per_cpu(svm_data, raw_smp_processor_id()) = NULL;
 998         kfree(sd->sev_vmcbs);
 999         __free_page(sd->save_area);
1000         kfree(sd);
1001 }
1002
1003 static int svm_cpu_init(int cpu)
1004 {
1005         struct svm_cpu_data *sd;
1006         int r;
1007
1008         sd = kzalloc(sizeof(struct svm_cpu_data), GFP_KERNEL);
1009         if (!sd)
1010                 return -ENOMEM;
1011         sd->cpu = cpu;
1012         r = -ENOMEM;
1013         sd->save_area = alloc_page(GFP_KERNEL);
1014         if (!sd->save_area)
1015                 goto err_1;
1016
1017         if (svm_sev_enabled()) {
1018                 r = -ENOMEM;
1019                 sd->sev_vmcbs = kmalloc_array(max_sev_asid + 1,
1020                                               sizeof(void *),
1021                                               GFP_KERNEL);
1022                 if (!sd->sev_vmcbs)
1023                         goto err_1;
1024         }
1025
1026         per_cpu(svm_data, cpu) = sd;
1027
1028         return 0;
1029
1030 err_1:
1031         kfree(sd);
1032         return r;
1033
1034 }
1035
1036 static bool valid_msr_intercept(u32 index)
1037 {
1038         int i;
1039
1040         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++)
1041                 if (direct_access_msrs[i].index == index)
1042                         return true;
1043
1044         return false;
1045 }
1046
1047 static bool msr_write_intercepted(struct kvm_vcpu *vcpu, unsigned msr)
1048 {
1049         u8 bit_write;
1050         unsigned long tmp;
1051         u32 offset;
1052         u32 *msrpm;
1053
1054         msrpm = is_guest_mode(vcpu) ? to_svm(vcpu)->nested.msrpm:
1055                                       to_svm(vcpu)->msrpm;
1056
1057         offset    = svm_msrpm_offset(msr);
1058         bit_write = 2 * (msr & 0x0f) + 1;
1059         tmp       = msrpm[offset];
1060
1061         BUG_ON(offset == MSR_INVALID);
1062
1063         return !!test_bit(bit_write,  &tmp);
1064 }
1065
1066 static void set_msr_interception(u32 *msrpm, unsigned msr,
1067                                  int read, int write)
1068 {
1069         u8 bit_read, bit_write;
1070         unsigned long tmp;
1071         u32 offset;
1072
1073         /*
1074          * If this warning triggers extend the direct_access_msrs list at the
1075          * beginning of the file
1076          */
1077         WARN_ON(!valid_msr_intercept(msr));
1078
1079         offset    = svm_msrpm_offset(msr);
1080         bit_read  = 2 * (msr & 0x0f);
1081         bit_write = 2 * (msr & 0x0f) + 1;
1082         tmp       = msrpm[offset];
1083
1084         BUG_ON(offset == MSR_INVALID);
1085
1086         read  ? clear_bit(bit_read,  &tmp) : set_bit(bit_read,  &tmp);
1087         write ? clear_bit(bit_write, &tmp) : set_bit(bit_write, &tmp);
1088
1089         msrpm[offset] = tmp;
1090 }
1091
1092 static void svm_vcpu_init_msrpm(u32 *msrpm)
1093 {
1094         int i;
1095
1096         memset(msrpm, 0xff, PAGE_SIZE * (1 << MSRPM_ALLOC_ORDER));
1097
1098         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1099                 if (!direct_access_msrs[i].always)
1100                         continue;
1101
1102                 set_msr_interception(msrpm, direct_access_msrs[i].index, 1, 1);
1103         }
1104 }
1105
1106 static void add_msr_offset(u32 offset)
1107 {
1108         int i;
1109
1110         for (i = 0; i < MSRPM_OFFSETS; ++i) {
1111
1112                 /* Offset already in list? */
1113                 if (msrpm_offsets[i] == offset)
1114                         return;
1115
1116                 /* Slot used by another offset? */
1117                 if (msrpm_offsets[i] != MSR_INVALID)
1118                         continue;
1119
1120                 /* Add offset to list */
1121                 msrpm_offsets[i] = offset;
1122
1123                 return;
1124         }
1125
1126         /*
1127          * If this BUG triggers the msrpm_offsets table has an overflow. Just
1128          * increase MSRPM_OFFSETS in this case.
1129          */
1130         BUG();
1131 }
1132
1133 static void init_msrpm_offsets(void)
1134 {
1135         int i;
1136
1137         memset(msrpm_offsets, 0xff, sizeof(msrpm_offsets));
1138
1139         for (i = 0; direct_access_msrs[i].index != MSR_INVALID; i++) {
1140                 u32 offset;
1141
1142                 offset = svm_msrpm_offset(direct_access_msrs[i].index);
1143                 BUG_ON(offset == MSR_INVALID);
1144
1145                 add_msr_offset(offset);
1146         }
1147 }
1148
1149 static void svm_enable_lbrv(struct vcpu_svm *svm)
1150 {
1151         u32 *msrpm = svm->msrpm;
1152
1153         svm->vmcb->control.virt_ext |= LBR_CTL_ENABLE_MASK;
1154         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 1, 1);
1155         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 1, 1);
1156         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 1, 1);
1157         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 1, 1);
1158 }
1159
1160 static void svm_disable_lbrv(struct vcpu_svm *svm)
1161 {
1162         u32 *msrpm = svm->msrpm;
1163
1164         svm->vmcb->control.virt_ext &= ~LBR_CTL_ENABLE_MASK;
1165         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHFROMIP, 0, 0);
1166         set_msr_interception(msrpm, MSR_IA32_LASTBRANCHTOIP, 0, 0);
1167         set_msr_interception(msrpm, MSR_IA32_LASTINTFROMIP, 0, 0);
1168         set_msr_interception(msrpm, MSR_IA32_LASTINTTOIP, 0, 0);
1169 }
1170
1171 static void disable_nmi_singlestep(struct vcpu_svm *svm)
1172 {
1173         svm->nmi_singlestep = false;
1174
1175         if (!(svm->vcpu.guest_debug & KVM_GUESTDBG_SINGLESTEP)) {
1176                 /* Clear our flags if they were not set by the guest */
1177                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
1178                         svm->vmcb->save.rflags &= ~X86_EFLAGS_TF;
1179                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
1180                         svm->vmcb->save.rflags &= ~X86_EFLAGS_RF;
1181         }
1182 }
1183
1184 /* Note:
1185  * This hash table is used to map VM_ID to a struct kvm_svm,
1186  * when handling AMD IOMMU GALOG notification to schedule in
1187  * a particular vCPU.
1188  */
1189 #define SVM_VM_DATA_HASH_BITS   8
1190 static DEFINE_HASHTABLE(svm_vm_data_hash, SVM_VM_DATA_HASH_BITS);
1191 static u32 next_vm_id = 0;
1192 static bool next_vm_id_wrapped = 0;
1193 static DEFINE_SPINLOCK(svm_vm_data_hash_lock);
1194
1195 /* Note:
1196  * This function is called from IOMMU driver to notify
1197  * SVM to schedule in a particular vCPU of a particular VM.
1198  */
1199 static int avic_ga_log_notifier(u32 ga_tag)
1200 {
1201         unsigned long flags;
1202         struct kvm_svm *kvm_svm;
1203         struct kvm_vcpu *vcpu = NULL;
1204         u32 vm_id = AVIC_GATAG_TO_VMID(ga_tag);
1205         u32 vcpu_id = AVIC_GATAG_TO_VCPUID(ga_tag);
1206
1207         pr_debug("SVM: %s: vm_id=%#x, vcpu_id=%#x\n", __func__, vm_id, vcpu_id);
1208
1209         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1210         hash_for_each_possible(svm_vm_data_hash, kvm_svm, hnode, vm_id) {
1211                 if (kvm_svm->avic_vm_id != vm_id)
1212                         continue;
1213                 vcpu = kvm_get_vcpu_by_id(&kvm_svm->kvm, vcpu_id);
1214                 break;
1215         }
1216         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1217
1218         /* Note:
1219          * At this point, the IOMMU should have already set the pending
1220          * bit in the vAPIC backing page. So, we just need to schedule
1221          * in the vcpu.
1222          */
1223         if (vcpu)
1224                 kvm_vcpu_wake_up(vcpu);
1225
1226         return 0;
1227 }
1228
1229 static __init int sev_hardware_setup(void)
1230 {
1231         struct sev_user_data_status *status;
1232         int rc;
1233
1234         /* Maximum number of encrypted guests supported simultaneously */
1235         max_sev_asid = cpuid_ecx(0x8000001F);
1236
1237         if (!max_sev_asid)
1238                 return 1;
1239
1240         /* Minimum ASID value that should be used for SEV guest */
1241         min_sev_asid = cpuid_edx(0x8000001F);
1242
1243         /* Initialize SEV ASID bitmaps */
1244         sev_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1245         if (!sev_asid_bitmap)
1246                 return 1;
1247
1248         sev_reclaim_asid_bitmap = bitmap_zalloc(max_sev_asid, GFP_KERNEL);
1249         if (!sev_reclaim_asid_bitmap)
1250                 return 1;
1251
1252         status = kmalloc(sizeof(*status), GFP_KERNEL);
1253         if (!status)
1254                 return 1;
1255
1256         /*
1257          * Check SEV platform status.
1258          *
1259          * PLATFORM_STATUS can be called in any state, if we failed to query
1260          * the PLATFORM status then either PSP firmware does not support SEV
1261          * feature or SEV firmware is dead.
1262          */
1263         rc = sev_platform_status(status, NULL);
1264         if (rc)
1265                 goto err;
1266
1267         pr_info("SEV supported\n");
1268
1269 err:
1270         kfree(status);
1271         return rc;
1272 }
1273
1274 static void grow_ple_window(struct kvm_vcpu *vcpu)
1275 {
1276         struct vcpu_svm *svm = to_svm(vcpu);
1277         struct vmcb_control_area *control = &svm->vmcb->control;
1278         int old = control->pause_filter_count;
1279
1280         control->pause_filter_count = __grow_ple_window(old,
1281                                                         pause_filter_count,
1282                                                         pause_filter_count_grow,
1283                                                         pause_filter_count_max);
1284
1285         if (control->pause_filter_count != old) {
1286                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1287                 trace_kvm_ple_window_update(vcpu->vcpu_id,
1288                                             control->pause_filter_count, old);
1289         }
1290 }
1291
1292 static void shrink_ple_window(struct kvm_vcpu *vcpu)
1293 {
1294         struct vcpu_svm *svm = to_svm(vcpu);
1295         struct vmcb_control_area *control = &svm->vmcb->control;
1296         int old = control->pause_filter_count;
1297
1298         control->pause_filter_count =
1299                                 __shrink_ple_window(old,
1300                                                     pause_filter_count,
1301                                                     pause_filter_count_shrink,
1302                                                     pause_filter_count);
1303         if (control->pause_filter_count != old) {
1304                 mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1305                 trace_kvm_ple_window_update(vcpu->vcpu_id,
1306                                             control->pause_filter_count, old);
1307         }
1308 }
1309
1310 /*
1311  * The default MMIO mask is a single bit (excluding the present bit),
1312  * which could conflict with the memory encryption bit. Check for
1313  * memory encryption support and override the default MMIO mask if
1314  * memory encryption is enabled.
1315  */
1316 static __init void svm_adjust_mmio_mask(void)
1317 {
1318         unsigned int enc_bit, mask_bit;
1319         u64 msr, mask;
1320
1321         /* If there is no memory encryption support, use existing mask */
1322         if (cpuid_eax(0x80000000) < 0x8000001f)
1323                 return;
1324
1325         /* If memory encryption is not enabled, use existing mask */
1326         rdmsrl(MSR_K8_SYSCFG, msr);
1327         if (!(msr & MSR_K8_SYSCFG_MEM_ENCRYPT))
1328                 return;
1329
1330         enc_bit = cpuid_ebx(0x8000001f) & 0x3f;
1331         mask_bit = boot_cpu_data.x86_phys_bits;
1332
1333         /* Increment the mask bit if it is the same as the encryption bit */
1334         if (enc_bit == mask_bit)
1335                 mask_bit++;
1336
1337         /*
1338          * If the mask bit location is below 52, then some bits above the
1339          * physical addressing limit will always be reserved, so use the
1340          * rsvd_bits() function to generate the mask. This mask, along with
1341          * the present bit, will be used to generate a page fault with
1342          * PFER.RSV = 1.
1343          *
1344          * If the mask bit location is 52 (or above), then clear the mask.
1345          */
1346         mask = (mask_bit < 52) ? rsvd_bits(mask_bit, 51) | PT_PRESENT_MASK : 0;
1347
1348         kvm_mmu_set_mmio_spte_mask(mask, mask, PT_WRITABLE_MASK | PT_USER_MASK);
1349 }
1350
1351 static __init int svm_hardware_setup(void)
1352 {
1353         int cpu;
1354         struct page *iopm_pages;
1355         void *iopm_va;
1356         int r;
1357
1358         iopm_pages = alloc_pages(GFP_KERNEL, IOPM_ALLOC_ORDER);
1359
1360         if (!iopm_pages)
1361                 return -ENOMEM;
1362
1363         iopm_va = page_address(iopm_pages);
1364         memset(iopm_va, 0xff, PAGE_SIZE * (1 << IOPM_ALLOC_ORDER));
1365         iopm_base = page_to_pfn(iopm_pages) << PAGE_SHIFT;
1366
1367         init_msrpm_offsets();
1368
1369         if (boot_cpu_has(X86_FEATURE_NX))
1370                 kvm_enable_efer_bits(EFER_NX);
1371
1372         if (boot_cpu_has(X86_FEATURE_FXSR_OPT))
1373                 kvm_enable_efer_bits(EFER_FFXSR);
1374
1375         if (boot_cpu_has(X86_FEATURE_TSCRATEMSR)) {
1376                 kvm_has_tsc_control = true;
1377                 kvm_max_tsc_scaling_ratio = TSC_RATIO_MAX;
1378                 kvm_tsc_scaling_ratio_frac_bits = 32;
1379         }
1380
1381         /* Check for pause filtering support */
1382         if (!boot_cpu_has(X86_FEATURE_PAUSEFILTER)) {
1383                 pause_filter_count = 0;
1384                 pause_filter_thresh = 0;
1385         } else if (!boot_cpu_has(X86_FEATURE_PFTHRESHOLD)) {
1386                 pause_filter_thresh = 0;
1387         }
1388
1389         if (nested) {
1390                 printk(KERN_INFO "kvm: Nested Virtualization enabled\n");
1391                 kvm_enable_efer_bits(EFER_SVME | EFER_LMSLE);
1392         }
1393
1394         if (sev) {
1395                 if (boot_cpu_has(X86_FEATURE_SEV) &&
1396                     IS_ENABLED(CONFIG_KVM_AMD_SEV)) {
1397                         r = sev_hardware_setup();
1398                         if (r)
1399                                 sev = false;
1400                 } else {
1401                         sev = false;
1402                 }
1403         }
1404
1405         svm_adjust_mmio_mask();
1406
1407         for_each_possible_cpu(cpu) {
1408                 r = svm_cpu_init(cpu);
1409                 if (r)
1410                         goto err;
1411         }
1412
1413         if (!boot_cpu_has(X86_FEATURE_NPT))
1414                 npt_enabled = false;
1415
1416         if (npt_enabled && !npt) {
1417                 printk(KERN_INFO "kvm: Nested Paging disabled\n");
1418                 npt_enabled = false;
1419         }
1420
1421         if (npt_enabled) {
1422                 printk(KERN_INFO "kvm: Nested Paging enabled\n");
1423                 kvm_enable_tdp();
1424         } else
1425                 kvm_disable_tdp();
1426
1427         if (nrips) {
1428                 if (!boot_cpu_has(X86_FEATURE_NRIPS))
1429                         nrips = false;
1430         }
1431
1432         if (avic) {
1433                 if (!npt_enabled ||
1434                     !boot_cpu_has(X86_FEATURE_AVIC) ||
1435                     !IS_ENABLED(CONFIG_X86_LOCAL_APIC)) {
1436                         avic = false;
1437                 } else {
1438                         pr_info("AVIC enabled\n");
1439
1440                         amd_iommu_register_ga_log_notifier(&avic_ga_log_notifier);
1441                 }
1442         }
1443
1444         if (vls) {
1445                 if (!npt_enabled ||
1446                     !boot_cpu_has(X86_FEATURE_V_VMSAVE_VMLOAD) ||
1447                     !IS_ENABLED(CONFIG_X86_64)) {
1448                         vls = false;
1449                 } else {
1450                         pr_info("Virtual VMLOAD VMSAVE supported\n");
1451                 }
1452         }
1453
1454         if (vgif) {
1455                 if (!boot_cpu_has(X86_FEATURE_VGIF))
1456                         vgif = false;
1457                 else
1458                         pr_info("Virtual GIF supported\n");
1459         }
1460
1461         return 0;
1462
1463 err:
1464         __free_pages(iopm_pages, IOPM_ALLOC_ORDER);
1465         iopm_base = 0;
1466         return r;
1467 }
1468
1469 static __exit void svm_hardware_unsetup(void)
1470 {
1471         int cpu;
1472
1473         if (svm_sev_enabled()) {
1474                 bitmap_free(sev_asid_bitmap);
1475                 bitmap_free(sev_reclaim_asid_bitmap);
1476
1477                 sev_flush_asids();
1478         }
1479
1480         for_each_possible_cpu(cpu)
1481                 svm_cpu_uninit(cpu);
1482
1483         __free_pages(pfn_to_page(iopm_base >> PAGE_SHIFT), IOPM_ALLOC_ORDER);
1484         iopm_base = 0;
1485 }
1486
1487 static void init_seg(struct vmcb_seg *seg)
1488 {
1489         seg->selector = 0;
1490         seg->attrib = SVM_SELECTOR_P_MASK | SVM_SELECTOR_S_MASK |
1491                       SVM_SELECTOR_WRITE_MASK; /* Read/Write Data Segment */
1492         seg->limit = 0xffff;
1493         seg->base = 0;
1494 }
1495
1496 static void init_sys_seg(struct vmcb_seg *seg, uint32_t type)
1497 {
1498         seg->selector = 0;
1499         seg->attrib = SVM_SELECTOR_P_MASK | type;
1500         seg->limit = 0xffff;
1501         seg->base = 0;
1502 }
1503
1504 static u64 svm_read_l1_tsc_offset(struct kvm_vcpu *vcpu)
1505 {
1506         struct vcpu_svm *svm = to_svm(vcpu);
1507
1508         if (is_guest_mode(vcpu))
1509                 return svm->nested.hsave->control.tsc_offset;
1510
1511         return vcpu->arch.tsc_offset;
1512 }
1513
1514 static u64 svm_write_l1_tsc_offset(struct kvm_vcpu *vcpu, u64 offset)
1515 {
1516         struct vcpu_svm *svm = to_svm(vcpu);
1517         u64 g_tsc_offset = 0;
1518
1519         if (is_guest_mode(vcpu)) {
1520                 /* Write L1's TSC offset.  */
1521                 g_tsc_offset = svm->vmcb->control.tsc_offset -
1522                                svm->nested.hsave->control.tsc_offset;
1523                 svm->nested.hsave->control.tsc_offset = offset;
1524         }
1525
1526         trace_kvm_write_tsc_offset(vcpu->vcpu_id,
1527                                    svm->vmcb->control.tsc_offset - g_tsc_offset,
1528                                    offset);
1529
1530         svm->vmcb->control.tsc_offset = offset + g_tsc_offset;
1531
1532         mark_dirty(svm->vmcb, VMCB_INTERCEPTS);
1533         return svm->vmcb->control.tsc_offset;
1534 }
1535
1536 static void avic_init_vmcb(struct vcpu_svm *svm)
1537 {
1538         struct vmcb *vmcb = svm->vmcb;
1539         struct kvm_svm *kvm_svm = to_kvm_svm(svm->vcpu.kvm);
1540         phys_addr_t bpa = __sme_set(page_to_phys(svm->avic_backing_page));
1541         phys_addr_t lpa = __sme_set(page_to_phys(kvm_svm->avic_logical_id_table_page));
1542         phys_addr_t ppa = __sme_set(page_to_phys(kvm_svm->avic_physical_id_table_page));
1543
1544         vmcb->control.avic_backing_page = bpa & AVIC_HPA_MASK;
1545         vmcb->control.avic_logical_id = lpa & AVIC_HPA_MASK;
1546         vmcb->control.avic_physical_id = ppa & AVIC_HPA_MASK;
1547         vmcb->control.avic_physical_id |= AVIC_MAX_PHYSICAL_ID_COUNT;
1548         vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
1549 }
1550
1551 static void init_vmcb(struct vcpu_svm *svm)
1552 {
1553         struct vmcb_control_area *control = &svm->vmcb->control;
1554         struct vmcb_save_area *save = &svm->vmcb->save;
1555
1556         svm->vcpu.arch.hflags = 0;
1557
1558         set_cr_intercept(svm, INTERCEPT_CR0_READ);
1559         set_cr_intercept(svm, INTERCEPT_CR3_READ);
1560         set_cr_intercept(svm, INTERCEPT_CR4_READ);
1561         set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
1562         set_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1563         set_cr_intercept(svm, INTERCEPT_CR4_WRITE);
1564         if (!kvm_vcpu_apicv_active(&svm->vcpu))
1565                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
1566
1567         set_dr_intercepts(svm);
1568
1569         set_exception_intercept(svm, PF_VECTOR);
1570         set_exception_intercept(svm, UD_VECTOR);
1571         set_exception_intercept(svm, MC_VECTOR);
1572         set_exception_intercept(svm, AC_VECTOR);
1573         set_exception_intercept(svm, DB_VECTOR);
1574         /*
1575          * Guest access to VMware backdoor ports could legitimately
1576          * trigger #GP because of TSS I/O permission bitmap.
1577          * We intercept those #GP and allow access to them anyway
1578          * as VMware does.
1579          */
1580         if (enable_vmware_backdoor)
1581                 set_exception_intercept(svm, GP_VECTOR);
1582
1583         set_intercept(svm, INTERCEPT_INTR);
1584         set_intercept(svm, INTERCEPT_NMI);
1585         set_intercept(svm, INTERCEPT_SMI);
1586         set_intercept(svm, INTERCEPT_SELECTIVE_CR0);
1587         set_intercept(svm, INTERCEPT_RDPMC);
1588         set_intercept(svm, INTERCEPT_CPUID);
1589         set_intercept(svm, INTERCEPT_INVD);
1590         set_intercept(svm, INTERCEPT_INVLPG);
1591         set_intercept(svm, INTERCEPT_INVLPGA);
1592         set_intercept(svm, INTERCEPT_IOIO_PROT);
1593         set_intercept(svm, INTERCEPT_MSR_PROT);
1594         set_intercept(svm, INTERCEPT_TASK_SWITCH);
1595         set_intercept(svm, INTERCEPT_SHUTDOWN);
1596         set_intercept(svm, INTERCEPT_VMRUN);
1597         set_intercept(svm, INTERCEPT_VMMCALL);
1598         set_intercept(svm, INTERCEPT_VMLOAD);
1599         set_intercept(svm, INTERCEPT_VMSAVE);
1600         set_intercept(svm, INTERCEPT_STGI);
1601         set_intercept(svm, INTERCEPT_CLGI);
1602         set_intercept(svm, INTERCEPT_SKINIT);
1603         set_intercept(svm, INTERCEPT_WBINVD);
1604         set_intercept(svm, INTERCEPT_XSETBV);
1605         set_intercept(svm, INTERCEPT_RDPRU);
1606         set_intercept(svm, INTERCEPT_RSM);
1607
1608         if (!kvm_mwait_in_guest(svm->vcpu.kvm)) {
1609                 set_intercept(svm, INTERCEPT_MONITOR);
1610                 set_intercept(svm, INTERCEPT_MWAIT);
1611         }
1612
1613         if (!kvm_hlt_in_guest(svm->vcpu.kvm))
1614                 set_intercept(svm, INTERCEPT_HLT);
1615
1616         control->iopm_base_pa = __sme_set(iopm_base);
1617         control->msrpm_base_pa = __sme_set(__pa(svm->msrpm));
1618         control->int_ctl = V_INTR_MASKING_MASK;
1619
1620         init_seg(&save->es);
1621         init_seg(&save->ss);
1622         init_seg(&save->ds);
1623         init_seg(&save->fs);
1624         init_seg(&save->gs);
1625
1626         save->cs.selector = 0xf000;
1627         save->cs.base = 0xffff0000;
1628         /* Executable/Readable Code Segment */
1629         save->cs.attrib = SVM_SELECTOR_READ_MASK | SVM_SELECTOR_P_MASK |
1630                 SVM_SELECTOR_S_MASK | SVM_SELECTOR_CODE_MASK;
1631         save->cs.limit = 0xffff;
1632
1633         save->gdtr.limit = 0xffff;
1634         save->idtr.limit = 0xffff;
1635
1636         init_sys_seg(&save->ldtr, SEG_TYPE_LDT);
1637         init_sys_seg(&save->tr, SEG_TYPE_BUSY_TSS16);
1638
1639         svm_set_efer(&svm->vcpu, 0);
1640         save->dr6 = 0xffff0ff0;
1641         kvm_set_rflags(&svm->vcpu, 2);
1642         save->rip = 0x0000fff0;
1643         svm->vcpu.arch.regs[VCPU_REGS_RIP] = save->rip;
1644
1645         /*
1646          * svm_set_cr0() sets PG and WP and clears NW and CD on save->cr0.
1647          * It also updates the guest-visible cr0 value.
1648          */
1649         svm_set_cr0(&svm->vcpu, X86_CR0_NW | X86_CR0_CD | X86_CR0_ET);
1650         kvm_mmu_reset_context(&svm->vcpu);
1651
1652         save->cr4 = X86_CR4_PAE;
1653         /* rdx = ?? */
1654
1655         if (npt_enabled) {
1656                 /* Setup VMCB for Nested Paging */
1657                 control->nested_ctl |= SVM_NESTED_CTL_NP_ENABLE;
1658                 clr_intercept(svm, INTERCEPT_INVLPG);
1659                 clr_exception_intercept(svm, PF_VECTOR);
1660                 clr_cr_intercept(svm, INTERCEPT_CR3_READ);
1661                 clr_cr_intercept(svm, INTERCEPT_CR3_WRITE);
1662                 save->g_pat = svm->vcpu.arch.pat;
1663                 save->cr3 = 0;
1664                 save->cr4 = 0;
1665         }
1666         svm->asid_generation = 0;
1667
1668         svm->nested.vmcb = 0;
1669         svm->vcpu.arch.hflags = 0;
1670
1671         if (pause_filter_count) {
1672                 control->pause_filter_count = pause_filter_count;
1673                 if (pause_filter_thresh)
1674                         control->pause_filter_thresh = pause_filter_thresh;
1675                 set_intercept(svm, INTERCEPT_PAUSE);
1676         } else {
1677                 clr_intercept(svm, INTERCEPT_PAUSE);
1678         }
1679
1680         if (kvm_vcpu_apicv_active(&svm->vcpu))
1681                 avic_init_vmcb(svm);
1682
1683         /*
1684          * If hardware supports Virtual VMLOAD VMSAVE then enable it
1685          * in VMCB and clear intercepts to avoid #VMEXIT.
1686          */
1687         if (vls) {
1688                 clr_intercept(svm, INTERCEPT_VMLOAD);
1689                 clr_intercept(svm, INTERCEPT_VMSAVE);
1690                 svm->vmcb->control.virt_ext |= VIRTUAL_VMLOAD_VMSAVE_ENABLE_MASK;
1691         }
1692
1693         if (vgif) {
1694                 clr_intercept(svm, INTERCEPT_STGI);
1695                 clr_intercept(svm, INTERCEPT_CLGI);
1696                 svm->vmcb->control.int_ctl |= V_GIF_ENABLE_MASK;
1697         }
1698
1699         if (sev_guest(svm->vcpu.kvm)) {
1700                 svm->vmcb->control.nested_ctl |= SVM_NESTED_CTL_SEV_ENABLE;
1701                 clr_exception_intercept(svm, UD_VECTOR);
1702         }
1703
1704         mark_all_dirty(svm->vmcb);
1705
1706         enable_gif(svm);
1707
1708 }
1709
1710 static u64 *avic_get_physical_id_entry(struct kvm_vcpu *vcpu,
1711                                        unsigned int index)
1712 {
1713         u64 *avic_physical_id_table;
1714         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
1715
1716         if (index >= AVIC_MAX_PHYSICAL_ID_COUNT)
1717                 return NULL;
1718
1719         avic_physical_id_table = page_address(kvm_svm->avic_physical_id_table_page);
1720
1721         return &avic_physical_id_table[index];
1722 }
1723
1724 /**
1725  * Note:
1726  * AVIC hardware walks the nested page table to check permissions,
1727  * but does not use the SPA address specified in the leaf page
1728  * table entry since it uses  address in the AVIC_BACKING_PAGE pointer
1729  * field of the VMCB. Therefore, we set up the
1730  * APIC_ACCESS_PAGE_PRIVATE_MEMSLOT (4KB) here.
1731  */
1732 static int avic_init_access_page(struct kvm_vcpu *vcpu)
1733 {
1734         struct kvm *kvm = vcpu->kvm;
1735         int ret = 0;
1736
1737         mutex_lock(&kvm->slots_lock);
1738         if (kvm->arch.apic_access_page_done)
1739                 goto out;
1740
1741         ret = __x86_set_memory_region(kvm,
1742                                       APIC_ACCESS_PAGE_PRIVATE_MEMSLOT,
1743                                       APIC_DEFAULT_PHYS_BASE,
1744                                       PAGE_SIZE);
1745         if (ret)
1746                 goto out;
1747
1748         kvm->arch.apic_access_page_done = true;
1749 out:
1750         mutex_unlock(&kvm->slots_lock);
1751         return ret;
1752 }
1753
1754 static int avic_init_backing_page(struct kvm_vcpu *vcpu)
1755 {
1756         int ret;
1757         u64 *entry, new_entry;
1758         int id = vcpu->vcpu_id;
1759         struct vcpu_svm *svm = to_svm(vcpu);
1760
1761         ret = avic_init_access_page(vcpu);
1762         if (ret)
1763                 return ret;
1764
1765         if (id >= AVIC_MAX_PHYSICAL_ID_COUNT)
1766                 return -EINVAL;
1767
1768         if (!svm->vcpu.arch.apic->regs)
1769                 return -EINVAL;
1770
1771         svm->avic_backing_page = virt_to_page(svm->vcpu.arch.apic->regs);
1772
1773         /* Setting AVIC backing page address in the phy APIC ID table */
1774         entry = avic_get_physical_id_entry(vcpu, id);
1775         if (!entry)
1776                 return -EINVAL;
1777
1778         new_entry = __sme_set((page_to_phys(svm->avic_backing_page) &
1779                               AVIC_PHYSICAL_ID_ENTRY_BACKING_PAGE_MASK) |
1780                               AVIC_PHYSICAL_ID_ENTRY_VALID_MASK);
1781         WRITE_ONCE(*entry, new_entry);
1782
1783         svm->avic_physical_id_cache = entry;
1784
1785         return 0;
1786 }
1787
1788 static void sev_asid_free(int asid)
1789 {
1790         struct svm_cpu_data *sd;
1791         int cpu, pos;
1792
1793         mutex_lock(&sev_bitmap_lock);
1794
1795         pos = asid - 1;
1796         __set_bit(pos, sev_reclaim_asid_bitmap);
1797
1798         for_each_possible_cpu(cpu) {
1799                 sd = per_cpu(svm_data, cpu);
1800                 sd->sev_vmcbs[pos] = NULL;
1801         }
1802
1803         mutex_unlock(&sev_bitmap_lock);
1804 }
1805
1806 static void sev_unbind_asid(struct kvm *kvm, unsigned int handle)
1807 {
1808         struct sev_data_decommission *decommission;
1809         struct sev_data_deactivate *data;
1810
1811         if (!handle)
1812                 return;
1813
1814         data = kzalloc(sizeof(*data), GFP_KERNEL);
1815         if (!data)
1816                 return;
1817
1818         /* deactivate handle */
1819         data->handle = handle;
1820
1821         /* Guard DEACTIVATE against WBINVD/DF_FLUSH used in ASID recycling */
1822         down_read(&sev_deactivate_lock);
1823         sev_guest_deactivate(data, NULL);
1824         up_read(&sev_deactivate_lock);
1825
1826         kfree(data);
1827
1828         decommission = kzalloc(sizeof(*decommission), GFP_KERNEL);
1829         if (!decommission)
1830                 return;
1831
1832         /* decommission handle */
1833         decommission->handle = handle;
1834         sev_guest_decommission(decommission, NULL);
1835
1836         kfree(decommission);
1837 }
1838
1839 static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr,
1840                                     unsigned long ulen, unsigned long *n,
1841                                     int write)
1842 {
1843         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1844         unsigned long npages, npinned, size;
1845         unsigned long locked, lock_limit;
1846         struct page **pages;
1847         unsigned long first, last;
1848
1849         if (ulen == 0 || uaddr + ulen < uaddr)
1850                 return NULL;
1851
1852         /* Calculate number of pages. */
1853         first = (uaddr & PAGE_MASK) >> PAGE_SHIFT;
1854         last = ((uaddr + ulen - 1) & PAGE_MASK) >> PAGE_SHIFT;
1855         npages = (last - first + 1);
1856
1857         locked = sev->pages_locked + npages;
1858         lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT;
1859         if (locked > lock_limit && !capable(CAP_IPC_LOCK)) {
1860                 pr_err("SEV: %lu locked pages exceed the lock limit of %lu.\n", locked, lock_limit);
1861                 return NULL;
1862         }
1863
1864         /* Avoid using vmalloc for smaller buffers. */
1865         size = npages * sizeof(struct page *);
1866         if (size > PAGE_SIZE)
1867                 pages = __vmalloc(size, GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1868                                   PAGE_KERNEL);
1869         else
1870                 pages = kmalloc(size, GFP_KERNEL_ACCOUNT);
1871
1872         if (!pages)
1873                 return NULL;
1874
1875         /* Pin the user virtual address. */
1876         npinned = get_user_pages_fast(uaddr, npages, FOLL_WRITE, pages);
1877         if (npinned != npages) {
1878                 pr_err("SEV: Failure locking %lu pages.\n", npages);
1879                 goto err;
1880         }
1881
1882         *n = npages;
1883         sev->pages_locked = locked;
1884
1885         return pages;
1886
1887 err:
1888         if (npinned > 0)
1889                 release_pages(pages, npinned);
1890
1891         kvfree(pages);
1892         return NULL;
1893 }
1894
1895 static void sev_unpin_memory(struct kvm *kvm, struct page **pages,
1896                              unsigned long npages)
1897 {
1898         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1899
1900         release_pages(pages, npages);
1901         kvfree(pages);
1902         sev->pages_locked -= npages;
1903 }
1904
1905 static void sev_clflush_pages(struct page *pages[], unsigned long npages)
1906 {
1907         uint8_t *page_virtual;
1908         unsigned long i;
1909
1910         if (npages == 0 || pages == NULL)
1911                 return;
1912
1913         for (i = 0; i < npages; i++) {
1914                 page_virtual = kmap_atomic(pages[i]);
1915                 clflush_cache_range(page_virtual, PAGE_SIZE);
1916                 kunmap_atomic(page_virtual);
1917         }
1918 }
1919
1920 static void __unregister_enc_region_locked(struct kvm *kvm,
1921                                            struct enc_region *region)
1922 {
1923         /*
1924          * The guest may change the memory encryption attribute from C=0 -> C=1
1925          * or vice versa for this memory range. Lets make sure caches are
1926          * flushed to ensure that guest data gets written into memory with
1927          * correct C-bit.
1928          */
1929         sev_clflush_pages(region->pages, region->npages);
1930
1931         sev_unpin_memory(kvm, region->pages, region->npages);
1932         list_del(&region->list);
1933         kfree(region);
1934 }
1935
1936 static struct kvm *svm_vm_alloc(void)
1937 {
1938         struct kvm_svm *kvm_svm = __vmalloc(sizeof(struct kvm_svm),
1939                                             GFP_KERNEL_ACCOUNT | __GFP_ZERO,
1940                                             PAGE_KERNEL);
1941         return &kvm_svm->kvm;
1942 }
1943
1944 static void svm_vm_free(struct kvm *kvm)
1945 {
1946         vfree(to_kvm_svm(kvm));
1947 }
1948
1949 static void sev_vm_destroy(struct kvm *kvm)
1950 {
1951         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
1952         struct list_head *head = &sev->regions_list;
1953         struct list_head *pos, *q;
1954
1955         if (!sev_guest(kvm))
1956                 return;
1957
1958         mutex_lock(&kvm->lock);
1959
1960         /*
1961          * if userspace was terminated before unregistering the memory regions
1962          * then lets unpin all the registered memory.
1963          */
1964         if (!list_empty(head)) {
1965                 list_for_each_safe(pos, q, head) {
1966                         __unregister_enc_region_locked(kvm,
1967                                 list_entry(pos, struct enc_region, list));
1968                 }
1969         }
1970
1971         mutex_unlock(&kvm->lock);
1972
1973         sev_unbind_asid(kvm, sev->handle);
1974         sev_asid_free(sev->asid);
1975 }
1976
1977 static void avic_vm_destroy(struct kvm *kvm)
1978 {
1979         unsigned long flags;
1980         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
1981
1982         if (!avic)
1983                 return;
1984
1985         if (kvm_svm->avic_logical_id_table_page)
1986                 __free_page(kvm_svm->avic_logical_id_table_page);
1987         if (kvm_svm->avic_physical_id_table_page)
1988                 __free_page(kvm_svm->avic_physical_id_table_page);
1989
1990         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
1991         hash_del(&kvm_svm->hnode);
1992         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
1993 }
1994
1995 static void svm_vm_destroy(struct kvm *kvm)
1996 {
1997         avic_vm_destroy(kvm);
1998         sev_vm_destroy(kvm);
1999 }
2000
2001 static int avic_vm_init(struct kvm *kvm)
2002 {
2003         unsigned long flags;
2004         int err = -ENOMEM;
2005         struct kvm_svm *kvm_svm = to_kvm_svm(kvm);
2006         struct kvm_svm *k2;
2007         struct page *p_page;
2008         struct page *l_page;
2009         u32 vm_id;
2010
2011         if (!avic)
2012                 return 0;
2013
2014         /* Allocating physical APIC ID table (4KB) */
2015         p_page = alloc_page(GFP_KERNEL_ACCOUNT);
2016         if (!p_page)
2017                 goto free_avic;
2018
2019         kvm_svm->avic_physical_id_table_page = p_page;
2020         clear_page(page_address(p_page));
2021
2022         /* Allocating logical APIC ID table (4KB) */
2023         l_page = alloc_page(GFP_KERNEL_ACCOUNT);
2024         if (!l_page)
2025                 goto free_avic;
2026
2027         kvm_svm->avic_logical_id_table_page = l_page;
2028         clear_page(page_address(l_page));
2029
2030         spin_lock_irqsave(&svm_vm_data_hash_lock, flags);
2031  again:
2032         vm_id = next_vm_id = (next_vm_id + 1) & AVIC_VM_ID_MASK;
2033         if (vm_id == 0) { /* id is 1-based, zero is not okay */
2034                 next_vm_id_wrapped = 1;
2035                 goto again;
2036         }
2037         /* Is it still in use? Only possible if wrapped at least once */
2038         if (next_vm_id_wrapped) {
2039                 hash_for_each_possible(svm_vm_data_hash, k2, hnode, vm_id) {
2040                         if (k2->avic_vm_id == vm_id)
2041                                 goto again;
2042                 }
2043         }
2044         kvm_svm->avic_vm_id = vm_id;
2045         hash_add(svm_vm_data_hash, &kvm_svm->hnode, kvm_svm->avic_vm_id);
2046         spin_unlock_irqrestore(&svm_vm_data_hash_lock, flags);
2047
2048         return 0;
2049
2050 free_avic:
2051         avic_vm_destroy(kvm);
2052         return err;
2053 }
2054
2055 static inline int
2056 avic_update_iommu_vcpu_affinity(struct kvm_vcpu *vcpu, int cpu, bool r)
2057 {
2058         int ret = 0;
2059         unsigned long flags;
2060         struct amd_svm_iommu_ir *ir;
2061         struct vcpu_svm *svm = to_svm(vcpu);
2062
2063         if (!kvm_arch_has_assigned_device(vcpu->kvm))
2064                 return 0;
2065
2066         /*
2067          * Here, we go through the per-vcpu ir_list to update all existing
2068          * interrupt remapping table entry targeting this vcpu.
2069          */
2070         spin_lock_irqsave(&svm->ir_list_lock, flags);
2071
2072         if (list_empty(&svm->ir_list))
2073                 goto out;
2074
2075         list_for_each_entry(ir, &svm->ir_list, node) {
2076                 ret = amd_iommu_update_ga(cpu, r, ir->data);
2077                 if (ret)
2078                         break;
2079         }
2080 out:
2081         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
2082         return ret;
2083 }
2084
2085 static void avic_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2086 {
2087         u64 entry;
2088         /* ID = 0xff (broadcast), ID > 0xff (reserved) */
2089         int h_physical_id = kvm_cpu_get_apicid(cpu);
2090         struct vcpu_svm *svm = to_svm(vcpu);
2091
2092         if (!kvm_vcpu_apicv_active(vcpu))
2093                 return;
2094
2095         /*
2096          * Since the host physical APIC id is 8 bits,
2097          * we can support host APIC ID upto 255.
2098          */
2099         if (WARN_ON(h_physical_id > AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK))
2100                 return;
2101
2102         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2103         WARN_ON(entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK);
2104
2105         entry &= ~AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK;
2106         entry |= (h_physical_id & AVIC_PHYSICAL_ID_ENTRY_HOST_PHYSICAL_ID_MASK);
2107
2108         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2109         if (svm->avic_is_running)
2110                 entry |= AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2111
2112         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2113         avic_update_iommu_vcpu_affinity(vcpu, h_physical_id,
2114                                         svm->avic_is_running);
2115 }
2116
2117 static void avic_vcpu_put(struct kvm_vcpu *vcpu)
2118 {
2119         u64 entry;
2120         struct vcpu_svm *svm = to_svm(vcpu);
2121
2122         if (!kvm_vcpu_apicv_active(vcpu))
2123                 return;
2124
2125         entry = READ_ONCE(*(svm->avic_physical_id_cache));
2126         if (entry & AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK)
2127                 avic_update_iommu_vcpu_affinity(vcpu, -1, 0);
2128
2129         entry &= ~AVIC_PHYSICAL_ID_ENTRY_IS_RUNNING_MASK;
2130         WRITE_ONCE(*(svm->avic_physical_id_cache), entry);
2131 }
2132
2133 /**
2134  * This function is called during VCPU halt/unhalt.
2135  */
2136 static void avic_set_running(struct kvm_vcpu *vcpu, bool is_run)
2137 {
2138         struct vcpu_svm *svm = to_svm(vcpu);
2139
2140         svm->avic_is_running = is_run;
2141         if (is_run)
2142                 avic_vcpu_load(vcpu, vcpu->cpu);
2143         else
2144                 avic_vcpu_put(vcpu);
2145 }
2146
2147 static void svm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
2148 {
2149         struct vcpu_svm *svm = to_svm(vcpu);
2150         u32 dummy;
2151         u32 eax = 1;
2152
2153         vcpu->arch.microcode_version = 0x01000065;
2154         svm->spec_ctrl = 0;
2155         svm->virt_spec_ctrl = 0;
2156
2157         if (!init_event) {
2158                 svm->vcpu.arch.apic_base = APIC_DEFAULT_PHYS_BASE |
2159                                            MSR_IA32_APICBASE_ENABLE;
2160                 if (kvm_vcpu_is_reset_bsp(&svm->vcpu))
2161                         svm->vcpu.arch.apic_base |= MSR_IA32_APICBASE_BSP;
2162         }
2163         init_vmcb(svm);
2164
2165         kvm_cpuid(vcpu, &eax, &dummy, &dummy, &dummy, true);
2166         kvm_rdx_write(vcpu, eax);
2167
2168         if (kvm_vcpu_apicv_active(vcpu) && !init_event)
2169                 avic_update_vapic_bar(svm, APIC_DEFAULT_PHYS_BASE);
2170 }
2171
2172 static int avic_init_vcpu(struct vcpu_svm *svm)
2173 {
2174         int ret;
2175
2176         if (!kvm_vcpu_apicv_active(&svm->vcpu))
2177                 return 0;
2178
2179         ret = avic_init_backing_page(&svm->vcpu);
2180         if (ret)
2181                 return ret;
2182
2183         INIT_LIST_HEAD(&svm->ir_list);
2184         spin_lock_init(&svm->ir_list_lock);
2185         svm->dfr_reg = APIC_DFR_FLAT;
2186
2187         return ret;
2188 }
2189
2190 static int svm_create_vcpu(struct kvm_vcpu *vcpu)
2191 {
2192         struct vcpu_svm *svm;
2193         struct page *page;
2194         struct page *msrpm_pages;
2195         struct page *hsave_page;
2196         struct page *nested_msrpm_pages;
2197         int err;
2198
2199         BUILD_BUG_ON(offsetof(struct vcpu_svm, vcpu) != 0);
2200         svm = to_svm(vcpu);
2201
2202         err = -ENOMEM;
2203         page = alloc_page(GFP_KERNEL_ACCOUNT);
2204         if (!page)
2205                 goto out;
2206
2207         msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
2208         if (!msrpm_pages)
2209                 goto free_page1;
2210
2211         nested_msrpm_pages = alloc_pages(GFP_KERNEL_ACCOUNT, MSRPM_ALLOC_ORDER);
2212         if (!nested_msrpm_pages)
2213                 goto free_page2;
2214
2215         hsave_page = alloc_page(GFP_KERNEL_ACCOUNT);
2216         if (!hsave_page)
2217                 goto free_page3;
2218
2219         err = avic_init_vcpu(svm);
2220         if (err)
2221                 goto free_page4;
2222
2223         /* We initialize this flag to true to make sure that the is_running
2224          * bit would be set the first time the vcpu is loaded.
2225          */
2226         svm->avic_is_running = true;
2227
2228         svm->nested.hsave = page_address(hsave_page);
2229
2230         svm->msrpm = page_address(msrpm_pages);
2231         svm_vcpu_init_msrpm(svm->msrpm);
2232
2233         svm->nested.msrpm = page_address(nested_msrpm_pages);
2234         svm_vcpu_init_msrpm(svm->nested.msrpm);
2235
2236         svm->vmcb = page_address(page);
2237         clear_page(svm->vmcb);
2238         svm->vmcb_pa = __sme_set(page_to_pfn(page) << PAGE_SHIFT);
2239         svm->asid_generation = 0;
2240         init_vmcb(svm);
2241
2242         svm_init_osvw(vcpu);
2243
2244         return 0;
2245
2246 free_page4:
2247         __free_page(hsave_page);
2248 free_page3:
2249         __free_pages(nested_msrpm_pages, MSRPM_ALLOC_ORDER);
2250 free_page2:
2251         __free_pages(msrpm_pages, MSRPM_ALLOC_ORDER);
2252 free_page1:
2253         __free_page(page);
2254 out:
2255         return err;
2256 }
2257
2258 static void svm_clear_current_vmcb(struct vmcb *vmcb)
2259 {
2260         int i;
2261
2262         for_each_online_cpu(i)
2263                 cmpxchg(&per_cpu(svm_data, i)->current_vmcb, vmcb, NULL);
2264 }
2265
2266 static void svm_free_vcpu(struct kvm_vcpu *vcpu)
2267 {
2268         struct vcpu_svm *svm = to_svm(vcpu);
2269
2270         /*
2271          * The vmcb page can be recycled, causing a false negative in
2272          * svm_vcpu_load(). So, ensure that no logical CPU has this
2273          * vmcb page recorded as its current vmcb.
2274          */
2275         svm_clear_current_vmcb(svm->vmcb);
2276
2277         __free_page(pfn_to_page(__sme_clr(svm->vmcb_pa) >> PAGE_SHIFT));
2278         __free_pages(virt_to_page(svm->msrpm), MSRPM_ALLOC_ORDER);
2279         __free_page(virt_to_page(svm->nested.hsave));
2280         __free_pages(virt_to_page(svm->nested.msrpm), MSRPM_ALLOC_ORDER);
2281 }
2282
2283 static void svm_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
2284 {
2285         struct vcpu_svm *svm = to_svm(vcpu);
2286         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
2287         int i;
2288
2289         if (unlikely(cpu != vcpu->cpu)) {
2290                 svm->asid_generation = 0;
2291                 mark_all_dirty(svm->vmcb);
2292         }
2293
2294 #ifdef CONFIG_X86_64
2295         rdmsrl(MSR_GS_BASE, to_svm(vcpu)->host.gs_base);
2296 #endif
2297         savesegment(fs, svm->host.fs);
2298         savesegment(gs, svm->host.gs);
2299         svm->host.ldt = kvm_read_ldt();
2300
2301         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2302                 rdmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2303
2304         if (static_cpu_has(X86_FEATURE_TSCRATEMSR)) {
2305                 u64 tsc_ratio = vcpu->arch.tsc_scaling_ratio;
2306                 if (tsc_ratio != __this_cpu_read(current_tsc_ratio)) {
2307                         __this_cpu_write(current_tsc_ratio, tsc_ratio);
2308                         wrmsrl(MSR_AMD64_TSC_RATIO, tsc_ratio);
2309                 }
2310         }
2311         /* This assumes that the kernel never uses MSR_TSC_AUX */
2312         if (static_cpu_has(X86_FEATURE_RDTSCP))
2313                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
2314
2315         if (sd->current_vmcb != svm->vmcb) {
2316                 sd->current_vmcb = svm->vmcb;
2317                 indirect_branch_prediction_barrier();
2318         }
2319         avic_vcpu_load(vcpu, cpu);
2320 }
2321
2322 static void svm_vcpu_put(struct kvm_vcpu *vcpu)
2323 {
2324         struct vcpu_svm *svm = to_svm(vcpu);
2325         int i;
2326
2327         avic_vcpu_put(vcpu);
2328
2329         ++vcpu->stat.host_state_reload;
2330         kvm_load_ldt(svm->host.ldt);
2331 #ifdef CONFIG_X86_64
2332         loadsegment(fs, svm->host.fs);
2333         wrmsrl(MSR_KERNEL_GS_BASE, current->thread.gsbase);
2334         load_gs_index(svm->host.gs);
2335 #else
2336 #ifdef CONFIG_X86_32_LAZY_GS
2337         loadsegment(gs, svm->host.gs);
2338 #endif
2339 #endif
2340         for (i = 0; i < NR_HOST_SAVE_USER_MSRS; i++)
2341                 wrmsrl(host_save_user_msrs[i], svm->host_user_msrs[i]);
2342 }
2343
2344 static void svm_vcpu_blocking(struct kvm_vcpu *vcpu)
2345 {
2346         avic_set_running(vcpu, false);
2347 }
2348
2349 static void svm_vcpu_unblocking(struct kvm_vcpu *vcpu)
2350 {
2351         avic_set_running(vcpu, true);
2352 }
2353
2354 static unsigned long svm_get_rflags(struct kvm_vcpu *vcpu)
2355 {
2356         struct vcpu_svm *svm = to_svm(vcpu);
2357         unsigned long rflags = svm->vmcb->save.rflags;
2358
2359         if (svm->nmi_singlestep) {
2360                 /* Hide our flags if they were not set by the guest */
2361                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF))
2362                         rflags &= ~X86_EFLAGS_TF;
2363                 if (!(svm->nmi_singlestep_guest_rflags & X86_EFLAGS_RF))
2364                         rflags &= ~X86_EFLAGS_RF;
2365         }
2366         return rflags;
2367 }
2368
2369 static void svm_set_rflags(struct kvm_vcpu *vcpu, unsigned long rflags)
2370 {
2371         if (to_svm(vcpu)->nmi_singlestep)
2372                 rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
2373
2374        /*
2375         * Any change of EFLAGS.VM is accompanied by a reload of SS
2376         * (caused by either a task switch or an inter-privilege IRET),
2377         * so we do not need to update the CPL here.
2378         */
2379         to_svm(vcpu)->vmcb->save.rflags = rflags;
2380 }
2381
2382 static void svm_cache_reg(struct kvm_vcpu *vcpu, enum kvm_reg reg)
2383 {
2384         switch (reg) {
2385         case VCPU_EXREG_PDPTR:
2386                 BUG_ON(!npt_enabled);
2387                 load_pdptrs(vcpu, vcpu->arch.walk_mmu, kvm_read_cr3(vcpu));
2388                 break;
2389         default:
2390                 WARN_ON_ONCE(1);
2391         }
2392 }
2393
2394 static void svm_set_vintr(struct vcpu_svm *svm)
2395 {
2396         set_intercept(svm, INTERCEPT_VINTR);
2397 }
2398
2399 static void svm_clear_vintr(struct vcpu_svm *svm)
2400 {
2401         clr_intercept(svm, INTERCEPT_VINTR);
2402 }
2403
2404 static struct vmcb_seg *svm_seg(struct kvm_vcpu *vcpu, int seg)
2405 {
2406         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2407
2408         switch (seg) {
2409         case VCPU_SREG_CS: return &save->cs;
2410         case VCPU_SREG_DS: return &save->ds;
2411         case VCPU_SREG_ES: return &save->es;
2412         case VCPU_SREG_FS: return &save->fs;
2413         case VCPU_SREG_GS: return &save->gs;
2414         case VCPU_SREG_SS: return &save->ss;
2415         case VCPU_SREG_TR: return &save->tr;
2416         case VCPU_SREG_LDTR: return &save->ldtr;
2417         }
2418         BUG();
2419         return NULL;
2420 }
2421
2422 static u64 svm_get_segment_base(struct kvm_vcpu *vcpu, int seg)
2423 {
2424         struct vmcb_seg *s = svm_seg(vcpu, seg);
2425
2426         return s->base;
2427 }
2428
2429 static void svm_get_segment(struct kvm_vcpu *vcpu,
2430                             struct kvm_segment *var, int seg)
2431 {
2432         struct vmcb_seg *s = svm_seg(vcpu, seg);
2433
2434         var->base = s->base;
2435         var->limit = s->limit;
2436         var->selector = s->selector;
2437         var->type = s->attrib & SVM_SELECTOR_TYPE_MASK;
2438         var->s = (s->attrib >> SVM_SELECTOR_S_SHIFT) & 1;
2439         var->dpl = (s->attrib >> SVM_SELECTOR_DPL_SHIFT) & 3;
2440         var->present = (s->attrib >> SVM_SELECTOR_P_SHIFT) & 1;
2441         var->avl = (s->attrib >> SVM_SELECTOR_AVL_SHIFT) & 1;
2442         var->l = (s->attrib >> SVM_SELECTOR_L_SHIFT) & 1;
2443         var->db = (s->attrib >> SVM_SELECTOR_DB_SHIFT) & 1;
2444
2445         /*
2446          * AMD CPUs circa 2014 track the G bit for all segments except CS.
2447          * However, the SVM spec states that the G bit is not observed by the
2448          * CPU, and some VMware virtual CPUs drop the G bit for all segments.
2449          * So let's synthesize a legal G bit for all segments, this helps
2450          * running KVM nested. It also helps cross-vendor migration, because
2451          * Intel's vmentry has a check on the 'G' bit.
2452          */
2453         var->g = s->limit > 0xfffff;
2454
2455         /*
2456          * AMD's VMCB does not have an explicit unusable field, so emulate it
2457          * for cross vendor migration purposes by "not present"
2458          */
2459         var->unusable = !var->present;
2460
2461         switch (seg) {
2462         case VCPU_SREG_TR:
2463                 /*
2464                  * Work around a bug where the busy flag in the tr selector
2465                  * isn't exposed
2466                  */
2467                 var->type |= 0x2;
2468                 break;
2469         case VCPU_SREG_DS:
2470         case VCPU_SREG_ES:
2471         case VCPU_SREG_FS:
2472         case VCPU_SREG_GS:
2473                 /*
2474                  * The accessed bit must always be set in the segment
2475                  * descriptor cache, although it can be cleared in the
2476                  * descriptor, the cached bit always remains at 1. Since
2477                  * Intel has a check on this, set it here to support
2478                  * cross-vendor migration.
2479                  */
2480                 if (!var->unusable)
2481                         var->type |= 0x1;
2482                 break;
2483         case VCPU_SREG_SS:
2484                 /*
2485                  * On AMD CPUs sometimes the DB bit in the segment
2486                  * descriptor is left as 1, although the whole segment has
2487                  * been made unusable. Clear it here to pass an Intel VMX
2488                  * entry check when cross vendor migrating.
2489                  */
2490                 if (var->unusable)
2491                         var->db = 0;
2492                 /* This is symmetric with svm_set_segment() */
2493                 var->dpl = to_svm(vcpu)->vmcb->save.cpl;
2494                 break;
2495         }
2496 }
2497
2498 static int svm_get_cpl(struct kvm_vcpu *vcpu)
2499 {
2500         struct vmcb_save_area *save = &to_svm(vcpu)->vmcb->save;
2501
2502         return save->cpl;
2503 }
2504
2505 static void svm_get_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2506 {
2507         struct vcpu_svm *svm = to_svm(vcpu);
2508
2509         dt->size = svm->vmcb->save.idtr.limit;
2510         dt->address = svm->vmcb->save.idtr.base;
2511 }
2512
2513 static void svm_set_idt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2514 {
2515         struct vcpu_svm *svm = to_svm(vcpu);
2516
2517         svm->vmcb->save.idtr.limit = dt->size;
2518         svm->vmcb->save.idtr.base = dt->address ;
2519         mark_dirty(svm->vmcb, VMCB_DT);
2520 }
2521
2522 static void svm_get_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2523 {
2524         struct vcpu_svm *svm = to_svm(vcpu);
2525
2526         dt->size = svm->vmcb->save.gdtr.limit;
2527         dt->address = svm->vmcb->save.gdtr.base;
2528 }
2529
2530 static void svm_set_gdt(struct kvm_vcpu *vcpu, struct desc_ptr *dt)
2531 {
2532         struct vcpu_svm *svm = to_svm(vcpu);
2533
2534         svm->vmcb->save.gdtr.limit = dt->size;
2535         svm->vmcb->save.gdtr.base = dt->address ;
2536         mark_dirty(svm->vmcb, VMCB_DT);
2537 }
2538
2539 static void svm_decache_cr0_guest_bits(struct kvm_vcpu *vcpu)
2540 {
2541 }
2542
2543 static void svm_decache_cr4_guest_bits(struct kvm_vcpu *vcpu)
2544 {
2545 }
2546
2547 static void update_cr0_intercept(struct vcpu_svm *svm)
2548 {
2549         ulong gcr0 = svm->vcpu.arch.cr0;
2550         u64 *hcr0 = &svm->vmcb->save.cr0;
2551
2552         *hcr0 = (*hcr0 & ~SVM_CR0_SELECTIVE_MASK)
2553                 | (gcr0 & SVM_CR0_SELECTIVE_MASK);
2554
2555         mark_dirty(svm->vmcb, VMCB_CR);
2556
2557         if (gcr0 == *hcr0) {
2558                 clr_cr_intercept(svm, INTERCEPT_CR0_READ);
2559                 clr_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2560         } else {
2561                 set_cr_intercept(svm, INTERCEPT_CR0_READ);
2562                 set_cr_intercept(svm, INTERCEPT_CR0_WRITE);
2563         }
2564 }
2565
2566 static void svm_set_cr0(struct kvm_vcpu *vcpu, unsigned long cr0)
2567 {
2568         struct vcpu_svm *svm = to_svm(vcpu);
2569
2570 #ifdef CONFIG_X86_64
2571         if (vcpu->arch.efer & EFER_LME) {
2572                 if (!is_paging(vcpu) && (cr0 & X86_CR0_PG)) {
2573                         vcpu->arch.efer |= EFER_LMA;
2574                         svm->vmcb->save.efer |= EFER_LMA | EFER_LME;
2575                 }
2576
2577                 if (is_paging(vcpu) && !(cr0 & X86_CR0_PG)) {
2578                         vcpu->arch.efer &= ~EFER_LMA;
2579                         svm->vmcb->save.efer &= ~(EFER_LMA | EFER_LME);
2580                 }
2581         }
2582 #endif
2583         vcpu->arch.cr0 = cr0;
2584
2585         if (!npt_enabled)
2586                 cr0 |= X86_CR0_PG | X86_CR0_WP;
2587
2588         /*
2589          * re-enable caching here because the QEMU bios
2590          * does not do it - this results in some delay at
2591          * reboot
2592          */
2593         if (kvm_check_has_quirk(vcpu->kvm, KVM_X86_QUIRK_CD_NW_CLEARED))
2594                 cr0 &= ~(X86_CR0_CD | X86_CR0_NW);
2595         svm->vmcb->save.cr0 = cr0;
2596         mark_dirty(svm->vmcb, VMCB_CR);
2597         update_cr0_intercept(svm);
2598 }
2599
2600 static int svm_set_cr4(struct kvm_vcpu *vcpu, unsigned long cr4)
2601 {
2602         unsigned long host_cr4_mce = cr4_read_shadow() & X86_CR4_MCE;
2603         unsigned long old_cr4 = to_svm(vcpu)->vmcb->save.cr4;
2604
2605         if (cr4 & X86_CR4_VMXE)
2606                 return 1;
2607
2608         if (npt_enabled && ((old_cr4 ^ cr4) & X86_CR4_PGE))
2609                 svm_flush_tlb(vcpu, true);
2610
2611         vcpu->arch.cr4 = cr4;
2612         if (!npt_enabled)
2613                 cr4 |= X86_CR4_PAE;
2614         cr4 |= host_cr4_mce;
2615         to_svm(vcpu)->vmcb->save.cr4 = cr4;
2616         mark_dirty(to_svm(vcpu)->vmcb, VMCB_CR);
2617         return 0;
2618 }
2619
2620 static void svm_set_segment(struct kvm_vcpu *vcpu,
2621                             struct kvm_segment *var, int seg)
2622 {
2623         struct vcpu_svm *svm = to_svm(vcpu);
2624         struct vmcb_seg *s = svm_seg(vcpu, seg);
2625
2626         s->base = var->base;
2627         s->limit = var->limit;
2628         s->selector = var->selector;
2629         s->attrib = (var->type & SVM_SELECTOR_TYPE_MASK);
2630         s->attrib |= (var->s & 1) << SVM_SELECTOR_S_SHIFT;
2631         s->attrib |= (var->dpl & 3) << SVM_SELECTOR_DPL_SHIFT;
2632         s->attrib |= ((var->present & 1) && !var->unusable) << SVM_SELECTOR_P_SHIFT;
2633         s->attrib |= (var->avl & 1) << SVM_SELECTOR_AVL_SHIFT;
2634         s->attrib |= (var->l & 1) << SVM_SELECTOR_L_SHIFT;
2635         s->attrib |= (var->db & 1) << SVM_SELECTOR_DB_SHIFT;
2636         s->attrib |= (var->g & 1) << SVM_SELECTOR_G_SHIFT;
2637
2638         /*
2639          * This is always accurate, except if SYSRET returned to a segment
2640          * with SS.DPL != 3.  Intel does not have this quirk, and always
2641          * forces SS.DPL to 3 on sysret, so we ignore that case; fixing it
2642          * would entail passing the CPL to userspace and back.
2643          */
2644         if (seg == VCPU_SREG_SS)
2645                 /* This is symmetric with svm_get_segment() */
2646                 svm->vmcb->save.cpl = (var->dpl & 3);
2647
2648         mark_dirty(svm->vmcb, VMCB_SEG);
2649 }
2650
2651 static void update_bp_intercept(struct kvm_vcpu *vcpu)
2652 {
2653         struct vcpu_svm *svm = to_svm(vcpu);
2654
2655         clr_exception_intercept(svm, BP_VECTOR);
2656
2657         if (vcpu->guest_debug & KVM_GUESTDBG_ENABLE) {
2658                 if (vcpu->guest_debug & KVM_GUESTDBG_USE_SW_BP)
2659                         set_exception_intercept(svm, BP_VECTOR);
2660         } else
2661                 vcpu->guest_debug = 0;
2662 }
2663
2664 static void new_asid(struct vcpu_svm *svm, struct svm_cpu_data *sd)
2665 {
2666         if (sd->next_asid > sd->max_asid) {
2667                 ++sd->asid_generation;
2668                 sd->next_asid = sd->min_asid;
2669                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ALL_ASID;
2670         }
2671
2672         svm->asid_generation = sd->asid_generation;
2673         svm->vmcb->control.asid = sd->next_asid++;
2674
2675         mark_dirty(svm->vmcb, VMCB_ASID);
2676 }
2677
2678 static u64 svm_get_dr6(struct kvm_vcpu *vcpu)
2679 {
2680         return to_svm(vcpu)->vmcb->save.dr6;
2681 }
2682
2683 static void svm_set_dr6(struct kvm_vcpu *vcpu, unsigned long value)
2684 {
2685         struct vcpu_svm *svm = to_svm(vcpu);
2686
2687         svm->vmcb->save.dr6 = value;
2688         mark_dirty(svm->vmcb, VMCB_DR);
2689 }
2690
2691 static void svm_sync_dirty_debug_regs(struct kvm_vcpu *vcpu)
2692 {
2693         struct vcpu_svm *svm = to_svm(vcpu);
2694
2695         get_debugreg(vcpu->arch.db[0], 0);
2696         get_debugreg(vcpu->arch.db[1], 1);
2697         get_debugreg(vcpu->arch.db[2], 2);
2698         get_debugreg(vcpu->arch.db[3], 3);
2699         vcpu->arch.dr6 = svm_get_dr6(vcpu);
2700         vcpu->arch.dr7 = svm->vmcb->save.dr7;
2701
2702         vcpu->arch.switch_db_regs &= ~KVM_DEBUGREG_WONT_EXIT;
2703         set_dr_intercepts(svm);
2704 }
2705
2706 static void svm_set_dr7(struct kvm_vcpu *vcpu, unsigned long value)
2707 {
2708         struct vcpu_svm *svm = to_svm(vcpu);
2709
2710         svm->vmcb->save.dr7 = value;
2711         mark_dirty(svm->vmcb, VMCB_DR);
2712 }
2713
2714 static int pf_interception(struct vcpu_svm *svm)
2715 {
2716         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2717         u64 error_code = svm->vmcb->control.exit_info_1;
2718
2719         return kvm_handle_page_fault(&svm->vcpu, error_code, fault_address,
2720                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2721                         svm->vmcb->control.insn_bytes : NULL,
2722                         svm->vmcb->control.insn_len);
2723 }
2724
2725 static int npf_interception(struct vcpu_svm *svm)
2726 {
2727         u64 fault_address = __sme_clr(svm->vmcb->control.exit_info_2);
2728         u64 error_code = svm->vmcb->control.exit_info_1;
2729
2730         trace_kvm_page_fault(fault_address, error_code);
2731         return kvm_mmu_page_fault(&svm->vcpu, fault_address, error_code,
2732                         static_cpu_has(X86_FEATURE_DECODEASSISTS) ?
2733                         svm->vmcb->control.insn_bytes : NULL,
2734                         svm->vmcb->control.insn_len);
2735 }
2736
2737 static int db_interception(struct vcpu_svm *svm)
2738 {
2739         struct kvm_run *kvm_run = svm->vcpu.run;
2740         struct kvm_vcpu *vcpu = &svm->vcpu;
2741
2742         if (!(svm->vcpu.guest_debug &
2743               (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) &&
2744                 !svm->nmi_singlestep) {
2745                 kvm_queue_exception(&svm->vcpu, DB_VECTOR);
2746                 return 1;
2747         }
2748
2749         if (svm->nmi_singlestep) {
2750                 disable_nmi_singlestep(svm);
2751                 /* Make sure we check for pending NMIs upon entry */
2752                 kvm_make_request(KVM_REQ_EVENT, vcpu);
2753         }
2754
2755         if (svm->vcpu.guest_debug &
2756             (KVM_GUESTDBG_SINGLESTEP | KVM_GUESTDBG_USE_HW_BP)) {
2757                 kvm_run->exit_reason = KVM_EXIT_DEBUG;
2758                 kvm_run->debug.arch.pc =
2759                         svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2760                 kvm_run->debug.arch.exception = DB_VECTOR;
2761                 return 0;
2762         }
2763
2764         return 1;
2765 }
2766
2767 static int bp_interception(struct vcpu_svm *svm)
2768 {
2769         struct kvm_run *kvm_run = svm->vcpu.run;
2770
2771         kvm_run->exit_reason = KVM_EXIT_DEBUG;
2772         kvm_run->debug.arch.pc = svm->vmcb->save.cs.base + svm->vmcb->save.rip;
2773         kvm_run->debug.arch.exception = BP_VECTOR;
2774         return 0;
2775 }
2776
2777 static int ud_interception(struct vcpu_svm *svm)
2778 {
2779         return handle_ud(&svm->vcpu);
2780 }
2781
2782 static int ac_interception(struct vcpu_svm *svm)
2783 {
2784         kvm_queue_exception_e(&svm->vcpu, AC_VECTOR, 0);
2785         return 1;
2786 }
2787
2788 static int gp_interception(struct vcpu_svm *svm)
2789 {
2790         struct kvm_vcpu *vcpu = &svm->vcpu;
2791         u32 error_code = svm->vmcb->control.exit_info_1;
2792
2793         WARN_ON_ONCE(!enable_vmware_backdoor);
2794
2795         /*
2796          * VMware backdoor emulation on #GP interception only handles IN{S},
2797          * OUT{S}, and RDPMC, none of which generate a non-zero error code.
2798          */
2799         if (error_code) {
2800                 kvm_queue_exception_e(vcpu, GP_VECTOR, error_code);
2801                 return 1;
2802         }
2803         return kvm_emulate_instruction(vcpu, EMULTYPE_VMWARE_GP);
2804 }
2805
2806 static bool is_erratum_383(void)
2807 {
2808         int err, i;
2809         u64 value;
2810
2811         if (!erratum_383_found)
2812                 return false;
2813
2814         value = native_read_msr_safe(MSR_IA32_MC0_STATUS, &err);
2815         if (err)
2816                 return false;
2817
2818         /* Bit 62 may or may not be set for this mce */
2819         value &= ~(1ULL << 62);
2820
2821         if (value != 0xb600000000010015ULL)
2822                 return false;
2823
2824         /* Clear MCi_STATUS registers */
2825         for (i = 0; i < 6; ++i)
2826                 native_write_msr_safe(MSR_IA32_MCx_STATUS(i), 0, 0);
2827
2828         value = native_read_msr_safe(MSR_IA32_MCG_STATUS, &err);
2829         if (!err) {
2830                 u32 low, high;
2831
2832                 value &= ~(1ULL << 2);
2833                 low    = lower_32_bits(value);
2834                 high   = upper_32_bits(value);
2835
2836                 native_write_msr_safe(MSR_IA32_MCG_STATUS, low, high);
2837         }
2838
2839         /* Flush tlb to evict multi-match entries */
2840         __flush_tlb_all();
2841
2842         return true;
2843 }
2844
2845 static void svm_handle_mce(struct vcpu_svm *svm)
2846 {
2847         if (is_erratum_383()) {
2848                 /*
2849                  * Erratum 383 triggered. Guest state is corrupt so kill the
2850                  * guest.
2851                  */
2852                 pr_err("KVM: Guest triggered AMD Erratum 383\n");
2853
2854                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, &svm->vcpu);
2855
2856                 return;
2857         }
2858
2859         /*
2860          * On an #MC intercept the MCE handler is not called automatically in
2861          * the host. So do it by hand here.
2862          */
2863         asm volatile (
2864                 "int $0x12\n");
2865         /* not sure if we ever come back to this point */
2866
2867         return;
2868 }
2869
2870 static int mc_interception(struct vcpu_svm *svm)
2871 {
2872         return 1;
2873 }
2874
2875 static int shutdown_interception(struct vcpu_svm *svm)
2876 {
2877         struct kvm_run *kvm_run = svm->vcpu.run;
2878
2879         /*
2880          * VMCB is undefined after a SHUTDOWN intercept
2881          * so reinitialize it.
2882          */
2883         clear_page(svm->vmcb);
2884         init_vmcb(svm);
2885
2886         kvm_run->exit_reason = KVM_EXIT_SHUTDOWN;
2887         return 0;
2888 }
2889
2890 static int io_interception(struct vcpu_svm *svm)
2891 {
2892         struct kvm_vcpu *vcpu = &svm->vcpu;
2893         u32 io_info = svm->vmcb->control.exit_info_1; /* address size bug? */
2894         int size, in, string;
2895         unsigned port;
2896
2897         ++svm->vcpu.stat.io_exits;
2898         string = (io_info & SVM_IOIO_STR_MASK) != 0;
2899         in = (io_info & SVM_IOIO_TYPE_MASK) != 0;
2900         if (string)
2901                 return kvm_emulate_instruction(vcpu, 0);
2902
2903         port = io_info >> 16;
2904         size = (io_info & SVM_IOIO_SIZE_MASK) >> SVM_IOIO_SIZE_SHIFT;
2905         svm->next_rip = svm->vmcb->control.exit_info_2;
2906
2907         return kvm_fast_pio(&svm->vcpu, size, port, in);
2908 }
2909
2910 static int nmi_interception(struct vcpu_svm *svm)
2911 {
2912         return 1;
2913 }
2914
2915 static int intr_interception(struct vcpu_svm *svm)
2916 {
2917         ++svm->vcpu.stat.irq_exits;
2918         return 1;
2919 }
2920
2921 static int nop_on_interception(struct vcpu_svm *svm)
2922 {
2923         return 1;
2924 }
2925
2926 static int halt_interception(struct vcpu_svm *svm)
2927 {
2928         return kvm_emulate_halt(&svm->vcpu);
2929 }
2930
2931 static int vmmcall_interception(struct vcpu_svm *svm)
2932 {
2933         return kvm_emulate_hypercall(&svm->vcpu);
2934 }
2935
2936 static unsigned long nested_svm_get_tdp_cr3(struct kvm_vcpu *vcpu)
2937 {
2938         struct vcpu_svm *svm = to_svm(vcpu);
2939
2940         return svm->nested.nested_cr3;
2941 }
2942
2943 static u64 nested_svm_get_tdp_pdptr(struct kvm_vcpu *vcpu, int index)
2944 {
2945         struct vcpu_svm *svm = to_svm(vcpu);
2946         u64 cr3 = svm->nested.nested_cr3;
2947         u64 pdpte;
2948         int ret;
2949
2950         ret = kvm_vcpu_read_guest_page(vcpu, gpa_to_gfn(__sme_clr(cr3)), &pdpte,
2951                                        offset_in_page(cr3) + index * 8, 8);
2952         if (ret)
2953                 return 0;
2954         return pdpte;
2955 }
2956
2957 static void nested_svm_set_tdp_cr3(struct kvm_vcpu *vcpu,
2958                                    unsigned long root)
2959 {
2960         struct vcpu_svm *svm = to_svm(vcpu);
2961
2962         svm->vmcb->control.nested_cr3 = __sme_set(root);
2963         mark_dirty(svm->vmcb, VMCB_NPT);
2964 }
2965
2966 static void nested_svm_inject_npf_exit(struct kvm_vcpu *vcpu,
2967                                        struct x86_exception *fault)
2968 {
2969         struct vcpu_svm *svm = to_svm(vcpu);
2970
2971         if (svm->vmcb->control.exit_code != SVM_EXIT_NPF) {
2972                 /*
2973                  * TODO: track the cause of the nested page fault, and
2974                  * correctly fill in the high bits of exit_info_1.
2975                  */
2976                 svm->vmcb->control.exit_code = SVM_EXIT_NPF;
2977                 svm->vmcb->control.exit_code_hi = 0;
2978                 svm->vmcb->control.exit_info_1 = (1ULL << 32);
2979                 svm->vmcb->control.exit_info_2 = fault->address;
2980         }
2981
2982         svm->vmcb->control.exit_info_1 &= ~0xffffffffULL;
2983         svm->vmcb->control.exit_info_1 |= fault->error_code;
2984
2985         /*
2986          * The present bit is always zero for page structure faults on real
2987          * hardware.
2988          */
2989         if (svm->vmcb->control.exit_info_1 & (2ULL << 32))
2990                 svm->vmcb->control.exit_info_1 &= ~1;
2991
2992         nested_svm_vmexit(svm);
2993 }
2994
2995 static void nested_svm_init_mmu_context(struct kvm_vcpu *vcpu)
2996 {
2997         WARN_ON(mmu_is_nested(vcpu));
2998
2999         vcpu->arch.mmu = &vcpu->arch.guest_mmu;
3000         kvm_init_shadow_mmu(vcpu);
3001         vcpu->arch.mmu->set_cr3           = nested_svm_set_tdp_cr3;
3002         vcpu->arch.mmu->get_cr3           = nested_svm_get_tdp_cr3;
3003         vcpu->arch.mmu->get_pdptr         = nested_svm_get_tdp_pdptr;
3004         vcpu->arch.mmu->inject_page_fault = nested_svm_inject_npf_exit;
3005         vcpu->arch.mmu->shadow_root_level = get_npt_level(vcpu);
3006         reset_shadow_zero_bits_mask(vcpu, vcpu->arch.mmu);
3007         vcpu->arch.walk_mmu              = &vcpu->arch.nested_mmu;
3008 }
3009
3010 static void nested_svm_uninit_mmu_context(struct kvm_vcpu *vcpu)
3011 {
3012         vcpu->arch.mmu = &vcpu->arch.root_mmu;
3013         vcpu->arch.walk_mmu = &vcpu->arch.root_mmu;
3014 }
3015
3016 static int nested_svm_check_permissions(struct vcpu_svm *svm)
3017 {
3018         if (!(svm->vcpu.arch.efer & EFER_SVME) ||
3019             !is_paging(&svm->vcpu)) {
3020                 kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3021                 return 1;
3022         }
3023
3024         if (svm->vmcb->save.cpl) {
3025                 kvm_inject_gp(&svm->vcpu, 0);
3026                 return 1;
3027         }
3028
3029         return 0;
3030 }
3031
3032 static int nested_svm_check_exception(struct vcpu_svm *svm, unsigned nr,
3033                                       bool has_error_code, u32 error_code)
3034 {
3035         int vmexit;
3036
3037         if (!is_guest_mode(&svm->vcpu))
3038                 return 0;
3039
3040         vmexit = nested_svm_intercept(svm);
3041         if (vmexit != NESTED_EXIT_DONE)
3042                 return 0;
3043
3044         svm->vmcb->control.exit_code = SVM_EXIT_EXCP_BASE + nr;
3045         svm->vmcb->control.exit_code_hi = 0;
3046         svm->vmcb->control.exit_info_1 = error_code;
3047
3048         /*
3049          * EXITINFO2 is undefined for all exception intercepts other
3050          * than #PF.
3051          */
3052         if (svm->vcpu.arch.exception.nested_apf)
3053                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.apf.nested_apf_token;
3054         else if (svm->vcpu.arch.exception.has_payload)
3055                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.exception.payload;
3056         else
3057                 svm->vmcb->control.exit_info_2 = svm->vcpu.arch.cr2;
3058
3059         svm->nested.exit_required = true;
3060         return vmexit;
3061 }
3062
3063 /* This function returns true if it is save to enable the irq window */
3064 static inline bool nested_svm_intr(struct vcpu_svm *svm)
3065 {
3066         if (!is_guest_mode(&svm->vcpu))
3067                 return true;
3068
3069         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3070                 return true;
3071
3072         if (!(svm->vcpu.arch.hflags & HF_HIF_MASK))
3073                 return false;
3074
3075         /*
3076          * if vmexit was already requested (by intercepted exception
3077          * for instance) do not overwrite it with "external interrupt"
3078          * vmexit.
3079          */
3080         if (svm->nested.exit_required)
3081                 return false;
3082
3083         svm->vmcb->control.exit_code   = SVM_EXIT_INTR;
3084         svm->vmcb->control.exit_info_1 = 0;
3085         svm->vmcb->control.exit_info_2 = 0;
3086
3087         if (svm->nested.intercept & 1ULL) {
3088                 /*
3089                  * The #vmexit can't be emulated here directly because this
3090                  * code path runs with irqs and preemption disabled. A
3091                  * #vmexit emulation might sleep. Only signal request for
3092                  * the #vmexit here.
3093                  */
3094                 svm->nested.exit_required = true;
3095                 trace_kvm_nested_intr_vmexit(svm->vmcb->save.rip);
3096                 return false;
3097         }
3098
3099         return true;
3100 }
3101
3102 /* This function returns true if it is save to enable the nmi window */
3103 static inline bool nested_svm_nmi(struct vcpu_svm *svm)
3104 {
3105         if (!is_guest_mode(&svm->vcpu))
3106                 return true;
3107
3108         if (!(svm->nested.intercept & (1ULL << INTERCEPT_NMI)))
3109                 return true;
3110
3111         svm->vmcb->control.exit_code = SVM_EXIT_NMI;
3112         svm->nested.exit_required = true;
3113
3114         return false;
3115 }
3116
3117 static int nested_svm_intercept_ioio(struct vcpu_svm *svm)
3118 {
3119         unsigned port, size, iopm_len;
3120         u16 val, mask;
3121         u8 start_bit;
3122         u64 gpa;
3123
3124         if (!(svm->nested.intercept & (1ULL << INTERCEPT_IOIO_PROT)))
3125                 return NESTED_EXIT_HOST;
3126
3127         port = svm->vmcb->control.exit_info_1 >> 16;
3128         size = (svm->vmcb->control.exit_info_1 & SVM_IOIO_SIZE_MASK) >>
3129                 SVM_IOIO_SIZE_SHIFT;
3130         gpa  = svm->nested.vmcb_iopm + (port / 8);
3131         start_bit = port % 8;
3132         iopm_len = (start_bit + size > 8) ? 2 : 1;
3133         mask = (0xf >> (4 - size)) << start_bit;
3134         val = 0;
3135
3136         if (kvm_vcpu_read_guest(&svm->vcpu, gpa, &val, iopm_len))
3137                 return NESTED_EXIT_DONE;
3138
3139         return (val & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3140 }
3141
3142 static int nested_svm_exit_handled_msr(struct vcpu_svm *svm)
3143 {
3144         u32 offset, msr, value;
3145         int write, mask;
3146
3147         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3148                 return NESTED_EXIT_HOST;
3149
3150         msr    = svm->vcpu.arch.regs[VCPU_REGS_RCX];
3151         offset = svm_msrpm_offset(msr);
3152         write  = svm->vmcb->control.exit_info_1 & 1;
3153         mask   = 1 << ((2 * (msr & 0xf)) + write);
3154
3155         if (offset == MSR_INVALID)
3156                 return NESTED_EXIT_DONE;
3157
3158         /* Offset is in 32 bit units but need in 8 bit units */
3159         offset *= 4;
3160
3161         if (kvm_vcpu_read_guest(&svm->vcpu, svm->nested.vmcb_msrpm + offset, &value, 4))
3162                 return NESTED_EXIT_DONE;
3163
3164         return (value & mask) ? NESTED_EXIT_DONE : NESTED_EXIT_HOST;
3165 }
3166
3167 /* DB exceptions for our internal use must not cause vmexit */
3168 static int nested_svm_intercept_db(struct vcpu_svm *svm)
3169 {
3170         unsigned long dr6;
3171
3172         /* if we're not singlestepping, it's not ours */
3173         if (!svm->nmi_singlestep)
3174                 return NESTED_EXIT_DONE;
3175
3176         /* if it's not a singlestep exception, it's not ours */
3177         if (kvm_get_dr(&svm->vcpu, 6, &dr6))
3178                 return NESTED_EXIT_DONE;
3179         if (!(dr6 & DR6_BS))
3180                 return NESTED_EXIT_DONE;
3181
3182         /* if the guest is singlestepping, it should get the vmexit */
3183         if (svm->nmi_singlestep_guest_rflags & X86_EFLAGS_TF) {
3184                 disable_nmi_singlestep(svm);
3185                 return NESTED_EXIT_DONE;
3186         }
3187
3188         /* it's ours, the nested hypervisor must not see this one */
3189         return NESTED_EXIT_HOST;
3190 }
3191
3192 static int nested_svm_exit_special(struct vcpu_svm *svm)
3193 {
3194         u32 exit_code = svm->vmcb->control.exit_code;
3195
3196         switch (exit_code) {
3197         case SVM_EXIT_INTR:
3198         case SVM_EXIT_NMI:
3199         case SVM_EXIT_EXCP_BASE + MC_VECTOR:
3200                 return NESTED_EXIT_HOST;
3201         case SVM_EXIT_NPF:
3202                 /* For now we are always handling NPFs when using them */
3203                 if (npt_enabled)
3204                         return NESTED_EXIT_HOST;
3205                 break;
3206         case SVM_EXIT_EXCP_BASE + PF_VECTOR:
3207                 /* When we're shadowing, trap PFs, but not async PF */
3208                 if (!npt_enabled && svm->vcpu.arch.apf.host_apf_reason == 0)
3209                         return NESTED_EXIT_HOST;
3210                 break;
3211         default:
3212                 break;
3213         }
3214
3215         return NESTED_EXIT_CONTINUE;
3216 }
3217
3218 /*
3219  * If this function returns true, this #vmexit was already handled
3220  */
3221 static int nested_svm_intercept(struct vcpu_svm *svm)
3222 {
3223         u32 exit_code = svm->vmcb->control.exit_code;
3224         int vmexit = NESTED_EXIT_HOST;
3225
3226         switch (exit_code) {
3227         case SVM_EXIT_MSR:
3228                 vmexit = nested_svm_exit_handled_msr(svm);
3229                 break;
3230         case SVM_EXIT_IOIO:
3231                 vmexit = nested_svm_intercept_ioio(svm);
3232                 break;
3233         case SVM_EXIT_READ_CR0 ... SVM_EXIT_WRITE_CR8: {
3234                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_CR0);
3235                 if (svm->nested.intercept_cr & bit)
3236                         vmexit = NESTED_EXIT_DONE;
3237                 break;
3238         }
3239         case SVM_EXIT_READ_DR0 ... SVM_EXIT_WRITE_DR7: {
3240                 u32 bit = 1U << (exit_code - SVM_EXIT_READ_DR0);
3241                 if (svm->nested.intercept_dr & bit)
3242                         vmexit = NESTED_EXIT_DONE;
3243                 break;
3244         }
3245         case SVM_EXIT_EXCP_BASE ... SVM_EXIT_EXCP_BASE + 0x1f: {
3246                 u32 excp_bits = 1 << (exit_code - SVM_EXIT_EXCP_BASE);
3247                 if (svm->nested.intercept_exceptions & excp_bits) {
3248                         if (exit_code == SVM_EXIT_EXCP_BASE + DB_VECTOR)
3249                                 vmexit = nested_svm_intercept_db(svm);
3250                         else
3251                                 vmexit = NESTED_EXIT_DONE;
3252                 }
3253                 /* async page fault always cause vmexit */
3254                 else if ((exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR) &&
3255                          svm->vcpu.arch.exception.nested_apf != 0)
3256                         vmexit = NESTED_EXIT_DONE;
3257                 break;
3258         }
3259         case SVM_EXIT_ERR: {
3260                 vmexit = NESTED_EXIT_DONE;
3261                 break;
3262         }
3263         default: {
3264                 u64 exit_bits = 1ULL << (exit_code - SVM_EXIT_INTR);
3265                 if (svm->nested.intercept & exit_bits)
3266                         vmexit = NESTED_EXIT_DONE;
3267         }
3268         }
3269
3270         return vmexit;
3271 }
3272
3273 static int nested_svm_exit_handled(struct vcpu_svm *svm)
3274 {
3275         int vmexit;
3276
3277         vmexit = nested_svm_intercept(svm);
3278
3279         if (vmexit == NESTED_EXIT_DONE)
3280                 nested_svm_vmexit(svm);
3281
3282         return vmexit;
3283 }
3284
3285 static inline void copy_vmcb_control_area(struct vmcb *dst_vmcb, struct vmcb *from_vmcb)
3286 {
3287         struct vmcb_control_area *dst  = &dst_vmcb->control;
3288         struct vmcb_control_area *from = &from_vmcb->control;
3289
3290         dst->intercept_cr         = from->intercept_cr;
3291         dst->intercept_dr         = from->intercept_dr;
3292         dst->intercept_exceptions = from->intercept_exceptions;
3293         dst->intercept            = from->intercept;
3294         dst->iopm_base_pa         = from->iopm_base_pa;
3295         dst->msrpm_base_pa        = from->msrpm_base_pa;
3296         dst->tsc_offset           = from->tsc_offset;
3297         dst->asid                 = from->asid;
3298         dst->tlb_ctl              = from->tlb_ctl;
3299         dst->int_ctl              = from->int_ctl;
3300         dst->int_vector           = from->int_vector;
3301         dst->int_state            = from->int_state;
3302         dst->exit_code            = from->exit_code;
3303         dst->exit_code_hi         = from->exit_code_hi;
3304         dst->exit_info_1          = from->exit_info_1;
3305         dst->exit_info_2          = from->exit_info_2;
3306         dst->exit_int_info        = from->exit_int_info;
3307         dst->exit_int_info_err    = from->exit_int_info_err;
3308         dst->nested_ctl           = from->nested_ctl;
3309         dst->event_inj            = from->event_inj;
3310         dst->event_inj_err        = from->event_inj_err;
3311         dst->nested_cr3           = from->nested_cr3;
3312         dst->virt_ext              = from->virt_ext;
3313         dst->pause_filter_count   = from->pause_filter_count;
3314         dst->pause_filter_thresh  = from->pause_filter_thresh;
3315 }
3316
3317 static int nested_svm_vmexit(struct vcpu_svm *svm)
3318 {
3319         int rc;
3320         struct vmcb *nested_vmcb;
3321         struct vmcb *hsave = svm->nested.hsave;
3322         struct vmcb *vmcb = svm->vmcb;
3323         struct kvm_host_map map;
3324
3325         trace_kvm_nested_vmexit_inject(vmcb->control.exit_code,
3326                                        vmcb->control.exit_info_1,
3327                                        vmcb->control.exit_info_2,
3328                                        vmcb->control.exit_int_info,
3329                                        vmcb->control.exit_int_info_err,
3330                                        KVM_ISA_SVM);
3331
3332         rc = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->nested.vmcb), &map);
3333         if (rc) {
3334                 if (rc == -EINVAL)
3335                         kvm_inject_gp(&svm->vcpu, 0);
3336                 return 1;
3337         }
3338
3339         nested_vmcb = map.hva;
3340
3341         /* Exit Guest-Mode */
3342         leave_guest_mode(&svm->vcpu);
3343         svm->nested.vmcb = 0;
3344
3345         /* Give the current vmcb to the guest */
3346         disable_gif(svm);
3347
3348         nested_vmcb->save.es     = vmcb->save.es;
3349         nested_vmcb->save.cs     = vmcb->save.cs;
3350         nested_vmcb->save.ss     = vmcb->save.ss;
3351         nested_vmcb->save.ds     = vmcb->save.ds;
3352         nested_vmcb->save.gdtr   = vmcb->save.gdtr;
3353         nested_vmcb->save.idtr   = vmcb->save.idtr;
3354         nested_vmcb->save.efer   = svm->vcpu.arch.efer;
3355         nested_vmcb->save.cr0    = kvm_read_cr0(&svm->vcpu);
3356         nested_vmcb->save.cr3    = kvm_read_cr3(&svm->vcpu);
3357         nested_vmcb->save.cr2    = vmcb->save.cr2;
3358         nested_vmcb->save.cr4    = svm->vcpu.arch.cr4;
3359         nested_vmcb->save.rflags = kvm_get_rflags(&svm->vcpu);
3360         nested_vmcb->save.rip    = vmcb->save.rip;
3361         nested_vmcb->save.rsp    = vmcb->save.rsp;
3362         nested_vmcb->save.rax    = vmcb->save.rax;
3363         nested_vmcb->save.dr7    = vmcb->save.dr7;
3364         nested_vmcb->save.dr6    = vmcb->save.dr6;
3365         nested_vmcb->save.cpl    = vmcb->save.cpl;
3366
3367         nested_vmcb->control.int_ctl           = vmcb->control.int_ctl;
3368         nested_vmcb->control.int_vector        = vmcb->control.int_vector;
3369         nested_vmcb->control.int_state         = vmcb->control.int_state;
3370         nested_vmcb->control.exit_code         = vmcb->control.exit_code;
3371         nested_vmcb->control.exit_code_hi      = vmcb->control.exit_code_hi;
3372         nested_vmcb->control.exit_info_1       = vmcb->control.exit_info_1;
3373         nested_vmcb->control.exit_info_2       = vmcb->control.exit_info_2;
3374         nested_vmcb->control.exit_int_info     = vmcb->control.exit_int_info;
3375         nested_vmcb->control.exit_int_info_err = vmcb->control.exit_int_info_err;
3376
3377         if (svm->nrips_enabled)
3378                 nested_vmcb->control.next_rip  = vmcb->control.next_rip;
3379
3380         /*
3381          * If we emulate a VMRUN/#VMEXIT in the same host #vmexit cycle we have
3382          * to make sure that we do not lose injected events. So check event_inj
3383          * here and copy it to exit_int_info if it is valid.
3384          * Exit_int_info and event_inj can't be both valid because the case
3385          * below only happens on a VMRUN instruction intercept which has
3386          * no valid exit_int_info set.
3387          */
3388         if (vmcb->control.event_inj & SVM_EVTINJ_VALID) {
3389                 struct vmcb_control_area *nc = &nested_vmcb->control;
3390
3391                 nc->exit_int_info     = vmcb->control.event_inj;
3392                 nc->exit_int_info_err = vmcb->control.event_inj_err;
3393         }
3394
3395         nested_vmcb->control.tlb_ctl           = 0;
3396         nested_vmcb->control.event_inj         = 0;
3397         nested_vmcb->control.event_inj_err     = 0;
3398
3399         nested_vmcb->control.pause_filter_count =
3400                 svm->vmcb->control.pause_filter_count;
3401         nested_vmcb->control.pause_filter_thresh =
3402                 svm->vmcb->control.pause_filter_thresh;
3403
3404         /* We always set V_INTR_MASKING and remember the old value in hflags */
3405         if (!(svm->vcpu.arch.hflags & HF_VINTR_MASK))
3406                 nested_vmcb->control.int_ctl &= ~V_INTR_MASKING_MASK;
3407
3408         /* Restore the original control entries */
3409         copy_vmcb_control_area(vmcb, hsave);
3410
3411         svm->vcpu.arch.tsc_offset = svm->vmcb->control.tsc_offset;
3412         kvm_clear_exception_queue(&svm->vcpu);
3413         kvm_clear_interrupt_queue(&svm->vcpu);
3414
3415         svm->nested.nested_cr3 = 0;
3416
3417         /* Restore selected save entries */
3418         svm->vmcb->save.es = hsave->save.es;
3419         svm->vmcb->save.cs = hsave->save.cs;
3420         svm->vmcb->save.ss = hsave->save.ss;
3421         svm->vmcb->save.ds = hsave->save.ds;
3422         svm->vmcb->save.gdtr = hsave->save.gdtr;
3423         svm->vmcb->save.idtr = hsave->save.idtr;
3424         kvm_set_rflags(&svm->vcpu, hsave->save.rflags);
3425         svm_set_efer(&svm->vcpu, hsave->save.efer);
3426         svm_set_cr0(&svm->vcpu, hsave->save.cr0 | X86_CR0_PE);
3427         svm_set_cr4(&svm->vcpu, hsave->save.cr4);
3428         if (npt_enabled) {
3429                 svm->vmcb->save.cr3 = hsave->save.cr3;
3430                 svm->vcpu.arch.cr3 = hsave->save.cr3;
3431         } else {
3432                 (void)kvm_set_cr3(&svm->vcpu, hsave->save.cr3);
3433         }
3434         kvm_rax_write(&svm->vcpu, hsave->save.rax);
3435         kvm_rsp_write(&svm->vcpu, hsave->save.rsp);
3436         kvm_rip_write(&svm->vcpu, hsave->save.rip);
3437         svm->vmcb->save.dr7 = 0;
3438         svm->vmcb->save.cpl = 0;
3439         svm->vmcb->control.exit_int_info = 0;
3440
3441         mark_all_dirty(svm->vmcb);
3442
3443         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3444
3445         nested_svm_uninit_mmu_context(&svm->vcpu);
3446         kvm_mmu_reset_context(&svm->vcpu);
3447         kvm_mmu_load(&svm->vcpu);
3448
3449         /*
3450          * Drop what we picked up for L2 via svm_complete_interrupts() so it
3451          * doesn't end up in L1.
3452          */
3453         svm->vcpu.arch.nmi_injected = false;
3454         kvm_clear_exception_queue(&svm->vcpu);
3455         kvm_clear_interrupt_queue(&svm->vcpu);
3456
3457         return 0;
3458 }
3459
3460 static bool nested_svm_vmrun_msrpm(struct vcpu_svm *svm)
3461 {
3462         /*
3463          * This function merges the msr permission bitmaps of kvm and the
3464          * nested vmcb. It is optimized in that it only merges the parts where
3465          * the kvm msr permission bitmap may contain zero bits
3466          */
3467         int i;
3468
3469         if (!(svm->nested.intercept & (1ULL << INTERCEPT_MSR_PROT)))
3470                 return true;
3471
3472         for (i = 0; i < MSRPM_OFFSETS; i++) {
3473                 u32 value, p;
3474                 u64 offset;
3475
3476                 if (msrpm_offsets[i] == 0xffffffff)
3477                         break;
3478
3479                 p      = msrpm_offsets[i];
3480                 offset = svm->nested.vmcb_msrpm + (p * 4);
3481
3482                 if (kvm_vcpu_read_guest(&svm->vcpu, offset, &value, 4))
3483                         return false;
3484
3485                 svm->nested.msrpm[p] = svm->msrpm[p] | value;
3486         }
3487
3488         svm->vmcb->control.msrpm_base_pa = __sme_set(__pa(svm->nested.msrpm));
3489
3490         return true;
3491 }
3492
3493 static bool nested_vmcb_checks(struct vmcb *vmcb)
3494 {
3495         if ((vmcb->control.intercept & (1ULL << INTERCEPT_VMRUN)) == 0)
3496                 return false;
3497
3498         if (vmcb->control.asid == 0)
3499                 return false;
3500
3501         if ((vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) &&
3502             !npt_enabled)
3503                 return false;
3504
3505         return true;
3506 }
3507
3508 static void enter_svm_guest_mode(struct vcpu_svm *svm, u64 vmcb_gpa,
3509                                  struct vmcb *nested_vmcb, struct kvm_host_map *map)
3510 {
3511         if (kvm_get_rflags(&svm->vcpu) & X86_EFLAGS_IF)
3512                 svm->vcpu.arch.hflags |= HF_HIF_MASK;
3513         else
3514                 svm->vcpu.arch.hflags &= ~HF_HIF_MASK;
3515
3516         if (nested_vmcb->control.nested_ctl & SVM_NESTED_CTL_NP_ENABLE) {
3517                 svm->nested.nested_cr3 = nested_vmcb->control.nested_cr3;
3518                 nested_svm_init_mmu_context(&svm->vcpu);
3519         }
3520
3521         /* Load the nested guest state */
3522         svm->vmcb->save.es = nested_vmcb->save.es;
3523         svm->vmcb->save.cs = nested_vmcb->save.cs;
3524         svm->vmcb->save.ss = nested_vmcb->save.ss;
3525         svm->vmcb->save.ds = nested_vmcb->save.ds;
3526         svm->vmcb->save.gdtr = nested_vmcb->save.gdtr;
3527         svm->vmcb->save.idtr = nested_vmcb->save.idtr;
3528         kvm_set_rflags(&svm->vcpu, nested_vmcb->save.rflags);
3529         svm_set_efer(&svm->vcpu, nested_vmcb->save.efer);
3530         svm_set_cr0(&svm->vcpu, nested_vmcb->save.cr0);
3531         svm_set_cr4(&svm->vcpu, nested_vmcb->save.cr4);
3532         if (npt_enabled) {
3533                 svm->vmcb->save.cr3 = nested_vmcb->save.cr3;
3534                 svm->vcpu.arch.cr3 = nested_vmcb->save.cr3;
3535         } else
3536                 (void)kvm_set_cr3(&svm->vcpu, nested_vmcb->save.cr3);
3537
3538         /* Guest paging mode is active - reset mmu */
3539         kvm_mmu_reset_context(&svm->vcpu);
3540
3541         svm->vmcb->save.cr2 = svm->vcpu.arch.cr2 = nested_vmcb->save.cr2;
3542         kvm_rax_write(&svm->vcpu, nested_vmcb->save.rax);
3543         kvm_rsp_write(&svm->vcpu, nested_vmcb->save.rsp);
3544         kvm_rip_write(&svm->vcpu, nested_vmcb->save.rip);
3545
3546         /* In case we don't even reach vcpu_run, the fields are not updated */
3547         svm->vmcb->save.rax = nested_vmcb->save.rax;
3548         svm->vmcb->save.rsp = nested_vmcb->save.rsp;
3549         svm->vmcb->save.rip = nested_vmcb->save.rip;
3550         svm->vmcb->save.dr7 = nested_vmcb->save.dr7;
3551         svm->vmcb->save.dr6 = nested_vmcb->save.dr6;
3552         svm->vmcb->save.cpl = nested_vmcb->save.cpl;
3553
3554         svm->nested.vmcb_msrpm = nested_vmcb->control.msrpm_base_pa & ~0x0fffULL;
3555         svm->nested.vmcb_iopm  = nested_vmcb->control.iopm_base_pa  & ~0x0fffULL;
3556
3557         /* cache intercepts */
3558         svm->nested.intercept_cr         = nested_vmcb->control.intercept_cr;
3559         svm->nested.intercept_dr         = nested_vmcb->control.intercept_dr;
3560         svm->nested.intercept_exceptions = nested_vmcb->control.intercept_exceptions;
3561         svm->nested.intercept            = nested_vmcb->control.intercept;
3562
3563         svm_flush_tlb(&svm->vcpu, true);
3564         svm->vmcb->control.int_ctl = nested_vmcb->control.int_ctl | V_INTR_MASKING_MASK;
3565         if (nested_vmcb->control.int_ctl & V_INTR_MASKING_MASK)
3566                 svm->vcpu.arch.hflags |= HF_VINTR_MASK;
3567         else
3568                 svm->vcpu.arch.hflags &= ~HF_VINTR_MASK;
3569
3570         if (svm->vcpu.arch.hflags & HF_VINTR_MASK) {
3571                 /* We only want the cr8 intercept bits of the guest */
3572                 clr_cr_intercept(svm, INTERCEPT_CR8_READ);
3573                 clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
3574         }
3575
3576         /* We don't want to see VMMCALLs from a nested guest */
3577         clr_intercept(svm, INTERCEPT_VMMCALL);
3578
3579         svm->vcpu.arch.tsc_offset += nested_vmcb->control.tsc_offset;
3580         svm->vmcb->control.tsc_offset = svm->vcpu.arch.tsc_offset;
3581
3582         svm->vmcb->control.virt_ext = nested_vmcb->control.virt_ext;
3583         svm->vmcb->control.int_vector = nested_vmcb->control.int_vector;
3584         svm->vmcb->control.int_state = nested_vmcb->control.int_state;
3585         svm->vmcb->control.event_inj = nested_vmcb->control.event_inj;
3586         svm->vmcb->control.event_inj_err = nested_vmcb->control.event_inj_err;
3587
3588         svm->vmcb->control.pause_filter_count =
3589                 nested_vmcb->control.pause_filter_count;
3590         svm->vmcb->control.pause_filter_thresh =
3591                 nested_vmcb->control.pause_filter_thresh;
3592
3593         kvm_vcpu_unmap(&svm->vcpu, map, true);
3594
3595         /* Enter Guest-Mode */
3596         enter_guest_mode(&svm->vcpu);
3597
3598         /*
3599          * Merge guest and host intercepts - must be called  with vcpu in
3600          * guest-mode to take affect here
3601          */
3602         recalc_intercepts(svm);
3603
3604         svm->nested.vmcb = vmcb_gpa;
3605
3606         enable_gif(svm);
3607
3608         mark_all_dirty(svm->vmcb);
3609 }
3610
3611 static int nested_svm_vmrun(struct vcpu_svm *svm)
3612 {
3613         int ret;
3614         struct vmcb *nested_vmcb;
3615         struct vmcb *hsave = svm->nested.hsave;
3616         struct vmcb *vmcb = svm->vmcb;
3617         struct kvm_host_map map;
3618         u64 vmcb_gpa;
3619
3620         vmcb_gpa = svm->vmcb->save.rax;
3621
3622         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb_gpa), &map);
3623         if (ret == -EINVAL) {
3624                 kvm_inject_gp(&svm->vcpu, 0);
3625                 return 1;
3626         } else if (ret) {
3627                 return kvm_skip_emulated_instruction(&svm->vcpu);
3628         }
3629
3630         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3631
3632         nested_vmcb = map.hva;
3633
3634         if (!nested_vmcb_checks(nested_vmcb)) {
3635                 nested_vmcb->control.exit_code    = SVM_EXIT_ERR;
3636                 nested_vmcb->control.exit_code_hi = 0;
3637                 nested_vmcb->control.exit_info_1  = 0;
3638                 nested_vmcb->control.exit_info_2  = 0;
3639
3640                 kvm_vcpu_unmap(&svm->vcpu, &map, true);
3641
3642                 return ret;
3643         }
3644
3645         trace_kvm_nested_vmrun(svm->vmcb->save.rip, vmcb_gpa,
3646                                nested_vmcb->save.rip,
3647                                nested_vmcb->control.int_ctl,
3648                                nested_vmcb->control.event_inj,
3649                                nested_vmcb->control.nested_ctl);
3650
3651         trace_kvm_nested_intercepts(nested_vmcb->control.intercept_cr & 0xffff,
3652                                     nested_vmcb->control.intercept_cr >> 16,
3653                                     nested_vmcb->control.intercept_exceptions,
3654                                     nested_vmcb->control.intercept);
3655
3656         /* Clear internal status */
3657         kvm_clear_exception_queue(&svm->vcpu);
3658         kvm_clear_interrupt_queue(&svm->vcpu);
3659
3660         /*
3661          * Save the old vmcb, so we don't need to pick what we save, but can
3662          * restore everything when a VMEXIT occurs
3663          */
3664         hsave->save.es     = vmcb->save.es;
3665         hsave->save.cs     = vmcb->save.cs;
3666         hsave->save.ss     = vmcb->save.ss;
3667         hsave->save.ds     = vmcb->save.ds;
3668         hsave->save.gdtr   = vmcb->save.gdtr;
3669         hsave->save.idtr   = vmcb->save.idtr;
3670         hsave->save.efer   = svm->vcpu.arch.efer;
3671         hsave->save.cr0    = kvm_read_cr0(&svm->vcpu);
3672         hsave->save.cr4    = svm->vcpu.arch.cr4;
3673         hsave->save.rflags = kvm_get_rflags(&svm->vcpu);
3674         hsave->save.rip    = kvm_rip_read(&svm->vcpu);
3675         hsave->save.rsp    = vmcb->save.rsp;
3676         hsave->save.rax    = vmcb->save.rax;
3677         if (npt_enabled)
3678                 hsave->save.cr3    = vmcb->save.cr3;
3679         else
3680                 hsave->save.cr3    = kvm_read_cr3(&svm->vcpu);
3681
3682         copy_vmcb_control_area(hsave, vmcb);
3683
3684         enter_svm_guest_mode(svm, vmcb_gpa, nested_vmcb, &map);
3685
3686         if (!nested_svm_vmrun_msrpm(svm)) {
3687                 svm->vmcb->control.exit_code    = SVM_EXIT_ERR;
3688                 svm->vmcb->control.exit_code_hi = 0;
3689                 svm->vmcb->control.exit_info_1  = 0;
3690                 svm->vmcb->control.exit_info_2  = 0;
3691
3692                 nested_svm_vmexit(svm);
3693         }
3694
3695         return ret;
3696 }
3697
3698 static void nested_svm_vmloadsave(struct vmcb *from_vmcb, struct vmcb *to_vmcb)
3699 {
3700         to_vmcb->save.fs = from_vmcb->save.fs;
3701         to_vmcb->save.gs = from_vmcb->save.gs;
3702         to_vmcb->save.tr = from_vmcb->save.tr;
3703         to_vmcb->save.ldtr = from_vmcb->save.ldtr;
3704         to_vmcb->save.kernel_gs_base = from_vmcb->save.kernel_gs_base;
3705         to_vmcb->save.star = from_vmcb->save.star;
3706         to_vmcb->save.lstar = from_vmcb->save.lstar;
3707         to_vmcb->save.cstar = from_vmcb->save.cstar;
3708         to_vmcb->save.sfmask = from_vmcb->save.sfmask;
3709         to_vmcb->save.sysenter_cs = from_vmcb->save.sysenter_cs;
3710         to_vmcb->save.sysenter_esp = from_vmcb->save.sysenter_esp;
3711         to_vmcb->save.sysenter_eip = from_vmcb->save.sysenter_eip;
3712 }
3713
3714 static int vmload_interception(struct vcpu_svm *svm)
3715 {
3716         struct vmcb *nested_vmcb;
3717         struct kvm_host_map map;
3718         int ret;
3719
3720         if (nested_svm_check_permissions(svm))
3721                 return 1;
3722
3723         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3724         if (ret) {
3725                 if (ret == -EINVAL)
3726                         kvm_inject_gp(&svm->vcpu, 0);
3727                 return 1;
3728         }
3729
3730         nested_vmcb = map.hva;
3731
3732         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3733
3734         nested_svm_vmloadsave(nested_vmcb, svm->vmcb);
3735         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3736
3737         return ret;
3738 }
3739
3740 static int vmsave_interception(struct vcpu_svm *svm)
3741 {
3742         struct vmcb *nested_vmcb;
3743         struct kvm_host_map map;
3744         int ret;
3745
3746         if (nested_svm_check_permissions(svm))
3747                 return 1;
3748
3749         ret = kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(svm->vmcb->save.rax), &map);
3750         if (ret) {
3751                 if (ret == -EINVAL)
3752                         kvm_inject_gp(&svm->vcpu, 0);
3753                 return 1;
3754         }
3755
3756         nested_vmcb = map.hva;
3757
3758         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3759
3760         nested_svm_vmloadsave(svm->vmcb, nested_vmcb);
3761         kvm_vcpu_unmap(&svm->vcpu, &map, true);
3762
3763         return ret;
3764 }
3765
3766 static int vmrun_interception(struct vcpu_svm *svm)
3767 {
3768         if (nested_svm_check_permissions(svm))
3769                 return 1;
3770
3771         return nested_svm_vmrun(svm);
3772 }
3773
3774 static int stgi_interception(struct vcpu_svm *svm)
3775 {
3776         int ret;
3777
3778         if (nested_svm_check_permissions(svm))
3779                 return 1;
3780
3781         /*
3782          * If VGIF is enabled, the STGI intercept is only added to
3783          * detect the opening of the SMI/NMI window; remove it now.
3784          */
3785         if (vgif_enabled(svm))
3786                 clr_intercept(svm, INTERCEPT_STGI);
3787
3788         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3789         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3790
3791         enable_gif(svm);
3792
3793         return ret;
3794 }
3795
3796 static int clgi_interception(struct vcpu_svm *svm)
3797 {
3798         int ret;
3799
3800         if (nested_svm_check_permissions(svm))
3801                 return 1;
3802
3803         ret = kvm_skip_emulated_instruction(&svm->vcpu);
3804
3805         disable_gif(svm);
3806
3807         /* After a CLGI no interrupts should come */
3808         if (!kvm_vcpu_apicv_active(&svm->vcpu)) {
3809                 svm_clear_vintr(svm);
3810                 svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
3811                 mark_dirty(svm->vmcb, VMCB_INTR);
3812         }
3813
3814         return ret;
3815 }
3816
3817 static int invlpga_interception(struct vcpu_svm *svm)
3818 {
3819         struct kvm_vcpu *vcpu = &svm->vcpu;
3820
3821         trace_kvm_invlpga(svm->vmcb->save.rip, kvm_rcx_read(&svm->vcpu),
3822                           kvm_rax_read(&svm->vcpu));
3823
3824         /* Let's treat INVLPGA the same as INVLPG (can be optimized!) */
3825         kvm_mmu_invlpg(vcpu, kvm_rax_read(&svm->vcpu));
3826
3827         return kvm_skip_emulated_instruction(&svm->vcpu);
3828 }
3829
3830 static int skinit_interception(struct vcpu_svm *svm)
3831 {
3832         trace_kvm_skinit(svm->vmcb->save.rip, kvm_rax_read(&svm->vcpu));
3833
3834         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3835         return 1;
3836 }
3837
3838 static int wbinvd_interception(struct vcpu_svm *svm)
3839 {
3840         return kvm_emulate_wbinvd(&svm->vcpu);
3841 }
3842
3843 static int xsetbv_interception(struct vcpu_svm *svm)
3844 {
3845         u64 new_bv = kvm_read_edx_eax(&svm->vcpu);
3846         u32 index = kvm_rcx_read(&svm->vcpu);
3847
3848         if (kvm_set_xcr(&svm->vcpu, index, new_bv) == 0) {
3849                 return kvm_skip_emulated_instruction(&svm->vcpu);
3850         }
3851
3852         return 1;
3853 }
3854
3855 static int rdpru_interception(struct vcpu_svm *svm)
3856 {
3857         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
3858         return 1;
3859 }
3860
3861 static int task_switch_interception(struct vcpu_svm *svm)
3862 {
3863         u16 tss_selector;
3864         int reason;
3865         int int_type = svm->vmcb->control.exit_int_info &
3866                 SVM_EXITINTINFO_TYPE_MASK;
3867         int int_vec = svm->vmcb->control.exit_int_info & SVM_EVTINJ_VEC_MASK;
3868         uint32_t type =
3869                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_TYPE_MASK;
3870         uint32_t idt_v =
3871                 svm->vmcb->control.exit_int_info & SVM_EXITINTINFO_VALID;
3872         bool has_error_code = false;
3873         u32 error_code = 0;
3874
3875         tss_selector = (u16)svm->vmcb->control.exit_info_1;
3876
3877         if (svm->vmcb->control.exit_info_2 &
3878             (1ULL << SVM_EXITINFOSHIFT_TS_REASON_IRET))
3879                 reason = TASK_SWITCH_IRET;
3880         else if (svm->vmcb->control.exit_info_2 &
3881                  (1ULL << SVM_EXITINFOSHIFT_TS_REASON_JMP))
3882                 reason = TASK_SWITCH_JMP;
3883         else if (idt_v)
3884                 reason = TASK_SWITCH_GATE;
3885         else
3886                 reason = TASK_SWITCH_CALL;
3887
3888         if (reason == TASK_SWITCH_GATE) {
3889                 switch (type) {
3890                 case SVM_EXITINTINFO_TYPE_NMI:
3891                         svm->vcpu.arch.nmi_injected = false;
3892                         break;
3893                 case SVM_EXITINTINFO_TYPE_EXEPT:
3894                         if (svm->vmcb->control.exit_info_2 &
3895                             (1ULL << SVM_EXITINFOSHIFT_TS_HAS_ERROR_CODE)) {
3896                                 has_error_code = true;
3897                                 error_code =
3898                                         (u32)svm->vmcb->control.exit_info_2;
3899                         }
3900                         kvm_clear_exception_queue(&svm->vcpu);
3901                         break;
3902                 case SVM_EXITINTINFO_TYPE_INTR:
3903                         kvm_clear_interrupt_queue(&svm->vcpu);
3904                         break;
3905                 default:
3906                         break;
3907                 }
3908         }
3909
3910         if (reason != TASK_SWITCH_GATE ||
3911             int_type == SVM_EXITINTINFO_TYPE_SOFT ||
3912             (int_type == SVM_EXITINTINFO_TYPE_EXEPT &&
3913              (int_vec == OF_VECTOR || int_vec == BP_VECTOR))) {
3914                 if (!skip_emulated_instruction(&svm->vcpu))
3915                         return 0;
3916         }
3917
3918         if (int_type != SVM_EXITINTINFO_TYPE_SOFT)
3919                 int_vec = -1;
3920
3921         return kvm_task_switch(&svm->vcpu, tss_selector, int_vec, reason,
3922                                has_error_code, error_code);
3923 }
3924
3925 static int cpuid_interception(struct vcpu_svm *svm)
3926 {
3927         return kvm_emulate_cpuid(&svm->vcpu);
3928 }
3929
3930 static int iret_interception(struct vcpu_svm *svm)
3931 {
3932         ++svm->vcpu.stat.nmi_window_exits;
3933         clr_intercept(svm, INTERCEPT_IRET);
3934         svm->vcpu.arch.hflags |= HF_IRET_MASK;
3935         svm->nmi_iret_rip = kvm_rip_read(&svm->vcpu);
3936         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
3937         return 1;
3938 }
3939
3940 static int invlpg_interception(struct vcpu_svm *svm)
3941 {
3942         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
3943                 return kvm_emulate_instruction(&svm->vcpu, 0);
3944
3945         kvm_mmu_invlpg(&svm->vcpu, svm->vmcb->control.exit_info_1);
3946         return kvm_skip_emulated_instruction(&svm->vcpu);
3947 }
3948
3949 static int emulate_on_interception(struct vcpu_svm *svm)
3950 {
3951         return kvm_emulate_instruction(&svm->vcpu, 0);
3952 }
3953
3954 static int rsm_interception(struct vcpu_svm *svm)
3955 {
3956         return kvm_emulate_instruction_from_buffer(&svm->vcpu, rsm_ins_bytes, 2);
3957 }
3958
3959 static int rdpmc_interception(struct vcpu_svm *svm)
3960 {
3961         int err;
3962
3963         if (!nrips)
3964                 return emulate_on_interception(svm);
3965
3966         err = kvm_rdpmc(&svm->vcpu);
3967         return kvm_complete_insn_gp(&svm->vcpu, err);
3968 }
3969
3970 static bool check_selective_cr0_intercepted(struct vcpu_svm *svm,
3971                                             unsigned long val)
3972 {
3973         unsigned long cr0 = svm->vcpu.arch.cr0;
3974         bool ret = false;
3975         u64 intercept;
3976
3977         intercept = svm->nested.intercept;
3978
3979         if (!is_guest_mode(&svm->vcpu) ||
3980             (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0))))
3981                 return false;
3982
3983         cr0 &= ~SVM_CR0_SELECTIVE_MASK;
3984         val &= ~SVM_CR0_SELECTIVE_MASK;
3985
3986         if (cr0 ^ val) {
3987                 svm->vmcb->control.exit_code = SVM_EXIT_CR0_SEL_WRITE;
3988                 ret = (nested_svm_exit_handled(svm) == NESTED_EXIT_DONE);
3989         }
3990
3991         return ret;
3992 }
3993
3994 #define CR_VALID (1ULL << 63)
3995
3996 static int cr_interception(struct vcpu_svm *svm)
3997 {
3998         int reg, cr;
3999         unsigned long val;
4000         int err;
4001
4002         if (!static_cpu_has(X86_FEATURE_DECODEASSISTS))
4003                 return emulate_on_interception(svm);
4004
4005         if (unlikely((svm->vmcb->control.exit_info_1 & CR_VALID) == 0))
4006                 return emulate_on_interception(svm);
4007
4008         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
4009         if (svm->vmcb->control.exit_code == SVM_EXIT_CR0_SEL_WRITE)
4010                 cr = SVM_EXIT_WRITE_CR0 - SVM_EXIT_READ_CR0;
4011         else
4012                 cr = svm->vmcb->control.exit_code - SVM_EXIT_READ_CR0;
4013
4014         err = 0;
4015         if (cr >= 16) { /* mov to cr */
4016                 cr -= 16;
4017                 val = kvm_register_read(&svm->vcpu, reg);
4018                 switch (cr) {
4019                 case 0:
4020                         if (!check_selective_cr0_intercepted(svm, val))
4021                                 err = kvm_set_cr0(&svm->vcpu, val);
4022                         else
4023                                 return 1;
4024
4025                         break;
4026                 case 3:
4027                         err = kvm_set_cr3(&svm->vcpu, val);
4028                         break;
4029                 case 4:
4030                         err = kvm_set_cr4(&svm->vcpu, val);
4031                         break;
4032                 case 8:
4033                         err = kvm_set_cr8(&svm->vcpu, val);
4034                         break;
4035                 default:
4036                         WARN(1, "unhandled write to CR%d", cr);
4037                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4038                         return 1;
4039                 }
4040         } else { /* mov from cr */
4041                 switch (cr) {
4042                 case 0:
4043                         val = kvm_read_cr0(&svm->vcpu);
4044                         break;
4045                 case 2:
4046                         val = svm->vcpu.arch.cr2;
4047                         break;
4048                 case 3:
4049                         val = kvm_read_cr3(&svm->vcpu);
4050                         break;
4051                 case 4:
4052                         val = kvm_read_cr4(&svm->vcpu);
4053                         break;
4054                 case 8:
4055                         val = kvm_get_cr8(&svm->vcpu);
4056                         break;
4057                 default:
4058                         WARN(1, "unhandled read from CR%d", cr);
4059                         kvm_queue_exception(&svm->vcpu, UD_VECTOR);
4060                         return 1;
4061                 }
4062                 kvm_register_write(&svm->vcpu, reg, val);
4063         }
4064         return kvm_complete_insn_gp(&svm->vcpu, err);
4065 }
4066
4067 static int dr_interception(struct vcpu_svm *svm)
4068 {
4069         int reg, dr;
4070         unsigned long val;
4071
4072         if (svm->vcpu.guest_debug == 0) {
4073                 /*
4074                  * No more DR vmexits; force a reload of the debug registers
4075                  * and reenter on this instruction.  The next vmexit will
4076                  * retrieve the full state of the debug registers.
4077                  */
4078                 clr_dr_intercepts(svm);
4079                 svm->vcpu.arch.switch_db_regs |= KVM_DEBUGREG_WONT_EXIT;
4080                 return 1;
4081         }
4082
4083         if (!boot_cpu_has(X86_FEATURE_DECODEASSISTS))
4084                 return emulate_on_interception(svm);
4085
4086         reg = svm->vmcb->control.exit_info_1 & SVM_EXITINFO_REG_MASK;
4087         dr = svm->vmcb->control.exit_code - SVM_EXIT_READ_DR0;
4088
4089         if (dr >= 16) { /* mov to DRn */
4090                 if (!kvm_require_dr(&svm->vcpu, dr - 16))
4091                         return 1;
4092                 val = kvm_register_read(&svm->vcpu, reg);
4093                 kvm_set_dr(&svm->vcpu, dr - 16, val);
4094         } else {
4095                 if (!kvm_require_dr(&svm->vcpu, dr))
4096                         return 1;
4097                 kvm_get_dr(&svm->vcpu, dr, &val);
4098                 kvm_register_write(&svm->vcpu, reg, val);
4099         }
4100
4101         return kvm_skip_emulated_instruction(&svm->vcpu);
4102 }
4103
4104 static int cr8_write_interception(struct vcpu_svm *svm)
4105 {
4106         struct kvm_run *kvm_run = svm->vcpu.run;
4107         int r;
4108
4109         u8 cr8_prev = kvm_get_cr8(&svm->vcpu);
4110         /* instruction emulation calls kvm_set_cr8() */
4111         r = cr_interception(svm);
4112         if (lapic_in_kernel(&svm->vcpu))
4113                 return r;
4114         if (cr8_prev <= kvm_get_cr8(&svm->vcpu))
4115                 return r;
4116         kvm_run->exit_reason = KVM_EXIT_SET_TPR;
4117         return 0;
4118 }
4119
4120 static int svm_get_msr_feature(struct kvm_msr_entry *msr)
4121 {
4122         msr->data = 0;
4123
4124         switch (msr->index) {
4125         case MSR_F10H_DECFG:
4126                 if (boot_cpu_has(X86_FEATURE_LFENCE_RDTSC))
4127                         msr->data |= MSR_F10H_DECFG_LFENCE_SERIALIZE;
4128                 break;
4129         default:
4130                 return 1;
4131         }
4132
4133         return 0;
4134 }
4135
4136 static int svm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
4137 {
4138         struct vcpu_svm *svm = to_svm(vcpu);
4139
4140         switch (msr_info->index) {
4141         case MSR_STAR:
4142                 msr_info->data = svm->vmcb->save.star;
4143                 break;
4144 #ifdef CONFIG_X86_64
4145         case MSR_LSTAR:
4146                 msr_info->data = svm->vmcb->save.lstar;
4147                 break;
4148         case MSR_CSTAR:
4149                 msr_info->data = svm->vmcb->save.cstar;
4150                 break;
4151         case MSR_KERNEL_GS_BASE:
4152                 msr_info->data = svm->vmcb->save.kernel_gs_base;
4153                 break;
4154         case MSR_SYSCALL_MASK:
4155                 msr_info->data = svm->vmcb->save.sfmask;
4156                 break;
4157 #endif
4158         case MSR_IA32_SYSENTER_CS:
4159                 msr_info->data = svm->vmcb->save.sysenter_cs;
4160                 break;
4161         case MSR_IA32_SYSENTER_EIP:
4162                 msr_info->data = svm->sysenter_eip;
4163                 break;
4164         case MSR_IA32_SYSENTER_ESP:
4165                 msr_info->data = svm->sysenter_esp;
4166                 break;
4167         case MSR_TSC_AUX:
4168                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4169                         return 1;
4170                 msr_info->data = svm->tsc_aux;
4171                 break;
4172         /*
4173          * Nobody will change the following 5 values in the VMCB so we can
4174          * safely return them on rdmsr. They will always be 0 until LBRV is
4175          * implemented.
4176          */
4177         case MSR_IA32_DEBUGCTLMSR:
4178                 msr_info->data = svm->vmcb->save.dbgctl;
4179                 break;
4180         case MSR_IA32_LASTBRANCHFROMIP:
4181                 msr_info->data = svm->vmcb->save.br_from;
4182                 break;
4183         case MSR_IA32_LASTBRANCHTOIP:
4184                 msr_info->data = svm->vmcb->save.br_to;
4185                 break;
4186         case MSR_IA32_LASTINTFROMIP:
4187                 msr_info->data = svm->vmcb->save.last_excp_from;
4188                 break;
4189         case MSR_IA32_LASTINTTOIP:
4190                 msr_info->data = svm->vmcb->save.last_excp_to;
4191                 break;
4192         case MSR_VM_HSAVE_PA:
4193                 msr_info->data = svm->nested.hsave_msr;
4194                 break;
4195         case MSR_VM_CR:
4196                 msr_info->data = svm->nested.vm_cr_msr;
4197                 break;
4198         case MSR_IA32_SPEC_CTRL:
4199                 if (!msr_info->host_initiated &&
4200                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4201                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4202                         return 1;
4203
4204                 msr_info->data = svm->spec_ctrl;
4205                 break;
4206         case MSR_AMD64_VIRT_SPEC_CTRL:
4207                 if (!msr_info->host_initiated &&
4208                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4209                         return 1;
4210
4211                 msr_info->data = svm->virt_spec_ctrl;
4212                 break;
4213         case MSR_F15H_IC_CFG: {
4214
4215                 int family, model;
4216
4217                 family = guest_cpuid_family(vcpu);
4218                 model  = guest_cpuid_model(vcpu);
4219
4220                 if (family < 0 || model < 0)
4221                         return kvm_get_msr_common(vcpu, msr_info);
4222
4223                 msr_info->data = 0;
4224
4225                 if (family == 0x15 &&
4226                     (model >= 0x2 && model < 0x20))
4227                         msr_info->data = 0x1E;
4228                 }
4229                 break;
4230         case MSR_F10H_DECFG:
4231                 msr_info->data = svm->msr_decfg;
4232                 break;
4233         default:
4234                 return kvm_get_msr_common(vcpu, msr_info);
4235         }
4236         return 0;
4237 }
4238
4239 static int rdmsr_interception(struct vcpu_svm *svm)
4240 {
4241         return kvm_emulate_rdmsr(&svm->vcpu);
4242 }
4243
4244 static int svm_set_vm_cr(struct kvm_vcpu *vcpu, u64 data)
4245 {
4246         struct vcpu_svm *svm = to_svm(vcpu);
4247         int svm_dis, chg_mask;
4248
4249         if (data & ~SVM_VM_CR_VALID_MASK)
4250                 return 1;
4251
4252         chg_mask = SVM_VM_CR_VALID_MASK;
4253
4254         if (svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK)
4255                 chg_mask &= ~(SVM_VM_CR_SVM_LOCK_MASK | SVM_VM_CR_SVM_DIS_MASK);
4256
4257         svm->nested.vm_cr_msr &= ~chg_mask;
4258         svm->nested.vm_cr_msr |= (data & chg_mask);
4259
4260         svm_dis = svm->nested.vm_cr_msr & SVM_VM_CR_SVM_DIS_MASK;
4261
4262         /* check for svm_disable while efer.svme is set */
4263         if (svm_dis && (vcpu->arch.efer & EFER_SVME))
4264                 return 1;
4265
4266         return 0;
4267 }
4268
4269 static int svm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr)
4270 {
4271         struct vcpu_svm *svm = to_svm(vcpu);
4272
4273         u32 ecx = msr->index;
4274         u64 data = msr->data;
4275         switch (ecx) {
4276         case MSR_IA32_CR_PAT:
4277                 if (!kvm_mtrr_valid(vcpu, MSR_IA32_CR_PAT, data))
4278                         return 1;
4279                 vcpu->arch.pat = data;
4280                 svm->vmcb->save.g_pat = data;
4281                 mark_dirty(svm->vmcb, VMCB_NPT);
4282                 break;
4283         case MSR_IA32_SPEC_CTRL:
4284                 if (!msr->host_initiated &&
4285                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBRS) &&
4286                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_SSBD))
4287                         return 1;
4288
4289                 if (data & ~kvm_spec_ctrl_valid_bits(vcpu))
4290                         return 1;
4291
4292                 svm->spec_ctrl = data;
4293                 if (!data)
4294                         break;
4295
4296                 /*
4297                  * For non-nested:
4298                  * When it's written (to non-zero) for the first time, pass
4299                  * it through.
4300                  *
4301                  * For nested:
4302                  * The handling of the MSR bitmap for L2 guests is done in
4303                  * nested_svm_vmrun_msrpm.
4304                  * We update the L1 MSR bit as well since it will end up
4305                  * touching the MSR anyway now.
4306                  */
4307                 set_msr_interception(svm->msrpm, MSR_IA32_SPEC_CTRL, 1, 1);
4308                 break;
4309         case MSR_IA32_PRED_CMD:
4310                 if (!msr->host_initiated &&
4311                     !guest_cpuid_has(vcpu, X86_FEATURE_AMD_IBPB))
4312                         return 1;
4313
4314                 if (data & ~PRED_CMD_IBPB)
4315                         return 1;
4316                 if (!boot_cpu_has(X86_FEATURE_AMD_IBPB))
4317                         return 1;
4318                 if (!data)
4319                         break;
4320
4321                 wrmsrl(MSR_IA32_PRED_CMD, PRED_CMD_IBPB);
4322                 set_msr_interception(svm->msrpm, MSR_IA32_PRED_CMD, 0, 1);
4323                 break;
4324         case MSR_AMD64_VIRT_SPEC_CTRL:
4325                 if (!msr->host_initiated &&
4326                     !guest_cpuid_has(vcpu, X86_FEATURE_VIRT_SSBD))
4327                         return 1;
4328
4329                 if (data & ~SPEC_CTRL_SSBD)
4330                         return 1;
4331
4332                 svm->virt_spec_ctrl = data;
4333                 break;
4334         case MSR_STAR:
4335                 svm->vmcb->save.star = data;
4336                 break;
4337 #ifdef CONFIG_X86_64
4338         case MSR_LSTAR:
4339                 svm->vmcb->save.lstar = data;
4340                 break;
4341         case MSR_CSTAR:
4342                 svm->vmcb->save.cstar = data;
4343                 break;
4344         case MSR_KERNEL_GS_BASE:
4345                 svm->vmcb->save.kernel_gs_base = data;
4346                 break;
4347         case MSR_SYSCALL_MASK:
4348                 svm->vmcb->save.sfmask = data;
4349                 break;
4350 #endif
4351         case MSR_IA32_SYSENTER_CS:
4352                 svm->vmcb->save.sysenter_cs = data;
4353                 break;
4354         case MSR_IA32_SYSENTER_EIP:
4355                 svm->sysenter_eip = data;
4356                 svm->vmcb->save.sysenter_eip = data;
4357                 break;
4358         case MSR_IA32_SYSENTER_ESP:
4359                 svm->sysenter_esp = data;
4360                 svm->vmcb->save.sysenter_esp = data;
4361                 break;
4362         case MSR_TSC_AUX:
4363                 if (!boot_cpu_has(X86_FEATURE_RDTSCP))
4364                         return 1;
4365
4366                 /*
4367                  * This is rare, so we update the MSR here instead of using
4368                  * direct_access_msrs.  Doing that would require a rdmsr in
4369                  * svm_vcpu_put.
4370                  */
4371                 svm->tsc_aux = data;
4372                 wrmsrl(MSR_TSC_AUX, svm->tsc_aux);
4373                 break;
4374         case MSR_IA32_DEBUGCTLMSR:
4375                 if (!boot_cpu_has(X86_FEATURE_LBRV)) {
4376                         vcpu_unimpl(vcpu, "%s: MSR_IA32_DEBUGCTL 0x%llx, nop\n",
4377                                     __func__, data);
4378                         break;
4379                 }
4380                 if (data & DEBUGCTL_RESERVED_BITS)
4381                         return 1;
4382
4383                 svm->vmcb->save.dbgctl = data;
4384                 mark_dirty(svm->vmcb, VMCB_LBR);
4385                 if (data & (1ULL<<0))
4386                         svm_enable_lbrv(svm);
4387                 else
4388                         svm_disable_lbrv(svm);
4389                 break;
4390         case MSR_VM_HSAVE_PA:
4391                 svm->nested.hsave_msr = data;
4392                 break;
4393         case MSR_VM_CR:
4394                 return svm_set_vm_cr(vcpu, data);
4395         case MSR_VM_IGNNE:
4396                 vcpu_unimpl(vcpu, "unimplemented wrmsr: 0x%x data 0x%llx\n", ecx, data);
4397                 break;
4398         case MSR_F10H_DECFG: {
4399                 struct kvm_msr_entry msr_entry;
4400
4401                 msr_entry.index = msr->index;
4402                 if (svm_get_msr_feature(&msr_entry))
4403                         return 1;
4404
4405                 /* Check the supported bits */
4406                 if (data & ~msr_entry.data)
4407                         return 1;
4408
4409                 /* Don't allow the guest to change a bit, #GP */
4410                 if (!msr->host_initiated && (data ^ msr_entry.data))
4411                         return 1;
4412
4413                 svm->msr_decfg = data;
4414                 break;
4415         }
4416         case MSR_IA32_APICBASE:
4417                 if (kvm_vcpu_apicv_active(vcpu))
4418                         avic_update_vapic_bar(to_svm(vcpu), data);
4419                 /* Fall through */
4420         default:
4421                 return kvm_set_msr_common(vcpu, msr);
4422         }
4423         return 0;
4424 }
4425
4426 static int wrmsr_interception(struct vcpu_svm *svm)
4427 {
4428         return kvm_emulate_wrmsr(&svm->vcpu);
4429 }
4430
4431 static int msr_interception(struct vcpu_svm *svm)
4432 {
4433         if (svm->vmcb->control.exit_info_1)
4434                 return wrmsr_interception(svm);
4435         else
4436                 return rdmsr_interception(svm);
4437 }
4438
4439 static int interrupt_window_interception(struct vcpu_svm *svm)
4440 {
4441         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
4442         svm_clear_vintr(svm);
4443         svm->vmcb->control.int_ctl &= ~V_IRQ_MASK;
4444         mark_dirty(svm->vmcb, VMCB_INTR);
4445         ++svm->vcpu.stat.irq_window_exits;
4446         return 1;
4447 }
4448
4449 static int pause_interception(struct vcpu_svm *svm)
4450 {
4451         struct kvm_vcpu *vcpu = &svm->vcpu;
4452         bool in_kernel = (svm_get_cpl(vcpu) == 0);
4453
4454         if (pause_filter_thresh)
4455                 grow_ple_window(vcpu);
4456
4457         kvm_vcpu_on_spin(vcpu, in_kernel);
4458         return 1;
4459 }
4460
4461 static int nop_interception(struct vcpu_svm *svm)
4462 {
4463         return kvm_skip_emulated_instruction(&(svm->vcpu));
4464 }
4465
4466 static int monitor_interception(struct vcpu_svm *svm)
4467 {
4468         printk_once(KERN_WARNING "kvm: MONITOR instruction emulated as NOP!\n");
4469         return nop_interception(svm);
4470 }
4471
4472 static int mwait_interception(struct vcpu_svm *svm)
4473 {
4474         printk_once(KERN_WARNING "kvm: MWAIT instruction emulated as NOP!\n");
4475         return nop_interception(svm);
4476 }
4477
4478 enum avic_ipi_failure_cause {
4479         AVIC_IPI_FAILURE_INVALID_INT_TYPE,
4480         AVIC_IPI_FAILURE_TARGET_NOT_RUNNING,
4481         AVIC_IPI_FAILURE_INVALID_TARGET,
4482         AVIC_IPI_FAILURE_INVALID_BACKING_PAGE,
4483 };
4484
4485 static int avic_incomplete_ipi_interception(struct vcpu_svm *svm)
4486 {
4487         u32 icrh = svm->vmcb->control.exit_info_1 >> 32;
4488         u32 icrl = svm->vmcb->control.exit_info_1;
4489         u32 id = svm->vmcb->control.exit_info_2 >> 32;
4490         u32 index = svm->vmcb->control.exit_info_2 & 0xFF;
4491         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4492
4493         trace_kvm_avic_incomplete_ipi(svm->vcpu.vcpu_id, icrh, icrl, id, index);
4494
4495         switch (id) {
4496         case AVIC_IPI_FAILURE_INVALID_INT_TYPE:
4497                 /*
4498                  * AVIC hardware handles the generation of
4499                  * IPIs when the specified Message Type is Fixed
4500                  * (also known as fixed delivery mode) and
4501                  * the Trigger Mode is edge-triggered. The hardware
4502                  * also supports self and broadcast delivery modes
4503                  * specified via the Destination Shorthand(DSH)
4504                  * field of the ICRL. Logical and physical APIC ID
4505                  * formats are supported. All other IPI types cause
4506                  * a #VMEXIT, which needs to emulated.
4507                  */
4508                 kvm_lapic_reg_write(apic, APIC_ICR2, icrh);
4509                 kvm_lapic_reg_write(apic, APIC_ICR, icrl);
4510                 break;
4511         case AVIC_IPI_FAILURE_TARGET_NOT_RUNNING: {
4512                 int i;
4513                 struct kvm_vcpu *vcpu;
4514                 struct kvm *kvm = svm->vcpu.kvm;
4515                 struct kvm_lapic *apic = svm->vcpu.arch.apic;
4516
4517                 /*
4518                  * At this point, we expect that the AVIC HW has already
4519                  * set the appropriate IRR bits on the valid target
4520                  * vcpus. So, we just need to kick the appropriate vcpu.
4521                  */
4522                 kvm_for_each_vcpu(i, vcpu, kvm) {
4523                         bool m = kvm_apic_match_dest(vcpu, apic,
4524                                                      icrl & APIC_SHORT_MASK,
4525                                                      GET_APIC_DEST_FIELD(icrh),
4526                                                      icrl & APIC_DEST_MASK);
4527
4528                         if (m && !avic_vcpu_is_running(vcpu))
4529                                 kvm_vcpu_wake_up(vcpu);
4530                 }
4531                 break;
4532         }
4533         case AVIC_IPI_FAILURE_INVALID_TARGET:
4534                 WARN_ONCE(1, "Invalid IPI target: index=%u, vcpu=%d, icr=%#0x:%#0x\n",
4535                           index, svm->vcpu.vcpu_id, icrh, icrl);
4536                 break;
4537         case AVIC_IPI_FAILURE_INVALID_BACKING_PAGE:
4538                 WARN_ONCE(1, "Invalid backing page\n");
4539                 break;
4540         default:
4541                 pr_err("Unknown IPI interception\n");
4542         }
4543
4544         return 1;
4545 }
4546
4547 static u32 *avic_get_logical_id_entry(struct kvm_vcpu *vcpu, u32 ldr, bool flat)
4548 {
4549         struct kvm_svm *kvm_svm = to_kvm_svm(vcpu->kvm);
4550         int index;
4551         u32 *logical_apic_id_table;
4552         int dlid = GET_APIC_LOGICAL_ID(ldr);
4553
4554         if (!dlid)
4555                 return NULL;
4556
4557         if (flat) { /* flat */
4558                 index = ffs(dlid) - 1;
4559                 if (index > 7)
4560                         return NULL;
4561         } else { /* cluster */
4562                 int cluster = (dlid & 0xf0) >> 4;
4563                 int apic = ffs(dlid & 0x0f) - 1;
4564
4565                 if ((apic < 0) || (apic > 7) ||
4566                     (cluster >= 0xf))
4567                         return NULL;
4568                 index = (cluster << 2) + apic;
4569         }
4570
4571         logical_apic_id_table = (u32 *) page_address(kvm_svm->avic_logical_id_table_page);
4572
4573         return &logical_apic_id_table[index];
4574 }
4575
4576 static int avic_ldr_write(struct kvm_vcpu *vcpu, u8 g_physical_id, u32 ldr)
4577 {
4578         bool flat;
4579         u32 *entry, new_entry;
4580
4581         flat = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR) == APIC_DFR_FLAT;
4582         entry = avic_get_logical_id_entry(vcpu, ldr, flat);
4583         if (!entry)
4584                 return -EINVAL;
4585
4586         new_entry = READ_ONCE(*entry);
4587         new_entry &= ~AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK;
4588         new_entry |= (g_physical_id & AVIC_LOGICAL_ID_ENTRY_GUEST_PHYSICAL_ID_MASK);
4589         new_entry |= AVIC_LOGICAL_ID_ENTRY_VALID_MASK;
4590         WRITE_ONCE(*entry, new_entry);
4591
4592         return 0;
4593 }
4594
4595 static void avic_invalidate_logical_id_entry(struct kvm_vcpu *vcpu)
4596 {
4597         struct vcpu_svm *svm = to_svm(vcpu);
4598         bool flat = svm->dfr_reg == APIC_DFR_FLAT;
4599         u32 *entry = avic_get_logical_id_entry(vcpu, svm->ldr_reg, flat);
4600
4601         if (entry)
4602                 clear_bit(AVIC_LOGICAL_ID_ENTRY_VALID_BIT, (unsigned long *)entry);
4603 }
4604
4605 static int avic_handle_ldr_update(struct kvm_vcpu *vcpu)
4606 {
4607         int ret = 0;
4608         struct vcpu_svm *svm = to_svm(vcpu);
4609         u32 ldr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_LDR);
4610         u32 id = kvm_xapic_id(vcpu->arch.apic);
4611
4612         if (ldr == svm->ldr_reg)
4613                 return 0;
4614
4615         avic_invalidate_logical_id_entry(vcpu);
4616
4617         if (ldr)
4618                 ret = avic_ldr_write(vcpu, id, ldr);
4619
4620         if (!ret)
4621                 svm->ldr_reg = ldr;
4622
4623         return ret;
4624 }
4625
4626 static int avic_handle_apic_id_update(struct kvm_vcpu *vcpu)
4627 {
4628         u64 *old, *new;
4629         struct vcpu_svm *svm = to_svm(vcpu);
4630         u32 id = kvm_xapic_id(vcpu->arch.apic);
4631
4632         if (vcpu->vcpu_id == id)
4633                 return 0;
4634
4635         old = avic_get_physical_id_entry(vcpu, vcpu->vcpu_id);
4636         new = avic_get_physical_id_entry(vcpu, id);
4637         if (!new || !old)
4638                 return 1;
4639
4640         /* We need to move physical_id_entry to new offset */
4641         *new = *old;
4642         *old = 0ULL;
4643         to_svm(vcpu)->avic_physical_id_cache = new;
4644
4645         /*
4646          * Also update the guest physical APIC ID in the logical
4647          * APIC ID table entry if already setup the LDR.
4648          */
4649         if (svm->ldr_reg)
4650                 avic_handle_ldr_update(vcpu);
4651
4652         return 0;
4653 }
4654
4655 static void avic_handle_dfr_update(struct kvm_vcpu *vcpu)
4656 {
4657         struct vcpu_svm *svm = to_svm(vcpu);
4658         u32 dfr = kvm_lapic_get_reg(vcpu->arch.apic, APIC_DFR);
4659
4660         if (svm->dfr_reg == dfr)
4661                 return;
4662
4663         avic_invalidate_logical_id_entry(vcpu);
4664         svm->dfr_reg = dfr;
4665 }
4666
4667 static int avic_unaccel_trap_write(struct vcpu_svm *svm)
4668 {
4669         struct kvm_lapic *apic = svm->vcpu.arch.apic;
4670         u32 offset = svm->vmcb->control.exit_info_1 &
4671                                 AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4672
4673         switch (offset) {
4674         case APIC_ID:
4675                 if (avic_handle_apic_id_update(&svm->vcpu))
4676                         return 0;
4677                 break;
4678         case APIC_LDR:
4679                 if (avic_handle_ldr_update(&svm->vcpu))
4680                         return 0;
4681                 break;
4682         case APIC_DFR:
4683                 avic_handle_dfr_update(&svm->vcpu);
4684                 break;
4685         default:
4686                 break;
4687         }
4688
4689         kvm_lapic_reg_write(apic, offset, kvm_lapic_get_reg(apic, offset));
4690
4691         return 1;
4692 }
4693
4694 static bool is_avic_unaccelerated_access_trap(u32 offset)
4695 {
4696         bool ret = false;
4697
4698         switch (offset) {
4699         case APIC_ID:
4700         case APIC_EOI:
4701         case APIC_RRR:
4702         case APIC_LDR:
4703         case APIC_DFR:
4704         case APIC_SPIV:
4705         case APIC_ESR:
4706         case APIC_ICR:
4707         case APIC_LVTT:
4708         case APIC_LVTTHMR:
4709         case APIC_LVTPC:
4710         case APIC_LVT0:
4711         case APIC_LVT1:
4712         case APIC_LVTERR:
4713         case APIC_TMICT:
4714         case APIC_TDCR:
4715                 ret = true;
4716                 break;
4717         default:
4718                 break;
4719         }
4720         return ret;
4721 }
4722
4723 static int avic_unaccelerated_access_interception(struct vcpu_svm *svm)
4724 {
4725         int ret = 0;
4726         u32 offset = svm->vmcb->control.exit_info_1 &
4727                      AVIC_UNACCEL_ACCESS_OFFSET_MASK;
4728         u32 vector = svm->vmcb->control.exit_info_2 &
4729                      AVIC_UNACCEL_ACCESS_VECTOR_MASK;
4730         bool write = (svm->vmcb->control.exit_info_1 >> 32) &
4731                      AVIC_UNACCEL_ACCESS_WRITE_MASK;
4732         bool trap = is_avic_unaccelerated_access_trap(offset);
4733
4734         trace_kvm_avic_unaccelerated_access(svm->vcpu.vcpu_id, offset,
4735                                             trap, write, vector);
4736         if (trap) {
4737                 /* Handling Trap */
4738                 WARN_ONCE(!write, "svm: Handling trap read.\n");
4739                 ret = avic_unaccel_trap_write(svm);
4740         } else {
4741                 /* Handling Fault */
4742                 ret = kvm_emulate_instruction(&svm->vcpu, 0);
4743         }
4744
4745         return ret;
4746 }
4747
4748 static int (*const svm_exit_handlers[])(struct vcpu_svm *svm) = {
4749         [SVM_EXIT_READ_CR0]                     = cr_interception,
4750         [SVM_EXIT_READ_CR3]                     = cr_interception,
4751         [SVM_EXIT_READ_CR4]                     = cr_interception,
4752         [SVM_EXIT_READ_CR8]                     = cr_interception,
4753         [SVM_EXIT_CR0_SEL_WRITE]                = cr_interception,
4754         [SVM_EXIT_WRITE_CR0]                    = cr_interception,
4755         [SVM_EXIT_WRITE_CR3]                    = cr_interception,
4756         [SVM_EXIT_WRITE_CR4]                    = cr_interception,
4757         [SVM_EXIT_WRITE_CR8]                    = cr8_write_interception,
4758         [SVM_EXIT_READ_DR0]                     = dr_interception,
4759         [SVM_EXIT_READ_DR1]                     = dr_interception,
4760         [SVM_EXIT_READ_DR2]                     = dr_interception,
4761         [SVM_EXIT_READ_DR3]                     = dr_interception,
4762         [SVM_EXIT_READ_DR4]                     = dr_interception,
4763         [SVM_EXIT_READ_DR5]                     = dr_interception,
4764         [SVM_EXIT_READ_DR6]                     = dr_interception,
4765         [SVM_EXIT_READ_DR7]                     = dr_interception,
4766         [SVM_EXIT_WRITE_DR0]                    = dr_interception,
4767         [SVM_EXIT_WRITE_DR1]                    = dr_interception,
4768         [SVM_EXIT_WRITE_DR2]                    = dr_interception,
4769         [SVM_EXIT_WRITE_DR3]                    = dr_interception,
4770         [SVM_EXIT_WRITE_DR4]                    = dr_interception,
4771         [SVM_EXIT_WRITE_DR5]                    = dr_interception,
4772         [SVM_EXIT_WRITE_DR6]                    = dr_interception,
4773         [SVM_EXIT_WRITE_DR7]                    = dr_interception,
4774         [SVM_EXIT_EXCP_BASE + DB_VECTOR]        = db_interception,
4775         [SVM_EXIT_EXCP_BASE + BP_VECTOR]        = bp_interception,
4776         [SVM_EXIT_EXCP_BASE + UD_VECTOR]        = ud_interception,
4777         [SVM_EXIT_EXCP_BASE + PF_VECTOR]        = pf_interception,
4778         [SVM_EXIT_EXCP_BASE + MC_VECTOR]        = mc_interception,
4779         [SVM_EXIT_EXCP_BASE + AC_VECTOR]        = ac_interception,
4780         [SVM_EXIT_EXCP_BASE + GP_VECTOR]        = gp_interception,
4781         [SVM_EXIT_INTR]                         = intr_interception,
4782         [SVM_EXIT_NMI]                          = nmi_interception,
4783         [SVM_EXIT_SMI]                          = nop_on_interception,
4784         [SVM_EXIT_INIT]                         = nop_on_interception,
4785         [SVM_EXIT_VINTR]                        = interrupt_window_interception,
4786         [SVM_EXIT_RDPMC]                        = rdpmc_interception,
4787         [SVM_EXIT_CPUID]                        = cpuid_interception,
4788         [SVM_EXIT_IRET]                         = iret_interception,
4789         [SVM_EXIT_INVD]                         = emulate_on_interception,
4790         [SVM_EXIT_PAUSE]                        = pause_interception,
4791         [SVM_EXIT_HLT]                          = halt_interception,
4792         [SVM_EXIT_INVLPG]                       = invlpg_interception,
4793         [SVM_EXIT_INVLPGA]                      = invlpga_interception,
4794         [SVM_EXIT_IOIO]                         = io_interception,
4795         [SVM_EXIT_MSR]                          = msr_interception,
4796         [SVM_EXIT_TASK_SWITCH]                  = task_switch_interception,
4797         [SVM_EXIT_SHUTDOWN]                     = shutdown_interception,
4798         [SVM_EXIT_VMRUN]                        = vmrun_interception,
4799         [SVM_EXIT_VMMCALL]                      = vmmcall_interception,
4800         [SVM_EXIT_VMLOAD]                       = vmload_interception,
4801         [SVM_EXIT_VMSAVE]                       = vmsave_interception,
4802         [SVM_EXIT_STGI]                         = stgi_interception,
4803         [SVM_EXIT_CLGI]                         = clgi_interception,
4804         [SVM_EXIT_SKINIT]                       = skinit_interception,
4805         [SVM_EXIT_WBINVD]                       = wbinvd_interception,
4806         [SVM_EXIT_MONITOR]                      = monitor_interception,
4807         [SVM_EXIT_MWAIT]                        = mwait_interception,
4808         [SVM_EXIT_XSETBV]                       = xsetbv_interception,
4809         [SVM_EXIT_RDPRU]                        = rdpru_interception,
4810         [SVM_EXIT_NPF]                          = npf_interception,
4811         [SVM_EXIT_RSM]                          = rsm_interception,
4812         [SVM_EXIT_AVIC_INCOMPLETE_IPI]          = avic_incomplete_ipi_interception,
4813         [SVM_EXIT_AVIC_UNACCELERATED_ACCESS]    = avic_unaccelerated_access_interception,
4814 };
4815
4816 static void dump_vmcb(struct kvm_vcpu *vcpu)
4817 {
4818         struct vcpu_svm *svm = to_svm(vcpu);
4819         struct vmcb_control_area *control = &svm->vmcb->control;
4820         struct vmcb_save_area *save = &svm->vmcb->save;
4821
4822         if (!dump_invalid_vmcb) {
4823                 pr_warn_ratelimited("set kvm_amd.dump_invalid_vmcb=1 to dump internal KVM state.\n");
4824                 return;
4825         }
4826
4827         pr_err("VMCB Control Area:\n");
4828         pr_err("%-20s%04x\n", "cr_read:", control->intercept_cr & 0xffff);
4829         pr_err("%-20s%04x\n", "cr_write:", control->intercept_cr >> 16);
4830         pr_err("%-20s%04x\n", "dr_read:", control->intercept_dr & 0xffff);
4831         pr_err("%-20s%04x\n", "dr_write:", control->intercept_dr >> 16);
4832         pr_err("%-20s%08x\n", "exceptions:", control->intercept_exceptions);
4833         pr_err("%-20s%016llx\n", "intercepts:", control->intercept);
4834         pr_err("%-20s%d\n", "pause filter count:", control->pause_filter_count);
4835         pr_err("%-20s%d\n", "pause filter threshold:",
4836                control->pause_filter_thresh);
4837         pr_err("%-20s%016llx\n", "iopm_base_pa:", control->iopm_base_pa);
4838         pr_err("%-20s%016llx\n", "msrpm_base_pa:", control->msrpm_base_pa);
4839         pr_err("%-20s%016llx\n", "tsc_offset:", control->tsc_offset);
4840         pr_err("%-20s%d\n", "asid:", control->asid);
4841         pr_err("%-20s%d\n", "tlb_ctl:", control->tlb_ctl);
4842         pr_err("%-20s%08x\n", "int_ctl:", control->int_ctl);
4843         pr_err("%-20s%08x\n", "int_vector:", control->int_vector);
4844         pr_err("%-20s%08x\n", "int_state:", control->int_state);
4845         pr_err("%-20s%08x\n", "exit_code:", control->exit_code);
4846         pr_err("%-20s%016llx\n", "exit_info1:", control->exit_info_1);
4847         pr_err("%-20s%016llx\n", "exit_info2:", control->exit_info_2);
4848         pr_err("%-20s%08x\n", "exit_int_info:", control->exit_int_info);
4849         pr_err("%-20s%08x\n", "exit_int_info_err:", control->exit_int_info_err);
4850         pr_err("%-20s%lld\n", "nested_ctl:", control->nested_ctl);
4851         pr_err("%-20s%016llx\n", "nested_cr3:", control->nested_cr3);
4852         pr_err("%-20s%016llx\n", "avic_vapic_bar:", control->avic_vapic_bar);
4853         pr_err("%-20s%08x\n", "event_inj:", control->event_inj);
4854         pr_err("%-20s%08x\n", "event_inj_err:", control->event_inj_err);
4855         pr_err("%-20s%lld\n", "virt_ext:", control->virt_ext);
4856         pr_err("%-20s%016llx\n", "next_rip:", control->next_rip);
4857         pr_err("%-20s%016llx\n", "avic_backing_page:", control->avic_backing_page);
4858         pr_err("%-20s%016llx\n", "avic_logical_id:", control->avic_logical_id);
4859         pr_err("%-20s%016llx\n", "avic_physical_id:", control->avic_physical_id);
4860         pr_err("VMCB State Save Area:\n");
4861         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4862                "es:",
4863                save->es.selector, save->es.attrib,
4864                save->es.limit, save->es.base);
4865         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4866                "cs:",
4867                save->cs.selector, save->cs.attrib,
4868                save->cs.limit, save->cs.base);
4869         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4870                "ss:",
4871                save->ss.selector, save->ss.attrib,
4872                save->ss.limit, save->ss.base);
4873         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4874                "ds:",
4875                save->ds.selector, save->ds.attrib,
4876                save->ds.limit, save->ds.base);
4877         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4878                "fs:",
4879                save->fs.selector, save->fs.attrib,
4880                save->fs.limit, save->fs.base);
4881         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4882                "gs:",
4883                save->gs.selector, save->gs.attrib,
4884                save->gs.limit, save->gs.base);
4885         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4886                "gdtr:",
4887                save->gdtr.selector, save->gdtr.attrib,
4888                save->gdtr.limit, save->gdtr.base);
4889         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4890                "ldtr:",
4891                save->ldtr.selector, save->ldtr.attrib,
4892                save->ldtr.limit, save->ldtr.base);
4893         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4894                "idtr:",
4895                save->idtr.selector, save->idtr.attrib,
4896                save->idtr.limit, save->idtr.base);
4897         pr_err("%-5s s: %04x a: %04x l: %08x b: %016llx\n",
4898                "tr:",
4899                save->tr.selector, save->tr.attrib,
4900                save->tr.limit, save->tr.base);
4901         pr_err("cpl:            %d                efer:         %016llx\n",
4902                 save->cpl, save->efer);
4903         pr_err("%-15s %016llx %-13s %016llx\n",
4904                "cr0:", save->cr0, "cr2:", save->cr2);
4905         pr_err("%-15s %016llx %-13s %016llx\n",
4906                "cr3:", save->cr3, "cr4:", save->cr4);
4907         pr_err("%-15s %016llx %-13s %016llx\n",
4908                "dr6:", save->dr6, "dr7:", save->dr7);
4909         pr_err("%-15s %016llx %-13s %016llx\n",
4910                "rip:", save->rip, "rflags:", save->rflags);
4911         pr_err("%-15s %016llx %-13s %016llx\n",
4912                "rsp:", save->rsp, "rax:", save->rax);
4913         pr_err("%-15s %016llx %-13s %016llx\n",
4914                "star:", save->star, "lstar:", save->lstar);
4915         pr_err("%-15s %016llx %-13s %016llx\n",
4916                "cstar:", save->cstar, "sfmask:", save->sfmask);
4917         pr_err("%-15s %016llx %-13s %016llx\n",
4918                "kernel_gs_base:", save->kernel_gs_base,
4919                "sysenter_cs:", save->sysenter_cs);
4920         pr_err("%-15s %016llx %-13s %016llx\n",
4921                "sysenter_esp:", save->sysenter_esp,
4922                "sysenter_eip:", save->sysenter_eip);
4923         pr_err("%-15s %016llx %-13s %016llx\n",
4924                "gpat:", save->g_pat, "dbgctl:", save->dbgctl);
4925         pr_err("%-15s %016llx %-13s %016llx\n",
4926                "br_from:", save->br_from, "br_to:", save->br_to);
4927         pr_err("%-15s %016llx %-13s %016llx\n",
4928                "excp_from:", save->last_excp_from,
4929                "excp_to:", save->last_excp_to);
4930 }
4931
4932 static void svm_get_exit_info(struct kvm_vcpu *vcpu, u64 *info1, u64 *info2)
4933 {
4934         struct vmcb_control_area *control = &to_svm(vcpu)->vmcb->control;
4935
4936         *info1 = control->exit_info_1;
4937         *info2 = control->exit_info_2;
4938 }
4939
4940 static int handle_exit(struct kvm_vcpu *vcpu,
4941         enum exit_fastpath_completion exit_fastpath)
4942 {
4943         struct vcpu_svm *svm = to_svm(vcpu);
4944         struct kvm_run *kvm_run = vcpu->run;
4945         u32 exit_code = svm->vmcb->control.exit_code;
4946
4947         trace_kvm_exit(exit_code, vcpu, KVM_ISA_SVM);
4948
4949         if (!is_cr_intercept(svm, INTERCEPT_CR0_WRITE))
4950                 vcpu->arch.cr0 = svm->vmcb->save.cr0;
4951         if (npt_enabled)
4952                 vcpu->arch.cr3 = svm->vmcb->save.cr3;
4953
4954         if (unlikely(svm->nested.exit_required)) {
4955                 nested_svm_vmexit(svm);
4956                 svm->nested.exit_required = false;
4957
4958                 return 1;
4959         }
4960
4961         if (is_guest_mode(vcpu)) {
4962                 int vmexit;
4963
4964                 trace_kvm_nested_vmexit(svm->vmcb->save.rip, exit_code,
4965                                         svm->vmcb->control.exit_info_1,
4966                                         svm->vmcb->control.exit_info_2,
4967                                         svm->vmcb->control.exit_int_info,
4968                                         svm->vmcb->control.exit_int_info_err,
4969                                         KVM_ISA_SVM);
4970
4971                 vmexit = nested_svm_exit_special(svm);
4972
4973                 if (vmexit == NESTED_EXIT_CONTINUE)
4974                         vmexit = nested_svm_exit_handled(svm);
4975
4976                 if (vmexit == NESTED_EXIT_DONE)
4977                         return 1;
4978         }
4979
4980         svm_complete_interrupts(svm);
4981
4982         if (svm->vmcb->control.exit_code == SVM_EXIT_ERR) {
4983                 kvm_run->exit_reason = KVM_EXIT_FAIL_ENTRY;
4984                 kvm_run->fail_entry.hardware_entry_failure_reason
4985                         = svm->vmcb->control.exit_code;
4986                 dump_vmcb(vcpu);
4987                 return 0;
4988         }
4989
4990         if (is_external_interrupt(svm->vmcb->control.exit_int_info) &&
4991             exit_code != SVM_EXIT_EXCP_BASE + PF_VECTOR &&
4992             exit_code != SVM_EXIT_NPF && exit_code != SVM_EXIT_TASK_SWITCH &&
4993             exit_code != SVM_EXIT_INTR && exit_code != SVM_EXIT_NMI)
4994                 printk(KERN_ERR "%s: unexpected exit_int_info 0x%x "
4995                        "exit_code 0x%x\n",
4996                        __func__, svm->vmcb->control.exit_int_info,
4997                        exit_code);
4998
4999         if (exit_fastpath == EXIT_FASTPATH_SKIP_EMUL_INS) {
5000                 kvm_skip_emulated_instruction(vcpu);
5001                 return 1;
5002         } else if (exit_code >= ARRAY_SIZE(svm_exit_handlers)
5003             || !svm_exit_handlers[exit_code]) {
5004                 vcpu_unimpl(vcpu, "svm: unexpected exit reason 0x%x\n", exit_code);
5005                 dump_vmcb(vcpu);
5006                 vcpu->run->exit_reason = KVM_EXIT_INTERNAL_ERROR;
5007                 vcpu->run->internal.suberror =
5008                         KVM_INTERNAL_ERROR_UNEXPECTED_EXIT_REASON;
5009                 vcpu->run->internal.ndata = 1;
5010                 vcpu->run->internal.data[0] = exit_code;
5011                 return 0;
5012         }
5013
5014 #ifdef CONFIG_RETPOLINE
5015         if (exit_code == SVM_EXIT_MSR)
5016                 return msr_interception(svm);
5017         else if (exit_code == SVM_EXIT_VINTR)
5018                 return interrupt_window_interception(svm);
5019         else if (exit_code == SVM_EXIT_INTR)
5020                 return intr_interception(svm);
5021         else if (exit_code == SVM_EXIT_HLT)
5022                 return halt_interception(svm);
5023         else if (exit_code == SVM_EXIT_NPF)
5024                 return npf_interception(svm);
5025 #endif
5026         return svm_exit_handlers[exit_code](svm);
5027 }
5028
5029 static void reload_tss(struct kvm_vcpu *vcpu)
5030 {
5031         int cpu = raw_smp_processor_id();
5032
5033         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5034         sd->tss_desc->type = 9; /* available 32/64-bit TSS */
5035         load_TR_desc();
5036 }
5037
5038 static void pre_sev_run(struct vcpu_svm *svm, int cpu)
5039 {
5040         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5041         int asid = sev_get_asid(svm->vcpu.kvm);
5042
5043         /* Assign the asid allocated with this SEV guest */
5044         svm->vmcb->control.asid = asid;
5045
5046         /*
5047          * Flush guest TLB:
5048          *
5049          * 1) when different VMCB for the same ASID is to be run on the same host CPU.
5050          * 2) or this VMCB was executed on different host CPU in previous VMRUNs.
5051          */
5052         if (sd->sev_vmcbs[asid] == svm->vmcb &&
5053             svm->last_cpu == cpu)
5054                 return;
5055
5056         svm->last_cpu = cpu;
5057         sd->sev_vmcbs[asid] = svm->vmcb;
5058         svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5059         mark_dirty(svm->vmcb, VMCB_ASID);
5060 }
5061
5062 static void pre_svm_run(struct vcpu_svm *svm)
5063 {
5064         int cpu = raw_smp_processor_id();
5065
5066         struct svm_cpu_data *sd = per_cpu(svm_data, cpu);
5067
5068         if (sev_guest(svm->vcpu.kvm))
5069                 return pre_sev_run(svm, cpu);
5070
5071         /* FIXME: handle wraparound of asid_generation */
5072         if (svm->asid_generation != sd->asid_generation)
5073                 new_asid(svm, sd);
5074 }
5075
5076 static void svm_inject_nmi(struct kvm_vcpu *vcpu)
5077 {
5078         struct vcpu_svm *svm = to_svm(vcpu);
5079
5080         svm->vmcb->control.event_inj = SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_NMI;
5081         vcpu->arch.hflags |= HF_NMI_MASK;
5082         set_intercept(svm, INTERCEPT_IRET);
5083         ++vcpu->stat.nmi_injections;
5084 }
5085
5086 static inline void svm_inject_irq(struct vcpu_svm *svm, int irq)
5087 {
5088         struct vmcb_control_area *control;
5089
5090         /* The following fields are ignored when AVIC is enabled */
5091         control = &svm->vmcb->control;
5092         control->int_vector = irq;
5093         control->int_ctl &= ~V_INTR_PRIO_MASK;
5094         control->int_ctl |= V_IRQ_MASK |
5095                 ((/*control->int_vector >> 4*/ 0xf) << V_INTR_PRIO_SHIFT);
5096         mark_dirty(svm->vmcb, VMCB_INTR);
5097 }
5098
5099 static void svm_set_irq(struct kvm_vcpu *vcpu)
5100 {
5101         struct vcpu_svm *svm = to_svm(vcpu);
5102
5103         BUG_ON(!(gif_set(svm)));
5104
5105         trace_kvm_inj_virq(vcpu->arch.interrupt.nr);
5106         ++vcpu->stat.irq_injections;
5107
5108         svm->vmcb->control.event_inj = vcpu->arch.interrupt.nr |
5109                 SVM_EVTINJ_VALID | SVM_EVTINJ_TYPE_INTR;
5110 }
5111
5112 static inline bool svm_nested_virtualize_tpr(struct kvm_vcpu *vcpu)
5113 {
5114         return is_guest_mode(vcpu) && (vcpu->arch.hflags & HF_VINTR_MASK);
5115 }
5116
5117 static void update_cr8_intercept(struct kvm_vcpu *vcpu, int tpr, int irr)
5118 {
5119         struct vcpu_svm *svm = to_svm(vcpu);
5120
5121         if (svm_nested_virtualize_tpr(vcpu))
5122                 return;
5123
5124         clr_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5125
5126         if (irr == -1)
5127                 return;
5128
5129         if (tpr >= irr)
5130                 set_cr_intercept(svm, INTERCEPT_CR8_WRITE);
5131 }
5132
5133 static void svm_set_virtual_apic_mode(struct kvm_vcpu *vcpu)
5134 {
5135         return;
5136 }
5137
5138 static bool svm_get_enable_apicv(struct kvm *kvm)
5139 {
5140         return avic && irqchip_split(kvm);
5141 }
5142
5143 static void svm_hwapic_irr_update(struct kvm_vcpu *vcpu, int max_irr)
5144 {
5145 }
5146
5147 static void svm_hwapic_isr_update(struct kvm_vcpu *vcpu, int max_isr)
5148 {
5149 }
5150
5151 /* Note: Currently only used by Hyper-V. */
5152 static void svm_refresh_apicv_exec_ctrl(struct kvm_vcpu *vcpu)
5153 {
5154         struct vcpu_svm *svm = to_svm(vcpu);
5155         struct vmcb *vmcb = svm->vmcb;
5156
5157         if (kvm_vcpu_apicv_active(vcpu))
5158                 vmcb->control.int_ctl |= AVIC_ENABLE_MASK;
5159         else
5160                 vmcb->control.int_ctl &= ~AVIC_ENABLE_MASK;
5161         mark_dirty(vmcb, VMCB_AVIC);
5162 }
5163
5164 static void svm_load_eoi_exitmap(struct kvm_vcpu *vcpu, u64 *eoi_exit_bitmap)
5165 {
5166         return;
5167 }
5168
5169 static void svm_deliver_avic_intr(struct kvm_vcpu *vcpu, int vec)
5170 {
5171         kvm_lapic_set_irr(vec, vcpu->arch.apic);
5172         smp_mb__after_atomic();
5173
5174         if (avic_vcpu_is_running(vcpu)) {
5175                 int cpuid = vcpu->cpu;
5176
5177                 if (cpuid != get_cpu())
5178                         wrmsrl(SVM_AVIC_DOORBELL, kvm_cpu_get_apicid(cpuid));
5179                 put_cpu();
5180         } else
5181                 kvm_vcpu_wake_up(vcpu);
5182 }
5183
5184 static bool svm_dy_apicv_has_pending_interrupt(struct kvm_vcpu *vcpu)
5185 {
5186         return false;
5187 }
5188
5189 static void svm_ir_list_del(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5190 {
5191         unsigned long flags;
5192         struct amd_svm_iommu_ir *cur;
5193
5194         spin_lock_irqsave(&svm->ir_list_lock, flags);
5195         list_for_each_entry(cur, &svm->ir_list, node) {
5196                 if (cur->data != pi->ir_data)
5197                         continue;
5198                 list_del(&cur->node);
5199                 kfree(cur);
5200                 break;
5201         }
5202         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5203 }
5204
5205 static int svm_ir_list_add(struct vcpu_svm *svm, struct amd_iommu_pi_data *pi)
5206 {
5207         int ret = 0;
5208         unsigned long flags;
5209         struct amd_svm_iommu_ir *ir;
5210
5211         /**
5212          * In some cases, the existing irte is updaed and re-set,
5213          * so we need to check here if it's already been * added
5214          * to the ir_list.
5215          */
5216         if (pi->ir_data && (pi->prev_ga_tag != 0)) {
5217                 struct kvm *kvm = svm->vcpu.kvm;
5218                 u32 vcpu_id = AVIC_GATAG_TO_VCPUID(pi->prev_ga_tag);
5219                 struct kvm_vcpu *prev_vcpu = kvm_get_vcpu_by_id(kvm, vcpu_id);
5220                 struct vcpu_svm *prev_svm;
5221
5222                 if (!prev_vcpu) {
5223                         ret = -EINVAL;
5224                         goto out;
5225                 }
5226
5227                 prev_svm = to_svm(prev_vcpu);
5228                 svm_ir_list_del(prev_svm, pi);
5229         }
5230
5231         /**
5232          * Allocating new amd_iommu_pi_data, which will get
5233          * add to the per-vcpu ir_list.
5234          */
5235         ir = kzalloc(sizeof(struct amd_svm_iommu_ir), GFP_KERNEL_ACCOUNT);
5236         if (!ir) {
5237                 ret = -ENOMEM;
5238                 goto out;
5239         }
5240         ir->data = pi->ir_data;
5241
5242         spin_lock_irqsave(&svm->ir_list_lock, flags);
5243         list_add(&ir->node, &svm->ir_list);
5244         spin_unlock_irqrestore(&svm->ir_list_lock, flags);
5245 out:
5246         return ret;
5247 }
5248
5249 /**
5250  * Note:
5251  * The HW cannot support posting multicast/broadcast
5252  * interrupts to a vCPU. So, we still use legacy interrupt
5253  * remapping for these kind of interrupts.
5254  *
5255  * For lowest-priority interrupts, we only support
5256  * those with single CPU as the destination, e.g. user
5257  * configures the interrupts via /proc/irq or uses
5258  * irqbalance to make the interrupts single-CPU.
5259  */
5260 static int
5261 get_pi_vcpu_info(struct kvm *kvm, struct kvm_kernel_irq_routing_entry *e,
5262                  struct vcpu_data *vcpu_info, struct vcpu_svm **svm)
5263 {
5264         struct kvm_lapic_irq irq;
5265         struct kvm_vcpu *vcpu = NULL;
5266
5267         kvm_set_msi_irq(kvm, e, &irq);
5268
5269         if (!kvm_intr_is_single_vcpu(kvm, &irq, &vcpu) ||
5270             !kvm_irq_is_postable(&irq)) {
5271                 pr_debug("SVM: %s: use legacy intr remap mode for irq %u\n",
5272                          __func__, irq.vector);
5273                 return -1;
5274         }
5275
5276         pr_debug("SVM: %s: use GA mode for irq %u\n", __func__,
5277                  irq.vector);
5278         *svm = to_svm(vcpu);
5279         vcpu_info->pi_desc_addr = __sme_set(page_to_phys((*svm)->avic_backing_page));
5280         vcpu_info->vector = irq.vector;
5281
5282         return 0;
5283 }
5284
5285 /*
5286  * svm_update_pi_irte - set IRTE for Posted-Interrupts
5287  *
5288  * @kvm: kvm
5289  * @host_irq: host irq of the interrupt
5290  * @guest_irq: gsi of the interrupt
5291  * @set: set or unset PI
5292  * returns 0 on success, < 0 on failure
5293  */
5294 static int svm_update_pi_irte(struct kvm *kvm, unsigned int host_irq,
5295                               uint32_t guest_irq, bool set)
5296 {
5297         struct kvm_kernel_irq_routing_entry *e;
5298         struct kvm_irq_routing_table *irq_rt;
5299         int idx, ret = -EINVAL;
5300
5301         if (!kvm_arch_has_assigned_device(kvm) ||
5302             !irq_remapping_cap(IRQ_POSTING_CAP))
5303                 return 0;
5304
5305         pr_debug("SVM: %s: host_irq=%#x, guest_irq=%#x, set=%#x\n",
5306                  __func__, host_irq, guest_irq, set);
5307
5308         idx = srcu_read_lock(&kvm->irq_srcu);
5309         irq_rt = srcu_dereference(kvm->irq_routing, &kvm->irq_srcu);
5310         WARN_ON(guest_irq >= irq_rt->nr_rt_entries);
5311
5312         hlist_for_each_entry(e, &irq_rt->map[guest_irq], link) {
5313                 struct vcpu_data vcpu_info;
5314                 struct vcpu_svm *svm = NULL;
5315
5316                 if (e->type != KVM_IRQ_ROUTING_MSI)
5317                         continue;
5318
5319                 /**
5320                  * Here, we setup with legacy mode in the following cases:
5321                  * 1. When cannot target interrupt to a specific vcpu.
5322                  * 2. Unsetting posted interrupt.
5323                  * 3. APIC virtialization is disabled for the vcpu.
5324                  * 4. IRQ has incompatible delivery mode (SMI, INIT, etc)
5325                  */
5326                 if (!get_pi_vcpu_info(kvm, e, &vcpu_info, &svm) && set &&
5327                     kvm_vcpu_apicv_active(&svm->vcpu)) {
5328                         struct amd_iommu_pi_data pi;
5329
5330                         /* Try to enable guest_mode in IRTE */
5331                         pi.base = __sme_set(page_to_phys(svm->avic_backing_page) &
5332                                             AVIC_HPA_MASK);
5333                         pi.ga_tag = AVIC_GATAG(to_kvm_svm(kvm)->avic_vm_id,
5334                                                      svm->vcpu.vcpu_id);
5335                         pi.is_guest_mode = true;
5336                         pi.vcpu_data = &vcpu_info;
5337                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5338
5339                         /**
5340                          * Here, we successfully setting up vcpu affinity in
5341                          * IOMMU guest mode. Now, we need to store the posted
5342                          * interrupt information in a per-vcpu ir_list so that
5343                          * we can reference to them directly when we update vcpu
5344                          * scheduling information in IOMMU irte.
5345                          */
5346                         if (!ret && pi.is_guest_mode)
5347                                 svm_ir_list_add(svm, &pi);
5348                 } else {
5349                         /* Use legacy mode in IRTE */
5350                         struct amd_iommu_pi_data pi;
5351
5352                         /**
5353                          * Here, pi is used to:
5354                          * - Tell IOMMU to use legacy mode for this interrupt.
5355                          * - Retrieve ga_tag of prior interrupt remapping data.
5356                          */
5357                         pi.is_guest_mode = false;
5358                         ret = irq_set_vcpu_affinity(host_irq, &pi);
5359
5360                         /**
5361                          * Check if the posted interrupt was previously
5362                          * setup with the guest_mode by checking if the ga_tag
5363                          * was cached. If so, we need to clean up the per-vcpu
5364                          * ir_list.
5365                          */
5366                         if (!ret && pi.prev_ga_tag) {
5367                                 int id = AVIC_GATAG_TO_VCPUID(pi.prev_ga_tag);
5368                                 struct kvm_vcpu *vcpu;
5369
5370                                 vcpu = kvm_get_vcpu_by_id(kvm, id);
5371                                 if (vcpu)
5372                                         svm_ir_list_del(to_svm(vcpu), &pi);
5373                         }
5374                 }
5375
5376                 if (!ret && svm) {
5377                         trace_kvm_pi_irte_update(host_irq, svm->vcpu.vcpu_id,
5378                                                  e->gsi, vcpu_info.vector,
5379                                                  vcpu_info.pi_desc_addr, set);
5380                 }
5381
5382                 if (ret < 0) {
5383                         pr_err("%s: failed to update PI IRTE\n", __func__);
5384                         goto out;
5385                 }
5386         }
5387
5388         ret = 0;
5389 out:
5390         srcu_read_unlock(&kvm->irq_srcu, idx);
5391         return ret;
5392 }
5393
5394 static int svm_nmi_allowed(struct kvm_vcpu *vcpu)
5395 {
5396         struct vcpu_svm *svm = to_svm(vcpu);
5397         struct vmcb *vmcb = svm->vmcb;
5398         int ret;
5399         ret = !(vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK) &&
5400               !(svm->vcpu.arch.hflags & HF_NMI_MASK);
5401         ret = ret && gif_set(svm) && nested_svm_nmi(svm);
5402
5403         return ret;
5404 }
5405
5406 static bool svm_get_nmi_mask(struct kvm_vcpu *vcpu)
5407 {
5408         struct vcpu_svm *svm = to_svm(vcpu);
5409
5410         return !!(svm->vcpu.arch.hflags & HF_NMI_MASK);
5411 }
5412
5413 static void svm_set_nmi_mask(struct kvm_vcpu *vcpu, bool masked)
5414 {
5415         struct vcpu_svm *svm = to_svm(vcpu);
5416
5417         if (masked) {
5418                 svm->vcpu.arch.hflags |= HF_NMI_MASK;
5419                 set_intercept(svm, INTERCEPT_IRET);
5420         } else {
5421                 svm->vcpu.arch.hflags &= ~HF_NMI_MASK;
5422                 clr_intercept(svm, INTERCEPT_IRET);
5423         }
5424 }
5425
5426 static int svm_interrupt_allowed(struct kvm_vcpu *vcpu)
5427 {
5428         struct vcpu_svm *svm = to_svm(vcpu);
5429         struct vmcb *vmcb = svm->vmcb;
5430         int ret;
5431
5432         if (!gif_set(svm) ||
5433              (vmcb->control.int_state & SVM_INTERRUPT_SHADOW_MASK))
5434                 return 0;
5435
5436         ret = !!(kvm_get_rflags(vcpu) & X86_EFLAGS_IF);
5437
5438         if (is_guest_mode(vcpu))
5439                 return ret && !(svm->vcpu.arch.hflags & HF_VINTR_MASK);
5440
5441         return ret;
5442 }
5443
5444 static void enable_irq_window(struct kvm_vcpu *vcpu)
5445 {
5446         struct vcpu_svm *svm = to_svm(vcpu);
5447
5448         if (kvm_vcpu_apicv_active(vcpu))
5449                 return;
5450
5451         /*
5452          * In case GIF=0 we can't rely on the CPU to tell us when GIF becomes
5453          * 1, because that's a separate STGI/VMRUN intercept.  The next time we
5454          * get that intercept, this function will be called again though and
5455          * we'll get the vintr intercept. However, if the vGIF feature is
5456          * enabled, the STGI interception will not occur. Enable the irq
5457          * window under the assumption that the hardware will set the GIF.
5458          */
5459         if ((vgif_enabled(svm) || gif_set(svm)) && nested_svm_intr(svm)) {
5460                 svm_set_vintr(svm);
5461                 svm_inject_irq(svm, 0x0);
5462         }
5463 }
5464
5465 static void enable_nmi_window(struct kvm_vcpu *vcpu)
5466 {
5467         struct vcpu_svm *svm = to_svm(vcpu);
5468
5469         if ((svm->vcpu.arch.hflags & (HF_NMI_MASK | HF_IRET_MASK))
5470             == HF_NMI_MASK)
5471                 return; /* IRET will cause a vm exit */
5472
5473         if (!gif_set(svm)) {
5474                 if (vgif_enabled(svm))
5475                         set_intercept(svm, INTERCEPT_STGI);
5476                 return; /* STGI will cause a vm exit */
5477         }
5478
5479         if (svm->nested.exit_required)
5480                 return; /* we're not going to run the guest yet */
5481
5482         /*
5483          * Something prevents NMI from been injected. Single step over possible
5484          * problem (IRET or exception injection or interrupt shadow)
5485          */
5486         svm->nmi_singlestep_guest_rflags = svm_get_rflags(vcpu);
5487         svm->nmi_singlestep = true;
5488         svm->vmcb->save.rflags |= (X86_EFLAGS_TF | X86_EFLAGS_RF);
5489 }
5490
5491 static int svm_set_tss_addr(struct kvm *kvm, unsigned int addr)
5492 {
5493         return 0;
5494 }
5495
5496 static int svm_set_identity_map_addr(struct kvm *kvm, u64 ident_addr)
5497 {
5498         return 0;
5499 }
5500
5501 static void svm_flush_tlb(struct kvm_vcpu *vcpu, bool invalidate_gpa)
5502 {
5503         struct vcpu_svm *svm = to_svm(vcpu);
5504
5505         if (static_cpu_has(X86_FEATURE_FLUSHBYASID))
5506                 svm->vmcb->control.tlb_ctl = TLB_CONTROL_FLUSH_ASID;
5507         else
5508                 svm->asid_generation--;
5509 }
5510
5511 static void svm_flush_tlb_gva(struct kvm_vcpu *vcpu, gva_t gva)
5512 {
5513         struct vcpu_svm *svm = to_svm(vcpu);
5514
5515         invlpga(gva, svm->vmcb->control.asid);
5516 }
5517
5518 static void svm_prepare_guest_switch(struct kvm_vcpu *vcpu)
5519 {
5520 }
5521
5522 static inline void sync_cr8_to_lapic(struct kvm_vcpu *vcpu)
5523 {
5524         struct vcpu_svm *svm = to_svm(vcpu);
5525
5526         if (svm_nested_virtualize_tpr(vcpu))
5527                 return;
5528
5529         if (!is_cr_intercept(svm, INTERCEPT_CR8_WRITE)) {
5530                 int cr8 = svm->vmcb->control.int_ctl & V_TPR_MASK;
5531                 kvm_set_cr8(vcpu, cr8);
5532         }
5533 }
5534
5535 static inline void sync_lapic_to_cr8(struct kvm_vcpu *vcpu)
5536 {
5537         struct vcpu_svm *svm = to_svm(vcpu);
5538         u64 cr8;
5539
5540         if (svm_nested_virtualize_tpr(vcpu) ||
5541             kvm_vcpu_apicv_active(vcpu))
5542                 return;
5543
5544         cr8 = kvm_get_cr8(vcpu);
5545         svm->vmcb->control.int_ctl &= ~V_TPR_MASK;
5546         svm->vmcb->control.int_ctl |= cr8 & V_TPR_MASK;
5547 }
5548
5549 static void svm_complete_interrupts(struct vcpu_svm *svm)
5550 {
5551         u8 vector;
5552         int type;
5553         u32 exitintinfo = svm->vmcb->control.exit_int_info;
5554         unsigned int3_injected = svm->int3_injected;
5555
5556         svm->int3_injected = 0;
5557
5558         /*
5559          * If we've made progress since setting HF_IRET_MASK, we've
5560          * executed an IRET and can allow NMI injection.
5561          */
5562         if ((svm->vcpu.arch.hflags & HF_IRET_MASK)
5563             && kvm_rip_read(&svm->vcpu) != svm->nmi_iret_rip) {
5564                 svm->vcpu.arch.hflags &= ~(HF_NMI_MASK | HF_IRET_MASK);
5565                 kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5566         }
5567
5568         svm->vcpu.arch.nmi_injected = false;
5569         kvm_clear_exception_queue(&svm->vcpu);
5570         kvm_clear_interrupt_queue(&svm->vcpu);
5571
5572         if (!(exitintinfo & SVM_EXITINTINFO_VALID))
5573                 return;
5574
5575         kvm_make_request(KVM_REQ_EVENT, &svm->vcpu);
5576
5577         vector = exitintinfo & SVM_EXITINTINFO_VEC_MASK;
5578         type = exitintinfo & SVM_EXITINTINFO_TYPE_MASK;
5579
5580         switch (type) {
5581         case SVM_EXITINTINFO_TYPE_NMI:
5582                 svm->vcpu.arch.nmi_injected = true;
5583                 break;
5584         case SVM_EXITINTINFO_TYPE_EXEPT:
5585                 /*
5586                  * In case of software exceptions, do not reinject the vector,
5587                  * but re-execute the instruction instead. Rewind RIP first
5588                  * if we emulated INT3 before.
5589                  */
5590                 if (kvm_exception_is_soft(vector)) {
5591                         if (vector == BP_VECTOR && int3_injected &&
5592                             kvm_is_linear_rip(&svm->vcpu, svm->int3_rip))
5593                                 kvm_rip_write(&svm->vcpu,
5594                                               kvm_rip_read(&svm->vcpu) -
5595                                               int3_injected);
5596                         break;
5597                 }
5598                 if (exitintinfo & SVM_EXITINTINFO_VALID_ERR) {
5599                         u32 err = svm->vmcb->control.exit_int_info_err;
5600                         kvm_requeue_exception_e(&svm->vcpu, vector, err);
5601
5602                 } else
5603                         kvm_requeue_exception(&svm->vcpu, vector);
5604                 break;
5605         case SVM_EXITINTINFO_TYPE_INTR:
5606                 kvm_queue_interrupt(&svm->vcpu, vector, false);
5607                 break;
5608         default:
5609                 break;
5610         }
5611 }
5612
5613 static void svm_cancel_injection(struct kvm_vcpu *vcpu)
5614 {
5615         struct vcpu_svm *svm = to_svm(vcpu);
5616         struct vmcb_control_area *control = &svm->vmcb->control;
5617
5618         control->exit_int_info = control->event_inj;
5619         control->exit_int_info_err = control->event_inj_err;
5620         control->event_inj = 0;
5621         svm_complete_interrupts(svm);
5622 }
5623
5624 static void svm_vcpu_run(struct kvm_vcpu *vcpu)
5625 {
5626         struct vcpu_svm *svm = to_svm(vcpu);
5627
5628         svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
5629         svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
5630         svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
5631
5632         /*
5633          * A vmexit emulation is required before the vcpu can be executed
5634          * again.
5635          */
5636         if (unlikely(svm->nested.exit_required))
5637                 return;
5638
5639         /*
5640          * Disable singlestep if we're injecting an interrupt/exception.
5641          * We don't want our modified rflags to be pushed on the stack where
5642          * we might not be able to easily reset them if we disabled NMI
5643          * singlestep later.
5644          */
5645         if (svm->nmi_singlestep && svm->vmcb->control.event_inj) {
5646                 /*
5647                  * Event injection happens before external interrupts cause a
5648                  * vmexit and interrupts are disabled here, so smp_send_reschedule
5649                  * is enough to force an immediate vmexit.
5650                  */
5651                 disable_nmi_singlestep(svm);
5652                 smp_send_reschedule(vcpu->cpu);
5653         }
5654
5655         pre_svm_run(svm);
5656
5657         sync_lapic_to_cr8(vcpu);
5658
5659         svm->vmcb->save.cr2 = vcpu->arch.cr2;
5660
5661         clgi();
5662         kvm_load_guest_xsave_state(vcpu);
5663
5664         if (lapic_in_kernel(vcpu) &&
5665                 vcpu->arch.apic->lapic_timer.timer_advance_ns)
5666                 kvm_wait_lapic_expire(vcpu);
5667
5668         /*
5669          * If this vCPU has touched SPEC_CTRL, restore the guest's value if
5670          * it's non-zero. Since vmentry is serialising on affected CPUs, there
5671          * is no need to worry about the conditional branch over the wrmsr
5672          * being speculatively taken.
5673          */
5674         x86_spec_ctrl_set_guest(svm->spec_ctrl, svm->virt_spec_ctrl);
5675
5676         local_irq_enable();
5677
5678         asm volatile (
5679                 "push %%" _ASM_BP "; \n\t"
5680                 "mov %c[rbx](%[svm]), %%" _ASM_BX " \n\t"
5681                 "mov %c[rcx](%[svm]), %%" _ASM_CX " \n\t"
5682                 "mov %c[rdx](%[svm]), %%" _ASM_DX " \n\t"
5683                 "mov %c[rsi](%[svm]), %%" _ASM_SI " \n\t"
5684                 "mov %c[rdi](%[svm]), %%" _ASM_DI " \n\t"
5685                 "mov %c[rbp](%[svm]), %%" _ASM_BP " \n\t"
5686 #ifdef CONFIG_X86_64
5687                 "mov %c[r8](%[svm]),  %%r8  \n\t"
5688                 "mov %c[r9](%[svm]),  %%r9  \n\t"
5689                 "mov %c[r10](%[svm]), %%r10 \n\t"
5690                 "mov %c[r11](%[svm]), %%r11 \n\t"
5691                 "mov %c[r12](%[svm]), %%r12 \n\t"
5692                 "mov %c[r13](%[svm]), %%r13 \n\t"
5693                 "mov %c[r14](%[svm]), %%r14 \n\t"
5694                 "mov %c[r15](%[svm]), %%r15 \n\t"
5695 #endif
5696
5697                 /* Enter guest mode */
5698                 "push %%" _ASM_AX " \n\t"
5699                 "mov %c[vmcb](%[svm]), %%" _ASM_AX " \n\t"
5700                 __ex("vmload %%" _ASM_AX) "\n\t"
5701                 __ex("vmrun %%" _ASM_AX) "\n\t"
5702                 __ex("vmsave %%" _ASM_AX) "\n\t"
5703                 "pop %%" _ASM_AX " \n\t"
5704
5705                 /* Save guest registers, load host registers */
5706                 "mov %%" _ASM_BX ", %c[rbx](%[svm]) \n\t"
5707                 "mov %%" _ASM_CX ", %c[rcx](%[svm]) \n\t"
5708                 "mov %%" _ASM_DX ", %c[rdx](%[svm]) \n\t"
5709                 "mov %%" _ASM_SI ", %c[rsi](%[svm]) \n\t"
5710                 "mov %%" _ASM_DI ", %c[rdi](%[svm]) \n\t"
5711                 "mov %%" _ASM_BP ", %c[rbp](%[svm]) \n\t"
5712 #ifdef CONFIG_X86_64
5713                 "mov %%r8,  %c[r8](%[svm]) \n\t"
5714                 "mov %%r9,  %c[r9](%[svm]) \n\t"
5715                 "mov %%r10, %c[r10](%[svm]) \n\t"
5716                 "mov %%r11, %c[r11](%[svm]) \n\t"
5717                 "mov %%r12, %c[r12](%[svm]) \n\t"
5718                 "mov %%r13, %c[r13](%[svm]) \n\t"
5719                 "mov %%r14, %c[r14](%[svm]) \n\t"
5720                 "mov %%r15, %c[r15](%[svm]) \n\t"
5721                 /*
5722                 * Clear host registers marked as clobbered to prevent
5723                 * speculative use.
5724                 */
5725                 "xor %%r8d, %%r8d \n\t"
5726                 "xor %%r9d, %%r9d \n\t"
5727                 "xor %%r10d, %%r10d \n\t"
5728                 "xor %%r11d, %%r11d \n\t"
5729                 "xor %%r12d, %%r12d \n\t"
5730                 "xor %%r13d, %%r13d \n\t"
5731                 "xor %%r14d, %%r14d \n\t"
5732                 "xor %%r15d, %%r15d \n\t"
5733 #endif
5734                 "xor %%ebx, %%ebx \n\t"
5735                 "xor %%ecx, %%ecx \n\t"
5736                 "xor %%edx, %%edx \n\t"
5737                 "xor %%esi, %%esi \n\t"
5738                 "xor %%edi, %%edi \n\t"
5739                 "pop %%" _ASM_BP
5740                 :
5741                 : [svm]"a"(svm),
5742                   [vmcb]"i"(offsetof(struct vcpu_svm, vmcb_pa)),
5743                   [rbx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBX])),
5744                   [rcx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RCX])),
5745                   [rdx]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDX])),
5746                   [rsi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RSI])),
5747                   [rdi]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RDI])),
5748                   [rbp]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_RBP]))
5749 #ifdef CONFIG_X86_64
5750                   , [r8]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R8])),
5751                   [r9]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R9])),
5752                   [r10]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R10])),
5753                   [r11]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R11])),
5754                   [r12]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R12])),
5755                   [r13]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R13])),
5756                   [r14]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R14])),
5757                   [r15]"i"(offsetof(struct vcpu_svm, vcpu.arch.regs[VCPU_REGS_R15]))
5758 #endif
5759                 : "cc", "memory"
5760 #ifdef CONFIG_X86_64
5761                 , "rbx", "rcx", "rdx", "rsi", "rdi"
5762                 , "r8", "r9", "r10", "r11" , "r12", "r13", "r14", "r15"
5763 #else
5764                 , "ebx", "ecx", "edx", "esi", "edi"
5765 #endif
5766                 );
5767
5768         /* Eliminate branch target predictions from guest mode */
5769         vmexit_fill_RSB();
5770
5771 #ifdef CONFIG_X86_64
5772         wrmsrl(MSR_GS_BASE, svm->host.gs_base);
5773 #else
5774         loadsegment(fs, svm->host.fs);
5775 #ifndef CONFIG_X86_32_LAZY_GS
5776         loadsegment(gs, svm->host.gs);
5777 #endif
5778 #endif
5779
5780         /*
5781          * We do not use IBRS in the kernel. If this vCPU has used the
5782          * SPEC_CTRL MSR it may have left it on; save the value and
5783          * turn it off. This is much more efficient than blindly adding
5784          * it to the atomic save/restore list. Especially as the former
5785          * (Saving guest MSRs on vmexit) doesn't even exist in KVM.
5786          *
5787          * For non-nested case:
5788          * If the L01 MSR bitmap does not intercept the MSR, then we need to
5789          * save it.
5790          *
5791          * For nested case:
5792          * If the L02 MSR bitmap does not intercept the MSR, then we need to
5793          * save it.
5794          */
5795         if (unlikely(!msr_write_intercepted(vcpu, MSR_IA32_SPEC_CTRL)))
5796                 svm->spec_ctrl = native_read_msr(MSR_IA32_SPEC_CTRL);
5797
5798         reload_tss(vcpu);
5799
5800         local_irq_disable();
5801
5802         x86_spec_ctrl_restore_host(svm->spec_ctrl, svm->virt_spec_ctrl);
5803
5804         vcpu->arch.cr2 = svm->vmcb->save.cr2;
5805         vcpu->arch.regs[VCPU_REGS_RAX] = svm->vmcb->save.rax;
5806         vcpu->arch.regs[VCPU_REGS_RSP] = svm->vmcb->save.rsp;
5807         vcpu->arch.regs[VCPU_REGS_RIP] = svm->vmcb->save.rip;
5808
5809         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5810                 kvm_before_interrupt(&svm->vcpu);
5811
5812         kvm_load_host_xsave_state(vcpu);
5813         stgi();
5814
5815         /* Any pending NMI will happen here */
5816
5817         if (unlikely(svm->vmcb->control.exit_code == SVM_EXIT_NMI))
5818                 kvm_after_interrupt(&svm->vcpu);
5819
5820         sync_cr8_to_lapic(vcpu);
5821
5822         svm->next_rip = 0;
5823
5824         svm->vmcb->control.tlb_ctl = TLB_CONTROL_DO_NOTHING;
5825
5826         /* if exit due to PF check for async PF */
5827         if (svm->vmcb->control.exit_code == SVM_EXIT_EXCP_BASE + PF_VECTOR)
5828                 svm->vcpu.arch.apf.host_apf_reason = kvm_read_and_reset_pf_reason();
5829
5830         if (npt_enabled) {
5831                 vcpu->arch.regs_avail &= ~(1 << VCPU_EXREG_PDPTR);
5832                 vcpu->arch.regs_dirty &= ~(1 << VCPU_EXREG_PDPTR);
5833         }
5834
5835         /*
5836          * We need to handle MC intercepts here before the vcpu has a chance to
5837          * change the physical cpu
5838          */
5839         if (unlikely(svm->vmcb->control.exit_code ==
5840                      SVM_EXIT_EXCP_BASE + MC_VECTOR))
5841                 svm_handle_mce(svm);
5842
5843         mark_all_clean(svm->vmcb);
5844 }
5845 STACK_FRAME_NON_STANDARD(svm_vcpu_run);
5846
5847 static void svm_set_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5848 {
5849         struct vcpu_svm *svm = to_svm(vcpu);
5850
5851         svm->vmcb->save.cr3 = __sme_set(root);
5852         mark_dirty(svm->vmcb, VMCB_CR);
5853 }
5854
5855 static void set_tdp_cr3(struct kvm_vcpu *vcpu, unsigned long root)
5856 {
5857         struct vcpu_svm *svm = to_svm(vcpu);
5858
5859         svm->vmcb->control.nested_cr3 = __sme_set(root);
5860         mark_dirty(svm->vmcb, VMCB_NPT);
5861
5862         /* Also sync guest cr3 here in case we live migrate */
5863         svm->vmcb->save.cr3 = kvm_read_cr3(vcpu);
5864         mark_dirty(svm->vmcb, VMCB_CR);
5865 }
5866
5867 static int is_disabled(void)
5868 {
5869         u64 vm_cr;
5870
5871         rdmsrl(MSR_VM_CR, vm_cr);
5872         if (vm_cr & (1 << SVM_VM_CR_SVM_DISABLE))
5873                 return 1;
5874
5875         return 0;
5876 }
5877
5878 static void
5879 svm_patch_hypercall(struct kvm_vcpu *vcpu, unsigned char *hypercall)
5880 {
5881         /*
5882          * Patch in the VMMCALL instruction:
5883          */
5884         hypercall[0] = 0x0f;
5885         hypercall[1] = 0x01;
5886         hypercall[2] = 0xd9;
5887 }
5888
5889 static int __init svm_check_processor_compat(void)
5890 {
5891         return 0;
5892 }
5893
5894 static bool svm_cpu_has_accelerated_tpr(void)
5895 {
5896         return false;
5897 }
5898
5899 static bool svm_has_emulated_msr(int index)
5900 {
5901         switch (index) {
5902         case MSR_IA32_MCG_EXT_CTL:
5903         case MSR_IA32_VMX_BASIC ... MSR_IA32_VMX_VMFUNC:
5904                 return false;
5905         default:
5906                 break;
5907         }
5908
5909         return true;
5910 }
5911
5912 static u64 svm_get_mt_mask(struct kvm_vcpu *vcpu, gfn_t gfn, bool is_mmio)
5913 {
5914         return 0;
5915 }
5916
5917 static void svm_cpuid_update(struct kvm_vcpu *vcpu)
5918 {
5919         struct vcpu_svm *svm = to_svm(vcpu);
5920
5921         vcpu->arch.xsaves_enabled = guest_cpuid_has(vcpu, X86_FEATURE_XSAVE) &&
5922                                     boot_cpu_has(X86_FEATURE_XSAVE) &&
5923                                     boot_cpu_has(X86_FEATURE_XSAVES);
5924
5925         /* Update nrips enabled cache */
5926         svm->nrips_enabled = !!guest_cpuid_has(&svm->vcpu, X86_FEATURE_NRIPS);
5927
5928         if (!kvm_vcpu_apicv_active(vcpu))
5929                 return;
5930
5931         guest_cpuid_clear(vcpu, X86_FEATURE_X2APIC);
5932 }
5933
5934 #define F feature_bit
5935
5936 static void svm_set_supported_cpuid(u32 func, struct kvm_cpuid_entry2 *entry)
5937 {
5938         switch (func) {
5939         case 0x1:
5940                 if (avic)
5941                         entry->ecx &= ~F(X2APIC);
5942                 break;
5943         case 0x80000001:
5944                 if (nested)
5945                         entry->ecx |= (1 << 2); /* Set SVM bit */
5946                 break;
5947         case 0x80000008:
5948                 if (boot_cpu_has(X86_FEATURE_LS_CFG_SSBD) ||
5949                      boot_cpu_has(X86_FEATURE_AMD_SSBD))
5950                         entry->ebx |= F(VIRT_SSBD);
5951                 break;
5952         case 0x8000000A:
5953                 entry->eax = 1; /* SVM revision 1 */
5954                 entry->ebx = 8; /* Lets support 8 ASIDs in case we add proper
5955                                    ASID emulation to nested SVM */
5956                 entry->ecx = 0; /* Reserved */
5957                 entry->edx = 0; /* Per default do not support any
5958                                    additional features */
5959
5960                 /* Support next_rip if host supports it */
5961                 if (boot_cpu_has(X86_FEATURE_NRIPS))
5962                         entry->edx |= F(NRIPS);
5963
5964                 /* Support NPT for the guest if enabled */
5965                 if (npt_enabled)
5966                         entry->edx |= F(NPT);
5967
5968         }
5969 }
5970
5971 static int svm_get_lpage_level(void)
5972 {
5973         return PT_PDPE_LEVEL;
5974 }
5975
5976 static bool svm_rdtscp_supported(void)
5977 {
5978         return boot_cpu_has(X86_FEATURE_RDTSCP);
5979 }
5980
5981 static bool svm_invpcid_supported(void)
5982 {
5983         return false;
5984 }
5985
5986 static bool svm_mpx_supported(void)
5987 {
5988         return false;
5989 }
5990
5991 static bool svm_xsaves_supported(void)
5992 {
5993         return boot_cpu_has(X86_FEATURE_XSAVES);
5994 }
5995
5996 static bool svm_umip_emulated(void)
5997 {
5998         return false;
5999 }
6000
6001 static bool svm_pt_supported(void)
6002 {
6003         return false;
6004 }
6005
6006 static bool svm_has_wbinvd_exit(void)
6007 {
6008         return true;
6009 }
6010
6011 static bool svm_pku_supported(void)
6012 {
6013         return false;
6014 }
6015
6016 #define PRE_EX(exit)  { .exit_code = (exit), \
6017                         .stage = X86_ICPT_PRE_EXCEPT, }
6018 #define POST_EX(exit) { .exit_code = (exit), \
6019                         .stage = X86_ICPT_POST_EXCEPT, }
6020 #define POST_MEM(exit) { .exit_code = (exit), \
6021                         .stage = X86_ICPT_POST_MEMACCESS, }
6022
6023 static const struct __x86_intercept {
6024         u32 exit_code;
6025         enum x86_intercept_stage stage;
6026 } x86_intercept_map[] = {
6027         [x86_intercept_cr_read]         = POST_EX(SVM_EXIT_READ_CR0),
6028         [x86_intercept_cr_write]        = POST_EX(SVM_EXIT_WRITE_CR0),
6029         [x86_intercept_clts]            = POST_EX(SVM_EXIT_WRITE_CR0),
6030         [x86_intercept_lmsw]            = POST_EX(SVM_EXIT_WRITE_CR0),
6031         [x86_intercept_smsw]            = POST_EX(SVM_EXIT_READ_CR0),
6032         [x86_intercept_dr_read]         = POST_EX(SVM_EXIT_READ_DR0),
6033         [x86_intercept_dr_write]        = POST_EX(SVM_EXIT_WRITE_DR0),
6034         [x86_intercept_sldt]            = POST_EX(SVM_EXIT_LDTR_READ),
6035         [x86_intercept_str]             = POST_EX(SVM_EXIT_TR_READ),
6036         [x86_intercept_lldt]            = POST_EX(SVM_EXIT_LDTR_WRITE),
6037         [x86_intercept_ltr]             = POST_EX(SVM_EXIT_TR_WRITE),
6038         [x86_intercept_sgdt]            = POST_EX(SVM_EXIT_GDTR_READ),
6039         [x86_intercept_sidt]            = POST_EX(SVM_EXIT_IDTR_READ),
6040         [x86_intercept_lgdt]            = POST_EX(SVM_EXIT_GDTR_WRITE),
6041         [x86_intercept_lidt]            = POST_EX(SVM_EXIT_IDTR_WRITE),
6042         [x86_intercept_vmrun]           = POST_EX(SVM_EXIT_VMRUN),
6043         [x86_intercept_vmmcall]         = POST_EX(SVM_EXIT_VMMCALL),
6044         [x86_intercept_vmload]          = POST_EX(SVM_EXIT_VMLOAD),
6045         [x86_intercept_vmsave]          = POST_EX(SVM_EXIT_VMSAVE),
6046         [x86_intercept_stgi]            = POST_EX(SVM_EXIT_STGI),
6047         [x86_intercept_clgi]            = POST_EX(SVM_EXIT_CLGI),
6048         [x86_intercept_skinit]          = POST_EX(SVM_EXIT_SKINIT),
6049         [x86_intercept_invlpga]         = POST_EX(SVM_EXIT_INVLPGA),
6050         [x86_intercept_rdtscp]          = POST_EX(SVM_EXIT_RDTSCP),
6051         [x86_intercept_monitor]         = POST_MEM(SVM_EXIT_MONITOR),
6052         [x86_intercept_mwait]           = POST_EX(SVM_EXIT_MWAIT),
6053         [x86_intercept_invlpg]          = POST_EX(SVM_EXIT_INVLPG),
6054         [x86_intercept_invd]            = POST_EX(SVM_EXIT_INVD),
6055         [x86_intercept_wbinvd]          = POST_EX(SVM_EXIT_WBINVD),
6056         [x86_intercept_wrmsr]           = POST_EX(SVM_EXIT_MSR),
6057         [x86_intercept_rdtsc]           = POST_EX(SVM_EXIT_RDTSC),
6058         [x86_intercept_rdmsr]           = POST_EX(SVM_EXIT_MSR),
6059         [x86_intercept_rdpmc]           = POST_EX(SVM_EXIT_RDPMC),
6060         [x86_intercept_cpuid]           = PRE_EX(SVM_EXIT_CPUID),
6061         [x86_intercept_rsm]             = PRE_EX(SVM_EXIT_RSM),
6062         [x86_intercept_pause]           = PRE_EX(SVM_EXIT_PAUSE),
6063         [x86_intercept_pushf]           = PRE_EX(SVM_EXIT_PUSHF),
6064         [x86_intercept_popf]            = PRE_EX(SVM_EXIT_POPF),
6065         [x86_intercept_intn]            = PRE_EX(SVM_EXIT_SWINT),
6066         [x86_intercept_iret]            = PRE_EX(SVM_EXIT_IRET),
6067         [x86_intercept_icebp]           = PRE_EX(SVM_EXIT_ICEBP),
6068         [x86_intercept_hlt]             = POST_EX(SVM_EXIT_HLT),
6069         [x86_intercept_in]              = POST_EX(SVM_EXIT_IOIO),
6070         [x86_intercept_ins]             = POST_EX(SVM_EXIT_IOIO),
6071         [x86_intercept_out]             = POST_EX(SVM_EXIT_IOIO),
6072         [x86_intercept_outs]            = POST_EX(SVM_EXIT_IOIO),
6073         [x86_intercept_xsetbv]          = PRE_EX(SVM_EXIT_XSETBV),
6074 };
6075
6076 #undef PRE_EX
6077 #undef POST_EX
6078 #undef POST_MEM
6079
6080 static int svm_check_intercept(struct kvm_vcpu *vcpu,
6081                                struct x86_instruction_info *info,
6082                                enum x86_intercept_stage stage)
6083 {
6084         struct vcpu_svm *svm = to_svm(vcpu);
6085         int vmexit, ret = X86EMUL_CONTINUE;
6086         struct __x86_intercept icpt_info;
6087         struct vmcb *vmcb = svm->vmcb;
6088
6089         if (info->intercept >= ARRAY_SIZE(x86_intercept_map))
6090                 goto out;
6091
6092         icpt_info = x86_intercept_map[info->intercept];
6093
6094         if (stage != icpt_info.stage)
6095                 goto out;
6096
6097         switch (icpt_info.exit_code) {
6098         case SVM_EXIT_READ_CR0:
6099                 if (info->intercept == x86_intercept_cr_read)
6100                         icpt_info.exit_code += info->modrm_reg;
6101                 break;
6102         case SVM_EXIT_WRITE_CR0: {
6103                 unsigned long cr0, val;
6104                 u64 intercept;
6105
6106                 if (info->intercept == x86_intercept_cr_write)
6107                         icpt_info.exit_code += info->modrm_reg;
6108
6109                 if (icpt_info.exit_code != SVM_EXIT_WRITE_CR0 ||
6110                     info->intercept == x86_intercept_clts)
6111                         break;
6112
6113                 intercept = svm->nested.intercept;
6114
6115                 if (!(intercept & (1ULL << INTERCEPT_SELECTIVE_CR0)))
6116                         break;
6117
6118                 cr0 = vcpu->arch.cr0 & ~SVM_CR0_SELECTIVE_MASK;
6119                 val = info->src_val  & ~SVM_CR0_SELECTIVE_MASK;
6120
6121                 if (info->intercept == x86_intercept_lmsw) {
6122                         cr0 &= 0xfUL;
6123                         val &= 0xfUL;
6124                         /* lmsw can't clear PE - catch this here */
6125                         if (cr0 & X86_CR0_PE)
6126                                 val |= X86_CR0_PE;
6127                 }
6128
6129                 if (cr0 ^ val)
6130                         icpt_info.exit_code = SVM_EXIT_CR0_SEL_WRITE;
6131
6132                 break;
6133         }
6134         case SVM_EXIT_READ_DR0:
6135         case SVM_EXIT_WRITE_DR0:
6136                 icpt_info.exit_code += info->modrm_reg;
6137                 break;
6138         case SVM_EXIT_MSR:
6139                 if (info->intercept == x86_intercept_wrmsr)
6140                         vmcb->control.exit_info_1 = 1;
6141                 else
6142                         vmcb->control.exit_info_1 = 0;
6143                 break;
6144         case SVM_EXIT_PAUSE:
6145                 /*
6146                  * We get this for NOP only, but pause
6147                  * is rep not, check this here
6148                  */
6149                 if (info->rep_prefix != REPE_PREFIX)
6150                         goto out;
6151                 break;
6152         case SVM_EXIT_IOIO: {
6153                 u64 exit_info;
6154                 u32 bytes;
6155
6156                 if (info->intercept == x86_intercept_in ||
6157                     info->intercept == x86_intercept_ins) {
6158                         exit_info = ((info->src_val & 0xffff) << 16) |
6159                                 SVM_IOIO_TYPE_MASK;
6160                         bytes = info->dst_bytes;
6161                 } else {
6162                         exit_info = (info->dst_val & 0xffff) << 16;
6163                         bytes = info->src_bytes;
6164                 }
6165
6166                 if (info->intercept == x86_intercept_outs ||
6167                     info->intercept == x86_intercept_ins)
6168                         exit_info |= SVM_IOIO_STR_MASK;
6169
6170                 if (info->rep_prefix)
6171                         exit_info |= SVM_IOIO_REP_MASK;
6172
6173                 bytes = min(bytes, 4u);
6174
6175                 exit_info |= bytes << SVM_IOIO_SIZE_SHIFT;
6176
6177                 exit_info |= (u32)info->ad_bytes << (SVM_IOIO_ASIZE_SHIFT - 1);
6178
6179                 vmcb->control.exit_info_1 = exit_info;
6180                 vmcb->control.exit_info_2 = info->next_rip;
6181
6182                 break;
6183         }
6184         default:
6185                 break;
6186         }
6187
6188         /* TODO: Advertise NRIPS to guest hypervisor unconditionally */
6189         if (static_cpu_has(X86_FEATURE_NRIPS))
6190                 vmcb->control.next_rip  = info->next_rip;
6191         vmcb->control.exit_code = icpt_info.exit_code;
6192         vmexit = nested_svm_exit_handled(svm);
6193
6194         ret = (vmexit == NESTED_EXIT_DONE) ? X86EMUL_INTERCEPTED
6195                                            : X86EMUL_CONTINUE;
6196
6197 out:
6198         return ret;
6199 }
6200
6201 static void svm_handle_exit_irqoff(struct kvm_vcpu *vcpu,
6202         enum exit_fastpath_completion *exit_fastpath)
6203 {
6204         if (!is_guest_mode(vcpu) &&
6205                 to_svm(vcpu)->vmcb->control.exit_code == EXIT_REASON_MSR_WRITE)
6206                 *exit_fastpath = handle_fastpath_set_msr_irqoff(vcpu);
6207 }
6208
6209 static void svm_sched_in(struct kvm_vcpu *vcpu, int cpu)
6210 {
6211         if (pause_filter_thresh)
6212                 shrink_ple_window(vcpu);
6213 }
6214
6215 static inline void avic_post_state_restore(struct kvm_vcpu *vcpu)
6216 {
6217         if (avic_handle_apic_id_update(vcpu) != 0)
6218                 return;
6219         avic_handle_dfr_update(vcpu);
6220         avic_handle_ldr_update(vcpu);
6221 }
6222
6223 static void svm_setup_mce(struct kvm_vcpu *vcpu)
6224 {
6225         /* [63:9] are reserved. */
6226         vcpu->arch.mcg_cap &= 0x1ff;
6227 }
6228
6229 static int svm_smi_allowed(struct kvm_vcpu *vcpu)
6230 {
6231         struct vcpu_svm *svm = to_svm(vcpu);
6232
6233         /* Per APM Vol.2 15.22.2 "Response to SMI" */
6234         if (!gif_set(svm))
6235                 return 0;
6236
6237         if (is_guest_mode(&svm->vcpu) &&
6238             svm->nested.intercept & (1ULL << INTERCEPT_SMI)) {
6239                 /* TODO: Might need to set exit_info_1 and exit_info_2 here */
6240                 svm->vmcb->control.exit_code = SVM_EXIT_SMI;
6241                 svm->nested.exit_required = true;
6242                 return 0;
6243         }
6244
6245         return 1;
6246 }
6247
6248 static int svm_pre_enter_smm(struct kvm_vcpu *vcpu, char *smstate)
6249 {
6250         struct vcpu_svm *svm = to_svm(vcpu);
6251         int ret;
6252
6253         if (is_guest_mode(vcpu)) {
6254                 /* FED8h - SVM Guest */
6255                 put_smstate(u64, smstate, 0x7ed8, 1);
6256                 /* FEE0h - SVM Guest VMCB Physical Address */
6257                 put_smstate(u64, smstate, 0x7ee0, svm->nested.vmcb);
6258
6259                 svm->vmcb->save.rax = vcpu->arch.regs[VCPU_REGS_RAX];
6260                 svm->vmcb->save.rsp = vcpu->arch.regs[VCPU_REGS_RSP];
6261                 svm->vmcb->save.rip = vcpu->arch.regs[VCPU_REGS_RIP];
6262
6263                 ret = nested_svm_vmexit(svm);
6264                 if (ret)
6265                         return ret;
6266         }
6267         return 0;
6268 }
6269
6270 static int svm_pre_leave_smm(struct kvm_vcpu *vcpu, const char *smstate)
6271 {
6272         struct vcpu_svm *svm = to_svm(vcpu);
6273         struct vmcb *nested_vmcb;
6274         struct kvm_host_map map;
6275         u64 guest;
6276         u64 vmcb;
6277
6278         guest = GET_SMSTATE(u64, smstate, 0x7ed8);
6279         vmcb = GET_SMSTATE(u64, smstate, 0x7ee0);
6280
6281         if (guest) {
6282                 if (kvm_vcpu_map(&svm->vcpu, gpa_to_gfn(vmcb), &map) == -EINVAL)
6283                         return 1;
6284                 nested_vmcb = map.hva;
6285                 enter_svm_guest_mode(svm, vmcb, nested_vmcb, &map);
6286         }
6287         return 0;
6288 }
6289
6290 static int enable_smi_window(struct kvm_vcpu *vcpu)
6291 {
6292         struct vcpu_svm *svm = to_svm(vcpu);
6293
6294         if (!gif_set(svm)) {
6295                 if (vgif_enabled(svm))
6296                         set_intercept(svm, INTERCEPT_STGI);
6297                 /* STGI will cause a vm exit */
6298                 return 1;
6299         }
6300         return 0;
6301 }
6302
6303 static int sev_flush_asids(void)
6304 {
6305         int ret, error;
6306
6307         /*
6308          * DEACTIVATE will clear the WBINVD indicator causing DF_FLUSH to fail,
6309          * so it must be guarded.
6310          */
6311         down_write(&sev_deactivate_lock);
6312
6313         wbinvd_on_all_cpus();
6314         ret = sev_guest_df_flush(&error);
6315
6316         up_write(&sev_deactivate_lock);
6317
6318         if (ret)
6319                 pr_err("SEV: DF_FLUSH failed, ret=%d, error=%#x\n", ret, error);
6320
6321         return ret;
6322 }
6323
6324 /* Must be called with the sev_bitmap_lock held */
6325 static bool __sev_recycle_asids(void)
6326 {
6327         int pos;
6328
6329         /* Check if there are any ASIDs to reclaim before performing a flush */
6330         pos = find_next_bit(sev_reclaim_asid_bitmap,
6331                             max_sev_asid, min_sev_asid - 1);
6332         if (pos >= max_sev_asid)
6333                 return false;
6334
6335         if (sev_flush_asids())
6336                 return false;
6337
6338         bitmap_xor(sev_asid_bitmap, sev_asid_bitmap, sev_reclaim_asid_bitmap,
6339                    max_sev_asid);
6340         bitmap_zero(sev_reclaim_asid_bitmap, max_sev_asid);
6341
6342         return true;
6343 }
6344
6345 static int sev_asid_new(void)
6346 {
6347         bool retry = true;
6348         int pos;
6349
6350         mutex_lock(&sev_bitmap_lock);
6351
6352         /*
6353          * SEV-enabled guest must use asid from min_sev_asid to max_sev_asid.
6354          */
6355 again:
6356         pos = find_next_zero_bit(sev_asid_bitmap, max_sev_asid, min_sev_asid - 1);
6357         if (pos >= max_sev_asid) {
6358                 if (retry && __sev_recycle_asids()) {
6359                         retry = false;
6360                         goto again;
6361                 }
6362                 mutex_unlock(&sev_bitmap_lock);
6363                 return -EBUSY;
6364         }
6365
6366         __set_bit(pos, sev_asid_bitmap);
6367
6368         mutex_unlock(&sev_bitmap_lock);
6369
6370         return pos + 1;
6371 }
6372
6373 static int sev_guest_init(struct kvm *kvm, struct kvm_sev_cmd *argp)
6374 {
6375         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6376         int asid, ret;
6377
6378         ret = -EBUSY;
6379         if (unlikely(sev->active))
6380                 return ret;
6381
6382         asid = sev_asid_new();
6383         if (asid < 0)
6384                 return ret;
6385
6386         ret = sev_platform_init(&argp->error);
6387         if (ret)
6388                 goto e_free;
6389
6390         sev->active = true;
6391         sev->asid = asid;
6392         INIT_LIST_HEAD(&sev->regions_list);
6393
6394         return 0;
6395
6396 e_free:
6397         sev_asid_free(asid);
6398         return ret;
6399 }
6400
6401 static int sev_bind_asid(struct kvm *kvm, unsigned int handle, int *error)
6402 {
6403         struct sev_data_activate *data;
6404         int asid = sev_get_asid(kvm);
6405         int ret;
6406
6407         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6408         if (!data)
6409                 return -ENOMEM;
6410
6411         /* activate ASID on the given handle */
6412         data->handle = handle;
6413         data->asid   = asid;
6414         ret = sev_guest_activate(data, error);
6415         kfree(data);
6416
6417         return ret;
6418 }
6419
6420 static int __sev_issue_cmd(int fd, int id, void *data, int *error)
6421 {
6422         struct fd f;
6423         int ret;
6424
6425         f = fdget(fd);
6426         if (!f.file)
6427                 return -EBADF;
6428
6429         ret = sev_issue_cmd_external_user(f.file, id, data, error);
6430
6431         fdput(f);
6432         return ret;
6433 }
6434
6435 static int sev_issue_cmd(struct kvm *kvm, int id, void *data, int *error)
6436 {
6437         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6438
6439         return __sev_issue_cmd(sev->fd, id, data, error);
6440 }
6441
6442 static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp)
6443 {
6444         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6445         struct sev_data_launch_start *start;
6446         struct kvm_sev_launch_start params;
6447         void *dh_blob, *session_blob;
6448         int *error = &argp->error;
6449         int ret;
6450
6451         if (!sev_guest(kvm))
6452                 return -ENOTTY;
6453
6454         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6455                 return -EFAULT;
6456
6457         start = kzalloc(sizeof(*start), GFP_KERNEL_ACCOUNT);
6458         if (!start)
6459                 return -ENOMEM;
6460
6461         dh_blob = NULL;
6462         if (params.dh_uaddr) {
6463                 dh_blob = psp_copy_user_blob(params.dh_uaddr, params.dh_len);
6464                 if (IS_ERR(dh_blob)) {
6465                         ret = PTR_ERR(dh_blob);
6466                         goto e_free;
6467                 }
6468
6469                 start->dh_cert_address = __sme_set(__pa(dh_blob));
6470                 start->dh_cert_len = params.dh_len;
6471         }
6472
6473         session_blob = NULL;
6474         if (params.session_uaddr) {
6475                 session_blob = psp_copy_user_blob(params.session_uaddr, params.session_len);
6476                 if (IS_ERR(session_blob)) {
6477                         ret = PTR_ERR(session_blob);
6478                         goto e_free_dh;
6479                 }
6480
6481                 start->session_address = __sme_set(__pa(session_blob));
6482                 start->session_len = params.session_len;
6483         }
6484
6485         start->handle = params.handle;
6486         start->policy = params.policy;
6487
6488         /* create memory encryption context */
6489         ret = __sev_issue_cmd(argp->sev_fd, SEV_CMD_LAUNCH_START, start, error);
6490         if (ret)
6491                 goto e_free_session;
6492
6493         /* Bind ASID to this guest */
6494         ret = sev_bind_asid(kvm, start->handle, error);
6495         if (ret)
6496                 goto e_free_session;
6497
6498         /* return handle to userspace */
6499         params.handle = start->handle;
6500         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params))) {
6501                 sev_unbind_asid(kvm, start->handle);
6502                 ret = -EFAULT;
6503                 goto e_free_session;
6504         }
6505
6506         sev->handle = start->handle;
6507         sev->fd = argp->sev_fd;
6508
6509 e_free_session:
6510         kfree(session_blob);
6511 e_free_dh:
6512         kfree(dh_blob);
6513 e_free:
6514         kfree(start);
6515         return ret;
6516 }
6517
6518 static unsigned long get_num_contig_pages(unsigned long idx,
6519                                 struct page **inpages, unsigned long npages)
6520 {
6521         unsigned long paddr, next_paddr;
6522         unsigned long i = idx + 1, pages = 1;
6523
6524         /* find the number of contiguous pages starting from idx */
6525         paddr = __sme_page_pa(inpages[idx]);
6526         while (i < npages) {
6527                 next_paddr = __sme_page_pa(inpages[i++]);
6528                 if ((paddr + PAGE_SIZE) == next_paddr) {
6529                         pages++;
6530                         paddr = next_paddr;
6531                         continue;
6532                 }
6533                 break;
6534         }
6535
6536         return pages;
6537 }
6538
6539 static int sev_launch_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp)
6540 {
6541         unsigned long vaddr, vaddr_end, next_vaddr, npages, pages, size, i;
6542         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6543         struct kvm_sev_launch_update_data params;
6544         struct sev_data_launch_update_data *data;
6545         struct page **inpages;
6546         int ret;
6547
6548         if (!sev_guest(kvm))
6549                 return -ENOTTY;
6550
6551         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6552                 return -EFAULT;
6553
6554         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6555         if (!data)
6556                 return -ENOMEM;
6557
6558         vaddr = params.uaddr;
6559         size = params.len;
6560         vaddr_end = vaddr + size;
6561
6562         /* Lock the user memory. */
6563         inpages = sev_pin_memory(kvm, vaddr, size, &npages, 1);
6564         if (!inpages) {
6565                 ret = -ENOMEM;
6566                 goto e_free;
6567         }
6568
6569         /*
6570          * The LAUNCH_UPDATE command will perform in-place encryption of the
6571          * memory content (i.e it will write the same memory region with C=1).
6572          * It's possible that the cache may contain the data with C=0, i.e.,
6573          * unencrypted so invalidate it first.
6574          */
6575         sev_clflush_pages(inpages, npages);
6576
6577         for (i = 0; vaddr < vaddr_end; vaddr = next_vaddr, i += pages) {
6578                 int offset, len;
6579
6580                 /*
6581                  * If the user buffer is not page-aligned, calculate the offset
6582                  * within the page.
6583                  */
6584                 offset = vaddr & (PAGE_SIZE - 1);
6585
6586                 /* Calculate the number of pages that can be encrypted in one go. */
6587                 pages = get_num_contig_pages(i, inpages, npages);
6588
6589                 len = min_t(size_t, ((pages * PAGE_SIZE) - offset), size);
6590
6591                 data->handle = sev->handle;
6592                 data->len = len;
6593                 data->address = __sme_page_pa(inpages[i]) + offset;
6594                 ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_DATA, data, &argp->error);
6595                 if (ret)
6596                         goto e_unpin;
6597
6598                 size -= len;
6599                 next_vaddr = vaddr + len;
6600         }
6601
6602 e_unpin:
6603         /* content of memory is updated, mark pages dirty */
6604         for (i = 0; i < npages; i++) {
6605                 set_page_dirty_lock(inpages[i]);
6606                 mark_page_accessed(inpages[i]);
6607         }
6608         /* unlock the user pages */
6609         sev_unpin_memory(kvm, inpages, npages);
6610 e_free:
6611         kfree(data);
6612         return ret;
6613 }
6614
6615 static int sev_launch_measure(struct kvm *kvm, struct kvm_sev_cmd *argp)
6616 {
6617         void __user *measure = (void __user *)(uintptr_t)argp->data;
6618         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6619         struct sev_data_launch_measure *data;
6620         struct kvm_sev_launch_measure params;
6621         void __user *p = NULL;
6622         void *blob = NULL;
6623         int ret;
6624
6625         if (!sev_guest(kvm))
6626                 return -ENOTTY;
6627
6628         if (copy_from_user(&params, measure, sizeof(params)))
6629                 return -EFAULT;
6630
6631         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6632         if (!data)
6633                 return -ENOMEM;
6634
6635         /* User wants to query the blob length */
6636         if (!params.len)
6637                 goto cmd;
6638
6639         p = (void __user *)(uintptr_t)params.uaddr;
6640         if (p) {
6641                 if (params.len > SEV_FW_BLOB_MAX_SIZE) {
6642                         ret = -EINVAL;
6643                         goto e_free;
6644                 }
6645
6646                 ret = -ENOMEM;
6647                 blob = kmalloc(params.len, GFP_KERNEL);
6648                 if (!blob)
6649                         goto e_free;
6650
6651                 data->address = __psp_pa(blob);
6652                 data->len = params.len;
6653         }
6654
6655 cmd:
6656         data->handle = sev->handle;
6657         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_MEASURE, data, &argp->error);
6658
6659         /*
6660          * If we query the session length, FW responded with expected data.
6661          */
6662         if (!params.len)
6663                 goto done;
6664
6665         if (ret)
6666                 goto e_free_blob;
6667
6668         if (blob) {
6669                 if (copy_to_user(p, blob, params.len))
6670                         ret = -EFAULT;
6671         }
6672
6673 done:
6674         params.len = data->len;
6675         if (copy_to_user(measure, &params, sizeof(params)))
6676                 ret = -EFAULT;
6677 e_free_blob:
6678         kfree(blob);
6679 e_free:
6680         kfree(data);
6681         return ret;
6682 }
6683
6684 static int sev_launch_finish(struct kvm *kvm, struct kvm_sev_cmd *argp)
6685 {
6686         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6687         struct sev_data_launch_finish *data;
6688         int ret;
6689
6690         if (!sev_guest(kvm))
6691                 return -ENOTTY;
6692
6693         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6694         if (!data)
6695                 return -ENOMEM;
6696
6697         data->handle = sev->handle;
6698         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_FINISH, data, &argp->error);
6699
6700         kfree(data);
6701         return ret;
6702 }
6703
6704 static int sev_guest_status(struct kvm *kvm, struct kvm_sev_cmd *argp)
6705 {
6706         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6707         struct kvm_sev_guest_status params;
6708         struct sev_data_guest_status *data;
6709         int ret;
6710
6711         if (!sev_guest(kvm))
6712                 return -ENOTTY;
6713
6714         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6715         if (!data)
6716                 return -ENOMEM;
6717
6718         data->handle = sev->handle;
6719         ret = sev_issue_cmd(kvm, SEV_CMD_GUEST_STATUS, data, &argp->error);
6720         if (ret)
6721                 goto e_free;
6722
6723         params.policy = data->policy;
6724         params.state = data->state;
6725         params.handle = data->handle;
6726
6727         if (copy_to_user((void __user *)(uintptr_t)argp->data, &params, sizeof(params)))
6728                 ret = -EFAULT;
6729 e_free:
6730         kfree(data);
6731         return ret;
6732 }
6733
6734 static int __sev_issue_dbg_cmd(struct kvm *kvm, unsigned long src,
6735                                unsigned long dst, int size,
6736                                int *error, bool enc)
6737 {
6738         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6739         struct sev_data_dbg *data;
6740         int ret;
6741
6742         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
6743         if (!data)
6744                 return -ENOMEM;
6745
6746         data->handle = sev->handle;
6747         data->dst_addr = dst;
6748         data->src_addr = src;
6749         data->len = size;
6750
6751         ret = sev_issue_cmd(kvm,
6752                             enc ? SEV_CMD_DBG_ENCRYPT : SEV_CMD_DBG_DECRYPT,
6753                             data, error);
6754         kfree(data);
6755         return ret;
6756 }
6757
6758 static int __sev_dbg_decrypt(struct kvm *kvm, unsigned long src_paddr,
6759                              unsigned long dst_paddr, int sz, int *err)
6760 {
6761         int offset;
6762
6763         /*
6764          * Its safe to read more than we are asked, caller should ensure that
6765          * destination has enough space.
6766          */
6767         src_paddr = round_down(src_paddr, 16);
6768         offset = src_paddr & 15;
6769         sz = round_up(sz + offset, 16);
6770
6771         return __sev_issue_dbg_cmd(kvm, src_paddr, dst_paddr, sz, err, false);
6772 }
6773
6774 static int __sev_dbg_decrypt_user(struct kvm *kvm, unsigned long paddr,
6775                                   unsigned long __user dst_uaddr,
6776                                   unsigned long dst_paddr,
6777                                   int size, int *err)
6778 {
6779         struct page *tpage = NULL;
6780         int ret, offset;
6781
6782         /* if inputs are not 16-byte then use intermediate buffer */
6783         if (!IS_ALIGNED(dst_paddr, 16) ||
6784             !IS_ALIGNED(paddr,     16) ||
6785             !IS_ALIGNED(size,      16)) {
6786                 tpage = (void *)alloc_page(GFP_KERNEL);
6787                 if (!tpage)
6788                         return -ENOMEM;
6789
6790                 dst_paddr = __sme_page_pa(tpage);
6791         }
6792
6793         ret = __sev_dbg_decrypt(kvm, paddr, dst_paddr, size, err);
6794         if (ret)
6795                 goto e_free;
6796
6797         if (tpage) {
6798                 offset = paddr & 15;
6799                 if (copy_to_user((void __user *)(uintptr_t)dst_uaddr,
6800                                  page_address(tpage) + offset, size))
6801                         ret = -EFAULT;
6802         }
6803
6804 e_free:
6805         if (tpage)
6806                 __free_page(tpage);
6807
6808         return ret;
6809 }
6810
6811 static int __sev_dbg_encrypt_user(struct kvm *kvm, unsigned long paddr,
6812                                   unsigned long __user vaddr,
6813                                   unsigned long dst_paddr,
6814                                   unsigned long __user dst_vaddr,
6815                                   int size, int *error)
6816 {
6817         struct page *src_tpage = NULL;
6818         struct page *dst_tpage = NULL;
6819         int ret, len = size;
6820
6821         /* If source buffer is not aligned then use an intermediate buffer */
6822         if (!IS_ALIGNED(vaddr, 16)) {
6823                 src_tpage = alloc_page(GFP_KERNEL);
6824                 if (!src_tpage)
6825                         return -ENOMEM;
6826
6827                 if (copy_from_user(page_address(src_tpage),
6828                                 (void __user *)(uintptr_t)vaddr, size)) {
6829                         __free_page(src_tpage);
6830                         return -EFAULT;
6831                 }
6832
6833                 paddr = __sme_page_pa(src_tpage);
6834         }
6835
6836         /*
6837          *  If destination buffer or length is not aligned then do read-modify-write:
6838          *   - decrypt destination in an intermediate buffer
6839          *   - copy the source buffer in an intermediate buffer
6840          *   - use the intermediate buffer as source buffer
6841          */
6842         if (!IS_ALIGNED(dst_vaddr, 16) || !IS_ALIGNED(size, 16)) {
6843                 int dst_offset;
6844
6845                 dst_tpage = alloc_page(GFP_KERNEL);
6846                 if (!dst_tpage) {
6847                         ret = -ENOMEM;
6848                         goto e_free;
6849                 }
6850
6851                 ret = __sev_dbg_decrypt(kvm, dst_paddr,
6852                                         __sme_page_pa(dst_tpage), size, error);
6853                 if (ret)
6854                         goto e_free;
6855
6856                 /*
6857                  *  If source is kernel buffer then use memcpy() otherwise
6858                  *  copy_from_user().
6859                  */
6860                 dst_offset = dst_paddr & 15;
6861
6862                 if (src_tpage)
6863                         memcpy(page_address(dst_tpage) + dst_offset,
6864                                page_address(src_tpage), size);
6865                 else {
6866                         if (copy_from_user(page_address(dst_tpage) + dst_offset,
6867                                            (void __user *)(uintptr_t)vaddr, size)) {
6868                                 ret = -EFAULT;
6869                                 goto e_free;
6870                         }
6871                 }
6872
6873                 paddr = __sme_page_pa(dst_tpage);
6874                 dst_paddr = round_down(dst_paddr, 16);
6875                 len = round_up(size, 16);
6876         }
6877
6878         ret = __sev_issue_dbg_cmd(kvm, paddr, dst_paddr, len, error, true);
6879
6880 e_free:
6881         if (src_tpage)
6882                 __free_page(src_tpage);
6883         if (dst_tpage)
6884                 __free_page(dst_tpage);
6885         return ret;
6886 }
6887
6888 static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec)
6889 {
6890         unsigned long vaddr, vaddr_end, next_vaddr;
6891         unsigned long dst_vaddr;
6892         struct page **src_p, **dst_p;
6893         struct kvm_sev_dbg debug;
6894         unsigned long n;
6895         unsigned int size;
6896         int ret;
6897
6898         if (!sev_guest(kvm))
6899                 return -ENOTTY;
6900
6901         if (copy_from_user(&debug, (void __user *)(uintptr_t)argp->data, sizeof(debug)))
6902                 return -EFAULT;
6903
6904         if (!debug.len || debug.src_uaddr + debug.len < debug.src_uaddr)
6905                 return -EINVAL;
6906         if (!debug.dst_uaddr)
6907                 return -EINVAL;
6908
6909         vaddr = debug.src_uaddr;
6910         size = debug.len;
6911         vaddr_end = vaddr + size;
6912         dst_vaddr = debug.dst_uaddr;
6913
6914         for (; vaddr < vaddr_end; vaddr = next_vaddr) {
6915                 int len, s_off, d_off;
6916
6917                 /* lock userspace source and destination page */
6918                 src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0);
6919                 if (!src_p)
6920                         return -EFAULT;
6921
6922                 dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, 1);
6923                 if (!dst_p) {
6924                         sev_unpin_memory(kvm, src_p, n);
6925                         return -EFAULT;
6926                 }
6927
6928                 /*
6929                  * The DBG_{DE,EN}CRYPT commands will perform {dec,en}cryption of the
6930                  * memory content (i.e it will write the same memory region with C=1).
6931                  * It's possible that the cache may contain the data with C=0, i.e.,
6932                  * unencrypted so invalidate it first.
6933                  */
6934                 sev_clflush_pages(src_p, 1);
6935                 sev_clflush_pages(dst_p, 1);
6936
6937                 /*
6938                  * Since user buffer may not be page aligned, calculate the
6939                  * offset within the page.
6940                  */
6941                 s_off = vaddr & ~PAGE_MASK;
6942                 d_off = dst_vaddr & ~PAGE_MASK;
6943                 len = min_t(size_t, (PAGE_SIZE - s_off), size);
6944
6945                 if (dec)
6946                         ret = __sev_dbg_decrypt_user(kvm,
6947                                                      __sme_page_pa(src_p[0]) + s_off,
6948                                                      dst_vaddr,
6949                                                      __sme_page_pa(dst_p[0]) + d_off,
6950                                                      len, &argp->error);
6951                 else
6952                         ret = __sev_dbg_encrypt_user(kvm,
6953                                                      __sme_page_pa(src_p[0]) + s_off,
6954                                                      vaddr,
6955                                                      __sme_page_pa(dst_p[0]) + d_off,
6956                                                      dst_vaddr,
6957                                                      len, &argp->error);
6958
6959                 sev_unpin_memory(kvm, src_p, n);
6960                 sev_unpin_memory(kvm, dst_p, n);
6961
6962                 if (ret)
6963                         goto err;
6964
6965                 next_vaddr = vaddr + len;
6966                 dst_vaddr = dst_vaddr + len;
6967                 size -= len;
6968         }
6969 err:
6970         return ret;
6971 }
6972
6973 static int sev_launch_secret(struct kvm *kvm, struct kvm_sev_cmd *argp)
6974 {
6975         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
6976         struct sev_data_launch_secret *data;
6977         struct kvm_sev_launch_secret params;
6978         struct page **pages;
6979         void *blob, *hdr;
6980         unsigned long n;
6981         int ret, offset;
6982
6983         if (!sev_guest(kvm))
6984                 return -ENOTTY;
6985
6986         if (copy_from_user(&params, (void __user *)(uintptr_t)argp->data, sizeof(params)))
6987                 return -EFAULT;
6988
6989         pages = sev_pin_memory(kvm, params.guest_uaddr, params.guest_len, &n, 1);
6990         if (!pages)
6991                 return -ENOMEM;
6992
6993         /*
6994          * The secret must be copied into contiguous memory region, lets verify
6995          * that userspace memory pages are contiguous before we issue command.
6996          */
6997         if (get_num_contig_pages(0, pages, n) != n) {
6998                 ret = -EINVAL;
6999                 goto e_unpin_memory;
7000         }
7001
7002         ret = -ENOMEM;
7003         data = kzalloc(sizeof(*data), GFP_KERNEL_ACCOUNT);
7004         if (!data)
7005                 goto e_unpin_memory;
7006
7007         offset = params.guest_uaddr & (PAGE_SIZE - 1);
7008         data->guest_address = __sme_page_pa(pages[0]) + offset;
7009         data->guest_len = params.guest_len;
7010
7011         blob = psp_copy_user_blob(params.trans_uaddr, params.trans_len);
7012         if (IS_ERR(blob)) {
7013                 ret = PTR_ERR(blob);
7014                 goto e_free;
7015         }
7016
7017         data->trans_address = __psp_pa(blob);
7018         data->trans_len = params.trans_len;
7019
7020         hdr = psp_copy_user_blob(params.hdr_uaddr, params.hdr_len);
7021         if (IS_ERR(hdr)) {
7022                 ret = PTR_ERR(hdr);
7023                 goto e_free_blob;
7024         }
7025         data->hdr_address = __psp_pa(hdr);
7026         data->hdr_len = params.hdr_len;
7027
7028         data->handle = sev->handle;
7029         ret = sev_issue_cmd(kvm, SEV_CMD_LAUNCH_UPDATE_SECRET, data, &argp->error);
7030
7031         kfree(hdr);
7032
7033 e_free_blob:
7034         kfree(blob);
7035 e_free:
7036         kfree(data);
7037 e_unpin_memory:
7038         sev_unpin_memory(kvm, pages, n);
7039         return ret;
7040 }
7041
7042 static int svm_mem_enc_op(struct kvm *kvm, void __user *argp)
7043 {
7044         struct kvm_sev_cmd sev_cmd;
7045         int r;
7046
7047         if (!svm_sev_enabled())
7048                 return -ENOTTY;
7049
7050         if (copy_from_user(&sev_cmd, argp, sizeof(struct kvm_sev_cmd)))
7051                 return -EFAULT;
7052
7053         mutex_lock(&kvm->lock);
7054
7055         switch (sev_cmd.id) {
7056         case KVM_SEV_INIT:
7057                 r = sev_guest_init(kvm, &sev_cmd);
7058                 break;
7059         case KVM_SEV_LAUNCH_START:
7060                 r = sev_launch_start(kvm, &sev_cmd);
7061                 break;
7062         case KVM_SEV_LAUNCH_UPDATE_DATA:
7063                 r = sev_launch_update_data(kvm, &sev_cmd);
7064                 break;
7065         case KVM_SEV_LAUNCH_MEASURE:
7066                 r = sev_launch_measure(kvm, &sev_cmd);
7067                 break;
7068         case KVM_SEV_LAUNCH_FINISH:
7069                 r = sev_launch_finish(kvm, &sev_cmd);
7070                 break;
7071         case KVM_SEV_GUEST_STATUS:
7072                 r = sev_guest_status(kvm, &sev_cmd);
7073                 break;
7074         case KVM_SEV_DBG_DECRYPT:
7075                 r = sev_dbg_crypt(kvm, &sev_cmd, true);
7076                 break;
7077         case KVM_SEV_DBG_ENCRYPT:
7078                 r = sev_dbg_crypt(kvm, &sev_cmd, false);
7079                 break;
7080         case KVM_SEV_LAUNCH_SECRET:
7081                 r = sev_launch_secret(kvm, &sev_cmd);
7082                 break;
7083         default:
7084                 r = -EINVAL;
7085                 goto out;
7086         }
7087
7088         if (copy_to_user(argp, &sev_cmd, sizeof(struct kvm_sev_cmd)))
7089                 r = -EFAULT;
7090
7091 out:
7092         mutex_unlock(&kvm->lock);
7093         return r;
7094 }
7095
7096 static int svm_register_enc_region(struct kvm *kvm,
7097                                    struct kvm_enc_region *range)
7098 {
7099         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
7100         struct enc_region *region;
7101         int ret = 0;
7102
7103         if (!sev_guest(kvm))
7104                 return -ENOTTY;
7105
7106         if (range->addr > ULONG_MAX || range->size > ULONG_MAX)
7107                 return -EINVAL;
7108
7109         region = kzalloc(sizeof(*region), GFP_KERNEL_ACCOUNT);
7110         if (!region)
7111                 return -ENOMEM;
7112
7113         region->pages = sev_pin_memory(kvm, range->addr, range->size, &region->npages, 1);
7114         if (!region->pages) {
7115                 ret = -ENOMEM;
7116                 goto e_free;
7117         }
7118
7119         /*
7120          * The guest may change the memory encryption attribute from C=0 -> C=1
7121          * or vice versa for this memory range. Lets make sure caches are
7122          * flushed to ensure that guest data gets written into memory with
7123          * correct C-bit.
7124          */
7125         sev_clflush_pages(region->pages, region->npages);
7126
7127         region->uaddr = range->addr;
7128         region->size = range->size;
7129
7130         mutex_lock(&kvm->lock);
7131         list_add_tail(&region->list, &sev->regions_list);
7132         mutex_unlock(&kvm->lock);
7133
7134         return ret;
7135
7136 e_free:
7137         kfree(region);
7138         return ret;
7139 }
7140
7141 static struct enc_region *
7142 find_enc_region(struct kvm *kvm, struct kvm_enc_region *range)
7143 {
7144         struct kvm_sev_info *sev = &to_kvm_svm(kvm)->sev_info;
7145         struct list_head *head = &sev->regions_list;
7146         struct enc_region *i;
7147
7148         list_for_each_entry(i, head, list) {
7149                 if (i->uaddr == range->addr &&
7150                     i->size == range->size)
7151                         return i;
7152         }
7153
7154         return NULL;
7155 }
7156
7157
7158 static int svm_unregister_enc_region(struct kvm *kvm,
7159                                      struct kvm_enc_region *range)
7160 {
7161         struct enc_region *region;
7162         int ret;
7163
7164         mutex_lock(&kvm->lock);
7165
7166         if (!sev_guest(kvm)) {
7167                 ret = -ENOTTY;
7168                 goto failed;
7169         }
7170
7171         region = find_enc_region(kvm, range);
7172         if (!region) {
7173                 ret = -EINVAL;
7174                 goto failed;
7175         }
7176
7177         __unregister_enc_region_locked(kvm, region);
7178
7179         mutex_unlock(&kvm->lock);
7180         return 0;
7181
7182 failed:
7183         mutex_unlock(&kvm->lock);
7184         return ret;
7185 }
7186
7187 static bool svm_need_emulation_on_page_fault(struct kvm_vcpu *vcpu)
7188 {
7189         unsigned long cr4 = kvm_read_cr4(vcpu);
7190         bool smep = cr4 & X86_CR4_SMEP;
7191         bool smap = cr4 & X86_CR4_SMAP;
7192         bool is_user = svm_get_cpl(vcpu) == 3;
7193
7194         /*
7195          * Detect and workaround Errata 1096 Fam_17h_00_0Fh.
7196          *
7197          * Errata:
7198          * When CPU raise #NPF on guest data access and vCPU CR4.SMAP=1, it is
7199          * possible that CPU microcode implementing DecodeAssist will fail
7200          * to read bytes of instruction which caused #NPF. In this case,
7201          * GuestIntrBytes field of the VMCB on a VMEXIT will incorrectly
7202          * return 0 instead of the correct guest instruction bytes.
7203          *
7204          * This happens because CPU microcode reading instruction bytes
7205          * uses a special opcode which attempts to read data using CPL=0
7206          * priviledges. The microcode reads CS:RIP and if it hits a SMAP
7207          * fault, it gives up and returns no instruction bytes.
7208          *
7209          * Detection:
7210          * We reach here in case CPU supports DecodeAssist, raised #NPF and
7211          * returned 0 in GuestIntrBytes field of the VMCB.
7212          * First, errata can only be triggered in case vCPU CR4.SMAP=1.
7213          * Second, if vCPU CR4.SMEP=1, errata could only be triggered
7214          * in case vCPU CPL==3 (Because otherwise guest would have triggered
7215          * a SMEP fault instead of #NPF).
7216          * Otherwise, vCPU CR4.SMEP=0, errata could be triggered by any vCPU CPL.
7217          * As most guests enable SMAP if they have also enabled SMEP, use above
7218          * logic in order to attempt minimize false-positive of detecting errata
7219          * while still preserving all cases semantic correctness.
7220          *
7221          * Workaround:
7222          * To determine what instruction the guest was executing, the hypervisor
7223          * will have to decode the instruction at the instruction pointer.
7224          *
7225          * In non SEV guest, hypervisor will be able to read the guest
7226          * memory to decode the instruction pointer when insn_len is zero
7227          * so we return true to indicate that decoding is possible.
7228          *
7229          * But in the SEV guest, the guest memory is encrypted with the
7230          * guest specific key and hypervisor will not be able to decode the
7231          * instruction pointer so we will not able to workaround it. Lets
7232          * print the error and request to kill the guest.
7233          */
7234         if (smap && (!smep || is_user)) {
7235                 if (!sev_guest(vcpu->kvm))
7236                         return true;
7237
7238                 pr_err_ratelimited("KVM: SEV Guest triggered AMD Erratum 1096\n");
7239                 kvm_make_request(KVM_REQ_TRIPLE_FAULT, vcpu);
7240         }
7241
7242         return false;
7243 }
7244
7245 static bool svm_apic_init_signal_blocked(struct kvm_vcpu *vcpu)
7246 {
7247         struct vcpu_svm *svm = to_svm(vcpu);
7248
7249         /*
7250          * TODO: Last condition latch INIT signals on vCPU when
7251          * vCPU is in guest-mode and vmcb12 defines intercept on INIT.
7252          * To properly emulate the INIT intercept, SVM should implement
7253          * kvm_x86_ops->check_nested_events() and call nested_svm_vmexit()
7254          * there if an INIT signal is pending.
7255          */
7256         return !gif_set(svm) ||
7257                    (svm->vmcb->control.intercept & (1ULL << INTERCEPT_INIT));
7258 }
7259
7260 static struct kvm_x86_ops svm_x86_ops __ro_after_init = {
7261         .cpu_has_kvm_support = has_svm,
7262         .disabled_by_bios = is_disabled,
7263         .hardware_setup = svm_hardware_setup,
7264         .hardware_unsetup = svm_hardware_unsetup,
7265         .check_processor_compatibility = svm_check_processor_compat,
7266         .hardware_enable = svm_hardware_enable,
7267         .hardware_disable = svm_hardware_disable,
7268         .cpu_has_accelerated_tpr = svm_cpu_has_accelerated_tpr,
7269         .has_emulated_msr = svm_has_emulated_msr,
7270
7271         .vcpu_create = svm_create_vcpu,
7272         .vcpu_free = svm_free_vcpu,
7273         .vcpu_reset = svm_vcpu_reset,
7274
7275         .vm_alloc = svm_vm_alloc,
7276         .vm_free = svm_vm_free,
7277         .vm_init = avic_vm_init,
7278         .vm_destroy = svm_vm_destroy,
7279
7280         .prepare_guest_switch = svm_prepare_guest_switch,
7281         .vcpu_load = svm_vcpu_load,
7282         .vcpu_put = svm_vcpu_put,
7283         .vcpu_blocking = svm_vcpu_blocking,
7284         .vcpu_unblocking = svm_vcpu_unblocking,
7285
7286         .update_bp_intercept = update_bp_intercept,
7287         .get_msr_feature = svm_get_msr_feature,
7288         .get_msr = svm_get_msr,
7289         .set_msr = svm_set_msr,
7290         .get_segment_base = svm_get_segment_base,
7291         .get_segment = svm_get_segment,
7292         .set_segment = svm_set_segment,
7293         .get_cpl = svm_get_cpl,
7294         .get_cs_db_l_bits = kvm_get_cs_db_l_bits,
7295         .decache_cr0_guest_bits = svm_decache_cr0_guest_bits,
7296         .decache_cr4_guest_bits = svm_decache_cr4_guest_bits,
7297         .set_cr0 = svm_set_cr0,
7298         .set_cr3 = svm_set_cr3,
7299         .set_cr4 = svm_set_cr4,
7300         .set_efer = svm_set_efer,
7301         .get_idt = svm_get_idt,
7302         .set_idt = svm_set_idt,
7303         .get_gdt = svm_get_gdt,
7304         .set_gdt = svm_set_gdt,
7305         .get_dr6 = svm_get_dr6,
7306         .set_dr6 = svm_set_dr6,
7307         .set_dr7 = svm_set_dr7,
7308         .sync_dirty_debug_regs = svm_sync_dirty_debug_regs,
7309         .cache_reg = svm_cache_reg,
7310         .get_rflags = svm_get_rflags,
7311         .set_rflags = svm_set_rflags,
7312
7313         .tlb_flush = svm_flush_tlb,
7314         .tlb_flush_gva = svm_flush_tlb_gva,
7315
7316         .run = svm_vcpu_run,
7317         .handle_exit = handle_exit,
7318         .skip_emulated_instruction = skip_emulated_instruction,
7319         .set_interrupt_shadow = svm_set_interrupt_shadow,
7320         .get_interrupt_shadow = svm_get_interrupt_shadow,
7321         .patch_hypercall = svm_patch_hypercall,
7322         .set_irq = svm_set_irq,
7323         .set_nmi = svm_inject_nmi,
7324         .queue_exception = svm_queue_exception,
7325         .cancel_injection = svm_cancel_injection,
7326         .interrupt_allowed = svm_interrupt_allowed,
7327         .nmi_allowed = svm_nmi_allowed,
7328         .get_nmi_mask = svm_get_nmi_mask,
7329         .set_nmi_mask = svm_set_nmi_mask,
7330         .enable_nmi_window = enable_nmi_window,
7331         .enable_irq_window = enable_irq_window,
7332         .update_cr8_intercept = update_cr8_intercept,
7333         .set_virtual_apic_mode = svm_set_virtual_apic_mode,
7334         .get_enable_apicv = svm_get_enable_apicv,
7335         .refresh_apicv_exec_ctrl = svm_refresh_apicv_exec_ctrl,
7336         .load_eoi_exitmap = svm_load_eoi_exitmap,
7337         .hwapic_irr_update = svm_hwapic_irr_update,
7338         .hwapic_isr_update = svm_hwapic_isr_update,
7339         .sync_pir_to_irr = kvm_lapic_find_highest_irr,
7340         .apicv_post_state_restore = avic_post_state_restore,
7341
7342         .set_tss_addr = svm_set_tss_addr,
7343         .set_identity_map_addr = svm_set_identity_map_addr,
7344         .get_tdp_level = get_npt_level,
7345         .get_mt_mask = svm_get_mt_mask,
7346
7347         .get_exit_info = svm_get_exit_info,
7348
7349         .get_lpage_level = svm_get_lpage_level,
7350
7351         .cpuid_update = svm_cpuid_update,
7352
7353         .rdtscp_supported = svm_rdtscp_supported,
7354         .invpcid_supported = svm_invpcid_supported,
7355         .mpx_supported = svm_mpx_supported,
7356         .xsaves_supported = svm_xsaves_supported,
7357         .umip_emulated = svm_umip_emulated,
7358         .pt_supported = svm_pt_supported,
7359         .pku_supported = svm_pku_supported,
7360
7361         .set_supported_cpuid = svm_set_supported_cpuid,
7362
7363         .has_wbinvd_exit = svm_has_wbinvd_exit,
7364
7365         .read_l1_tsc_offset = svm_read_l1_tsc_offset,
7366         .write_l1_tsc_offset = svm_write_l1_tsc_offset,
7367
7368         .set_tdp_cr3 = set_tdp_cr3,
7369
7370         .check_intercept = svm_check_intercept,
7371         .handle_exit_irqoff = svm_handle_exit_irqoff,
7372
7373         .request_immediate_exit = __kvm_request_immediate_exit,
7374
7375         .sched_in = svm_sched_in,
7376
7377         .pmu_ops = &amd_pmu_ops,
7378         .deliver_posted_interrupt = svm_deliver_avic_intr,
7379         .dy_apicv_has_pending_interrupt = svm_dy_apicv_has_pending_interrupt,
7380         .update_pi_irte = svm_update_pi_irte,
7381         .setup_mce = svm_setup_mce,
7382
7383         .smi_allowed = svm_smi_allowed,
7384         .pre_enter_smm = svm_pre_enter_smm,
7385         .pre_leave_smm = svm_pre_leave_smm,
7386         .enable_smi_window = enable_smi_window,
7387
7388         .mem_enc_op = svm_mem_enc_op,
7389         .mem_enc_reg_region = svm_register_enc_region,
7390         .mem_enc_unreg_region = svm_unregister_enc_region,
7391
7392         .nested_enable_evmcs = NULL,
7393         .nested_get_evmcs_version = NULL,
7394
7395         .need_emulation_on_page_fault = svm_need_emulation_on_page_fault,
7396
7397         .apic_init_signal_blocked = svm_apic_init_signal_blocked,
7398 };
7399
7400 static int __init svm_init(void)
7401 {
7402         return kvm_init(&svm_x86_ops, sizeof(struct vcpu_svm),
7403                         __alignof__(struct vcpu_svm), THIS_MODULE);
7404 }
7405
7406 static void __exit svm_exit(void)
7407 {
7408         kvm_exit();
7409 }
7410
7411 module_init(svm_init)
7412 module_exit(svm_exit)