tools/testing/selftests/kvm/lib/x86_64/processor.c

   1 /*
   2  * tools/testing/selftests/kvm/lib/x86_64/processor.c
   3  *
   4  * Copyright (C) 2018, Google LLC.
   5  *
   6  * This work is licensed under the terms of the GNU GPL, version 2.
   7  */
   8
   9 #define _GNU_SOURCE /* for program_invocation_name */
  10
  11 #include "test_util.h"
  12 #include "kvm_util.h"
  13 #include "../kvm_util_internal.h"
  14 #include "processor.h"
  15
  16 /* Minimum physical address used for virtual translation tables. */
  17 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000
  18
  19 /* Virtual translation table structure declarations */
  20 struct pageMapL4Entry {
  21         uint64_t present:1;
  22         uint64_t writable:1;
  23         uint64_t user:1;
  24         uint64_t write_through:1;
  25         uint64_t cache_disable:1;
  26         uint64_t accessed:1;
  27         uint64_t ignored_06:1;
  28         uint64_t page_size:1;
  29         uint64_t ignored_11_08:4;
  30         uint64_t address:40;
  31         uint64_t ignored_62_52:11;
  32         uint64_t execute_disable:1;
  33 };
  34
  35 struct pageDirectoryPointerEntry {
  36         uint64_t present:1;
  37         uint64_t writable:1;
  38         uint64_t user:1;
  39         uint64_t write_through:1;
  40         uint64_t cache_disable:1;
  41         uint64_t accessed:1;
  42         uint64_t ignored_06:1;
  43         uint64_t page_size:1;
  44         uint64_t ignored_11_08:4;
  45         uint64_t address:40;
  46         uint64_t ignored_62_52:11;
  47         uint64_t execute_disable:1;
  48 };
  49
  50 struct pageDirectoryEntry {
  51         uint64_t present:1;
  52         uint64_t writable:1;
  53         uint64_t user:1;
  54         uint64_t write_through:1;
  55         uint64_t cache_disable:1;
  56         uint64_t accessed:1;
  57         uint64_t ignored_06:1;
  58         uint64_t page_size:1;
  59         uint64_t ignored_11_08:4;
  60         uint64_t address:40;
  61         uint64_t ignored_62_52:11;
  62         uint64_t execute_disable:1;
  63 };
  64
  65 struct pageTableEntry {
  66         uint64_t present:1;
  67         uint64_t writable:1;
  68         uint64_t user:1;
  69         uint64_t write_through:1;
  70         uint64_t cache_disable:1;
  71         uint64_t accessed:1;
  72         uint64_t dirty:1;
  73         uint64_t reserved_07:1;
  74         uint64_t global:1;
  75         uint64_t ignored_11_09:3;
  76         uint64_t address:40;
  77         uint64_t ignored_62_52:11;
  78         uint64_t execute_disable:1;
  79 };
  80
  81 /* Register Dump
  82  *
  83  * Input Args:
  84  *   indent - Left margin indent amount
  85  *   regs - register
  86  *
  87  * Output Args:
  88  *   stream - Output FILE stream
  89  *
  90  * Return: None
  91  *
  92  * Dumps the state of the registers given by regs, to the FILE stream
  93  * given by steam.
  94  */
  95 void regs_dump(FILE *stream, struct kvm_regs *regs,
  96                uint8_t indent)
  97 {
  98         fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
  99                 "rcx: 0x%.16llx rdx: 0x%.16llx\n",
 100                 indent, "",
 101                 regs->rax, regs->rbx, regs->rcx, regs->rdx);
 102         fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
 103                 "rsp: 0x%.16llx rbp: 0x%.16llx\n",
 104                 indent, "",
 105                 regs->rsi, regs->rdi, regs->rsp, regs->rbp);
 106         fprintf(stream, "%*sr8:  0x%.16llx r9:  0x%.16llx "
 107                 "r10: 0x%.16llx r11: 0x%.16llx\n",
 108                 indent, "",
 109                 regs->r8, regs->r9, regs->r10, regs->r11);
 110         fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
 111                 "r14: 0x%.16llx r15: 0x%.16llx\n",
 112                 indent, "",
 113                 regs->r12, regs->r13, regs->r14, regs->r15);
 114         fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
 115                 indent, "",
 116                 regs->rip, regs->rflags);
 117 }
 118
 119 /* Segment Dump
 120  *
 121  * Input Args:
 122  *   indent - Left margin indent amount
 123  *   segment - KVM segment
 124  *
 125  * Output Args:
 126  *   stream - Output FILE stream
 127  *
 128  * Return: None
 129  *
 130  * Dumps the state of the KVM segment given by segment, to the FILE stream
 131  * given by steam.
 132  */
 133 static void segment_dump(FILE *stream, struct kvm_segment *segment,
 134                          uint8_t indent)
 135 {
 136         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
 137                 "selector: 0x%.4x type: 0x%.2x\n",
 138                 indent, "", segment->base, segment->limit,
 139                 segment->selector, segment->type);
 140         fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
 141                 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
 142                 indent, "", segment->present, segment->dpl,
 143                 segment->db, segment->s, segment->l);
 144         fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
 145                 "unusable: 0x%.2x padding: 0x%.2x\n",
 146                 indent, "", segment->g, segment->avl,
 147                 segment->unusable, segment->padding);
 148 }
 149
 150 /* dtable Dump
 151  *
 152  * Input Args:
 153  *   indent - Left margin indent amount
 154  *   dtable - KVM dtable
 155  *
 156  * Output Args:
 157  *   stream - Output FILE stream
 158  *
 159  * Return: None
 160  *
 161  * Dumps the state of the KVM dtable given by dtable, to the FILE stream
 162  * given by steam.
 163  */
 164 static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
 165                         uint8_t indent)
 166 {
 167         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
 168                 "padding: 0x%.4x 0x%.4x 0x%.4x\n",
 169                 indent, "", dtable->base, dtable->limit,
 170                 dtable->padding[0], dtable->padding[1], dtable->padding[2]);
 171 }
 172
 173 /* System Register Dump
 174  *
 175  * Input Args:
 176  *   indent - Left margin indent amount
 177  *   sregs - System registers
 178  *
 179  * Output Args:
 180  *   stream - Output FILE stream
 181  *
 182  * Return: None
 183  *
 184  * Dumps the state of the system registers given by sregs, to the FILE stream
 185  * given by steam.
 186  */
 187 void sregs_dump(FILE *stream, struct kvm_sregs *sregs,
 188                 uint8_t indent)
 189 {
 190         unsigned int i;
 191
 192         fprintf(stream, "%*scs:\n", indent, "");
 193         segment_dump(stream, &sregs->cs, indent + 2);
 194         fprintf(stream, "%*sds:\n", indent, "");
 195         segment_dump(stream, &sregs->ds, indent + 2);
 196         fprintf(stream, "%*ses:\n", indent, "");
 197         segment_dump(stream, &sregs->es, indent + 2);
 198         fprintf(stream, "%*sfs:\n", indent, "");
 199         segment_dump(stream, &sregs->fs, indent + 2);
 200         fprintf(stream, "%*sgs:\n", indent, "");
 201         segment_dump(stream, &sregs->gs, indent + 2);
 202         fprintf(stream, "%*sss:\n", indent, "");
 203         segment_dump(stream, &sregs->ss, indent + 2);
 204         fprintf(stream, "%*str:\n", indent, "");
 205         segment_dump(stream, &sregs->tr, indent + 2);
 206         fprintf(stream, "%*sldt:\n", indent, "");
 207         segment_dump(stream, &sregs->ldt, indent + 2);
 208
 209         fprintf(stream, "%*sgdt:\n", indent, "");
 210         dtable_dump(stream, &sregs->gdt, indent + 2);
 211         fprintf(stream, "%*sidt:\n", indent, "");
 212         dtable_dump(stream, &sregs->idt, indent + 2);
 213
 214         fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
 215                 "cr3: 0x%.16llx cr4: 0x%.16llx\n",
 216                 indent, "",
 217                 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
 218         fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
 219                 "apic_base: 0x%.16llx\n",
 220                 indent, "",
 221                 sregs->cr8, sregs->efer, sregs->apic_base);
 222
 223         fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
 224         for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
 225                 fprintf(stream, "%*s%.16llx\n", indent + 2, "",
 226                         sregs->interrupt_bitmap[i]);
 227         }
 228 }
 229
 230 void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot)
 231 {
 232         TEST_ASSERT(vm->mode == VM_MODE_P52V48_4K, "Attempt to use "
 233                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 234
 235         /* If needed, create page map l4 table. */
 236         if (!vm->pgd_created) {
 237                 vm_paddr_t paddr = vm_phy_page_alloc(vm,
 238                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot);
 239                 vm->pgd = paddr;
 240                 vm->pgd_created = true;
 241         }
 242 }
 243
 244 /* VM Virtual Page Map
 245  *
 246  * Input Args:
 247  *   vm - Virtual Machine
 248  *   vaddr - VM Virtual Address
 249  *   paddr - VM Physical Address
 250  *   pgd_memslot - Memory region slot for new virtual translation tables
 251  *
 252  * Output Args: None
 253  *
 254  * Return: None
 255  *
 256  * Within the VM given by vm, creates a virtual translation for the page
 257  * starting at vaddr to the page starting at paddr.
 258  */
 259 void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
 260         uint32_t pgd_memslot)
 261 {
 262         uint16_t index[4];
 263         struct pageMapL4Entry *pml4e;
 264
 265         TEST_ASSERT(vm->mode == VM_MODE_P52V48_4K, "Attempt to use "
 266                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 267
 268         TEST_ASSERT((vaddr % vm->page_size) == 0,
 269                 "Virtual address not on page boundary,\n"
 270                 "  vaddr: 0x%lx vm->page_size: 0x%x",
 271                 vaddr, vm->page_size);
 272         TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
 273                 (vaddr >> vm->page_shift)),
 274                 "Invalid virtual address, vaddr: 0x%lx",
 275                 vaddr);
 276         TEST_ASSERT((paddr % vm->page_size) == 0,
 277                 "Physical address not on page boundary,\n"
 278                 "  paddr: 0x%lx vm->page_size: 0x%x",
 279                 paddr, vm->page_size);
 280         TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
 281                 "Physical address beyond beyond maximum supported,\n"
 282                 "  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
 283                 paddr, vm->max_gfn, vm->page_size);
 284
 285         index[0] = (vaddr >> 12) & 0x1ffu;
 286         index[1] = (vaddr >> 21) & 0x1ffu;
 287         index[2] = (vaddr >> 30) & 0x1ffu;
 288         index[3] = (vaddr >> 39) & 0x1ffu;
 289
 290         /* Allocate page directory pointer table if not present. */
 291         pml4e = addr_gpa2hva(vm, vm->pgd);
 292         if (!pml4e[index[3]].present) {
 293                 pml4e[index[3]].address = vm_phy_page_alloc(vm,
 294                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
 295                         >> vm->page_shift;
 296                 pml4e[index[3]].writable = true;
 297                 pml4e[index[3]].present = true;
 298         }
 299
 300         /* Allocate page directory table if not present. */
 301         struct pageDirectoryPointerEntry *pdpe;
 302         pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size);
 303         if (!pdpe[index[2]].present) {
 304                 pdpe[index[2]].address = vm_phy_page_alloc(vm,
 305                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
 306                         >> vm->page_shift;
 307                 pdpe[index[2]].writable = true;
 308                 pdpe[index[2]].present = true;
 309         }
 310
 311         /* Allocate page table if not present. */
 312         struct pageDirectoryEntry *pde;
 313         pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size);
 314         if (!pde[index[1]].present) {
 315                 pde[index[1]].address = vm_phy_page_alloc(vm,
 316                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
 317                         >> vm->page_shift;
 318                 pde[index[1]].writable = true;
 319                 pde[index[1]].present = true;
 320         }
 321
 322         /* Fill in page table entry. */
 323         struct pageTableEntry *pte;
 324         pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size);
 325         pte[index[0]].address = paddr >> vm->page_shift;
 326         pte[index[0]].writable = true;
 327         pte[index[0]].present = 1;
 328 }
 329
 330 /* Virtual Translation Tables Dump
 331  *
 332  * Input Args:
 333  *   vm - Virtual Machine
 334  *   indent - Left margin indent amount
 335  *
 336  * Output Args:
 337  *   stream - Output FILE stream
 338  *
 339  * Return: None
 340  *
 341  * Dumps to the FILE stream given by stream, the contents of all the
 342  * virtual translation tables for the VM given by vm.
 343  */
 344 void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
 345 {
 346         struct pageMapL4Entry *pml4e, *pml4e_start;
 347         struct pageDirectoryPointerEntry *pdpe, *pdpe_start;
 348         struct pageDirectoryEntry *pde, *pde_start;
 349         struct pageTableEntry *pte, *pte_start;
 350
 351         if (!vm->pgd_created)
 352                 return;
 353
 354         fprintf(stream, "%*s                                          "
 355                 "                no\n", indent, "");
 356         fprintf(stream, "%*s      index hvaddr         gpaddr         "
 357                 "addr         w exec dirty\n",
 358                 indent, "");
 359         pml4e_start = (struct pageMapL4Entry *) addr_gpa2hva(vm,
 360                 vm->pgd);
 361         for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
 362                 pml4e = &pml4e_start[n1];
 363                 if (!pml4e->present)
 364                         continue;
 365                 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10lx %u "
 366                         " %u\n",
 367                         indent, "",
 368                         pml4e - pml4e_start, pml4e,
 369                         addr_hva2gpa(vm, pml4e), (uint64_t) pml4e->address,
 370                         pml4e->writable, pml4e->execute_disable);
 371
 372                 pdpe_start = addr_gpa2hva(vm, pml4e->address
 373                         * vm->page_size);
 374                 for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
 375                         pdpe = &pdpe_start[n2];
 376                         if (!pdpe->present)
 377                                 continue;
 378                         fprintf(stream, "%*spdpe  0x%-3zx %p 0x%-12lx 0x%-10lx "
 379                                 "%u  %u\n",
 380                                 indent, "",
 381                                 pdpe - pdpe_start, pdpe,
 382                                 addr_hva2gpa(vm, pdpe),
 383                                 (uint64_t) pdpe->address, pdpe->writable,
 384                                 pdpe->execute_disable);
 385
 386                         pde_start = addr_gpa2hva(vm,
 387                                 pdpe->address * vm->page_size);
 388                         for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
 389                                 pde = &pde_start[n3];
 390                                 if (!pde->present)
 391                                         continue;
 392                                 fprintf(stream, "%*spde   0x%-3zx %p "
 393                                         "0x%-12lx 0x%-10lx %u  %u\n",
 394                                         indent, "", pde - pde_start, pde,
 395                                         addr_hva2gpa(vm, pde),
 396                                         (uint64_t) pde->address, pde->writable,
 397                                         pde->execute_disable);
 398
 399                                 pte_start = addr_gpa2hva(vm,
 400                                         pde->address * vm->page_size);
 401                                 for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
 402                                         pte = &pte_start[n4];
 403                                         if (!pte->present)
 404                                                 continue;
 405                                         fprintf(stream, "%*spte   0x%-3zx %p "
 406                                                 "0x%-12lx 0x%-10lx %u  %u "
 407                                                 "    %u    0x%-10lx\n",
 408                                                 indent, "",
 409                                                 pte - pte_start, pte,
 410                                                 addr_hva2gpa(vm, pte),
 411                                                 (uint64_t) pte->address,
 412                                                 pte->writable,
 413                                                 pte->execute_disable,
 414                                                 pte->dirty,
 415                                                 ((uint64_t) n1 << 27)
 416                                                         | ((uint64_t) n2 << 18)
 417                                                         | ((uint64_t) n3 << 9)
 418                                                         | ((uint64_t) n4));
 419                                 }
 420                         }
 421                 }
 422         }
 423 }
 424
 425 /* Set Unusable Segment
 426  *
 427  * Input Args: None
 428  *
 429  * Output Args:
 430  *   segp - Pointer to segment register
 431  *
 432  * Return: None
 433  *
 434  * Sets the segment register pointed to by segp to an unusable state.
 435  */
 436 static void kvm_seg_set_unusable(struct kvm_segment *segp)
 437 {
 438         memset(segp, 0, sizeof(*segp));
 439         segp->unusable = true;
 440 }
 441
 442 static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
 443 {
 444         void *gdt = addr_gva2hva(vm, vm->gdt);
 445         struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
 446
 447         desc->limit0 = segp->limit & 0xFFFF;
 448         desc->base0 = segp->base & 0xFFFF;
 449         desc->base1 = segp->base >> 16;
 450         desc->s = segp->s;
 451         desc->type = segp->type;
 452         desc->dpl = segp->dpl;
 453         desc->p = segp->present;
 454         desc->limit1 = segp->limit >> 16;
 455         desc->l = segp->l;
 456         desc->db = segp->db;
 457         desc->g = segp->g;
 458         desc->base2 = segp->base >> 24;
 459         if (!segp->s)
 460                 desc->base3 = segp->base >> 32;
 461 }
 462
 463
 464 /* Set Long Mode Flat Kernel Code Segment
 465  *
 466  * Input Args:
 467  *   vm - VM whose GDT is being filled, or NULL to only write segp
 468  *   selector - selector value
 469  *
 470  * Output Args:
 471  *   segp - Pointer to KVM segment
 472  *
 473  * Return: None
 474  *
 475  * Sets up the KVM segment pointed to by segp, to be a code segment
 476  * with the selector value given by selector.
 477  */
 478 static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector,
 479         struct kvm_segment *segp)
 480 {
 481         memset(segp, 0, sizeof(*segp));
 482         segp->selector = selector;
 483         segp->limit = 0xFFFFFFFFu;
 484         segp->s = 0x1; /* kTypeCodeData */
 485         segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
 486                                           * | kFlagCodeReadable
 487                                           */
 488         segp->g = true;
 489         segp->l = true;
 490         segp->present = 1;
 491         if (vm)
 492                 kvm_seg_fill_gdt_64bit(vm, segp);
 493 }
 494
 495 /* Set Long Mode Flat Kernel Data Segment
 496  *
 497  * Input Args:
 498  *   vm - VM whose GDT is being filled, or NULL to only write segp
 499  *   selector - selector value
 500  *
 501  * Output Args:
 502  *   segp - Pointer to KVM segment
 503  *
 504  * Return: None
 505  *
 506  * Sets up the KVM segment pointed to by segp, to be a data segment
 507  * with the selector value given by selector.
 508  */
 509 static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
 510         struct kvm_segment *segp)
 511 {
 512         memset(segp, 0, sizeof(*segp));
 513         segp->selector = selector;
 514         segp->limit = 0xFFFFFFFFu;
 515         segp->s = 0x1; /* kTypeCodeData */
 516         segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
 517                                           * | kFlagDataWritable
 518                                           */
 519         segp->g = true;
 520         segp->present = true;
 521         if (vm)
 522                 kvm_seg_fill_gdt_64bit(vm, segp);
 523 }
 524
 525 /* Address Guest Virtual to Guest Physical
 526  *
 527  * Input Args:
 528  *   vm - Virtual Machine
 529  *   gpa - VM virtual address
 530  *
 531  * Output Args: None
 532  *
 533  * Return:
 534  *   Equivalent VM physical address
 535  *
 536  * Translates the VM virtual address given by gva to a VM physical
 537  * address and then locates the memory region containing the VM
 538  * physical address, within the VM given by vm.  When found, the host
 539  * virtual address providing the memory to the vm physical address is returned.
 540  * A TEST_ASSERT failure occurs if no region containing translated
 541  * VM virtual address exists.
 542  */
 543 vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
 544 {
 545         uint16_t index[4];
 546         struct pageMapL4Entry *pml4e;
 547         struct pageDirectoryPointerEntry *pdpe;
 548         struct pageDirectoryEntry *pde;
 549         struct pageTableEntry *pte;
 550
 551         TEST_ASSERT(vm->mode == VM_MODE_P52V48_4K, "Attempt to use "
 552                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 553
 554         index[0] = (gva >> 12) & 0x1ffu;
 555         index[1] = (gva >> 21) & 0x1ffu;
 556         index[2] = (gva >> 30) & 0x1ffu;
 557         index[3] = (gva >> 39) & 0x1ffu;
 558
 559         if (!vm->pgd_created)
 560                 goto unmapped_gva;
 561         pml4e = addr_gpa2hva(vm, vm->pgd);
 562         if (!pml4e[index[3]].present)
 563                 goto unmapped_gva;
 564
 565         pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size);
 566         if (!pdpe[index[2]].present)
 567                 goto unmapped_gva;
 568
 569         pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size);
 570         if (!pde[index[1]].present)
 571                 goto unmapped_gva;
 572
 573         pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size);
 574         if (!pte[index[0]].present)
 575                 goto unmapped_gva;
 576
 577         return (pte[index[0]].address * vm->page_size) + (gva & 0xfffu);
 578
 579 unmapped_gva:
 580         TEST_ASSERT(false, "No mapping for vm virtual address, "
 581                     "gva: 0x%lx", gva);
 582         exit(EXIT_FAILURE);
 583 }
 584
 585 static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt, int gdt_memslot,
 586                           int pgd_memslot)
 587 {
 588         if (!vm->gdt)
 589                 vm->gdt = vm_vaddr_alloc(vm, getpagesize(),
 590                         KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot);
 591
 592         dt->base = vm->gdt;
 593         dt->limit = getpagesize();
 594 }
 595
 596 static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp,
 597                                 int selector, int gdt_memslot,
 598                                 int pgd_memslot)
 599 {
 600         if (!vm->tss)
 601                 vm->tss = vm_vaddr_alloc(vm, getpagesize(),
 602                         KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot);
 603
 604         memset(segp, 0, sizeof(*segp));
 605         segp->base = vm->tss;
 606         segp->limit = 0x67;
 607         segp->selector = selector;
 608         segp->type = 0xb;
 609         segp->present = 1;
 610         kvm_seg_fill_gdt_64bit(vm, segp);
 611 }
 612
 613 void vcpu_setup(struct kvm_vm *vm, int vcpuid, int pgd_memslot, int gdt_memslot)
 614 {
 615         struct kvm_sregs sregs;
 616
 617         /* Set mode specific system register values. */
 618         vcpu_sregs_get(vm, vcpuid, &sregs);
 619
 620         sregs.idt.limit = 0;
 621
 622         kvm_setup_gdt(vm, &sregs.gdt, gdt_memslot, pgd_memslot);
 623
 624         switch (vm->mode) {
 625         case VM_MODE_P52V48_4K:
 626                 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
 627                 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
 628                 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
 629
 630                 kvm_seg_set_unusable(&sregs.ldt);
 631                 kvm_seg_set_kernel_code_64bit(vm, 0x8, &sregs.cs);
 632                 kvm_seg_set_kernel_data_64bit(vm, 0x10, &sregs.ds);
 633                 kvm_seg_set_kernel_data_64bit(vm, 0x10, &sregs.es);
 634                 kvm_setup_tss_64bit(vm, &sregs.tr, 0x18, gdt_memslot, pgd_memslot);
 635                 break;
 636
 637         default:
 638                 TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", vm->mode);
 639         }
 640
 641         sregs.cr3 = vm->pgd;
 642         vcpu_sregs_set(vm, vcpuid, &sregs);
 643 }
 644 /* Adds a vCPU with reasonable defaults (i.e., a stack)
 645  *
 646  * Input Args:
 647  *   vcpuid - The id of the VCPU to add to the VM.
 648  *   guest_code - The vCPU's entry point
 649  */
 650 void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)
 651 {
 652         struct kvm_mp_state mp_state;
 653         struct kvm_regs regs;
 654         vm_vaddr_t stack_vaddr;
 655         stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
 656                                      DEFAULT_GUEST_STACK_VADDR_MIN, 0, 0);
 657
 658         /* Create VCPU */
 659         vm_vcpu_add(vm, vcpuid, 0, 0);
 660
 661         /* Setup guest general purpose registers */
 662         vcpu_regs_get(vm, vcpuid, &regs);
 663         regs.rflags = regs.rflags | 0x2;
 664         regs.rsp = stack_vaddr + (DEFAULT_STACK_PGS * getpagesize());
 665         regs.rip = (unsigned long) guest_code;
 666         vcpu_regs_set(vm, vcpuid, &regs);
 667
 668         /* Setup the MP state */
 669         mp_state.mp_state = 0;
 670         vcpu_set_mp_state(vm, vcpuid, &mp_state);
 671 }
 672
 673 /* Allocate an instance of struct kvm_cpuid2
 674  *
 675  * Input Args: None
 676  *
 677  * Output Args: None
 678  *
 679  * Return: A pointer to the allocated struct. The caller is responsible
 680  * for freeing this struct.
 681  *
 682  * Since kvm_cpuid2 uses a 0-length array to allow a the size of the
 683  * array to be decided at allocation time, allocation is slightly
 684  * complicated. This function uses a reasonable default length for
 685  * the array and performs the appropriate allocation.
 686  */
 687 static struct kvm_cpuid2 *allocate_kvm_cpuid2(void)
 688 {
 689         struct kvm_cpuid2 *cpuid;
 690         int nent = 100;
 691         size_t size;
 692
 693         size = sizeof(*cpuid);
 694         size += nent * sizeof(struct kvm_cpuid_entry2);
 695         cpuid = malloc(size);
 696         if (!cpuid) {
 697                 perror("malloc");
 698                 abort();
 699         }
 700
 701         cpuid->nent = nent;
 702
 703         return cpuid;
 704 }
 705
 706 /* KVM Supported CPUID Get
 707  *
 708  * Input Args: None
 709  *
 710  * Output Args:
 711  *
 712  * Return: The supported KVM CPUID
 713  *
 714  * Get the guest CPUID supported by KVM.
 715  */
 716 struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
 717 {
 718         static struct kvm_cpuid2 *cpuid;
 719         int ret;
 720         int kvm_fd;
 721
 722         if (cpuid)
 723                 return cpuid;
 724
 725         cpuid = allocate_kvm_cpuid2();
 726         kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
 727         if (kvm_fd < 0)
 728                 exit(KSFT_SKIP);
 729
 730         ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID, cpuid);
 731         TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_CPUID failed %d %d\n",
 732                     ret, errno);
 733
 734         close(kvm_fd);
 735         return cpuid;
 736 }
 737
 738 /* Locate a cpuid entry.
 739  *
 740  * Input Args:
 741  *   cpuid: The cpuid.
 742  *   function: The function of the cpuid entry to find.
 743  *
 744  * Output Args: None
 745  *
 746  * Return: A pointer to the cpuid entry. Never returns NULL.
 747  */
 748 struct kvm_cpuid_entry2 *
 749 kvm_get_supported_cpuid_index(uint32_t function, uint32_t index)
 750 {
 751         struct kvm_cpuid2 *cpuid;
 752         struct kvm_cpuid_entry2 *entry = NULL;
 753         int i;
 754
 755         cpuid = kvm_get_supported_cpuid();
 756         for (i = 0; i < cpuid->nent; i++) {
 757                 if (cpuid->entries[i].function == function &&
 758                     cpuid->entries[i].index == index) {
 759                         entry = &cpuid->entries[i];
 760                         break;
 761                 }
 762         }
 763
 764         TEST_ASSERT(entry, "Guest CPUID entry not found: (EAX=%x, ECX=%x).",
 765                     function, index);
 766         return entry;
 767 }
 768
 769 /* VM VCPU CPUID Set
 770  *
 771  * Input Args:
 772  *   vm - Virtual Machine
 773  *   vcpuid - VCPU id
 774  *   cpuid - The CPUID values to set.
 775  *
 776  * Output Args: None
 777  *
 778  * Return: void
 779  *
 780  * Set the VCPU's CPUID.
 781  */
 782 void vcpu_set_cpuid(struct kvm_vm *vm,
 783                 uint32_t vcpuid, struct kvm_cpuid2 *cpuid)
 784 {
 785         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
 786         int rc;
 787
 788         TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
 789
 790         rc = ioctl(vcpu->fd, KVM_SET_CPUID2, cpuid);
 791         TEST_ASSERT(rc == 0, "KVM_SET_CPUID2 failed, rc: %i errno: %i",
 792                     rc, errno);
 793
 794 }
 795
 796 /* Create a VM with reasonable defaults
 797  *
 798  * Input Args:
 799  *   vcpuid - The id of the single VCPU to add to the VM.
 800  *   extra_mem_pages - The size of extra memories to add (this will
 801  *                     decide how much extra space we will need to
 802  *                     setup the page tables using mem slot 0)
 803  *   guest_code - The vCPU's entry point
 804  *
 805  * Output Args: None
 806  *
 807  * Return:
 808  *   Pointer to opaque structure that describes the created VM.
 809  */
 810 struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages,
 811                                  void *guest_code)
 812 {
 813         struct kvm_vm *vm;
 814         /*
 815          * For x86 the maximum page table size for a memory region
 816          * will be when only 4K pages are used.  In that case the
 817          * total extra size for page tables (for extra N pages) will
 818          * be: N/512+N/512^2+N/512^3+... which is definitely smaller
 819          * than N/512*2.
 820          */
 821         uint64_t extra_pg_pages = extra_mem_pages / 512 * 2;
 822
 823         /* Create VM */
 824         vm = vm_create(VM_MODE_P52V48_4K,
 825                        DEFAULT_GUEST_PHY_PAGES + extra_pg_pages,
 826                        O_RDWR);
 827
 828         /* Setup guest code */
 829         kvm_vm_elf_load(vm, program_invocation_name, 0, 0);
 830
 831         /* Setup IRQ Chip */
 832         vm_create_irqchip(vm);
 833
 834         /* Add the first vCPU. */
 835         vm_vcpu_add_default(vm, vcpuid, guest_code);
 836
 837         return vm;
 838 }
 839
 840 /* VCPU Get MSR
 841  *
 842  * Input Args:
 843  *   vm - Virtual Machine
 844  *   vcpuid - VCPU ID
 845  *   msr_index - Index of MSR
 846  *
 847  * Output Args: None
 848  *
 849  * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
 850  *
 851  * Get value of MSR for VCPU.
 852  */
 853 uint64_t vcpu_get_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index)
 854 {
 855         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
 856         struct {
 857                 struct kvm_msrs header;
 858                 struct kvm_msr_entry entry;
 859         } buffer = {};
 860         int r;
 861
 862         TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
 863         buffer.header.nmsrs = 1;
 864         buffer.entry.index = msr_index;
 865         r = ioctl(vcpu->fd, KVM_GET_MSRS, &buffer.header);
 866         TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
 867                 "  rc: %i errno: %i", r, errno);
 868
 869         return buffer.entry.data;
 870 }
 871
 872 /* VCPU Set MSR
 873  *
 874  * Input Args:
 875  *   vm - Virtual Machine
 876  *   vcpuid - VCPU ID
 877  *   msr_index - Index of MSR
 878  *   msr_value - New value of MSR
 879  *
 880  * Output Args: None
 881  *
 882  * Return: On success, nothing. On failure a TEST_ASSERT is produced.
 883  *
 884  * Set value of MSR for VCPU.
 885  */
 886 void vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index,
 887         uint64_t msr_value)
 888 {
 889         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
 890         struct {
 891                 struct kvm_msrs header;
 892                 struct kvm_msr_entry entry;
 893         } buffer = {};
 894         int r;
 895
 896         TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
 897         memset(&buffer, 0, sizeof(buffer));
 898         buffer.header.nmsrs = 1;
 899         buffer.entry.index = msr_index;
 900         buffer.entry.data = msr_value;
 901         r = ioctl(vcpu->fd, KVM_SET_MSRS, &buffer.header);
 902         TEST_ASSERT(r == 1, "KVM_SET_MSRS IOCTL failed,\n"
 903                 "  rc: %i errno: %i", r, errno);
 904 }
 905
 906 /* VM VCPU Args Set
 907  *
 908  * Input Args:
 909  *   vm - Virtual Machine
 910  *   vcpuid - VCPU ID
 911  *   num - number of arguments
 912  *   ... - arguments, each of type uint64_t
 913  *
 914  * Output Args: None
 915  *
 916  * Return: None
 917  *
 918  * Sets the first num function input arguments to the values
 919  * given as variable args.  Each of the variable args is expected to
 920  * be of type uint64_t.
 921  */
 922 void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...)
 923 {
 924         va_list ap;
 925         struct kvm_regs regs;
 926
 927         TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n"
 928                     "  num: %u\n",
 929                     num);
 930
 931         va_start(ap, num);
 932         vcpu_regs_get(vm, vcpuid, &regs);
 933
 934         if (num >= 1)
 935                 regs.rdi = va_arg(ap, uint64_t);
 936
 937         if (num >= 2)
 938                 regs.rsi = va_arg(ap, uint64_t);
 939
 940         if (num >= 3)
 941                 regs.rdx = va_arg(ap, uint64_t);
 942
 943         if (num >= 4)
 944                 regs.rcx = va_arg(ap, uint64_t);
 945
 946         if (num >= 5)
 947                 regs.r8 = va_arg(ap, uint64_t);
 948
 949         if (num >= 6)
 950                 regs.r9 = va_arg(ap, uint64_t);
 951
 952         vcpu_regs_set(vm, vcpuid, &regs);
 953         va_end(ap);
 954 }
 955
 956 /*
 957  * VM VCPU Dump
 958  *
 959  * Input Args:
 960  *   vm - Virtual Machine
 961  *   vcpuid - VCPU ID
 962  *   indent - Left margin indent amount
 963  *
 964  * Output Args:
 965  *   stream - Output FILE stream
 966  *
 967  * Return: None
 968  *
 969  * Dumps the current state of the VCPU specified by vcpuid, within the VM
 970  * given by vm, to the FILE stream given by stream.
 971  */
 972 void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
 973 {
 974         struct kvm_regs regs;
 975         struct kvm_sregs sregs;
 976
 977         fprintf(stream, "%*scpuid: %u\n", indent, "", vcpuid);
 978
 979         fprintf(stream, "%*sregs:\n", indent + 2, "");
 980         vcpu_regs_get(vm, vcpuid, &regs);
 981         regs_dump(stream, &regs, indent + 4);
 982
 983         fprintf(stream, "%*ssregs:\n", indent + 2, "");
 984         vcpu_sregs_get(vm, vcpuid, &sregs);
 985         sregs_dump(stream, &sregs, indent + 4);
 986 }
 987
 988 struct kvm_x86_state {
 989         struct kvm_vcpu_events events;
 990         struct kvm_mp_state mp_state;
 991         struct kvm_regs regs;
 992         struct kvm_xsave xsave;
 993         struct kvm_xcrs xcrs;
 994         struct kvm_sregs sregs;
 995         struct kvm_debugregs debugregs;
 996         union {
 997                 struct kvm_nested_state nested;
 998                 char nested_[16384];
 999         };
1000         struct kvm_msrs msrs;
1001 };
1002
1003 static int kvm_get_num_msrs(struct kvm_vm *vm)
1004 {
1005         struct kvm_msr_list nmsrs;
1006         int r;
1007
1008         nmsrs.nmsrs = 0;
1009         r = ioctl(vm->kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs);
1010         TEST_ASSERT(r == -1 && errno == E2BIG, "Unexpected result from KVM_GET_MSR_INDEX_LIST probe, r: %i",
1011                 r);
1012
1013         return nmsrs.nmsrs;
1014 }
1015
1016 struct kvm_x86_state *vcpu_save_state(struct kvm_vm *vm, uint32_t vcpuid)
1017 {
1018         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1019         struct kvm_msr_list *list;
1020         struct kvm_x86_state *state;
1021         int nmsrs, r, i;
1022         static int nested_size = -1;
1023
1024         if (nested_size == -1) {
1025                 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE);
1026                 TEST_ASSERT(nested_size <= sizeof(state->nested_),
1027                             "Nested state size too big, %i > %zi",
1028                             nested_size, sizeof(state->nested_));
1029         }
1030
1031         /*
1032          * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees
1033          * guest state is consistent only after userspace re-enters the
1034          * kernel with KVM_RUN.  Complete IO prior to migrating state
1035          * to a new VM.
1036          */
1037         vcpu_run_complete_io(vm, vcpuid);
1038
1039         nmsrs = kvm_get_num_msrs(vm);
1040         list = malloc(sizeof(*list) + nmsrs * sizeof(list->indices[0]));
1041         list->nmsrs = nmsrs;
1042         r = ioctl(vm->kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
1043         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MSR_INDEX_LIST, r: %i",
1044                 r);
1045
1046         state = malloc(sizeof(*state) + nmsrs * sizeof(state->msrs.entries[0]));
1047         r = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, &state->events);
1048         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_VCPU_EVENTS, r: %i",
1049                 r);
1050
1051         r = ioctl(vcpu->fd, KVM_GET_MP_STATE, &state->mp_state);
1052         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MP_STATE, r: %i",
1053                 r);
1054
1055         r = ioctl(vcpu->fd, KVM_GET_REGS, &state->regs);
1056         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_REGS, r: %i",
1057                 r);
1058
1059         r = ioctl(vcpu->fd, KVM_GET_XSAVE, &state->xsave);
1060         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XSAVE, r: %i",
1061                 r);
1062
1063         r = ioctl(vcpu->fd, KVM_GET_XCRS, &state->xcrs);
1064         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XCRS, r: %i",
1065                 r);
1066
1067         r = ioctl(vcpu->fd, KVM_GET_SREGS, &state->sregs);
1068         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_SREGS, r: %i",
1069                 r);
1070
1071         if (nested_size) {
1072                 state->nested.size = sizeof(state->nested_);
1073                 r = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, &state->nested);
1074                 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_NESTED_STATE, r: %i",
1075                         r);
1076                 TEST_ASSERT(state->nested.size <= nested_size,
1077                         "Nested state size too big, %i (KVM_CHECK_CAP gave %i)",
1078                         state->nested.size, nested_size);
1079         } else
1080                 state->nested.size = 0;
1081
1082         state->msrs.nmsrs = nmsrs;
1083         for (i = 0; i < nmsrs; i++)
1084                 state->msrs.entries[i].index = list->indices[i];
1085         r = ioctl(vcpu->fd, KVM_GET_MSRS, &state->msrs);
1086         TEST_ASSERT(r == nmsrs, "Unexpected result from KVM_GET_MSRS, r: %i (failed at %x)",
1087                 r, r == nmsrs ? -1 : list->indices[r]);
1088
1089         r = ioctl(vcpu->fd, KVM_GET_DEBUGREGS, &state->debugregs);
1090         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_DEBUGREGS, r: %i",
1091                 r);
1092
1093         free(list);
1094         return state;
1095 }
1096
1097 void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *state)
1098 {
1099         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1100         int r;
1101
1102         r = ioctl(vcpu->fd, KVM_SET_XSAVE, &state->xsave);
1103         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XSAVE, r: %i",
1104                 r);
1105
1106         r = ioctl(vcpu->fd, KVM_SET_XCRS, &state->xcrs);
1107         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XCRS, r: %i",
1108                 r);
1109
1110         r = ioctl(vcpu->fd, KVM_SET_SREGS, &state->sregs);
1111         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_SREGS, r: %i",
1112                 r);
1113
1114         r = ioctl(vcpu->fd, KVM_SET_MSRS, &state->msrs);
1115         TEST_ASSERT(r == state->msrs.nmsrs, "Unexpected result from KVM_SET_MSRS, r: %i (failed at %x)",
1116                 r, r == state->msrs.nmsrs ? -1 : state->msrs.entries[r].index);
1117
1118         r = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, &state->events);
1119         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_VCPU_EVENTS, r: %i",
1120                 r);
1121
1122         r = ioctl(vcpu->fd, KVM_SET_MP_STATE, &state->mp_state);
1123         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_MP_STATE, r: %i",
1124                 r);
1125
1126         r = ioctl(vcpu->fd, KVM_SET_DEBUGREGS, &state->debugregs);
1127         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_DEBUGREGS, r: %i",
1128                 r);
1129
1130         r = ioctl(vcpu->fd, KVM_SET_REGS, &state->regs);
1131         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_REGS, r: %i",
1132                 r);
1133
1134         if (state->nested.size) {
1135                 r = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, &state->nested);
1136                 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_NESTED_STATE, r: %i",
1137                         r);
1138         }
1139 }