tools/testing/selftests/kvm/lib/x86_64/processor.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * tools/testing/selftests/kvm/lib/x86_64/processor.c
   4  *
   5  * Copyright (C) 2018, Google LLC.
   6  */
   7
   8 #define _GNU_SOURCE /* for program_invocation_name */
   9
  10 #include "test_util.h"
  11 #include "kvm_util.h"
  12 #include "../kvm_util_internal.h"
  13 #include "processor.h"
  14
  15 /* Minimum physical address used for virtual translation tables. */
  16 #define KVM_GUEST_PAGE_TABLE_MIN_PADDR 0x180000
  17
  18 /* Virtual translation table structure declarations */
  19 struct pageMapL4Entry {
  20         uint64_t present:1;
  21         uint64_t writable:1;
  22         uint64_t user:1;
  23         uint64_t write_through:1;
  24         uint64_t cache_disable:1;
  25         uint64_t accessed:1;
  26         uint64_t ignored_06:1;
  27         uint64_t page_size:1;
  28         uint64_t ignored_11_08:4;
  29         uint64_t address:40;
  30         uint64_t ignored_62_52:11;
  31         uint64_t execute_disable:1;
  32 };
  33
  34 struct pageDirectoryPointerEntry {
  35         uint64_t present:1;
  36         uint64_t writable:1;
  37         uint64_t user:1;
  38         uint64_t write_through:1;
  39         uint64_t cache_disable:1;
  40         uint64_t accessed:1;
  41         uint64_t ignored_06:1;
  42         uint64_t page_size:1;
  43         uint64_t ignored_11_08:4;
  44         uint64_t address:40;
  45         uint64_t ignored_62_52:11;
  46         uint64_t execute_disable:1;
  47 };
  48
  49 struct pageDirectoryEntry {
  50         uint64_t present:1;
  51         uint64_t writable:1;
  52         uint64_t user:1;
  53         uint64_t write_through:1;
  54         uint64_t cache_disable:1;
  55         uint64_t accessed:1;
  56         uint64_t ignored_06:1;
  57         uint64_t page_size:1;
  58         uint64_t ignored_11_08:4;
  59         uint64_t address:40;
  60         uint64_t ignored_62_52:11;
  61         uint64_t execute_disable:1;
  62 };
  63
  64 struct pageTableEntry {
  65         uint64_t present:1;
  66         uint64_t writable:1;
  67         uint64_t user:1;
  68         uint64_t write_through:1;
  69         uint64_t cache_disable:1;
  70         uint64_t accessed:1;
  71         uint64_t dirty:1;
  72         uint64_t reserved_07:1;
  73         uint64_t global:1;
  74         uint64_t ignored_11_09:3;
  75         uint64_t address:40;
  76         uint64_t ignored_62_52:11;
  77         uint64_t execute_disable:1;
  78 };
  79
  80 /* Register Dump
  81  *
  82  * Input Args:
  83  *   indent - Left margin indent amount
  84  *   regs - register
  85  *
  86  * Output Args:
  87  *   stream - Output FILE stream
  88  *
  89  * Return: None
  90  *
  91  * Dumps the state of the registers given by regs, to the FILE stream
  92  * given by steam.
  93  */
  94 void regs_dump(FILE *stream, struct kvm_regs *regs,
  95                uint8_t indent)
  96 {
  97         fprintf(stream, "%*srax: 0x%.16llx rbx: 0x%.16llx "
  98                 "rcx: 0x%.16llx rdx: 0x%.16llx\n",
  99                 indent, "",
 100                 regs->rax, regs->rbx, regs->rcx, regs->rdx);
 101         fprintf(stream, "%*srsi: 0x%.16llx rdi: 0x%.16llx "
 102                 "rsp: 0x%.16llx rbp: 0x%.16llx\n",
 103                 indent, "",
 104                 regs->rsi, regs->rdi, regs->rsp, regs->rbp);
 105         fprintf(stream, "%*sr8:  0x%.16llx r9:  0x%.16llx "
 106                 "r10: 0x%.16llx r11: 0x%.16llx\n",
 107                 indent, "",
 108                 regs->r8, regs->r9, regs->r10, regs->r11);
 109         fprintf(stream, "%*sr12: 0x%.16llx r13: 0x%.16llx "
 110                 "r14: 0x%.16llx r15: 0x%.16llx\n",
 111                 indent, "",
 112                 regs->r12, regs->r13, regs->r14, regs->r15);
 113         fprintf(stream, "%*srip: 0x%.16llx rfl: 0x%.16llx\n",
 114                 indent, "",
 115                 regs->rip, regs->rflags);
 116 }
 117
 118 /* Segment Dump
 119  *
 120  * Input Args:
 121  *   indent - Left margin indent amount
 122  *   segment - KVM segment
 123  *
 124  * Output Args:
 125  *   stream - Output FILE stream
 126  *
 127  * Return: None
 128  *
 129  * Dumps the state of the KVM segment given by segment, to the FILE stream
 130  * given by steam.
 131  */
 132 static void segment_dump(FILE *stream, struct kvm_segment *segment,
 133                          uint8_t indent)
 134 {
 135         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.8x "
 136                 "selector: 0x%.4x type: 0x%.2x\n",
 137                 indent, "", segment->base, segment->limit,
 138                 segment->selector, segment->type);
 139         fprintf(stream, "%*spresent: 0x%.2x dpl: 0x%.2x "
 140                 "db: 0x%.2x s: 0x%.2x l: 0x%.2x\n",
 141                 indent, "", segment->present, segment->dpl,
 142                 segment->db, segment->s, segment->l);
 143         fprintf(stream, "%*sg: 0x%.2x avl: 0x%.2x "
 144                 "unusable: 0x%.2x padding: 0x%.2x\n",
 145                 indent, "", segment->g, segment->avl,
 146                 segment->unusable, segment->padding);
 147 }
 148
 149 /* dtable Dump
 150  *
 151  * Input Args:
 152  *   indent - Left margin indent amount
 153  *   dtable - KVM dtable
 154  *
 155  * Output Args:
 156  *   stream - Output FILE stream
 157  *
 158  * Return: None
 159  *
 160  * Dumps the state of the KVM dtable given by dtable, to the FILE stream
 161  * given by steam.
 162  */
 163 static void dtable_dump(FILE *stream, struct kvm_dtable *dtable,
 164                         uint8_t indent)
 165 {
 166         fprintf(stream, "%*sbase: 0x%.16llx limit: 0x%.4x "
 167                 "padding: 0x%.4x 0x%.4x 0x%.4x\n",
 168                 indent, "", dtable->base, dtable->limit,
 169                 dtable->padding[0], dtable->padding[1], dtable->padding[2]);
 170 }
 171
 172 /* System Register Dump
 173  *
 174  * Input Args:
 175  *   indent - Left margin indent amount
 176  *   sregs - System registers
 177  *
 178  * Output Args:
 179  *   stream - Output FILE stream
 180  *
 181  * Return: None
 182  *
 183  * Dumps the state of the system registers given by sregs, to the FILE stream
 184  * given by steam.
 185  */
 186 void sregs_dump(FILE *stream, struct kvm_sregs *sregs,
 187                 uint8_t indent)
 188 {
 189         unsigned int i;
 190
 191         fprintf(stream, "%*scs:\n", indent, "");
 192         segment_dump(stream, &sregs->cs, indent + 2);
 193         fprintf(stream, "%*sds:\n", indent, "");
 194         segment_dump(stream, &sregs->ds, indent + 2);
 195         fprintf(stream, "%*ses:\n", indent, "");
 196         segment_dump(stream, &sregs->es, indent + 2);
 197         fprintf(stream, "%*sfs:\n", indent, "");
 198         segment_dump(stream, &sregs->fs, indent + 2);
 199         fprintf(stream, "%*sgs:\n", indent, "");
 200         segment_dump(stream, &sregs->gs, indent + 2);
 201         fprintf(stream, "%*sss:\n", indent, "");
 202         segment_dump(stream, &sregs->ss, indent + 2);
 203         fprintf(stream, "%*str:\n", indent, "");
 204         segment_dump(stream, &sregs->tr, indent + 2);
 205         fprintf(stream, "%*sldt:\n", indent, "");
 206         segment_dump(stream, &sregs->ldt, indent + 2);
 207
 208         fprintf(stream, "%*sgdt:\n", indent, "");
 209         dtable_dump(stream, &sregs->gdt, indent + 2);
 210         fprintf(stream, "%*sidt:\n", indent, "");
 211         dtable_dump(stream, &sregs->idt, indent + 2);
 212
 213         fprintf(stream, "%*scr0: 0x%.16llx cr2: 0x%.16llx "
 214                 "cr3: 0x%.16llx cr4: 0x%.16llx\n",
 215                 indent, "",
 216                 sregs->cr0, sregs->cr2, sregs->cr3, sregs->cr4);
 217         fprintf(stream, "%*scr8: 0x%.16llx efer: 0x%.16llx "
 218                 "apic_base: 0x%.16llx\n",
 219                 indent, "",
 220                 sregs->cr8, sregs->efer, sregs->apic_base);
 221
 222         fprintf(stream, "%*sinterrupt_bitmap:\n", indent, "");
 223         for (i = 0; i < (KVM_NR_INTERRUPTS + 63) / 64; i++) {
 224                 fprintf(stream, "%*s%.16llx\n", indent + 2, "",
 225                         sregs->interrupt_bitmap[i]);
 226         }
 227 }
 228
 229 void virt_pgd_alloc(struct kvm_vm *vm, uint32_t pgd_memslot)
 230 {
 231         TEST_ASSERT(vm->mode == VM_MODE_P52V48_4K, "Attempt to use "
 232                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 233
 234         /* If needed, create page map l4 table. */
 235         if (!vm->pgd_created) {
 236                 vm_paddr_t paddr = vm_phy_page_alloc(vm,
 237                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot);
 238                 vm->pgd = paddr;
 239                 vm->pgd_created = true;
 240         }
 241 }
 242
 243 /* VM Virtual Page Map
 244  *
 245  * Input Args:
 246  *   vm - Virtual Machine
 247  *   vaddr - VM Virtual Address
 248  *   paddr - VM Physical Address
 249  *   pgd_memslot - Memory region slot for new virtual translation tables
 250  *
 251  * Output Args: None
 252  *
 253  * Return: None
 254  *
 255  * Within the VM given by vm, creates a virtual translation for the page
 256  * starting at vaddr to the page starting at paddr.
 257  */
 258 void virt_pg_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
 259         uint32_t pgd_memslot)
 260 {
 261         uint16_t index[4];
 262         struct pageMapL4Entry *pml4e;
 263
 264         TEST_ASSERT(vm->mode == VM_MODE_P52V48_4K, "Attempt to use "
 265                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 266
 267         TEST_ASSERT((vaddr % vm->page_size) == 0,
 268                 "Virtual address not on page boundary,\n"
 269                 "  vaddr: 0x%lx vm->page_size: 0x%x",
 270                 vaddr, vm->page_size);
 271         TEST_ASSERT(sparsebit_is_set(vm->vpages_valid,
 272                 (vaddr >> vm->page_shift)),
 273                 "Invalid virtual address, vaddr: 0x%lx",
 274                 vaddr);
 275         TEST_ASSERT((paddr % vm->page_size) == 0,
 276                 "Physical address not on page boundary,\n"
 277                 "  paddr: 0x%lx vm->page_size: 0x%x",
 278                 paddr, vm->page_size);
 279         TEST_ASSERT((paddr >> vm->page_shift) <= vm->max_gfn,
 280                 "Physical address beyond beyond maximum supported,\n"
 281                 "  paddr: 0x%lx vm->max_gfn: 0x%lx vm->page_size: 0x%x",
 282                 paddr, vm->max_gfn, vm->page_size);
 283
 284         index[0] = (vaddr >> 12) & 0x1ffu;
 285         index[1] = (vaddr >> 21) & 0x1ffu;
 286         index[2] = (vaddr >> 30) & 0x1ffu;
 287         index[3] = (vaddr >> 39) & 0x1ffu;
 288
 289         /* Allocate page directory pointer table if not present. */
 290         pml4e = addr_gpa2hva(vm, vm->pgd);
 291         if (!pml4e[index[3]].present) {
 292                 pml4e[index[3]].address = vm_phy_page_alloc(vm,
 293                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
 294                         >> vm->page_shift;
 295                 pml4e[index[3]].writable = true;
 296                 pml4e[index[3]].present = true;
 297         }
 298
 299         /* Allocate page directory table if not present. */
 300         struct pageDirectoryPointerEntry *pdpe;
 301         pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size);
 302         if (!pdpe[index[2]].present) {
 303                 pdpe[index[2]].address = vm_phy_page_alloc(vm,
 304                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
 305                         >> vm->page_shift;
 306                 pdpe[index[2]].writable = true;
 307                 pdpe[index[2]].present = true;
 308         }
 309
 310         /* Allocate page table if not present. */
 311         struct pageDirectoryEntry *pde;
 312         pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size);
 313         if (!pde[index[1]].present) {
 314                 pde[index[1]].address = vm_phy_page_alloc(vm,
 315                         KVM_GUEST_PAGE_TABLE_MIN_PADDR, pgd_memslot)
 316                         >> vm->page_shift;
 317                 pde[index[1]].writable = true;
 318                 pde[index[1]].present = true;
 319         }
 320
 321         /* Fill in page table entry. */
 322         struct pageTableEntry *pte;
 323         pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size);
 324         pte[index[0]].address = paddr >> vm->page_shift;
 325         pte[index[0]].writable = true;
 326         pte[index[0]].present = 1;
 327 }
 328
 329 /* Virtual Translation Tables Dump
 330  *
 331  * Input Args:
 332  *   vm - Virtual Machine
 333  *   indent - Left margin indent amount
 334  *
 335  * Output Args:
 336  *   stream - Output FILE stream
 337  *
 338  * Return: None
 339  *
 340  * Dumps to the FILE stream given by stream, the contents of all the
 341  * virtual translation tables for the VM given by vm.
 342  */
 343 void virt_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
 344 {
 345         struct pageMapL4Entry *pml4e, *pml4e_start;
 346         struct pageDirectoryPointerEntry *pdpe, *pdpe_start;
 347         struct pageDirectoryEntry *pde, *pde_start;
 348         struct pageTableEntry *pte, *pte_start;
 349
 350         if (!vm->pgd_created)
 351                 return;
 352
 353         fprintf(stream, "%*s                                          "
 354                 "                no\n", indent, "");
 355         fprintf(stream, "%*s      index hvaddr         gpaddr         "
 356                 "addr         w exec dirty\n",
 357                 indent, "");
 358         pml4e_start = (struct pageMapL4Entry *) addr_gpa2hva(vm,
 359                 vm->pgd);
 360         for (uint16_t n1 = 0; n1 <= 0x1ffu; n1++) {
 361                 pml4e = &pml4e_start[n1];
 362                 if (!pml4e->present)
 363                         continue;
 364                 fprintf(stream, "%*spml4e 0x%-3zx %p 0x%-12lx 0x%-10lx %u "
 365                         " %u\n",
 366                         indent, "",
 367                         pml4e - pml4e_start, pml4e,
 368                         addr_hva2gpa(vm, pml4e), (uint64_t) pml4e->address,
 369                         pml4e->writable, pml4e->execute_disable);
 370
 371                 pdpe_start = addr_gpa2hva(vm, pml4e->address
 372                         * vm->page_size);
 373                 for (uint16_t n2 = 0; n2 <= 0x1ffu; n2++) {
 374                         pdpe = &pdpe_start[n2];
 375                         if (!pdpe->present)
 376                                 continue;
 377                         fprintf(stream, "%*spdpe  0x%-3zx %p 0x%-12lx 0x%-10lx "
 378                                 "%u  %u\n",
 379                                 indent, "",
 380                                 pdpe - pdpe_start, pdpe,
 381                                 addr_hva2gpa(vm, pdpe),
 382                                 (uint64_t) pdpe->address, pdpe->writable,
 383                                 pdpe->execute_disable);
 384
 385                         pde_start = addr_gpa2hva(vm,
 386                                 pdpe->address * vm->page_size);
 387                         for (uint16_t n3 = 0; n3 <= 0x1ffu; n3++) {
 388                                 pde = &pde_start[n3];
 389                                 if (!pde->present)
 390                                         continue;
 391                                 fprintf(stream, "%*spde   0x%-3zx %p "
 392                                         "0x%-12lx 0x%-10lx %u  %u\n",
 393                                         indent, "", pde - pde_start, pde,
 394                                         addr_hva2gpa(vm, pde),
 395                                         (uint64_t) pde->address, pde->writable,
 396                                         pde->execute_disable);
 397
 398                                 pte_start = addr_gpa2hva(vm,
 399                                         pde->address * vm->page_size);
 400                                 for (uint16_t n4 = 0; n4 <= 0x1ffu; n4++) {
 401                                         pte = &pte_start[n4];
 402                                         if (!pte->present)
 403                                                 continue;
 404                                         fprintf(stream, "%*spte   0x%-3zx %p "
 405                                                 "0x%-12lx 0x%-10lx %u  %u "
 406                                                 "    %u    0x%-10lx\n",
 407                                                 indent, "",
 408                                                 pte - pte_start, pte,
 409                                                 addr_hva2gpa(vm, pte),
 410                                                 (uint64_t) pte->address,
 411                                                 pte->writable,
 412                                                 pte->execute_disable,
 413                                                 pte->dirty,
 414                                                 ((uint64_t) n1 << 27)
 415                                                         | ((uint64_t) n2 << 18)
 416                                                         | ((uint64_t) n3 << 9)
 417                                                         | ((uint64_t) n4));
 418                                 }
 419                         }
 420                 }
 421         }
 422 }
 423
 424 /* Set Unusable Segment
 425  *
 426  * Input Args: None
 427  *
 428  * Output Args:
 429  *   segp - Pointer to segment register
 430  *
 431  * Return: None
 432  *
 433  * Sets the segment register pointed to by segp to an unusable state.
 434  */
 435 static void kvm_seg_set_unusable(struct kvm_segment *segp)
 436 {
 437         memset(segp, 0, sizeof(*segp));
 438         segp->unusable = true;
 439 }
 440
 441 static void kvm_seg_fill_gdt_64bit(struct kvm_vm *vm, struct kvm_segment *segp)
 442 {
 443         void *gdt = addr_gva2hva(vm, vm->gdt);
 444         struct desc64 *desc = gdt + (segp->selector >> 3) * 8;
 445
 446         desc->limit0 = segp->limit & 0xFFFF;
 447         desc->base0 = segp->base & 0xFFFF;
 448         desc->base1 = segp->base >> 16;
 449         desc->s = segp->s;
 450         desc->type = segp->type;
 451         desc->dpl = segp->dpl;
 452         desc->p = segp->present;
 453         desc->limit1 = segp->limit >> 16;
 454         desc->l = segp->l;
 455         desc->db = segp->db;
 456         desc->g = segp->g;
 457         desc->base2 = segp->base >> 24;
 458         if (!segp->s)
 459                 desc->base3 = segp->base >> 32;
 460 }
 461
 462
 463 /* Set Long Mode Flat Kernel Code Segment
 464  *
 465  * Input Args:
 466  *   vm - VM whose GDT is being filled, or NULL to only write segp
 467  *   selector - selector value
 468  *
 469  * Output Args:
 470  *   segp - Pointer to KVM segment
 471  *
 472  * Return: None
 473  *
 474  * Sets up the KVM segment pointed to by segp, to be a code segment
 475  * with the selector value given by selector.
 476  */
 477 static void kvm_seg_set_kernel_code_64bit(struct kvm_vm *vm, uint16_t selector,
 478         struct kvm_segment *segp)
 479 {
 480         memset(segp, 0, sizeof(*segp));
 481         segp->selector = selector;
 482         segp->limit = 0xFFFFFFFFu;
 483         segp->s = 0x1; /* kTypeCodeData */
 484         segp->type = 0x08 | 0x01 | 0x02; /* kFlagCode | kFlagCodeAccessed
 485                                           * | kFlagCodeReadable
 486                                           */
 487         segp->g = true;
 488         segp->l = true;
 489         segp->present = 1;
 490         if (vm)
 491                 kvm_seg_fill_gdt_64bit(vm, segp);
 492 }
 493
 494 /* Set Long Mode Flat Kernel Data Segment
 495  *
 496  * Input Args:
 497  *   vm - VM whose GDT is being filled, or NULL to only write segp
 498  *   selector - selector value
 499  *
 500  * Output Args:
 501  *   segp - Pointer to KVM segment
 502  *
 503  * Return: None
 504  *
 505  * Sets up the KVM segment pointed to by segp, to be a data segment
 506  * with the selector value given by selector.
 507  */
 508 static void kvm_seg_set_kernel_data_64bit(struct kvm_vm *vm, uint16_t selector,
 509         struct kvm_segment *segp)
 510 {
 511         memset(segp, 0, sizeof(*segp));
 512         segp->selector = selector;
 513         segp->limit = 0xFFFFFFFFu;
 514         segp->s = 0x1; /* kTypeCodeData */
 515         segp->type = 0x00 | 0x01 | 0x02; /* kFlagData | kFlagDataAccessed
 516                                           * | kFlagDataWritable
 517                                           */
 518         segp->g = true;
 519         segp->present = true;
 520         if (vm)
 521                 kvm_seg_fill_gdt_64bit(vm, segp);
 522 }
 523
 524 /* Address Guest Virtual to Guest Physical
 525  *
 526  * Input Args:
 527  *   vm - Virtual Machine
 528  *   gpa - VM virtual address
 529  *
 530  * Output Args: None
 531  *
 532  * Return:
 533  *   Equivalent VM physical address
 534  *
 535  * Translates the VM virtual address given by gva to a VM physical
 536  * address and then locates the memory region containing the VM
 537  * physical address, within the VM given by vm.  When found, the host
 538  * virtual address providing the memory to the vm physical address is returned.
 539  * A TEST_ASSERT failure occurs if no region containing translated
 540  * VM virtual address exists.
 541  */
 542 vm_paddr_t addr_gva2gpa(struct kvm_vm *vm, vm_vaddr_t gva)
 543 {
 544         uint16_t index[4];
 545         struct pageMapL4Entry *pml4e;
 546         struct pageDirectoryPointerEntry *pdpe;
 547         struct pageDirectoryEntry *pde;
 548         struct pageTableEntry *pte;
 549
 550         TEST_ASSERT(vm->mode == VM_MODE_P52V48_4K, "Attempt to use "
 551                 "unknown or unsupported guest mode, mode: 0x%x", vm->mode);
 552
 553         index[0] = (gva >> 12) & 0x1ffu;
 554         index[1] = (gva >> 21) & 0x1ffu;
 555         index[2] = (gva >> 30) & 0x1ffu;
 556         index[3] = (gva >> 39) & 0x1ffu;
 557
 558         if (!vm->pgd_created)
 559                 goto unmapped_gva;
 560         pml4e = addr_gpa2hva(vm, vm->pgd);
 561         if (!pml4e[index[3]].present)
 562                 goto unmapped_gva;
 563
 564         pdpe = addr_gpa2hva(vm, pml4e[index[3]].address * vm->page_size);
 565         if (!pdpe[index[2]].present)
 566                 goto unmapped_gva;
 567
 568         pde = addr_gpa2hva(vm, pdpe[index[2]].address * vm->page_size);
 569         if (!pde[index[1]].present)
 570                 goto unmapped_gva;
 571
 572         pte = addr_gpa2hva(vm, pde[index[1]].address * vm->page_size);
 573         if (!pte[index[0]].present)
 574                 goto unmapped_gva;
 575
 576         return (pte[index[0]].address * vm->page_size) + (gva & 0xfffu);
 577
 578 unmapped_gva:
 579         TEST_ASSERT(false, "No mapping for vm virtual address, "
 580                     "gva: 0x%lx", gva);
 581         exit(EXIT_FAILURE);
 582 }
 583
 584 static void kvm_setup_gdt(struct kvm_vm *vm, struct kvm_dtable *dt, int gdt_memslot,
 585                           int pgd_memslot)
 586 {
 587         if (!vm->gdt)
 588                 vm->gdt = vm_vaddr_alloc(vm, getpagesize(),
 589                         KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot);
 590
 591         dt->base = vm->gdt;
 592         dt->limit = getpagesize();
 593 }
 594
 595 static void kvm_setup_tss_64bit(struct kvm_vm *vm, struct kvm_segment *segp,
 596                                 int selector, int gdt_memslot,
 597                                 int pgd_memslot)
 598 {
 599         if (!vm->tss)
 600                 vm->tss = vm_vaddr_alloc(vm, getpagesize(),
 601                         KVM_UTIL_MIN_VADDR, gdt_memslot, pgd_memslot);
 602
 603         memset(segp, 0, sizeof(*segp));
 604         segp->base = vm->tss;
 605         segp->limit = 0x67;
 606         segp->selector = selector;
 607         segp->type = 0xb;
 608         segp->present = 1;
 609         kvm_seg_fill_gdt_64bit(vm, segp);
 610 }
 611
 612 static void vcpu_setup(struct kvm_vm *vm, int vcpuid, int pgd_memslot, int gdt_memslot)
 613 {
 614         struct kvm_sregs sregs;
 615
 616         /* Set mode specific system register values. */
 617         vcpu_sregs_get(vm, vcpuid, &sregs);
 618
 619         sregs.idt.limit = 0;
 620
 621         kvm_setup_gdt(vm, &sregs.gdt, gdt_memslot, pgd_memslot);
 622
 623         switch (vm->mode) {
 624         case VM_MODE_P52V48_4K:
 625                 sregs.cr0 = X86_CR0_PE | X86_CR0_NE | X86_CR0_PG;
 626                 sregs.cr4 |= X86_CR4_PAE | X86_CR4_OSFXSR;
 627                 sregs.efer |= (EFER_LME | EFER_LMA | EFER_NX);
 628
 629                 kvm_seg_set_unusable(&sregs.ldt);
 630                 kvm_seg_set_kernel_code_64bit(vm, 0x8, &sregs.cs);
 631                 kvm_seg_set_kernel_data_64bit(vm, 0x10, &sregs.ds);
 632                 kvm_seg_set_kernel_data_64bit(vm, 0x10, &sregs.es);
 633                 kvm_setup_tss_64bit(vm, &sregs.tr, 0x18, gdt_memslot, pgd_memslot);
 634                 break;
 635
 636         default:
 637                 TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", vm->mode);
 638         }
 639
 640         sregs.cr3 = vm->pgd;
 641         vcpu_sregs_set(vm, vcpuid, &sregs);
 642 }
 643 /* Adds a vCPU with reasonable defaults (i.e., a stack)
 644  *
 645  * Input Args:
 646  *   vcpuid - The id of the VCPU to add to the VM.
 647  *   guest_code - The vCPU's entry point
 648  */
 649 void vm_vcpu_add_default(struct kvm_vm *vm, uint32_t vcpuid, void *guest_code)
 650 {
 651         struct kvm_mp_state mp_state;
 652         struct kvm_regs regs;
 653         vm_vaddr_t stack_vaddr;
 654         stack_vaddr = vm_vaddr_alloc(vm, DEFAULT_STACK_PGS * getpagesize(),
 655                                      DEFAULT_GUEST_STACK_VADDR_MIN, 0, 0);
 656
 657         /* Create VCPU */
 658         vm_vcpu_add(vm, vcpuid);
 659         vcpu_setup(vm, vcpuid, 0, 0);
 660
 661         /* Setup guest general purpose registers */
 662         vcpu_regs_get(vm, vcpuid, &regs);
 663         regs.rflags = regs.rflags | 0x2;
 664         regs.rsp = stack_vaddr + (DEFAULT_STACK_PGS * getpagesize());
 665         regs.rip = (unsigned long) guest_code;
 666         vcpu_regs_set(vm, vcpuid, &regs);
 667
 668         /* Setup the MP state */
 669         mp_state.mp_state = 0;
 670         vcpu_set_mp_state(vm, vcpuid, &mp_state);
 671 }
 672
 673 /* Allocate an instance of struct kvm_cpuid2
 674  *
 675  * Input Args: None
 676  *
 677  * Output Args: None
 678  *
 679  * Return: A pointer to the allocated struct. The caller is responsible
 680  * for freeing this struct.
 681  *
 682  * Since kvm_cpuid2 uses a 0-length array to allow a the size of the
 683  * array to be decided at allocation time, allocation is slightly
 684  * complicated. This function uses a reasonable default length for
 685  * the array and performs the appropriate allocation.
 686  */
 687 static struct kvm_cpuid2 *allocate_kvm_cpuid2(void)
 688 {
 689         struct kvm_cpuid2 *cpuid;
 690         int nent = 100;
 691         size_t size;
 692
 693         size = sizeof(*cpuid);
 694         size += nent * sizeof(struct kvm_cpuid_entry2);
 695         cpuid = malloc(size);
 696         if (!cpuid) {
 697                 perror("malloc");
 698                 abort();
 699         }
 700
 701         cpuid->nent = nent;
 702
 703         return cpuid;
 704 }
 705
 706 /* KVM Supported CPUID Get
 707  *
 708  * Input Args: None
 709  *
 710  * Output Args:
 711  *
 712  * Return: The supported KVM CPUID
 713  *
 714  * Get the guest CPUID supported by KVM.
 715  */
 716 struct kvm_cpuid2 *kvm_get_supported_cpuid(void)
 717 {
 718         static struct kvm_cpuid2 *cpuid;
 719         int ret;
 720         int kvm_fd;
 721
 722         if (cpuid)
 723                 return cpuid;
 724
 725         cpuid = allocate_kvm_cpuid2();
 726         kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
 727         if (kvm_fd < 0)
 728                 exit(KSFT_SKIP);
 729
 730         ret = ioctl(kvm_fd, KVM_GET_SUPPORTED_CPUID, cpuid);
 731         TEST_ASSERT(ret == 0, "KVM_GET_SUPPORTED_CPUID failed %d %d\n",
 732                     ret, errno);
 733
 734         close(kvm_fd);
 735         return cpuid;
 736 }
 737
 738 /* Locate a cpuid entry.
 739  *
 740  * Input Args:
 741  *   cpuid: The cpuid.
 742  *   function: The function of the cpuid entry to find.
 743  *
 744  * Output Args: None
 745  *
 746  * Return: A pointer to the cpuid entry. Never returns NULL.
 747  */
 748 struct kvm_cpuid_entry2 *
 749 kvm_get_supported_cpuid_index(uint32_t function, uint32_t index)
 750 {
 751         struct kvm_cpuid2 *cpuid;
 752         struct kvm_cpuid_entry2 *entry = NULL;
 753         int i;
 754
 755         cpuid = kvm_get_supported_cpuid();
 756         for (i = 0; i < cpuid->nent; i++) {
 757                 if (cpuid->entries[i].function == function &&
 758                     cpuid->entries[i].index == index) {
 759                         entry = &cpuid->entries[i];
 760                         break;
 761                 }
 762         }
 763
 764         TEST_ASSERT(entry, "Guest CPUID entry not found: (EAX=%x, ECX=%x).",
 765                     function, index);
 766         return entry;
 767 }
 768
 769 /* VM VCPU CPUID Set
 770  *
 771  * Input Args:
 772  *   vm - Virtual Machine
 773  *   vcpuid - VCPU id
 774  *   cpuid - The CPUID values to set.
 775  *
 776  * Output Args: None
 777  *
 778  * Return: void
 779  *
 780  * Set the VCPU's CPUID.
 781  */
 782 void vcpu_set_cpuid(struct kvm_vm *vm,
 783                 uint32_t vcpuid, struct kvm_cpuid2 *cpuid)
 784 {
 785         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
 786         int rc;
 787
 788         TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
 789
 790         rc = ioctl(vcpu->fd, KVM_SET_CPUID2, cpuid);
 791         TEST_ASSERT(rc == 0, "KVM_SET_CPUID2 failed, rc: %i errno: %i",
 792                     rc, errno);
 793
 794 }
 795
 796 /* Create a VM with reasonable defaults
 797  *
 798  * Input Args:
 799  *   vcpuid - The id of the single VCPU to add to the VM.
 800  *   extra_mem_pages - The size of extra memories to add (this will
 801  *                     decide how much extra space we will need to
 802  *                     setup the page tables using mem slot 0)
 803  *   guest_code - The vCPU's entry point
 804  *
 805  * Output Args: None
 806  *
 807  * Return:
 808  *   Pointer to opaque structure that describes the created VM.
 809  */
 810 struct kvm_vm *vm_create_default(uint32_t vcpuid, uint64_t extra_mem_pages,
 811                                  void *guest_code)
 812 {
 813         struct kvm_vm *vm;
 814         /*
 815          * For x86 the maximum page table size for a memory region
 816          * will be when only 4K pages are used.  In that case the
 817          * total extra size for page tables (for extra N pages) will
 818          * be: N/512+N/512^2+N/512^3+... which is definitely smaller
 819          * than N/512*2.
 820          */
 821         uint64_t extra_pg_pages = extra_mem_pages / 512 * 2;
 822
 823         /* Create VM */
 824         vm = vm_create(VM_MODE_DEFAULT,
 825                        DEFAULT_GUEST_PHY_PAGES + extra_pg_pages,
 826                        O_RDWR);
 827
 828         /* Setup guest code */
 829         kvm_vm_elf_load(vm, program_invocation_name, 0, 0);
 830
 831         /* Setup IRQ Chip */
 832         vm_create_irqchip(vm);
 833
 834         /* Add the first vCPU. */
 835         vm_vcpu_add_default(vm, vcpuid, guest_code);
 836
 837         return vm;
 838 }
 839
 840 /* VCPU Get MSR
 841  *
 842  * Input Args:
 843  *   vm - Virtual Machine
 844  *   vcpuid - VCPU ID
 845  *   msr_index - Index of MSR
 846  *
 847  * Output Args: None
 848  *
 849  * Return: On success, value of the MSR. On failure a TEST_ASSERT is produced.
 850  *
 851  * Get value of MSR for VCPU.
 852  */
 853 uint64_t vcpu_get_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index)
 854 {
 855         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
 856         struct {
 857                 struct kvm_msrs header;
 858                 struct kvm_msr_entry entry;
 859         } buffer = {};
 860         int r;
 861
 862         TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
 863         buffer.header.nmsrs = 1;
 864         buffer.entry.index = msr_index;
 865         r = ioctl(vcpu->fd, KVM_GET_MSRS, &buffer.header);
 866         TEST_ASSERT(r == 1, "KVM_GET_MSRS IOCTL failed,\n"
 867                 "  rc: %i errno: %i", r, errno);
 868
 869         return buffer.entry.data;
 870 }
 871
 872 /* VCPU Set MSR
 873  *
 874  * Input Args:
 875  *   vm - Virtual Machine
 876  *   vcpuid - VCPU ID
 877  *   msr_index - Index of MSR
 878  *   msr_value - New value of MSR
 879  *
 880  * Output Args: None
 881  *
 882  * Return: On success, nothing. On failure a TEST_ASSERT is produced.
 883  *
 884  * Set value of MSR for VCPU.
 885  */
 886 void vcpu_set_msr(struct kvm_vm *vm, uint32_t vcpuid, uint64_t msr_index,
 887         uint64_t msr_value)
 888 {
 889         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
 890         struct {
 891                 struct kvm_msrs header;
 892                 struct kvm_msr_entry entry;
 893         } buffer = {};
 894         int r;
 895
 896         TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
 897         memset(&buffer, 0, sizeof(buffer));
 898         buffer.header.nmsrs = 1;
 899         buffer.entry.index = msr_index;
 900         buffer.entry.data = msr_value;
 901         r = ioctl(vcpu->fd, KVM_SET_MSRS, &buffer.header);
 902         TEST_ASSERT(r == 1, "KVM_SET_MSRS IOCTL failed,\n"
 903                 "  rc: %i errno: %i", r, errno);
 904 }
 905
 906 /* VM VCPU Args Set
 907  *
 908  * Input Args:
 909  *   vm - Virtual Machine
 910  *   vcpuid - VCPU ID
 911  *   num - number of arguments
 912  *   ... - arguments, each of type uint64_t
 913  *
 914  * Output Args: None
 915  *
 916  * Return: None
 917  *
 918  * Sets the first num function input arguments to the values
 919  * given as variable args.  Each of the variable args is expected to
 920  * be of type uint64_t.
 921  */
 922 void vcpu_args_set(struct kvm_vm *vm, uint32_t vcpuid, unsigned int num, ...)
 923 {
 924         va_list ap;
 925         struct kvm_regs regs;
 926
 927         TEST_ASSERT(num >= 1 && num <= 6, "Unsupported number of args,\n"
 928                     "  num: %u\n",
 929                     num);
 930
 931         va_start(ap, num);
 932         vcpu_regs_get(vm, vcpuid, &regs);
 933
 934         if (num >= 1)
 935                 regs.rdi = va_arg(ap, uint64_t);
 936
 937         if (num >= 2)
 938                 regs.rsi = va_arg(ap, uint64_t);
 939
 940         if (num >= 3)
 941                 regs.rdx = va_arg(ap, uint64_t);
 942
 943         if (num >= 4)
 944                 regs.rcx = va_arg(ap, uint64_t);
 945
 946         if (num >= 5)
 947                 regs.r8 = va_arg(ap, uint64_t);
 948
 949         if (num >= 6)
 950                 regs.r9 = va_arg(ap, uint64_t);
 951
 952         vcpu_regs_set(vm, vcpuid, &regs);
 953         va_end(ap);
 954 }
 955
 956 /*
 957  * VM VCPU Dump
 958  *
 959  * Input Args:
 960  *   vm - Virtual Machine
 961  *   vcpuid - VCPU ID
 962  *   indent - Left margin indent amount
 963  *
 964  * Output Args:
 965  *   stream - Output FILE stream
 966  *
 967  * Return: None
 968  *
 969  * Dumps the current state of the VCPU specified by vcpuid, within the VM
 970  * given by vm, to the FILE stream given by stream.
 971  */
 972 void vcpu_dump(FILE *stream, struct kvm_vm *vm, uint32_t vcpuid, uint8_t indent)
 973 {
 974         struct kvm_regs regs;
 975         struct kvm_sregs sregs;
 976
 977         fprintf(stream, "%*scpuid: %u\n", indent, "", vcpuid);
 978
 979         fprintf(stream, "%*sregs:\n", indent + 2, "");
 980         vcpu_regs_get(vm, vcpuid, &regs);
 981         regs_dump(stream, &regs, indent + 4);
 982
 983         fprintf(stream, "%*ssregs:\n", indent + 2, "");
 984         vcpu_sregs_get(vm, vcpuid, &sregs);
 985         sregs_dump(stream, &sregs, indent + 4);
 986 }
 987
 988 struct kvm_x86_state {
 989         struct kvm_vcpu_events events;
 990         struct kvm_mp_state mp_state;
 991         struct kvm_regs regs;
 992         struct kvm_xsave xsave;
 993         struct kvm_xcrs xcrs;
 994         struct kvm_sregs sregs;
 995         struct kvm_debugregs debugregs;
 996         union {
 997                 struct kvm_nested_state nested;
 998                 char nested_[16384];
 999         };
1000         struct kvm_msrs msrs;
1001 };
1002
1003 static int kvm_get_num_msrs(struct kvm_vm *vm)
1004 {
1005         struct kvm_msr_list nmsrs;
1006         int r;
1007
1008         nmsrs.nmsrs = 0;
1009         r = ioctl(vm->kvm_fd, KVM_GET_MSR_INDEX_LIST, &nmsrs);
1010         TEST_ASSERT(r == -1 && errno == E2BIG, "Unexpected result from KVM_GET_MSR_INDEX_LIST probe, r: %i",
1011                 r);
1012
1013         return nmsrs.nmsrs;
1014 }
1015
1016 struct kvm_x86_state *vcpu_save_state(struct kvm_vm *vm, uint32_t vcpuid)
1017 {
1018         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1019         struct kvm_msr_list *list;
1020         struct kvm_x86_state *state;
1021         int nmsrs, r, i;
1022         static int nested_size = -1;
1023
1024         if (nested_size == -1) {
1025                 nested_size = kvm_check_cap(KVM_CAP_NESTED_STATE);
1026                 TEST_ASSERT(nested_size <= sizeof(state->nested_),
1027                             "Nested state size too big, %i > %zi",
1028                             nested_size, sizeof(state->nested_));
1029         }
1030
1031         /*
1032          * When KVM exits to userspace with KVM_EXIT_IO, KVM guarantees
1033          * guest state is consistent only after userspace re-enters the
1034          * kernel with KVM_RUN.  Complete IO prior to migrating state
1035          * to a new VM.
1036          */
1037         vcpu_run_complete_io(vm, vcpuid);
1038
1039         nmsrs = kvm_get_num_msrs(vm);
1040         list = malloc(sizeof(*list) + nmsrs * sizeof(list->indices[0]));
1041         list->nmsrs = nmsrs;
1042         r = ioctl(vm->kvm_fd, KVM_GET_MSR_INDEX_LIST, list);
1043         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MSR_INDEX_LIST, r: %i",
1044                 r);
1045
1046         state = malloc(sizeof(*state) + nmsrs * sizeof(state->msrs.entries[0]));
1047         r = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, &state->events);
1048         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_VCPU_EVENTS, r: %i",
1049                 r);
1050
1051         r = ioctl(vcpu->fd, KVM_GET_MP_STATE, &state->mp_state);
1052         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_MP_STATE, r: %i",
1053                 r);
1054
1055         r = ioctl(vcpu->fd, KVM_GET_REGS, &state->regs);
1056         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_REGS, r: %i",
1057                 r);
1058
1059         r = ioctl(vcpu->fd, KVM_GET_XSAVE, &state->xsave);
1060         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XSAVE, r: %i",
1061                 r);
1062
1063         if (kvm_check_cap(KVM_CAP_XCRS)) {
1064                 r = ioctl(vcpu->fd, KVM_GET_XCRS, &state->xcrs);
1065                 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_XCRS, r: %i",
1066                             r);
1067         }
1068
1069         r = ioctl(vcpu->fd, KVM_GET_SREGS, &state->sregs);
1070         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_SREGS, r: %i",
1071                 r);
1072
1073         if (nested_size) {
1074                 state->nested.size = sizeof(state->nested_);
1075                 r = ioctl(vcpu->fd, KVM_GET_NESTED_STATE, &state->nested);
1076                 TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_NESTED_STATE, r: %i",
1077                         r);
1078                 TEST_ASSERT(state->nested.size <= nested_size,
1079                         "Nested state size too big, %i (KVM_CHECK_CAP gave %i)",
1080                         state->nested.size, nested_size);
1081         } else
1082                 state->nested.size = 0;
1083
1084         state->msrs.nmsrs = nmsrs;
1085         for (i = 0; i < nmsrs; i++)
1086                 state->msrs.entries[i].index = list->indices[i];
1087         r = ioctl(vcpu->fd, KVM_GET_MSRS, &state->msrs);
1088         TEST_ASSERT(r == nmsrs, "Unexpected result from KVM_GET_MSRS, r: %i (failed at %x)",
1089                 r, r == nmsrs ? -1 : list->indices[r]);
1090
1091         r = ioctl(vcpu->fd, KVM_GET_DEBUGREGS, &state->debugregs);
1092         TEST_ASSERT(r == 0, "Unexpected result from KVM_GET_DEBUGREGS, r: %i",
1093                 r);
1094
1095         free(list);
1096         return state;
1097 }
1098
1099 void vcpu_load_state(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_x86_state *state)
1100 {
1101         struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1102         int r;
1103
1104         r = ioctl(vcpu->fd, KVM_SET_XSAVE, &state->xsave);
1105         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XSAVE, r: %i",
1106                 r);
1107
1108         if (kvm_check_cap(KVM_CAP_XCRS)) {
1109                 r = ioctl(vcpu->fd, KVM_SET_XCRS, &state->xcrs);
1110                 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_XCRS, r: %i",
1111                             r);
1112         }
1113
1114         r = ioctl(vcpu->fd, KVM_SET_SREGS, &state->sregs);
1115         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_SREGS, r: %i",
1116                 r);
1117
1118         r = ioctl(vcpu->fd, KVM_SET_MSRS, &state->msrs);
1119         TEST_ASSERT(r == state->msrs.nmsrs, "Unexpected result from KVM_SET_MSRS, r: %i (failed at %x)",
1120                 r, r == state->msrs.nmsrs ? -1 : state->msrs.entries[r].index);
1121
1122         r = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, &state->events);
1123         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_VCPU_EVENTS, r: %i",
1124                 r);
1125
1126         r = ioctl(vcpu->fd, KVM_SET_MP_STATE, &state->mp_state);
1127         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_MP_STATE, r: %i",
1128                 r);
1129
1130         r = ioctl(vcpu->fd, KVM_SET_DEBUGREGS, &state->debugregs);
1131         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_DEBUGREGS, r: %i",
1132                 r);
1133
1134         r = ioctl(vcpu->fd, KVM_SET_REGS, &state->regs);
1135         TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_REGS, r: %i",
1136                 r);
1137
1138         if (state->nested.size) {
1139                 r = ioctl(vcpu->fd, KVM_SET_NESTED_STATE, &state->nested);
1140                 TEST_ASSERT(r == 0, "Unexpected result from KVM_SET_NESTED_STATE, r: %i",
1141                         r);
1142         }
1143 }
1144
1145 bool is_intel_cpu(void)
1146 {
1147         int eax, ebx, ecx, edx;
1148         const uint32_t *chunk;
1149         const int leaf = 0;
1150
1151         __asm__ __volatile__(
1152                 "cpuid"
1153                 : /* output */ "=a"(eax), "=b"(ebx),
1154                   "=c"(ecx), "=d"(edx)
1155                 : /* input */ "0"(leaf), "2"(0));
1156
1157         chunk = (const uint32_t *)("GenuineIntel");
1158         return (ebx == chunk[0] && edx == chunk[1] && ecx == chunk[2]);
1159 }