2 * tools/testing/selftests/kvm/lib/kvm_util.c
4 * Copyright (C) 2018, Google LLC.
6 * This work is licensed under the terms of the GNU GPL, version 2.
11 #include "kvm_util_internal.h"
15 #include <sys/types.h>
17 #include <linux/kernel.h>
19 #define KVM_UTIL_PGS_PER_HUGEPG 512
20 #define KVM_UTIL_MIN_PFN 2
22 /* Aligns x up to the next multiple of size. Size must be a power of 2. */
23 static void *align(void *x, size_t size)
25 size_t mask = size - 1;
26 TEST_ASSERT(size != 0 && !(size & (size - 1)),
27 "size not a power of 2: %lu", size);
28 return (void *) (((size_t) x + mask) & ~mask);
40 * On success, the Value corresponding to the capability (KVM_CAP_*)
41 * specified by the value of cap. On failure a TEST_ASSERT failure
44 * Looks up and returns the value corresponding to the capability
45 * (KVM_CAP_*) given by cap.
47 int kvm_check_cap(long cap)
52 kvm_fd = open(KVM_DEV_PATH, O_RDONLY);
56 ret = ioctl(kvm_fd, KVM_CHECK_EXTENSION, cap);
57 TEST_ASSERT(ret != -1, "KVM_CHECK_EXTENSION IOCTL failed,\n"
58 " rc: %i errno: %i", ret, errno);
65 /* VM Enable Capability
68 * vm - Virtual Machine
73 * Return: On success, 0. On failure a TEST_ASSERT failure is produced.
75 * Enables a capability (KVM_CAP_*) on the VM.
77 int vm_enable_cap(struct kvm_vm *vm, struct kvm_enable_cap *cap)
81 ret = ioctl(vm->fd, KVM_ENABLE_CAP, cap);
82 TEST_ASSERT(ret == 0, "KVM_ENABLE_CAP IOCTL failed,\n"
83 " rc: %i errno: %i", ret, errno);
88 static void vm_open(struct kvm_vm *vm, int perm, unsigned long type)
90 vm->kvm_fd = open(KVM_DEV_PATH, perm);
94 vm->fd = ioctl(vm->kvm_fd, KVM_CREATE_VM, type);
95 TEST_ASSERT(vm->fd >= 0, "KVM_CREATE_VM ioctl failed, "
96 "rc: %i errno: %i", vm->fd, errno);
99 const char * const vm_guest_mode_string[] = {
100 "PA-bits:52, VA-bits:48, 4K pages",
101 "PA-bits:52, VA-bits:48, 64K pages",
102 "PA-bits:48, VA-bits:48, 4K pages",
103 "PA-bits:48, VA-bits:48, 64K pages",
104 "PA-bits:40, VA-bits:48, 4K pages",
105 "PA-bits:40, VA-bits:48, 64K pages",
107 _Static_assert(sizeof(vm_guest_mode_string)/sizeof(char *) == NUM_VM_MODES,
108 "Missing new mode strings?");
114 * mode - VM Mode (e.g. VM_MODE_P52V48_4K)
115 * phy_pages - Physical memory pages
121 * Pointer to opaque structure that describes the created VM.
123 * Creates a VM with the mode specified by mode (e.g. VM_MODE_P52V48_4K).
124 * When phy_pages is non-zero, a memory region of phy_pages physical pages
125 * is created and mapped starting at guest physical address 0. The file
126 * descriptor to control the created VM is created with the permissions
127 * given by perm (e.g. O_RDWR).
129 struct kvm_vm *_vm_create(enum vm_guest_mode mode, uint64_t phy_pages,
130 int perm, unsigned long type)
135 vm = calloc(1, sizeof(*vm));
136 TEST_ASSERT(vm != NULL, "Insufficient Memory");
140 vm_open(vm, perm, type);
142 /* Setup mode specific traits. */
144 case VM_MODE_P52V48_4K:
145 vm->pgtable_levels = 4;
148 vm->page_size = 0x1000;
151 case VM_MODE_P52V48_64K:
152 vm->pgtable_levels = 3;
155 vm->page_size = 0x10000;
158 case VM_MODE_P48V48_4K:
159 vm->pgtable_levels = 4;
162 vm->page_size = 0x1000;
165 case VM_MODE_P48V48_64K:
166 vm->pgtable_levels = 3;
169 vm->page_size = 0x10000;
172 case VM_MODE_P40V48_4K:
173 vm->pgtable_levels = 4;
176 vm->page_size = 0x1000;
179 case VM_MODE_P40V48_64K:
180 vm->pgtable_levels = 3;
183 vm->page_size = 0x10000;
187 TEST_ASSERT(false, "Unknown guest mode, mode: 0x%x", mode);
190 /* Limit to VA-bit canonical virtual addresses. */
191 vm->vpages_valid = sparsebit_alloc();
192 sparsebit_set_num(vm->vpages_valid,
193 0, (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
194 sparsebit_set_num(vm->vpages_valid,
195 (~((1ULL << (vm->va_bits - 1)) - 1)) >> vm->page_shift,
196 (1ULL << (vm->va_bits - 1)) >> vm->page_shift);
198 /* Limit physical addresses to PA-bits. */
199 vm->max_gfn = ((1ULL << vm->pa_bits) >> vm->page_shift) - 1;
201 /* Allocate and setup memory for guest. */
202 vm->vpages_mapped = sparsebit_alloc();
204 vm_userspace_mem_region_add(vm, VM_MEM_SRC_ANONYMOUS,
210 struct kvm_vm *vm_create(enum vm_guest_mode mode, uint64_t phy_pages, int perm)
212 return _vm_create(mode, phy_pages, perm, 0);
219 * vm - VM that has been released before
224 * Reopens the file descriptors associated to the VM and reinstates the
225 * global state, such as the irqchip and the memory regions that are mapped
228 void kvm_vm_restart(struct kvm_vm *vmp, int perm)
230 struct userspace_mem_region *region;
232 vm_open(vmp, perm, vmp->type);
233 if (vmp->has_irqchip)
234 vm_create_irqchip(vmp);
236 for (region = vmp->userspace_mem_region_head; region;
237 region = region->next) {
238 int ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
239 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
240 " rc: %i errno: %i\n"
241 " slot: %u flags: 0x%x\n"
242 " guest_phys_addr: 0x%lx size: 0x%lx",
243 ret, errno, region->region.slot,
244 region->region.flags,
245 region->region.guest_phys_addr,
246 region->region.memory_size);
250 void kvm_vm_get_dirty_log(struct kvm_vm *vm, int slot, void *log)
252 struct kvm_dirty_log args = { .dirty_bitmap = log, .slot = slot };
255 ret = ioctl(vm->fd, KVM_GET_DIRTY_LOG, &args);
256 TEST_ASSERT(ret == 0, "%s: KVM_GET_DIRTY_LOG failed: %s",
260 void kvm_vm_clear_dirty_log(struct kvm_vm *vm, int slot, void *log,
261 uint64_t first_page, uint32_t num_pages)
263 struct kvm_clear_dirty_log args = { .dirty_bitmap = log, .slot = slot,
264 .first_page = first_page,
265 .num_pages = num_pages };
268 ret = ioctl(vm->fd, KVM_CLEAR_DIRTY_LOG, &args);
269 TEST_ASSERT(ret == 0, "%s: KVM_CLEAR_DIRTY_LOG failed: %s",
274 * Userspace Memory Region Find
277 * vm - Virtual Machine
278 * start - Starting VM physical address
279 * end - Ending VM physical address, inclusive.
284 * Pointer to overlapping region, NULL if no such region.
286 * Searches for a region with any physical memory that overlaps with
287 * any portion of the guest physical addresses from start to end
288 * inclusive. If multiple overlapping regions exist, a pointer to any
289 * of the regions is returned. Null is returned only when no overlapping
292 static struct userspace_mem_region *
293 userspace_mem_region_find(struct kvm_vm *vm, uint64_t start, uint64_t end)
295 struct userspace_mem_region *region;
297 for (region = vm->userspace_mem_region_head; region;
298 region = region->next) {
299 uint64_t existing_start = region->region.guest_phys_addr;
300 uint64_t existing_end = region->region.guest_phys_addr
301 + region->region.memory_size - 1;
302 if (start <= existing_end && end >= existing_start)
310 * KVM Userspace Memory Region Find
313 * vm - Virtual Machine
314 * start - Starting VM physical address
315 * end - Ending VM physical address, inclusive.
320 * Pointer to overlapping region, NULL if no such region.
322 * Public interface to userspace_mem_region_find. Allows tests to look up
323 * the memslot datastructure for a given range of guest physical memory.
325 struct kvm_userspace_memory_region *
326 kvm_userspace_memory_region_find(struct kvm_vm *vm, uint64_t start,
329 struct userspace_mem_region *region;
331 region = userspace_mem_region_find(vm, start, end);
335 return ®ion->region;
342 * vm - Virtual Machine
348 * Pointer to VCPU structure
350 * Locates a vcpu structure that describes the VCPU specified by vcpuid and
351 * returns a pointer to it. Returns NULL if the VM doesn't contain a VCPU
352 * for the specified vcpuid.
354 struct vcpu *vcpu_find(struct kvm_vm *vm, uint32_t vcpuid)
358 for (vcpup = vm->vcpu_head; vcpup; vcpup = vcpup->next) {
359 if (vcpup->id == vcpuid)
370 * vm - Virtual Machine
375 * Return: None, TEST_ASSERT failures for all error conditions
377 * Within the VM specified by vm, removes the VCPU given by vcpuid.
379 static void vm_vcpu_rm(struct kvm_vm *vm, uint32_t vcpuid)
381 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
384 ret = munmap(vcpu->state, sizeof(*vcpu->state));
385 TEST_ASSERT(ret == 0, "munmap of VCPU fd failed, rc: %i "
386 "errno: %i", ret, errno);
388 TEST_ASSERT(ret == 0, "Close of VCPU fd failed, rc: %i "
389 "errno: %i", ret, errno);
392 vcpu->next->prev = vcpu->prev;
394 vcpu->prev->next = vcpu->next;
396 vm->vcpu_head = vcpu->next;
400 void kvm_vm_release(struct kvm_vm *vmp)
404 while (vmp->vcpu_head)
405 vm_vcpu_rm(vmp, vmp->vcpu_head->id);
407 ret = close(vmp->fd);
408 TEST_ASSERT(ret == 0, "Close of vm fd failed,\n"
409 " vmp->fd: %i rc: %i errno: %i", vmp->fd, ret, errno);
412 TEST_ASSERT(ret == 0, "Close of /dev/kvm fd failed,\n"
413 " vmp->kvm_fd: %i rc: %i errno: %i", vmp->kvm_fd, ret, errno);
417 * Destroys and frees the VM pointed to by vmp.
419 void kvm_vm_free(struct kvm_vm *vmp)
426 /* Free userspace_mem_regions. */
427 while (vmp->userspace_mem_region_head) {
428 struct userspace_mem_region *region
429 = vmp->userspace_mem_region_head;
431 region->region.memory_size = 0;
432 ret = ioctl(vmp->fd, KVM_SET_USER_MEMORY_REGION,
434 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed, "
435 "rc: %i errno: %i", ret, errno);
437 vmp->userspace_mem_region_head = region->next;
438 sparsebit_free(®ion->unused_phy_pages);
439 ret = munmap(region->mmap_start, region->mmap_size);
440 TEST_ASSERT(ret == 0, "munmap failed, rc: %i errno: %i",
446 /* Free sparsebit arrays. */
447 sparsebit_free(&vmp->vpages_valid);
448 sparsebit_free(&vmp->vpages_mapped);
452 /* Free the structure describing the VM. */
457 * Memory Compare, host virtual to guest virtual
460 * hva - Starting host virtual address
461 * vm - Virtual Machine
462 * gva - Starting guest virtual address
463 * len - number of bytes to compare
467 * Input/Output Args: None
470 * Returns 0 if the bytes starting at hva for a length of len
471 * are equal the guest virtual bytes starting at gva. Returns
472 * a value < 0, if bytes at hva are less than those at gva.
473 * Otherwise a value > 0 is returned.
475 * Compares the bytes starting at the host virtual address hva, for
476 * a length of len, to the guest bytes starting at the guest virtual
477 * address given by gva.
479 int kvm_memcmp_hva_gva(void *hva, struct kvm_vm *vm, vm_vaddr_t gva, size_t len)
484 * Compare a batch of bytes until either a match is found
485 * or all the bytes have been compared.
487 for (uintptr_t offset = 0; offset < len; offset += amt) {
488 uintptr_t ptr1 = (uintptr_t)hva + offset;
491 * Determine host address for guest virtual address
494 uintptr_t ptr2 = (uintptr_t)addr_gva2hva(vm, gva + offset);
497 * Determine amount to compare on this pass.
498 * Don't allow the comparsion to cross a page boundary.
501 if ((ptr1 >> vm->page_shift) != ((ptr1 + amt) >> vm->page_shift))
502 amt = vm->page_size - (ptr1 % vm->page_size);
503 if ((ptr2 >> vm->page_shift) != ((ptr2 + amt) >> vm->page_shift))
504 amt = vm->page_size - (ptr2 % vm->page_size);
506 assert((ptr1 >> vm->page_shift) == ((ptr1 + amt - 1) >> vm->page_shift));
507 assert((ptr2 >> vm->page_shift) == ((ptr2 + amt - 1) >> vm->page_shift));
510 * Perform the comparison. If there is a difference
511 * return that result to the caller, otherwise need
512 * to continue on looking for a mismatch.
514 int ret = memcmp((void *)ptr1, (void *)ptr2, amt);
520 * No mismatch found. Let the caller know the two memory
527 * VM Userspace Memory Region Add
530 * vm - Virtual Machine
531 * backing_src - Storage source for this region.
532 * NULL to use anonymous memory.
533 * guest_paddr - Starting guest physical address
534 * slot - KVM region slot
535 * npages - Number of physical pages
536 * flags - KVM memory region flags (e.g. KVM_MEM_LOG_DIRTY_PAGES)
542 * Allocates a memory area of the number of pages specified by npages
543 * and maps it to the VM specified by vm, at a starting physical address
544 * given by guest_paddr. The region is created with a KVM region slot
545 * given by slot, which must be unique and < KVM_MEM_SLOTS_NUM. The
546 * region is created with the flags given by flags.
548 void vm_userspace_mem_region_add(struct kvm_vm *vm,
549 enum vm_mem_backing_src_type src_type,
550 uint64_t guest_paddr, uint32_t slot, uint64_t npages,
554 unsigned long pmem_size = 0;
555 struct userspace_mem_region *region;
556 size_t huge_page_size = KVM_UTIL_PGS_PER_HUGEPG * vm->page_size;
558 TEST_ASSERT((guest_paddr % vm->page_size) == 0, "Guest physical "
559 "address not on a page boundary.\n"
560 " guest_paddr: 0x%lx vm->page_size: 0x%x",
561 guest_paddr, vm->page_size);
562 TEST_ASSERT((((guest_paddr >> vm->page_shift) + npages) - 1)
563 <= vm->max_gfn, "Physical range beyond maximum "
564 "supported physical address,\n"
565 " guest_paddr: 0x%lx npages: 0x%lx\n"
566 " vm->max_gfn: 0x%lx vm->page_size: 0x%x",
567 guest_paddr, npages, vm->max_gfn, vm->page_size);
570 * Confirm a mem region with an overlapping address doesn't
573 region = (struct userspace_mem_region *) userspace_mem_region_find(
574 vm, guest_paddr, (guest_paddr + npages * vm->page_size) - 1);
576 TEST_ASSERT(false, "overlapping userspace_mem_region already "
578 " requested guest_paddr: 0x%lx npages: 0x%lx "
580 " existing guest_paddr: 0x%lx size: 0x%lx",
581 guest_paddr, npages, vm->page_size,
582 (uint64_t) region->region.guest_phys_addr,
583 (uint64_t) region->region.memory_size);
585 /* Confirm no region with the requested slot already exists. */
586 for (region = vm->userspace_mem_region_head; region;
587 region = region->next) {
588 if (region->region.slot == slot)
592 TEST_ASSERT(false, "A mem region with the requested slot "
594 " requested slot: %u paddr: 0x%lx npages: 0x%lx\n"
595 " existing slot: %u paddr: 0x%lx size: 0x%lx",
596 slot, guest_paddr, npages,
598 (uint64_t) region->region.guest_phys_addr,
599 (uint64_t) region->region.memory_size);
601 /* Allocate and initialize new mem region structure. */
602 region = calloc(1, sizeof(*region));
603 TEST_ASSERT(region != NULL, "Insufficient Memory");
604 region->mmap_size = npages * vm->page_size;
606 /* Enough memory to align up to a huge page. */
607 if (src_type == VM_MEM_SRC_ANONYMOUS_THP)
608 region->mmap_size += huge_page_size;
609 region->mmap_start = mmap(NULL, region->mmap_size,
610 PROT_READ | PROT_WRITE,
611 MAP_PRIVATE | MAP_ANONYMOUS
612 | (src_type == VM_MEM_SRC_ANONYMOUS_HUGETLB ? MAP_HUGETLB : 0),
614 TEST_ASSERT(region->mmap_start != MAP_FAILED,
615 "test_malloc failed, mmap_start: %p errno: %i",
616 region->mmap_start, errno);
618 /* Align THP allocation up to start of a huge page. */
619 region->host_mem = align(region->mmap_start,
620 src_type == VM_MEM_SRC_ANONYMOUS_THP ? huge_page_size : 1);
622 /* As needed perform madvise */
623 if (src_type == VM_MEM_SRC_ANONYMOUS || src_type == VM_MEM_SRC_ANONYMOUS_THP) {
624 ret = madvise(region->host_mem, npages * vm->page_size,
625 src_type == VM_MEM_SRC_ANONYMOUS ? MADV_NOHUGEPAGE : MADV_HUGEPAGE);
626 TEST_ASSERT(ret == 0, "madvise failed,\n"
630 region->host_mem, npages * vm->page_size, src_type);
633 region->unused_phy_pages = sparsebit_alloc();
634 sparsebit_set_num(region->unused_phy_pages,
635 guest_paddr >> vm->page_shift, npages);
636 region->region.slot = slot;
637 region->region.flags = flags;
638 region->region.guest_phys_addr = guest_paddr;
639 region->region.memory_size = npages * vm->page_size;
640 region->region.userspace_addr = (uintptr_t) region->host_mem;
641 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
642 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
643 " rc: %i errno: %i\n"
644 " slot: %u flags: 0x%x\n"
645 " guest_phys_addr: 0x%lx size: 0x%lx",
646 ret, errno, slot, flags,
647 guest_paddr, (uint64_t) region->region.memory_size);
649 /* Add to linked-list of memory regions. */
650 if (vm->userspace_mem_region_head)
651 vm->userspace_mem_region_head->prev = region;
652 region->next = vm->userspace_mem_region_head;
653 vm->userspace_mem_region_head = region;
660 * vm - Virtual Machine
661 * memslot - KVM memory slot ID
666 * Pointer to memory region structure that describe memory region
667 * using kvm memory slot ID given by memslot. TEST_ASSERT failure
668 * on error (e.g. currently no memory region using memslot as a KVM
671 static struct userspace_mem_region *
672 memslot2region(struct kvm_vm *vm, uint32_t memslot)
674 struct userspace_mem_region *region;
676 for (region = vm->userspace_mem_region_head; region;
677 region = region->next) {
678 if (region->region.slot == memslot)
681 if (region == NULL) {
682 fprintf(stderr, "No mem region with the requested slot found,\n"
683 " requested slot: %u\n", memslot);
684 fputs("---- vm dump ----\n", stderr);
685 vm_dump(stderr, vm, 2);
686 TEST_ASSERT(false, "Mem region not found");
693 * VM Memory Region Flags Set
696 * vm - Virtual Machine
697 * flags - Starting guest physical address
703 * Sets the flags of the memory region specified by the value of slot,
704 * to the values given by flags.
706 void vm_mem_region_set_flags(struct kvm_vm *vm, uint32_t slot, uint32_t flags)
709 struct userspace_mem_region *region;
711 region = memslot2region(vm, slot);
713 region->region.flags = flags;
715 ret = ioctl(vm->fd, KVM_SET_USER_MEMORY_REGION, ®ion->region);
717 TEST_ASSERT(ret == 0, "KVM_SET_USER_MEMORY_REGION IOCTL failed,\n"
718 " rc: %i errno: %i slot: %u flags: 0x%x",
719 ret, errno, slot, flags);
732 * Returns the size of the structure pointed to by the return value
735 static int vcpu_mmap_sz(void)
739 dev_fd = open(KVM_DEV_PATH, O_RDONLY);
743 ret = ioctl(dev_fd, KVM_GET_VCPU_MMAP_SIZE, NULL);
744 TEST_ASSERT(ret >= sizeof(struct kvm_run),
745 "%s KVM_GET_VCPU_MMAP_SIZE ioctl failed, rc: %i errno: %i",
746 __func__, ret, errno);
757 * vm - Virtual Machine
764 * Creates and adds to the VM specified by vm and virtual CPU with
765 * the ID given by vcpuid.
767 void vm_vcpu_add(struct kvm_vm *vm, uint32_t vcpuid, int pgd_memslot,
772 /* Confirm a vcpu with the specified id doesn't already exist. */
773 vcpu = vcpu_find(vm, vcpuid);
775 TEST_ASSERT(false, "vcpu with the specified id "
777 " requested vcpuid: %u\n"
778 " existing vcpuid: %u state: %p",
779 vcpuid, vcpu->id, vcpu->state);
781 /* Allocate and initialize new vcpu structure. */
782 vcpu = calloc(1, sizeof(*vcpu));
783 TEST_ASSERT(vcpu != NULL, "Insufficient Memory");
785 vcpu->fd = ioctl(vm->fd, KVM_CREATE_VCPU, vcpuid);
786 TEST_ASSERT(vcpu->fd >= 0, "KVM_CREATE_VCPU failed, rc: %i errno: %i",
789 TEST_ASSERT(vcpu_mmap_sz() >= sizeof(*vcpu->state), "vcpu mmap size "
790 "smaller than expected, vcpu_mmap_sz: %i expected_min: %zi",
791 vcpu_mmap_sz(), sizeof(*vcpu->state));
792 vcpu->state = (struct kvm_run *) mmap(NULL, sizeof(*vcpu->state),
793 PROT_READ | PROT_WRITE, MAP_SHARED, vcpu->fd, 0);
794 TEST_ASSERT(vcpu->state != MAP_FAILED, "mmap vcpu_state failed, "
795 "vcpu id: %u errno: %i", vcpuid, errno);
797 /* Add to linked-list of VCPUs. */
799 vm->vcpu_head->prev = vcpu;
800 vcpu->next = vm->vcpu_head;
801 vm->vcpu_head = vcpu;
803 vcpu_setup(vm, vcpuid, pgd_memslot, gdt_memslot);
807 * VM Virtual Address Unused Gap
810 * vm - Virtual Machine
812 * vaddr_min - Minimum Virtual Address
817 * Lowest virtual address at or below vaddr_min, with at least
818 * sz unused bytes. TEST_ASSERT failure if no area of at least
819 * size sz is available.
821 * Within the VM specified by vm, locates the lowest starting virtual
822 * address >= vaddr_min, that has at least sz unallocated bytes. A
823 * TEST_ASSERT failure occurs for invalid input or no area of at least
824 * sz unallocated bytes >= vaddr_min is available.
826 static vm_vaddr_t vm_vaddr_unused_gap(struct kvm_vm *vm, size_t sz,
827 vm_vaddr_t vaddr_min)
829 uint64_t pages = (sz + vm->page_size - 1) >> vm->page_shift;
831 /* Determine lowest permitted virtual page index. */
832 uint64_t pgidx_start = (vaddr_min + vm->page_size - 1) >> vm->page_shift;
833 if ((pgidx_start * vm->page_size) < vaddr_min)
836 /* Loop over section with enough valid virtual page indexes. */
837 if (!sparsebit_is_set_num(vm->vpages_valid,
839 pgidx_start = sparsebit_next_set_num(vm->vpages_valid,
843 * Are there enough unused virtual pages available at
844 * the currently proposed starting virtual page index.
845 * If not, adjust proposed starting index to next
848 if (sparsebit_is_clear_num(vm->vpages_mapped,
851 pgidx_start = sparsebit_next_clear_num(vm->vpages_mapped,
853 if (pgidx_start == 0)
857 * If needed, adjust proposed starting virtual address,
858 * to next range of valid virtual addresses.
860 if (!sparsebit_is_set_num(vm->vpages_valid,
861 pgidx_start, pages)) {
862 pgidx_start = sparsebit_next_set_num(
863 vm->vpages_valid, pgidx_start, pages);
864 if (pgidx_start == 0)
867 } while (pgidx_start != 0);
870 TEST_ASSERT(false, "No vaddr of specified pages available, "
871 "pages: 0x%lx", pages);
877 TEST_ASSERT(sparsebit_is_set_num(vm->vpages_valid,
879 "Unexpected, invalid virtual page index range,\n"
880 " pgidx_start: 0x%lx\n"
883 TEST_ASSERT(sparsebit_is_clear_num(vm->vpages_mapped,
885 "Unexpected, pages already mapped,\n"
886 " pgidx_start: 0x%lx\n"
890 return pgidx_start * vm->page_size;
894 * VM Virtual Address Allocate
897 * vm - Virtual Machine
899 * vaddr_min - Minimum starting virtual address
900 * data_memslot - Memory region slot for data pages
901 * pgd_memslot - Memory region slot for new virtual translation tables
906 * Starting guest virtual address
908 * Allocates at least sz bytes within the virtual address space of the vm
909 * given by vm. The allocated bytes are mapped to a virtual address >=
910 * the address given by vaddr_min. Note that each allocation uses a
911 * a unique set of pages, with the minimum real allocation being at least
914 vm_vaddr_t vm_vaddr_alloc(struct kvm_vm *vm, size_t sz, vm_vaddr_t vaddr_min,
915 uint32_t data_memslot, uint32_t pgd_memslot)
917 uint64_t pages = (sz >> vm->page_shift) + ((sz % vm->page_size) != 0);
919 virt_pgd_alloc(vm, pgd_memslot);
922 * Find an unused range of virtual page addresses of at least
925 vm_vaddr_t vaddr_start = vm_vaddr_unused_gap(vm, sz, vaddr_min);
927 /* Map the virtual pages. */
928 for (vm_vaddr_t vaddr = vaddr_start; pages > 0;
929 pages--, vaddr += vm->page_size) {
932 paddr = vm_phy_page_alloc(vm,
933 KVM_UTIL_MIN_PFN * vm->page_size, data_memslot);
935 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
937 sparsebit_set(vm->vpages_mapped,
938 vaddr >> vm->page_shift);
945 * Map a range of VM virtual address to the VM's physical address
948 * vm - Virtual Machine
949 * vaddr - Virtuall address to map
950 * paddr - VM Physical Address
951 * size - The size of the range to map
952 * pgd_memslot - Memory region slot for new virtual translation tables
958 * Within the VM given by vm, creates a virtual translation for the
959 * page range starting at vaddr to the page range starting at paddr.
961 void virt_map(struct kvm_vm *vm, uint64_t vaddr, uint64_t paddr,
962 size_t size, uint32_t pgd_memslot)
964 size_t page_size = vm->page_size;
965 size_t npages = size / page_size;
967 TEST_ASSERT(vaddr + size > vaddr, "Vaddr overflow");
968 TEST_ASSERT(paddr + size > paddr, "Paddr overflow");
971 virt_pg_map(vm, vaddr, paddr, pgd_memslot);
978 * Address VM Physical to Host Virtual
981 * vm - Virtual Machine
982 * gpa - VM physical address
987 * Equivalent host virtual address
989 * Locates the memory region containing the VM physical address given
990 * by gpa, within the VM given by vm. When found, the host virtual
991 * address providing the memory to the vm physical address is returned.
992 * A TEST_ASSERT failure occurs if no region containing gpa exists.
994 void *addr_gpa2hva(struct kvm_vm *vm, vm_paddr_t gpa)
996 struct userspace_mem_region *region;
997 for (region = vm->userspace_mem_region_head; region;
998 region = region->next) {
999 if ((gpa >= region->region.guest_phys_addr)
1000 && (gpa <= (region->region.guest_phys_addr
1001 + region->region.memory_size - 1)))
1002 return (void *) ((uintptr_t) region->host_mem
1003 + (gpa - region->region.guest_phys_addr));
1006 TEST_ASSERT(false, "No vm physical memory at 0x%lx", gpa);
1011 * Address Host Virtual to VM Physical
1014 * vm - Virtual Machine
1015 * hva - Host virtual address
1020 * Equivalent VM physical address
1022 * Locates the memory region containing the host virtual address given
1023 * by hva, within the VM given by vm. When found, the equivalent
1024 * VM physical address is returned. A TEST_ASSERT failure occurs if no
1025 * region containing hva exists.
1027 vm_paddr_t addr_hva2gpa(struct kvm_vm *vm, void *hva)
1029 struct userspace_mem_region *region;
1030 for (region = vm->userspace_mem_region_head; region;
1031 region = region->next) {
1032 if ((hva >= region->host_mem)
1033 && (hva <= (region->host_mem
1034 + region->region.memory_size - 1)))
1035 return (vm_paddr_t) ((uintptr_t)
1036 region->region.guest_phys_addr
1037 + (hva - (uintptr_t) region->host_mem));
1040 TEST_ASSERT(false, "No mapping to a guest physical address, "
1046 * VM Create IRQ Chip
1049 * vm - Virtual Machine
1055 * Creates an interrupt controller chip for the VM specified by vm.
1057 void vm_create_irqchip(struct kvm_vm *vm)
1061 ret = ioctl(vm->fd, KVM_CREATE_IRQCHIP, 0);
1062 TEST_ASSERT(ret == 0, "KVM_CREATE_IRQCHIP IOCTL failed, "
1063 "rc: %i errno: %i", ret, errno);
1065 vm->has_irqchip = true;
1072 * vm - Virtual Machine
1078 * Pointer to structure that describes the state of the VCPU.
1080 * Locates and returns a pointer to a structure that describes the
1081 * state of the VCPU with the given vcpuid.
1083 struct kvm_run *vcpu_state(struct kvm_vm *vm, uint32_t vcpuid)
1085 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1086 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1095 * vm - Virtual Machine
1102 * Switch to executing the code for the VCPU given by vcpuid, within the VM
1105 void vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1107 int ret = _vcpu_run(vm, vcpuid);
1108 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1109 "rc: %i errno: %i", ret, errno);
1112 int _vcpu_run(struct kvm_vm *vm, uint32_t vcpuid)
1114 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1117 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1119 rc = ioctl(vcpu->fd, KVM_RUN, NULL);
1120 } while (rc == -1 && errno == EINTR);
1124 void vcpu_run_complete_io(struct kvm_vm *vm, uint32_t vcpuid)
1126 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1129 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1131 vcpu->state->immediate_exit = 1;
1132 ret = ioctl(vcpu->fd, KVM_RUN, NULL);
1133 vcpu->state->immediate_exit = 0;
1135 TEST_ASSERT(ret == -1 && errno == EINTR,
1136 "KVM_RUN IOCTL didn't exit immediately, rc: %i, errno: %i",
1141 * VM VCPU Set MP State
1144 * vm - Virtual Machine
1146 * mp_state - mp_state to be set
1152 * Sets the MP state of the VCPU given by vcpuid, to the state given
1155 void vcpu_set_mp_state(struct kvm_vm *vm, uint32_t vcpuid,
1156 struct kvm_mp_state *mp_state)
1158 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1161 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1163 ret = ioctl(vcpu->fd, KVM_SET_MP_STATE, mp_state);
1164 TEST_ASSERT(ret == 0, "KVM_SET_MP_STATE IOCTL failed, "
1165 "rc: %i errno: %i", ret, errno);
1172 * vm - Virtual Machine
1176 * regs - current state of VCPU regs
1180 * Obtains the current register state for the VCPU specified by vcpuid
1181 * and stores it at the location given by regs.
1183 void vcpu_regs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1185 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1188 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1190 ret = ioctl(vcpu->fd, KVM_GET_REGS, regs);
1191 TEST_ASSERT(ret == 0, "KVM_GET_REGS failed, rc: %i errno: %i",
1199 * vm - Virtual Machine
1201 * regs - Values to set VCPU regs to
1207 * Sets the regs of the VCPU specified by vcpuid to the values
1210 void vcpu_regs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_regs *regs)
1212 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1215 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1217 ret = ioctl(vcpu->fd, KVM_SET_REGS, regs);
1218 TEST_ASSERT(ret == 0, "KVM_SET_REGS failed, rc: %i errno: %i",
1222 void vcpu_events_get(struct kvm_vm *vm, uint32_t vcpuid,
1223 struct kvm_vcpu_events *events)
1225 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1228 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1230 ret = ioctl(vcpu->fd, KVM_GET_VCPU_EVENTS, events);
1231 TEST_ASSERT(ret == 0, "KVM_GET_VCPU_EVENTS, failed, rc: %i errno: %i",
1235 void vcpu_events_set(struct kvm_vm *vm, uint32_t vcpuid,
1236 struct kvm_vcpu_events *events)
1238 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1241 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1243 ret = ioctl(vcpu->fd, KVM_SET_VCPU_EVENTS, events);
1244 TEST_ASSERT(ret == 0, "KVM_SET_VCPU_EVENTS, failed, rc: %i errno: %i",
1249 * VM VCPU System Regs Get
1252 * vm - Virtual Machine
1256 * sregs - current state of VCPU system regs
1260 * Obtains the current system register state for the VCPU specified by
1261 * vcpuid and stores it at the location given by sregs.
1263 void vcpu_sregs_get(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1265 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1268 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1270 ret = ioctl(vcpu->fd, KVM_GET_SREGS, sregs);
1271 TEST_ASSERT(ret == 0, "KVM_GET_SREGS failed, rc: %i errno: %i",
1276 * VM VCPU System Regs Set
1279 * vm - Virtual Machine
1281 * sregs - Values to set VCPU system regs to
1287 * Sets the system regs of the VCPU specified by vcpuid to the values
1290 void vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1292 int ret = _vcpu_sregs_set(vm, vcpuid, sregs);
1293 TEST_ASSERT(ret == 0, "KVM_RUN IOCTL failed, "
1294 "rc: %i errno: %i", ret, errno);
1297 int _vcpu_sregs_set(struct kvm_vm *vm, uint32_t vcpuid, struct kvm_sregs *sregs)
1299 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1302 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1304 return ioctl(vcpu->fd, KVM_SET_SREGS, sregs);
1311 * vm - Virtual Machine
1313 * cmd - Ioctl number
1314 * arg - Argument to pass to the ioctl
1318 * Issues an arbitrary ioctl on a VCPU fd.
1320 void vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1321 unsigned long cmd, void *arg)
1325 ret = _vcpu_ioctl(vm, vcpuid, cmd, arg);
1326 TEST_ASSERT(ret == 0, "vcpu ioctl %lu failed, rc: %i errno: %i (%s)",
1327 cmd, ret, errno, strerror(errno));
1330 int _vcpu_ioctl(struct kvm_vm *vm, uint32_t vcpuid,
1331 unsigned long cmd, void *arg)
1333 struct vcpu *vcpu = vcpu_find(vm, vcpuid);
1336 TEST_ASSERT(vcpu != NULL, "vcpu not found, vcpuid: %u", vcpuid);
1338 ret = ioctl(vcpu->fd, cmd, arg);
1347 * vm - Virtual Machine
1348 * cmd - Ioctl number
1349 * arg - Argument to pass to the ioctl
1353 * Issues an arbitrary ioctl on a VM fd.
1355 void vm_ioctl(struct kvm_vm *vm, unsigned long cmd, void *arg)
1359 ret = ioctl(vm->fd, cmd, arg);
1360 TEST_ASSERT(ret == 0, "vm ioctl %lu failed, rc: %i errno: %i (%s)",
1361 cmd, ret, errno, strerror(errno));
1368 * vm - Virtual Machine
1369 * indent - Left margin indent amount
1372 * stream - Output FILE stream
1376 * Dumps the current state of the VM given by vm, to the FILE stream
1379 void vm_dump(FILE *stream, struct kvm_vm *vm, uint8_t indent)
1381 struct userspace_mem_region *region;
1384 fprintf(stream, "%*smode: 0x%x\n", indent, "", vm->mode);
1385 fprintf(stream, "%*sfd: %i\n", indent, "", vm->fd);
1386 fprintf(stream, "%*spage_size: 0x%x\n", indent, "", vm->page_size);
1387 fprintf(stream, "%*sMem Regions:\n", indent, "");
1388 for (region = vm->userspace_mem_region_head; region;
1389 region = region->next) {
1390 fprintf(stream, "%*sguest_phys: 0x%lx size: 0x%lx "
1391 "host_virt: %p\n", indent + 2, "",
1392 (uint64_t) region->region.guest_phys_addr,
1393 (uint64_t) region->region.memory_size,
1395 fprintf(stream, "%*sunused_phy_pages: ", indent + 2, "");
1396 sparsebit_dump(stream, region->unused_phy_pages, 0);
1398 fprintf(stream, "%*sMapped Virtual Pages:\n", indent, "");
1399 sparsebit_dump(stream, vm->vpages_mapped, indent + 2);
1400 fprintf(stream, "%*spgd_created: %u\n", indent, "",
1402 if (vm->pgd_created) {
1403 fprintf(stream, "%*sVirtual Translation Tables:\n",
1405 virt_dump(stream, vm, indent + 4);
1407 fprintf(stream, "%*sVCPUs:\n", indent, "");
1408 for (vcpu = vm->vcpu_head; vcpu; vcpu = vcpu->next)
1409 vcpu_dump(stream, vm, vcpu->id, indent + 2);
1412 /* Known KVM exit reasons */
1413 static struct exit_reason {
1414 unsigned int reason;
1416 } exit_reasons_known[] = {
1417 {KVM_EXIT_UNKNOWN, "UNKNOWN"},
1418 {KVM_EXIT_EXCEPTION, "EXCEPTION"},
1419 {KVM_EXIT_IO, "IO"},
1420 {KVM_EXIT_HYPERCALL, "HYPERCALL"},
1421 {KVM_EXIT_DEBUG, "DEBUG"},
1422 {KVM_EXIT_HLT, "HLT"},
1423 {KVM_EXIT_MMIO, "MMIO"},
1424 {KVM_EXIT_IRQ_WINDOW_OPEN, "IRQ_WINDOW_OPEN"},
1425 {KVM_EXIT_SHUTDOWN, "SHUTDOWN"},
1426 {KVM_EXIT_FAIL_ENTRY, "FAIL_ENTRY"},
1427 {KVM_EXIT_INTR, "INTR"},
1428 {KVM_EXIT_SET_TPR, "SET_TPR"},
1429 {KVM_EXIT_TPR_ACCESS, "TPR_ACCESS"},
1430 {KVM_EXIT_S390_SIEIC, "S390_SIEIC"},
1431 {KVM_EXIT_S390_RESET, "S390_RESET"},
1432 {KVM_EXIT_DCR, "DCR"},
1433 {KVM_EXIT_NMI, "NMI"},
1434 {KVM_EXIT_INTERNAL_ERROR, "INTERNAL_ERROR"},
1435 {KVM_EXIT_OSI, "OSI"},
1436 {KVM_EXIT_PAPR_HCALL, "PAPR_HCALL"},
1437 #ifdef KVM_EXIT_MEMORY_NOT_PRESENT
1438 {KVM_EXIT_MEMORY_NOT_PRESENT, "MEMORY_NOT_PRESENT"},
1443 * Exit Reason String
1446 * exit_reason - Exit reason
1451 * Constant string pointer describing the exit reason.
1453 * Locates and returns a constant string that describes the KVM exit
1454 * reason given by exit_reason. If no such string is found, a constant
1455 * string of "Unknown" is returned.
1457 const char *exit_reason_str(unsigned int exit_reason)
1461 for (n1 = 0; n1 < ARRAY_SIZE(exit_reasons_known); n1++) {
1462 if (exit_reason == exit_reasons_known[n1].reason)
1463 return exit_reasons_known[n1].name;
1470 * Physical Contiguous Page Allocator
1473 * vm - Virtual Machine
1474 * num - number of pages
1475 * paddr_min - Physical address minimum
1476 * memslot - Memory region to allocate page from
1481 * Starting physical address
1483 * Within the VM specified by vm, locates a range of available physical
1484 * pages at or above paddr_min. If found, the pages are marked as in use
1485 * and their base address is returned. A TEST_ASSERT failure occurs if
1486 * not enough pages are available at or above paddr_min.
1488 vm_paddr_t vm_phy_pages_alloc(struct kvm_vm *vm, size_t num,
1489 vm_paddr_t paddr_min, uint32_t memslot)
1491 struct userspace_mem_region *region;
1492 sparsebit_idx_t pg, base;
1494 TEST_ASSERT(num > 0, "Must allocate at least one page");
1496 TEST_ASSERT((paddr_min % vm->page_size) == 0, "Min physical address "
1497 "not divisible by page size.\n"
1498 " paddr_min: 0x%lx page_size: 0x%x",
1499 paddr_min, vm->page_size);
1501 region = memslot2region(vm, memslot);
1502 base = pg = paddr_min >> vm->page_shift;
1505 for (; pg < base + num; ++pg) {
1506 if (!sparsebit_is_set(region->unused_phy_pages, pg)) {
1507 base = pg = sparsebit_next_set(region->unused_phy_pages, pg);
1511 } while (pg && pg != base + num);
1514 fprintf(stderr, "No guest physical page available, "
1515 "paddr_min: 0x%lx page_size: 0x%x memslot: %u\n",
1516 paddr_min, vm->page_size, memslot);
1517 fputs("---- vm dump ----\n", stderr);
1518 vm_dump(stderr, vm, 2);
1522 for (pg = base; pg < base + num; ++pg)
1523 sparsebit_clear(region->unused_phy_pages, pg);
1525 return base * vm->page_size;
1528 vm_paddr_t vm_phy_page_alloc(struct kvm_vm *vm, vm_paddr_t paddr_min,
1531 return vm_phy_pages_alloc(vm, 1, paddr_min, memslot);
1535 * Address Guest Virtual to Host Virtual
1538 * vm - Virtual Machine
1539 * gva - VM virtual address
1544 * Equivalent host virtual address
1546 void *addr_gva2hva(struct kvm_vm *vm, vm_vaddr_t gva)
1548 return addr_gpa2hva(vm, addr_gva2gpa(vm, gva));