drivers/gpu/drm/i915/gvt/kvmgt.c

   1 /*
   2  * KVMGT - the implementation of Intel mediated pass-through framework for KVM
   3  *
   4  * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the next
  14  * paragraph) shall be included in all copies or substantial portions of the
  15  * Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  23  * SOFTWARE.
  24  *
  25  * Authors:
  26  *    Kevin Tian <kevin.tian@intel.com>
  27  *    Jike Song <jike.song@intel.com>
  28  *    Xiaoguang Chen <xiaoguang.chen@intel.com>
  29  */
  30
  31 #include <linux/init.h>
  32 #include <linux/device.h>
  33 #include <linux/mm.h>
  34 #include <linux/mmu_context.h>
  35 #include <linux/sched/mm.h>
  36 #include <linux/types.h>
  37 #include <linux/list.h>
  38 #include <linux/rbtree.h>
  39 #include <linux/spinlock.h>
  40 #include <linux/eventfd.h>
  41 #include <linux/uuid.h>
  42 #include <linux/kvm_host.h>
  43 #include <linux/vfio.h>
  44 #include <linux/mdev.h>
  45 #include <linux/debugfs.h>
  46
  47 #include <linux/nospec.h>
  48
  49 #include "i915_drv.h"
  50 #include "gvt.h"
  51
  52 static const struct intel_gvt_ops *intel_gvt_ops;
  53
  54 /* helper macros copied from vfio-pci */
  55 #define VFIO_PCI_OFFSET_SHIFT   40
  56 #define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
  57 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
  58 #define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
  59
  60 #define OPREGION_SIGNATURE "IntelGraphicsMem"
  61
  62 struct vfio_region;
  63 struct intel_vgpu_regops {
  64         size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
  65                         size_t count, loff_t *ppos, bool iswrite);
  66         void (*release)(struct intel_vgpu *vgpu,
  67                         struct vfio_region *region);
  68 };
  69
  70 struct vfio_region {
  71         u32                             type;
  72         u32                             subtype;
  73         size_t                          size;
  74         u32                             flags;
  75         const struct intel_vgpu_regops  *ops;
  76         void                            *data;
  77 };
  78
  79 struct kvmgt_pgfn {
  80         gfn_t gfn;
  81         struct hlist_node hnode;
  82 };
  83
  84 struct kvmgt_guest_info {
  85         struct kvm *kvm;
  86         struct intel_vgpu *vgpu;
  87         struct kvm_page_track_notifier_node track_node;
  88 #define NR_BKT (1 << 18)
  89         struct hlist_head ptable[NR_BKT];
  90 #undef NR_BKT
  91         struct dentry *debugfs_cache_entries;
  92 };
  93
  94 struct gvt_dma {
  95         struct intel_vgpu *vgpu;
  96         struct rb_node gfn_node;
  97         struct rb_node dma_addr_node;
  98         gfn_t gfn;
  99         dma_addr_t dma_addr;
 100         unsigned long size;
 101         struct kref ref;
 102 };
 103
 104 static inline bool handle_valid(unsigned long handle)
 105 {
 106         return !!(handle & ~0xff);
 107 }
 108
 109 static int kvmgt_guest_init(struct mdev_device *mdev);
 110 static void intel_vgpu_release_work(struct work_struct *work);
 111 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
 112
 113 static void gvt_unpin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 114                 unsigned long size)
 115 {
 116         int total_pages;
 117         int npage;
 118         int ret;
 119
 120         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
 121
 122         for (npage = 0; npage < total_pages; npage++) {
 123                 unsigned long cur_gfn = gfn + npage;
 124
 125                 ret = vfio_unpin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1);
 126                 WARN_ON(ret != 1);
 127         }
 128 }
 129
 130 /* Pin a normal or compound guest page for dma. */
 131 static int gvt_pin_guest_page(struct intel_vgpu *vgpu, unsigned long gfn,
 132                 unsigned long size, struct page **page)
 133 {
 134         unsigned long base_pfn = 0;
 135         int total_pages;
 136         int npage;
 137         int ret;
 138
 139         total_pages = roundup(size, PAGE_SIZE) / PAGE_SIZE;
 140         /*
 141          * We pin the pages one-by-one to avoid allocating a big arrary
 142          * on stack to hold pfns.
 143          */
 144         for (npage = 0; npage < total_pages; npage++) {
 145                 unsigned long cur_gfn = gfn + npage;
 146                 unsigned long pfn;
 147
 148                 ret = vfio_pin_pages(mdev_dev(vgpu->vdev.mdev), &cur_gfn, 1,
 149                                      IOMMU_READ | IOMMU_WRITE, &pfn);
 150                 if (ret != 1) {
 151                         gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx, ret %d\n",
 152                                      cur_gfn, ret);
 153                         goto err;
 154                 }
 155
 156                 if (!pfn_valid(pfn)) {
 157                         gvt_vgpu_err("pfn 0x%lx is not mem backed\n", pfn);
 158                         npage++;
 159                         ret = -EFAULT;
 160                         goto err;
 161                 }
 162
 163                 if (npage == 0)
 164                         base_pfn = pfn;
 165                 else if (base_pfn + npage != pfn) {
 166                         gvt_vgpu_err("The pages are not continuous\n");
 167                         ret = -EINVAL;
 168                         npage++;
 169                         goto err;
 170                 }
 171         }
 172
 173         *page = pfn_to_page(base_pfn);
 174         return 0;
 175 err:
 176         gvt_unpin_guest_page(vgpu, gfn, npage * PAGE_SIZE);
 177         return ret;
 178 }
 179
 180 static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn,
 181                 dma_addr_t *dma_addr, unsigned long size)
 182 {
 183         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 184         struct page *page = NULL;
 185         int ret;
 186
 187         ret = gvt_pin_guest_page(vgpu, gfn, size, &page);
 188         if (ret)
 189                 return ret;
 190
 191         /* Setup DMA mapping. */
 192         *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL);
 193         if (dma_mapping_error(dev, *dma_addr)) {
 194                 gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n",
 195                              page_to_pfn(page), ret);
 196                 gvt_unpin_guest_page(vgpu, gfn, size);
 197                 return -ENOMEM;
 198         }
 199
 200         return 0;
 201 }
 202
 203 static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn,
 204                 dma_addr_t dma_addr, unsigned long size)
 205 {
 206         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
 207
 208         dma_unmap_page(dev, dma_addr, size, PCI_DMA_BIDIRECTIONAL);
 209         gvt_unpin_guest_page(vgpu, gfn, size);
 210 }
 211
 212 static struct gvt_dma *__gvt_cache_find_dma_addr(struct intel_vgpu *vgpu,
 213                 dma_addr_t dma_addr)
 214 {
 215         struct rb_node *node = vgpu->vdev.dma_addr_cache.rb_node;
 216         struct gvt_dma *itr;
 217
 218         while (node) {
 219                 itr = rb_entry(node, struct gvt_dma, dma_addr_node);
 220
 221                 if (dma_addr < itr->dma_addr)
 222                         node = node->rb_left;
 223                 else if (dma_addr > itr->dma_addr)
 224                         node = node->rb_right;
 225                 else
 226                         return itr;
 227         }
 228         return NULL;
 229 }
 230
 231 static struct gvt_dma *__gvt_cache_find_gfn(struct intel_vgpu *vgpu, gfn_t gfn)
 232 {
 233         struct rb_node *node = vgpu->vdev.gfn_cache.rb_node;
 234         struct gvt_dma *itr;
 235
 236         while (node) {
 237                 itr = rb_entry(node, struct gvt_dma, gfn_node);
 238
 239                 if (gfn < itr->gfn)
 240                         node = node->rb_left;
 241                 else if (gfn > itr->gfn)
 242                         node = node->rb_right;
 243                 else
 244                         return itr;
 245         }
 246         return NULL;
 247 }
 248
 249 static int __gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
 250                 dma_addr_t dma_addr, unsigned long size)
 251 {
 252         struct gvt_dma *new, *itr;
 253         struct rb_node **link, *parent = NULL;
 254
 255         new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
 256         if (!new)
 257                 return -ENOMEM;
 258
 259         new->vgpu = vgpu;
 260         new->gfn = gfn;
 261         new->dma_addr = dma_addr;
 262         new->size = size;
 263         kref_init(&new->ref);
 264
 265         /* gfn_cache maps gfn to struct gvt_dma. */
 266         link = &vgpu->vdev.gfn_cache.rb_node;
 267         while (*link) {
 268                 parent = *link;
 269                 itr = rb_entry(parent, struct gvt_dma, gfn_node);
 270
 271                 if (gfn < itr->gfn)
 272                         link = &parent->rb_left;
 273                 else
 274                         link = &parent->rb_right;
 275         }
 276         rb_link_node(&new->gfn_node, parent, link);
 277         rb_insert_color(&new->gfn_node, &vgpu->vdev.gfn_cache);
 278
 279         /* dma_addr_cache maps dma addr to struct gvt_dma. */
 280         parent = NULL;
 281         link = &vgpu->vdev.dma_addr_cache.rb_node;
 282         while (*link) {
 283                 parent = *link;
 284                 itr = rb_entry(parent, struct gvt_dma, dma_addr_node);
 285
 286                 if (dma_addr < itr->dma_addr)
 287                         link = &parent->rb_left;
 288                 else
 289                         link = &parent->rb_right;
 290         }
 291         rb_link_node(&new->dma_addr_node, parent, link);
 292         rb_insert_color(&new->dma_addr_node, &vgpu->vdev.dma_addr_cache);
 293
 294         vgpu->vdev.nr_cache_entries++;
 295         return 0;
 296 }
 297
 298 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
 299                                 struct gvt_dma *entry)
 300 {
 301         rb_erase(&entry->gfn_node, &vgpu->vdev.gfn_cache);
 302         rb_erase(&entry->dma_addr_node, &vgpu->vdev.dma_addr_cache);
 303         kfree(entry);
 304         vgpu->vdev.nr_cache_entries--;
 305 }
 306
 307 static void gvt_cache_destroy(struct intel_vgpu *vgpu)
 308 {
 309         struct gvt_dma *dma;
 310         struct rb_node *node = NULL;
 311
 312         for (;;) {
 313                 mutex_lock(&vgpu->vdev.cache_lock);
 314                 node = rb_first(&vgpu->vdev.gfn_cache);
 315                 if (!node) {
 316                         mutex_unlock(&vgpu->vdev.cache_lock);
 317                         break;
 318                 }
 319                 dma = rb_entry(node, struct gvt_dma, gfn_node);
 320                 gvt_dma_unmap_page(vgpu, dma->gfn, dma->dma_addr, dma->size);
 321                 __gvt_cache_remove_entry(vgpu, dma);
 322                 mutex_unlock(&vgpu->vdev.cache_lock);
 323         }
 324 }
 325
 326 static void gvt_cache_init(struct intel_vgpu *vgpu)
 327 {
 328         vgpu->vdev.gfn_cache = RB_ROOT;
 329         vgpu->vdev.dma_addr_cache = RB_ROOT;
 330         vgpu->vdev.nr_cache_entries = 0;
 331         mutex_init(&vgpu->vdev.cache_lock);
 332 }
 333
 334 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
 335 {
 336         hash_init(info->ptable);
 337 }
 338
 339 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
 340 {
 341         struct kvmgt_pgfn *p;
 342         struct hlist_node *tmp;
 343         int i;
 344
 345         hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
 346                 hash_del(&p->hnode);
 347                 kfree(p);
 348         }
 349 }
 350
 351 static struct kvmgt_pgfn *
 352 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
 353 {
 354         struct kvmgt_pgfn *p, *res = NULL;
 355
 356         hash_for_each_possible(info->ptable, p, hnode, gfn) {
 357                 if (gfn == p->gfn) {
 358                         res = p;
 359                         break;
 360                 }
 361         }
 362
 363         return res;
 364 }
 365
 366 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
 367                                 gfn_t gfn)
 368 {
 369         struct kvmgt_pgfn *p;
 370
 371         p = __kvmgt_protect_table_find(info, gfn);
 372         return !!p;
 373 }
 374
 375 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
 376 {
 377         struct kvmgt_pgfn *p;
 378
 379         if (kvmgt_gfn_is_write_protected(info, gfn))
 380                 return;
 381
 382         p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
 383         if (WARN(!p, "gfn: 0x%llx\n", gfn))
 384                 return;
 385
 386         p->gfn = gfn;
 387         hash_add(info->ptable, &p->hnode, gfn);
 388 }
 389
 390 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
 391                                 gfn_t gfn)
 392 {
 393         struct kvmgt_pgfn *p;
 394
 395         p = __kvmgt_protect_table_find(info, gfn);
 396         if (p) {
 397                 hash_del(&p->hnode);
 398                 kfree(p);
 399         }
 400 }
 401
 402 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
 403                 size_t count, loff_t *ppos, bool iswrite)
 404 {
 405         unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
 406                         VFIO_PCI_NUM_REGIONS;
 407         void *base = vgpu->vdev.region[i].data;
 408         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 409
 410         if (pos >= vgpu->vdev.region[i].size || iswrite) {
 411                 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
 412                 return -EINVAL;
 413         }
 414         count = min(count, (size_t)(vgpu->vdev.region[i].size - pos));
 415         memcpy(buf, base + pos, count);
 416
 417         return count;
 418 }
 419
 420 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
 421                 struct vfio_region *region)
 422 {
 423 }
 424
 425 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
 426         .rw = intel_vgpu_reg_rw_opregion,
 427         .release = intel_vgpu_reg_release_opregion,
 428 };
 429
 430 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
 431                 unsigned int type, unsigned int subtype,
 432                 const struct intel_vgpu_regops *ops,
 433                 size_t size, u32 flags, void *data)
 434 {
 435         struct vfio_region *region;
 436
 437         region = krealloc(vgpu->vdev.region,
 438                         (vgpu->vdev.num_regions + 1) * sizeof(*region),
 439                         GFP_KERNEL);
 440         if (!region)
 441                 return -ENOMEM;
 442
 443         vgpu->vdev.region = region;
 444         vgpu->vdev.region[vgpu->vdev.num_regions].type = type;
 445         vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype;
 446         vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops;
 447         vgpu->vdev.region[vgpu->vdev.num_regions].size = size;
 448         vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags;
 449         vgpu->vdev.region[vgpu->vdev.num_regions].data = data;
 450         vgpu->vdev.num_regions++;
 451         return 0;
 452 }
 453
 454 static int kvmgt_get_vfio_device(void *p_vgpu)
 455 {
 456         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
 457
 458         vgpu->vdev.vfio_device = vfio_device_get_from_dev(
 459                 mdev_dev(vgpu->vdev.mdev));
 460         if (!vgpu->vdev.vfio_device) {
 461                 gvt_vgpu_err("failed to get vfio device\n");
 462                 return -ENODEV;
 463         }
 464         return 0;
 465 }
 466
 467
 468 static int kvmgt_set_opregion(void *p_vgpu)
 469 {
 470         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
 471         void *base;
 472         int ret;
 473
 474         /* Each vgpu has its own opregion, although VFIO would create another
 475          * one later. This one is used to expose opregion to VFIO. And the
 476          * other one created by VFIO later, is used by guest actually.
 477          */
 478         base = vgpu_opregion(vgpu)->va;
 479         if (!base)
 480                 return -ENOMEM;
 481
 482         if (memcmp(base, OPREGION_SIGNATURE, 16)) {
 483                 memunmap(base);
 484                 return -EINVAL;
 485         }
 486
 487         ret = intel_vgpu_register_reg(vgpu,
 488                         PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
 489                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
 490                         &intel_vgpu_regops_opregion, OPREGION_SIZE,
 491                         VFIO_REGION_INFO_FLAG_READ, base);
 492
 493         return ret;
 494 }
 495
 496 static void kvmgt_put_vfio_device(void *vgpu)
 497 {
 498         if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device))
 499                 return;
 500
 501         vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device);
 502 }
 503
 504 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
 505 {
 506         struct intel_vgpu *vgpu = NULL;
 507         struct intel_vgpu_type *type;
 508         struct device *pdev;
 509         void *gvt;
 510         int ret;
 511
 512         pdev = mdev_parent_dev(mdev);
 513         gvt = kdev_to_i915(pdev)->gvt;
 514
 515         type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
 516         if (!type) {
 517                 gvt_vgpu_err("failed to find type %s to create\n",
 518                                                 kobject_name(kobj));
 519                 ret = -EINVAL;
 520                 goto out;
 521         }
 522
 523         vgpu = intel_gvt_ops->vgpu_create(gvt, type);
 524         if (IS_ERR_OR_NULL(vgpu)) {
 525                 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu);
 526                 gvt_err("failed to create intel vgpu: %d\n", ret);
 527                 goto out;
 528         }
 529
 530         INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work);
 531
 532         vgpu->vdev.mdev = mdev;
 533         mdev_set_drvdata(mdev, vgpu);
 534
 535         gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
 536                      dev_name(mdev_dev(mdev)));
 537         ret = 0;
 538
 539 out:
 540         return ret;
 541 }
 542
 543 static int intel_vgpu_remove(struct mdev_device *mdev)
 544 {
 545         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 546
 547         if (handle_valid(vgpu->handle))
 548                 return -EBUSY;
 549
 550         intel_gvt_ops->vgpu_destroy(vgpu);
 551         return 0;
 552 }
 553
 554 static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
 555                                      unsigned long action, void *data)
 556 {
 557         struct intel_vgpu *vgpu = container_of(nb,
 558                                         struct intel_vgpu,
 559                                         vdev.iommu_notifier);
 560
 561         if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
 562                 struct vfio_iommu_type1_dma_unmap *unmap = data;
 563                 struct gvt_dma *entry;
 564                 unsigned long iov_pfn, end_iov_pfn;
 565
 566                 iov_pfn = unmap->iova >> PAGE_SHIFT;
 567                 end_iov_pfn = iov_pfn + unmap->size / PAGE_SIZE;
 568
 569                 mutex_lock(&vgpu->vdev.cache_lock);
 570                 for (; iov_pfn < end_iov_pfn; iov_pfn++) {
 571                         entry = __gvt_cache_find_gfn(vgpu, iov_pfn);
 572                         if (!entry)
 573                                 continue;
 574
 575                         gvt_dma_unmap_page(vgpu, entry->gfn, entry->dma_addr,
 576                                            entry->size);
 577                         __gvt_cache_remove_entry(vgpu, entry);
 578                 }
 579                 mutex_unlock(&vgpu->vdev.cache_lock);
 580         }
 581
 582         return NOTIFY_OK;
 583 }
 584
 585 static int intel_vgpu_group_notifier(struct notifier_block *nb,
 586                                      unsigned long action, void *data)
 587 {
 588         struct intel_vgpu *vgpu = container_of(nb,
 589                                         struct intel_vgpu,
 590                                         vdev.group_notifier);
 591
 592         /* the only action we care about */
 593         if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
 594                 vgpu->vdev.kvm = data;
 595
 596                 if (!data)
 597                         schedule_work(&vgpu->vdev.release_work);
 598         }
 599
 600         return NOTIFY_OK;
 601 }
 602
 603 static int intel_vgpu_open(struct mdev_device *mdev)
 604 {
 605         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 606         unsigned long events;
 607         int ret;
 608
 609         vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier;
 610         vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier;
 611
 612         events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
 613         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events,
 614                                 &vgpu->vdev.iommu_notifier);
 615         if (ret != 0) {
 616                 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n",
 617                         ret);
 618                 goto out;
 619         }
 620
 621         events = VFIO_GROUP_NOTIFY_SET_KVM;
 622         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events,
 623                                 &vgpu->vdev.group_notifier);
 624         if (ret != 0) {
 625                 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n",
 626                         ret);
 627                 goto undo_iommu;
 628         }
 629
 630         ret = kvmgt_guest_init(mdev);
 631         if (ret)
 632                 goto undo_group;
 633
 634         intel_gvt_ops->vgpu_activate(vgpu);
 635
 636         atomic_set(&vgpu->vdev.released, 0);
 637         return ret;
 638
 639 undo_group:
 640         vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
 641                                         &vgpu->vdev.group_notifier);
 642
 643 undo_iommu:
 644         vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
 645                                         &vgpu->vdev.iommu_notifier);
 646 out:
 647         return ret;
 648 }
 649
 650 static void intel_vgpu_release_msi_eventfd_ctx(struct intel_vgpu *vgpu)
 651 {
 652         struct eventfd_ctx *trigger;
 653
 654         trigger = vgpu->vdev.msi_trigger;
 655         if (trigger) {
 656                 eventfd_ctx_put(trigger);
 657                 vgpu->vdev.msi_trigger = NULL;
 658         }
 659 }
 660
 661 static void __intel_vgpu_release(struct intel_vgpu *vgpu)
 662 {
 663         struct kvmgt_guest_info *info;
 664         int ret;
 665
 666         if (!handle_valid(vgpu->handle))
 667                 return;
 668
 669         if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1))
 670                 return;
 671
 672         intel_gvt_ops->vgpu_release(vgpu);
 673
 674         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY,
 675                                         &vgpu->vdev.iommu_notifier);
 676         WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret);
 677
 678         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY,
 679                                         &vgpu->vdev.group_notifier);
 680         WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret);
 681
 682         info = (struct kvmgt_guest_info *)vgpu->handle;
 683         kvmgt_guest_exit(info);
 684
 685         intel_vgpu_release_msi_eventfd_ctx(vgpu);
 686
 687         vgpu->vdev.kvm = NULL;
 688         vgpu->handle = 0;
 689 }
 690
 691 static void intel_vgpu_release(struct mdev_device *mdev)
 692 {
 693         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 694
 695         __intel_vgpu_release(vgpu);
 696 }
 697
 698 static void intel_vgpu_release_work(struct work_struct *work)
 699 {
 700         struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu,
 701                                         vdev.release_work);
 702
 703         __intel_vgpu_release(vgpu);
 704 }
 705
 706 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
 707 {
 708         u32 start_lo, start_hi;
 709         u32 mem_type;
 710
 711         start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 712                         PCI_BASE_ADDRESS_MEM_MASK;
 713         mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
 714                         PCI_BASE_ADDRESS_MEM_TYPE_MASK;
 715
 716         switch (mem_type) {
 717         case PCI_BASE_ADDRESS_MEM_TYPE_64:
 718                 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
 719                                                 + bar + 4));
 720                 break;
 721         case PCI_BASE_ADDRESS_MEM_TYPE_32:
 722         case PCI_BASE_ADDRESS_MEM_TYPE_1M:
 723                 /* 1M mem BAR treated as 32-bit BAR */
 724         default:
 725                 /* mem unknown type treated as 32-bit BAR */
 726                 start_hi = 0;
 727                 break;
 728         }
 729
 730         return ((u64)start_hi << 32) | start_lo;
 731 }
 732
 733 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
 734                              void *buf, unsigned int count, bool is_write)
 735 {
 736         uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
 737         int ret;
 738
 739         if (is_write)
 740                 ret = intel_gvt_ops->emulate_mmio_write(vgpu,
 741                                         bar_start + off, buf, count);
 742         else
 743                 ret = intel_gvt_ops->emulate_mmio_read(vgpu,
 744                                         bar_start + off, buf, count);
 745         return ret;
 746 }
 747
 748 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off)
 749 {
 750         return off >= vgpu_aperture_offset(vgpu) &&
 751                off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
 752 }
 753
 754 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off,
 755                 void *buf, unsigned long count, bool is_write)
 756 {
 757         void *aperture_va;
 758
 759         if (!intel_vgpu_in_aperture(vgpu, off) ||
 760             !intel_vgpu_in_aperture(vgpu, off + count)) {
 761                 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
 762                 return -EINVAL;
 763         }
 764
 765         aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap,
 766                                         ALIGN_DOWN(off, PAGE_SIZE),
 767                                         count + offset_in_page(off));
 768         if (!aperture_va)
 769                 return -EIO;
 770
 771         if (is_write)
 772                 memcpy(aperture_va + offset_in_page(off), buf, count);
 773         else
 774                 memcpy(buf, aperture_va + offset_in_page(off), count);
 775
 776         io_mapping_unmap(aperture_va);
 777
 778         return 0;
 779 }
 780
 781 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
 782                         size_t count, loff_t *ppos, bool is_write)
 783 {
 784         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 785         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 786         uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
 787         int ret = -EINVAL;
 788
 789
 790         if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) {
 791                 gvt_vgpu_err("invalid index: %u\n", index);
 792                 return -EINVAL;
 793         }
 794
 795         switch (index) {
 796         case VFIO_PCI_CONFIG_REGION_INDEX:
 797                 if (is_write)
 798                         ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos,
 799                                                 buf, count);
 800                 else
 801                         ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos,
 802                                                 buf, count);
 803                 break;
 804         case VFIO_PCI_BAR0_REGION_INDEX:
 805                 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
 806                                         buf, count, is_write);
 807                 break;
 808         case VFIO_PCI_BAR2_REGION_INDEX:
 809                 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
 810                 break;
 811         case VFIO_PCI_BAR1_REGION_INDEX:
 812         case VFIO_PCI_BAR3_REGION_INDEX:
 813         case VFIO_PCI_BAR4_REGION_INDEX:
 814         case VFIO_PCI_BAR5_REGION_INDEX:
 815         case VFIO_PCI_VGA_REGION_INDEX:
 816         case VFIO_PCI_ROM_REGION_INDEX:
 817                 break;
 818         default:
 819                 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions)
 820                         return -EINVAL;
 821
 822                 index -= VFIO_PCI_NUM_REGIONS;
 823                 return vgpu->vdev.region[index].ops->rw(vgpu, buf, count,
 824                                 ppos, is_write);
 825         }
 826
 827         return ret == 0 ? count : ret;
 828 }
 829
 830 static bool gtt_entry(struct mdev_device *mdev, loff_t *ppos)
 831 {
 832         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
 833         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 834         struct intel_gvt *gvt = vgpu->gvt;
 835         int offset;
 836
 837         /* Only allow MMIO GGTT entry access */
 838         if (index != PCI_BASE_ADDRESS_0)
 839                 return false;
 840
 841         offset = (u64)(*ppos & VFIO_PCI_OFFSET_MASK) -
 842                 intel_vgpu_get_bar_gpa(vgpu, PCI_BASE_ADDRESS_0);
 843
 844         return (offset >= gvt->device_info.gtt_start_offset &&
 845                 offset < gvt->device_info.gtt_start_offset + gvt_ggtt_sz(gvt)) ?
 846                         true : false;
 847 }
 848
 849 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
 850                         size_t count, loff_t *ppos)
 851 {
 852         unsigned int done = 0;
 853         int ret;
 854
 855         while (count) {
 856                 size_t filled;
 857
 858                 /* Only support GGTT entry 8 bytes read */
 859                 if (count >= 8 && !(*ppos % 8) &&
 860                         gtt_entry(mdev, ppos)) {
 861                         u64 val;
 862
 863                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 864                                         ppos, false);
 865                         if (ret <= 0)
 866                                 goto read_err;
 867
 868                         if (copy_to_user(buf, &val, sizeof(val)))
 869                                 goto read_err;
 870
 871                         filled = 8;
 872                 } else if (count >= 4 && !(*ppos % 4)) {
 873                         u32 val;
 874
 875                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 876                                         ppos, false);
 877                         if (ret <= 0)
 878                                 goto read_err;
 879
 880                         if (copy_to_user(buf, &val, sizeof(val)))
 881                                 goto read_err;
 882
 883                         filled = 4;
 884                 } else if (count >= 2 && !(*ppos % 2)) {
 885                         u16 val;
 886
 887                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 888                                         ppos, false);
 889                         if (ret <= 0)
 890                                 goto read_err;
 891
 892                         if (copy_to_user(buf, &val, sizeof(val)))
 893                                 goto read_err;
 894
 895                         filled = 2;
 896                 } else {
 897                         u8 val;
 898
 899                         ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos,
 900                                         false);
 901                         if (ret <= 0)
 902                                 goto read_err;
 903
 904                         if (copy_to_user(buf, &val, sizeof(val)))
 905                                 goto read_err;
 906
 907                         filled = 1;
 908                 }
 909
 910                 count -= filled;
 911                 done += filled;
 912                 *ppos += filled;
 913                 buf += filled;
 914         }
 915
 916         return done;
 917
 918 read_err:
 919         return -EFAULT;
 920 }
 921
 922 static ssize_t intel_vgpu_write(struct mdev_device *mdev,
 923                                 const char __user *buf,
 924                                 size_t count, loff_t *ppos)
 925 {
 926         unsigned int done = 0;
 927         int ret;
 928
 929         while (count) {
 930                 size_t filled;
 931
 932                 /* Only support GGTT entry 8 bytes write */
 933                 if (count >= 8 && !(*ppos % 8) &&
 934                         gtt_entry(mdev, ppos)) {
 935                         u64 val;
 936
 937                         if (copy_from_user(&val, buf, sizeof(val)))
 938                                 goto write_err;
 939
 940                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 941                                         ppos, true);
 942                         if (ret <= 0)
 943                                 goto write_err;
 944
 945                         filled = 8;
 946                 } else if (count >= 4 && !(*ppos % 4)) {
 947                         u32 val;
 948
 949                         if (copy_from_user(&val, buf, sizeof(val)))
 950                                 goto write_err;
 951
 952                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
 953                                         ppos, true);
 954                         if (ret <= 0)
 955                                 goto write_err;
 956
 957                         filled = 4;
 958                 } else if (count >= 2 && !(*ppos % 2)) {
 959                         u16 val;
 960
 961                         if (copy_from_user(&val, buf, sizeof(val)))
 962                                 goto write_err;
 963
 964                         ret = intel_vgpu_rw(mdev, (char *)&val,
 965                                         sizeof(val), ppos, true);
 966                         if (ret <= 0)
 967                                 goto write_err;
 968
 969                         filled = 2;
 970                 } else {
 971                         u8 val;
 972
 973                         if (copy_from_user(&val, buf, sizeof(val)))
 974                                 goto write_err;
 975
 976                         ret = intel_vgpu_rw(mdev, &val, sizeof(val),
 977                                         ppos, true);
 978                         if (ret <= 0)
 979                                 goto write_err;
 980
 981                         filled = 1;
 982                 }
 983
 984                 count -= filled;
 985                 done += filled;
 986                 *ppos += filled;
 987                 buf += filled;
 988         }
 989
 990         return done;
 991 write_err:
 992         return -EFAULT;
 993 }
 994
 995 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma)
 996 {
 997         unsigned int index;
 998         u64 virtaddr;
 999         unsigned long req_size, pgoff = 0;
1000         pgprot_t pg_prot;
1001         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1002
1003         index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
1004         if (index >= VFIO_PCI_ROM_REGION_INDEX)
1005                 return -EINVAL;
1006
1007         if (vma->vm_end < vma->vm_start)
1008                 return -EINVAL;
1009         if ((vma->vm_flags & VM_SHARED) == 0)
1010                 return -EINVAL;
1011         if (index != VFIO_PCI_BAR2_REGION_INDEX)
1012                 return -EINVAL;
1013
1014         pg_prot = vma->vm_page_prot;
1015         virtaddr = vma->vm_start;
1016         req_size = vma->vm_end - vma->vm_start;
1017         pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
1018
1019         return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
1020 }
1021
1022 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
1023 {
1024         if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
1025                 return 1;
1026
1027         return 0;
1028 }
1029
1030 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
1031                         unsigned int index, unsigned int start,
1032                         unsigned int count, uint32_t flags,
1033                         void *data)
1034 {
1035         return 0;
1036 }
1037
1038 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
1039                         unsigned int index, unsigned int start,
1040                         unsigned int count, uint32_t flags, void *data)
1041 {
1042         return 0;
1043 }
1044
1045 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
1046                 unsigned int index, unsigned int start, unsigned int count,
1047                 uint32_t flags, void *data)
1048 {
1049         return 0;
1050 }
1051
1052 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
1053                 unsigned int index, unsigned int start, unsigned int count,
1054                 uint32_t flags, void *data)
1055 {
1056         struct eventfd_ctx *trigger;
1057
1058         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1059                 int fd = *(int *)data;
1060
1061                 trigger = eventfd_ctx_fdget(fd);
1062                 if (IS_ERR(trigger)) {
1063                         gvt_vgpu_err("eventfd_ctx_fdget failed\n");
1064                         return PTR_ERR(trigger);
1065                 }
1066                 vgpu->vdev.msi_trigger = trigger;
1067         } else if ((flags & VFIO_IRQ_SET_DATA_NONE) && !count)
1068                 intel_vgpu_release_msi_eventfd_ctx(vgpu);
1069
1070         return 0;
1071 }
1072
1073 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags,
1074                 unsigned int index, unsigned int start, unsigned int count,
1075                 void *data)
1076 {
1077         int (*func)(struct intel_vgpu *vgpu, unsigned int index,
1078                         unsigned int start, unsigned int count, uint32_t flags,
1079                         void *data) = NULL;
1080
1081         switch (index) {
1082         case VFIO_PCI_INTX_IRQ_INDEX:
1083                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1084                 case VFIO_IRQ_SET_ACTION_MASK:
1085                         func = intel_vgpu_set_intx_mask;
1086                         break;
1087                 case VFIO_IRQ_SET_ACTION_UNMASK:
1088                         func = intel_vgpu_set_intx_unmask;
1089                         break;
1090                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1091                         func = intel_vgpu_set_intx_trigger;
1092                         break;
1093                 }
1094                 break;
1095         case VFIO_PCI_MSI_IRQ_INDEX:
1096                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1097                 case VFIO_IRQ_SET_ACTION_MASK:
1098                 case VFIO_IRQ_SET_ACTION_UNMASK:
1099                         /* XXX Need masking support exported */
1100                         break;
1101                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1102                         func = intel_vgpu_set_msi_trigger;
1103                         break;
1104                 }
1105                 break;
1106         }
1107
1108         if (!func)
1109                 return -ENOTTY;
1110
1111         return func(vgpu, index, start, count, flags, data);
1112 }
1113
1114 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
1115                              unsigned long arg)
1116 {
1117         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
1118         unsigned long minsz;
1119
1120         gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
1121
1122         if (cmd == VFIO_DEVICE_GET_INFO) {
1123                 struct vfio_device_info info;
1124
1125                 minsz = offsetofend(struct vfio_device_info, num_irqs);
1126
1127                 if (copy_from_user(&info, (void __user *)arg, minsz))
1128                         return -EFAULT;
1129
1130                 if (info.argsz < minsz)
1131                         return -EINVAL;
1132
1133                 info.flags = VFIO_DEVICE_FLAGS_PCI;
1134                 info.flags |= VFIO_DEVICE_FLAGS_RESET;
1135                 info.num_regions = VFIO_PCI_NUM_REGIONS +
1136                                 vgpu->vdev.num_regions;
1137                 info.num_irqs = VFIO_PCI_NUM_IRQS;
1138
1139                 return copy_to_user((void __user *)arg, &info, minsz) ?
1140                         -EFAULT : 0;
1141
1142         } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1143                 struct vfio_region_info info;
1144                 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1145                 unsigned int i;
1146                 int ret;
1147                 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1148                 size_t size;
1149                 int nr_areas = 1;
1150                 int cap_type_id;
1151
1152                 minsz = offsetofend(struct vfio_region_info, offset);
1153
1154                 if (copy_from_user(&info, (void __user *)arg, minsz))
1155                         return -EFAULT;
1156
1157                 if (info.argsz < minsz)
1158                         return -EINVAL;
1159
1160                 switch (info.index) {
1161                 case VFIO_PCI_CONFIG_REGION_INDEX:
1162                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1163                         info.size = vgpu->gvt->device_info.cfg_space_size;
1164                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1165                                      VFIO_REGION_INFO_FLAG_WRITE;
1166                         break;
1167                 case VFIO_PCI_BAR0_REGION_INDEX:
1168                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1169                         info.size = vgpu->cfg_space.bar[info.index].size;
1170                         if (!info.size) {
1171                                 info.flags = 0;
1172                                 break;
1173                         }
1174
1175                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1176                                      VFIO_REGION_INFO_FLAG_WRITE;
1177                         break;
1178                 case VFIO_PCI_BAR1_REGION_INDEX:
1179                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1180                         info.size = 0;
1181                         info.flags = 0;
1182                         break;
1183                 case VFIO_PCI_BAR2_REGION_INDEX:
1184                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1185                         info.flags = VFIO_REGION_INFO_FLAG_CAPS |
1186                                         VFIO_REGION_INFO_FLAG_MMAP |
1187                                         VFIO_REGION_INFO_FLAG_READ |
1188                                         VFIO_REGION_INFO_FLAG_WRITE;
1189                         info.size = gvt_aperture_sz(vgpu->gvt);
1190
1191                         size = sizeof(*sparse) +
1192                                         (nr_areas * sizeof(*sparse->areas));
1193                         sparse = kzalloc(size, GFP_KERNEL);
1194                         if (!sparse)
1195                                 return -ENOMEM;
1196
1197                         sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1198                         sparse->header.version = 1;
1199                         sparse->nr_areas = nr_areas;
1200                         cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1201                         sparse->areas[0].offset =
1202                                         PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1203                         sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1204                         break;
1205
1206                 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1207                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1208                         info.size = 0;
1209                         info.flags = 0;
1210
1211                         gvt_dbg_core("get region info bar:%d\n", info.index);
1212                         break;
1213
1214                 case VFIO_PCI_ROM_REGION_INDEX:
1215                 case VFIO_PCI_VGA_REGION_INDEX:
1216                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1217                         info.size = 0;
1218                         info.flags = 0;
1219
1220                         gvt_dbg_core("get region info index:%d\n", info.index);
1221                         break;
1222                 default:
1223                         {
1224                                 struct vfio_region_info_cap_type cap_type = {
1225                                         .header.id = VFIO_REGION_INFO_CAP_TYPE,
1226                                         .header.version = 1 };
1227
1228                                 if (info.index >= VFIO_PCI_NUM_REGIONS +
1229                                                 vgpu->vdev.num_regions)
1230                                         return -EINVAL;
1231                                 info.index =
1232                                         array_index_nospec(info.index,
1233                                                         VFIO_PCI_NUM_REGIONS +
1234                                                         vgpu->vdev.num_regions);
1235
1236                                 i = info.index - VFIO_PCI_NUM_REGIONS;
1237
1238                                 info.offset =
1239                                         VFIO_PCI_INDEX_TO_OFFSET(info.index);
1240                                 info.size = vgpu->vdev.region[i].size;
1241                                 info.flags = vgpu->vdev.region[i].flags;
1242
1243                                 cap_type.type = vgpu->vdev.region[i].type;
1244                                 cap_type.subtype = vgpu->vdev.region[i].subtype;
1245
1246                                 ret = vfio_info_add_capability(&caps,
1247                                                         &cap_type.header,
1248                                                         sizeof(cap_type));
1249                                 if (ret)
1250                                         return ret;
1251                         }
1252                 }
1253
1254                 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1255                         switch (cap_type_id) {
1256                         case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
1257                                 ret = vfio_info_add_capability(&caps,
1258                                         &sparse->header, sizeof(*sparse) +
1259                                         (sparse->nr_areas *
1260                                                 sizeof(*sparse->areas)));
1261                                 if (ret) {
1262                                         kfree(sparse);
1263                                         return ret;
1264                                 }
1265                                 break;
1266                         default:
1267                                 kfree(sparse);
1268                                 return -EINVAL;
1269                         }
1270                 }
1271
1272                 if (caps.size) {
1273                         info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1274                         if (info.argsz < sizeof(info) + caps.size) {
1275                                 info.argsz = sizeof(info) + caps.size;
1276                                 info.cap_offset = 0;
1277                         } else {
1278                                 vfio_info_cap_shift(&caps, sizeof(info));
1279                                 if (copy_to_user((void __user *)arg +
1280                                                   sizeof(info), caps.buf,
1281                                                   caps.size)) {
1282                                         kfree(caps.buf);
1283                                         kfree(sparse);
1284                                         return -EFAULT;
1285                                 }
1286                                 info.cap_offset = sizeof(info);
1287                         }
1288
1289                         kfree(caps.buf);
1290                 }
1291
1292                 kfree(sparse);
1293                 return copy_to_user((void __user *)arg, &info, minsz) ?
1294                         -EFAULT : 0;
1295         } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1296                 struct vfio_irq_info info;
1297
1298                 minsz = offsetofend(struct vfio_irq_info, count);
1299
1300                 if (copy_from_user(&info, (void __user *)arg, minsz))
1301                         return -EFAULT;
1302
1303                 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1304                         return -EINVAL;
1305
1306                 switch (info.index) {
1307                 case VFIO_PCI_INTX_IRQ_INDEX:
1308                 case VFIO_PCI_MSI_IRQ_INDEX:
1309                         break;
1310                 default:
1311                         return -EINVAL;
1312                 }
1313
1314                 info.flags = VFIO_IRQ_INFO_EVENTFD;
1315
1316                 info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1317
1318                 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1319                         info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1320                                        VFIO_IRQ_INFO_AUTOMASKED);
1321                 else
1322                         info.flags |= VFIO_IRQ_INFO_NORESIZE;
1323
1324                 return copy_to_user((void __user *)arg, &info, minsz) ?
1325                         -EFAULT : 0;
1326         } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1327                 struct vfio_irq_set hdr;
1328                 u8 *data = NULL;
1329                 int ret = 0;
1330                 size_t data_size = 0;
1331
1332                 minsz = offsetofend(struct vfio_irq_set, count);
1333
1334                 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1335                         return -EFAULT;
1336
1337                 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1338                         int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1339
1340                         ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1341                                                 VFIO_PCI_NUM_IRQS, &data_size);
1342                         if (ret) {
1343                                 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1344                                 return -EINVAL;
1345                         }
1346                         if (data_size) {
1347                                 data = memdup_user((void __user *)(arg + minsz),
1348                                                    data_size);
1349                                 if (IS_ERR(data))
1350                                         return PTR_ERR(data);
1351                         }
1352                 }
1353
1354                 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1355                                         hdr.start, hdr.count, data);
1356                 kfree(data);
1357
1358                 return ret;
1359         } else if (cmd == VFIO_DEVICE_RESET) {
1360                 intel_gvt_ops->vgpu_reset(vgpu);
1361                 return 0;
1362         } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1363                 struct vfio_device_gfx_plane_info dmabuf;
1364                 int ret = 0;
1365
1366                 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1367                                     dmabuf_id);
1368                 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1369                         return -EFAULT;
1370                 if (dmabuf.argsz < minsz)
1371                         return -EINVAL;
1372
1373                 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf);
1374                 if (ret != 0)
1375                         return ret;
1376
1377                 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1378                                                                 -EFAULT : 0;
1379         } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1380                 __u32 dmabuf_id;
1381                 __s32 dmabuf_fd;
1382
1383                 if (get_user(dmabuf_id, (__u32 __user *)arg))
1384                         return -EFAULT;
1385
1386                 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id);
1387                 return dmabuf_fd;
1388
1389         }
1390
1391         return -ENOTTY;
1392 }
1393
1394 static ssize_t
1395 vgpu_id_show(struct device *dev, struct device_attribute *attr,
1396              char *buf)
1397 {
1398         struct mdev_device *mdev = mdev_from_dev(dev);
1399
1400         if (mdev) {
1401                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1402                         mdev_get_drvdata(mdev);
1403                 return sprintf(buf, "%d\n", vgpu->id);
1404         }
1405         return sprintf(buf, "\n");
1406 }
1407
1408 static ssize_t
1409 hw_id_show(struct device *dev, struct device_attribute *attr,
1410            char *buf)
1411 {
1412         struct mdev_device *mdev = mdev_from_dev(dev);
1413
1414         if (mdev) {
1415                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1416                         mdev_get_drvdata(mdev);
1417                 return sprintf(buf, "%u\n",
1418                                vgpu->submission.shadow_ctx->hw_id);
1419         }
1420         return sprintf(buf, "\n");
1421 }
1422
1423 static DEVICE_ATTR_RO(vgpu_id);
1424 static DEVICE_ATTR_RO(hw_id);
1425
1426 static struct attribute *intel_vgpu_attrs[] = {
1427         &dev_attr_vgpu_id.attr,
1428         &dev_attr_hw_id.attr,
1429         NULL
1430 };
1431
1432 static const struct attribute_group intel_vgpu_group = {
1433         .name = "intel_vgpu",
1434         .attrs = intel_vgpu_attrs,
1435 };
1436
1437 static const struct attribute_group *intel_vgpu_groups[] = {
1438         &intel_vgpu_group,
1439         NULL,
1440 };
1441
1442 static struct mdev_parent_ops intel_vgpu_ops = {
1443         .mdev_attr_groups       = intel_vgpu_groups,
1444         .create                 = intel_vgpu_create,
1445         .remove                 = intel_vgpu_remove,
1446
1447         .open                   = intel_vgpu_open,
1448         .release                = intel_vgpu_release,
1449
1450         .read                   = intel_vgpu_read,
1451         .write                  = intel_vgpu_write,
1452         .mmap                   = intel_vgpu_mmap,
1453         .ioctl                  = intel_vgpu_ioctl,
1454 };
1455
1456 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
1457 {
1458         struct attribute **kvm_type_attrs;
1459         struct attribute_group **kvm_vgpu_type_groups;
1460
1461         intel_gvt_ops = ops;
1462         if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs,
1463                         &kvm_vgpu_type_groups))
1464                 return -EFAULT;
1465         intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
1466
1467         return mdev_register_device(dev, &intel_vgpu_ops);
1468 }
1469
1470 static void kvmgt_host_exit(struct device *dev, void *gvt)
1471 {
1472         mdev_unregister_device(dev);
1473 }
1474
1475 static int kvmgt_page_track_add(unsigned long handle, u64 gfn)
1476 {
1477         struct kvmgt_guest_info *info;
1478         struct kvm *kvm;
1479         struct kvm_memory_slot *slot;
1480         int idx;
1481
1482         if (!handle_valid(handle))
1483                 return -ESRCH;
1484
1485         info = (struct kvmgt_guest_info *)handle;
1486         kvm = info->kvm;
1487
1488         idx = srcu_read_lock(&kvm->srcu);
1489         slot = gfn_to_memslot(kvm, gfn);
1490         if (!slot) {
1491                 srcu_read_unlock(&kvm->srcu, idx);
1492                 return -EINVAL;
1493         }
1494
1495         spin_lock(&kvm->mmu_lock);
1496
1497         if (kvmgt_gfn_is_write_protected(info, gfn))
1498                 goto out;
1499
1500         kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1501         kvmgt_protect_table_add(info, gfn);
1502
1503 out:
1504         spin_unlock(&kvm->mmu_lock);
1505         srcu_read_unlock(&kvm->srcu, idx);
1506         return 0;
1507 }
1508
1509 static int kvmgt_page_track_remove(unsigned long handle, u64 gfn)
1510 {
1511         struct kvmgt_guest_info *info;
1512         struct kvm *kvm;
1513         struct kvm_memory_slot *slot;
1514         int idx;
1515
1516         if (!handle_valid(handle))
1517                 return 0;
1518
1519         info = (struct kvmgt_guest_info *)handle;
1520         kvm = info->kvm;
1521
1522         idx = srcu_read_lock(&kvm->srcu);
1523         slot = gfn_to_memslot(kvm, gfn);
1524         if (!slot) {
1525                 srcu_read_unlock(&kvm->srcu, idx);
1526                 return -EINVAL;
1527         }
1528
1529         spin_lock(&kvm->mmu_lock);
1530
1531         if (!kvmgt_gfn_is_write_protected(info, gfn))
1532                 goto out;
1533
1534         kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1535         kvmgt_protect_table_del(info, gfn);
1536
1537 out:
1538         spin_unlock(&kvm->mmu_lock);
1539         srcu_read_unlock(&kvm->srcu, idx);
1540         return 0;
1541 }
1542
1543 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1544                 const u8 *val, int len,
1545                 struct kvm_page_track_notifier_node *node)
1546 {
1547         struct kvmgt_guest_info *info = container_of(node,
1548                                         struct kvmgt_guest_info, track_node);
1549
1550         if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
1551                 intel_gvt_ops->write_protect_handler(info->vgpu, gpa,
1552                                                      (void *)val, len);
1553 }
1554
1555 static void kvmgt_page_track_flush_slot(struct kvm *kvm,
1556                 struct kvm_memory_slot *slot,
1557                 struct kvm_page_track_notifier_node *node)
1558 {
1559         int i;
1560         gfn_t gfn;
1561         struct kvmgt_guest_info *info = container_of(node,
1562                                         struct kvmgt_guest_info, track_node);
1563
1564         spin_lock(&kvm->mmu_lock);
1565         for (i = 0; i < slot->npages; i++) {
1566                 gfn = slot->base_gfn + i;
1567                 if (kvmgt_gfn_is_write_protected(info, gfn)) {
1568                         kvm_slot_page_track_remove_page(kvm, slot, gfn,
1569                                                 KVM_PAGE_TRACK_WRITE);
1570                         kvmgt_protect_table_del(info, gfn);
1571                 }
1572         }
1573         spin_unlock(&kvm->mmu_lock);
1574 }
1575
1576 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm)
1577 {
1578         struct intel_vgpu *itr;
1579         struct kvmgt_guest_info *info;
1580         int id;
1581         bool ret = false;
1582
1583         mutex_lock(&vgpu->gvt->lock);
1584         for_each_active_vgpu(vgpu->gvt, itr, id) {
1585                 if (!handle_valid(itr->handle))
1586                         continue;
1587
1588                 info = (struct kvmgt_guest_info *)itr->handle;
1589                 if (kvm && kvm == info->kvm) {
1590                         ret = true;
1591                         goto out;
1592                 }
1593         }
1594 out:
1595         mutex_unlock(&vgpu->gvt->lock);
1596         return ret;
1597 }
1598
1599 static int kvmgt_guest_init(struct mdev_device *mdev)
1600 {
1601         struct kvmgt_guest_info *info;
1602         struct intel_vgpu *vgpu;
1603         struct kvm *kvm;
1604
1605         vgpu = mdev_get_drvdata(mdev);
1606         if (handle_valid(vgpu->handle))
1607                 return -EEXIST;
1608
1609         kvm = vgpu->vdev.kvm;
1610         if (!kvm || kvm->mm != current->mm) {
1611                 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
1612                 return -ESRCH;
1613         }
1614
1615         if (__kvmgt_vgpu_exist(vgpu, kvm))
1616                 return -EEXIST;
1617
1618         info = vzalloc(sizeof(struct kvmgt_guest_info));
1619         if (!info)
1620                 return -ENOMEM;
1621
1622         vgpu->handle = (unsigned long)info;
1623         info->vgpu = vgpu;
1624         info->kvm = kvm;
1625         kvm_get_kvm(info->kvm);
1626
1627         kvmgt_protect_table_init(info);
1628         gvt_cache_init(vgpu);
1629
1630         init_completion(&vgpu->vblank_done);
1631
1632         info->track_node.track_write = kvmgt_page_track_write;
1633         info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
1634         kvm_page_track_register_notifier(kvm, &info->track_node);
1635
1636         info->debugfs_cache_entries = debugfs_create_ulong(
1637                                                 "kvmgt_nr_cache_entries",
1638                                                 0444, vgpu->debugfs,
1639                                                 &vgpu->vdev.nr_cache_entries);
1640         if (!info->debugfs_cache_entries)
1641                 gvt_vgpu_err("Cannot create kvmgt debugfs entry\n");
1642
1643         return 0;
1644 }
1645
1646 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info)
1647 {
1648         debugfs_remove(info->debugfs_cache_entries);
1649
1650         kvm_page_track_unregister_notifier(info->kvm, &info->track_node);
1651         kvm_put_kvm(info->kvm);
1652         kvmgt_protect_table_destroy(info);
1653         gvt_cache_destroy(info->vgpu);
1654         vfree(info);
1655
1656         return true;
1657 }
1658
1659 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle)
1660 {
1661         /* nothing to do here */
1662         return 0;
1663 }
1664
1665 static void kvmgt_detach_vgpu(unsigned long handle)
1666 {
1667         /* nothing to do here */
1668 }
1669
1670 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
1671 {
1672         struct kvmgt_guest_info *info;
1673         struct intel_vgpu *vgpu;
1674
1675         if (!handle_valid(handle))
1676                 return -ESRCH;
1677
1678         info = (struct kvmgt_guest_info *)handle;
1679         vgpu = info->vgpu;
1680
1681         /*
1682          * When guest is poweroff, msi_trigger is set to NULL, but vgpu's
1683          * config and mmio register isn't restored to default during guest
1684          * poweroff. If this vgpu is still used in next vm, this vgpu's pipe
1685          * may be enabled, then once this vgpu is active, it will get inject
1686          * vblank interrupt request. But msi_trigger is null until msi is
1687          * enabled by guest. so if msi_trigger is null, success is still
1688          * returned and don't inject interrupt into guest.
1689          */
1690         if (vgpu->vdev.msi_trigger == NULL)
1691                 return 0;
1692
1693         if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1)
1694                 return 0;
1695
1696         return -EFAULT;
1697 }
1698
1699 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
1700 {
1701         struct kvmgt_guest_info *info;
1702         kvm_pfn_t pfn;
1703
1704         if (!handle_valid(handle))
1705                 return INTEL_GVT_INVALID_ADDR;
1706
1707         info = (struct kvmgt_guest_info *)handle;
1708
1709         pfn = gfn_to_pfn(info->kvm, gfn);
1710         if (is_error_noslot_pfn(pfn))
1711                 return INTEL_GVT_INVALID_ADDR;
1712
1713         return pfn;
1714 }
1715
1716 static int kvmgt_dma_map_guest_page(unsigned long handle, unsigned long gfn,
1717                 unsigned long size, dma_addr_t *dma_addr)
1718 {
1719         struct kvmgt_guest_info *info;
1720         struct intel_vgpu *vgpu;
1721         struct gvt_dma *entry;
1722         int ret;
1723
1724         if (!handle_valid(handle))
1725                 return -EINVAL;
1726
1727         info = (struct kvmgt_guest_info *)handle;
1728         vgpu = info->vgpu;
1729
1730         mutex_lock(&info->vgpu->vdev.cache_lock);
1731
1732         entry = __gvt_cache_find_gfn(info->vgpu, gfn);
1733         if (!entry) {
1734                 ret = gvt_dma_map_page(vgpu, gfn, dma_addr, size);
1735                 if (ret)
1736                         goto err_unlock;
1737
1738                 ret = __gvt_cache_add(info->vgpu, gfn, *dma_addr, size);
1739                 if (ret)
1740                         goto err_unmap;
1741         } else {
1742                 kref_get(&entry->ref);
1743                 *dma_addr = entry->dma_addr;
1744         }
1745
1746         mutex_unlock(&info->vgpu->vdev.cache_lock);
1747         return 0;
1748
1749 err_unmap:
1750         gvt_dma_unmap_page(vgpu, gfn, *dma_addr, size);
1751 err_unlock:
1752         mutex_unlock(&info->vgpu->vdev.cache_lock);
1753         return ret;
1754 }
1755
1756 static void __gvt_dma_release(struct kref *ref)
1757 {
1758         struct gvt_dma *entry = container_of(ref, typeof(*entry), ref);
1759
1760         gvt_dma_unmap_page(entry->vgpu, entry->gfn, entry->dma_addr,
1761                            entry->size);
1762         __gvt_cache_remove_entry(entry->vgpu, entry);
1763 }
1764
1765 static void kvmgt_dma_unmap_guest_page(unsigned long handle, dma_addr_t dma_addr)
1766 {
1767         struct kvmgt_guest_info *info;
1768         struct gvt_dma *entry;
1769
1770         if (!handle_valid(handle))
1771                 return;
1772
1773         info = (struct kvmgt_guest_info *)handle;
1774
1775         mutex_lock(&info->vgpu->vdev.cache_lock);
1776         entry = __gvt_cache_find_dma_addr(info->vgpu, dma_addr);
1777         if (entry)
1778                 kref_put(&entry->ref, __gvt_dma_release);
1779         mutex_unlock(&info->vgpu->vdev.cache_lock);
1780 }
1781
1782 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
1783                         void *buf, unsigned long len, bool write)
1784 {
1785         struct kvmgt_guest_info *info;
1786         struct kvm *kvm;
1787         int idx, ret;
1788         bool kthread = current->mm == NULL;
1789
1790         if (!handle_valid(handle))
1791                 return -ESRCH;
1792
1793         info = (struct kvmgt_guest_info *)handle;
1794         kvm = info->kvm;
1795
1796         if (kthread) {
1797                 if (!mmget_not_zero(kvm->mm))
1798                         return -EFAULT;
1799                 use_mm(kvm->mm);
1800         }
1801
1802         idx = srcu_read_lock(&kvm->srcu);
1803         ret = write ? kvm_write_guest(kvm, gpa, buf, len) :
1804                       kvm_read_guest(kvm, gpa, buf, len);
1805         srcu_read_unlock(&kvm->srcu, idx);
1806
1807         if (kthread) {
1808                 unuse_mm(kvm->mm);
1809                 mmput(kvm->mm);
1810         }
1811
1812         return ret;
1813 }
1814
1815 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
1816                         void *buf, unsigned long len)
1817 {
1818         return kvmgt_rw_gpa(handle, gpa, buf, len, false);
1819 }
1820
1821 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
1822                         void *buf, unsigned long len)
1823 {
1824         return kvmgt_rw_gpa(handle, gpa, buf, len, true);
1825 }
1826
1827 static unsigned long kvmgt_virt_to_pfn(void *addr)
1828 {
1829         return PFN_DOWN(__pa(addr));
1830 }
1831
1832 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn)
1833 {
1834         struct kvmgt_guest_info *info;
1835         struct kvm *kvm;
1836         int idx;
1837         bool ret;
1838
1839         if (!handle_valid(handle))
1840                 return false;
1841
1842         info = (struct kvmgt_guest_info *)handle;
1843         kvm = info->kvm;
1844
1845         idx = srcu_read_lock(&kvm->srcu);
1846         ret = kvm_is_visible_gfn(kvm, gfn);
1847         srcu_read_unlock(&kvm->srcu, idx);
1848
1849         return ret;
1850 }
1851
1852 struct intel_gvt_mpt kvmgt_mpt = {
1853         .host_init = kvmgt_host_init,
1854         .host_exit = kvmgt_host_exit,
1855         .attach_vgpu = kvmgt_attach_vgpu,
1856         .detach_vgpu = kvmgt_detach_vgpu,
1857         .inject_msi = kvmgt_inject_msi,
1858         .from_virt_to_mfn = kvmgt_virt_to_pfn,
1859         .enable_page_track = kvmgt_page_track_add,
1860         .disable_page_track = kvmgt_page_track_remove,
1861         .read_gpa = kvmgt_read_gpa,
1862         .write_gpa = kvmgt_write_gpa,
1863         .gfn_to_mfn = kvmgt_gfn_to_pfn,
1864         .dma_map_guest_page = kvmgt_dma_map_guest_page,
1865         .dma_unmap_guest_page = kvmgt_dma_unmap_guest_page,
1866         .set_opregion = kvmgt_set_opregion,
1867         .get_vfio_device = kvmgt_get_vfio_device,
1868         .put_vfio_device = kvmgt_put_vfio_device,
1869         .is_valid_gfn = kvmgt_is_valid_gfn,
1870 };
1871 EXPORT_SYMBOL_GPL(kvmgt_mpt);
1872
1873 static int __init kvmgt_init(void)
1874 {
1875         return 0;
1876 }
1877
1878 static void __exit kvmgt_exit(void)
1879 {
1880 }
1881
1882 module_init(kvmgt_init);
1883 module_exit(kvmgt_exit);
1884
1885 MODULE_LICENSE("GPL and additional rights");
1886 MODULE_AUTHOR("Intel Corporation");