Merge tag 'drm-for-v4.16-part2-fixes' of git://people.freedesktop.org/~airlied/linux
[sfrench/cifs-2.6.git] / drivers / gpu / drm / i915 / gvt / kvmgt.c
1 /*
2  * KVMGT - the implementation of Intel mediated pass-through framework for KVM
3  *
4  * Copyright(c) 2014-2016 Intel Corporation. All rights reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the next
14  * paragraph) shall be included in all copies or substantial portions of the
15  * Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23  * SOFTWARE.
24  *
25  * Authors:
26  *    Kevin Tian <kevin.tian@intel.com>
27  *    Jike Song <jike.song@intel.com>
28  *    Xiaoguang Chen <xiaoguang.chen@intel.com>
29  */
30
31 #include <linux/init.h>
32 #include <linux/device.h>
33 #include <linux/mm.h>
34 #include <linux/mmu_context.h>
35 #include <linux/types.h>
36 #include <linux/list.h>
37 #include <linux/rbtree.h>
38 #include <linux/spinlock.h>
39 #include <linux/eventfd.h>
40 #include <linux/uuid.h>
41 #include <linux/kvm_host.h>
42 #include <linux/vfio.h>
43 #include <linux/mdev.h>
44
45 #include "i915_drv.h"
46 #include "gvt.h"
47
48 static const struct intel_gvt_ops *intel_gvt_ops;
49
50 /* helper macros copied from vfio-pci */
51 #define VFIO_PCI_OFFSET_SHIFT   40
52 #define VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> VFIO_PCI_OFFSET_SHIFT)
53 #define VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << VFIO_PCI_OFFSET_SHIFT)
54 #define VFIO_PCI_OFFSET_MASK    (((u64)(1) << VFIO_PCI_OFFSET_SHIFT) - 1)
55
56 #define OPREGION_SIGNATURE "IntelGraphicsMem"
57
58 struct vfio_region;
59 struct intel_vgpu_regops {
60         size_t (*rw)(struct intel_vgpu *vgpu, char *buf,
61                         size_t count, loff_t *ppos, bool iswrite);
62         void (*release)(struct intel_vgpu *vgpu,
63                         struct vfio_region *region);
64 };
65
66 struct vfio_region {
67         u32                             type;
68         u32                             subtype;
69         size_t                          size;
70         u32                             flags;
71         const struct intel_vgpu_regops  *ops;
72         void                            *data;
73 };
74
75 struct kvmgt_pgfn {
76         gfn_t gfn;
77         struct hlist_node hnode;
78 };
79
80 struct kvmgt_guest_info {
81         struct kvm *kvm;
82         struct intel_vgpu *vgpu;
83         struct kvm_page_track_notifier_node track_node;
84 #define NR_BKT (1 << 18)
85         struct hlist_head ptable[NR_BKT];
86 #undef NR_BKT
87 };
88
89 struct gvt_dma {
90         struct rb_node node;
91         gfn_t gfn;
92         unsigned long iova;
93 };
94
95 static inline bool handle_valid(unsigned long handle)
96 {
97         return !!(handle & ~0xff);
98 }
99
100 static int kvmgt_guest_init(struct mdev_device *mdev);
101 static void intel_vgpu_release_work(struct work_struct *work);
102 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info);
103
104 static int gvt_dma_map_iova(struct intel_vgpu *vgpu, kvm_pfn_t pfn,
105                 unsigned long *iova)
106 {
107         struct page *page;
108         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
109         dma_addr_t daddr;
110
111         if (unlikely(!pfn_valid(pfn)))
112                 return -EFAULT;
113
114         page = pfn_to_page(pfn);
115         daddr = dma_map_page(dev, page, 0, PAGE_SIZE,
116                         PCI_DMA_BIDIRECTIONAL);
117         if (dma_mapping_error(dev, daddr))
118                 return -ENOMEM;
119
120         *iova = (unsigned long)(daddr >> PAGE_SHIFT);
121         return 0;
122 }
123
124 static void gvt_dma_unmap_iova(struct intel_vgpu *vgpu, unsigned long iova)
125 {
126         struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
127         dma_addr_t daddr;
128
129         daddr = (dma_addr_t)(iova << PAGE_SHIFT);
130         dma_unmap_page(dev, daddr, PAGE_SIZE, PCI_DMA_BIDIRECTIONAL);
131 }
132
133 static struct gvt_dma *__gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn)
134 {
135         struct rb_node *node = vgpu->vdev.cache.rb_node;
136         struct gvt_dma *ret = NULL;
137
138         while (node) {
139                 struct gvt_dma *itr = rb_entry(node, struct gvt_dma, node);
140
141                 if (gfn < itr->gfn)
142                         node = node->rb_left;
143                 else if (gfn > itr->gfn)
144                         node = node->rb_right;
145                 else {
146                         ret = itr;
147                         goto out;
148                 }
149         }
150
151 out:
152         return ret;
153 }
154
155 static unsigned long gvt_cache_find(struct intel_vgpu *vgpu, gfn_t gfn)
156 {
157         struct gvt_dma *entry;
158         unsigned long iova;
159
160         mutex_lock(&vgpu->vdev.cache_lock);
161
162         entry = __gvt_cache_find(vgpu, gfn);
163         iova = (entry == NULL) ? INTEL_GVT_INVALID_ADDR : entry->iova;
164
165         mutex_unlock(&vgpu->vdev.cache_lock);
166         return iova;
167 }
168
169 static void gvt_cache_add(struct intel_vgpu *vgpu, gfn_t gfn,
170                 unsigned long iova)
171 {
172         struct gvt_dma *new, *itr;
173         struct rb_node **link = &vgpu->vdev.cache.rb_node, *parent = NULL;
174
175         new = kzalloc(sizeof(struct gvt_dma), GFP_KERNEL);
176         if (!new)
177                 return;
178
179         new->gfn = gfn;
180         new->iova = iova;
181
182         mutex_lock(&vgpu->vdev.cache_lock);
183         while (*link) {
184                 parent = *link;
185                 itr = rb_entry(parent, struct gvt_dma, node);
186
187                 if (gfn == itr->gfn)
188                         goto out;
189                 else if (gfn < itr->gfn)
190                         link = &parent->rb_left;
191                 else
192                         link = &parent->rb_right;
193         }
194
195         rb_link_node(&new->node, parent, link);
196         rb_insert_color(&new->node, &vgpu->vdev.cache);
197         mutex_unlock(&vgpu->vdev.cache_lock);
198         return;
199
200 out:
201         mutex_unlock(&vgpu->vdev.cache_lock);
202         kfree(new);
203 }
204
205 static void __gvt_cache_remove_entry(struct intel_vgpu *vgpu,
206                                 struct gvt_dma *entry)
207 {
208         rb_erase(&entry->node, &vgpu->vdev.cache);
209         kfree(entry);
210 }
211
212 static void gvt_cache_remove(struct intel_vgpu *vgpu, gfn_t gfn)
213 {
214         struct device *dev = mdev_dev(vgpu->vdev.mdev);
215         struct gvt_dma *this;
216         unsigned long g1;
217         int rc;
218
219         mutex_lock(&vgpu->vdev.cache_lock);
220         this  = __gvt_cache_find(vgpu, gfn);
221         if (!this) {
222                 mutex_unlock(&vgpu->vdev.cache_lock);
223                 return;
224         }
225
226         g1 = gfn;
227         gvt_dma_unmap_iova(vgpu, this->iova);
228         rc = vfio_unpin_pages(dev, &g1, 1);
229         WARN_ON(rc != 1);
230         __gvt_cache_remove_entry(vgpu, this);
231         mutex_unlock(&vgpu->vdev.cache_lock);
232 }
233
234 static void gvt_cache_init(struct intel_vgpu *vgpu)
235 {
236         vgpu->vdev.cache = RB_ROOT;
237         mutex_init(&vgpu->vdev.cache_lock);
238 }
239
240 static void gvt_cache_destroy(struct intel_vgpu *vgpu)
241 {
242         struct gvt_dma *dma;
243         struct rb_node *node = NULL;
244         struct device *dev = mdev_dev(vgpu->vdev.mdev);
245         unsigned long gfn;
246
247         for (;;) {
248                 mutex_lock(&vgpu->vdev.cache_lock);
249                 node = rb_first(&vgpu->vdev.cache);
250                 if (!node) {
251                         mutex_unlock(&vgpu->vdev.cache_lock);
252                         break;
253                 }
254                 dma = rb_entry(node, struct gvt_dma, node);
255                 gvt_dma_unmap_iova(vgpu, dma->iova);
256                 gfn = dma->gfn;
257                 __gvt_cache_remove_entry(vgpu, dma);
258                 mutex_unlock(&vgpu->vdev.cache_lock);
259                 vfio_unpin_pages(dev, &gfn, 1);
260         }
261 }
262
263 static void kvmgt_protect_table_init(struct kvmgt_guest_info *info)
264 {
265         hash_init(info->ptable);
266 }
267
268 static void kvmgt_protect_table_destroy(struct kvmgt_guest_info *info)
269 {
270         struct kvmgt_pgfn *p;
271         struct hlist_node *tmp;
272         int i;
273
274         hash_for_each_safe(info->ptable, i, tmp, p, hnode) {
275                 hash_del(&p->hnode);
276                 kfree(p);
277         }
278 }
279
280 static struct kvmgt_pgfn *
281 __kvmgt_protect_table_find(struct kvmgt_guest_info *info, gfn_t gfn)
282 {
283         struct kvmgt_pgfn *p, *res = NULL;
284
285         hash_for_each_possible(info->ptable, p, hnode, gfn) {
286                 if (gfn == p->gfn) {
287                         res = p;
288                         break;
289                 }
290         }
291
292         return res;
293 }
294
295 static bool kvmgt_gfn_is_write_protected(struct kvmgt_guest_info *info,
296                                 gfn_t gfn)
297 {
298         struct kvmgt_pgfn *p;
299
300         p = __kvmgt_protect_table_find(info, gfn);
301         return !!p;
302 }
303
304 static void kvmgt_protect_table_add(struct kvmgt_guest_info *info, gfn_t gfn)
305 {
306         struct kvmgt_pgfn *p;
307
308         if (kvmgt_gfn_is_write_protected(info, gfn))
309                 return;
310
311         p = kzalloc(sizeof(struct kvmgt_pgfn), GFP_ATOMIC);
312         if (WARN(!p, "gfn: 0x%llx\n", gfn))
313                 return;
314
315         p->gfn = gfn;
316         hash_add(info->ptable, &p->hnode, gfn);
317 }
318
319 static void kvmgt_protect_table_del(struct kvmgt_guest_info *info,
320                                 gfn_t gfn)
321 {
322         struct kvmgt_pgfn *p;
323
324         p = __kvmgt_protect_table_find(info, gfn);
325         if (p) {
326                 hash_del(&p->hnode);
327                 kfree(p);
328         }
329 }
330
331 static size_t intel_vgpu_reg_rw_opregion(struct intel_vgpu *vgpu, char *buf,
332                 size_t count, loff_t *ppos, bool iswrite)
333 {
334         unsigned int i = VFIO_PCI_OFFSET_TO_INDEX(*ppos) -
335                         VFIO_PCI_NUM_REGIONS;
336         void *base = vgpu->vdev.region[i].data;
337         loff_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
338
339         if (pos >= vgpu->vdev.region[i].size || iswrite) {
340                 gvt_vgpu_err("invalid op or offset for Intel vgpu OpRegion\n");
341                 return -EINVAL;
342         }
343         count = min(count, (size_t)(vgpu->vdev.region[i].size - pos));
344         memcpy(buf, base + pos, count);
345
346         return count;
347 }
348
349 static void intel_vgpu_reg_release_opregion(struct intel_vgpu *vgpu,
350                 struct vfio_region *region)
351 {
352 }
353
354 static const struct intel_vgpu_regops intel_vgpu_regops_opregion = {
355         .rw = intel_vgpu_reg_rw_opregion,
356         .release = intel_vgpu_reg_release_opregion,
357 };
358
359 static int intel_vgpu_register_reg(struct intel_vgpu *vgpu,
360                 unsigned int type, unsigned int subtype,
361                 const struct intel_vgpu_regops *ops,
362                 size_t size, u32 flags, void *data)
363 {
364         struct vfio_region *region;
365
366         region = krealloc(vgpu->vdev.region,
367                         (vgpu->vdev.num_regions + 1) * sizeof(*region),
368                         GFP_KERNEL);
369         if (!region)
370                 return -ENOMEM;
371
372         vgpu->vdev.region = region;
373         vgpu->vdev.region[vgpu->vdev.num_regions].type = type;
374         vgpu->vdev.region[vgpu->vdev.num_regions].subtype = subtype;
375         vgpu->vdev.region[vgpu->vdev.num_regions].ops = ops;
376         vgpu->vdev.region[vgpu->vdev.num_regions].size = size;
377         vgpu->vdev.region[vgpu->vdev.num_regions].flags = flags;
378         vgpu->vdev.region[vgpu->vdev.num_regions].data = data;
379         vgpu->vdev.num_regions++;
380         return 0;
381 }
382
383 static int kvmgt_get_vfio_device(void *p_vgpu)
384 {
385         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
386
387         vgpu->vdev.vfio_device = vfio_device_get_from_dev(
388                 mdev_dev(vgpu->vdev.mdev));
389         if (!vgpu->vdev.vfio_device) {
390                 gvt_vgpu_err("failed to get vfio device\n");
391                 return -ENODEV;
392         }
393         return 0;
394 }
395
396
397 static int kvmgt_set_opregion(void *p_vgpu)
398 {
399         struct intel_vgpu *vgpu = (struct intel_vgpu *)p_vgpu;
400         void *base;
401         int ret;
402
403         /* Each vgpu has its own opregion, although VFIO would create another
404          * one later. This one is used to expose opregion to VFIO. And the
405          * other one created by VFIO later, is used by guest actually.
406          */
407         base = vgpu_opregion(vgpu)->va;
408         if (!base)
409                 return -ENOMEM;
410
411         if (memcmp(base, OPREGION_SIGNATURE, 16)) {
412                 memunmap(base);
413                 return -EINVAL;
414         }
415
416         ret = intel_vgpu_register_reg(vgpu,
417                         PCI_VENDOR_ID_INTEL | VFIO_REGION_TYPE_PCI_VENDOR_TYPE,
418                         VFIO_REGION_SUBTYPE_INTEL_IGD_OPREGION,
419                         &intel_vgpu_regops_opregion, OPREGION_SIZE,
420                         VFIO_REGION_INFO_FLAG_READ, base);
421
422         return ret;
423 }
424
425 static void kvmgt_put_vfio_device(void *vgpu)
426 {
427         if (WARN_ON(!((struct intel_vgpu *)vgpu)->vdev.vfio_device))
428                 return;
429
430         vfio_device_put(((struct intel_vgpu *)vgpu)->vdev.vfio_device);
431 }
432
433 static int intel_vgpu_create(struct kobject *kobj, struct mdev_device *mdev)
434 {
435         struct intel_vgpu *vgpu = NULL;
436         struct intel_vgpu_type *type;
437         struct device *pdev;
438         void *gvt;
439         int ret;
440
441         pdev = mdev_parent_dev(mdev);
442         gvt = kdev_to_i915(pdev)->gvt;
443
444         type = intel_gvt_ops->gvt_find_vgpu_type(gvt, kobject_name(kobj));
445         if (!type) {
446                 gvt_vgpu_err("failed to find type %s to create\n",
447                                                 kobject_name(kobj));
448                 ret = -EINVAL;
449                 goto out;
450         }
451
452         vgpu = intel_gvt_ops->vgpu_create(gvt, type);
453         if (IS_ERR_OR_NULL(vgpu)) {
454                 ret = vgpu == NULL ? -EFAULT : PTR_ERR(vgpu);
455                 gvt_vgpu_err("failed to create intel vgpu: %d\n", ret);
456                 goto out;
457         }
458
459         INIT_WORK(&vgpu->vdev.release_work, intel_vgpu_release_work);
460
461         vgpu->vdev.mdev = mdev;
462         mdev_set_drvdata(mdev, vgpu);
463
464         gvt_dbg_core("intel_vgpu_create succeeded for mdev: %s\n",
465                      dev_name(mdev_dev(mdev)));
466         ret = 0;
467
468 out:
469         return ret;
470 }
471
472 static int intel_vgpu_remove(struct mdev_device *mdev)
473 {
474         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
475
476         if (handle_valid(vgpu->handle))
477                 return -EBUSY;
478
479         intel_gvt_ops->vgpu_destroy(vgpu);
480         return 0;
481 }
482
483 static int intel_vgpu_iommu_notifier(struct notifier_block *nb,
484                                      unsigned long action, void *data)
485 {
486         struct intel_vgpu *vgpu = container_of(nb,
487                                         struct intel_vgpu,
488                                         vdev.iommu_notifier);
489
490         if (action == VFIO_IOMMU_NOTIFY_DMA_UNMAP) {
491                 struct vfio_iommu_type1_dma_unmap *unmap = data;
492                 unsigned long gfn, end_gfn;
493
494                 gfn = unmap->iova >> PAGE_SHIFT;
495                 end_gfn = gfn + unmap->size / PAGE_SIZE;
496
497                 while (gfn < end_gfn)
498                         gvt_cache_remove(vgpu, gfn++);
499         }
500
501         return NOTIFY_OK;
502 }
503
504 static int intel_vgpu_group_notifier(struct notifier_block *nb,
505                                      unsigned long action, void *data)
506 {
507         struct intel_vgpu *vgpu = container_of(nb,
508                                         struct intel_vgpu,
509                                         vdev.group_notifier);
510
511         /* the only action we care about */
512         if (action == VFIO_GROUP_NOTIFY_SET_KVM) {
513                 vgpu->vdev.kvm = data;
514
515                 if (!data)
516                         schedule_work(&vgpu->vdev.release_work);
517         }
518
519         return NOTIFY_OK;
520 }
521
522 static int intel_vgpu_open(struct mdev_device *mdev)
523 {
524         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
525         unsigned long events;
526         int ret;
527
528         vgpu->vdev.iommu_notifier.notifier_call = intel_vgpu_iommu_notifier;
529         vgpu->vdev.group_notifier.notifier_call = intel_vgpu_group_notifier;
530
531         events = VFIO_IOMMU_NOTIFY_DMA_UNMAP;
532         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY, &events,
533                                 &vgpu->vdev.iommu_notifier);
534         if (ret != 0) {
535                 gvt_vgpu_err("vfio_register_notifier for iommu failed: %d\n",
536                         ret);
537                 goto out;
538         }
539
540         events = VFIO_GROUP_NOTIFY_SET_KVM;
541         ret = vfio_register_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY, &events,
542                                 &vgpu->vdev.group_notifier);
543         if (ret != 0) {
544                 gvt_vgpu_err("vfio_register_notifier for group failed: %d\n",
545                         ret);
546                 goto undo_iommu;
547         }
548
549         ret = kvmgt_guest_init(mdev);
550         if (ret)
551                 goto undo_group;
552
553         intel_gvt_ops->vgpu_activate(vgpu);
554
555         atomic_set(&vgpu->vdev.released, 0);
556         return ret;
557
558 undo_group:
559         vfio_unregister_notifier(mdev_dev(mdev), VFIO_GROUP_NOTIFY,
560                                         &vgpu->vdev.group_notifier);
561
562 undo_iommu:
563         vfio_unregister_notifier(mdev_dev(mdev), VFIO_IOMMU_NOTIFY,
564                                         &vgpu->vdev.iommu_notifier);
565 out:
566         return ret;
567 }
568
569 static void __intel_vgpu_release(struct intel_vgpu *vgpu)
570 {
571         struct kvmgt_guest_info *info;
572         int ret;
573
574         if (!handle_valid(vgpu->handle))
575                 return;
576
577         if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1))
578                 return;
579
580         intel_gvt_ops->vgpu_deactivate(vgpu);
581
582         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY,
583                                         &vgpu->vdev.iommu_notifier);
584         WARN(ret, "vfio_unregister_notifier for iommu failed: %d\n", ret);
585
586         ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_GROUP_NOTIFY,
587                                         &vgpu->vdev.group_notifier);
588         WARN(ret, "vfio_unregister_notifier for group failed: %d\n", ret);
589
590         info = (struct kvmgt_guest_info *)vgpu->handle;
591         kvmgt_guest_exit(info);
592
593         vgpu->vdev.kvm = NULL;
594         vgpu->handle = 0;
595 }
596
597 static void intel_vgpu_release(struct mdev_device *mdev)
598 {
599         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
600
601         __intel_vgpu_release(vgpu);
602 }
603
604 static void intel_vgpu_release_work(struct work_struct *work)
605 {
606         struct intel_vgpu *vgpu = container_of(work, struct intel_vgpu,
607                                         vdev.release_work);
608
609         __intel_vgpu_release(vgpu);
610 }
611
612 static uint64_t intel_vgpu_get_bar_addr(struct intel_vgpu *vgpu, int bar)
613 {
614         u32 start_lo, start_hi;
615         u32 mem_type;
616
617         start_lo = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
618                         PCI_BASE_ADDRESS_MEM_MASK;
619         mem_type = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space + bar)) &
620                         PCI_BASE_ADDRESS_MEM_TYPE_MASK;
621
622         switch (mem_type) {
623         case PCI_BASE_ADDRESS_MEM_TYPE_64:
624                 start_hi = (*(u32 *)(vgpu->cfg_space.virtual_cfg_space
625                                                 + bar + 4));
626                 break;
627         case PCI_BASE_ADDRESS_MEM_TYPE_32:
628         case PCI_BASE_ADDRESS_MEM_TYPE_1M:
629                 /* 1M mem BAR treated as 32-bit BAR */
630         default:
631                 /* mem unknown type treated as 32-bit BAR */
632                 start_hi = 0;
633                 break;
634         }
635
636         return ((u64)start_hi << 32) | start_lo;
637 }
638
639 static int intel_vgpu_bar_rw(struct intel_vgpu *vgpu, int bar, uint64_t off,
640                              void *buf, unsigned int count, bool is_write)
641 {
642         uint64_t bar_start = intel_vgpu_get_bar_addr(vgpu, bar);
643         int ret;
644
645         if (is_write)
646                 ret = intel_gvt_ops->emulate_mmio_write(vgpu,
647                                         bar_start + off, buf, count);
648         else
649                 ret = intel_gvt_ops->emulate_mmio_read(vgpu,
650                                         bar_start + off, buf, count);
651         return ret;
652 }
653
654 static inline bool intel_vgpu_in_aperture(struct intel_vgpu *vgpu, uint64_t off)
655 {
656         return off >= vgpu_aperture_offset(vgpu) &&
657                off < vgpu_aperture_offset(vgpu) + vgpu_aperture_sz(vgpu);
658 }
659
660 static int intel_vgpu_aperture_rw(struct intel_vgpu *vgpu, uint64_t off,
661                 void *buf, unsigned long count, bool is_write)
662 {
663         void *aperture_va;
664
665         if (!intel_vgpu_in_aperture(vgpu, off) ||
666             !intel_vgpu_in_aperture(vgpu, off + count)) {
667                 gvt_vgpu_err("Invalid aperture offset %llu\n", off);
668                 return -EINVAL;
669         }
670
671         aperture_va = io_mapping_map_wc(&vgpu->gvt->dev_priv->ggtt.iomap,
672                                         ALIGN_DOWN(off, PAGE_SIZE),
673                                         count + offset_in_page(off));
674         if (!aperture_va)
675                 return -EIO;
676
677         if (is_write)
678                 memcpy(aperture_va + offset_in_page(off), buf, count);
679         else
680                 memcpy(buf, aperture_va + offset_in_page(off), count);
681
682         io_mapping_unmap(aperture_va);
683
684         return 0;
685 }
686
687 static ssize_t intel_vgpu_rw(struct mdev_device *mdev, char *buf,
688                         size_t count, loff_t *ppos, bool is_write)
689 {
690         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
691         unsigned int index = VFIO_PCI_OFFSET_TO_INDEX(*ppos);
692         uint64_t pos = *ppos & VFIO_PCI_OFFSET_MASK;
693         int ret = -EINVAL;
694
695
696         if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) {
697                 gvt_vgpu_err("invalid index: %u\n", index);
698                 return -EINVAL;
699         }
700
701         switch (index) {
702         case VFIO_PCI_CONFIG_REGION_INDEX:
703                 if (is_write)
704                         ret = intel_gvt_ops->emulate_cfg_write(vgpu, pos,
705                                                 buf, count);
706                 else
707                         ret = intel_gvt_ops->emulate_cfg_read(vgpu, pos,
708                                                 buf, count);
709                 break;
710         case VFIO_PCI_BAR0_REGION_INDEX:
711                 ret = intel_vgpu_bar_rw(vgpu, PCI_BASE_ADDRESS_0, pos,
712                                         buf, count, is_write);
713                 break;
714         case VFIO_PCI_BAR2_REGION_INDEX:
715                 ret = intel_vgpu_aperture_rw(vgpu, pos, buf, count, is_write);
716                 break;
717         case VFIO_PCI_BAR1_REGION_INDEX:
718         case VFIO_PCI_BAR3_REGION_INDEX:
719         case VFIO_PCI_BAR4_REGION_INDEX:
720         case VFIO_PCI_BAR5_REGION_INDEX:
721         case VFIO_PCI_VGA_REGION_INDEX:
722         case VFIO_PCI_ROM_REGION_INDEX:
723                 break;
724         default:
725                 if (index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions)
726                         return -EINVAL;
727
728                 index -= VFIO_PCI_NUM_REGIONS;
729                 return vgpu->vdev.region[index].ops->rw(vgpu, buf, count,
730                                 ppos, is_write);
731         }
732
733         return ret == 0 ? count : ret;
734 }
735
736 static ssize_t intel_vgpu_read(struct mdev_device *mdev, char __user *buf,
737                         size_t count, loff_t *ppos)
738 {
739         unsigned int done = 0;
740         int ret;
741
742         while (count) {
743                 size_t filled;
744
745                 if (count >= 4 && !(*ppos % 4)) {
746                         u32 val;
747
748                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
749                                         ppos, false);
750                         if (ret <= 0)
751                                 goto read_err;
752
753                         if (copy_to_user(buf, &val, sizeof(val)))
754                                 goto read_err;
755
756                         filled = 4;
757                 } else if (count >= 2 && !(*ppos % 2)) {
758                         u16 val;
759
760                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
761                                         ppos, false);
762                         if (ret <= 0)
763                                 goto read_err;
764
765                         if (copy_to_user(buf, &val, sizeof(val)))
766                                 goto read_err;
767
768                         filled = 2;
769                 } else {
770                         u8 val;
771
772                         ret = intel_vgpu_rw(mdev, &val, sizeof(val), ppos,
773                                         false);
774                         if (ret <= 0)
775                                 goto read_err;
776
777                         if (copy_to_user(buf, &val, sizeof(val)))
778                                 goto read_err;
779
780                         filled = 1;
781                 }
782
783                 count -= filled;
784                 done += filled;
785                 *ppos += filled;
786                 buf += filled;
787         }
788
789         return done;
790
791 read_err:
792         return -EFAULT;
793 }
794
795 static ssize_t intel_vgpu_write(struct mdev_device *mdev,
796                                 const char __user *buf,
797                                 size_t count, loff_t *ppos)
798 {
799         unsigned int done = 0;
800         int ret;
801
802         while (count) {
803                 size_t filled;
804
805                 if (count >= 4 && !(*ppos % 4)) {
806                         u32 val;
807
808                         if (copy_from_user(&val, buf, sizeof(val)))
809                                 goto write_err;
810
811                         ret = intel_vgpu_rw(mdev, (char *)&val, sizeof(val),
812                                         ppos, true);
813                         if (ret <= 0)
814                                 goto write_err;
815
816                         filled = 4;
817                 } else if (count >= 2 && !(*ppos % 2)) {
818                         u16 val;
819
820                         if (copy_from_user(&val, buf, sizeof(val)))
821                                 goto write_err;
822
823                         ret = intel_vgpu_rw(mdev, (char *)&val,
824                                         sizeof(val), ppos, true);
825                         if (ret <= 0)
826                                 goto write_err;
827
828                         filled = 2;
829                 } else {
830                         u8 val;
831
832                         if (copy_from_user(&val, buf, sizeof(val)))
833                                 goto write_err;
834
835                         ret = intel_vgpu_rw(mdev, &val, sizeof(val),
836                                         ppos, true);
837                         if (ret <= 0)
838                                 goto write_err;
839
840                         filled = 1;
841                 }
842
843                 count -= filled;
844                 done += filled;
845                 *ppos += filled;
846                 buf += filled;
847         }
848
849         return done;
850 write_err:
851         return -EFAULT;
852 }
853
854 static int intel_vgpu_mmap(struct mdev_device *mdev, struct vm_area_struct *vma)
855 {
856         unsigned int index;
857         u64 virtaddr;
858         unsigned long req_size, pgoff = 0;
859         pgprot_t pg_prot;
860         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
861
862         index = vma->vm_pgoff >> (VFIO_PCI_OFFSET_SHIFT - PAGE_SHIFT);
863         if (index >= VFIO_PCI_ROM_REGION_INDEX)
864                 return -EINVAL;
865
866         if (vma->vm_end < vma->vm_start)
867                 return -EINVAL;
868         if ((vma->vm_flags & VM_SHARED) == 0)
869                 return -EINVAL;
870         if (index != VFIO_PCI_BAR2_REGION_INDEX)
871                 return -EINVAL;
872
873         pg_prot = vma->vm_page_prot;
874         virtaddr = vma->vm_start;
875         req_size = vma->vm_end - vma->vm_start;
876         pgoff = vgpu_aperture_pa_base(vgpu) >> PAGE_SHIFT;
877
878         return remap_pfn_range(vma, virtaddr, pgoff, req_size, pg_prot);
879 }
880
881 static int intel_vgpu_get_irq_count(struct intel_vgpu *vgpu, int type)
882 {
883         if (type == VFIO_PCI_INTX_IRQ_INDEX || type == VFIO_PCI_MSI_IRQ_INDEX)
884                 return 1;
885
886         return 0;
887 }
888
889 static int intel_vgpu_set_intx_mask(struct intel_vgpu *vgpu,
890                         unsigned int index, unsigned int start,
891                         unsigned int count, uint32_t flags,
892                         void *data)
893 {
894         return 0;
895 }
896
897 static int intel_vgpu_set_intx_unmask(struct intel_vgpu *vgpu,
898                         unsigned int index, unsigned int start,
899                         unsigned int count, uint32_t flags, void *data)
900 {
901         return 0;
902 }
903
904 static int intel_vgpu_set_intx_trigger(struct intel_vgpu *vgpu,
905                 unsigned int index, unsigned int start, unsigned int count,
906                 uint32_t flags, void *data)
907 {
908         return 0;
909 }
910
911 static int intel_vgpu_set_msi_trigger(struct intel_vgpu *vgpu,
912                 unsigned int index, unsigned int start, unsigned int count,
913                 uint32_t flags, void *data)
914 {
915         struct eventfd_ctx *trigger;
916
917         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
918                 int fd = *(int *)data;
919
920                 trigger = eventfd_ctx_fdget(fd);
921                 if (IS_ERR(trigger)) {
922                         gvt_vgpu_err("eventfd_ctx_fdget failed\n");
923                         return PTR_ERR(trigger);
924                 }
925                 vgpu->vdev.msi_trigger = trigger;
926         }
927
928         return 0;
929 }
930
931 static int intel_vgpu_set_irqs(struct intel_vgpu *vgpu, uint32_t flags,
932                 unsigned int index, unsigned int start, unsigned int count,
933                 void *data)
934 {
935         int (*func)(struct intel_vgpu *vgpu, unsigned int index,
936                         unsigned int start, unsigned int count, uint32_t flags,
937                         void *data) = NULL;
938
939         switch (index) {
940         case VFIO_PCI_INTX_IRQ_INDEX:
941                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
942                 case VFIO_IRQ_SET_ACTION_MASK:
943                         func = intel_vgpu_set_intx_mask;
944                         break;
945                 case VFIO_IRQ_SET_ACTION_UNMASK:
946                         func = intel_vgpu_set_intx_unmask;
947                         break;
948                 case VFIO_IRQ_SET_ACTION_TRIGGER:
949                         func = intel_vgpu_set_intx_trigger;
950                         break;
951                 }
952                 break;
953         case VFIO_PCI_MSI_IRQ_INDEX:
954                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
955                 case VFIO_IRQ_SET_ACTION_MASK:
956                 case VFIO_IRQ_SET_ACTION_UNMASK:
957                         /* XXX Need masking support exported */
958                         break;
959                 case VFIO_IRQ_SET_ACTION_TRIGGER:
960                         func = intel_vgpu_set_msi_trigger;
961                         break;
962                 }
963                 break;
964         }
965
966         if (!func)
967                 return -ENOTTY;
968
969         return func(vgpu, index, start, count, flags, data);
970 }
971
972 static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd,
973                              unsigned long arg)
974 {
975         struct intel_vgpu *vgpu = mdev_get_drvdata(mdev);
976         unsigned long minsz;
977
978         gvt_dbg_core("vgpu%d ioctl, cmd: %d\n", vgpu->id, cmd);
979
980         if (cmd == VFIO_DEVICE_GET_INFO) {
981                 struct vfio_device_info info;
982
983                 minsz = offsetofend(struct vfio_device_info, num_irqs);
984
985                 if (copy_from_user(&info, (void __user *)arg, minsz))
986                         return -EFAULT;
987
988                 if (info.argsz < minsz)
989                         return -EINVAL;
990
991                 info.flags = VFIO_DEVICE_FLAGS_PCI;
992                 info.flags |= VFIO_DEVICE_FLAGS_RESET;
993                 info.num_regions = VFIO_PCI_NUM_REGIONS +
994                                 vgpu->vdev.num_regions;
995                 info.num_irqs = VFIO_PCI_NUM_IRQS;
996
997                 return copy_to_user((void __user *)arg, &info, minsz) ?
998                         -EFAULT : 0;
999
1000         } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) {
1001                 struct vfio_region_info info;
1002                 struct vfio_info_cap caps = { .buf = NULL, .size = 0 };
1003                 int i, ret;
1004                 struct vfio_region_info_cap_sparse_mmap *sparse = NULL;
1005                 size_t size;
1006                 int nr_areas = 1;
1007                 int cap_type_id;
1008
1009                 minsz = offsetofend(struct vfio_region_info, offset);
1010
1011                 if (copy_from_user(&info, (void __user *)arg, minsz))
1012                         return -EFAULT;
1013
1014                 if (info.argsz < minsz)
1015                         return -EINVAL;
1016
1017                 switch (info.index) {
1018                 case VFIO_PCI_CONFIG_REGION_INDEX:
1019                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1020                         info.size = vgpu->gvt->device_info.cfg_space_size;
1021                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1022                                      VFIO_REGION_INFO_FLAG_WRITE;
1023                         break;
1024                 case VFIO_PCI_BAR0_REGION_INDEX:
1025                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1026                         info.size = vgpu->cfg_space.bar[info.index].size;
1027                         if (!info.size) {
1028                                 info.flags = 0;
1029                                 break;
1030                         }
1031
1032                         info.flags = VFIO_REGION_INFO_FLAG_READ |
1033                                      VFIO_REGION_INFO_FLAG_WRITE;
1034                         break;
1035                 case VFIO_PCI_BAR1_REGION_INDEX:
1036                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1037                         info.size = 0;
1038                         info.flags = 0;
1039                         break;
1040                 case VFIO_PCI_BAR2_REGION_INDEX:
1041                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1042                         info.flags = VFIO_REGION_INFO_FLAG_CAPS |
1043                                         VFIO_REGION_INFO_FLAG_MMAP |
1044                                         VFIO_REGION_INFO_FLAG_READ |
1045                                         VFIO_REGION_INFO_FLAG_WRITE;
1046                         info.size = gvt_aperture_sz(vgpu->gvt);
1047
1048                         size = sizeof(*sparse) +
1049                                         (nr_areas * sizeof(*sparse->areas));
1050                         sparse = kzalloc(size, GFP_KERNEL);
1051                         if (!sparse)
1052                                 return -ENOMEM;
1053
1054                         sparse->header.id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1055                         sparse->header.version = 1;
1056                         sparse->nr_areas = nr_areas;
1057                         cap_type_id = VFIO_REGION_INFO_CAP_SPARSE_MMAP;
1058                         sparse->areas[0].offset =
1059                                         PAGE_ALIGN(vgpu_aperture_offset(vgpu));
1060                         sparse->areas[0].size = vgpu_aperture_sz(vgpu);
1061                         break;
1062
1063                 case VFIO_PCI_BAR3_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
1064                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1065                         info.size = 0;
1066                         info.flags = 0;
1067
1068                         gvt_dbg_core("get region info bar:%d\n", info.index);
1069                         break;
1070
1071                 case VFIO_PCI_ROM_REGION_INDEX:
1072                 case VFIO_PCI_VGA_REGION_INDEX:
1073                         info.offset = VFIO_PCI_INDEX_TO_OFFSET(info.index);
1074                         info.size = 0;
1075                         info.flags = 0;
1076
1077                         gvt_dbg_core("get region info index:%d\n", info.index);
1078                         break;
1079                 default:
1080                         {
1081                                 struct vfio_region_info_cap_type cap_type = {
1082                                         .header.id = VFIO_REGION_INFO_CAP_TYPE,
1083                                         .header.version = 1 };
1084
1085                                 if (info.index >= VFIO_PCI_NUM_REGIONS +
1086                                                 vgpu->vdev.num_regions)
1087                                         return -EINVAL;
1088
1089                                 i = info.index - VFIO_PCI_NUM_REGIONS;
1090
1091                                 info.offset =
1092                                         VFIO_PCI_INDEX_TO_OFFSET(info.index);
1093                                 info.size = vgpu->vdev.region[i].size;
1094                                 info.flags = vgpu->vdev.region[i].flags;
1095
1096                                 cap_type.type = vgpu->vdev.region[i].type;
1097                                 cap_type.subtype = vgpu->vdev.region[i].subtype;
1098
1099                                 ret = vfio_info_add_capability(&caps,
1100                                                         &cap_type.header,
1101                                                         sizeof(cap_type));
1102                                 if (ret)
1103                                         return ret;
1104                         }
1105                 }
1106
1107                 if ((info.flags & VFIO_REGION_INFO_FLAG_CAPS) && sparse) {
1108                         switch (cap_type_id) {
1109                         case VFIO_REGION_INFO_CAP_SPARSE_MMAP:
1110                                 ret = vfio_info_add_capability(&caps,
1111                                         &sparse->header, sizeof(*sparse) +
1112                                         (sparse->nr_areas *
1113                                                 sizeof(*sparse->areas)));
1114                                 kfree(sparse);
1115                                 if (ret)
1116                                         return ret;
1117                                 break;
1118                         default:
1119                                 return -EINVAL;
1120                         }
1121                 }
1122
1123                 if (caps.size) {
1124                         info.flags |= VFIO_REGION_INFO_FLAG_CAPS;
1125                         if (info.argsz < sizeof(info) + caps.size) {
1126                                 info.argsz = sizeof(info) + caps.size;
1127                                 info.cap_offset = 0;
1128                         } else {
1129                                 vfio_info_cap_shift(&caps, sizeof(info));
1130                                 if (copy_to_user((void __user *)arg +
1131                                                   sizeof(info), caps.buf,
1132                                                   caps.size)) {
1133                                         kfree(caps.buf);
1134                                         return -EFAULT;
1135                                 }
1136                                 info.cap_offset = sizeof(info);
1137                         }
1138
1139                         kfree(caps.buf);
1140                 }
1141
1142                 return copy_to_user((void __user *)arg, &info, minsz) ?
1143                         -EFAULT : 0;
1144         } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) {
1145                 struct vfio_irq_info info;
1146
1147                 minsz = offsetofend(struct vfio_irq_info, count);
1148
1149                 if (copy_from_user(&info, (void __user *)arg, minsz))
1150                         return -EFAULT;
1151
1152                 if (info.argsz < minsz || info.index >= VFIO_PCI_NUM_IRQS)
1153                         return -EINVAL;
1154
1155                 switch (info.index) {
1156                 case VFIO_PCI_INTX_IRQ_INDEX:
1157                 case VFIO_PCI_MSI_IRQ_INDEX:
1158                         break;
1159                 default:
1160                         return -EINVAL;
1161                 }
1162
1163                 info.flags = VFIO_IRQ_INFO_EVENTFD;
1164
1165                 info.count = intel_vgpu_get_irq_count(vgpu, info.index);
1166
1167                 if (info.index == VFIO_PCI_INTX_IRQ_INDEX)
1168                         info.flags |= (VFIO_IRQ_INFO_MASKABLE |
1169                                        VFIO_IRQ_INFO_AUTOMASKED);
1170                 else
1171                         info.flags |= VFIO_IRQ_INFO_NORESIZE;
1172
1173                 return copy_to_user((void __user *)arg, &info, minsz) ?
1174                         -EFAULT : 0;
1175         } else if (cmd == VFIO_DEVICE_SET_IRQS) {
1176                 struct vfio_irq_set hdr;
1177                 u8 *data = NULL;
1178                 int ret = 0;
1179                 size_t data_size = 0;
1180
1181                 minsz = offsetofend(struct vfio_irq_set, count);
1182
1183                 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1184                         return -EFAULT;
1185
1186                 if (!(hdr.flags & VFIO_IRQ_SET_DATA_NONE)) {
1187                         int max = intel_vgpu_get_irq_count(vgpu, hdr.index);
1188
1189                         ret = vfio_set_irqs_validate_and_prepare(&hdr, max,
1190                                                 VFIO_PCI_NUM_IRQS, &data_size);
1191                         if (ret) {
1192                                 gvt_vgpu_err("intel:vfio_set_irqs_validate_and_prepare failed\n");
1193                                 return -EINVAL;
1194                         }
1195                         if (data_size) {
1196                                 data = memdup_user((void __user *)(arg + minsz),
1197                                                    data_size);
1198                                 if (IS_ERR(data))
1199                                         return PTR_ERR(data);
1200                         }
1201                 }
1202
1203                 ret = intel_vgpu_set_irqs(vgpu, hdr.flags, hdr.index,
1204                                         hdr.start, hdr.count, data);
1205                 kfree(data);
1206
1207                 return ret;
1208         } else if (cmd == VFIO_DEVICE_RESET) {
1209                 intel_gvt_ops->vgpu_reset(vgpu);
1210                 return 0;
1211         } else if (cmd == VFIO_DEVICE_QUERY_GFX_PLANE) {
1212                 struct vfio_device_gfx_plane_info dmabuf;
1213                 int ret = 0;
1214
1215                 minsz = offsetofend(struct vfio_device_gfx_plane_info,
1216                                     dmabuf_id);
1217                 if (copy_from_user(&dmabuf, (void __user *)arg, minsz))
1218                         return -EFAULT;
1219                 if (dmabuf.argsz < minsz)
1220                         return -EINVAL;
1221
1222                 ret = intel_gvt_ops->vgpu_query_plane(vgpu, &dmabuf);
1223                 if (ret != 0)
1224                         return ret;
1225
1226                 return copy_to_user((void __user *)arg, &dmabuf, minsz) ?
1227                                                                 -EFAULT : 0;
1228         } else if (cmd == VFIO_DEVICE_GET_GFX_DMABUF) {
1229                 __u32 dmabuf_id;
1230                 __s32 dmabuf_fd;
1231
1232                 if (get_user(dmabuf_id, (__u32 __user *)arg))
1233                         return -EFAULT;
1234
1235                 dmabuf_fd = intel_gvt_ops->vgpu_get_dmabuf(vgpu, dmabuf_id);
1236                 return dmabuf_fd;
1237
1238         }
1239
1240         return 0;
1241 }
1242
1243 static ssize_t
1244 vgpu_id_show(struct device *dev, struct device_attribute *attr,
1245              char *buf)
1246 {
1247         struct mdev_device *mdev = mdev_from_dev(dev);
1248
1249         if (mdev) {
1250                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1251                         mdev_get_drvdata(mdev);
1252                 return sprintf(buf, "%d\n", vgpu->id);
1253         }
1254         return sprintf(buf, "\n");
1255 }
1256
1257 static ssize_t
1258 hw_id_show(struct device *dev, struct device_attribute *attr,
1259            char *buf)
1260 {
1261         struct mdev_device *mdev = mdev_from_dev(dev);
1262
1263         if (mdev) {
1264                 struct intel_vgpu *vgpu = (struct intel_vgpu *)
1265                         mdev_get_drvdata(mdev);
1266                 return sprintf(buf, "%u\n",
1267                                vgpu->submission.shadow_ctx->hw_id);
1268         }
1269         return sprintf(buf, "\n");
1270 }
1271
1272 static DEVICE_ATTR_RO(vgpu_id);
1273 static DEVICE_ATTR_RO(hw_id);
1274
1275 static struct attribute *intel_vgpu_attrs[] = {
1276         &dev_attr_vgpu_id.attr,
1277         &dev_attr_hw_id.attr,
1278         NULL
1279 };
1280
1281 static const struct attribute_group intel_vgpu_group = {
1282         .name = "intel_vgpu",
1283         .attrs = intel_vgpu_attrs,
1284 };
1285
1286 static const struct attribute_group *intel_vgpu_groups[] = {
1287         &intel_vgpu_group,
1288         NULL,
1289 };
1290
1291 static struct mdev_parent_ops intel_vgpu_ops = {
1292         .mdev_attr_groups       = intel_vgpu_groups,
1293         .create                 = intel_vgpu_create,
1294         .remove                 = intel_vgpu_remove,
1295
1296         .open                   = intel_vgpu_open,
1297         .release                = intel_vgpu_release,
1298
1299         .read                   = intel_vgpu_read,
1300         .write                  = intel_vgpu_write,
1301         .mmap                   = intel_vgpu_mmap,
1302         .ioctl                  = intel_vgpu_ioctl,
1303 };
1304
1305 static int kvmgt_host_init(struct device *dev, void *gvt, const void *ops)
1306 {
1307         struct attribute **kvm_type_attrs;
1308         struct attribute_group **kvm_vgpu_type_groups;
1309
1310         intel_gvt_ops = ops;
1311         if (!intel_gvt_ops->get_gvt_attrs(&kvm_type_attrs,
1312                         &kvm_vgpu_type_groups))
1313                 return -EFAULT;
1314         intel_vgpu_ops.supported_type_groups = kvm_vgpu_type_groups;
1315
1316         return mdev_register_device(dev, &intel_vgpu_ops);
1317 }
1318
1319 static void kvmgt_host_exit(struct device *dev, void *gvt)
1320 {
1321         mdev_unregister_device(dev);
1322 }
1323
1324 static int kvmgt_write_protect_add(unsigned long handle, u64 gfn)
1325 {
1326         struct kvmgt_guest_info *info;
1327         struct kvm *kvm;
1328         struct kvm_memory_slot *slot;
1329         int idx;
1330
1331         if (!handle_valid(handle))
1332                 return -ESRCH;
1333
1334         info = (struct kvmgt_guest_info *)handle;
1335         kvm = info->kvm;
1336
1337         idx = srcu_read_lock(&kvm->srcu);
1338         slot = gfn_to_memslot(kvm, gfn);
1339         if (!slot) {
1340                 srcu_read_unlock(&kvm->srcu, idx);
1341                 return -EINVAL;
1342         }
1343
1344         spin_lock(&kvm->mmu_lock);
1345
1346         if (kvmgt_gfn_is_write_protected(info, gfn))
1347                 goto out;
1348
1349         kvm_slot_page_track_add_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1350         kvmgt_protect_table_add(info, gfn);
1351
1352 out:
1353         spin_unlock(&kvm->mmu_lock);
1354         srcu_read_unlock(&kvm->srcu, idx);
1355         return 0;
1356 }
1357
1358 static int kvmgt_write_protect_remove(unsigned long handle, u64 gfn)
1359 {
1360         struct kvmgt_guest_info *info;
1361         struct kvm *kvm;
1362         struct kvm_memory_slot *slot;
1363         int idx;
1364
1365         if (!handle_valid(handle))
1366                 return 0;
1367
1368         info = (struct kvmgt_guest_info *)handle;
1369         kvm = info->kvm;
1370
1371         idx = srcu_read_lock(&kvm->srcu);
1372         slot = gfn_to_memslot(kvm, gfn);
1373         if (!slot) {
1374                 srcu_read_unlock(&kvm->srcu, idx);
1375                 return -EINVAL;
1376         }
1377
1378         spin_lock(&kvm->mmu_lock);
1379
1380         if (!kvmgt_gfn_is_write_protected(info, gfn))
1381                 goto out;
1382
1383         kvm_slot_page_track_remove_page(kvm, slot, gfn, KVM_PAGE_TRACK_WRITE);
1384         kvmgt_protect_table_del(info, gfn);
1385
1386 out:
1387         spin_unlock(&kvm->mmu_lock);
1388         srcu_read_unlock(&kvm->srcu, idx);
1389         return 0;
1390 }
1391
1392 static void kvmgt_page_track_write(struct kvm_vcpu *vcpu, gpa_t gpa,
1393                 const u8 *val, int len,
1394                 struct kvm_page_track_notifier_node *node)
1395 {
1396         struct kvmgt_guest_info *info = container_of(node,
1397                                         struct kvmgt_guest_info, track_node);
1398
1399         if (kvmgt_gfn_is_write_protected(info, gpa_to_gfn(gpa)))
1400                 intel_gvt_ops->write_protect_handler(info->vgpu, gpa,
1401                                                      (void *)val, len);
1402 }
1403
1404 static void kvmgt_page_track_flush_slot(struct kvm *kvm,
1405                 struct kvm_memory_slot *slot,
1406                 struct kvm_page_track_notifier_node *node)
1407 {
1408         int i;
1409         gfn_t gfn;
1410         struct kvmgt_guest_info *info = container_of(node,
1411                                         struct kvmgt_guest_info, track_node);
1412
1413         spin_lock(&kvm->mmu_lock);
1414         for (i = 0; i < slot->npages; i++) {
1415                 gfn = slot->base_gfn + i;
1416                 if (kvmgt_gfn_is_write_protected(info, gfn)) {
1417                         kvm_slot_page_track_remove_page(kvm, slot, gfn,
1418                                                 KVM_PAGE_TRACK_WRITE);
1419                         kvmgt_protect_table_del(info, gfn);
1420                 }
1421         }
1422         spin_unlock(&kvm->mmu_lock);
1423 }
1424
1425 static bool __kvmgt_vgpu_exist(struct intel_vgpu *vgpu, struct kvm *kvm)
1426 {
1427         struct intel_vgpu *itr;
1428         struct kvmgt_guest_info *info;
1429         int id;
1430         bool ret = false;
1431
1432         mutex_lock(&vgpu->gvt->lock);
1433         for_each_active_vgpu(vgpu->gvt, itr, id) {
1434                 if (!handle_valid(itr->handle))
1435                         continue;
1436
1437                 info = (struct kvmgt_guest_info *)itr->handle;
1438                 if (kvm && kvm == info->kvm) {
1439                         ret = true;
1440                         goto out;
1441                 }
1442         }
1443 out:
1444         mutex_unlock(&vgpu->gvt->lock);
1445         return ret;
1446 }
1447
1448 static int kvmgt_guest_init(struct mdev_device *mdev)
1449 {
1450         struct kvmgt_guest_info *info;
1451         struct intel_vgpu *vgpu;
1452         struct kvm *kvm;
1453
1454         vgpu = mdev_get_drvdata(mdev);
1455         if (handle_valid(vgpu->handle))
1456                 return -EEXIST;
1457
1458         kvm = vgpu->vdev.kvm;
1459         if (!kvm || kvm->mm != current->mm) {
1460                 gvt_vgpu_err("KVM is required to use Intel vGPU\n");
1461                 return -ESRCH;
1462         }
1463
1464         if (__kvmgt_vgpu_exist(vgpu, kvm))
1465                 return -EEXIST;
1466
1467         info = vzalloc(sizeof(struct kvmgt_guest_info));
1468         if (!info)
1469                 return -ENOMEM;
1470
1471         vgpu->handle = (unsigned long)info;
1472         info->vgpu = vgpu;
1473         info->kvm = kvm;
1474         kvm_get_kvm(info->kvm);
1475
1476         kvmgt_protect_table_init(info);
1477         gvt_cache_init(vgpu);
1478
1479         mutex_init(&vgpu->dmabuf_lock);
1480         init_completion(&vgpu->vblank_done);
1481
1482         info->track_node.track_write = kvmgt_page_track_write;
1483         info->track_node.track_flush_slot = kvmgt_page_track_flush_slot;
1484         kvm_page_track_register_notifier(kvm, &info->track_node);
1485
1486         return 0;
1487 }
1488
1489 static bool kvmgt_guest_exit(struct kvmgt_guest_info *info)
1490 {
1491         kvm_page_track_unregister_notifier(info->kvm, &info->track_node);
1492         kvm_put_kvm(info->kvm);
1493         kvmgt_protect_table_destroy(info);
1494         gvt_cache_destroy(info->vgpu);
1495         vfree(info);
1496
1497         return true;
1498 }
1499
1500 static int kvmgt_attach_vgpu(void *vgpu, unsigned long *handle)
1501 {
1502         /* nothing to do here */
1503         return 0;
1504 }
1505
1506 static void kvmgt_detach_vgpu(unsigned long handle)
1507 {
1508         /* nothing to do here */
1509 }
1510
1511 static int kvmgt_inject_msi(unsigned long handle, u32 addr, u16 data)
1512 {
1513         struct kvmgt_guest_info *info;
1514         struct intel_vgpu *vgpu;
1515
1516         if (!handle_valid(handle))
1517                 return -ESRCH;
1518
1519         info = (struct kvmgt_guest_info *)handle;
1520         vgpu = info->vgpu;
1521
1522         if (eventfd_signal(vgpu->vdev.msi_trigger, 1) == 1)
1523                 return 0;
1524
1525         return -EFAULT;
1526 }
1527
1528 static unsigned long kvmgt_gfn_to_pfn(unsigned long handle, unsigned long gfn)
1529 {
1530         unsigned long iova, pfn;
1531         struct kvmgt_guest_info *info;
1532         struct device *dev;
1533         struct intel_vgpu *vgpu;
1534         int rc;
1535
1536         if (!handle_valid(handle))
1537                 return INTEL_GVT_INVALID_ADDR;
1538
1539         info = (struct kvmgt_guest_info *)handle;
1540         vgpu = info->vgpu;
1541         iova = gvt_cache_find(info->vgpu, gfn);
1542         if (iova != INTEL_GVT_INVALID_ADDR)
1543                 return iova;
1544
1545         pfn = INTEL_GVT_INVALID_ADDR;
1546         dev = mdev_dev(info->vgpu->vdev.mdev);
1547         rc = vfio_pin_pages(dev, &gfn, 1, IOMMU_READ | IOMMU_WRITE, &pfn);
1548         if (rc != 1) {
1549                 gvt_vgpu_err("vfio_pin_pages failed for gfn 0x%lx: %d\n",
1550                         gfn, rc);
1551                 return INTEL_GVT_INVALID_ADDR;
1552         }
1553         /* transfer to host iova for GFX to use DMA */
1554         rc = gvt_dma_map_iova(info->vgpu, pfn, &iova);
1555         if (rc) {
1556                 gvt_vgpu_err("gvt_dma_map_iova failed for gfn: 0x%lx\n", gfn);
1557                 vfio_unpin_pages(dev, &gfn, 1);
1558                 return INTEL_GVT_INVALID_ADDR;
1559         }
1560
1561         gvt_cache_add(info->vgpu, gfn, iova);
1562         return iova;
1563 }
1564
1565 static int kvmgt_rw_gpa(unsigned long handle, unsigned long gpa,
1566                         void *buf, unsigned long len, bool write)
1567 {
1568         struct kvmgt_guest_info *info;
1569         struct kvm *kvm;
1570         int idx, ret;
1571         bool kthread = current->mm == NULL;
1572
1573         if (!handle_valid(handle))
1574                 return -ESRCH;
1575
1576         info = (struct kvmgt_guest_info *)handle;
1577         kvm = info->kvm;
1578
1579         if (kthread)
1580                 use_mm(kvm->mm);
1581
1582         idx = srcu_read_lock(&kvm->srcu);
1583         ret = write ? kvm_write_guest(kvm, gpa, buf, len) :
1584                       kvm_read_guest(kvm, gpa, buf, len);
1585         srcu_read_unlock(&kvm->srcu, idx);
1586
1587         if (kthread)
1588                 unuse_mm(kvm->mm);
1589
1590         return ret;
1591 }
1592
1593 static int kvmgt_read_gpa(unsigned long handle, unsigned long gpa,
1594                         void *buf, unsigned long len)
1595 {
1596         return kvmgt_rw_gpa(handle, gpa, buf, len, false);
1597 }
1598
1599 static int kvmgt_write_gpa(unsigned long handle, unsigned long gpa,
1600                         void *buf, unsigned long len)
1601 {
1602         return kvmgt_rw_gpa(handle, gpa, buf, len, true);
1603 }
1604
1605 static unsigned long kvmgt_virt_to_pfn(void *addr)
1606 {
1607         return PFN_DOWN(__pa(addr));
1608 }
1609
1610 static bool kvmgt_is_valid_gfn(unsigned long handle, unsigned long gfn)
1611 {
1612         struct kvmgt_guest_info *info;
1613         struct kvm *kvm;
1614
1615         if (!handle_valid(handle))
1616                 return false;
1617
1618         info = (struct kvmgt_guest_info *)handle;
1619         kvm = info->kvm;
1620
1621         return kvm_is_visible_gfn(kvm, gfn);
1622
1623 }
1624
1625 struct intel_gvt_mpt kvmgt_mpt = {
1626         .host_init = kvmgt_host_init,
1627         .host_exit = kvmgt_host_exit,
1628         .attach_vgpu = kvmgt_attach_vgpu,
1629         .detach_vgpu = kvmgt_detach_vgpu,
1630         .inject_msi = kvmgt_inject_msi,
1631         .from_virt_to_mfn = kvmgt_virt_to_pfn,
1632         .set_wp_page = kvmgt_write_protect_add,
1633         .unset_wp_page = kvmgt_write_protect_remove,
1634         .read_gpa = kvmgt_read_gpa,
1635         .write_gpa = kvmgt_write_gpa,
1636         .gfn_to_mfn = kvmgt_gfn_to_pfn,
1637         .set_opregion = kvmgt_set_opregion,
1638         .get_vfio_device = kvmgt_get_vfio_device,
1639         .put_vfio_device = kvmgt_put_vfio_device,
1640         .is_valid_gfn = kvmgt_is_valid_gfn,
1641 };
1642 EXPORT_SYMBOL_GPL(kvmgt_mpt);
1643
1644 static int __init kvmgt_init(void)
1645 {
1646         return 0;
1647 }
1648
1649 static void __exit kvmgt_exit(void)
1650 {
1651 }
1652
1653 module_init(kvmgt_init);
1654 module_exit(kvmgt_exit);
1655
1656 MODULE_LICENSE("GPL and additional rights");
1657 MODULE_AUTHOR("Intel Corporation");