drivers/vfio/vfio_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO core
   4  *
   5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6  *     Author: Alex Williamson <alex.williamson@redhat.com>
   7  *
   8  * Derived from original vfio:
   9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10  * Author: Tom Lyon, pugs@cisco.com
  11  */
  12
  13 #include <linux/cdev.h>
  14 #include <linux/compat.h>
  15 #include <linux/device.h>
  16 #include <linux/fs.h>
  17 #include <linux/idr.h>
  18 #include <linux/iommu.h>
  19 #if IS_ENABLED(CONFIG_KVM)
  20 #include <linux/kvm_host.h>
  21 #endif
  22 #include <linux/list.h>
  23 #include <linux/miscdevice.h>
  24 #include <linux/module.h>
  25 #include <linux/mount.h>
  26 #include <linux/mutex.h>
  27 #include <linux/pci.h>
  28 #include <linux/pseudo_fs.h>
  29 #include <linux/rwsem.h>
  30 #include <linux/sched.h>
  31 #include <linux/slab.h>
  32 #include <linux/stat.h>
  33 #include <linux/string.h>
  34 #include <linux/uaccess.h>
  35 #include <linux/vfio.h>
  36 #include <linux/wait.h>
  37 #include <linux/sched/signal.h>
  38 #include <linux/pm_runtime.h>
  39 #include <linux/interval_tree.h>
  40 #include <linux/iova_bitmap.h>
  41 #include <linux/iommufd.h>
  42 #include "vfio.h"
  43
  44 #define DRIVER_VERSION  "0.3"
  45 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
  46 #define DRIVER_DESC     "VFIO - User Level meta-driver"
  47
  48 #define VFIO_MAGIC 0x5646494f /* "VFIO" */
  49
  50 static struct vfio {
  51         struct class                    *device_class;
  52         struct ida                      device_ida;
  53         struct vfsmount                 *vfs_mount;
  54         int                             fs_count;
  55 } vfio;
  56
  57 #ifdef CONFIG_VFIO_NOIOMMU
  58 bool vfio_noiommu __read_mostly;
  59 module_param_named(enable_unsafe_noiommu_mode,
  60                    vfio_noiommu, bool, S_IRUGO | S_IWUSR);
  61 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode.  This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel.  If you do not know what this is for, step away. (default: false)");
  62 #endif
  63
  64 static DEFINE_XARRAY(vfio_device_set_xa);
  65
  66 int vfio_assign_device_set(struct vfio_device *device, void *set_id)
  67 {
  68         unsigned long idx = (unsigned long)set_id;
  69         struct vfio_device_set *new_dev_set;
  70         struct vfio_device_set *dev_set;
  71
  72         if (WARN_ON(!set_id))
  73                 return -EINVAL;
  74
  75         /*
  76          * Atomically acquire a singleton object in the xarray for this set_id
  77          */
  78         xa_lock(&vfio_device_set_xa);
  79         dev_set = xa_load(&vfio_device_set_xa, idx);
  80         if (dev_set)
  81                 goto found_get_ref;
  82         xa_unlock(&vfio_device_set_xa);
  83
  84         new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL);
  85         if (!new_dev_set)
  86                 return -ENOMEM;
  87         mutex_init(&new_dev_set->lock);
  88         INIT_LIST_HEAD(&new_dev_set->device_list);
  89         new_dev_set->set_id = set_id;
  90
  91         xa_lock(&vfio_device_set_xa);
  92         dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set,
  93                                GFP_KERNEL);
  94         if (!dev_set) {
  95                 dev_set = new_dev_set;
  96                 goto found_get_ref;
  97         }
  98
  99         kfree(new_dev_set);
 100         if (xa_is_err(dev_set)) {
 101                 xa_unlock(&vfio_device_set_xa);
 102                 return xa_err(dev_set);
 103         }
 104
 105 found_get_ref:
 106         dev_set->device_count++;
 107         xa_unlock(&vfio_device_set_xa);
 108         mutex_lock(&dev_set->lock);
 109         device->dev_set = dev_set;
 110         list_add_tail(&device->dev_set_list, &dev_set->device_list);
 111         mutex_unlock(&dev_set->lock);
 112         return 0;
 113 }
 114 EXPORT_SYMBOL_GPL(vfio_assign_device_set);
 115
 116 static void vfio_release_device_set(struct vfio_device *device)
 117 {
 118         struct vfio_device_set *dev_set = device->dev_set;
 119
 120         if (!dev_set)
 121                 return;
 122
 123         mutex_lock(&dev_set->lock);
 124         list_del(&device->dev_set_list);
 125         mutex_unlock(&dev_set->lock);
 126
 127         xa_lock(&vfio_device_set_xa);
 128         if (!--dev_set->device_count) {
 129                 __xa_erase(&vfio_device_set_xa,
 130                            (unsigned long)dev_set->set_id);
 131                 mutex_destroy(&dev_set->lock);
 132                 kfree(dev_set);
 133         }
 134         xa_unlock(&vfio_device_set_xa);
 135 }
 136
 137 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set)
 138 {
 139         struct vfio_device *cur;
 140         unsigned int open_count = 0;
 141
 142         lockdep_assert_held(&dev_set->lock);
 143
 144         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 145                 open_count += cur->open_count;
 146         return open_count;
 147 }
 148 EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
 149
 150 struct vfio_device *
 151 vfio_find_device_in_devset(struct vfio_device_set *dev_set,
 152                            struct device *dev)
 153 {
 154         struct vfio_device *cur;
 155
 156         lockdep_assert_held(&dev_set->lock);
 157
 158         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 159                 if (cur->dev == dev)
 160                         return cur;
 161         return NULL;
 162 }
 163 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset);
 164
 165 /*
 166  * Device objects - create, release, get, put, search
 167  */
 168 /* Device reference always implies a group reference */
 169 void vfio_device_put_registration(struct vfio_device *device)
 170 {
 171         if (refcount_dec_and_test(&device->refcount))
 172                 complete(&device->comp);
 173 }
 174
 175 bool vfio_device_try_get_registration(struct vfio_device *device)
 176 {
 177         return refcount_inc_not_zero(&device->refcount);
 178 }
 179
 180 /*
 181  * VFIO driver API
 182  */
 183 /* Release helper called by vfio_put_device() */
 184 static void vfio_device_release(struct device *dev)
 185 {
 186         struct vfio_device *device =
 187                         container_of(dev, struct vfio_device, device);
 188
 189         vfio_release_device_set(device);
 190         ida_free(&vfio.device_ida, device->index);
 191
 192         if (device->ops->release)
 193                 device->ops->release(device);
 194
 195         iput(device->inode);
 196         simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 197         kvfree(device);
 198 }
 199
 200 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 201                             const struct vfio_device_ops *ops);
 202
 203 /*
 204  * Allocate and initialize vfio_device so it can be registered to vfio
 205  * core.
 206  *
 207  * Drivers should use the wrapper vfio_alloc_device() for allocation.
 208  * @size is the size of the structure to be allocated, including any
 209  * private data used by the driver.
 210  *
 211  * Driver may provide an @init callback to cover device private data.
 212  *
 213  * Use vfio_put_device() to release the structure after success return.
 214  */
 215 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
 216                                        const struct vfio_device_ops *ops)
 217 {
 218         struct vfio_device *device;
 219         int ret;
 220
 221         if (WARN_ON(size < sizeof(struct vfio_device)))
 222                 return ERR_PTR(-EINVAL);
 223
 224         device = kvzalloc(size, GFP_KERNEL);
 225         if (!device)
 226                 return ERR_PTR(-ENOMEM);
 227
 228         ret = vfio_init_device(device, dev, ops);
 229         if (ret)
 230                 goto out_free;
 231         return device;
 232
 233 out_free:
 234         kvfree(device);
 235         return ERR_PTR(ret);
 236 }
 237 EXPORT_SYMBOL_GPL(_vfio_alloc_device);
 238
 239 static int vfio_fs_init_fs_context(struct fs_context *fc)
 240 {
 241         return init_pseudo(fc, VFIO_MAGIC) ? 0 : -ENOMEM;
 242 }
 243
 244 static struct file_system_type vfio_fs_type = {
 245         .name = "vfio",
 246         .owner = THIS_MODULE,
 247         .init_fs_context = vfio_fs_init_fs_context,
 248         .kill_sb = kill_anon_super,
 249 };
 250
 251 static struct inode *vfio_fs_inode_new(void)
 252 {
 253         struct inode *inode;
 254         int ret;
 255
 256         ret = simple_pin_fs(&vfio_fs_type, &vfio.vfs_mount, &vfio.fs_count);
 257         if (ret)
 258                 return ERR_PTR(ret);
 259
 260         inode = alloc_anon_inode(vfio.vfs_mount->mnt_sb);
 261         if (IS_ERR(inode))
 262                 simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 263
 264         return inode;
 265 }
 266
 267 /*
 268  * Initialize a vfio_device so it can be registered to vfio core.
 269  */
 270 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 271                             const struct vfio_device_ops *ops)
 272 {
 273         int ret;
 274
 275         ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
 276         if (ret < 0) {
 277                 dev_dbg(dev, "Error to alloc index\n");
 278                 return ret;
 279         }
 280
 281         device->index = ret;
 282         init_completion(&device->comp);
 283         device->dev = dev;
 284         device->ops = ops;
 285         device->inode = vfio_fs_inode_new();
 286         if (IS_ERR(device->inode)) {
 287                 ret = PTR_ERR(device->inode);
 288                 goto out_inode;
 289         }
 290
 291         if (ops->init) {
 292                 ret = ops->init(device);
 293                 if (ret)
 294                         goto out_uninit;
 295         }
 296
 297         device_initialize(&device->device);
 298         device->device.release = vfio_device_release;
 299         device->device.class = vfio.device_class;
 300         device->device.parent = device->dev;
 301         return 0;
 302
 303 out_uninit:
 304         iput(device->inode);
 305         simple_release_fs(&vfio.vfs_mount, &vfio.fs_count);
 306 out_inode:
 307         vfio_release_device_set(device);
 308         ida_free(&vfio.device_ida, device->index);
 309         return ret;
 310 }
 311
 312 static int __vfio_register_dev(struct vfio_device *device,
 313                                enum vfio_group_type type)
 314 {
 315         int ret;
 316
 317         if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) &&
 318                     (!device->ops->bind_iommufd ||
 319                      !device->ops->unbind_iommufd ||
 320                      !device->ops->attach_ioas ||
 321                      !device->ops->detach_ioas)))
 322                 return -EINVAL;
 323
 324         /*
 325          * If the driver doesn't specify a set then the device is added to a
 326          * singleton set just for itself.
 327          */
 328         if (!device->dev_set)
 329                 vfio_assign_device_set(device, device);
 330
 331         ret = dev_set_name(&device->device, "vfio%d", device->index);
 332         if (ret)
 333                 return ret;
 334
 335         ret = vfio_device_set_group(device, type);
 336         if (ret)
 337                 return ret;
 338
 339         /*
 340          * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
 341          * restore cache coherency. It has to be checked here because it is only
 342          * valid for cases where we are using iommu groups.
 343          */
 344         if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) &&
 345             !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) {
 346                 ret = -EINVAL;
 347                 goto err_out;
 348         }
 349
 350         ret = vfio_device_add(device);
 351         if (ret)
 352                 goto err_out;
 353
 354         /* Refcounting can't start until the driver calls register */
 355         refcount_set(&device->refcount, 1);
 356
 357         vfio_device_group_register(device);
 358         vfio_device_debugfs_init(device);
 359
 360         return 0;
 361 err_out:
 362         vfio_device_remove_group(device);
 363         return ret;
 364 }
 365
 366 int vfio_register_group_dev(struct vfio_device *device)
 367 {
 368         return __vfio_register_dev(device, VFIO_IOMMU);
 369 }
 370 EXPORT_SYMBOL_GPL(vfio_register_group_dev);
 371
 372 /*
 373  * Register a virtual device without IOMMU backing.  The user of this
 374  * device must not be able to directly trigger unmediated DMA.
 375  */
 376 int vfio_register_emulated_iommu_dev(struct vfio_device *device)
 377 {
 378         return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
 379 }
 380 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
 381
 382 /*
 383  * Decrement the device reference count and wait for the device to be
 384  * removed.  Open file descriptors for the device... */
 385 void vfio_unregister_group_dev(struct vfio_device *device)
 386 {
 387         unsigned int i = 0;
 388         bool interrupted = false;
 389         long rc;
 390
 391         /*
 392          * Prevent new device opened by userspace via the
 393          * VFIO_GROUP_GET_DEVICE_FD in the group path.
 394          */
 395         vfio_device_group_unregister(device);
 396
 397         /*
 398          * Balances vfio_device_add() in register path, also prevents
 399          * new device opened by userspace in the cdev path.
 400          */
 401         vfio_device_del(device);
 402
 403         vfio_device_put_registration(device);
 404         rc = try_wait_for_completion(&device->comp);
 405         while (rc <= 0) {
 406                 if (device->ops->request)
 407                         device->ops->request(device, i++);
 408
 409                 if (interrupted) {
 410                         rc = wait_for_completion_timeout(&device->comp,
 411                                                          HZ * 10);
 412                 } else {
 413                         rc = wait_for_completion_interruptible_timeout(
 414                                 &device->comp, HZ * 10);
 415                         if (rc < 0) {
 416                                 interrupted = true;
 417                                 dev_warn(device->dev,
 418                                          "Device is currently in use, task"
 419                                          " \"%s\" (%d) "
 420                                          "blocked until device is released",
 421                                          current->comm, task_pid_nr(current));
 422                         }
 423                 }
 424         }
 425
 426         vfio_device_debugfs_exit(device);
 427         /* Balances vfio_device_set_group in register path */
 428         vfio_device_remove_group(device);
 429 }
 430 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
 431
 432 #if IS_ENABLED(CONFIG_KVM)
 433 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm)
 434 {
 435         void (*pfn)(struct kvm *kvm);
 436         bool (*fn)(struct kvm *kvm);
 437         bool ret;
 438
 439         lockdep_assert_held(&device->dev_set->lock);
 440
 441         if (!kvm)
 442                 return;
 443
 444         pfn = symbol_get(kvm_put_kvm);
 445         if (WARN_ON(!pfn))
 446                 return;
 447
 448         fn = symbol_get(kvm_get_kvm_safe);
 449         if (WARN_ON(!fn)) {
 450                 symbol_put(kvm_put_kvm);
 451                 return;
 452         }
 453
 454         ret = fn(kvm);
 455         symbol_put(kvm_get_kvm_safe);
 456         if (!ret) {
 457                 symbol_put(kvm_put_kvm);
 458                 return;
 459         }
 460
 461         device->put_kvm = pfn;
 462         device->kvm = kvm;
 463 }
 464
 465 void vfio_device_put_kvm(struct vfio_device *device)
 466 {
 467         lockdep_assert_held(&device->dev_set->lock);
 468
 469         if (!device->kvm)
 470                 return;
 471
 472         if (WARN_ON(!device->put_kvm))
 473                 goto clear;
 474
 475         device->put_kvm(device->kvm);
 476         device->put_kvm = NULL;
 477         symbol_put(kvm_put_kvm);
 478
 479 clear:
 480         device->kvm = NULL;
 481 }
 482 #endif
 483
 484 /* true if the vfio_device has open_device() called but not close_device() */
 485 static bool vfio_assert_device_open(struct vfio_device *device)
 486 {
 487         return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
 488 }
 489
 490 struct vfio_device_file *
 491 vfio_allocate_device_file(struct vfio_device *device)
 492 {
 493         struct vfio_device_file *df;
 494
 495         df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT);
 496         if (!df)
 497                 return ERR_PTR(-ENOMEM);
 498
 499         df->device = device;
 500         spin_lock_init(&df->kvm_ref_lock);
 501
 502         return df;
 503 }
 504
 505 static int vfio_df_device_first_open(struct vfio_device_file *df)
 506 {
 507         struct vfio_device *device = df->device;
 508         struct iommufd_ctx *iommufd = df->iommufd;
 509         int ret;
 510
 511         lockdep_assert_held(&device->dev_set->lock);
 512
 513         if (!try_module_get(device->dev->driver->owner))
 514                 return -ENODEV;
 515
 516         if (iommufd)
 517                 ret = vfio_df_iommufd_bind(df);
 518         else
 519                 ret = vfio_device_group_use_iommu(device);
 520         if (ret)
 521                 goto err_module_put;
 522
 523         if (device->ops->open_device) {
 524                 ret = device->ops->open_device(device);
 525                 if (ret)
 526                         goto err_unuse_iommu;
 527         }
 528         return 0;
 529
 530 err_unuse_iommu:
 531         if (iommufd)
 532                 vfio_df_iommufd_unbind(df);
 533         else
 534                 vfio_device_group_unuse_iommu(device);
 535 err_module_put:
 536         module_put(device->dev->driver->owner);
 537         return ret;
 538 }
 539
 540 static void vfio_df_device_last_close(struct vfio_device_file *df)
 541 {
 542         struct vfio_device *device = df->device;
 543         struct iommufd_ctx *iommufd = df->iommufd;
 544
 545         lockdep_assert_held(&device->dev_set->lock);
 546
 547         if (device->ops->close_device)
 548                 device->ops->close_device(device);
 549         if (iommufd)
 550                 vfio_df_iommufd_unbind(df);
 551         else
 552                 vfio_device_group_unuse_iommu(device);
 553         module_put(device->dev->driver->owner);
 554 }
 555
 556 int vfio_df_open(struct vfio_device_file *df)
 557 {
 558         struct vfio_device *device = df->device;
 559         int ret = 0;
 560
 561         lockdep_assert_held(&device->dev_set->lock);
 562
 563         /*
 564          * Only the group path allows the device to be opened multiple
 565          * times.  The device cdev path doesn't have a secure way for it.
 566          */
 567         if (device->open_count != 0 && !df->group)
 568                 return -EINVAL;
 569
 570         device->open_count++;
 571         if (device->open_count == 1) {
 572                 ret = vfio_df_device_first_open(df);
 573                 if (ret)
 574                         device->open_count--;
 575         }
 576
 577         return ret;
 578 }
 579
 580 void vfio_df_close(struct vfio_device_file *df)
 581 {
 582         struct vfio_device *device = df->device;
 583
 584         lockdep_assert_held(&device->dev_set->lock);
 585
 586         vfio_assert_device_open(device);
 587         if (device->open_count == 1)
 588                 vfio_df_device_last_close(df);
 589         device->open_count--;
 590 }
 591
 592 /*
 593  * Wrapper around pm_runtime_resume_and_get().
 594  * Return error code on failure or 0 on success.
 595  */
 596 static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
 597 {
 598         struct device *dev = device->dev;
 599
 600         if (dev->driver && dev->driver->pm) {
 601                 int ret;
 602
 603                 ret = pm_runtime_resume_and_get(dev);
 604                 if (ret) {
 605                         dev_info_ratelimited(dev,
 606                                 "vfio: runtime resume failed %d\n", ret);
 607                         return -EIO;
 608                 }
 609         }
 610
 611         return 0;
 612 }
 613
 614 /*
 615  * Wrapper around pm_runtime_put().
 616  */
 617 static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
 618 {
 619         struct device *dev = device->dev;
 620
 621         if (dev->driver && dev->driver->pm)
 622                 pm_runtime_put(dev);
 623 }
 624
 625 /*
 626  * VFIO Device fd
 627  */
 628 static int vfio_device_fops_release(struct inode *inode, struct file *filep)
 629 {
 630         struct vfio_device_file *df = filep->private_data;
 631         struct vfio_device *device = df->device;
 632
 633         if (df->group)
 634                 vfio_df_group_close(df);
 635         else
 636                 vfio_df_unbind_iommufd(df);
 637
 638         vfio_device_put_registration(device);
 639
 640         kfree(df);
 641
 642         return 0;
 643 }
 644
 645 /*
 646  * vfio_mig_get_next_state - Compute the next step in the FSM
 647  * @cur_fsm - The current state the device is in
 648  * @new_fsm - The target state to reach
 649  * @next_fsm - Pointer to the next step to get to new_fsm
 650  *
 651  * Return 0 upon success, otherwise -errno
 652  * Upon success the next step in the state progression between cur_fsm and
 653  * new_fsm will be set in next_fsm.
 654  *
 655  * This breaks down requests for combination transitions into smaller steps and
 656  * returns the next step to get to new_fsm. The function may need to be called
 657  * multiple times before reaching new_fsm.
 658  *
 659  */
 660 int vfio_mig_get_next_state(struct vfio_device *device,
 661                             enum vfio_device_mig_state cur_fsm,
 662                             enum vfio_device_mig_state new_fsm,
 663                             enum vfio_device_mig_state *next_fsm)
 664 {
 665         enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 };
 666         /*
 667          * The coding in this table requires the driver to implement the
 668          * following FSM arcs:
 669          *         RESUMING -> STOP
 670          *         STOP -> RESUMING
 671          *         STOP -> STOP_COPY
 672          *         STOP_COPY -> STOP
 673          *
 674          * If P2P is supported then the driver must also implement these FSM
 675          * arcs:
 676          *         RUNNING -> RUNNING_P2P
 677          *         RUNNING_P2P -> RUNNING
 678          *         RUNNING_P2P -> STOP
 679          *         STOP -> RUNNING_P2P
 680          *
 681          * If precopy is supported then the driver must support these additional
 682          * FSM arcs:
 683          *         RUNNING -> PRE_COPY
 684          *         PRE_COPY -> RUNNING
 685          *         PRE_COPY -> STOP_COPY
 686          * However, if precopy and P2P are supported together then the driver
 687          * must support these additional arcs beyond the P2P arcs above:
 688          *         PRE_COPY -> RUNNING
 689          *         PRE_COPY -> PRE_COPY_P2P
 690          *         PRE_COPY_P2P -> PRE_COPY
 691          *         PRE_COPY_P2P -> RUNNING_P2P
 692          *         PRE_COPY_P2P -> STOP_COPY
 693          *         RUNNING -> PRE_COPY
 694          *         RUNNING_P2P -> PRE_COPY_P2P
 695          *
 696          * Without P2P and precopy the driver must implement:
 697          *         RUNNING -> STOP
 698          *         STOP -> RUNNING
 699          *
 700          * The coding will step through multiple states for some combination
 701          * transitions; if all optional features are supported, this means the
 702          * following ones:
 703          *         PRE_COPY -> PRE_COPY_P2P -> STOP_COPY
 704          *         PRE_COPY -> RUNNING -> RUNNING_P2P
 705          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP
 706          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 707          *         PRE_COPY_P2P -> RUNNING_P2P -> RUNNING
 708          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP
 709          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING
 710          *         RESUMING -> STOP -> RUNNING_P2P
 711          *         RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P
 712          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING
 713          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 714          *         RESUMING -> STOP -> STOP_COPY
 715          *         RUNNING -> RUNNING_P2P -> PRE_COPY_P2P
 716          *         RUNNING -> RUNNING_P2P -> STOP
 717          *         RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 718          *         RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY
 719          *         RUNNING_P2P -> RUNNING -> PRE_COPY
 720          *         RUNNING_P2P -> STOP -> RESUMING
 721          *         RUNNING_P2P -> STOP -> STOP_COPY
 722          *         STOP -> RUNNING_P2P -> PRE_COPY_P2P
 723          *         STOP -> RUNNING_P2P -> RUNNING
 724          *         STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 725          *         STOP_COPY -> STOP -> RESUMING
 726          *         STOP_COPY -> STOP -> RUNNING_P2P
 727          *         STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING
 728          *
 729          *  The following transitions are blocked:
 730          *         STOP_COPY -> PRE_COPY
 731          *         STOP_COPY -> PRE_COPY_P2P
 732          */
 733         static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = {
 734                 [VFIO_DEVICE_STATE_STOP] = {
 735                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 736                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 737                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 738                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 739                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 740                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 741                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 742                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 743                 },
 744                 [VFIO_DEVICE_STATE_RUNNING] = {
 745                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 746                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 747                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 748                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 749                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 750                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 751                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 752                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 753                 },
 754                 [VFIO_DEVICE_STATE_PRE_COPY] = {
 755                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING,
 756                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 757                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 758                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 759                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 760                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING,
 761                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING,
 762                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 763                 },
 764                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = {
 765                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 766                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 767                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 768                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 769                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 770                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 771                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 772                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 773                 },
 774                 [VFIO_DEVICE_STATE_STOP_COPY] = {
 775                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 776                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 777                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 778                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 779                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 780                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 781                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 782                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 783                 },
 784                 [VFIO_DEVICE_STATE_RESUMING] = {
 785                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 786                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 787                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP,
 788                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP,
 789                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 790                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 791                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 792                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 793                 },
 794                 [VFIO_DEVICE_STATE_RUNNING_P2P] = {
 795                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 796                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 797                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING,
 798                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 799                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 800                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 801                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 802                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 803                 },
 804                 [VFIO_DEVICE_STATE_ERROR] = {
 805                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR,
 806                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR,
 807                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 808                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 809                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR,
 810                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR,
 811                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR,
 812                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 813                 },
 814         };
 815
 816         static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = {
 817                 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY,
 818                 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY,
 819                 [VFIO_DEVICE_STATE_PRE_COPY] =
 820                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY,
 821                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY |
 822                                                    VFIO_MIGRATION_P2P |
 823                                                    VFIO_MIGRATION_PRE_COPY,
 824                 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY,
 825                 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY,
 826                 [VFIO_DEVICE_STATE_RUNNING_P2P] =
 827                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P,
 828                 [VFIO_DEVICE_STATE_ERROR] = ~0U,
 829         };
 830
 831         if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 832                     (state_flags_table[cur_fsm] & device->migration_flags) !=
 833                         state_flags_table[cur_fsm]))
 834                 return -EINVAL;
 835
 836         if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 837            (state_flags_table[new_fsm] & device->migration_flags) !=
 838                         state_flags_table[new_fsm])
 839                 return -EINVAL;
 840
 841         /*
 842          * Arcs touching optional and unsupported states are skipped over. The
 843          * driver will instead see an arc from the original state to the next
 844          * logical state, as per the above comment.
 845          */
 846         *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm];
 847         while ((state_flags_table[*next_fsm] & device->migration_flags) !=
 848                         state_flags_table[*next_fsm])
 849                 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm];
 850
 851         return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL;
 852 }
 853 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state);
 854
 855 /*
 856  * Convert the drivers's struct file into a FD number and return it to userspace
 857  */
 858 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg,
 859                                    struct vfio_device_feature_mig_state *mig)
 860 {
 861         int ret;
 862         int fd;
 863
 864         fd = get_unused_fd_flags(O_CLOEXEC);
 865         if (fd < 0) {
 866                 ret = fd;
 867                 goto out_fput;
 868         }
 869
 870         mig->data_fd = fd;
 871         if (copy_to_user(arg, mig, sizeof(*mig))) {
 872                 ret = -EFAULT;
 873                 goto out_put_unused;
 874         }
 875         fd_install(fd, filp);
 876         return 0;
 877
 878 out_put_unused:
 879         put_unused_fd(fd);
 880 out_fput:
 881         fput(filp);
 882         return ret;
 883 }
 884
 885 static int
 886 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device,
 887                                            u32 flags, void __user *arg,
 888                                            size_t argsz)
 889 {
 890         size_t minsz =
 891                 offsetofend(struct vfio_device_feature_mig_state, data_fd);
 892         struct vfio_device_feature_mig_state mig;
 893         struct file *filp = NULL;
 894         int ret;
 895
 896         if (!device->mig_ops)
 897                 return -ENOTTY;
 898
 899         ret = vfio_check_feature(flags, argsz,
 900                                  VFIO_DEVICE_FEATURE_SET |
 901                                  VFIO_DEVICE_FEATURE_GET,
 902                                  sizeof(mig));
 903         if (ret != 1)
 904                 return ret;
 905
 906         if (copy_from_user(&mig, arg, minsz))
 907                 return -EFAULT;
 908
 909         if (flags & VFIO_DEVICE_FEATURE_GET) {
 910                 enum vfio_device_mig_state curr_state;
 911
 912                 ret = device->mig_ops->migration_get_state(device,
 913                                                            &curr_state);
 914                 if (ret)
 915                         return ret;
 916                 mig.device_state = curr_state;
 917                 goto out_copy;
 918         }
 919
 920         /* Handle the VFIO_DEVICE_FEATURE_SET */
 921         filp = device->mig_ops->migration_set_state(device, mig.device_state);
 922         if (IS_ERR(filp) || !filp)
 923                 goto out_copy;
 924
 925         return vfio_ioct_mig_return_fd(filp, arg, &mig);
 926 out_copy:
 927         mig.data_fd = -1;
 928         if (copy_to_user(arg, &mig, sizeof(mig)))
 929                 return -EFAULT;
 930         if (IS_ERR(filp))
 931                 return PTR_ERR(filp);
 932         return 0;
 933 }
 934
 935 static int
 936 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device,
 937                                               u32 flags, void __user *arg,
 938                                               size_t argsz)
 939 {
 940         struct vfio_device_feature_mig_data_size data_size = {};
 941         unsigned long stop_copy_length;
 942         int ret;
 943
 944         if (!device->mig_ops)
 945                 return -ENOTTY;
 946
 947         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 948                                  sizeof(data_size));
 949         if (ret != 1)
 950                 return ret;
 951
 952         ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length);
 953         if (ret)
 954                 return ret;
 955
 956         data_size.stop_copy_length = stop_copy_length;
 957         if (copy_to_user(arg, &data_size, sizeof(data_size)))
 958                 return -EFAULT;
 959
 960         return 0;
 961 }
 962
 963 static int vfio_ioctl_device_feature_migration(struct vfio_device *device,
 964                                                u32 flags, void __user *arg,
 965                                                size_t argsz)
 966 {
 967         struct vfio_device_feature_migration mig = {
 968                 .flags = device->migration_flags,
 969         };
 970         int ret;
 971
 972         if (!device->mig_ops)
 973                 return -ENOTTY;
 974
 975         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 976                                  sizeof(mig));
 977         if (ret != 1)
 978                 return ret;
 979         if (copy_to_user(arg, &mig, sizeof(mig)))
 980                 return -EFAULT;
 981         return 0;
 982 }
 983
 984 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes,
 985                               u32 req_nodes)
 986 {
 987         struct interval_tree_node *prev, *curr, *comb_start, *comb_end;
 988         unsigned long min_gap, curr_gap;
 989
 990         /* Special shortcut when a single range is required */
 991         if (req_nodes == 1) {
 992                 unsigned long last;
 993
 994                 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX);
 995
 996                 /* Empty list */
 997                 if (WARN_ON_ONCE(!comb_start))
 998                         return;
 999
1000                 curr = comb_start;
1001                 while (curr) {
1002                         last = curr->last;
1003                         prev = curr;
1004                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1005                         if (prev != comb_start)
1006                                 interval_tree_remove(prev, root);
1007                 }
1008                 comb_start->last = last;
1009                 return;
1010         }
1011
1012         /* Combine ranges which have the smallest gap */
1013         while (cur_nodes > req_nodes) {
1014                 prev = NULL;
1015                 min_gap = ULONG_MAX;
1016                 curr = interval_tree_iter_first(root, 0, ULONG_MAX);
1017                 while (curr) {
1018                         if (prev) {
1019                                 curr_gap = curr->start - prev->last;
1020                                 if (curr_gap < min_gap) {
1021                                         min_gap = curr_gap;
1022                                         comb_start = prev;
1023                                         comb_end = curr;
1024                                 }
1025                         }
1026                         prev = curr;
1027                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
1028                 }
1029
1030                 /* Empty list or no nodes to combine */
1031                 if (WARN_ON_ONCE(min_gap == ULONG_MAX))
1032                         break;
1033
1034                 comb_start->last = comb_end->last;
1035                 interval_tree_remove(comb_end, root);
1036                 cur_nodes--;
1037         }
1038 }
1039 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges);
1040
1041 /* Ranges should fit into a single kernel page */
1042 #define LOG_MAX_RANGES \
1043         (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
1044
1045 static int
1046 vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
1047                                         u32 flags, void __user *arg,
1048                                         size_t argsz)
1049 {
1050         size_t minsz =
1051                 offsetofend(struct vfio_device_feature_dma_logging_control,
1052                             ranges);
1053         struct vfio_device_feature_dma_logging_range __user *ranges;
1054         struct vfio_device_feature_dma_logging_control control;
1055         struct vfio_device_feature_dma_logging_range range;
1056         struct rb_root_cached root = RB_ROOT_CACHED;
1057         struct interval_tree_node *nodes;
1058         u64 iova_end;
1059         u32 nnodes;
1060         int i, ret;
1061
1062         if (!device->log_ops)
1063                 return -ENOTTY;
1064
1065         ret = vfio_check_feature(flags, argsz,
1066                                  VFIO_DEVICE_FEATURE_SET,
1067                                  sizeof(control));
1068         if (ret != 1)
1069                 return ret;
1070
1071         if (copy_from_user(&control, arg, minsz))
1072                 return -EFAULT;
1073
1074         nnodes = control.num_ranges;
1075         if (!nnodes)
1076                 return -EINVAL;
1077
1078         if (nnodes > LOG_MAX_RANGES)
1079                 return -E2BIG;
1080
1081         ranges = u64_to_user_ptr(control.ranges);
1082         nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
1083                               GFP_KERNEL);
1084         if (!nodes)
1085                 return -ENOMEM;
1086
1087         for (i = 0; i < nnodes; i++) {
1088                 if (copy_from_user(&range, &ranges[i], sizeof(range))) {
1089                         ret = -EFAULT;
1090                         goto end;
1091                 }
1092                 if (!IS_ALIGNED(range.iova, control.page_size) ||
1093                     !IS_ALIGNED(range.length, control.page_size)) {
1094                         ret = -EINVAL;
1095                         goto end;
1096                 }
1097
1098                 if (check_add_overflow(range.iova, range.length, &iova_end) ||
1099                     iova_end > ULONG_MAX) {
1100                         ret = -EOVERFLOW;
1101                         goto end;
1102                 }
1103
1104                 nodes[i].start = range.iova;
1105                 nodes[i].last = range.iova + range.length - 1;
1106                 if (interval_tree_iter_first(&root, nodes[i].start,
1107                                              nodes[i].last)) {
1108                         /* Range overlapping */
1109                         ret = -EINVAL;
1110                         goto end;
1111                 }
1112                 interval_tree_insert(nodes + i, &root);
1113         }
1114
1115         ret = device->log_ops->log_start(device, &root, nnodes,
1116                                          &control.page_size);
1117         if (ret)
1118                 goto end;
1119
1120         if (copy_to_user(arg, &control, sizeof(control))) {
1121                 ret = -EFAULT;
1122                 device->log_ops->log_stop(device);
1123         }
1124
1125 end:
1126         kfree(nodes);
1127         return ret;
1128 }
1129
1130 static int
1131 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
1132                                        u32 flags, void __user *arg,
1133                                        size_t argsz)
1134 {
1135         int ret;
1136
1137         if (!device->log_ops)
1138                 return -ENOTTY;
1139
1140         ret = vfio_check_feature(flags, argsz,
1141                                  VFIO_DEVICE_FEATURE_SET, 0);
1142         if (ret != 1)
1143                 return ret;
1144
1145         return device->log_ops->log_stop(device);
1146 }
1147
1148 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
1149                                           unsigned long iova, size_t length,
1150                                           void *opaque)
1151 {
1152         struct vfio_device *device = opaque;
1153
1154         return device->log_ops->log_read_and_clear(device, iova, length, iter);
1155 }
1156
1157 static int
1158 vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
1159                                          u32 flags, void __user *arg,
1160                                          size_t argsz)
1161 {
1162         size_t minsz =
1163                 offsetofend(struct vfio_device_feature_dma_logging_report,
1164                             bitmap);
1165         struct vfio_device_feature_dma_logging_report report;
1166         struct iova_bitmap *iter;
1167         u64 iova_end;
1168         int ret;
1169
1170         if (!device->log_ops)
1171                 return -ENOTTY;
1172
1173         ret = vfio_check_feature(flags, argsz,
1174                                  VFIO_DEVICE_FEATURE_GET,
1175                                  sizeof(report));
1176         if (ret != 1)
1177                 return ret;
1178
1179         if (copy_from_user(&report, arg, minsz))
1180                 return -EFAULT;
1181
1182         if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
1183                 return -EINVAL;
1184
1185         if (check_add_overflow(report.iova, report.length, &iova_end) ||
1186             iova_end > ULONG_MAX)
1187                 return -EOVERFLOW;
1188
1189         iter = iova_bitmap_alloc(report.iova, report.length,
1190                                  report.page_size,
1191                                  u64_to_user_ptr(report.bitmap));
1192         if (IS_ERR(iter))
1193                 return PTR_ERR(iter);
1194
1195         ret = iova_bitmap_for_each(iter, device,
1196                                    vfio_device_log_read_and_clear);
1197
1198         iova_bitmap_free(iter);
1199         return ret;
1200 }
1201
1202 static int vfio_ioctl_device_feature(struct vfio_device *device,
1203                                      struct vfio_device_feature __user *arg)
1204 {
1205         size_t minsz = offsetofend(struct vfio_device_feature, flags);
1206         struct vfio_device_feature feature;
1207
1208         if (copy_from_user(&feature, arg, minsz))
1209                 return -EFAULT;
1210
1211         if (feature.argsz < minsz)
1212                 return -EINVAL;
1213
1214         /* Check unknown flags */
1215         if (feature.flags &
1216             ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET |
1217               VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE))
1218                 return -EINVAL;
1219
1220         /* GET & SET are mutually exclusive except with PROBE */
1221         if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1222             (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1223             (feature.flags & VFIO_DEVICE_FEATURE_GET))
1224                 return -EINVAL;
1225
1226         switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1227         case VFIO_DEVICE_FEATURE_MIGRATION:
1228                 return vfio_ioctl_device_feature_migration(
1229                         device, feature.flags, arg->data,
1230                         feature.argsz - minsz);
1231         case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE:
1232                 return vfio_ioctl_device_feature_mig_device_state(
1233                         device, feature.flags, arg->data,
1234                         feature.argsz - minsz);
1235         case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
1236                 return vfio_ioctl_device_feature_logging_start(
1237                         device, feature.flags, arg->data,
1238                         feature.argsz - minsz);
1239         case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
1240                 return vfio_ioctl_device_feature_logging_stop(
1241                         device, feature.flags, arg->data,
1242                         feature.argsz - minsz);
1243         case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
1244                 return vfio_ioctl_device_feature_logging_report(
1245                         device, feature.flags, arg->data,
1246                         feature.argsz - minsz);
1247         case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE:
1248                 return vfio_ioctl_device_feature_migration_data_size(
1249                         device, feature.flags, arg->data,
1250                         feature.argsz - minsz);
1251         default:
1252                 if (unlikely(!device->ops->device_feature))
1253                         return -EINVAL;
1254                 return device->ops->device_feature(device, feature.flags,
1255                                                    arg->data,
1256                                                    feature.argsz - minsz);
1257         }
1258 }
1259
1260 static long vfio_device_fops_unl_ioctl(struct file *filep,
1261                                        unsigned int cmd, unsigned long arg)
1262 {
1263         struct vfio_device_file *df = filep->private_data;
1264         struct vfio_device *device = df->device;
1265         void __user *uptr = (void __user *)arg;
1266         int ret;
1267
1268         if (cmd == VFIO_DEVICE_BIND_IOMMUFD)
1269                 return vfio_df_ioctl_bind_iommufd(df, uptr);
1270
1271         /* Paired with smp_store_release() following vfio_df_open() */
1272         if (!smp_load_acquire(&df->access_granted))
1273                 return -EINVAL;
1274
1275         ret = vfio_device_pm_runtime_get(device);
1276         if (ret)
1277                 return ret;
1278
1279         /* cdev only ioctls */
1280         if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) {
1281                 switch (cmd) {
1282                 case VFIO_DEVICE_ATTACH_IOMMUFD_PT:
1283                         ret = vfio_df_ioctl_attach_pt(df, uptr);
1284                         goto out;
1285
1286                 case VFIO_DEVICE_DETACH_IOMMUFD_PT:
1287                         ret = vfio_df_ioctl_detach_pt(df, uptr);
1288                         goto out;
1289                 }
1290         }
1291
1292         switch (cmd) {
1293         case VFIO_DEVICE_FEATURE:
1294                 ret = vfio_ioctl_device_feature(device, uptr);
1295                 break;
1296
1297         default:
1298                 if (unlikely(!device->ops->ioctl))
1299                         ret = -EINVAL;
1300                 else
1301                         ret = device->ops->ioctl(device, cmd, arg);
1302                 break;
1303         }
1304 out:
1305         vfio_device_pm_runtime_put(device);
1306         return ret;
1307 }
1308
1309 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1310                                      size_t count, loff_t *ppos)
1311 {
1312         struct vfio_device_file *df = filep->private_data;
1313         struct vfio_device *device = df->device;
1314
1315         /* Paired with smp_store_release() following vfio_df_open() */
1316         if (!smp_load_acquire(&df->access_granted))
1317                 return -EINVAL;
1318
1319         if (unlikely(!device->ops->read))
1320                 return -EINVAL;
1321
1322         return device->ops->read(device, buf, count, ppos);
1323 }
1324
1325 static ssize_t vfio_device_fops_write(struct file *filep,
1326                                       const char __user *buf,
1327                                       size_t count, loff_t *ppos)
1328 {
1329         struct vfio_device_file *df = filep->private_data;
1330         struct vfio_device *device = df->device;
1331
1332         /* Paired with smp_store_release() following vfio_df_open() */
1333         if (!smp_load_acquire(&df->access_granted))
1334                 return -EINVAL;
1335
1336         if (unlikely(!device->ops->write))
1337                 return -EINVAL;
1338
1339         return device->ops->write(device, buf, count, ppos);
1340 }
1341
1342 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1343 {
1344         struct vfio_device_file *df = filep->private_data;
1345         struct vfio_device *device = df->device;
1346
1347         /* Paired with smp_store_release() following vfio_df_open() */
1348         if (!smp_load_acquire(&df->access_granted))
1349                 return -EINVAL;
1350
1351         if (unlikely(!device->ops->mmap))
1352                 return -EINVAL;
1353
1354         return device->ops->mmap(device, vma);
1355 }
1356
1357 const struct file_operations vfio_device_fops = {
1358         .owner          = THIS_MODULE,
1359         .open           = vfio_device_fops_cdev_open,
1360         .release        = vfio_device_fops_release,
1361         .read           = vfio_device_fops_read,
1362         .write          = vfio_device_fops_write,
1363         .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1364         .compat_ioctl   = compat_ptr_ioctl,
1365         .mmap           = vfio_device_fops_mmap,
1366 };
1367
1368 static struct vfio_device *vfio_device_from_file(struct file *file)
1369 {
1370         struct vfio_device_file *df = file->private_data;
1371
1372         if (file->f_op != &vfio_device_fops)
1373                 return NULL;
1374         return df->device;
1375 }
1376
1377 /**
1378  * vfio_file_is_valid - True if the file is valid vfio file
1379  * @file: VFIO group file or VFIO device file
1380  */
1381 bool vfio_file_is_valid(struct file *file)
1382 {
1383         return vfio_group_from_file(file) ||
1384                vfio_device_from_file(file);
1385 }
1386 EXPORT_SYMBOL_GPL(vfio_file_is_valid);
1387
1388 /**
1389  * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1390  *        is always CPU cache coherent
1391  * @file: VFIO group file or VFIO device file
1392  *
1393  * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1394  * bit in DMA transactions. A return of false indicates that the user has
1395  * rights to access additional instructions such as wbinvd on x86.
1396  */
1397 bool vfio_file_enforced_coherent(struct file *file)
1398 {
1399         struct vfio_device *device;
1400         struct vfio_group *group;
1401
1402         group = vfio_group_from_file(file);
1403         if (group)
1404                 return vfio_group_enforced_coherent(group);
1405
1406         device = vfio_device_from_file(file);
1407         if (device)
1408                 return device_iommu_capable(device->dev,
1409                                             IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
1410
1411         return true;
1412 }
1413 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1414
1415 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm)
1416 {
1417         struct vfio_device_file *df = file->private_data;
1418
1419         /*
1420          * The kvm is first recorded in the vfio_device_file, and will
1421          * be propagated to vfio_device::kvm when the file is bound to
1422          * iommufd successfully in the vfio device cdev path.
1423          */
1424         spin_lock(&df->kvm_ref_lock);
1425         df->kvm = kvm;
1426         spin_unlock(&df->kvm_ref_lock);
1427 }
1428
1429 /**
1430  * vfio_file_set_kvm - Link a kvm with VFIO drivers
1431  * @file: VFIO group file or VFIO device file
1432  * @kvm: KVM to link
1433  *
1434  * When a VFIO device is first opened the KVM will be available in
1435  * device->kvm if one was associated with the file.
1436  */
1437 void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
1438 {
1439         struct vfio_group *group;
1440
1441         group = vfio_group_from_file(file);
1442         if (group)
1443                 vfio_group_set_kvm(group, kvm);
1444
1445         if (vfio_device_from_file(file))
1446                 vfio_device_file_set_kvm(file, kvm);
1447 }
1448 EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1449
1450 /*
1451  * Sub-module support
1452  */
1453 /*
1454  * Helper for managing a buffer of info chain capabilities, allocate or
1455  * reallocate a buffer with additional @size, filling in @id and @version
1456  * of the capability.  A pointer to the new capability is returned.
1457  *
1458  * NB. The chain is based at the head of the buffer, so new entries are
1459  * added to the tail, vfio_info_cap_shift() should be called to fixup the
1460  * next offsets prior to copying to the user buffer.
1461  */
1462 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
1463                                                size_t size, u16 id, u16 version)
1464 {
1465         void *buf;
1466         struct vfio_info_cap_header *header, *tmp;
1467
1468         /* Ensure that the next capability struct will be aligned */
1469         size = ALIGN(size, sizeof(u64));
1470
1471         buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
1472         if (!buf) {
1473                 kfree(caps->buf);
1474                 caps->buf = NULL;
1475                 caps->size = 0;
1476                 return ERR_PTR(-ENOMEM);
1477         }
1478
1479         caps->buf = buf;
1480         header = buf + caps->size;
1481
1482         /* Eventually copied to user buffer, zero */
1483         memset(header, 0, size);
1484
1485         header->id = id;
1486         header->version = version;
1487
1488         /* Add to the end of the capability chain */
1489         for (tmp = buf; tmp->next; tmp = buf + tmp->next)
1490                 ; /* nothing */
1491
1492         tmp->next = caps->size;
1493         caps->size += size;
1494
1495         return header;
1496 }
1497 EXPORT_SYMBOL_GPL(vfio_info_cap_add);
1498
1499 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
1500 {
1501         struct vfio_info_cap_header *tmp;
1502         void *buf = (void *)caps->buf;
1503
1504         /* Capability structs should start with proper alignment */
1505         WARN_ON(!IS_ALIGNED(offset, sizeof(u64)));
1506
1507         for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
1508                 tmp->next += offset;
1509 }
1510 EXPORT_SYMBOL(vfio_info_cap_shift);
1511
1512 int vfio_info_add_capability(struct vfio_info_cap *caps,
1513                              struct vfio_info_cap_header *cap, size_t size)
1514 {
1515         struct vfio_info_cap_header *header;
1516
1517         header = vfio_info_cap_add(caps, size, cap->id, cap->version);
1518         if (IS_ERR(header))
1519                 return PTR_ERR(header);
1520
1521         memcpy(header + 1, cap + 1, size - sizeof(*header));
1522
1523         return 0;
1524 }
1525 EXPORT_SYMBOL(vfio_info_add_capability);
1526
1527 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
1528                                        int max_irq_type, size_t *data_size)
1529 {
1530         unsigned long minsz;
1531         size_t size;
1532
1533         minsz = offsetofend(struct vfio_irq_set, count);
1534
1535         if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
1536             (hdr->count >= (U32_MAX - hdr->start)) ||
1537             (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
1538                                 VFIO_IRQ_SET_ACTION_TYPE_MASK)))
1539                 return -EINVAL;
1540
1541         if (data_size)
1542                 *data_size = 0;
1543
1544         if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
1545                 return -EINVAL;
1546
1547         switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
1548         case VFIO_IRQ_SET_DATA_NONE:
1549                 size = 0;
1550                 break;
1551         case VFIO_IRQ_SET_DATA_BOOL:
1552                 size = sizeof(uint8_t);
1553                 break;
1554         case VFIO_IRQ_SET_DATA_EVENTFD:
1555                 size = sizeof(int32_t);
1556                 break;
1557         default:
1558                 return -EINVAL;
1559         }
1560
1561         if (size) {
1562                 if (hdr->argsz - minsz < hdr->count * size)
1563                         return -EINVAL;
1564
1565                 if (!data_size)
1566                         return -EINVAL;
1567
1568                 *data_size = hdr->count * size;
1569         }
1570
1571         return 0;
1572 }
1573 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
1574
1575 /*
1576  * Pin contiguous user pages and return their associated host pages for local
1577  * domain only.
1578  * @device [in]  : device
1579  * @iova [in]    : starting IOVA of user pages to be pinned.
1580  * @npage [in]   : count of pages to be pinned.  This count should not
1581  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1582  * @prot [in]    : protection flags
1583  * @pages[out]   : array of host pages
1584  * Return error or number of pages pinned.
1585  *
1586  * A driver may only call this function if the vfio_device was created
1587  * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
1588  */
1589 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
1590                    int npage, int prot, struct page **pages)
1591 {
1592         /* group->container cannot change while a vfio device is open */
1593         if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
1594                 return -EINVAL;
1595         if (!device->ops->dma_unmap)
1596                 return -EINVAL;
1597         if (vfio_device_has_container(device))
1598                 return vfio_device_container_pin_pages(device, iova,
1599                                                        npage, prot, pages);
1600         if (device->iommufd_access) {
1601                 int ret;
1602
1603                 if (iova > ULONG_MAX)
1604                         return -EINVAL;
1605                 /*
1606                  * VFIO ignores the sub page offset, npages is from the start of
1607                  * a PAGE_SIZE chunk of IOVA. The caller is expected to recover
1608                  * the sub page offset by doing:
1609                  *     pages[0] + (iova % PAGE_SIZE)
1610                  */
1611                 ret = iommufd_access_pin_pages(
1612                         device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
1613                         npage * PAGE_SIZE, pages,
1614                         (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
1615                 if (ret)
1616                         return ret;
1617                 return npage;
1618         }
1619         return -EINVAL;
1620 }
1621 EXPORT_SYMBOL(vfio_pin_pages);
1622
1623 /*
1624  * Unpin contiguous host pages for local domain only.
1625  * @device [in]  : device
1626  * @iova [in]    : starting address of user pages to be unpinned.
1627  * @npage [in]   : count of pages to be unpinned.  This count should not
1628  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1629  */
1630 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
1631 {
1632         if (WARN_ON(!vfio_assert_device_open(device)))
1633                 return;
1634         if (WARN_ON(!device->ops->dma_unmap))
1635                 return;
1636
1637         if (vfio_device_has_container(device)) {
1638                 vfio_device_container_unpin_pages(device, iova, npage);
1639                 return;
1640         }
1641         if (device->iommufd_access) {
1642                 if (WARN_ON(iova > ULONG_MAX))
1643                         return;
1644                 iommufd_access_unpin_pages(device->iommufd_access,
1645                                            ALIGN_DOWN(iova, PAGE_SIZE),
1646                                            npage * PAGE_SIZE);
1647                 return;
1648         }
1649 }
1650 EXPORT_SYMBOL(vfio_unpin_pages);
1651
1652 /*
1653  * This interface allows the CPUs to perform some sort of virtual DMA on
1654  * behalf of the device.
1655  *
1656  * CPUs read/write from/into a range of IOVAs pointing to user space memory
1657  * into/from a kernel buffer.
1658  *
1659  * As the read/write of user space memory is conducted via the CPUs and is
1660  * not a real device DMA, it is not necessary to pin the user space memory.
1661  *
1662  * @device [in]         : VFIO device
1663  * @iova [in]           : base IOVA of a user space buffer
1664  * @data [in]           : pointer to kernel buffer
1665  * @len [in]            : kernel buffer length
1666  * @write               : indicate read or write
1667  * Return error code on failure or 0 on success.
1668  */
1669 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
1670                 size_t len, bool write)
1671 {
1672         if (!data || len <= 0 || !vfio_assert_device_open(device))
1673                 return -EINVAL;
1674
1675         if (vfio_device_has_container(device))
1676                 return vfio_device_container_dma_rw(device, iova,
1677                                                     data, len, write);
1678
1679         if (device->iommufd_access) {
1680                 unsigned int flags = 0;
1681
1682                 if (iova > ULONG_MAX)
1683                         return -EINVAL;
1684
1685                 /* VFIO historically tries to auto-detect a kthread */
1686                 if (!current->mm)
1687                         flags |= IOMMUFD_ACCESS_RW_KTHREAD;
1688                 if (write)
1689                         flags |= IOMMUFD_ACCESS_RW_WRITE;
1690                 return iommufd_access_rw(device->iommufd_access, iova, data,
1691                                          len, flags);
1692         }
1693         return -EINVAL;
1694 }
1695 EXPORT_SYMBOL(vfio_dma_rw);
1696
1697 /*
1698  * Module/class support
1699  */
1700 static int __init vfio_init(void)
1701 {
1702         int ret;
1703
1704         ida_init(&vfio.device_ida);
1705
1706         ret = vfio_group_init();
1707         if (ret)
1708                 return ret;
1709
1710         ret = vfio_virqfd_init();
1711         if (ret)
1712                 goto err_virqfd;
1713
1714         /* /sys/class/vfio-dev/vfioX */
1715         vfio.device_class = class_create("vfio-dev");
1716         if (IS_ERR(vfio.device_class)) {
1717                 ret = PTR_ERR(vfio.device_class);
1718                 goto err_dev_class;
1719         }
1720
1721         ret = vfio_cdev_init(vfio.device_class);
1722         if (ret)
1723                 goto err_alloc_dev_chrdev;
1724
1725         vfio_debugfs_create_root();
1726         pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1727         return 0;
1728
1729 err_alloc_dev_chrdev:
1730         class_destroy(vfio.device_class);
1731         vfio.device_class = NULL;
1732 err_dev_class:
1733         vfio_virqfd_exit();
1734 err_virqfd:
1735         vfio_group_cleanup();
1736         return ret;
1737 }
1738
1739 static void __exit vfio_cleanup(void)
1740 {
1741         vfio_debugfs_remove_root();
1742         ida_destroy(&vfio.device_ida);
1743         vfio_cdev_cleanup();
1744         class_destroy(vfio.device_class);
1745         vfio.device_class = NULL;
1746         vfio_virqfd_exit();
1747         vfio_group_cleanup();
1748         xa_destroy(&vfio_device_set_xa);
1749 }
1750
1751 module_init(vfio_init);
1752 module_exit(vfio_cleanup);
1753
1754 MODULE_IMPORT_NS(IOMMUFD);
1755 MODULE_VERSION(DRIVER_VERSION);
1756 MODULE_LICENSE("GPL v2");
1757 MODULE_AUTHOR(DRIVER_AUTHOR);
1758 MODULE_DESCRIPTION(DRIVER_DESC);
1759 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");