drivers/vfio/vfio_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VFIO core
   4  *
   5  * Copyright (C) 2012 Red Hat, Inc.  All rights reserved.
   6  *     Author: Alex Williamson <alex.williamson@redhat.com>
   7  *
   8  * Derived from original vfio:
   9  * Copyright 2010 Cisco Systems, Inc.  All rights reserved.
  10  * Author: Tom Lyon, pugs@cisco.com
  11  */
  12
  13 #include <linux/cdev.h>
  14 #include <linux/compat.h>
  15 #include <linux/device.h>
  16 #include <linux/fs.h>
  17 #include <linux/idr.h>
  18 #include <linux/iommu.h>
  19 #ifdef CONFIG_HAVE_KVM
  20 #include <linux/kvm_host.h>
  21 #endif
  22 #include <linux/list.h>
  23 #include <linux/miscdevice.h>
  24 #include <linux/module.h>
  25 #include <linux/mutex.h>
  26 #include <linux/pci.h>
  27 #include <linux/rwsem.h>
  28 #include <linux/sched.h>
  29 #include <linux/slab.h>
  30 #include <linux/stat.h>
  31 #include <linux/string.h>
  32 #include <linux/uaccess.h>
  33 #include <linux/vfio.h>
  34 #include <linux/wait.h>
  35 #include <linux/sched/signal.h>
  36 #include <linux/pm_runtime.h>
  37 #include <linux/interval_tree.h>
  38 #include <linux/iova_bitmap.h>
  39 #include <linux/iommufd.h>
  40 #include "vfio.h"
  41
  42 #define DRIVER_VERSION  "0.3"
  43 #define DRIVER_AUTHOR   "Alex Williamson <alex.williamson@redhat.com>"
  44 #define DRIVER_DESC     "VFIO - User Level meta-driver"
  45
  46 static struct vfio {
  47         struct class                    *device_class;
  48         struct ida                      device_ida;
  49 } vfio;
  50
  51 #ifdef CONFIG_VFIO_NOIOMMU
  52 bool vfio_noiommu __read_mostly;
  53 module_param_named(enable_unsafe_noiommu_mode,
  54                    vfio_noiommu, bool, S_IRUGO | S_IWUSR);
  55 MODULE_PARM_DESC(enable_unsafe_noiommu_mode, "Enable UNSAFE, no-IOMMU mode.  This mode provides no device isolation, no DMA translation, no host kernel protection, cannot be used for device assignment to virtual machines, requires RAWIO permissions, and will taint the kernel.  If you do not know what this is for, step away. (default: false)");
  56 #endif
  57
  58 static DEFINE_XARRAY(vfio_device_set_xa);
  59
  60 int vfio_assign_device_set(struct vfio_device *device, void *set_id)
  61 {
  62         unsigned long idx = (unsigned long)set_id;
  63         struct vfio_device_set *new_dev_set;
  64         struct vfio_device_set *dev_set;
  65
  66         if (WARN_ON(!set_id))
  67                 return -EINVAL;
  68
  69         /*
  70          * Atomically acquire a singleton object in the xarray for this set_id
  71          */
  72         xa_lock(&vfio_device_set_xa);
  73         dev_set = xa_load(&vfio_device_set_xa, idx);
  74         if (dev_set)
  75                 goto found_get_ref;
  76         xa_unlock(&vfio_device_set_xa);
  77
  78         new_dev_set = kzalloc(sizeof(*new_dev_set), GFP_KERNEL);
  79         if (!new_dev_set)
  80                 return -ENOMEM;
  81         mutex_init(&new_dev_set->lock);
  82         INIT_LIST_HEAD(&new_dev_set->device_list);
  83         new_dev_set->set_id = set_id;
  84
  85         xa_lock(&vfio_device_set_xa);
  86         dev_set = __xa_cmpxchg(&vfio_device_set_xa, idx, NULL, new_dev_set,
  87                                GFP_KERNEL);
  88         if (!dev_set) {
  89                 dev_set = new_dev_set;
  90                 goto found_get_ref;
  91         }
  92
  93         kfree(new_dev_set);
  94         if (xa_is_err(dev_set)) {
  95                 xa_unlock(&vfio_device_set_xa);
  96                 return xa_err(dev_set);
  97         }
  98
  99 found_get_ref:
 100         dev_set->device_count++;
 101         xa_unlock(&vfio_device_set_xa);
 102         mutex_lock(&dev_set->lock);
 103         device->dev_set = dev_set;
 104         list_add_tail(&device->dev_set_list, &dev_set->device_list);
 105         mutex_unlock(&dev_set->lock);
 106         return 0;
 107 }
 108 EXPORT_SYMBOL_GPL(vfio_assign_device_set);
 109
 110 static void vfio_release_device_set(struct vfio_device *device)
 111 {
 112         struct vfio_device_set *dev_set = device->dev_set;
 113
 114         if (!dev_set)
 115                 return;
 116
 117         mutex_lock(&dev_set->lock);
 118         list_del(&device->dev_set_list);
 119         mutex_unlock(&dev_set->lock);
 120
 121         xa_lock(&vfio_device_set_xa);
 122         if (!--dev_set->device_count) {
 123                 __xa_erase(&vfio_device_set_xa,
 124                            (unsigned long)dev_set->set_id);
 125                 mutex_destroy(&dev_set->lock);
 126                 kfree(dev_set);
 127         }
 128         xa_unlock(&vfio_device_set_xa);
 129 }
 130
 131 unsigned int vfio_device_set_open_count(struct vfio_device_set *dev_set)
 132 {
 133         struct vfio_device *cur;
 134         unsigned int open_count = 0;
 135
 136         lockdep_assert_held(&dev_set->lock);
 137
 138         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 139                 open_count += cur->open_count;
 140         return open_count;
 141 }
 142 EXPORT_SYMBOL_GPL(vfio_device_set_open_count);
 143
 144 struct vfio_device *
 145 vfio_find_device_in_devset(struct vfio_device_set *dev_set,
 146                            struct device *dev)
 147 {
 148         struct vfio_device *cur;
 149
 150         lockdep_assert_held(&dev_set->lock);
 151
 152         list_for_each_entry(cur, &dev_set->device_list, dev_set_list)
 153                 if (cur->dev == dev)
 154                         return cur;
 155         return NULL;
 156 }
 157 EXPORT_SYMBOL_GPL(vfio_find_device_in_devset);
 158
 159 /*
 160  * Device objects - create, release, get, put, search
 161  */
 162 /* Device reference always implies a group reference */
 163 void vfio_device_put_registration(struct vfio_device *device)
 164 {
 165         if (refcount_dec_and_test(&device->refcount))
 166                 complete(&device->comp);
 167 }
 168
 169 bool vfio_device_try_get_registration(struct vfio_device *device)
 170 {
 171         return refcount_inc_not_zero(&device->refcount);
 172 }
 173
 174 /*
 175  * VFIO driver API
 176  */
 177 /* Release helper called by vfio_put_device() */
 178 static void vfio_device_release(struct device *dev)
 179 {
 180         struct vfio_device *device =
 181                         container_of(dev, struct vfio_device, device);
 182
 183         vfio_release_device_set(device);
 184         ida_free(&vfio.device_ida, device->index);
 185
 186         if (device->ops->release)
 187                 device->ops->release(device);
 188
 189         kvfree(device);
 190 }
 191
 192 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 193                             const struct vfio_device_ops *ops);
 194
 195 /*
 196  * Allocate and initialize vfio_device so it can be registered to vfio
 197  * core.
 198  *
 199  * Drivers should use the wrapper vfio_alloc_device() for allocation.
 200  * @size is the size of the structure to be allocated, including any
 201  * private data used by the driver.
 202  *
 203  * Driver may provide an @init callback to cover device private data.
 204  *
 205  * Use vfio_put_device() to release the structure after success return.
 206  */
 207 struct vfio_device *_vfio_alloc_device(size_t size, struct device *dev,
 208                                        const struct vfio_device_ops *ops)
 209 {
 210         struct vfio_device *device;
 211         int ret;
 212
 213         if (WARN_ON(size < sizeof(struct vfio_device)))
 214                 return ERR_PTR(-EINVAL);
 215
 216         device = kvzalloc(size, GFP_KERNEL);
 217         if (!device)
 218                 return ERR_PTR(-ENOMEM);
 219
 220         ret = vfio_init_device(device, dev, ops);
 221         if (ret)
 222                 goto out_free;
 223         return device;
 224
 225 out_free:
 226         kvfree(device);
 227         return ERR_PTR(ret);
 228 }
 229 EXPORT_SYMBOL_GPL(_vfio_alloc_device);
 230
 231 /*
 232  * Initialize a vfio_device so it can be registered to vfio core.
 233  */
 234 static int vfio_init_device(struct vfio_device *device, struct device *dev,
 235                             const struct vfio_device_ops *ops)
 236 {
 237         int ret;
 238
 239         ret = ida_alloc_max(&vfio.device_ida, MINORMASK, GFP_KERNEL);
 240         if (ret < 0) {
 241                 dev_dbg(dev, "Error to alloc index\n");
 242                 return ret;
 243         }
 244
 245         device->index = ret;
 246         init_completion(&device->comp);
 247         device->dev = dev;
 248         device->ops = ops;
 249
 250         if (ops->init) {
 251                 ret = ops->init(device);
 252                 if (ret)
 253                         goto out_uninit;
 254         }
 255
 256         device_initialize(&device->device);
 257         device->device.release = vfio_device_release;
 258         device->device.class = vfio.device_class;
 259         device->device.parent = device->dev;
 260         return 0;
 261
 262 out_uninit:
 263         vfio_release_device_set(device);
 264         ida_free(&vfio.device_ida, device->index);
 265         return ret;
 266 }
 267
 268 static int __vfio_register_dev(struct vfio_device *device,
 269                                enum vfio_group_type type)
 270 {
 271         int ret;
 272
 273         if (WARN_ON(IS_ENABLED(CONFIG_IOMMUFD) &&
 274                     (!device->ops->bind_iommufd ||
 275                      !device->ops->unbind_iommufd ||
 276                      !device->ops->attach_ioas ||
 277                      !device->ops->detach_ioas)))
 278                 return -EINVAL;
 279
 280         /*
 281          * If the driver doesn't specify a set then the device is added to a
 282          * singleton set just for itself.
 283          */
 284         if (!device->dev_set)
 285                 vfio_assign_device_set(device, device);
 286
 287         ret = dev_set_name(&device->device, "vfio%d", device->index);
 288         if (ret)
 289                 return ret;
 290
 291         ret = vfio_device_set_group(device, type);
 292         if (ret)
 293                 return ret;
 294
 295         /*
 296          * VFIO always sets IOMMU_CACHE because we offer no way for userspace to
 297          * restore cache coherency. It has to be checked here because it is only
 298          * valid for cases where we are using iommu groups.
 299          */
 300         if (type == VFIO_IOMMU && !vfio_device_is_noiommu(device) &&
 301             !device_iommu_capable(device->dev, IOMMU_CAP_CACHE_COHERENCY)) {
 302                 ret = -EINVAL;
 303                 goto err_out;
 304         }
 305
 306         ret = vfio_device_add(device);
 307         if (ret)
 308                 goto err_out;
 309
 310         /* Refcounting can't start until the driver calls register */
 311         refcount_set(&device->refcount, 1);
 312
 313         vfio_device_group_register(device);
 314
 315         return 0;
 316 err_out:
 317         vfio_device_remove_group(device);
 318         return ret;
 319 }
 320
 321 int vfio_register_group_dev(struct vfio_device *device)
 322 {
 323         return __vfio_register_dev(device, VFIO_IOMMU);
 324 }
 325 EXPORT_SYMBOL_GPL(vfio_register_group_dev);
 326
 327 /*
 328  * Register a virtual device without IOMMU backing.  The user of this
 329  * device must not be able to directly trigger unmediated DMA.
 330  */
 331 int vfio_register_emulated_iommu_dev(struct vfio_device *device)
 332 {
 333         return __vfio_register_dev(device, VFIO_EMULATED_IOMMU);
 334 }
 335 EXPORT_SYMBOL_GPL(vfio_register_emulated_iommu_dev);
 336
 337 /*
 338  * Decrement the device reference count and wait for the device to be
 339  * removed.  Open file descriptors for the device... */
 340 void vfio_unregister_group_dev(struct vfio_device *device)
 341 {
 342         unsigned int i = 0;
 343         bool interrupted = false;
 344         long rc;
 345
 346         /*
 347          * Prevent new device opened by userspace via the
 348          * VFIO_GROUP_GET_DEVICE_FD in the group path.
 349          */
 350         vfio_device_group_unregister(device);
 351
 352         /*
 353          * Balances vfio_device_add() in register path, also prevents
 354          * new device opened by userspace in the cdev path.
 355          */
 356         vfio_device_del(device);
 357
 358         vfio_device_put_registration(device);
 359         rc = try_wait_for_completion(&device->comp);
 360         while (rc <= 0) {
 361                 if (device->ops->request)
 362                         device->ops->request(device, i++);
 363
 364                 if (interrupted) {
 365                         rc = wait_for_completion_timeout(&device->comp,
 366                                                          HZ * 10);
 367                 } else {
 368                         rc = wait_for_completion_interruptible_timeout(
 369                                 &device->comp, HZ * 10);
 370                         if (rc < 0) {
 371                                 interrupted = true;
 372                                 dev_warn(device->dev,
 373                                          "Device is currently in use, task"
 374                                          " \"%s\" (%d) "
 375                                          "blocked until device is released",
 376                                          current->comm, task_pid_nr(current));
 377                         }
 378                 }
 379         }
 380
 381         /* Balances vfio_device_set_group in register path */
 382         vfio_device_remove_group(device);
 383 }
 384 EXPORT_SYMBOL_GPL(vfio_unregister_group_dev);
 385
 386 #ifdef CONFIG_HAVE_KVM
 387 void vfio_device_get_kvm_safe(struct vfio_device *device, struct kvm *kvm)
 388 {
 389         void (*pfn)(struct kvm *kvm);
 390         bool (*fn)(struct kvm *kvm);
 391         bool ret;
 392
 393         lockdep_assert_held(&device->dev_set->lock);
 394
 395         if (!kvm)
 396                 return;
 397
 398         pfn = symbol_get(kvm_put_kvm);
 399         if (WARN_ON(!pfn))
 400                 return;
 401
 402         fn = symbol_get(kvm_get_kvm_safe);
 403         if (WARN_ON(!fn)) {
 404                 symbol_put(kvm_put_kvm);
 405                 return;
 406         }
 407
 408         ret = fn(kvm);
 409         symbol_put(kvm_get_kvm_safe);
 410         if (!ret) {
 411                 symbol_put(kvm_put_kvm);
 412                 return;
 413         }
 414
 415         device->put_kvm = pfn;
 416         device->kvm = kvm;
 417 }
 418
 419 void vfio_device_put_kvm(struct vfio_device *device)
 420 {
 421         lockdep_assert_held(&device->dev_set->lock);
 422
 423         if (!device->kvm)
 424                 return;
 425
 426         if (WARN_ON(!device->put_kvm))
 427                 goto clear;
 428
 429         device->put_kvm(device->kvm);
 430         device->put_kvm = NULL;
 431         symbol_put(kvm_put_kvm);
 432
 433 clear:
 434         device->kvm = NULL;
 435 }
 436 #endif
 437
 438 /* true if the vfio_device has open_device() called but not close_device() */
 439 static bool vfio_assert_device_open(struct vfio_device *device)
 440 {
 441         return !WARN_ON_ONCE(!READ_ONCE(device->open_count));
 442 }
 443
 444 struct vfio_device_file *
 445 vfio_allocate_device_file(struct vfio_device *device)
 446 {
 447         struct vfio_device_file *df;
 448
 449         df = kzalloc(sizeof(*df), GFP_KERNEL_ACCOUNT);
 450         if (!df)
 451                 return ERR_PTR(-ENOMEM);
 452
 453         df->device = device;
 454         spin_lock_init(&df->kvm_ref_lock);
 455
 456         return df;
 457 }
 458
 459 static int vfio_df_device_first_open(struct vfio_device_file *df)
 460 {
 461         struct vfio_device *device = df->device;
 462         struct iommufd_ctx *iommufd = df->iommufd;
 463         int ret;
 464
 465         lockdep_assert_held(&device->dev_set->lock);
 466
 467         if (!try_module_get(device->dev->driver->owner))
 468                 return -ENODEV;
 469
 470         if (iommufd)
 471                 ret = vfio_df_iommufd_bind(df);
 472         else
 473                 ret = vfio_device_group_use_iommu(device);
 474         if (ret)
 475                 goto err_module_put;
 476
 477         if (device->ops->open_device) {
 478                 ret = device->ops->open_device(device);
 479                 if (ret)
 480                         goto err_unuse_iommu;
 481         }
 482         return 0;
 483
 484 err_unuse_iommu:
 485         if (iommufd)
 486                 vfio_df_iommufd_unbind(df);
 487         else
 488                 vfio_device_group_unuse_iommu(device);
 489 err_module_put:
 490         module_put(device->dev->driver->owner);
 491         return ret;
 492 }
 493
 494 static void vfio_df_device_last_close(struct vfio_device_file *df)
 495 {
 496         struct vfio_device *device = df->device;
 497         struct iommufd_ctx *iommufd = df->iommufd;
 498
 499         lockdep_assert_held(&device->dev_set->lock);
 500
 501         if (device->ops->close_device)
 502                 device->ops->close_device(device);
 503         if (iommufd)
 504                 vfio_df_iommufd_unbind(df);
 505         else
 506                 vfio_device_group_unuse_iommu(device);
 507         module_put(device->dev->driver->owner);
 508 }
 509
 510 int vfio_df_open(struct vfio_device_file *df)
 511 {
 512         struct vfio_device *device = df->device;
 513         int ret = 0;
 514
 515         lockdep_assert_held(&device->dev_set->lock);
 516
 517         /*
 518          * Only the group path allows the device to be opened multiple
 519          * times.  The device cdev path doesn't have a secure way for it.
 520          */
 521         if (device->open_count != 0 && !df->group)
 522                 return -EINVAL;
 523
 524         device->open_count++;
 525         if (device->open_count == 1) {
 526                 ret = vfio_df_device_first_open(df);
 527                 if (ret)
 528                         device->open_count--;
 529         }
 530
 531         return ret;
 532 }
 533
 534 void vfio_df_close(struct vfio_device_file *df)
 535 {
 536         struct vfio_device *device = df->device;
 537
 538         lockdep_assert_held(&device->dev_set->lock);
 539
 540         vfio_assert_device_open(device);
 541         if (device->open_count == 1)
 542                 vfio_df_device_last_close(df);
 543         device->open_count--;
 544 }
 545
 546 /*
 547  * Wrapper around pm_runtime_resume_and_get().
 548  * Return error code on failure or 0 on success.
 549  */
 550 static inline int vfio_device_pm_runtime_get(struct vfio_device *device)
 551 {
 552         struct device *dev = device->dev;
 553
 554         if (dev->driver && dev->driver->pm) {
 555                 int ret;
 556
 557                 ret = pm_runtime_resume_and_get(dev);
 558                 if (ret) {
 559                         dev_info_ratelimited(dev,
 560                                 "vfio: runtime resume failed %d\n", ret);
 561                         return -EIO;
 562                 }
 563         }
 564
 565         return 0;
 566 }
 567
 568 /*
 569  * Wrapper around pm_runtime_put().
 570  */
 571 static inline void vfio_device_pm_runtime_put(struct vfio_device *device)
 572 {
 573         struct device *dev = device->dev;
 574
 575         if (dev->driver && dev->driver->pm)
 576                 pm_runtime_put(dev);
 577 }
 578
 579 /*
 580  * VFIO Device fd
 581  */
 582 static int vfio_device_fops_release(struct inode *inode, struct file *filep)
 583 {
 584         struct vfio_device_file *df = filep->private_data;
 585         struct vfio_device *device = df->device;
 586
 587         if (df->group)
 588                 vfio_df_group_close(df);
 589         else
 590                 vfio_df_unbind_iommufd(df);
 591
 592         vfio_device_put_registration(device);
 593
 594         kfree(df);
 595
 596         return 0;
 597 }
 598
 599 /*
 600  * vfio_mig_get_next_state - Compute the next step in the FSM
 601  * @cur_fsm - The current state the device is in
 602  * @new_fsm - The target state to reach
 603  * @next_fsm - Pointer to the next step to get to new_fsm
 604  *
 605  * Return 0 upon success, otherwise -errno
 606  * Upon success the next step in the state progression between cur_fsm and
 607  * new_fsm will be set in next_fsm.
 608  *
 609  * This breaks down requests for combination transitions into smaller steps and
 610  * returns the next step to get to new_fsm. The function may need to be called
 611  * multiple times before reaching new_fsm.
 612  *
 613  */
 614 int vfio_mig_get_next_state(struct vfio_device *device,
 615                             enum vfio_device_mig_state cur_fsm,
 616                             enum vfio_device_mig_state new_fsm,
 617                             enum vfio_device_mig_state *next_fsm)
 618 {
 619         enum { VFIO_DEVICE_NUM_STATES = VFIO_DEVICE_STATE_PRE_COPY_P2P + 1 };
 620         /*
 621          * The coding in this table requires the driver to implement the
 622          * following FSM arcs:
 623          *         RESUMING -> STOP
 624          *         STOP -> RESUMING
 625          *         STOP -> STOP_COPY
 626          *         STOP_COPY -> STOP
 627          *
 628          * If P2P is supported then the driver must also implement these FSM
 629          * arcs:
 630          *         RUNNING -> RUNNING_P2P
 631          *         RUNNING_P2P -> RUNNING
 632          *         RUNNING_P2P -> STOP
 633          *         STOP -> RUNNING_P2P
 634          *
 635          * If precopy is supported then the driver must support these additional
 636          * FSM arcs:
 637          *         RUNNING -> PRE_COPY
 638          *         PRE_COPY -> RUNNING
 639          *         PRE_COPY -> STOP_COPY
 640          * However, if precopy and P2P are supported together then the driver
 641          * must support these additional arcs beyond the P2P arcs above:
 642          *         PRE_COPY -> RUNNING
 643          *         PRE_COPY -> PRE_COPY_P2P
 644          *         PRE_COPY_P2P -> PRE_COPY
 645          *         PRE_COPY_P2P -> RUNNING_P2P
 646          *         PRE_COPY_P2P -> STOP_COPY
 647          *         RUNNING -> PRE_COPY
 648          *         RUNNING_P2P -> PRE_COPY_P2P
 649          *
 650          * Without P2P and precopy the driver must implement:
 651          *         RUNNING -> STOP
 652          *         STOP -> RUNNING
 653          *
 654          * The coding will step through multiple states for some combination
 655          * transitions; if all optional features are supported, this means the
 656          * following ones:
 657          *         PRE_COPY -> PRE_COPY_P2P -> STOP_COPY
 658          *         PRE_COPY -> RUNNING -> RUNNING_P2P
 659          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP
 660          *         PRE_COPY -> RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 661          *         PRE_COPY_P2P -> RUNNING_P2P -> RUNNING
 662          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP
 663          *         PRE_COPY_P2P -> RUNNING_P2P -> STOP -> RESUMING
 664          *         RESUMING -> STOP -> RUNNING_P2P
 665          *         RESUMING -> STOP -> RUNNING_P2P -> PRE_COPY_P2P
 666          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING
 667          *         RESUMING -> STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 668          *         RESUMING -> STOP -> STOP_COPY
 669          *         RUNNING -> RUNNING_P2P -> PRE_COPY_P2P
 670          *         RUNNING -> RUNNING_P2P -> STOP
 671          *         RUNNING -> RUNNING_P2P -> STOP -> RESUMING
 672          *         RUNNING -> RUNNING_P2P -> STOP -> STOP_COPY
 673          *         RUNNING_P2P -> RUNNING -> PRE_COPY
 674          *         RUNNING_P2P -> STOP -> RESUMING
 675          *         RUNNING_P2P -> STOP -> STOP_COPY
 676          *         STOP -> RUNNING_P2P -> PRE_COPY_P2P
 677          *         STOP -> RUNNING_P2P -> RUNNING
 678          *         STOP -> RUNNING_P2P -> RUNNING -> PRE_COPY
 679          *         STOP_COPY -> STOP -> RESUMING
 680          *         STOP_COPY -> STOP -> RUNNING_P2P
 681          *         STOP_COPY -> STOP -> RUNNING_P2P -> RUNNING
 682          *
 683          *  The following transitions are blocked:
 684          *         STOP_COPY -> PRE_COPY
 685          *         STOP_COPY -> PRE_COPY_P2P
 686          */
 687         static const u8 vfio_from_fsm_table[VFIO_DEVICE_NUM_STATES][VFIO_DEVICE_NUM_STATES] = {
 688                 [VFIO_DEVICE_STATE_STOP] = {
 689                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 690                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 691                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 692                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 693                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 694                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 695                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 696                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 697                 },
 698                 [VFIO_DEVICE_STATE_RUNNING] = {
 699                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 700                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 701                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 702                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 703                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_RUNNING_P2P,
 704                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 705                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 706                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 707                 },
 708                 [VFIO_DEVICE_STATE_PRE_COPY] = {
 709                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING,
 710                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 711                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 712                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 713                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 714                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING,
 715                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING,
 716                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 717                 },
 718                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = {
 719                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_RUNNING_P2P,
 720                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 721                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_PRE_COPY,
 722                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 723                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 724                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RUNNING_P2P,
 725                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 726                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 727                 },
 728                 [VFIO_DEVICE_STATE_STOP_COPY] = {
 729                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 730                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 731                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 732                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 733                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP_COPY,
 734                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 735                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 736                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 737                 },
 738                 [VFIO_DEVICE_STATE_RESUMING] = {
 739                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 740                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_STOP,
 741                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_STOP,
 742                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_STOP,
 743                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 744                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_RESUMING,
 745                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_STOP,
 746                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 747                 },
 748                 [VFIO_DEVICE_STATE_RUNNING_P2P] = {
 749                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_STOP,
 750                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_RUNNING,
 751                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_RUNNING,
 752                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_PRE_COPY_P2P,
 753                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_STOP,
 754                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_STOP,
 755                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_RUNNING_P2P,
 756                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 757                 },
 758                 [VFIO_DEVICE_STATE_ERROR] = {
 759                         [VFIO_DEVICE_STATE_STOP] = VFIO_DEVICE_STATE_ERROR,
 760                         [VFIO_DEVICE_STATE_RUNNING] = VFIO_DEVICE_STATE_ERROR,
 761                         [VFIO_DEVICE_STATE_PRE_COPY] = VFIO_DEVICE_STATE_ERROR,
 762                         [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_DEVICE_STATE_ERROR,
 763                         [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_DEVICE_STATE_ERROR,
 764                         [VFIO_DEVICE_STATE_RESUMING] = VFIO_DEVICE_STATE_ERROR,
 765                         [VFIO_DEVICE_STATE_RUNNING_P2P] = VFIO_DEVICE_STATE_ERROR,
 766                         [VFIO_DEVICE_STATE_ERROR] = VFIO_DEVICE_STATE_ERROR,
 767                 },
 768         };
 769
 770         static const unsigned int state_flags_table[VFIO_DEVICE_NUM_STATES] = {
 771                 [VFIO_DEVICE_STATE_STOP] = VFIO_MIGRATION_STOP_COPY,
 772                 [VFIO_DEVICE_STATE_RUNNING] = VFIO_MIGRATION_STOP_COPY,
 773                 [VFIO_DEVICE_STATE_PRE_COPY] =
 774                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_PRE_COPY,
 775                 [VFIO_DEVICE_STATE_PRE_COPY_P2P] = VFIO_MIGRATION_STOP_COPY |
 776                                                    VFIO_MIGRATION_P2P |
 777                                                    VFIO_MIGRATION_PRE_COPY,
 778                 [VFIO_DEVICE_STATE_STOP_COPY] = VFIO_MIGRATION_STOP_COPY,
 779                 [VFIO_DEVICE_STATE_RESUMING] = VFIO_MIGRATION_STOP_COPY,
 780                 [VFIO_DEVICE_STATE_RUNNING_P2P] =
 781                         VFIO_MIGRATION_STOP_COPY | VFIO_MIGRATION_P2P,
 782                 [VFIO_DEVICE_STATE_ERROR] = ~0U,
 783         };
 784
 785         if (WARN_ON(cur_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 786                     (state_flags_table[cur_fsm] & device->migration_flags) !=
 787                         state_flags_table[cur_fsm]))
 788                 return -EINVAL;
 789
 790         if (new_fsm >= ARRAY_SIZE(vfio_from_fsm_table) ||
 791            (state_flags_table[new_fsm] & device->migration_flags) !=
 792                         state_flags_table[new_fsm])
 793                 return -EINVAL;
 794
 795         /*
 796          * Arcs touching optional and unsupported states are skipped over. The
 797          * driver will instead see an arc from the original state to the next
 798          * logical state, as per the above comment.
 799          */
 800         *next_fsm = vfio_from_fsm_table[cur_fsm][new_fsm];
 801         while ((state_flags_table[*next_fsm] & device->migration_flags) !=
 802                         state_flags_table[*next_fsm])
 803                 *next_fsm = vfio_from_fsm_table[*next_fsm][new_fsm];
 804
 805         return (*next_fsm != VFIO_DEVICE_STATE_ERROR) ? 0 : -EINVAL;
 806 }
 807 EXPORT_SYMBOL_GPL(vfio_mig_get_next_state);
 808
 809 /*
 810  * Convert the drivers's struct file into a FD number and return it to userspace
 811  */
 812 static int vfio_ioct_mig_return_fd(struct file *filp, void __user *arg,
 813                                    struct vfio_device_feature_mig_state *mig)
 814 {
 815         int ret;
 816         int fd;
 817
 818         fd = get_unused_fd_flags(O_CLOEXEC);
 819         if (fd < 0) {
 820                 ret = fd;
 821                 goto out_fput;
 822         }
 823
 824         mig->data_fd = fd;
 825         if (copy_to_user(arg, mig, sizeof(*mig))) {
 826                 ret = -EFAULT;
 827                 goto out_put_unused;
 828         }
 829         fd_install(fd, filp);
 830         return 0;
 831
 832 out_put_unused:
 833         put_unused_fd(fd);
 834 out_fput:
 835         fput(filp);
 836         return ret;
 837 }
 838
 839 static int
 840 vfio_ioctl_device_feature_mig_device_state(struct vfio_device *device,
 841                                            u32 flags, void __user *arg,
 842                                            size_t argsz)
 843 {
 844         size_t minsz =
 845                 offsetofend(struct vfio_device_feature_mig_state, data_fd);
 846         struct vfio_device_feature_mig_state mig;
 847         struct file *filp = NULL;
 848         int ret;
 849
 850         if (!device->mig_ops)
 851                 return -ENOTTY;
 852
 853         ret = vfio_check_feature(flags, argsz,
 854                                  VFIO_DEVICE_FEATURE_SET |
 855                                  VFIO_DEVICE_FEATURE_GET,
 856                                  sizeof(mig));
 857         if (ret != 1)
 858                 return ret;
 859
 860         if (copy_from_user(&mig, arg, minsz))
 861                 return -EFAULT;
 862
 863         if (flags & VFIO_DEVICE_FEATURE_GET) {
 864                 enum vfio_device_mig_state curr_state;
 865
 866                 ret = device->mig_ops->migration_get_state(device,
 867                                                            &curr_state);
 868                 if (ret)
 869                         return ret;
 870                 mig.device_state = curr_state;
 871                 goto out_copy;
 872         }
 873
 874         /* Handle the VFIO_DEVICE_FEATURE_SET */
 875         filp = device->mig_ops->migration_set_state(device, mig.device_state);
 876         if (IS_ERR(filp) || !filp)
 877                 goto out_copy;
 878
 879         return vfio_ioct_mig_return_fd(filp, arg, &mig);
 880 out_copy:
 881         mig.data_fd = -1;
 882         if (copy_to_user(arg, &mig, sizeof(mig)))
 883                 return -EFAULT;
 884         if (IS_ERR(filp))
 885                 return PTR_ERR(filp);
 886         return 0;
 887 }
 888
 889 static int
 890 vfio_ioctl_device_feature_migration_data_size(struct vfio_device *device,
 891                                               u32 flags, void __user *arg,
 892                                               size_t argsz)
 893 {
 894         struct vfio_device_feature_mig_data_size data_size = {};
 895         unsigned long stop_copy_length;
 896         int ret;
 897
 898         if (!device->mig_ops)
 899                 return -ENOTTY;
 900
 901         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 902                                  sizeof(data_size));
 903         if (ret != 1)
 904                 return ret;
 905
 906         ret = device->mig_ops->migration_get_data_size(device, &stop_copy_length);
 907         if (ret)
 908                 return ret;
 909
 910         data_size.stop_copy_length = stop_copy_length;
 911         if (copy_to_user(arg, &data_size, sizeof(data_size)))
 912                 return -EFAULT;
 913
 914         return 0;
 915 }
 916
 917 static int vfio_ioctl_device_feature_migration(struct vfio_device *device,
 918                                                u32 flags, void __user *arg,
 919                                                size_t argsz)
 920 {
 921         struct vfio_device_feature_migration mig = {
 922                 .flags = device->migration_flags,
 923         };
 924         int ret;
 925
 926         if (!device->mig_ops)
 927                 return -ENOTTY;
 928
 929         ret = vfio_check_feature(flags, argsz, VFIO_DEVICE_FEATURE_GET,
 930                                  sizeof(mig));
 931         if (ret != 1)
 932                 return ret;
 933         if (copy_to_user(arg, &mig, sizeof(mig)))
 934                 return -EFAULT;
 935         return 0;
 936 }
 937
 938 void vfio_combine_iova_ranges(struct rb_root_cached *root, u32 cur_nodes,
 939                               u32 req_nodes)
 940 {
 941         struct interval_tree_node *prev, *curr, *comb_start, *comb_end;
 942         unsigned long min_gap, curr_gap;
 943
 944         /* Special shortcut when a single range is required */
 945         if (req_nodes == 1) {
 946                 unsigned long last;
 947
 948                 comb_start = interval_tree_iter_first(root, 0, ULONG_MAX);
 949
 950                 /* Empty list */
 951                 if (WARN_ON_ONCE(!comb_start))
 952                         return;
 953
 954                 curr = comb_start;
 955                 while (curr) {
 956                         last = curr->last;
 957                         prev = curr;
 958                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
 959                         if (prev != comb_start)
 960                                 interval_tree_remove(prev, root);
 961                 }
 962                 comb_start->last = last;
 963                 return;
 964         }
 965
 966         /* Combine ranges which have the smallest gap */
 967         while (cur_nodes > req_nodes) {
 968                 prev = NULL;
 969                 min_gap = ULONG_MAX;
 970                 curr = interval_tree_iter_first(root, 0, ULONG_MAX);
 971                 while (curr) {
 972                         if (prev) {
 973                                 curr_gap = curr->start - prev->last;
 974                                 if (curr_gap < min_gap) {
 975                                         min_gap = curr_gap;
 976                                         comb_start = prev;
 977                                         comb_end = curr;
 978                                 }
 979                         }
 980                         prev = curr;
 981                         curr = interval_tree_iter_next(curr, 0, ULONG_MAX);
 982                 }
 983
 984                 /* Empty list or no nodes to combine */
 985                 if (WARN_ON_ONCE(min_gap == ULONG_MAX))
 986                         break;
 987
 988                 comb_start->last = comb_end->last;
 989                 interval_tree_remove(comb_end, root);
 990                 cur_nodes--;
 991         }
 992 }
 993 EXPORT_SYMBOL_GPL(vfio_combine_iova_ranges);
 994
 995 /* Ranges should fit into a single kernel page */
 996 #define LOG_MAX_RANGES \
 997         (PAGE_SIZE / sizeof(struct vfio_device_feature_dma_logging_range))
 998
 999 static int
1000 vfio_ioctl_device_feature_logging_start(struct vfio_device *device,
1001                                         u32 flags, void __user *arg,
1002                                         size_t argsz)
1003 {
1004         size_t minsz =
1005                 offsetofend(struct vfio_device_feature_dma_logging_control,
1006                             ranges);
1007         struct vfio_device_feature_dma_logging_range __user *ranges;
1008         struct vfio_device_feature_dma_logging_control control;
1009         struct vfio_device_feature_dma_logging_range range;
1010         struct rb_root_cached root = RB_ROOT_CACHED;
1011         struct interval_tree_node *nodes;
1012         u64 iova_end;
1013         u32 nnodes;
1014         int i, ret;
1015
1016         if (!device->log_ops)
1017                 return -ENOTTY;
1018
1019         ret = vfio_check_feature(flags, argsz,
1020                                  VFIO_DEVICE_FEATURE_SET,
1021                                  sizeof(control));
1022         if (ret != 1)
1023                 return ret;
1024
1025         if (copy_from_user(&control, arg, minsz))
1026                 return -EFAULT;
1027
1028         nnodes = control.num_ranges;
1029         if (!nnodes)
1030                 return -EINVAL;
1031
1032         if (nnodes > LOG_MAX_RANGES)
1033                 return -E2BIG;
1034
1035         ranges = u64_to_user_ptr(control.ranges);
1036         nodes = kmalloc_array(nnodes, sizeof(struct interval_tree_node),
1037                               GFP_KERNEL);
1038         if (!nodes)
1039                 return -ENOMEM;
1040
1041         for (i = 0; i < nnodes; i++) {
1042                 if (copy_from_user(&range, &ranges[i], sizeof(range))) {
1043                         ret = -EFAULT;
1044                         goto end;
1045                 }
1046                 if (!IS_ALIGNED(range.iova, control.page_size) ||
1047                     !IS_ALIGNED(range.length, control.page_size)) {
1048                         ret = -EINVAL;
1049                         goto end;
1050                 }
1051
1052                 if (check_add_overflow(range.iova, range.length, &iova_end) ||
1053                     iova_end > ULONG_MAX) {
1054                         ret = -EOVERFLOW;
1055                         goto end;
1056                 }
1057
1058                 nodes[i].start = range.iova;
1059                 nodes[i].last = range.iova + range.length - 1;
1060                 if (interval_tree_iter_first(&root, nodes[i].start,
1061                                              nodes[i].last)) {
1062                         /* Range overlapping */
1063                         ret = -EINVAL;
1064                         goto end;
1065                 }
1066                 interval_tree_insert(nodes + i, &root);
1067         }
1068
1069         ret = device->log_ops->log_start(device, &root, nnodes,
1070                                          &control.page_size);
1071         if (ret)
1072                 goto end;
1073
1074         if (copy_to_user(arg, &control, sizeof(control))) {
1075                 ret = -EFAULT;
1076                 device->log_ops->log_stop(device);
1077         }
1078
1079 end:
1080         kfree(nodes);
1081         return ret;
1082 }
1083
1084 static int
1085 vfio_ioctl_device_feature_logging_stop(struct vfio_device *device,
1086                                        u32 flags, void __user *arg,
1087                                        size_t argsz)
1088 {
1089         int ret;
1090
1091         if (!device->log_ops)
1092                 return -ENOTTY;
1093
1094         ret = vfio_check_feature(flags, argsz,
1095                                  VFIO_DEVICE_FEATURE_SET, 0);
1096         if (ret != 1)
1097                 return ret;
1098
1099         return device->log_ops->log_stop(device);
1100 }
1101
1102 static int vfio_device_log_read_and_clear(struct iova_bitmap *iter,
1103                                           unsigned long iova, size_t length,
1104                                           void *opaque)
1105 {
1106         struct vfio_device *device = opaque;
1107
1108         return device->log_ops->log_read_and_clear(device, iova, length, iter);
1109 }
1110
1111 static int
1112 vfio_ioctl_device_feature_logging_report(struct vfio_device *device,
1113                                          u32 flags, void __user *arg,
1114                                          size_t argsz)
1115 {
1116         size_t minsz =
1117                 offsetofend(struct vfio_device_feature_dma_logging_report,
1118                             bitmap);
1119         struct vfio_device_feature_dma_logging_report report;
1120         struct iova_bitmap *iter;
1121         u64 iova_end;
1122         int ret;
1123
1124         if (!device->log_ops)
1125                 return -ENOTTY;
1126
1127         ret = vfio_check_feature(flags, argsz,
1128                                  VFIO_DEVICE_FEATURE_GET,
1129                                  sizeof(report));
1130         if (ret != 1)
1131                 return ret;
1132
1133         if (copy_from_user(&report, arg, minsz))
1134                 return -EFAULT;
1135
1136         if (report.page_size < SZ_4K || !is_power_of_2(report.page_size))
1137                 return -EINVAL;
1138
1139         if (check_add_overflow(report.iova, report.length, &iova_end) ||
1140             iova_end > ULONG_MAX)
1141                 return -EOVERFLOW;
1142
1143         iter = iova_bitmap_alloc(report.iova, report.length,
1144                                  report.page_size,
1145                                  u64_to_user_ptr(report.bitmap));
1146         if (IS_ERR(iter))
1147                 return PTR_ERR(iter);
1148
1149         ret = iova_bitmap_for_each(iter, device,
1150                                    vfio_device_log_read_and_clear);
1151
1152         iova_bitmap_free(iter);
1153         return ret;
1154 }
1155
1156 static int vfio_ioctl_device_feature(struct vfio_device *device,
1157                                      struct vfio_device_feature __user *arg)
1158 {
1159         size_t minsz = offsetofend(struct vfio_device_feature, flags);
1160         struct vfio_device_feature feature;
1161
1162         if (copy_from_user(&feature, arg, minsz))
1163                 return -EFAULT;
1164
1165         if (feature.argsz < minsz)
1166                 return -EINVAL;
1167
1168         /* Check unknown flags */
1169         if (feature.flags &
1170             ~(VFIO_DEVICE_FEATURE_MASK | VFIO_DEVICE_FEATURE_SET |
1171               VFIO_DEVICE_FEATURE_GET | VFIO_DEVICE_FEATURE_PROBE))
1172                 return -EINVAL;
1173
1174         /* GET & SET are mutually exclusive except with PROBE */
1175         if (!(feature.flags & VFIO_DEVICE_FEATURE_PROBE) &&
1176             (feature.flags & VFIO_DEVICE_FEATURE_SET) &&
1177             (feature.flags & VFIO_DEVICE_FEATURE_GET))
1178                 return -EINVAL;
1179
1180         switch (feature.flags & VFIO_DEVICE_FEATURE_MASK) {
1181         case VFIO_DEVICE_FEATURE_MIGRATION:
1182                 return vfio_ioctl_device_feature_migration(
1183                         device, feature.flags, arg->data,
1184                         feature.argsz - minsz);
1185         case VFIO_DEVICE_FEATURE_MIG_DEVICE_STATE:
1186                 return vfio_ioctl_device_feature_mig_device_state(
1187                         device, feature.flags, arg->data,
1188                         feature.argsz - minsz);
1189         case VFIO_DEVICE_FEATURE_DMA_LOGGING_START:
1190                 return vfio_ioctl_device_feature_logging_start(
1191                         device, feature.flags, arg->data,
1192                         feature.argsz - minsz);
1193         case VFIO_DEVICE_FEATURE_DMA_LOGGING_STOP:
1194                 return vfio_ioctl_device_feature_logging_stop(
1195                         device, feature.flags, arg->data,
1196                         feature.argsz - minsz);
1197         case VFIO_DEVICE_FEATURE_DMA_LOGGING_REPORT:
1198                 return vfio_ioctl_device_feature_logging_report(
1199                         device, feature.flags, arg->data,
1200                         feature.argsz - minsz);
1201         case VFIO_DEVICE_FEATURE_MIG_DATA_SIZE:
1202                 return vfio_ioctl_device_feature_migration_data_size(
1203                         device, feature.flags, arg->data,
1204                         feature.argsz - minsz);
1205         default:
1206                 if (unlikely(!device->ops->device_feature))
1207                         return -EINVAL;
1208                 return device->ops->device_feature(device, feature.flags,
1209                                                    arg->data,
1210                                                    feature.argsz - minsz);
1211         }
1212 }
1213
1214 static long vfio_device_fops_unl_ioctl(struct file *filep,
1215                                        unsigned int cmd, unsigned long arg)
1216 {
1217         struct vfio_device_file *df = filep->private_data;
1218         struct vfio_device *device = df->device;
1219         void __user *uptr = (void __user *)arg;
1220         int ret;
1221
1222         if (cmd == VFIO_DEVICE_BIND_IOMMUFD)
1223                 return vfio_df_ioctl_bind_iommufd(df, uptr);
1224
1225         /* Paired with smp_store_release() following vfio_df_open() */
1226         if (!smp_load_acquire(&df->access_granted))
1227                 return -EINVAL;
1228
1229         ret = vfio_device_pm_runtime_get(device);
1230         if (ret)
1231                 return ret;
1232
1233         /* cdev only ioctls */
1234         if (IS_ENABLED(CONFIG_VFIO_DEVICE_CDEV) && !df->group) {
1235                 switch (cmd) {
1236                 case VFIO_DEVICE_ATTACH_IOMMUFD_PT:
1237                         ret = vfio_df_ioctl_attach_pt(df, uptr);
1238                         goto out;
1239
1240                 case VFIO_DEVICE_DETACH_IOMMUFD_PT:
1241                         ret = vfio_df_ioctl_detach_pt(df, uptr);
1242                         goto out;
1243                 }
1244         }
1245
1246         switch (cmd) {
1247         case VFIO_DEVICE_FEATURE:
1248                 ret = vfio_ioctl_device_feature(device, uptr);
1249                 break;
1250
1251         default:
1252                 if (unlikely(!device->ops->ioctl))
1253                         ret = -EINVAL;
1254                 else
1255                         ret = device->ops->ioctl(device, cmd, arg);
1256                 break;
1257         }
1258 out:
1259         vfio_device_pm_runtime_put(device);
1260         return ret;
1261 }
1262
1263 static ssize_t vfio_device_fops_read(struct file *filep, char __user *buf,
1264                                      size_t count, loff_t *ppos)
1265 {
1266         struct vfio_device_file *df = filep->private_data;
1267         struct vfio_device *device = df->device;
1268
1269         /* Paired with smp_store_release() following vfio_df_open() */
1270         if (!smp_load_acquire(&df->access_granted))
1271                 return -EINVAL;
1272
1273         if (unlikely(!device->ops->read))
1274                 return -EINVAL;
1275
1276         return device->ops->read(device, buf, count, ppos);
1277 }
1278
1279 static ssize_t vfio_device_fops_write(struct file *filep,
1280                                       const char __user *buf,
1281                                       size_t count, loff_t *ppos)
1282 {
1283         struct vfio_device_file *df = filep->private_data;
1284         struct vfio_device *device = df->device;
1285
1286         /* Paired with smp_store_release() following vfio_df_open() */
1287         if (!smp_load_acquire(&df->access_granted))
1288                 return -EINVAL;
1289
1290         if (unlikely(!device->ops->write))
1291                 return -EINVAL;
1292
1293         return device->ops->write(device, buf, count, ppos);
1294 }
1295
1296 static int vfio_device_fops_mmap(struct file *filep, struct vm_area_struct *vma)
1297 {
1298         struct vfio_device_file *df = filep->private_data;
1299         struct vfio_device *device = df->device;
1300
1301         /* Paired with smp_store_release() following vfio_df_open() */
1302         if (!smp_load_acquire(&df->access_granted))
1303                 return -EINVAL;
1304
1305         if (unlikely(!device->ops->mmap))
1306                 return -EINVAL;
1307
1308         return device->ops->mmap(device, vma);
1309 }
1310
1311 const struct file_operations vfio_device_fops = {
1312         .owner          = THIS_MODULE,
1313         .open           = vfio_device_fops_cdev_open,
1314         .release        = vfio_device_fops_release,
1315         .read           = vfio_device_fops_read,
1316         .write          = vfio_device_fops_write,
1317         .unlocked_ioctl = vfio_device_fops_unl_ioctl,
1318         .compat_ioctl   = compat_ptr_ioctl,
1319         .mmap           = vfio_device_fops_mmap,
1320 };
1321
1322 static struct vfio_device *vfio_device_from_file(struct file *file)
1323 {
1324         struct vfio_device_file *df = file->private_data;
1325
1326         if (file->f_op != &vfio_device_fops)
1327                 return NULL;
1328         return df->device;
1329 }
1330
1331 /**
1332  * vfio_file_is_valid - True if the file is valid vfio file
1333  * @file: VFIO group file or VFIO device file
1334  */
1335 bool vfio_file_is_valid(struct file *file)
1336 {
1337         return vfio_group_from_file(file) ||
1338                vfio_device_from_file(file);
1339 }
1340 EXPORT_SYMBOL_GPL(vfio_file_is_valid);
1341
1342 /**
1343  * vfio_file_enforced_coherent - True if the DMA associated with the VFIO file
1344  *        is always CPU cache coherent
1345  * @file: VFIO group file or VFIO device file
1346  *
1347  * Enforced coherency means that the IOMMU ignores things like the PCIe no-snoop
1348  * bit in DMA transactions. A return of false indicates that the user has
1349  * rights to access additional instructions such as wbinvd on x86.
1350  */
1351 bool vfio_file_enforced_coherent(struct file *file)
1352 {
1353         struct vfio_device *device;
1354         struct vfio_group *group;
1355
1356         group = vfio_group_from_file(file);
1357         if (group)
1358                 return vfio_group_enforced_coherent(group);
1359
1360         device = vfio_device_from_file(file);
1361         if (device)
1362                 return device_iommu_capable(device->dev,
1363                                             IOMMU_CAP_ENFORCE_CACHE_COHERENCY);
1364
1365         return true;
1366 }
1367 EXPORT_SYMBOL_GPL(vfio_file_enforced_coherent);
1368
1369 static void vfio_device_file_set_kvm(struct file *file, struct kvm *kvm)
1370 {
1371         struct vfio_device_file *df = file->private_data;
1372
1373         /*
1374          * The kvm is first recorded in the vfio_device_file, and will
1375          * be propagated to vfio_device::kvm when the file is bound to
1376          * iommufd successfully in the vfio device cdev path.
1377          */
1378         spin_lock(&df->kvm_ref_lock);
1379         df->kvm = kvm;
1380         spin_unlock(&df->kvm_ref_lock);
1381 }
1382
1383 /**
1384  * vfio_file_set_kvm - Link a kvm with VFIO drivers
1385  * @file: VFIO group file or VFIO device file
1386  * @kvm: KVM to link
1387  *
1388  * When a VFIO device is first opened the KVM will be available in
1389  * device->kvm if one was associated with the file.
1390  */
1391 void vfio_file_set_kvm(struct file *file, struct kvm *kvm)
1392 {
1393         struct vfio_group *group;
1394
1395         group = vfio_group_from_file(file);
1396         if (group)
1397                 vfio_group_set_kvm(group, kvm);
1398
1399         if (vfio_device_from_file(file))
1400                 vfio_device_file_set_kvm(file, kvm);
1401 }
1402 EXPORT_SYMBOL_GPL(vfio_file_set_kvm);
1403
1404 /*
1405  * Sub-module support
1406  */
1407 /*
1408  * Helper for managing a buffer of info chain capabilities, allocate or
1409  * reallocate a buffer with additional @size, filling in @id and @version
1410  * of the capability.  A pointer to the new capability is returned.
1411  *
1412  * NB. The chain is based at the head of the buffer, so new entries are
1413  * added to the tail, vfio_info_cap_shift() should be called to fixup the
1414  * next offsets prior to copying to the user buffer.
1415  */
1416 struct vfio_info_cap_header *vfio_info_cap_add(struct vfio_info_cap *caps,
1417                                                size_t size, u16 id, u16 version)
1418 {
1419         void *buf;
1420         struct vfio_info_cap_header *header, *tmp;
1421
1422         /* Ensure that the next capability struct will be aligned */
1423         size = ALIGN(size, sizeof(u64));
1424
1425         buf = krealloc(caps->buf, caps->size + size, GFP_KERNEL);
1426         if (!buf) {
1427                 kfree(caps->buf);
1428                 caps->buf = NULL;
1429                 caps->size = 0;
1430                 return ERR_PTR(-ENOMEM);
1431         }
1432
1433         caps->buf = buf;
1434         header = buf + caps->size;
1435
1436         /* Eventually copied to user buffer, zero */
1437         memset(header, 0, size);
1438
1439         header->id = id;
1440         header->version = version;
1441
1442         /* Add to the end of the capability chain */
1443         for (tmp = buf; tmp->next; tmp = buf + tmp->next)
1444                 ; /* nothing */
1445
1446         tmp->next = caps->size;
1447         caps->size += size;
1448
1449         return header;
1450 }
1451 EXPORT_SYMBOL_GPL(vfio_info_cap_add);
1452
1453 void vfio_info_cap_shift(struct vfio_info_cap *caps, size_t offset)
1454 {
1455         struct vfio_info_cap_header *tmp;
1456         void *buf = (void *)caps->buf;
1457
1458         /* Capability structs should start with proper alignment */
1459         WARN_ON(!IS_ALIGNED(offset, sizeof(u64)));
1460
1461         for (tmp = buf; tmp->next; tmp = buf + tmp->next - offset)
1462                 tmp->next += offset;
1463 }
1464 EXPORT_SYMBOL(vfio_info_cap_shift);
1465
1466 int vfio_info_add_capability(struct vfio_info_cap *caps,
1467                              struct vfio_info_cap_header *cap, size_t size)
1468 {
1469         struct vfio_info_cap_header *header;
1470
1471         header = vfio_info_cap_add(caps, size, cap->id, cap->version);
1472         if (IS_ERR(header))
1473                 return PTR_ERR(header);
1474
1475         memcpy(header + 1, cap + 1, size - sizeof(*header));
1476
1477         return 0;
1478 }
1479 EXPORT_SYMBOL(vfio_info_add_capability);
1480
1481 int vfio_set_irqs_validate_and_prepare(struct vfio_irq_set *hdr, int num_irqs,
1482                                        int max_irq_type, size_t *data_size)
1483 {
1484         unsigned long minsz;
1485         size_t size;
1486
1487         minsz = offsetofend(struct vfio_irq_set, count);
1488
1489         if ((hdr->argsz < minsz) || (hdr->index >= max_irq_type) ||
1490             (hdr->count >= (U32_MAX - hdr->start)) ||
1491             (hdr->flags & ~(VFIO_IRQ_SET_DATA_TYPE_MASK |
1492                                 VFIO_IRQ_SET_ACTION_TYPE_MASK)))
1493                 return -EINVAL;
1494
1495         if (data_size)
1496                 *data_size = 0;
1497
1498         if (hdr->start >= num_irqs || hdr->start + hdr->count > num_irqs)
1499                 return -EINVAL;
1500
1501         switch (hdr->flags & VFIO_IRQ_SET_DATA_TYPE_MASK) {
1502         case VFIO_IRQ_SET_DATA_NONE:
1503                 size = 0;
1504                 break;
1505         case VFIO_IRQ_SET_DATA_BOOL:
1506                 size = sizeof(uint8_t);
1507                 break;
1508         case VFIO_IRQ_SET_DATA_EVENTFD:
1509                 size = sizeof(int32_t);
1510                 break;
1511         default:
1512                 return -EINVAL;
1513         }
1514
1515         if (size) {
1516                 if (hdr->argsz - minsz < hdr->count * size)
1517                         return -EINVAL;
1518
1519                 if (!data_size)
1520                         return -EINVAL;
1521
1522                 *data_size = hdr->count * size;
1523         }
1524
1525         return 0;
1526 }
1527 EXPORT_SYMBOL(vfio_set_irqs_validate_and_prepare);
1528
1529 /*
1530  * Pin contiguous user pages and return their associated host pages for local
1531  * domain only.
1532  * @device [in]  : device
1533  * @iova [in]    : starting IOVA of user pages to be pinned.
1534  * @npage [in]   : count of pages to be pinned.  This count should not
1535  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1536  * @prot [in]    : protection flags
1537  * @pages[out]   : array of host pages
1538  * Return error or number of pages pinned.
1539  *
1540  * A driver may only call this function if the vfio_device was created
1541  * by vfio_register_emulated_iommu_dev() due to vfio_device_container_pin_pages().
1542  */
1543 int vfio_pin_pages(struct vfio_device *device, dma_addr_t iova,
1544                    int npage, int prot, struct page **pages)
1545 {
1546         /* group->container cannot change while a vfio device is open */
1547         if (!pages || !npage || WARN_ON(!vfio_assert_device_open(device)))
1548                 return -EINVAL;
1549         if (!device->ops->dma_unmap)
1550                 return -EINVAL;
1551         if (vfio_device_has_container(device))
1552                 return vfio_device_container_pin_pages(device, iova,
1553                                                        npage, prot, pages);
1554         if (device->iommufd_access) {
1555                 int ret;
1556
1557                 if (iova > ULONG_MAX)
1558                         return -EINVAL;
1559                 /*
1560                  * VFIO ignores the sub page offset, npages is from the start of
1561                  * a PAGE_SIZE chunk of IOVA. The caller is expected to recover
1562                  * the sub page offset by doing:
1563                  *     pages[0] + (iova % PAGE_SIZE)
1564                  */
1565                 ret = iommufd_access_pin_pages(
1566                         device->iommufd_access, ALIGN_DOWN(iova, PAGE_SIZE),
1567                         npage * PAGE_SIZE, pages,
1568                         (prot & IOMMU_WRITE) ? IOMMUFD_ACCESS_RW_WRITE : 0);
1569                 if (ret)
1570                         return ret;
1571                 return npage;
1572         }
1573         return -EINVAL;
1574 }
1575 EXPORT_SYMBOL(vfio_pin_pages);
1576
1577 /*
1578  * Unpin contiguous host pages for local domain only.
1579  * @device [in]  : device
1580  * @iova [in]    : starting address of user pages to be unpinned.
1581  * @npage [in]   : count of pages to be unpinned.  This count should not
1582  *                 be greater than VFIO_PIN_PAGES_MAX_ENTRIES.
1583  */
1584 void vfio_unpin_pages(struct vfio_device *device, dma_addr_t iova, int npage)
1585 {
1586         if (WARN_ON(!vfio_assert_device_open(device)))
1587                 return;
1588         if (WARN_ON(!device->ops->dma_unmap))
1589                 return;
1590
1591         if (vfio_device_has_container(device)) {
1592                 vfio_device_container_unpin_pages(device, iova, npage);
1593                 return;
1594         }
1595         if (device->iommufd_access) {
1596                 if (WARN_ON(iova > ULONG_MAX))
1597                         return;
1598                 iommufd_access_unpin_pages(device->iommufd_access,
1599                                            ALIGN_DOWN(iova, PAGE_SIZE),
1600                                            npage * PAGE_SIZE);
1601                 return;
1602         }
1603 }
1604 EXPORT_SYMBOL(vfio_unpin_pages);
1605
1606 /*
1607  * This interface allows the CPUs to perform some sort of virtual DMA on
1608  * behalf of the device.
1609  *
1610  * CPUs read/write from/into a range of IOVAs pointing to user space memory
1611  * into/from a kernel buffer.
1612  *
1613  * As the read/write of user space memory is conducted via the CPUs and is
1614  * not a real device DMA, it is not necessary to pin the user space memory.
1615  *
1616  * @device [in]         : VFIO device
1617  * @iova [in]           : base IOVA of a user space buffer
1618  * @data [in]           : pointer to kernel buffer
1619  * @len [in]            : kernel buffer length
1620  * @write               : indicate read or write
1621  * Return error code on failure or 0 on success.
1622  */
1623 int vfio_dma_rw(struct vfio_device *device, dma_addr_t iova, void *data,
1624                 size_t len, bool write)
1625 {
1626         if (!data || len <= 0 || !vfio_assert_device_open(device))
1627                 return -EINVAL;
1628
1629         if (vfio_device_has_container(device))
1630                 return vfio_device_container_dma_rw(device, iova,
1631                                                     data, len, write);
1632
1633         if (device->iommufd_access) {
1634                 unsigned int flags = 0;
1635
1636                 if (iova > ULONG_MAX)
1637                         return -EINVAL;
1638
1639                 /* VFIO historically tries to auto-detect a kthread */
1640                 if (!current->mm)
1641                         flags |= IOMMUFD_ACCESS_RW_KTHREAD;
1642                 if (write)
1643                         flags |= IOMMUFD_ACCESS_RW_WRITE;
1644                 return iommufd_access_rw(device->iommufd_access, iova, data,
1645                                          len, flags);
1646         }
1647         return -EINVAL;
1648 }
1649 EXPORT_SYMBOL(vfio_dma_rw);
1650
1651 /*
1652  * Module/class support
1653  */
1654 static int __init vfio_init(void)
1655 {
1656         int ret;
1657
1658         ida_init(&vfio.device_ida);
1659
1660         ret = vfio_group_init();
1661         if (ret)
1662                 return ret;
1663
1664         ret = vfio_virqfd_init();
1665         if (ret)
1666                 goto err_virqfd;
1667
1668         /* /sys/class/vfio-dev/vfioX */
1669         vfio.device_class = class_create("vfio-dev");
1670         if (IS_ERR(vfio.device_class)) {
1671                 ret = PTR_ERR(vfio.device_class);
1672                 goto err_dev_class;
1673         }
1674
1675         ret = vfio_cdev_init(vfio.device_class);
1676         if (ret)
1677                 goto err_alloc_dev_chrdev;
1678
1679         pr_info(DRIVER_DESC " version: " DRIVER_VERSION "\n");
1680         return 0;
1681
1682 err_alloc_dev_chrdev:
1683         class_destroy(vfio.device_class);
1684         vfio.device_class = NULL;
1685 err_dev_class:
1686         vfio_virqfd_exit();
1687 err_virqfd:
1688         vfio_group_cleanup();
1689         return ret;
1690 }
1691
1692 static void __exit vfio_cleanup(void)
1693 {
1694         ida_destroy(&vfio.device_ida);
1695         vfio_cdev_cleanup();
1696         class_destroy(vfio.device_class);
1697         vfio.device_class = NULL;
1698         vfio_virqfd_exit();
1699         vfio_group_cleanup();
1700         xa_destroy(&vfio_device_set_xa);
1701 }
1702
1703 module_init(vfio_init);
1704 module_exit(vfio_cleanup);
1705
1706 MODULE_IMPORT_NS(IOMMUFD);
1707 MODULE_VERSION(DRIVER_VERSION);
1708 MODULE_LICENSE("GPL v2");
1709 MODULE_AUTHOR(DRIVER_AUTHOR);
1710 MODULE_DESCRIPTION(DRIVER_DESC);
1711 MODULE_SOFTDEP("post: vfio_iommu_type1 vfio_iommu_spapr_tce");