drivers/vdpa/vdpa_user/vduse_dev.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * VDUSE: vDPA Device in Userspace
   4  *
   5  * Copyright (C) 2020-2021 Bytedance Inc. and/or its affiliates. All rights reserved.
   6  *
   7  * Author: Xie Yongji <xieyongji@bytedance.com>
   8  *
   9  */
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/cdev.h>
  14 #include <linux/device.h>
  15 #include <linux/eventfd.h>
  16 #include <linux/slab.h>
  17 #include <linux/wait.h>
  18 #include <linux/dma-map-ops.h>
  19 #include <linux/poll.h>
  20 #include <linux/file.h>
  21 #include <linux/uio.h>
  22 #include <linux/vdpa.h>
  23 #include <linux/nospec.h>
  24 #include <uapi/linux/vduse.h>
  25 #include <uapi/linux/vdpa.h>
  26 #include <uapi/linux/virtio_config.h>
  27 #include <uapi/linux/virtio_ids.h>
  28 #include <uapi/linux/virtio_blk.h>
  29 #include <linux/mod_devicetable.h>
  30
  31 #include "iova_domain.h"
  32
  33 #define DRV_AUTHOR   "Yongji Xie <xieyongji@bytedance.com>"
  34 #define DRV_DESC     "vDPA Device in Userspace"
  35 #define DRV_LICENSE  "GPL v2"
  36
  37 #define VDUSE_DEV_MAX (1U << MINORBITS)
  38 #define VDUSE_BOUNCE_SIZE (64 * 1024 * 1024)
  39 #define VDUSE_IOVA_SIZE (128 * 1024 * 1024)
  40 #define VDUSE_MSG_DEFAULT_TIMEOUT 30
  41
  42 struct vduse_virtqueue {
  43         u16 index;
  44         u16 num_max;
  45         u32 num;
  46         u64 desc_addr;
  47         u64 driver_addr;
  48         u64 device_addr;
  49         struct vdpa_vq_state state;
  50         bool ready;
  51         bool kicked;
  52         spinlock_t kick_lock;
  53         spinlock_t irq_lock;
  54         struct eventfd_ctx *kickfd;
  55         struct vdpa_callback cb;
  56         struct work_struct inject;
  57         struct work_struct kick;
  58 };
  59
  60 struct vduse_dev;
  61
  62 struct vduse_vdpa {
  63         struct vdpa_device vdpa;
  64         struct vduse_dev *dev;
  65 };
  66
  67 struct vduse_dev {
  68         struct vduse_vdpa *vdev;
  69         struct device *dev;
  70         struct vduse_virtqueue *vqs;
  71         struct vduse_iova_domain *domain;
  72         char *name;
  73         struct mutex lock;
  74         spinlock_t msg_lock;
  75         u64 msg_unique;
  76         u32 msg_timeout;
  77         wait_queue_head_t waitq;
  78         struct list_head send_list;
  79         struct list_head recv_list;
  80         struct vdpa_callback config_cb;
  81         struct work_struct inject;
  82         spinlock_t irq_lock;
  83         struct rw_semaphore rwsem;
  84         int minor;
  85         bool broken;
  86         bool connected;
  87         u64 api_version;
  88         u64 device_features;
  89         u64 driver_features;
  90         u32 device_id;
  91         u32 vendor_id;
  92         u32 generation;
  93         u32 config_size;
  94         void *config;
  95         u8 status;
  96         u32 vq_num;
  97         u32 vq_align;
  98 };
  99
 100 struct vduse_dev_msg {
 101         struct vduse_dev_request req;
 102         struct vduse_dev_response resp;
 103         struct list_head list;
 104         wait_queue_head_t waitq;
 105         bool completed;
 106 };
 107
 108 struct vduse_control {
 109         u64 api_version;
 110 };
 111
 112 static DEFINE_MUTEX(vduse_lock);
 113 static DEFINE_IDR(vduse_idr);
 114
 115 static dev_t vduse_major;
 116 static struct class *vduse_class;
 117 static struct cdev vduse_ctrl_cdev;
 118 static struct cdev vduse_cdev;
 119 static struct workqueue_struct *vduse_irq_wq;
 120
 121 static u32 allowed_device_id[] = {
 122         VIRTIO_ID_BLOCK,
 123 };
 124
 125 static inline struct vduse_dev *vdpa_to_vduse(struct vdpa_device *vdpa)
 126 {
 127         struct vduse_vdpa *vdev = container_of(vdpa, struct vduse_vdpa, vdpa);
 128
 129         return vdev->dev;
 130 }
 131
 132 static inline struct vduse_dev *dev_to_vduse(struct device *dev)
 133 {
 134         struct vdpa_device *vdpa = dev_to_vdpa(dev);
 135
 136         return vdpa_to_vduse(vdpa);
 137 }
 138
 139 static struct vduse_dev_msg *vduse_find_msg(struct list_head *head,
 140                                             uint32_t request_id)
 141 {
 142         struct vduse_dev_msg *msg;
 143
 144         list_for_each_entry(msg, head, list) {
 145                 if (msg->req.request_id == request_id) {
 146                         list_del(&msg->list);
 147                         return msg;
 148                 }
 149         }
 150
 151         return NULL;
 152 }
 153
 154 static struct vduse_dev_msg *vduse_dequeue_msg(struct list_head *head)
 155 {
 156         struct vduse_dev_msg *msg = NULL;
 157
 158         if (!list_empty(head)) {
 159                 msg = list_first_entry(head, struct vduse_dev_msg, list);
 160                 list_del(&msg->list);
 161         }
 162
 163         return msg;
 164 }
 165
 166 static void vduse_enqueue_msg(struct list_head *head,
 167                               struct vduse_dev_msg *msg)
 168 {
 169         list_add_tail(&msg->list, head);
 170 }
 171
 172 static void vduse_dev_broken(struct vduse_dev *dev)
 173 {
 174         struct vduse_dev_msg *msg, *tmp;
 175
 176         if (unlikely(dev->broken))
 177                 return;
 178
 179         list_splice_init(&dev->recv_list, &dev->send_list);
 180         list_for_each_entry_safe(msg, tmp, &dev->send_list, list) {
 181                 list_del(&msg->list);
 182                 msg->completed = 1;
 183                 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
 184                 wake_up(&msg->waitq);
 185         }
 186         dev->broken = true;
 187         wake_up(&dev->waitq);
 188 }
 189
 190 static int vduse_dev_msg_sync(struct vduse_dev *dev,
 191                               struct vduse_dev_msg *msg)
 192 {
 193         int ret;
 194
 195         if (unlikely(dev->broken))
 196                 return -EIO;
 197
 198         init_waitqueue_head(&msg->waitq);
 199         spin_lock(&dev->msg_lock);
 200         if (unlikely(dev->broken)) {
 201                 spin_unlock(&dev->msg_lock);
 202                 return -EIO;
 203         }
 204         msg->req.request_id = dev->msg_unique++;
 205         vduse_enqueue_msg(&dev->send_list, msg);
 206         wake_up(&dev->waitq);
 207         spin_unlock(&dev->msg_lock);
 208         if (dev->msg_timeout)
 209                 ret = wait_event_killable_timeout(msg->waitq, msg->completed,
 210                                                   (long)dev->msg_timeout * HZ);
 211         else
 212                 ret = wait_event_killable(msg->waitq, msg->completed);
 213
 214         spin_lock(&dev->msg_lock);
 215         if (!msg->completed) {
 216                 list_del(&msg->list);
 217                 msg->resp.result = VDUSE_REQ_RESULT_FAILED;
 218                 /* Mark the device as malfunction when there is a timeout */
 219                 if (!ret)
 220                         vduse_dev_broken(dev);
 221         }
 222         ret = (msg->resp.result == VDUSE_REQ_RESULT_OK) ? 0 : -EIO;
 223         spin_unlock(&dev->msg_lock);
 224
 225         return ret;
 226 }
 227
 228 static int vduse_dev_get_vq_state_packed(struct vduse_dev *dev,
 229                                          struct vduse_virtqueue *vq,
 230                                          struct vdpa_vq_state_packed *packed)
 231 {
 232         struct vduse_dev_msg msg = { 0 };
 233         int ret;
 234
 235         msg.req.type = VDUSE_GET_VQ_STATE;
 236         msg.req.vq_state.index = vq->index;
 237
 238         ret = vduse_dev_msg_sync(dev, &msg);
 239         if (ret)
 240                 return ret;
 241
 242         packed->last_avail_counter =
 243                         msg.resp.vq_state.packed.last_avail_counter & 0x0001;
 244         packed->last_avail_idx =
 245                         msg.resp.vq_state.packed.last_avail_idx & 0x7FFF;
 246         packed->last_used_counter =
 247                         msg.resp.vq_state.packed.last_used_counter & 0x0001;
 248         packed->last_used_idx =
 249                         msg.resp.vq_state.packed.last_used_idx & 0x7FFF;
 250
 251         return 0;
 252 }
 253
 254 static int vduse_dev_get_vq_state_split(struct vduse_dev *dev,
 255                                         struct vduse_virtqueue *vq,
 256                                         struct vdpa_vq_state_split *split)
 257 {
 258         struct vduse_dev_msg msg = { 0 };
 259         int ret;
 260
 261         msg.req.type = VDUSE_GET_VQ_STATE;
 262         msg.req.vq_state.index = vq->index;
 263
 264         ret = vduse_dev_msg_sync(dev, &msg);
 265         if (ret)
 266                 return ret;
 267
 268         split->avail_index = msg.resp.vq_state.split.avail_index;
 269
 270         return 0;
 271 }
 272
 273 static int vduse_dev_set_status(struct vduse_dev *dev, u8 status)
 274 {
 275         struct vduse_dev_msg msg = { 0 };
 276
 277         msg.req.type = VDUSE_SET_STATUS;
 278         msg.req.s.status = status;
 279
 280         return vduse_dev_msg_sync(dev, &msg);
 281 }
 282
 283 static int vduse_dev_update_iotlb(struct vduse_dev *dev,
 284                                   u64 start, u64 last)
 285 {
 286         struct vduse_dev_msg msg = { 0 };
 287
 288         if (last < start)
 289                 return -EINVAL;
 290
 291         msg.req.type = VDUSE_UPDATE_IOTLB;
 292         msg.req.iova.start = start;
 293         msg.req.iova.last = last;
 294
 295         return vduse_dev_msg_sync(dev, &msg);
 296 }
 297
 298 static ssize_t vduse_dev_read_iter(struct kiocb *iocb, struct iov_iter *to)
 299 {
 300         struct file *file = iocb->ki_filp;
 301         struct vduse_dev *dev = file->private_data;
 302         struct vduse_dev_msg *msg;
 303         int size = sizeof(struct vduse_dev_request);
 304         ssize_t ret;
 305
 306         if (iov_iter_count(to) < size)
 307                 return -EINVAL;
 308
 309         spin_lock(&dev->msg_lock);
 310         while (1) {
 311                 msg = vduse_dequeue_msg(&dev->send_list);
 312                 if (msg)
 313                         break;
 314
 315                 ret = -EAGAIN;
 316                 if (file->f_flags & O_NONBLOCK)
 317                         goto unlock;
 318
 319                 spin_unlock(&dev->msg_lock);
 320                 ret = wait_event_interruptible_exclusive(dev->waitq,
 321                                         !list_empty(&dev->send_list));
 322                 if (ret)
 323                         return ret;
 324
 325                 spin_lock(&dev->msg_lock);
 326         }
 327         spin_unlock(&dev->msg_lock);
 328         ret = copy_to_iter(&msg->req, size, to);
 329         spin_lock(&dev->msg_lock);
 330         if (ret != size) {
 331                 ret = -EFAULT;
 332                 vduse_enqueue_msg(&dev->send_list, msg);
 333                 goto unlock;
 334         }
 335         vduse_enqueue_msg(&dev->recv_list, msg);
 336 unlock:
 337         spin_unlock(&dev->msg_lock);
 338
 339         return ret;
 340 }
 341
 342 static bool is_mem_zero(const char *ptr, int size)
 343 {
 344         int i;
 345
 346         for (i = 0; i < size; i++) {
 347                 if (ptr[i])
 348                         return false;
 349         }
 350         return true;
 351 }
 352
 353 static ssize_t vduse_dev_write_iter(struct kiocb *iocb, struct iov_iter *from)
 354 {
 355         struct file *file = iocb->ki_filp;
 356         struct vduse_dev *dev = file->private_data;
 357         struct vduse_dev_response resp;
 358         struct vduse_dev_msg *msg;
 359         size_t ret;
 360
 361         ret = copy_from_iter(&resp, sizeof(resp), from);
 362         if (ret != sizeof(resp))
 363                 return -EINVAL;
 364
 365         if (!is_mem_zero((const char *)resp.reserved, sizeof(resp.reserved)))
 366                 return -EINVAL;
 367
 368         spin_lock(&dev->msg_lock);
 369         msg = vduse_find_msg(&dev->recv_list, resp.request_id);
 370         if (!msg) {
 371                 ret = -ENOENT;
 372                 goto unlock;
 373         }
 374
 375         memcpy(&msg->resp, &resp, sizeof(resp));
 376         msg->completed = 1;
 377         wake_up(&msg->waitq);
 378 unlock:
 379         spin_unlock(&dev->msg_lock);
 380
 381         return ret;
 382 }
 383
 384 static __poll_t vduse_dev_poll(struct file *file, poll_table *wait)
 385 {
 386         struct vduse_dev *dev = file->private_data;
 387         __poll_t mask = 0;
 388
 389         poll_wait(file, &dev->waitq, wait);
 390
 391         spin_lock(&dev->msg_lock);
 392
 393         if (unlikely(dev->broken))
 394                 mask |= EPOLLERR;
 395         if (!list_empty(&dev->send_list))
 396                 mask |= EPOLLIN | EPOLLRDNORM;
 397         if (!list_empty(&dev->recv_list))
 398                 mask |= EPOLLOUT | EPOLLWRNORM;
 399
 400         spin_unlock(&dev->msg_lock);
 401
 402         return mask;
 403 }
 404
 405 static void vduse_dev_reset(struct vduse_dev *dev)
 406 {
 407         int i;
 408         struct vduse_iova_domain *domain = dev->domain;
 409
 410         /* The coherent mappings are handled in vduse_dev_free_coherent() */
 411         if (domain->bounce_map)
 412                 vduse_domain_reset_bounce_map(domain);
 413
 414         down_write(&dev->rwsem);
 415
 416         dev->status = 0;
 417         dev->driver_features = 0;
 418         dev->generation++;
 419         spin_lock(&dev->irq_lock);
 420         dev->config_cb.callback = NULL;
 421         dev->config_cb.private = NULL;
 422         spin_unlock(&dev->irq_lock);
 423         flush_work(&dev->inject);
 424
 425         for (i = 0; i < dev->vq_num; i++) {
 426                 struct vduse_virtqueue *vq = &dev->vqs[i];
 427
 428                 vq->ready = false;
 429                 vq->desc_addr = 0;
 430                 vq->driver_addr = 0;
 431                 vq->device_addr = 0;
 432                 vq->num = 0;
 433                 memset(&vq->state, 0, sizeof(vq->state));
 434
 435                 spin_lock(&vq->kick_lock);
 436                 vq->kicked = false;
 437                 if (vq->kickfd)
 438                         eventfd_ctx_put(vq->kickfd);
 439                 vq->kickfd = NULL;
 440                 spin_unlock(&vq->kick_lock);
 441
 442                 spin_lock(&vq->irq_lock);
 443                 vq->cb.callback = NULL;
 444                 vq->cb.private = NULL;
 445                 spin_unlock(&vq->irq_lock);
 446                 flush_work(&vq->inject);
 447                 flush_work(&vq->kick);
 448         }
 449
 450         up_write(&dev->rwsem);
 451 }
 452
 453 static int vduse_vdpa_set_vq_address(struct vdpa_device *vdpa, u16 idx,
 454                                 u64 desc_area, u64 driver_area,
 455                                 u64 device_area)
 456 {
 457         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 458         struct vduse_virtqueue *vq = &dev->vqs[idx];
 459
 460         vq->desc_addr = desc_area;
 461         vq->driver_addr = driver_area;
 462         vq->device_addr = device_area;
 463
 464         return 0;
 465 }
 466
 467 static void vduse_vq_kick(struct vduse_virtqueue *vq)
 468 {
 469         spin_lock(&vq->kick_lock);
 470         if (!vq->ready)
 471                 goto unlock;
 472
 473         if (vq->kickfd)
 474                 eventfd_signal(vq->kickfd, 1);
 475         else
 476                 vq->kicked = true;
 477 unlock:
 478         spin_unlock(&vq->kick_lock);
 479 }
 480
 481 static void vduse_vq_kick_work(struct work_struct *work)
 482 {
 483         struct vduse_virtqueue *vq = container_of(work,
 484                                         struct vduse_virtqueue, kick);
 485
 486         vduse_vq_kick(vq);
 487 }
 488
 489 static void vduse_vdpa_kick_vq(struct vdpa_device *vdpa, u16 idx)
 490 {
 491         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 492         struct vduse_virtqueue *vq = &dev->vqs[idx];
 493
 494         if (!eventfd_signal_allowed()) {
 495                 schedule_work(&vq->kick);
 496                 return;
 497         }
 498         vduse_vq_kick(vq);
 499 }
 500
 501 static void vduse_vdpa_set_vq_cb(struct vdpa_device *vdpa, u16 idx,
 502                               struct vdpa_callback *cb)
 503 {
 504         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 505         struct vduse_virtqueue *vq = &dev->vqs[idx];
 506
 507         spin_lock(&vq->irq_lock);
 508         vq->cb.callback = cb->callback;
 509         vq->cb.private = cb->private;
 510         spin_unlock(&vq->irq_lock);
 511 }
 512
 513 static void vduse_vdpa_set_vq_num(struct vdpa_device *vdpa, u16 idx, u32 num)
 514 {
 515         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 516         struct vduse_virtqueue *vq = &dev->vqs[idx];
 517
 518         vq->num = num;
 519 }
 520
 521 static void vduse_vdpa_set_vq_ready(struct vdpa_device *vdpa,
 522                                         u16 idx, bool ready)
 523 {
 524         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 525         struct vduse_virtqueue *vq = &dev->vqs[idx];
 526
 527         vq->ready = ready;
 528 }
 529
 530 static bool vduse_vdpa_get_vq_ready(struct vdpa_device *vdpa, u16 idx)
 531 {
 532         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 533         struct vduse_virtqueue *vq = &dev->vqs[idx];
 534
 535         return vq->ready;
 536 }
 537
 538 static int vduse_vdpa_set_vq_state(struct vdpa_device *vdpa, u16 idx,
 539                                 const struct vdpa_vq_state *state)
 540 {
 541         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 542         struct vduse_virtqueue *vq = &dev->vqs[idx];
 543
 544         if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
 545                 vq->state.packed.last_avail_counter =
 546                                 state->packed.last_avail_counter;
 547                 vq->state.packed.last_avail_idx = state->packed.last_avail_idx;
 548                 vq->state.packed.last_used_counter =
 549                                 state->packed.last_used_counter;
 550                 vq->state.packed.last_used_idx = state->packed.last_used_idx;
 551         } else
 552                 vq->state.split.avail_index = state->split.avail_index;
 553
 554         return 0;
 555 }
 556
 557 static int vduse_vdpa_get_vq_state(struct vdpa_device *vdpa, u16 idx,
 558                                 struct vdpa_vq_state *state)
 559 {
 560         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 561         struct vduse_virtqueue *vq = &dev->vqs[idx];
 562
 563         if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED))
 564                 return vduse_dev_get_vq_state_packed(dev, vq, &state->packed);
 565
 566         return vduse_dev_get_vq_state_split(dev, vq, &state->split);
 567 }
 568
 569 static u32 vduse_vdpa_get_vq_align(struct vdpa_device *vdpa)
 570 {
 571         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 572
 573         return dev->vq_align;
 574 }
 575
 576 static u64 vduse_vdpa_get_device_features(struct vdpa_device *vdpa)
 577 {
 578         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 579
 580         return dev->device_features;
 581 }
 582
 583 static int vduse_vdpa_set_driver_features(struct vdpa_device *vdpa, u64 features)
 584 {
 585         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 586
 587         dev->driver_features = features;
 588         return 0;
 589 }
 590
 591 static u64 vduse_vdpa_get_driver_features(struct vdpa_device *vdpa)
 592 {
 593         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 594
 595         return dev->driver_features;
 596 }
 597
 598 static void vduse_vdpa_set_config_cb(struct vdpa_device *vdpa,
 599                                   struct vdpa_callback *cb)
 600 {
 601         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 602
 603         spin_lock(&dev->irq_lock);
 604         dev->config_cb.callback = cb->callback;
 605         dev->config_cb.private = cb->private;
 606         spin_unlock(&dev->irq_lock);
 607 }
 608
 609 static u16 vduse_vdpa_get_vq_num_max(struct vdpa_device *vdpa)
 610 {
 611         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 612         u16 num_max = 0;
 613         int i;
 614
 615         for (i = 0; i < dev->vq_num; i++)
 616                 if (num_max < dev->vqs[i].num_max)
 617                         num_max = dev->vqs[i].num_max;
 618
 619         return num_max;
 620 }
 621
 622 static u32 vduse_vdpa_get_device_id(struct vdpa_device *vdpa)
 623 {
 624         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 625
 626         return dev->device_id;
 627 }
 628
 629 static u32 vduse_vdpa_get_vendor_id(struct vdpa_device *vdpa)
 630 {
 631         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 632
 633         return dev->vendor_id;
 634 }
 635
 636 static u8 vduse_vdpa_get_status(struct vdpa_device *vdpa)
 637 {
 638         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 639
 640         return dev->status;
 641 }
 642
 643 static void vduse_vdpa_set_status(struct vdpa_device *vdpa, u8 status)
 644 {
 645         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 646
 647         if (vduse_dev_set_status(dev, status))
 648                 return;
 649
 650         dev->status = status;
 651 }
 652
 653 static size_t vduse_vdpa_get_config_size(struct vdpa_device *vdpa)
 654 {
 655         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 656
 657         return dev->config_size;
 658 }
 659
 660 static void vduse_vdpa_get_config(struct vdpa_device *vdpa, unsigned int offset,
 661                                   void *buf, unsigned int len)
 662 {
 663         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 664
 665         if (offset > dev->config_size ||
 666             len > dev->config_size - offset)
 667                 return;
 668
 669         memcpy(buf, dev->config + offset, len);
 670 }
 671
 672 static void vduse_vdpa_set_config(struct vdpa_device *vdpa, unsigned int offset,
 673                         const void *buf, unsigned int len)
 674 {
 675         /* Now we only support read-only configuration space */
 676 }
 677
 678 static int vduse_vdpa_reset(struct vdpa_device *vdpa)
 679 {
 680         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 681         int ret = vduse_dev_set_status(dev, 0);
 682
 683         vduse_dev_reset(dev);
 684
 685         return ret;
 686 }
 687
 688 static u32 vduse_vdpa_get_generation(struct vdpa_device *vdpa)
 689 {
 690         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 691
 692         return dev->generation;
 693 }
 694
 695 static int vduse_vdpa_set_map(struct vdpa_device *vdpa,
 696                                 unsigned int asid,
 697                                 struct vhost_iotlb *iotlb)
 698 {
 699         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 700         int ret;
 701
 702         ret = vduse_domain_set_map(dev->domain, iotlb);
 703         if (ret)
 704                 return ret;
 705
 706         ret = vduse_dev_update_iotlb(dev, 0ULL, ULLONG_MAX);
 707         if (ret) {
 708                 vduse_domain_clear_map(dev->domain, iotlb);
 709                 return ret;
 710         }
 711
 712         return 0;
 713 }
 714
 715 static void vduse_vdpa_free(struct vdpa_device *vdpa)
 716 {
 717         struct vduse_dev *dev = vdpa_to_vduse(vdpa);
 718
 719         dev->vdev = NULL;
 720 }
 721
 722 static const struct vdpa_config_ops vduse_vdpa_config_ops = {
 723         .set_vq_address         = vduse_vdpa_set_vq_address,
 724         .kick_vq                = vduse_vdpa_kick_vq,
 725         .set_vq_cb              = vduse_vdpa_set_vq_cb,
 726         .set_vq_num             = vduse_vdpa_set_vq_num,
 727         .set_vq_ready           = vduse_vdpa_set_vq_ready,
 728         .get_vq_ready           = vduse_vdpa_get_vq_ready,
 729         .set_vq_state           = vduse_vdpa_set_vq_state,
 730         .get_vq_state           = vduse_vdpa_get_vq_state,
 731         .get_vq_align           = vduse_vdpa_get_vq_align,
 732         .get_device_features    = vduse_vdpa_get_device_features,
 733         .set_driver_features    = vduse_vdpa_set_driver_features,
 734         .get_driver_features    = vduse_vdpa_get_driver_features,
 735         .set_config_cb          = vduse_vdpa_set_config_cb,
 736         .get_vq_num_max         = vduse_vdpa_get_vq_num_max,
 737         .get_device_id          = vduse_vdpa_get_device_id,
 738         .get_vendor_id          = vduse_vdpa_get_vendor_id,
 739         .get_status             = vduse_vdpa_get_status,
 740         .set_status             = vduse_vdpa_set_status,
 741         .get_config_size        = vduse_vdpa_get_config_size,
 742         .get_config             = vduse_vdpa_get_config,
 743         .set_config             = vduse_vdpa_set_config,
 744         .get_generation         = vduse_vdpa_get_generation,
 745         .reset                  = vduse_vdpa_reset,
 746         .set_map                = vduse_vdpa_set_map,
 747         .free                   = vduse_vdpa_free,
 748 };
 749
 750 static dma_addr_t vduse_dev_map_page(struct device *dev, struct page *page,
 751                                      unsigned long offset, size_t size,
 752                                      enum dma_data_direction dir,
 753                                      unsigned long attrs)
 754 {
 755         struct vduse_dev *vdev = dev_to_vduse(dev);
 756         struct vduse_iova_domain *domain = vdev->domain;
 757
 758         return vduse_domain_map_page(domain, page, offset, size, dir, attrs);
 759 }
 760
 761 static void vduse_dev_unmap_page(struct device *dev, dma_addr_t dma_addr,
 762                                 size_t size, enum dma_data_direction dir,
 763                                 unsigned long attrs)
 764 {
 765         struct vduse_dev *vdev = dev_to_vduse(dev);
 766         struct vduse_iova_domain *domain = vdev->domain;
 767
 768         return vduse_domain_unmap_page(domain, dma_addr, size, dir, attrs);
 769 }
 770
 771 static void *vduse_dev_alloc_coherent(struct device *dev, size_t size,
 772                                         dma_addr_t *dma_addr, gfp_t flag,
 773                                         unsigned long attrs)
 774 {
 775         struct vduse_dev *vdev = dev_to_vduse(dev);
 776         struct vduse_iova_domain *domain = vdev->domain;
 777         unsigned long iova;
 778         void *addr;
 779
 780         *dma_addr = DMA_MAPPING_ERROR;
 781         addr = vduse_domain_alloc_coherent(domain, size,
 782                                 (dma_addr_t *)&iova, flag, attrs);
 783         if (!addr)
 784                 return NULL;
 785
 786         *dma_addr = (dma_addr_t)iova;
 787
 788         return addr;
 789 }
 790
 791 static void vduse_dev_free_coherent(struct device *dev, size_t size,
 792                                         void *vaddr, dma_addr_t dma_addr,
 793                                         unsigned long attrs)
 794 {
 795         struct vduse_dev *vdev = dev_to_vduse(dev);
 796         struct vduse_iova_domain *domain = vdev->domain;
 797
 798         vduse_domain_free_coherent(domain, size, vaddr, dma_addr, attrs);
 799 }
 800
 801 static size_t vduse_dev_max_mapping_size(struct device *dev)
 802 {
 803         struct vduse_dev *vdev = dev_to_vduse(dev);
 804         struct vduse_iova_domain *domain = vdev->domain;
 805
 806         return domain->bounce_size;
 807 }
 808
 809 static const struct dma_map_ops vduse_dev_dma_ops = {
 810         .map_page = vduse_dev_map_page,
 811         .unmap_page = vduse_dev_unmap_page,
 812         .alloc = vduse_dev_alloc_coherent,
 813         .free = vduse_dev_free_coherent,
 814         .max_mapping_size = vduse_dev_max_mapping_size,
 815 };
 816
 817 static unsigned int perm_to_file_flags(u8 perm)
 818 {
 819         unsigned int flags = 0;
 820
 821         switch (perm) {
 822         case VDUSE_ACCESS_WO:
 823                 flags |= O_WRONLY;
 824                 break;
 825         case VDUSE_ACCESS_RO:
 826                 flags |= O_RDONLY;
 827                 break;
 828         case VDUSE_ACCESS_RW:
 829                 flags |= O_RDWR;
 830                 break;
 831         default:
 832                 WARN(1, "invalidate vhost IOTLB permission\n");
 833                 break;
 834         }
 835
 836         return flags;
 837 }
 838
 839 static int vduse_kickfd_setup(struct vduse_dev *dev,
 840                         struct vduse_vq_eventfd *eventfd)
 841 {
 842         struct eventfd_ctx *ctx = NULL;
 843         struct vduse_virtqueue *vq;
 844         u32 index;
 845
 846         if (eventfd->index >= dev->vq_num)
 847                 return -EINVAL;
 848
 849         index = array_index_nospec(eventfd->index, dev->vq_num);
 850         vq = &dev->vqs[index];
 851         if (eventfd->fd >= 0) {
 852                 ctx = eventfd_ctx_fdget(eventfd->fd);
 853                 if (IS_ERR(ctx))
 854                         return PTR_ERR(ctx);
 855         } else if (eventfd->fd != VDUSE_EVENTFD_DEASSIGN)
 856                 return 0;
 857
 858         spin_lock(&vq->kick_lock);
 859         if (vq->kickfd)
 860                 eventfd_ctx_put(vq->kickfd);
 861         vq->kickfd = ctx;
 862         if (vq->ready && vq->kicked && vq->kickfd) {
 863                 eventfd_signal(vq->kickfd, 1);
 864                 vq->kicked = false;
 865         }
 866         spin_unlock(&vq->kick_lock);
 867
 868         return 0;
 869 }
 870
 871 static bool vduse_dev_is_ready(struct vduse_dev *dev)
 872 {
 873         int i;
 874
 875         for (i = 0; i < dev->vq_num; i++)
 876                 if (!dev->vqs[i].num_max)
 877                         return false;
 878
 879         return true;
 880 }
 881
 882 static void vduse_dev_irq_inject(struct work_struct *work)
 883 {
 884         struct vduse_dev *dev = container_of(work, struct vduse_dev, inject);
 885
 886         spin_lock_irq(&dev->irq_lock);
 887         if (dev->config_cb.callback)
 888                 dev->config_cb.callback(dev->config_cb.private);
 889         spin_unlock_irq(&dev->irq_lock);
 890 }
 891
 892 static void vduse_vq_irq_inject(struct work_struct *work)
 893 {
 894         struct vduse_virtqueue *vq = container_of(work,
 895                                         struct vduse_virtqueue, inject);
 896
 897         spin_lock_irq(&vq->irq_lock);
 898         if (vq->ready && vq->cb.callback)
 899                 vq->cb.callback(vq->cb.private);
 900         spin_unlock_irq(&vq->irq_lock);
 901 }
 902
 903 static int vduse_dev_queue_irq_work(struct vduse_dev *dev,
 904                                     struct work_struct *irq_work)
 905 {
 906         int ret = -EINVAL;
 907
 908         down_read(&dev->rwsem);
 909         if (!(dev->status & VIRTIO_CONFIG_S_DRIVER_OK))
 910                 goto unlock;
 911
 912         ret = 0;
 913         queue_work(vduse_irq_wq, irq_work);
 914 unlock:
 915         up_read(&dev->rwsem);
 916
 917         return ret;
 918 }
 919
 920 static long vduse_dev_ioctl(struct file *file, unsigned int cmd,
 921                             unsigned long arg)
 922 {
 923         struct vduse_dev *dev = file->private_data;
 924         void __user *argp = (void __user *)arg;
 925         int ret;
 926
 927         if (unlikely(dev->broken))
 928                 return -EPERM;
 929
 930         switch (cmd) {
 931         case VDUSE_IOTLB_GET_FD: {
 932                 struct vduse_iotlb_entry entry;
 933                 struct vhost_iotlb_map *map;
 934                 struct vdpa_map_file *map_file;
 935                 struct vduse_iova_domain *domain = dev->domain;
 936                 struct file *f = NULL;
 937
 938                 ret = -EFAULT;
 939                 if (copy_from_user(&entry, argp, sizeof(entry)))
 940                         break;
 941
 942                 ret = -EINVAL;
 943                 if (entry.start > entry.last)
 944                         break;
 945
 946                 spin_lock(&domain->iotlb_lock);
 947                 map = vhost_iotlb_itree_first(domain->iotlb,
 948                                               entry.start, entry.last);
 949                 if (map) {
 950                         map_file = (struct vdpa_map_file *)map->opaque;
 951                         f = get_file(map_file->file);
 952                         entry.offset = map_file->offset;
 953                         entry.start = map->start;
 954                         entry.last = map->last;
 955                         entry.perm = map->perm;
 956                 }
 957                 spin_unlock(&domain->iotlb_lock);
 958                 ret = -EINVAL;
 959                 if (!f)
 960                         break;
 961
 962                 ret = -EFAULT;
 963                 if (copy_to_user(argp, &entry, sizeof(entry))) {
 964                         fput(f);
 965                         break;
 966                 }
 967                 ret = receive_fd(f, perm_to_file_flags(entry.perm));
 968                 fput(f);
 969                 break;
 970         }
 971         case VDUSE_DEV_GET_FEATURES:
 972                 /*
 973                  * Just mirror what driver wrote here.
 974                  * The driver is expected to check FEATURE_OK later.
 975                  */
 976                 ret = put_user(dev->driver_features, (u64 __user *)argp);
 977                 break;
 978         case VDUSE_DEV_SET_CONFIG: {
 979                 struct vduse_config_data config;
 980                 unsigned long size = offsetof(struct vduse_config_data,
 981                                               buffer);
 982
 983                 ret = -EFAULT;
 984                 if (copy_from_user(&config, argp, size))
 985                         break;
 986
 987                 ret = -EINVAL;
 988                 if (config.offset > dev->config_size ||
 989                     config.length == 0 ||
 990                     config.length > dev->config_size - config.offset)
 991                         break;
 992
 993                 ret = -EFAULT;
 994                 if (copy_from_user(dev->config + config.offset, argp + size,
 995                                    config.length))
 996                         break;
 997
 998                 ret = 0;
 999                 break;
1000         }
1001         case VDUSE_DEV_INJECT_CONFIG_IRQ:
1002                 ret = vduse_dev_queue_irq_work(dev, &dev->inject);
1003                 break;
1004         case VDUSE_VQ_SETUP: {
1005                 struct vduse_vq_config config;
1006                 u32 index;
1007
1008                 ret = -EFAULT;
1009                 if (copy_from_user(&config, argp, sizeof(config)))
1010                         break;
1011
1012                 ret = -EINVAL;
1013                 if (config.index >= dev->vq_num)
1014                         break;
1015
1016                 if (!is_mem_zero((const char *)config.reserved,
1017                                  sizeof(config.reserved)))
1018                         break;
1019
1020                 index = array_index_nospec(config.index, dev->vq_num);
1021                 dev->vqs[index].num_max = config.max_size;
1022                 ret = 0;
1023                 break;
1024         }
1025         case VDUSE_VQ_GET_INFO: {
1026                 struct vduse_vq_info vq_info;
1027                 struct vduse_virtqueue *vq;
1028                 u32 index;
1029
1030                 ret = -EFAULT;
1031                 if (copy_from_user(&vq_info, argp, sizeof(vq_info)))
1032                         break;
1033
1034                 ret = -EINVAL;
1035                 if (vq_info.index >= dev->vq_num)
1036                         break;
1037
1038                 index = array_index_nospec(vq_info.index, dev->vq_num);
1039                 vq = &dev->vqs[index];
1040                 vq_info.desc_addr = vq->desc_addr;
1041                 vq_info.driver_addr = vq->driver_addr;
1042                 vq_info.device_addr = vq->device_addr;
1043                 vq_info.num = vq->num;
1044
1045                 if (dev->driver_features & BIT_ULL(VIRTIO_F_RING_PACKED)) {
1046                         vq_info.packed.last_avail_counter =
1047                                 vq->state.packed.last_avail_counter;
1048                         vq_info.packed.last_avail_idx =
1049                                 vq->state.packed.last_avail_idx;
1050                         vq_info.packed.last_used_counter =
1051                                 vq->state.packed.last_used_counter;
1052                         vq_info.packed.last_used_idx =
1053                                 vq->state.packed.last_used_idx;
1054                 } else
1055                         vq_info.split.avail_index =
1056                                 vq->state.split.avail_index;
1057
1058                 vq_info.ready = vq->ready;
1059
1060                 ret = -EFAULT;
1061                 if (copy_to_user(argp, &vq_info, sizeof(vq_info)))
1062                         break;
1063
1064                 ret = 0;
1065                 break;
1066         }
1067         case VDUSE_VQ_SETUP_KICKFD: {
1068                 struct vduse_vq_eventfd eventfd;
1069
1070                 ret = -EFAULT;
1071                 if (copy_from_user(&eventfd, argp, sizeof(eventfd)))
1072                         break;
1073
1074                 ret = vduse_kickfd_setup(dev, &eventfd);
1075                 break;
1076         }
1077         case VDUSE_VQ_INJECT_IRQ: {
1078                 u32 index;
1079
1080                 ret = -EFAULT;
1081                 if (get_user(index, (u32 __user *)argp))
1082                         break;
1083
1084                 ret = -EINVAL;
1085                 if (index >= dev->vq_num)
1086                         break;
1087
1088                 index = array_index_nospec(index, dev->vq_num);
1089                 ret = vduse_dev_queue_irq_work(dev, &dev->vqs[index].inject);
1090                 break;
1091         }
1092         default:
1093                 ret = -ENOIOCTLCMD;
1094                 break;
1095         }
1096
1097         return ret;
1098 }
1099
1100 static int vduse_dev_release(struct inode *inode, struct file *file)
1101 {
1102         struct vduse_dev *dev = file->private_data;
1103
1104         spin_lock(&dev->msg_lock);
1105         /* Make sure the inflight messages can processed after reconncection */
1106         list_splice_init(&dev->recv_list, &dev->send_list);
1107         spin_unlock(&dev->msg_lock);
1108         dev->connected = false;
1109
1110         return 0;
1111 }
1112
1113 static struct vduse_dev *vduse_dev_get_from_minor(int minor)
1114 {
1115         struct vduse_dev *dev;
1116
1117         mutex_lock(&vduse_lock);
1118         dev = idr_find(&vduse_idr, minor);
1119         mutex_unlock(&vduse_lock);
1120
1121         return dev;
1122 }
1123
1124 static int vduse_dev_open(struct inode *inode, struct file *file)
1125 {
1126         int ret;
1127         struct vduse_dev *dev = vduse_dev_get_from_minor(iminor(inode));
1128
1129         if (!dev)
1130                 return -ENODEV;
1131
1132         ret = -EBUSY;
1133         mutex_lock(&dev->lock);
1134         if (dev->connected)
1135                 goto unlock;
1136
1137         ret = 0;
1138         dev->connected = true;
1139         file->private_data = dev;
1140 unlock:
1141         mutex_unlock(&dev->lock);
1142
1143         return ret;
1144 }
1145
1146 static const struct file_operations vduse_dev_fops = {
1147         .owner          = THIS_MODULE,
1148         .open           = vduse_dev_open,
1149         .release        = vduse_dev_release,
1150         .read_iter      = vduse_dev_read_iter,
1151         .write_iter     = vduse_dev_write_iter,
1152         .poll           = vduse_dev_poll,
1153         .unlocked_ioctl = vduse_dev_ioctl,
1154         .compat_ioctl   = compat_ptr_ioctl,
1155         .llseek         = noop_llseek,
1156 };
1157
1158 static struct vduse_dev *vduse_dev_create(void)
1159 {
1160         struct vduse_dev *dev = kzalloc(sizeof(*dev), GFP_KERNEL);
1161
1162         if (!dev)
1163                 return NULL;
1164
1165         mutex_init(&dev->lock);
1166         spin_lock_init(&dev->msg_lock);
1167         INIT_LIST_HEAD(&dev->send_list);
1168         INIT_LIST_HEAD(&dev->recv_list);
1169         spin_lock_init(&dev->irq_lock);
1170         init_rwsem(&dev->rwsem);
1171
1172         INIT_WORK(&dev->inject, vduse_dev_irq_inject);
1173         init_waitqueue_head(&dev->waitq);
1174
1175         return dev;
1176 }
1177
1178 static void vduse_dev_destroy(struct vduse_dev *dev)
1179 {
1180         kfree(dev);
1181 }
1182
1183 static struct vduse_dev *vduse_find_dev(const char *name)
1184 {
1185         struct vduse_dev *dev;
1186         int id;
1187
1188         idr_for_each_entry(&vduse_idr, dev, id)
1189                 if (!strcmp(dev->name, name))
1190                         return dev;
1191
1192         return NULL;
1193 }
1194
1195 static int vduse_destroy_dev(char *name)
1196 {
1197         struct vduse_dev *dev = vduse_find_dev(name);
1198
1199         if (!dev)
1200                 return -EINVAL;
1201
1202         mutex_lock(&dev->lock);
1203         if (dev->vdev || dev->connected) {
1204                 mutex_unlock(&dev->lock);
1205                 return -EBUSY;
1206         }
1207         dev->connected = true;
1208         mutex_unlock(&dev->lock);
1209
1210         vduse_dev_reset(dev);
1211         device_destroy(vduse_class, MKDEV(MAJOR(vduse_major), dev->minor));
1212         idr_remove(&vduse_idr, dev->minor);
1213         kvfree(dev->config);
1214         kfree(dev->vqs);
1215         vduse_domain_destroy(dev->domain);
1216         kfree(dev->name);
1217         vduse_dev_destroy(dev);
1218         module_put(THIS_MODULE);
1219
1220         return 0;
1221 }
1222
1223 static bool device_is_allowed(u32 device_id)
1224 {
1225         int i;
1226
1227         for (i = 0; i < ARRAY_SIZE(allowed_device_id); i++)
1228                 if (allowed_device_id[i] == device_id)
1229                         return true;
1230
1231         return false;
1232 }
1233
1234 static bool features_is_valid(u64 features)
1235 {
1236         if (!(features & (1ULL << VIRTIO_F_ACCESS_PLATFORM)))
1237                 return false;
1238
1239         /* Now we only support read-only configuration space */
1240         if (features & (1ULL << VIRTIO_BLK_F_CONFIG_WCE))
1241                 return false;
1242
1243         return true;
1244 }
1245
1246 static bool vduse_validate_config(struct vduse_dev_config *config)
1247 {
1248         if (!is_mem_zero((const char *)config->reserved,
1249                          sizeof(config->reserved)))
1250                 return false;
1251
1252         if (config->vq_align > PAGE_SIZE)
1253                 return false;
1254
1255         if (config->config_size > PAGE_SIZE)
1256                 return false;
1257
1258         if (!device_is_allowed(config->device_id))
1259                 return false;
1260
1261         if (!features_is_valid(config->features))
1262                 return false;
1263
1264         return true;
1265 }
1266
1267 static ssize_t msg_timeout_show(struct device *device,
1268                                 struct device_attribute *attr, char *buf)
1269 {
1270         struct vduse_dev *dev = dev_get_drvdata(device);
1271
1272         return sysfs_emit(buf, "%u\n", dev->msg_timeout);
1273 }
1274
1275 static ssize_t msg_timeout_store(struct device *device,
1276                                  struct device_attribute *attr,
1277                                  const char *buf, size_t count)
1278 {
1279         struct vduse_dev *dev = dev_get_drvdata(device);
1280         int ret;
1281
1282         ret = kstrtouint(buf, 10, &dev->msg_timeout);
1283         if (ret < 0)
1284                 return ret;
1285
1286         return count;
1287 }
1288
1289 static DEVICE_ATTR_RW(msg_timeout);
1290
1291 static struct attribute *vduse_dev_attrs[] = {
1292         &dev_attr_msg_timeout.attr,
1293         NULL
1294 };
1295
1296 ATTRIBUTE_GROUPS(vduse_dev);
1297
1298 static int vduse_create_dev(struct vduse_dev_config *config,
1299                             void *config_buf, u64 api_version)
1300 {
1301         int i, ret;
1302         struct vduse_dev *dev;
1303
1304         ret = -EEXIST;
1305         if (vduse_find_dev(config->name))
1306                 goto err;
1307
1308         ret = -ENOMEM;
1309         dev = vduse_dev_create();
1310         if (!dev)
1311                 goto err;
1312
1313         dev->api_version = api_version;
1314         dev->device_features = config->features;
1315         dev->device_id = config->device_id;
1316         dev->vendor_id = config->vendor_id;
1317         dev->name = kstrdup(config->name, GFP_KERNEL);
1318         if (!dev->name)
1319                 goto err_str;
1320
1321         dev->domain = vduse_domain_create(VDUSE_IOVA_SIZE - 1,
1322                                           VDUSE_BOUNCE_SIZE);
1323         if (!dev->domain)
1324                 goto err_domain;
1325
1326         dev->config = config_buf;
1327         dev->config_size = config->config_size;
1328         dev->vq_align = config->vq_align;
1329         dev->vq_num = config->vq_num;
1330         dev->vqs = kcalloc(dev->vq_num, sizeof(*dev->vqs), GFP_KERNEL);
1331         if (!dev->vqs)
1332                 goto err_vqs;
1333
1334         for (i = 0; i < dev->vq_num; i++) {
1335                 dev->vqs[i].index = i;
1336                 INIT_WORK(&dev->vqs[i].inject, vduse_vq_irq_inject);
1337                 INIT_WORK(&dev->vqs[i].kick, vduse_vq_kick_work);
1338                 spin_lock_init(&dev->vqs[i].kick_lock);
1339                 spin_lock_init(&dev->vqs[i].irq_lock);
1340         }
1341
1342         ret = idr_alloc(&vduse_idr, dev, 1, VDUSE_DEV_MAX, GFP_KERNEL);
1343         if (ret < 0)
1344                 goto err_idr;
1345
1346         dev->minor = ret;
1347         dev->msg_timeout = VDUSE_MSG_DEFAULT_TIMEOUT;
1348         dev->dev = device_create_with_groups(vduse_class, NULL,
1349                                 MKDEV(MAJOR(vduse_major), dev->minor),
1350                                 dev, vduse_dev_groups, "%s", config->name);
1351         if (IS_ERR(dev->dev)) {
1352                 ret = PTR_ERR(dev->dev);
1353                 goto err_dev;
1354         }
1355         __module_get(THIS_MODULE);
1356
1357         return 0;
1358 err_dev:
1359         idr_remove(&vduse_idr, dev->minor);
1360 err_idr:
1361         kfree(dev->vqs);
1362 err_vqs:
1363         vduse_domain_destroy(dev->domain);
1364 err_domain:
1365         kfree(dev->name);
1366 err_str:
1367         vduse_dev_destroy(dev);
1368 err:
1369         return ret;
1370 }
1371
1372 static long vduse_ioctl(struct file *file, unsigned int cmd,
1373                         unsigned long arg)
1374 {
1375         int ret;
1376         void __user *argp = (void __user *)arg;
1377         struct vduse_control *control = file->private_data;
1378
1379         mutex_lock(&vduse_lock);
1380         switch (cmd) {
1381         case VDUSE_GET_API_VERSION:
1382                 ret = put_user(control->api_version, (u64 __user *)argp);
1383                 break;
1384         case VDUSE_SET_API_VERSION: {
1385                 u64 api_version;
1386
1387                 ret = -EFAULT;
1388                 if (get_user(api_version, (u64 __user *)argp))
1389                         break;
1390
1391                 ret = -EINVAL;
1392                 if (api_version > VDUSE_API_VERSION)
1393                         break;
1394
1395                 ret = 0;
1396                 control->api_version = api_version;
1397                 break;
1398         }
1399         case VDUSE_CREATE_DEV: {
1400                 struct vduse_dev_config config;
1401                 unsigned long size = offsetof(struct vduse_dev_config, config);
1402                 void *buf;
1403
1404                 ret = -EFAULT;
1405                 if (copy_from_user(&config, argp, size))
1406                         break;
1407
1408                 ret = -EINVAL;
1409                 if (vduse_validate_config(&config) == false)
1410                         break;
1411
1412                 buf = vmemdup_user(argp + size, config.config_size);
1413                 if (IS_ERR(buf)) {
1414                         ret = PTR_ERR(buf);
1415                         break;
1416                 }
1417                 config.name[VDUSE_NAME_MAX - 1] = '\0';
1418                 ret = vduse_create_dev(&config, buf, control->api_version);
1419                 if (ret)
1420                         kvfree(buf);
1421                 break;
1422         }
1423         case VDUSE_DESTROY_DEV: {
1424                 char name[VDUSE_NAME_MAX];
1425
1426                 ret = -EFAULT;
1427                 if (copy_from_user(name, argp, VDUSE_NAME_MAX))
1428                         break;
1429
1430                 name[VDUSE_NAME_MAX - 1] = '\0';
1431                 ret = vduse_destroy_dev(name);
1432                 break;
1433         }
1434         default:
1435                 ret = -EINVAL;
1436                 break;
1437         }
1438         mutex_unlock(&vduse_lock);
1439
1440         return ret;
1441 }
1442
1443 static int vduse_release(struct inode *inode, struct file *file)
1444 {
1445         struct vduse_control *control = file->private_data;
1446
1447         kfree(control);
1448         return 0;
1449 }
1450
1451 static int vduse_open(struct inode *inode, struct file *file)
1452 {
1453         struct vduse_control *control;
1454
1455         control = kmalloc(sizeof(struct vduse_control), GFP_KERNEL);
1456         if (!control)
1457                 return -ENOMEM;
1458
1459         control->api_version = VDUSE_API_VERSION;
1460         file->private_data = control;
1461
1462         return 0;
1463 }
1464
1465 static const struct file_operations vduse_ctrl_fops = {
1466         .owner          = THIS_MODULE,
1467         .open           = vduse_open,
1468         .release        = vduse_release,
1469         .unlocked_ioctl = vduse_ioctl,
1470         .compat_ioctl   = compat_ptr_ioctl,
1471         .llseek         = noop_llseek,
1472 };
1473
1474 static char *vduse_devnode(struct device *dev, umode_t *mode)
1475 {
1476         return kasprintf(GFP_KERNEL, "vduse/%s", dev_name(dev));
1477 }
1478
1479 static void vduse_mgmtdev_release(struct device *dev)
1480 {
1481 }
1482
1483 static struct device vduse_mgmtdev = {
1484         .init_name = "vduse",
1485         .release = vduse_mgmtdev_release,
1486 };
1487
1488 static struct vdpa_mgmt_dev mgmt_dev;
1489
1490 static int vduse_dev_init_vdpa(struct vduse_dev *dev, const char *name)
1491 {
1492         struct vduse_vdpa *vdev;
1493         int ret;
1494
1495         if (dev->vdev)
1496                 return -EEXIST;
1497
1498         vdev = vdpa_alloc_device(struct vduse_vdpa, vdpa, dev->dev,
1499                                  &vduse_vdpa_config_ops, 1, 1, name, true);
1500         if (IS_ERR(vdev))
1501                 return PTR_ERR(vdev);
1502
1503         dev->vdev = vdev;
1504         vdev->dev = dev;
1505         vdev->vdpa.dev.dma_mask = &vdev->vdpa.dev.coherent_dma_mask;
1506         ret = dma_set_mask_and_coherent(&vdev->vdpa.dev, DMA_BIT_MASK(64));
1507         if (ret) {
1508                 put_device(&vdev->vdpa.dev);
1509                 return ret;
1510         }
1511         set_dma_ops(&vdev->vdpa.dev, &vduse_dev_dma_ops);
1512         vdev->vdpa.dma_dev = &vdev->vdpa.dev;
1513         vdev->vdpa.mdev = &mgmt_dev;
1514
1515         return 0;
1516 }
1517
1518 static int vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name,
1519                         const struct vdpa_dev_set_config *config)
1520 {
1521         struct vduse_dev *dev;
1522         int ret;
1523
1524         mutex_lock(&vduse_lock);
1525         dev = vduse_find_dev(name);
1526         if (!dev || !vduse_dev_is_ready(dev)) {
1527                 mutex_unlock(&vduse_lock);
1528                 return -EINVAL;
1529         }
1530         ret = vduse_dev_init_vdpa(dev, name);
1531         mutex_unlock(&vduse_lock);
1532         if (ret)
1533                 return ret;
1534
1535         ret = _vdpa_register_device(&dev->vdev->vdpa, dev->vq_num);
1536         if (ret) {
1537                 put_device(&dev->vdev->vdpa.dev);
1538                 return ret;
1539         }
1540
1541         return 0;
1542 }
1543
1544 static void vdpa_dev_del(struct vdpa_mgmt_dev *mdev, struct vdpa_device *dev)
1545 {
1546         _vdpa_unregister_device(dev);
1547 }
1548
1549 static const struct vdpa_mgmtdev_ops vdpa_dev_mgmtdev_ops = {
1550         .dev_add = vdpa_dev_add,
1551         .dev_del = vdpa_dev_del,
1552 };
1553
1554 static struct virtio_device_id id_table[] = {
1555         { VIRTIO_ID_BLOCK, VIRTIO_DEV_ANY_ID },
1556         { 0 },
1557 };
1558
1559 static struct vdpa_mgmt_dev mgmt_dev = {
1560         .device = &vduse_mgmtdev,
1561         .id_table = id_table,
1562         .ops = &vdpa_dev_mgmtdev_ops,
1563 };
1564
1565 static int vduse_mgmtdev_init(void)
1566 {
1567         int ret;
1568
1569         ret = device_register(&vduse_mgmtdev);
1570         if (ret)
1571                 return ret;
1572
1573         ret = vdpa_mgmtdev_register(&mgmt_dev);
1574         if (ret)
1575                 goto err;
1576
1577         return 0;
1578 err:
1579         device_unregister(&vduse_mgmtdev);
1580         return ret;
1581 }
1582
1583 static void vduse_mgmtdev_exit(void)
1584 {
1585         vdpa_mgmtdev_unregister(&mgmt_dev);
1586         device_unregister(&vduse_mgmtdev);
1587 }
1588
1589 static int vduse_init(void)
1590 {
1591         int ret;
1592         struct device *dev;
1593
1594         vduse_class = class_create(THIS_MODULE, "vduse");
1595         if (IS_ERR(vduse_class))
1596                 return PTR_ERR(vduse_class);
1597
1598         vduse_class->devnode = vduse_devnode;
1599
1600         ret = alloc_chrdev_region(&vduse_major, 0, VDUSE_DEV_MAX, "vduse");
1601         if (ret)
1602                 goto err_chardev_region;
1603
1604         /* /dev/vduse/control */
1605         cdev_init(&vduse_ctrl_cdev, &vduse_ctrl_fops);
1606         vduse_ctrl_cdev.owner = THIS_MODULE;
1607         ret = cdev_add(&vduse_ctrl_cdev, vduse_major, 1);
1608         if (ret)
1609                 goto err_ctrl_cdev;
1610
1611         dev = device_create(vduse_class, NULL, vduse_major, NULL, "control");
1612         if (IS_ERR(dev)) {
1613                 ret = PTR_ERR(dev);
1614                 goto err_device;
1615         }
1616
1617         /* /dev/vduse/$DEVICE */
1618         cdev_init(&vduse_cdev, &vduse_dev_fops);
1619         vduse_cdev.owner = THIS_MODULE;
1620         ret = cdev_add(&vduse_cdev, MKDEV(MAJOR(vduse_major), 1),
1621                        VDUSE_DEV_MAX - 1);
1622         if (ret)
1623                 goto err_cdev;
1624
1625         vduse_irq_wq = alloc_workqueue("vduse-irq",
1626                                 WQ_HIGHPRI | WQ_SYSFS | WQ_UNBOUND, 0);
1627         if (!vduse_irq_wq) {
1628                 ret = -ENOMEM;
1629                 goto err_wq;
1630         }
1631
1632         ret = vduse_domain_init();
1633         if (ret)
1634                 goto err_domain;
1635
1636         ret = vduse_mgmtdev_init();
1637         if (ret)
1638                 goto err_mgmtdev;
1639
1640         return 0;
1641 err_mgmtdev:
1642         vduse_domain_exit();
1643 err_domain:
1644         destroy_workqueue(vduse_irq_wq);
1645 err_wq:
1646         cdev_del(&vduse_cdev);
1647 err_cdev:
1648         device_destroy(vduse_class, vduse_major);
1649 err_device:
1650         cdev_del(&vduse_ctrl_cdev);
1651 err_ctrl_cdev:
1652         unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1653 err_chardev_region:
1654         class_destroy(vduse_class);
1655         return ret;
1656 }
1657 module_init(vduse_init);
1658
1659 static void vduse_exit(void)
1660 {
1661         vduse_mgmtdev_exit();
1662         vduse_domain_exit();
1663         destroy_workqueue(vduse_irq_wq);
1664         cdev_del(&vduse_cdev);
1665         device_destroy(vduse_class, vduse_major);
1666         cdev_del(&vduse_ctrl_cdev);
1667         unregister_chrdev_region(vduse_major, VDUSE_DEV_MAX);
1668         class_destroy(vduse_class);
1669 }
1670 module_exit(vduse_exit);
1671
1672 MODULE_LICENSE(DRV_LICENSE);
1673 MODULE_AUTHOR(DRV_AUTHOR);
1674 MODULE_DESCRIPTION(DRV_DESC);