samples/vfio-mdev/mtty.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Mediated virtual PCI serial host device driver
   4  *
   5  * Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved.
   6  *     Author: Neo Jia <cjia@nvidia.com>
   7  *             Kirti Wankhede <kwankhede@nvidia.com>
   8  *
   9  * Sample driver that creates mdev device that simulates serial port over PCI
  10  * card.
  11  */
  12
  13 #include <linux/init.h>
  14 #include <linux/module.h>
  15 #include <linux/kernel.h>
  16 #include <linux/fs.h>
  17 #include <linux/poll.h>
  18 #include <linux/slab.h>
  19 #include <linux/cdev.h>
  20 #include <linux/sched.h>
  21 #include <linux/wait.h>
  22 #include <linux/vfio.h>
  23 #include <linux/iommu.h>
  24 #include <linux/sysfs.h>
  25 #include <linux/ctype.h>
  26 #include <linux/file.h>
  27 #include <linux/mdev.h>
  28 #include <linux/pci.h>
  29 #include <linux/serial.h>
  30 #include <uapi/linux/serial_reg.h>
  31 #include <linux/eventfd.h>
  32 #include <linux/anon_inodes.h>
  33
  34 /*
  35  * #defines
  36  */
  37
  38 #define VERSION_STRING  "0.1"
  39 #define DRIVER_AUTHOR   "NVIDIA Corporation"
  40
  41 #define MTTY_CLASS_NAME "mtty"
  42
  43 #define MTTY_NAME       "mtty"
  44
  45 #define MTTY_STRING_LEN         16
  46
  47 #define MTTY_CONFIG_SPACE_SIZE  0xff
  48 #define MTTY_IO_BAR_SIZE        0x8
  49 #define MTTY_MMIO_BAR_SIZE      0x100000
  50
  51 #define STORE_LE16(addr, val)   (*(u16 *)addr = val)
  52 #define STORE_LE32(addr, val)   (*(u32 *)addr = val)
  53
  54 #define MAX_FIFO_SIZE   16
  55
  56 #define CIRCULAR_BUF_INC_IDX(idx)    (idx = (idx + 1) & (MAX_FIFO_SIZE - 1))
  57
  58 #define MTTY_VFIO_PCI_OFFSET_SHIFT   40
  59
  60 #define MTTY_VFIO_PCI_OFFSET_TO_INDEX(off)   (off >> MTTY_VFIO_PCI_OFFSET_SHIFT)
  61 #define MTTY_VFIO_PCI_INDEX_TO_OFFSET(index) \
  62                                 ((u64)(index) << MTTY_VFIO_PCI_OFFSET_SHIFT)
  63 #define MTTY_VFIO_PCI_OFFSET_MASK    \
  64                                 (((u64)(1) << MTTY_VFIO_PCI_OFFSET_SHIFT) - 1)
  65 #define MAX_MTTYS       24
  66
  67 /*
  68  * Global Structures
  69  */
  70
  71 static struct mtty_dev {
  72         dev_t           vd_devt;
  73         struct class    *vd_class;
  74         struct cdev     vd_cdev;
  75         struct idr      vd_idr;
  76         struct device   dev;
  77         struct mdev_parent parent;
  78 } mtty_dev;
  79
  80 struct mdev_region_info {
  81         u64 start;
  82         u64 phys_start;
  83         u32 size;
  84         u64 vfio_offset;
  85 };
  86
  87 #if defined(DEBUG_REGS)
  88 static const char *wr_reg[] = {
  89         "TX",
  90         "IER",
  91         "FCR",
  92         "LCR",
  93         "MCR",
  94         "LSR",
  95         "MSR",
  96         "SCR"
  97 };
  98
  99 static const char *rd_reg[] = {
 100         "RX",
 101         "IER",
 102         "IIR",
 103         "LCR",
 104         "MCR",
 105         "LSR",
 106         "MSR",
 107         "SCR"
 108 };
 109 #endif
 110
 111 /* loop back buffer */
 112 struct rxtx {
 113         u8 fifo[MAX_FIFO_SIZE];
 114         u8 head, tail;
 115         u8 count;
 116 };
 117
 118 struct serial_port {
 119         u8 uart_reg[8];         /* 8 registers */
 120         struct rxtx rxtx;       /* loop back buffer */
 121         bool dlab;
 122         bool overrun;
 123         u16 divisor;
 124         u8 fcr;                 /* FIFO control register */
 125         u8 max_fifo_size;
 126         u8 intr_trigger_level;  /* interrupt trigger level */
 127 };
 128
 129 struct mtty_data {
 130         u64 magic;
 131 #define MTTY_MAGIC 0x7e9d09898c3e2c4e /* Nothing clever, just random */
 132         u32 major_ver;
 133 #define MTTY_MAJOR_VER 1
 134         u32 minor_ver;
 135 #define MTTY_MINOR_VER 0
 136         u32 nr_ports;
 137         u32 flags;
 138         struct serial_port ports[2];
 139 };
 140
 141 struct mdev_state;
 142
 143 struct mtty_migration_file {
 144         struct file *filp;
 145         struct mutex lock;
 146         struct mdev_state *mdev_state;
 147         struct mtty_data data;
 148         ssize_t filled_size;
 149         u8 disabled:1;
 150 };
 151
 152 /* State of each mdev device */
 153 struct mdev_state {
 154         struct vfio_device vdev;
 155         struct eventfd_ctx *intx_evtfd;
 156         struct eventfd_ctx *msi_evtfd;
 157         int irq_index;
 158         u8 *vconfig;
 159         struct mutex ops_lock;
 160         struct mdev_device *mdev;
 161         struct mdev_region_info region_info[VFIO_PCI_NUM_REGIONS];
 162         u32 bar_mask[VFIO_PCI_NUM_REGIONS];
 163         struct list_head next;
 164         struct serial_port s[2];
 165         struct mutex rxtx_lock;
 166         struct vfio_device_info dev_info;
 167         int nr_ports;
 168         enum vfio_device_mig_state state;
 169         struct mutex state_mutex;
 170         struct mutex reset_mutex;
 171         struct mtty_migration_file *saving_migf;
 172         struct mtty_migration_file *resuming_migf;
 173         u8 deferred_reset:1;
 174         u8 intx_mask:1;
 175 };
 176
 177 static struct mtty_type {
 178         struct mdev_type type;
 179         int nr_ports;
 180 } mtty_types[2] = {
 181         { .nr_ports = 1, .type.sysfs_name = "1",
 182           .type.pretty_name = "Single port serial" },
 183         { .nr_ports = 2, .type.sysfs_name = "2",
 184           .type.pretty_name = "Dual port serial" },
 185 };
 186
 187 static struct mdev_type *mtty_mdev_types[] = {
 188         &mtty_types[0].type,
 189         &mtty_types[1].type,
 190 };
 191
 192 static atomic_t mdev_avail_ports = ATOMIC_INIT(MAX_MTTYS);
 193
 194 static const struct file_operations vd_fops = {
 195         .owner          = THIS_MODULE,
 196 };
 197
 198 static const struct vfio_device_ops mtty_dev_ops;
 199
 200 /* Helper functions */
 201
 202 static void dump_buffer(u8 *buf, uint32_t count)
 203 {
 204 #if defined(DEBUG)
 205         int i;
 206
 207         pr_info("Buffer:\n");
 208         for (i = 0; i < count; i++) {
 209                 pr_info("%2x ", *(buf + i));
 210                 if ((i + 1) % 16 == 0)
 211                         pr_info("\n");
 212         }
 213 #endif
 214 }
 215
 216 static bool is_intx(struct mdev_state *mdev_state)
 217 {
 218         return mdev_state->irq_index == VFIO_PCI_INTX_IRQ_INDEX;
 219 }
 220
 221 static bool is_msi(struct mdev_state *mdev_state)
 222 {
 223         return mdev_state->irq_index == VFIO_PCI_MSI_IRQ_INDEX;
 224 }
 225
 226 static bool is_noirq(struct mdev_state *mdev_state)
 227 {
 228         return !is_intx(mdev_state) && !is_msi(mdev_state);
 229 }
 230
 231 static void mtty_trigger_interrupt(struct mdev_state *mdev_state)
 232 {
 233         lockdep_assert_held(&mdev_state->ops_lock);
 234
 235         if (is_msi(mdev_state)) {
 236                 if (mdev_state->msi_evtfd)
 237                         eventfd_signal(mdev_state->msi_evtfd);
 238         } else if (is_intx(mdev_state)) {
 239                 if (mdev_state->intx_evtfd && !mdev_state->intx_mask) {
 240                         eventfd_signal(mdev_state->intx_evtfd);
 241                         mdev_state->intx_mask = true;
 242                 }
 243         }
 244 }
 245
 246 static void mtty_create_config_space(struct mdev_state *mdev_state)
 247 {
 248         /* PCI dev ID */
 249         STORE_LE32((u32 *) &mdev_state->vconfig[0x0], 0x32534348);
 250
 251         /* Control: I/O+, Mem-, BusMaster- */
 252         STORE_LE16((u16 *) &mdev_state->vconfig[0x4], 0x0001);
 253
 254         /* Status: capabilities list absent */
 255         STORE_LE16((u16 *) &mdev_state->vconfig[0x6], 0x0200);
 256
 257         /* Rev ID */
 258         mdev_state->vconfig[0x8] =  0x10;
 259
 260         /* programming interface class : 16550-compatible serial controller */
 261         mdev_state->vconfig[0x9] =  0x02;
 262
 263         /* Sub class : 00 */
 264         mdev_state->vconfig[0xa] =  0x00;
 265
 266         /* Base class : Simple Communication controllers */
 267         mdev_state->vconfig[0xb] =  0x07;
 268
 269         /* base address registers */
 270         /* BAR0: IO space */
 271         STORE_LE32((u32 *) &mdev_state->vconfig[0x10], 0x000001);
 272         mdev_state->bar_mask[0] = ~(MTTY_IO_BAR_SIZE) + 1;
 273
 274         if (mdev_state->nr_ports == 2) {
 275                 /* BAR1: IO space */
 276                 STORE_LE32((u32 *) &mdev_state->vconfig[0x14], 0x000001);
 277                 mdev_state->bar_mask[1] = ~(MTTY_IO_BAR_SIZE) + 1;
 278         }
 279
 280         /* Subsystem ID */
 281         STORE_LE32((u32 *) &mdev_state->vconfig[0x2c], 0x32534348);
 282
 283         mdev_state->vconfig[0x34] =  0x00;   /* Cap Ptr */
 284         mdev_state->vconfig[0x3d] =  0x01;   /* interrupt pin (INTA#) */
 285
 286         /* Vendor specific data */
 287         mdev_state->vconfig[0x40] =  0x23;
 288         mdev_state->vconfig[0x43] =  0x80;
 289         mdev_state->vconfig[0x44] =  0x23;
 290         mdev_state->vconfig[0x48] =  0x23;
 291         mdev_state->vconfig[0x4c] =  0x23;
 292
 293         mdev_state->vconfig[0x60] =  0x50;
 294         mdev_state->vconfig[0x61] =  0x43;
 295         mdev_state->vconfig[0x62] =  0x49;
 296         mdev_state->vconfig[0x63] =  0x20;
 297         mdev_state->vconfig[0x64] =  0x53;
 298         mdev_state->vconfig[0x65] =  0x65;
 299         mdev_state->vconfig[0x66] =  0x72;
 300         mdev_state->vconfig[0x67] =  0x69;
 301         mdev_state->vconfig[0x68] =  0x61;
 302         mdev_state->vconfig[0x69] =  0x6c;
 303         mdev_state->vconfig[0x6a] =  0x2f;
 304         mdev_state->vconfig[0x6b] =  0x55;
 305         mdev_state->vconfig[0x6c] =  0x41;
 306         mdev_state->vconfig[0x6d] =  0x52;
 307         mdev_state->vconfig[0x6e] =  0x54;
 308 }
 309
 310 static void handle_pci_cfg_write(struct mdev_state *mdev_state, u16 offset,
 311                                  u8 *buf, u32 count)
 312 {
 313         u32 cfg_addr, bar_mask, bar_index = 0;
 314
 315         switch (offset) {
 316         case 0x04: /* device control */
 317         case 0x06: /* device status */
 318                 /* do nothing */
 319                 break;
 320         case 0x3c:  /* interrupt line */
 321                 mdev_state->vconfig[0x3c] = buf[0];
 322                 break;
 323         case 0x3d:
 324                 /*
 325                  * Interrupt Pin is hardwired to INTA.
 326                  * This field is write protected by hardware
 327                  */
 328                 break;
 329         case 0x10:  /* BAR0 */
 330         case 0x14:  /* BAR1 */
 331                 if (offset == 0x10)
 332                         bar_index = 0;
 333                 else if (offset == 0x14)
 334                         bar_index = 1;
 335
 336                 if ((mdev_state->nr_ports == 1) && (bar_index == 1)) {
 337                         STORE_LE32(&mdev_state->vconfig[offset], 0);
 338                         break;
 339                 }
 340
 341                 cfg_addr = *(u32 *)buf;
 342                 pr_info("BAR%d addr 0x%x\n", bar_index, cfg_addr);
 343
 344                 if (cfg_addr == 0xffffffff) {
 345                         bar_mask = mdev_state->bar_mask[bar_index];
 346                         cfg_addr = (cfg_addr & bar_mask);
 347                 }
 348
 349                 cfg_addr |= (mdev_state->vconfig[offset] & 0x3ul);
 350                 STORE_LE32(&mdev_state->vconfig[offset], cfg_addr);
 351                 break;
 352         case 0x18:  /* BAR2 */
 353         case 0x1c:  /* BAR3 */
 354         case 0x20:  /* BAR4 */
 355                 STORE_LE32(&mdev_state->vconfig[offset], 0);
 356                 break;
 357         default:
 358                 pr_info("PCI config write @0x%x of %d bytes not handled\n",
 359                         offset, count);
 360                 break;
 361         }
 362 }
 363
 364 static void handle_bar_write(unsigned int index, struct mdev_state *mdev_state,
 365                                 u16 offset, u8 *buf, u32 count)
 366 {
 367         u8 data = *buf;
 368
 369         /* Handle data written by guest */
 370         switch (offset) {
 371         case UART_TX:
 372                 /* if DLAB set, data is LSB of divisor */
 373                 if (mdev_state->s[index].dlab) {
 374                         mdev_state->s[index].divisor |= data;
 375                         break;
 376                 }
 377
 378                 mutex_lock(&mdev_state->rxtx_lock);
 379
 380                 /* save in TX buffer */
 381                 if (mdev_state->s[index].rxtx.count <
 382                                 mdev_state->s[index].max_fifo_size) {
 383                         mdev_state->s[index].rxtx.fifo[
 384                                         mdev_state->s[index].rxtx.head] = data;
 385                         mdev_state->s[index].rxtx.count++;
 386                         CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.head);
 387                         mdev_state->s[index].overrun = false;
 388
 389                         /*
 390                          * Trigger interrupt if receive data interrupt is
 391                          * enabled and fifo reached trigger level
 392                          */
 393                         if ((mdev_state->s[index].uart_reg[UART_IER] &
 394                                                 UART_IER_RDI) &&
 395                            (mdev_state->s[index].rxtx.count ==
 396                                     mdev_state->s[index].intr_trigger_level)) {
 397                                 /* trigger interrupt */
 398 #if defined(DEBUG_INTR)
 399                                 pr_err("Serial port %d: Fifo level trigger\n",
 400                                         index);
 401 #endif
 402                                 mtty_trigger_interrupt(mdev_state);
 403                         }
 404                 } else {
 405 #if defined(DEBUG_INTR)
 406                         pr_err("Serial port %d: Buffer Overflow\n", index);
 407 #endif
 408                         mdev_state->s[index].overrun = true;
 409
 410                         /*
 411                          * Trigger interrupt if receiver line status interrupt
 412                          * is enabled
 413                          */
 414                         if (mdev_state->s[index].uart_reg[UART_IER] &
 415                                                                 UART_IER_RLSI)
 416                                 mtty_trigger_interrupt(mdev_state);
 417                 }
 418                 mutex_unlock(&mdev_state->rxtx_lock);
 419                 break;
 420
 421         case UART_IER:
 422                 /* if DLAB set, data is MSB of divisor */
 423                 if (mdev_state->s[index].dlab)
 424                         mdev_state->s[index].divisor |= (u16)data << 8;
 425                 else {
 426                         mdev_state->s[index].uart_reg[offset] = data;
 427                         mutex_lock(&mdev_state->rxtx_lock);
 428                         if ((data & UART_IER_THRI) &&
 429                             (mdev_state->s[index].rxtx.head ==
 430                                         mdev_state->s[index].rxtx.tail)) {
 431 #if defined(DEBUG_INTR)
 432                                 pr_err("Serial port %d: IER_THRI write\n",
 433                                         index);
 434 #endif
 435                                 mtty_trigger_interrupt(mdev_state);
 436                         }
 437
 438                         mutex_unlock(&mdev_state->rxtx_lock);
 439                 }
 440
 441                 break;
 442
 443         case UART_FCR:
 444                 mdev_state->s[index].fcr = data;
 445
 446                 mutex_lock(&mdev_state->rxtx_lock);
 447                 if (data & (UART_FCR_CLEAR_RCVR | UART_FCR_CLEAR_XMIT)) {
 448                         /* clear loop back FIFO */
 449                         mdev_state->s[index].rxtx.count = 0;
 450                         mdev_state->s[index].rxtx.head = 0;
 451                         mdev_state->s[index].rxtx.tail = 0;
 452                 }
 453                 mutex_unlock(&mdev_state->rxtx_lock);
 454
 455                 switch (data & UART_FCR_TRIGGER_MASK) {
 456                 case UART_FCR_TRIGGER_1:
 457                         mdev_state->s[index].intr_trigger_level = 1;
 458                         break;
 459
 460                 case UART_FCR_TRIGGER_4:
 461                         mdev_state->s[index].intr_trigger_level = 4;
 462                         break;
 463
 464                 case UART_FCR_TRIGGER_8:
 465                         mdev_state->s[index].intr_trigger_level = 8;
 466                         break;
 467
 468                 case UART_FCR_TRIGGER_14:
 469                         mdev_state->s[index].intr_trigger_level = 14;
 470                         break;
 471                 }
 472
 473                 /*
 474                  * Set trigger level to 1 otherwise or  implement timer with
 475                  * timeout of 4 characters and on expiring that timer set
 476                  * Recevice data timeout in IIR register
 477                  */
 478                 mdev_state->s[index].intr_trigger_level = 1;
 479                 if (data & UART_FCR_ENABLE_FIFO)
 480                         mdev_state->s[index].max_fifo_size = MAX_FIFO_SIZE;
 481                 else {
 482                         mdev_state->s[index].max_fifo_size = 1;
 483                         mdev_state->s[index].intr_trigger_level = 1;
 484                 }
 485
 486                 break;
 487
 488         case UART_LCR:
 489                 if (data & UART_LCR_DLAB) {
 490                         mdev_state->s[index].dlab = true;
 491                         mdev_state->s[index].divisor = 0;
 492                 } else
 493                         mdev_state->s[index].dlab = false;
 494
 495                 mdev_state->s[index].uart_reg[offset] = data;
 496                 break;
 497
 498         case UART_MCR:
 499                 mdev_state->s[index].uart_reg[offset] = data;
 500
 501                 if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) &&
 502                                 (data & UART_MCR_OUT2)) {
 503 #if defined(DEBUG_INTR)
 504                         pr_err("Serial port %d: MCR_OUT2 write\n", index);
 505 #endif
 506                         mtty_trigger_interrupt(mdev_state);
 507                 }
 508
 509                 if ((mdev_state->s[index].uart_reg[UART_IER] & UART_IER_MSI) &&
 510                                 (data & (UART_MCR_RTS | UART_MCR_DTR))) {
 511 #if defined(DEBUG_INTR)
 512                         pr_err("Serial port %d: MCR RTS/DTR write\n", index);
 513 #endif
 514                         mtty_trigger_interrupt(mdev_state);
 515                 }
 516                 break;
 517
 518         case UART_LSR:
 519         case UART_MSR:
 520                 /* do nothing */
 521                 break;
 522
 523         case UART_SCR:
 524                 mdev_state->s[index].uart_reg[offset] = data;
 525                 break;
 526
 527         default:
 528                 break;
 529         }
 530 }
 531
 532 static void handle_bar_read(unsigned int index, struct mdev_state *mdev_state,
 533                             u16 offset, u8 *buf, u32 count)
 534 {
 535         /* Handle read requests by guest */
 536         switch (offset) {
 537         case UART_RX:
 538                 /* if DLAB set, data is LSB of divisor */
 539                 if (mdev_state->s[index].dlab) {
 540                         *buf  = (u8)mdev_state->s[index].divisor;
 541                         break;
 542                 }
 543
 544                 mutex_lock(&mdev_state->rxtx_lock);
 545                 /* return data in tx buffer */
 546                 if (mdev_state->s[index].rxtx.head !=
 547                                  mdev_state->s[index].rxtx.tail) {
 548                         *buf = mdev_state->s[index].rxtx.fifo[
 549                                                 mdev_state->s[index].rxtx.tail];
 550                         mdev_state->s[index].rxtx.count--;
 551                         CIRCULAR_BUF_INC_IDX(mdev_state->s[index].rxtx.tail);
 552                 }
 553
 554                 if (mdev_state->s[index].rxtx.head ==
 555                                 mdev_state->s[index].rxtx.tail) {
 556                 /*
 557                  *  Trigger interrupt if tx buffer empty interrupt is
 558                  *  enabled and fifo is empty
 559                  */
 560 #if defined(DEBUG_INTR)
 561                         pr_err("Serial port %d: Buffer Empty\n", index);
 562 #endif
 563                         if (mdev_state->s[index].uart_reg[UART_IER] &
 564                                                          UART_IER_THRI)
 565                                 mtty_trigger_interrupt(mdev_state);
 566                 }
 567                 mutex_unlock(&mdev_state->rxtx_lock);
 568
 569                 break;
 570
 571         case UART_IER:
 572                 if (mdev_state->s[index].dlab) {
 573                         *buf = (u8)(mdev_state->s[index].divisor >> 8);
 574                         break;
 575                 }
 576                 *buf = mdev_state->s[index].uart_reg[offset] & 0x0f;
 577                 break;
 578
 579         case UART_IIR:
 580         {
 581                 u8 ier = mdev_state->s[index].uart_reg[UART_IER];
 582                 *buf = 0;
 583
 584                 mutex_lock(&mdev_state->rxtx_lock);
 585                 /* Interrupt priority 1: Parity, overrun, framing or break */
 586                 if ((ier & UART_IER_RLSI) && mdev_state->s[index].overrun)
 587                         *buf |= UART_IIR_RLSI;
 588
 589                 /* Interrupt priority 2: Fifo trigger level reached */
 590                 if ((ier & UART_IER_RDI) &&
 591                     (mdev_state->s[index].rxtx.count >=
 592                       mdev_state->s[index].intr_trigger_level))
 593                         *buf |= UART_IIR_RDI;
 594
 595                 /* Interrupt priotiry 3: transmitter holding register empty */
 596                 if ((ier & UART_IER_THRI) &&
 597                     (mdev_state->s[index].rxtx.head ==
 598                                 mdev_state->s[index].rxtx.tail))
 599                         *buf |= UART_IIR_THRI;
 600
 601                 /* Interrupt priotiry 4: Modem status: CTS, DSR, RI or DCD  */
 602                 if ((ier & UART_IER_MSI) &&
 603                     (mdev_state->s[index].uart_reg[UART_MCR] &
 604                                  (UART_MCR_RTS | UART_MCR_DTR)))
 605                         *buf |= UART_IIR_MSI;
 606
 607                 /* bit0: 0=> interrupt pending, 1=> no interrupt is pending */
 608                 if (*buf == 0)
 609                         *buf = UART_IIR_NO_INT;
 610
 611                 /* set bit 6 & 7 to be 16550 compatible */
 612                 *buf |= 0xC0;
 613                 mutex_unlock(&mdev_state->rxtx_lock);
 614         }
 615         break;
 616
 617         case UART_LCR:
 618         case UART_MCR:
 619                 *buf = mdev_state->s[index].uart_reg[offset];
 620                 break;
 621
 622         case UART_LSR:
 623         {
 624                 u8 lsr = 0;
 625
 626                 mutex_lock(&mdev_state->rxtx_lock);
 627                 /* atleast one char in FIFO */
 628                 if (mdev_state->s[index].rxtx.head !=
 629                                  mdev_state->s[index].rxtx.tail)
 630                         lsr |= UART_LSR_DR;
 631
 632                 /* if FIFO overrun */
 633                 if (mdev_state->s[index].overrun)
 634                         lsr |= UART_LSR_OE;
 635
 636                 /* transmit FIFO empty and tramsitter empty */
 637                 if (mdev_state->s[index].rxtx.head ==
 638                                  mdev_state->s[index].rxtx.tail)
 639                         lsr |= UART_LSR_TEMT | UART_LSR_THRE;
 640
 641                 mutex_unlock(&mdev_state->rxtx_lock);
 642                 *buf = lsr;
 643                 break;
 644         }
 645         case UART_MSR:
 646                 *buf = UART_MSR_DSR | UART_MSR_DDSR | UART_MSR_DCD;
 647
 648                 mutex_lock(&mdev_state->rxtx_lock);
 649                 /* if AFE is 1 and FIFO have space, set CTS bit */
 650                 if (mdev_state->s[index].uart_reg[UART_MCR] &
 651                                                  UART_MCR_AFE) {
 652                         if (mdev_state->s[index].rxtx.count <
 653                                         mdev_state->s[index].max_fifo_size)
 654                                 *buf |= UART_MSR_CTS | UART_MSR_DCTS;
 655                 } else
 656                         *buf |= UART_MSR_CTS | UART_MSR_DCTS;
 657                 mutex_unlock(&mdev_state->rxtx_lock);
 658
 659                 break;
 660
 661         case UART_SCR:
 662                 *buf = mdev_state->s[index].uart_reg[offset];
 663                 break;
 664
 665         default:
 666                 break;
 667         }
 668 }
 669
 670 static void mdev_read_base(struct mdev_state *mdev_state)
 671 {
 672         int index, pos;
 673         u32 start_lo, start_hi;
 674         u32 mem_type;
 675
 676         pos = PCI_BASE_ADDRESS_0;
 677
 678         for (index = 0; index <= VFIO_PCI_BAR5_REGION_INDEX; index++) {
 679
 680                 if (!mdev_state->region_info[index].size)
 681                         continue;
 682
 683                 start_lo = (*(u32 *)(mdev_state->vconfig + pos)) &
 684                         PCI_BASE_ADDRESS_MEM_MASK;
 685                 mem_type = (*(u32 *)(mdev_state->vconfig + pos)) &
 686                         PCI_BASE_ADDRESS_MEM_TYPE_MASK;
 687
 688                 switch (mem_type) {
 689                 case PCI_BASE_ADDRESS_MEM_TYPE_64:
 690                         start_hi = (*(u32 *)(mdev_state->vconfig + pos + 4));
 691                         pos += 4;
 692                         break;
 693                 case PCI_BASE_ADDRESS_MEM_TYPE_32:
 694                 case PCI_BASE_ADDRESS_MEM_TYPE_1M:
 695                         /* 1M mem BAR treated as 32-bit BAR */
 696                 default:
 697                         /* mem unknown type treated as 32-bit BAR */
 698                         start_hi = 0;
 699                         break;
 700                 }
 701                 pos += 4;
 702                 mdev_state->region_info[index].start = ((u64)start_hi << 32) |
 703                                                         start_lo;
 704         }
 705 }
 706
 707 static ssize_t mdev_access(struct mdev_state *mdev_state, u8 *buf, size_t count,
 708                            loff_t pos, bool is_write)
 709 {
 710         unsigned int index;
 711         loff_t offset;
 712         int ret = 0;
 713
 714         if (!buf)
 715                 return -EINVAL;
 716
 717         mutex_lock(&mdev_state->ops_lock);
 718
 719         index = MTTY_VFIO_PCI_OFFSET_TO_INDEX(pos);
 720         offset = pos & MTTY_VFIO_PCI_OFFSET_MASK;
 721         switch (index) {
 722         case VFIO_PCI_CONFIG_REGION_INDEX:
 723
 724 #if defined(DEBUG)
 725                 pr_info("%s: PCI config space %s at offset 0x%llx\n",
 726                          __func__, is_write ? "write" : "read", offset);
 727 #endif
 728                 if (is_write) {
 729                         dump_buffer(buf, count);
 730                         handle_pci_cfg_write(mdev_state, offset, buf, count);
 731                 } else {
 732                         memcpy(buf, (mdev_state->vconfig + offset), count);
 733                         dump_buffer(buf, count);
 734                 }
 735
 736                 break;
 737
 738         case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
 739                 if (!mdev_state->region_info[index].start)
 740                         mdev_read_base(mdev_state);
 741
 742                 if (is_write) {
 743                         dump_buffer(buf, count);
 744
 745 #if defined(DEBUG_REGS)
 746                         pr_info("%s: BAR%d  WR @0x%llx %s val:0x%02x dlab:%d\n",
 747                                 __func__, index, offset, wr_reg[offset],
 748                                 *buf, mdev_state->s[index].dlab);
 749 #endif
 750                         handle_bar_write(index, mdev_state, offset, buf, count);
 751                 } else {
 752                         handle_bar_read(index, mdev_state, offset, buf, count);
 753                         dump_buffer(buf, count);
 754
 755 #if defined(DEBUG_REGS)
 756                         pr_info("%s: BAR%d  RD @0x%llx %s val:0x%02x dlab:%d\n",
 757                                 __func__, index, offset, rd_reg[offset],
 758                                 *buf, mdev_state->s[index].dlab);
 759 #endif
 760                 }
 761                 break;
 762
 763         default:
 764                 ret = -1;
 765                 goto accessfailed;
 766         }
 767
 768         ret = count;
 769
 770
 771 accessfailed:
 772         mutex_unlock(&mdev_state->ops_lock);
 773
 774         return ret;
 775 }
 776
 777 static size_t mtty_data_size(struct mdev_state *mdev_state)
 778 {
 779         return offsetof(struct mtty_data, ports) +
 780                 (mdev_state->nr_ports * sizeof(struct serial_port));
 781 }
 782
 783 static void mtty_disable_file(struct mtty_migration_file *migf)
 784 {
 785         mutex_lock(&migf->lock);
 786         migf->disabled = true;
 787         migf->filled_size = 0;
 788         migf->filp->f_pos = 0;
 789         mutex_unlock(&migf->lock);
 790 }
 791
 792 static void mtty_disable_files(struct mdev_state *mdev_state)
 793 {
 794         if (mdev_state->saving_migf) {
 795                 mtty_disable_file(mdev_state->saving_migf);
 796                 fput(mdev_state->saving_migf->filp);
 797                 mdev_state->saving_migf = NULL;
 798         }
 799
 800         if (mdev_state->resuming_migf) {
 801                 mtty_disable_file(mdev_state->resuming_migf);
 802                 fput(mdev_state->resuming_migf->filp);
 803                 mdev_state->resuming_migf = NULL;
 804         }
 805 }
 806
 807 static void mtty_state_mutex_unlock(struct mdev_state *mdev_state)
 808 {
 809 again:
 810         mutex_lock(&mdev_state->reset_mutex);
 811         if (mdev_state->deferred_reset) {
 812                 mdev_state->deferred_reset = false;
 813                 mutex_unlock(&mdev_state->reset_mutex);
 814                 mdev_state->state = VFIO_DEVICE_STATE_RUNNING;
 815                 mtty_disable_files(mdev_state);
 816                 goto again;
 817         }
 818         mutex_unlock(&mdev_state->state_mutex);
 819         mutex_unlock(&mdev_state->reset_mutex);
 820 }
 821
 822 static int mtty_release_migf(struct inode *inode, struct file *filp)
 823 {
 824         struct mtty_migration_file *migf = filp->private_data;
 825
 826         mtty_disable_file(migf);
 827         mutex_destroy(&migf->lock);
 828         kfree(migf);
 829
 830         return 0;
 831 }
 832
 833 static long mtty_precopy_ioctl(struct file *filp, unsigned int cmd,
 834                                unsigned long arg)
 835 {
 836         struct mtty_migration_file *migf = filp->private_data;
 837         struct mdev_state *mdev_state = migf->mdev_state;
 838         loff_t *pos = &filp->f_pos;
 839         struct vfio_precopy_info info = {};
 840         unsigned long minsz;
 841         int ret;
 842
 843         if (cmd != VFIO_MIG_GET_PRECOPY_INFO)
 844                 return -ENOTTY;
 845
 846         minsz = offsetofend(struct vfio_precopy_info, dirty_bytes);
 847
 848         if (copy_from_user(&info, (void __user *)arg, minsz))
 849                 return -EFAULT;
 850         if (info.argsz < minsz)
 851                 return -EINVAL;
 852
 853         mutex_lock(&mdev_state->state_mutex);
 854         if (mdev_state->state != VFIO_DEVICE_STATE_PRE_COPY &&
 855             mdev_state->state != VFIO_DEVICE_STATE_PRE_COPY_P2P) {
 856                 ret = -EINVAL;
 857                 goto unlock;
 858         }
 859
 860         mutex_lock(&migf->lock);
 861
 862         if (migf->disabled) {
 863                 mutex_unlock(&migf->lock);
 864                 ret = -ENODEV;
 865                 goto unlock;
 866         }
 867
 868         if (*pos > migf->filled_size) {
 869                 mutex_unlock(&migf->lock);
 870                 ret = -EINVAL;
 871                 goto unlock;
 872         }
 873
 874         info.dirty_bytes = 0;
 875         info.initial_bytes = migf->filled_size - *pos;
 876         mutex_unlock(&migf->lock);
 877
 878         ret = copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0;
 879 unlock:
 880         mtty_state_mutex_unlock(mdev_state);
 881         return ret;
 882 }
 883
 884 static ssize_t mtty_save_read(struct file *filp, char __user *buf,
 885                               size_t len, loff_t *pos)
 886 {
 887         struct mtty_migration_file *migf = filp->private_data;
 888         ssize_t ret = 0;
 889
 890         if (pos)
 891                 return -ESPIPE;
 892
 893         pos = &filp->f_pos;
 894
 895         mutex_lock(&migf->lock);
 896
 897         dev_dbg(migf->mdev_state->vdev.dev, "%s ask %zu\n", __func__, len);
 898
 899         if (migf->disabled) {
 900                 ret = -ENODEV;
 901                 goto out_unlock;
 902         }
 903
 904         if (*pos > migf->filled_size) {
 905                 ret = -EINVAL;
 906                 goto out_unlock;
 907         }
 908
 909         len = min_t(size_t, migf->filled_size - *pos, len);
 910         if (len) {
 911                 if (copy_to_user(buf, (void *)&migf->data + *pos, len)) {
 912                         ret = -EFAULT;
 913                         goto out_unlock;
 914                 }
 915                 *pos += len;
 916                 ret = len;
 917         }
 918 out_unlock:
 919         dev_dbg(migf->mdev_state->vdev.dev, "%s read %zu\n", __func__, ret);
 920         mutex_unlock(&migf->lock);
 921         return ret;
 922 }
 923
 924 static const struct file_operations mtty_save_fops = {
 925         .owner = THIS_MODULE,
 926         .read = mtty_save_read,
 927         .unlocked_ioctl = mtty_precopy_ioctl,
 928         .compat_ioctl = compat_ptr_ioctl,
 929         .release = mtty_release_migf,
 930         .llseek = no_llseek,
 931 };
 932
 933 static void mtty_save_state(struct mdev_state *mdev_state)
 934 {
 935         struct mtty_migration_file *migf = mdev_state->saving_migf;
 936         int i;
 937
 938         mutex_lock(&migf->lock);
 939         for (i = 0; i < mdev_state->nr_ports; i++) {
 940                 memcpy(&migf->data.ports[i],
 941                         &mdev_state->s[i], sizeof(struct serial_port));
 942                 migf->filled_size += sizeof(struct serial_port);
 943         }
 944         dev_dbg(mdev_state->vdev.dev,
 945                 "%s filled to %zu\n", __func__, migf->filled_size);
 946         mutex_unlock(&migf->lock);
 947 }
 948
 949 static int mtty_load_state(struct mdev_state *mdev_state)
 950 {
 951         struct mtty_migration_file *migf = mdev_state->resuming_migf;
 952         int i;
 953
 954         mutex_lock(&migf->lock);
 955         /* magic and version already tested by resume write fn */
 956         if (migf->filled_size < mtty_data_size(mdev_state)) {
 957                 dev_dbg(mdev_state->vdev.dev, "%s expected %zu, got %zu\n",
 958                         __func__, mtty_data_size(mdev_state),
 959                         migf->filled_size);
 960                 mutex_unlock(&migf->lock);
 961                 return -EINVAL;
 962         }
 963
 964         for (i = 0; i < mdev_state->nr_ports; i++)
 965                 memcpy(&mdev_state->s[i],
 966                        &migf->data.ports[i], sizeof(struct serial_port));
 967
 968         mutex_unlock(&migf->lock);
 969         return 0;
 970 }
 971
 972 static struct mtty_migration_file *
 973 mtty_save_device_data(struct mdev_state *mdev_state,
 974                       enum vfio_device_mig_state state)
 975 {
 976         struct mtty_migration_file *migf = mdev_state->saving_migf;
 977         struct mtty_migration_file *ret = NULL;
 978
 979         if (migf) {
 980                 if (state == VFIO_DEVICE_STATE_STOP_COPY)
 981                         goto fill_data;
 982                 return ret;
 983         }
 984
 985         migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
 986         if (!migf)
 987                 return ERR_PTR(-ENOMEM);
 988
 989         migf->filp = anon_inode_getfile("mtty_mig", &mtty_save_fops,
 990                                         migf, O_RDONLY);
 991         if (IS_ERR(migf->filp)) {
 992                 int rc = PTR_ERR(migf->filp);
 993
 994                 kfree(migf);
 995                 return ERR_PTR(rc);
 996         }
 997
 998         stream_open(migf->filp->f_inode, migf->filp);
 999         mutex_init(&migf->lock);
1000         migf->mdev_state = mdev_state;
1001
1002         migf->data.magic = MTTY_MAGIC;
1003         migf->data.major_ver = MTTY_MAJOR_VER;
1004         migf->data.minor_ver = MTTY_MINOR_VER;
1005         migf->data.nr_ports = mdev_state->nr_ports;
1006
1007         migf->filled_size = offsetof(struct mtty_data, ports);
1008
1009         dev_dbg(mdev_state->vdev.dev, "%s filled header to %zu\n",
1010                 __func__, migf->filled_size);
1011
1012         ret = mdev_state->saving_migf = migf;
1013
1014 fill_data:
1015         if (state == VFIO_DEVICE_STATE_STOP_COPY)
1016                 mtty_save_state(mdev_state);
1017
1018         return ret;
1019 }
1020
1021 static ssize_t mtty_resume_write(struct file *filp, const char __user *buf,
1022                                  size_t len, loff_t *pos)
1023 {
1024         struct mtty_migration_file *migf = filp->private_data;
1025         struct mdev_state *mdev_state = migf->mdev_state;
1026         loff_t requested_length;
1027         ssize_t ret = 0;
1028
1029         if (pos)
1030                 return -ESPIPE;
1031
1032         pos = &filp->f_pos;
1033
1034         if (*pos < 0 ||
1035             check_add_overflow((loff_t)len, *pos, &requested_length))
1036                 return -EINVAL;
1037
1038         if (requested_length > mtty_data_size(mdev_state))
1039                 return -ENOMEM;
1040
1041         mutex_lock(&migf->lock);
1042
1043         if (migf->disabled) {
1044                 ret = -ENODEV;
1045                 goto out_unlock;
1046         }
1047
1048         if (copy_from_user((void *)&migf->data + *pos, buf, len)) {
1049                 ret = -EFAULT;
1050                 goto out_unlock;
1051         }
1052
1053         *pos += len;
1054         ret = len;
1055
1056         dev_dbg(migf->mdev_state->vdev.dev, "%s received %zu, total %zu\n",
1057                 __func__, len, migf->filled_size + len);
1058
1059         if (migf->filled_size < offsetof(struct mtty_data, ports) &&
1060             migf->filled_size + len >= offsetof(struct mtty_data, ports)) {
1061                 if (migf->data.magic != MTTY_MAGIC || migf->data.flags ||
1062                     migf->data.major_ver != MTTY_MAJOR_VER ||
1063                     migf->data.minor_ver != MTTY_MINOR_VER ||
1064                     migf->data.nr_ports != mdev_state->nr_ports) {
1065                         dev_dbg(migf->mdev_state->vdev.dev,
1066                                 "%s failed validation\n", __func__);
1067                         ret = -EFAULT;
1068                 } else {
1069                         dev_dbg(migf->mdev_state->vdev.dev,
1070                                 "%s header validated\n", __func__);
1071                 }
1072         }
1073
1074         migf->filled_size += len;
1075
1076 out_unlock:
1077         mutex_unlock(&migf->lock);
1078         return ret;
1079 }
1080
1081 static const struct file_operations mtty_resume_fops = {
1082         .owner = THIS_MODULE,
1083         .write = mtty_resume_write,
1084         .release = mtty_release_migf,
1085         .llseek = no_llseek,
1086 };
1087
1088 static struct mtty_migration_file *
1089 mtty_resume_device_data(struct mdev_state *mdev_state)
1090 {
1091         struct mtty_migration_file *migf;
1092         int ret;
1093
1094         migf = kzalloc(sizeof(*migf), GFP_KERNEL_ACCOUNT);
1095         if (!migf)
1096                 return ERR_PTR(-ENOMEM);
1097
1098         migf->filp = anon_inode_getfile("mtty_mig", &mtty_resume_fops,
1099                                         migf, O_WRONLY);
1100         if (IS_ERR(migf->filp)) {
1101                 ret = PTR_ERR(migf->filp);
1102                 kfree(migf);
1103                 return ERR_PTR(ret);
1104         }
1105
1106         stream_open(migf->filp->f_inode, migf->filp);
1107         mutex_init(&migf->lock);
1108         migf->mdev_state = mdev_state;
1109
1110         mdev_state->resuming_migf = migf;
1111
1112         return migf;
1113 }
1114
1115 static struct file *mtty_step_state(struct mdev_state *mdev_state,
1116                                      enum vfio_device_mig_state new)
1117 {
1118         enum vfio_device_mig_state cur = mdev_state->state;
1119
1120         dev_dbg(mdev_state->vdev.dev, "%s: %d -> %d\n", __func__, cur, new);
1121
1122         /*
1123          * The following state transitions are no-op considering
1124          * mtty does not do DMA nor require any explicit start/stop.
1125          *
1126          *         RUNNING -> RUNNING_P2P
1127          *         RUNNING_P2P -> RUNNING
1128          *         RUNNING_P2P -> STOP
1129          *         PRE_COPY -> PRE_COPY_P2P
1130          *         PRE_COPY_P2P -> PRE_COPY
1131          *         STOP -> RUNNING_P2P
1132          */
1133         if ((cur == VFIO_DEVICE_STATE_RUNNING &&
1134              new == VFIO_DEVICE_STATE_RUNNING_P2P) ||
1135             (cur == VFIO_DEVICE_STATE_RUNNING_P2P &&
1136              (new == VFIO_DEVICE_STATE_RUNNING ||
1137               new == VFIO_DEVICE_STATE_STOP)) ||
1138             (cur == VFIO_DEVICE_STATE_PRE_COPY &&
1139              new == VFIO_DEVICE_STATE_PRE_COPY_P2P) ||
1140             (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P &&
1141              new == VFIO_DEVICE_STATE_PRE_COPY) ||
1142             (cur == VFIO_DEVICE_STATE_STOP &&
1143              new == VFIO_DEVICE_STATE_RUNNING_P2P))
1144                 return NULL;
1145
1146         /*
1147          * The following state transitions simply close migration files,
1148          * with the exception of RESUMING -> STOP, which needs to load
1149          * the state first.
1150          *
1151          *         RESUMING -> STOP
1152          *         PRE_COPY -> RUNNING
1153          *         PRE_COPY_P2P -> RUNNING_P2P
1154          *         STOP_COPY -> STOP
1155          */
1156         if (cur == VFIO_DEVICE_STATE_RESUMING &&
1157             new == VFIO_DEVICE_STATE_STOP) {
1158                 int ret;
1159
1160                 ret = mtty_load_state(mdev_state);
1161                 if (ret)
1162                         return ERR_PTR(ret);
1163                 mtty_disable_files(mdev_state);
1164                 return NULL;
1165         }
1166
1167         if ((cur == VFIO_DEVICE_STATE_PRE_COPY &&
1168              new == VFIO_DEVICE_STATE_RUNNING) ||
1169             (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P &&
1170              new == VFIO_DEVICE_STATE_RUNNING_P2P) ||
1171             (cur == VFIO_DEVICE_STATE_STOP_COPY &&
1172              new == VFIO_DEVICE_STATE_STOP)) {
1173                 mtty_disable_files(mdev_state);
1174                 return NULL;
1175         }
1176
1177         /*
1178          * The following state transitions return migration files.
1179          *
1180          *         RUNNING -> PRE_COPY
1181          *         RUNNING_P2P -> PRE_COPY_P2P
1182          *         STOP -> STOP_COPY
1183          *         STOP -> RESUMING
1184          *         PRE_COPY_P2P -> STOP_COPY
1185          */
1186         if ((cur == VFIO_DEVICE_STATE_RUNNING &&
1187              new == VFIO_DEVICE_STATE_PRE_COPY) ||
1188             (cur == VFIO_DEVICE_STATE_RUNNING_P2P &&
1189              new == VFIO_DEVICE_STATE_PRE_COPY_P2P) ||
1190             (cur == VFIO_DEVICE_STATE_STOP &&
1191              new == VFIO_DEVICE_STATE_STOP_COPY) ||
1192             (cur == VFIO_DEVICE_STATE_PRE_COPY_P2P &&
1193              new == VFIO_DEVICE_STATE_STOP_COPY)) {
1194                 struct mtty_migration_file *migf;
1195
1196                 migf = mtty_save_device_data(mdev_state, new);
1197                 if (IS_ERR(migf))
1198                         return ERR_CAST(migf);
1199
1200                 if (migf) {
1201                         get_file(migf->filp);
1202
1203                         return migf->filp;
1204                 }
1205                 return NULL;
1206         }
1207
1208         if (cur == VFIO_DEVICE_STATE_STOP &&
1209             new == VFIO_DEVICE_STATE_RESUMING) {
1210                 struct mtty_migration_file *migf;
1211
1212                 migf = mtty_resume_device_data(mdev_state);
1213                 if (IS_ERR(migf))
1214                         return ERR_CAST(migf);
1215
1216                 get_file(migf->filp);
1217
1218                 return migf->filp;
1219         }
1220
1221         /* vfio_mig_get_next_state() does not use arcs other than the above */
1222         WARN_ON(true);
1223         return ERR_PTR(-EINVAL);
1224 }
1225
1226 static struct file *mtty_set_state(struct vfio_device *vdev,
1227                                    enum vfio_device_mig_state new_state)
1228 {
1229         struct mdev_state *mdev_state =
1230                 container_of(vdev, struct mdev_state, vdev);
1231         struct file *ret = NULL;
1232
1233         dev_dbg(vdev->dev, "%s -> %d\n", __func__, new_state);
1234
1235         mutex_lock(&mdev_state->state_mutex);
1236         while (mdev_state->state != new_state) {
1237                 enum vfio_device_mig_state next_state;
1238                 int rc = vfio_mig_get_next_state(vdev, mdev_state->state,
1239                                                  new_state, &next_state);
1240                 if (rc) {
1241                         ret = ERR_PTR(rc);
1242                         break;
1243                 }
1244
1245                 ret = mtty_step_state(mdev_state, next_state);
1246                 if (IS_ERR(ret))
1247                         break;
1248
1249                 mdev_state->state = next_state;
1250
1251                 if (WARN_ON(ret && new_state != next_state)) {
1252                         fput(ret);
1253                         ret = ERR_PTR(-EINVAL);
1254                         break;
1255                 }
1256         }
1257         mtty_state_mutex_unlock(mdev_state);
1258         return ret;
1259 }
1260
1261 static int mtty_get_state(struct vfio_device *vdev,
1262                           enum vfio_device_mig_state *current_state)
1263 {
1264         struct mdev_state *mdev_state =
1265                 container_of(vdev, struct mdev_state, vdev);
1266
1267         mutex_lock(&mdev_state->state_mutex);
1268         *current_state = mdev_state->state;
1269         mtty_state_mutex_unlock(mdev_state);
1270         return 0;
1271 }
1272
1273 static int mtty_get_data_size(struct vfio_device *vdev,
1274                               unsigned long *stop_copy_length)
1275 {
1276         struct mdev_state *mdev_state =
1277                 container_of(vdev, struct mdev_state, vdev);
1278
1279         *stop_copy_length = mtty_data_size(mdev_state);
1280         return 0;
1281 }
1282
1283 static const struct vfio_migration_ops mtty_migration_ops = {
1284         .migration_set_state = mtty_set_state,
1285         .migration_get_state = mtty_get_state,
1286         .migration_get_data_size = mtty_get_data_size,
1287 };
1288
1289 static int mtty_log_start(struct vfio_device *vdev,
1290                           struct rb_root_cached *ranges,
1291                           u32 nnodes, u64 *page_size)
1292 {
1293         return 0;
1294 }
1295
1296 static int mtty_log_stop(struct vfio_device *vdev)
1297 {
1298         return 0;
1299 }
1300
1301 static int mtty_log_read_and_clear(struct vfio_device *vdev,
1302                                    unsigned long iova, unsigned long length,
1303                                    struct iova_bitmap *dirty)
1304 {
1305         return 0;
1306 }
1307
1308 static const struct vfio_log_ops mtty_log_ops = {
1309         .log_start = mtty_log_start,
1310         .log_stop = mtty_log_stop,
1311         .log_read_and_clear = mtty_log_read_and_clear,
1312 };
1313
1314 static int mtty_init_dev(struct vfio_device *vdev)
1315 {
1316         struct mdev_state *mdev_state =
1317                 container_of(vdev, struct mdev_state, vdev);
1318         struct mdev_device *mdev = to_mdev_device(vdev->dev);
1319         struct mtty_type *type =
1320                 container_of(mdev->type, struct mtty_type, type);
1321         int avail_ports = atomic_read(&mdev_avail_ports);
1322         int ret;
1323
1324         do {
1325                 if (avail_ports < type->nr_ports)
1326                         return -ENOSPC;
1327         } while (!atomic_try_cmpxchg(&mdev_avail_ports,
1328                                      &avail_ports,
1329                                      avail_ports - type->nr_ports));
1330
1331         mdev_state->nr_ports = type->nr_ports;
1332         mdev_state->irq_index = -1;
1333         mdev_state->s[0].max_fifo_size = MAX_FIFO_SIZE;
1334         mdev_state->s[1].max_fifo_size = MAX_FIFO_SIZE;
1335         mutex_init(&mdev_state->rxtx_lock);
1336
1337         mdev_state->vconfig = kzalloc(MTTY_CONFIG_SPACE_SIZE, GFP_KERNEL);
1338         if (!mdev_state->vconfig) {
1339                 ret = -ENOMEM;
1340                 goto err_nr_ports;
1341         }
1342
1343         mutex_init(&mdev_state->ops_lock);
1344         mdev_state->mdev = mdev;
1345         mtty_create_config_space(mdev_state);
1346
1347         mutex_init(&mdev_state->state_mutex);
1348         mutex_init(&mdev_state->reset_mutex);
1349         vdev->migration_flags = VFIO_MIGRATION_STOP_COPY |
1350                                 VFIO_MIGRATION_P2P |
1351                                 VFIO_MIGRATION_PRE_COPY;
1352         vdev->mig_ops = &mtty_migration_ops;
1353         vdev->log_ops = &mtty_log_ops;
1354         mdev_state->state = VFIO_DEVICE_STATE_RUNNING;
1355
1356         return 0;
1357
1358 err_nr_ports:
1359         atomic_add(type->nr_ports, &mdev_avail_ports);
1360         return ret;
1361 }
1362
1363 static int mtty_probe(struct mdev_device *mdev)
1364 {
1365         struct mdev_state *mdev_state;
1366         int ret;
1367
1368         mdev_state = vfio_alloc_device(mdev_state, vdev, &mdev->dev,
1369                                        &mtty_dev_ops);
1370         if (IS_ERR(mdev_state))
1371                 return PTR_ERR(mdev_state);
1372
1373         ret = vfio_register_emulated_iommu_dev(&mdev_state->vdev);
1374         if (ret)
1375                 goto err_put_vdev;
1376         dev_set_drvdata(&mdev->dev, mdev_state);
1377         return 0;
1378
1379 err_put_vdev:
1380         vfio_put_device(&mdev_state->vdev);
1381         return ret;
1382 }
1383
1384 static void mtty_release_dev(struct vfio_device *vdev)
1385 {
1386         struct mdev_state *mdev_state =
1387                 container_of(vdev, struct mdev_state, vdev);
1388
1389         mutex_destroy(&mdev_state->reset_mutex);
1390         mutex_destroy(&mdev_state->state_mutex);
1391         atomic_add(mdev_state->nr_ports, &mdev_avail_ports);
1392         kfree(mdev_state->vconfig);
1393 }
1394
1395 static void mtty_remove(struct mdev_device *mdev)
1396 {
1397         struct mdev_state *mdev_state = dev_get_drvdata(&mdev->dev);
1398
1399         vfio_unregister_group_dev(&mdev_state->vdev);
1400         vfio_put_device(&mdev_state->vdev);
1401 }
1402
1403 static int mtty_reset(struct mdev_state *mdev_state)
1404 {
1405         pr_info("%s: called\n", __func__);
1406
1407         mutex_lock(&mdev_state->reset_mutex);
1408         mdev_state->deferred_reset = true;
1409         if (!mutex_trylock(&mdev_state->state_mutex)) {
1410                 mutex_unlock(&mdev_state->reset_mutex);
1411                 return 0;
1412         }
1413         mutex_unlock(&mdev_state->reset_mutex);
1414         mtty_state_mutex_unlock(mdev_state);
1415
1416         return 0;
1417 }
1418
1419 static ssize_t mtty_read(struct vfio_device *vdev, char __user *buf,
1420                          size_t count, loff_t *ppos)
1421 {
1422         struct mdev_state *mdev_state =
1423                 container_of(vdev, struct mdev_state, vdev);
1424         unsigned int done = 0;
1425         int ret;
1426
1427         while (count) {
1428                 size_t filled;
1429
1430                 if (count >= 4 && !(*ppos % 4)) {
1431                         u32 val;
1432
1433                         ret =  mdev_access(mdev_state, (u8 *)&val, sizeof(val),
1434                                            *ppos, false);
1435                         if (ret <= 0)
1436                                 goto read_err;
1437
1438                         if (copy_to_user(buf, &val, sizeof(val)))
1439                                 goto read_err;
1440
1441                         filled = 4;
1442                 } else if (count >= 2 && !(*ppos % 2)) {
1443                         u16 val;
1444
1445                         ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val),
1446                                           *ppos, false);
1447                         if (ret <= 0)
1448                                 goto read_err;
1449
1450                         if (copy_to_user(buf, &val, sizeof(val)))
1451                                 goto read_err;
1452
1453                         filled = 2;
1454                 } else {
1455                         u8 val;
1456
1457                         ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val),
1458                                           *ppos, false);
1459                         if (ret <= 0)
1460                                 goto read_err;
1461
1462                         if (copy_to_user(buf, &val, sizeof(val)))
1463                                 goto read_err;
1464
1465                         filled = 1;
1466                 }
1467
1468                 count -= filled;
1469                 done += filled;
1470                 *ppos += filled;
1471                 buf += filled;
1472         }
1473
1474         return done;
1475
1476 read_err:
1477         return -EFAULT;
1478 }
1479
1480 static ssize_t mtty_write(struct vfio_device *vdev, const char __user *buf,
1481                    size_t count, loff_t *ppos)
1482 {
1483         struct mdev_state *mdev_state =
1484                 container_of(vdev, struct mdev_state, vdev);
1485         unsigned int done = 0;
1486         int ret;
1487
1488         while (count) {
1489                 size_t filled;
1490
1491                 if (count >= 4 && !(*ppos % 4)) {
1492                         u32 val;
1493
1494                         if (copy_from_user(&val, buf, sizeof(val)))
1495                                 goto write_err;
1496
1497                         ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val),
1498                                           *ppos, true);
1499                         if (ret <= 0)
1500                                 goto write_err;
1501
1502                         filled = 4;
1503                 } else if (count >= 2 && !(*ppos % 2)) {
1504                         u16 val;
1505
1506                         if (copy_from_user(&val, buf, sizeof(val)))
1507                                 goto write_err;
1508
1509                         ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val),
1510                                           *ppos, true);
1511                         if (ret <= 0)
1512                                 goto write_err;
1513
1514                         filled = 2;
1515                 } else {
1516                         u8 val;
1517
1518                         if (copy_from_user(&val, buf, sizeof(val)))
1519                                 goto write_err;
1520
1521                         ret = mdev_access(mdev_state, (u8 *)&val, sizeof(val),
1522                                           *ppos, true);
1523                         if (ret <= 0)
1524                                 goto write_err;
1525
1526                         filled = 1;
1527                 }
1528                 count -= filled;
1529                 done += filled;
1530                 *ppos += filled;
1531                 buf += filled;
1532         }
1533
1534         return done;
1535 write_err:
1536         return -EFAULT;
1537 }
1538
1539 static void mtty_disable_intx(struct mdev_state *mdev_state)
1540 {
1541         if (mdev_state->intx_evtfd) {
1542                 eventfd_ctx_put(mdev_state->intx_evtfd);
1543                 mdev_state->intx_evtfd = NULL;
1544                 mdev_state->intx_mask = false;
1545                 mdev_state->irq_index = -1;
1546         }
1547 }
1548
1549 static void mtty_disable_msi(struct mdev_state *mdev_state)
1550 {
1551         if (mdev_state->msi_evtfd) {
1552                 eventfd_ctx_put(mdev_state->msi_evtfd);
1553                 mdev_state->msi_evtfd = NULL;
1554                 mdev_state->irq_index = -1;
1555         }
1556 }
1557
1558 static int mtty_set_irqs(struct mdev_state *mdev_state, uint32_t flags,
1559                          unsigned int index, unsigned int start,
1560                          unsigned int count, void *data)
1561 {
1562         int ret = 0;
1563
1564         mutex_lock(&mdev_state->ops_lock);
1565         switch (index) {
1566         case VFIO_PCI_INTX_IRQ_INDEX:
1567                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1568                 case VFIO_IRQ_SET_ACTION_MASK:
1569                         if (!is_intx(mdev_state) || start != 0 || count != 1) {
1570                                 ret = -EINVAL;
1571                                 break;
1572                         }
1573
1574                         if (flags & VFIO_IRQ_SET_DATA_NONE) {
1575                                 mdev_state->intx_mask = true;
1576                         } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
1577                                 uint8_t mask = *(uint8_t *)data;
1578
1579                                 if (mask)
1580                                         mdev_state->intx_mask = true;
1581                         } else if (flags &  VFIO_IRQ_SET_DATA_EVENTFD) {
1582                                 ret = -ENOTTY; /* No support for mask fd */
1583                         }
1584                         break;
1585                 case VFIO_IRQ_SET_ACTION_UNMASK:
1586                         if (!is_intx(mdev_state) || start != 0 || count != 1) {
1587                                 ret = -EINVAL;
1588                                 break;
1589                         }
1590
1591                         if (flags & VFIO_IRQ_SET_DATA_NONE) {
1592                                 mdev_state->intx_mask = false;
1593                         } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
1594                                 uint8_t mask = *(uint8_t *)data;
1595
1596                                 if (mask)
1597                                         mdev_state->intx_mask = false;
1598                         } else if (flags &  VFIO_IRQ_SET_DATA_EVENTFD) {
1599                                 ret = -ENOTTY; /* No support for unmask fd */
1600                         }
1601                         break;
1602                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1603                         if (is_intx(mdev_state) && !count &&
1604                             (flags & VFIO_IRQ_SET_DATA_NONE)) {
1605                                 mtty_disable_intx(mdev_state);
1606                                 break;
1607                         }
1608
1609                         if (!(is_intx(mdev_state) || is_noirq(mdev_state)) ||
1610                             start != 0 || count != 1) {
1611                                 ret = -EINVAL;
1612                                 break;
1613                         }
1614
1615                         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1616                                 int fd = *(int *)data;
1617                                 struct eventfd_ctx *evt;
1618
1619                                 mtty_disable_intx(mdev_state);
1620
1621                                 if (fd < 0)
1622                                         break;
1623
1624                                 evt = eventfd_ctx_fdget(fd);
1625                                 if (IS_ERR(evt)) {
1626                                         ret = PTR_ERR(evt);
1627                                         break;
1628                                 }
1629                                 mdev_state->intx_evtfd = evt;
1630                                 mdev_state->irq_index = index;
1631                                 break;
1632                         }
1633
1634                         if (!is_intx(mdev_state)) {
1635                                 ret = -EINVAL;
1636                                 break;
1637                         }
1638
1639                         if (flags & VFIO_IRQ_SET_DATA_NONE) {
1640                                 mtty_trigger_interrupt(mdev_state);
1641                         } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
1642                                 uint8_t trigger = *(uint8_t *)data;
1643
1644                                 if (trigger)
1645                                         mtty_trigger_interrupt(mdev_state);
1646                         }
1647                         break;
1648                 }
1649                 break;
1650         case VFIO_PCI_MSI_IRQ_INDEX:
1651                 switch (flags & VFIO_IRQ_SET_ACTION_TYPE_MASK) {
1652                 case VFIO_IRQ_SET_ACTION_MASK:
1653                 case VFIO_IRQ_SET_ACTION_UNMASK:
1654                         ret = -ENOTTY;
1655                         break;
1656                 case VFIO_IRQ_SET_ACTION_TRIGGER:
1657                         if (is_msi(mdev_state) && !count &&
1658                             (flags & VFIO_IRQ_SET_DATA_NONE)) {
1659                                 mtty_disable_msi(mdev_state);
1660                                 break;
1661                         }
1662
1663                         if (!(is_msi(mdev_state) || is_noirq(mdev_state)) ||
1664                             start != 0 || count != 1) {
1665                                 ret = -EINVAL;
1666                                 break;
1667                         }
1668
1669                         if (flags & VFIO_IRQ_SET_DATA_EVENTFD) {
1670                                 int fd = *(int *)data;
1671                                 struct eventfd_ctx *evt;
1672
1673                                 mtty_disable_msi(mdev_state);
1674
1675                                 if (fd < 0)
1676                                         break;
1677
1678                                 evt = eventfd_ctx_fdget(fd);
1679                                 if (IS_ERR(evt)) {
1680                                         ret = PTR_ERR(evt);
1681                                         break;
1682                                 }
1683                                 mdev_state->msi_evtfd = evt;
1684                                 mdev_state->irq_index = index;
1685                                 break;
1686                         }
1687
1688                         if (!is_msi(mdev_state)) {
1689                                 ret = -EINVAL;
1690                                 break;
1691                         }
1692
1693                         if (flags & VFIO_IRQ_SET_DATA_NONE) {
1694                                 mtty_trigger_interrupt(mdev_state);
1695                         } else if (flags & VFIO_IRQ_SET_DATA_BOOL) {
1696                                 uint8_t trigger = *(uint8_t *)data;
1697
1698                                 if (trigger)
1699                                         mtty_trigger_interrupt(mdev_state);
1700                         }
1701                         break;
1702                 }
1703                 break;
1704         case VFIO_PCI_MSIX_IRQ_INDEX:
1705                 dev_dbg(mdev_state->vdev.dev, "%s: MSIX_IRQ\n", __func__);
1706                 ret = -ENOTTY;
1707                 break;
1708         case VFIO_PCI_ERR_IRQ_INDEX:
1709                 dev_dbg(mdev_state->vdev.dev, "%s: ERR_IRQ\n", __func__);
1710                 ret = -ENOTTY;
1711                 break;
1712         case VFIO_PCI_REQ_IRQ_INDEX:
1713                 dev_dbg(mdev_state->vdev.dev, "%s: REQ_IRQ\n", __func__);
1714                 ret = -ENOTTY;
1715                 break;
1716         }
1717
1718         mutex_unlock(&mdev_state->ops_lock);
1719         return ret;
1720 }
1721
1722 static int mtty_get_region_info(struct mdev_state *mdev_state,
1723                          struct vfio_region_info *region_info,
1724                          u16 *cap_type_id, void **cap_type)
1725 {
1726         unsigned int size = 0;
1727         u32 bar_index;
1728
1729         bar_index = region_info->index;
1730         if (bar_index >= VFIO_PCI_NUM_REGIONS)
1731                 return -EINVAL;
1732
1733         mutex_lock(&mdev_state->ops_lock);
1734
1735         switch (bar_index) {
1736         case VFIO_PCI_CONFIG_REGION_INDEX:
1737                 size = MTTY_CONFIG_SPACE_SIZE;
1738                 break;
1739         case VFIO_PCI_BAR0_REGION_INDEX:
1740                 size = MTTY_IO_BAR_SIZE;
1741                 break;
1742         case VFIO_PCI_BAR1_REGION_INDEX:
1743                 if (mdev_state->nr_ports == 2)
1744                         size = MTTY_IO_BAR_SIZE;
1745                 break;
1746         default:
1747                 size = 0;
1748                 break;
1749         }
1750
1751         mdev_state->region_info[bar_index].size = size;
1752         mdev_state->region_info[bar_index].vfio_offset =
1753                 MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index);
1754
1755         region_info->size = size;
1756         region_info->offset = MTTY_VFIO_PCI_INDEX_TO_OFFSET(bar_index);
1757         region_info->flags = VFIO_REGION_INFO_FLAG_READ |
1758                 VFIO_REGION_INFO_FLAG_WRITE;
1759         mutex_unlock(&mdev_state->ops_lock);
1760         return 0;
1761 }
1762
1763 static int mtty_get_irq_info(struct vfio_irq_info *irq_info)
1764 {
1765         if (irq_info->index != VFIO_PCI_INTX_IRQ_INDEX &&
1766             irq_info->index != VFIO_PCI_MSI_IRQ_INDEX)
1767                 return -EINVAL;
1768
1769         irq_info->flags = VFIO_IRQ_INFO_EVENTFD;
1770         irq_info->count = 1;
1771
1772         if (irq_info->index == VFIO_PCI_INTX_IRQ_INDEX)
1773                 irq_info->flags |= VFIO_IRQ_INFO_MASKABLE |
1774                                    VFIO_IRQ_INFO_AUTOMASKED;
1775         else
1776                 irq_info->flags |= VFIO_IRQ_INFO_NORESIZE;
1777
1778         return 0;
1779 }
1780
1781 static int mtty_get_device_info(struct vfio_device_info *dev_info)
1782 {
1783         dev_info->flags = VFIO_DEVICE_FLAGS_PCI;
1784         dev_info->num_regions = VFIO_PCI_NUM_REGIONS;
1785         dev_info->num_irqs = VFIO_PCI_NUM_IRQS;
1786
1787         return 0;
1788 }
1789
1790 static long mtty_ioctl(struct vfio_device *vdev, unsigned int cmd,
1791                         unsigned long arg)
1792 {
1793         struct mdev_state *mdev_state =
1794                 container_of(vdev, struct mdev_state, vdev);
1795         int ret = 0;
1796         unsigned long minsz;
1797
1798         switch (cmd) {
1799         case VFIO_DEVICE_GET_INFO:
1800         {
1801                 struct vfio_device_info info;
1802
1803                 minsz = offsetofend(struct vfio_device_info, num_irqs);
1804
1805                 if (copy_from_user(&info, (void __user *)arg, minsz))
1806                         return -EFAULT;
1807
1808                 if (info.argsz < minsz)
1809                         return -EINVAL;
1810
1811                 ret = mtty_get_device_info(&info);
1812                 if (ret)
1813                         return ret;
1814
1815                 memcpy(&mdev_state->dev_info, &info, sizeof(info));
1816
1817                 if (copy_to_user((void __user *)arg, &info, minsz))
1818                         return -EFAULT;
1819
1820                 return 0;
1821         }
1822         case VFIO_DEVICE_GET_REGION_INFO:
1823         {
1824                 struct vfio_region_info info;
1825                 u16 cap_type_id = 0;
1826                 void *cap_type = NULL;
1827
1828                 minsz = offsetofend(struct vfio_region_info, offset);
1829
1830                 if (copy_from_user(&info, (void __user *)arg, minsz))
1831                         return -EFAULT;
1832
1833                 if (info.argsz < minsz)
1834                         return -EINVAL;
1835
1836                 ret = mtty_get_region_info(mdev_state, &info, &cap_type_id,
1837                                            &cap_type);
1838                 if (ret)
1839                         return ret;
1840
1841                 if (copy_to_user((void __user *)arg, &info, minsz))
1842                         return -EFAULT;
1843
1844                 return 0;
1845         }
1846
1847         case VFIO_DEVICE_GET_IRQ_INFO:
1848         {
1849                 struct vfio_irq_info info;
1850
1851                 minsz = offsetofend(struct vfio_irq_info, count);
1852
1853                 if (copy_from_user(&info, (void __user *)arg, minsz))
1854                         return -EFAULT;
1855
1856                 if ((info.argsz < minsz) ||
1857                     (info.index >= mdev_state->dev_info.num_irqs))
1858                         return -EINVAL;
1859
1860                 ret = mtty_get_irq_info(&info);
1861                 if (ret)
1862                         return ret;
1863
1864                 if (copy_to_user((void __user *)arg, &info, minsz))
1865                         return -EFAULT;
1866
1867                 return 0;
1868         }
1869         case VFIO_DEVICE_SET_IRQS:
1870         {
1871                 struct vfio_irq_set hdr;
1872                 u8 *data = NULL, *ptr = NULL;
1873                 size_t data_size = 0;
1874
1875                 minsz = offsetofend(struct vfio_irq_set, count);
1876
1877                 if (copy_from_user(&hdr, (void __user *)arg, minsz))
1878                         return -EFAULT;
1879
1880                 ret = vfio_set_irqs_validate_and_prepare(&hdr,
1881                                                 mdev_state->dev_info.num_irqs,
1882                                                 VFIO_PCI_NUM_IRQS,
1883                                                 &data_size);
1884                 if (ret)
1885                         return ret;
1886
1887                 if (data_size) {
1888                         ptr = data = memdup_user((void __user *)(arg + minsz),
1889                                                  data_size);
1890                         if (IS_ERR(data))
1891                                 return PTR_ERR(data);
1892                 }
1893
1894                 ret = mtty_set_irqs(mdev_state, hdr.flags, hdr.index, hdr.start,
1895                                     hdr.count, data);
1896
1897                 kfree(ptr);
1898                 return ret;
1899         }
1900         case VFIO_DEVICE_RESET:
1901                 return mtty_reset(mdev_state);
1902         }
1903         return -ENOTTY;
1904 }
1905
1906 static ssize_t
1907 sample_mdev_dev_show(struct device *dev, struct device_attribute *attr,
1908                      char *buf)
1909 {
1910         return sprintf(buf, "This is MDEV %s\n", dev_name(dev));
1911 }
1912
1913 static DEVICE_ATTR_RO(sample_mdev_dev);
1914
1915 static struct attribute *mdev_dev_attrs[] = {
1916         &dev_attr_sample_mdev_dev.attr,
1917         NULL,
1918 };
1919
1920 static const struct attribute_group mdev_dev_group = {
1921         .name  = "vendor",
1922         .attrs = mdev_dev_attrs,
1923 };
1924
1925 static const struct attribute_group *mdev_dev_groups[] = {
1926         &mdev_dev_group,
1927         NULL,
1928 };
1929
1930 static unsigned int mtty_get_available(struct mdev_type *mtype)
1931 {
1932         struct mtty_type *type = container_of(mtype, struct mtty_type, type);
1933
1934         return atomic_read(&mdev_avail_ports) / type->nr_ports;
1935 }
1936
1937 static void mtty_close(struct vfio_device *vdev)
1938 {
1939         struct mdev_state *mdev_state =
1940                                 container_of(vdev, struct mdev_state, vdev);
1941
1942         mtty_disable_files(mdev_state);
1943         mtty_disable_intx(mdev_state);
1944         mtty_disable_msi(mdev_state);
1945 }
1946
1947 static const struct vfio_device_ops mtty_dev_ops = {
1948         .name = "vfio-mtty",
1949         .init = mtty_init_dev,
1950         .release = mtty_release_dev,
1951         .read = mtty_read,
1952         .write = mtty_write,
1953         .ioctl = mtty_ioctl,
1954         .bind_iommufd   = vfio_iommufd_emulated_bind,
1955         .unbind_iommufd = vfio_iommufd_emulated_unbind,
1956         .attach_ioas    = vfio_iommufd_emulated_attach_ioas,
1957         .detach_ioas    = vfio_iommufd_emulated_detach_ioas,
1958         .close_device   = mtty_close,
1959 };
1960
1961 static struct mdev_driver mtty_driver = {
1962         .device_api = VFIO_DEVICE_API_PCI_STRING,
1963         .driver = {
1964                 .name = "mtty",
1965                 .owner = THIS_MODULE,
1966                 .mod_name = KBUILD_MODNAME,
1967                 .dev_groups = mdev_dev_groups,
1968         },
1969         .probe = mtty_probe,
1970         .remove = mtty_remove,
1971         .get_available = mtty_get_available,
1972 };
1973
1974 static void mtty_device_release(struct device *dev)
1975 {
1976         dev_dbg(dev, "mtty: released\n");
1977 }
1978
1979 static int __init mtty_dev_init(void)
1980 {
1981         int ret = 0;
1982
1983         pr_info("mtty_dev: %s\n", __func__);
1984
1985         memset(&mtty_dev, 0, sizeof(mtty_dev));
1986
1987         idr_init(&mtty_dev.vd_idr);
1988
1989         ret = alloc_chrdev_region(&mtty_dev.vd_devt, 0, MINORMASK + 1,
1990                                   MTTY_NAME);
1991
1992         if (ret < 0) {
1993                 pr_err("Error: failed to register mtty_dev, err:%d\n", ret);
1994                 return ret;
1995         }
1996
1997         cdev_init(&mtty_dev.vd_cdev, &vd_fops);
1998         cdev_add(&mtty_dev.vd_cdev, mtty_dev.vd_devt, MINORMASK + 1);
1999
2000         pr_info("major_number:%d\n", MAJOR(mtty_dev.vd_devt));
2001
2002         ret = mdev_register_driver(&mtty_driver);
2003         if (ret)
2004                 goto err_cdev;
2005
2006         mtty_dev.vd_class = class_create(MTTY_CLASS_NAME);
2007
2008         if (IS_ERR(mtty_dev.vd_class)) {
2009                 pr_err("Error: failed to register mtty_dev class\n");
2010                 ret = PTR_ERR(mtty_dev.vd_class);
2011                 goto err_driver;
2012         }
2013
2014         mtty_dev.dev.class = mtty_dev.vd_class;
2015         mtty_dev.dev.release = mtty_device_release;
2016         dev_set_name(&mtty_dev.dev, "%s", MTTY_NAME);
2017
2018         ret = device_register(&mtty_dev.dev);
2019         if (ret)
2020                 goto err_put;
2021
2022         ret = mdev_register_parent(&mtty_dev.parent, &mtty_dev.dev,
2023                                    &mtty_driver, mtty_mdev_types,
2024                                    ARRAY_SIZE(mtty_mdev_types));
2025         if (ret)
2026                 goto err_device;
2027         return 0;
2028
2029 err_device:
2030         device_del(&mtty_dev.dev);
2031 err_put:
2032         put_device(&mtty_dev.dev);
2033         class_destroy(mtty_dev.vd_class);
2034 err_driver:
2035         mdev_unregister_driver(&mtty_driver);
2036 err_cdev:
2037         cdev_del(&mtty_dev.vd_cdev);
2038         unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1);
2039         return ret;
2040 }
2041
2042 static void __exit mtty_dev_exit(void)
2043 {
2044         mtty_dev.dev.bus = NULL;
2045         mdev_unregister_parent(&mtty_dev.parent);
2046
2047         device_unregister(&mtty_dev.dev);
2048         idr_destroy(&mtty_dev.vd_idr);
2049         mdev_unregister_driver(&mtty_driver);
2050         cdev_del(&mtty_dev.vd_cdev);
2051         unregister_chrdev_region(mtty_dev.vd_devt, MINORMASK + 1);
2052         class_destroy(mtty_dev.vd_class);
2053         mtty_dev.vd_class = NULL;
2054         pr_info("mtty_dev: Unloaded!\n");
2055 }
2056
2057 module_init(mtty_dev_init)
2058 module_exit(mtty_dev_exit)
2059
2060 MODULE_LICENSE("GPL v2");
2061 MODULE_INFO(supported, "Test driver that simulate serial port over PCI");
2062 MODULE_VERSION(VERSION_STRING);
2063 MODULE_AUTHOR(DRIVER_AUTHOR);