drivers/misc/mic/vop/vop_main.c

   1 /*
   2  * Intel MIC Platform Software Stack (MPSS)
   3  *
   4  * Copyright(c) 2016 Intel Corporation.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License, version 2, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  13  * General Public License for more details.
  14  *
  15  * The full GNU General Public License is included in this distribution in
  16  * the file called "COPYING".
  17  *
  18  * Adapted from:
  19  *
  20  * virtio for kvm on s390
  21  *
  22  * Copyright IBM Corp. 2008
  23  *
  24  * This program is free software; you can redistribute it and/or modify
  25  * it under the terms of the GNU General Public License (version 2 only)
  26  * as published by the Free Software Foundation.
  27  *
  28  *    Author(s): Christian Borntraeger <borntraeger@de.ibm.com>
  29  *
  30  * Intel Virtio Over PCIe (VOP) driver.
  31  *
  32  */
  33 #include <linux/delay.h>
  34 #include <linux/module.h>
  35 #include <linux/sched.h>
  36 #include <linux/dma-mapping.h>
  37
  38 #include "vop_main.h"
  39
  40 #define VOP_MAX_VRINGS 4
  41
  42 /*
  43  * _vop_vdev - Allocated per virtio device instance injected by the peer.
  44  *
  45  * @vdev: Virtio device
  46  * @desc: Virtio device page descriptor
  47  * @dc: Virtio device control
  48  * @vpdev: VOP device which is the parent for this virtio device
  49  * @vr: Buffer for accessing the VRING
  50  * @used: Buffer for used
  51  * @used_size: Size of the used buffer
  52  * @reset_done: Track whether VOP reset is complete
  53  * @virtio_cookie: Cookie returned upon requesting a interrupt
  54  * @c2h_vdev_db: The doorbell used by the guest to interrupt the host
  55  * @h2c_vdev_db: The doorbell used by the host to interrupt the guest
  56  * @dnode: The destination node
  57  */
  58 struct _vop_vdev {
  59         struct virtio_device vdev;
  60         struct mic_device_desc __iomem *desc;
  61         struct mic_device_ctrl __iomem *dc;
  62         struct vop_device *vpdev;
  63         void __iomem *vr[VOP_MAX_VRINGS];
  64         dma_addr_t used[VOP_MAX_VRINGS];
  65         int used_size[VOP_MAX_VRINGS];
  66         struct completion reset_done;
  67         struct mic_irq *virtio_cookie;
  68         int c2h_vdev_db;
  69         int h2c_vdev_db;
  70         int dnode;
  71 };
  72
  73 #define to_vopvdev(vd) container_of(vd, struct _vop_vdev, vdev)
  74
  75 #define _vop_aligned_desc_size(d) __mic_align(_vop_desc_size(d), 8)
  76
  77 /* Helper API to obtain the parent of the virtio device */
  78 static inline struct device *_vop_dev(struct _vop_vdev *vdev)
  79 {
  80         return vdev->vdev.dev.parent;
  81 }
  82
  83 static inline unsigned _vop_desc_size(struct mic_device_desc __iomem *desc)
  84 {
  85         return sizeof(*desc)
  86                 + ioread8(&desc->num_vq) * sizeof(struct mic_vqconfig)
  87                 + ioread8(&desc->feature_len) * 2
  88                 + ioread8(&desc->config_len);
  89 }
  90
  91 static inline struct mic_vqconfig __iomem *
  92 _vop_vq_config(struct mic_device_desc __iomem *desc)
  93 {
  94         return (struct mic_vqconfig __iomem *)(desc + 1);
  95 }
  96
  97 static inline u8 __iomem *
  98 _vop_vq_features(struct mic_device_desc __iomem *desc)
  99 {
 100         return (u8 __iomem *)(_vop_vq_config(desc) + ioread8(&desc->num_vq));
 101 }
 102
 103 static inline u8 __iomem *
 104 _vop_vq_configspace(struct mic_device_desc __iomem *desc)
 105 {
 106         return _vop_vq_features(desc) + ioread8(&desc->feature_len) * 2;
 107 }
 108
 109 static inline unsigned
 110 _vop_total_desc_size(struct mic_device_desc __iomem *desc)
 111 {
 112         return _vop_aligned_desc_size(desc) + sizeof(struct mic_device_ctrl);
 113 }
 114
 115 /* This gets the device's feature bits. */
 116 static u64 vop_get_features(struct virtio_device *vdev)
 117 {
 118         unsigned int i, bits;
 119         u32 features = 0;
 120         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 121         u8 __iomem *in_features = _vop_vq_features(desc);
 122         int feature_len = ioread8(&desc->feature_len);
 123
 124         bits = min_t(unsigned, feature_len, sizeof(vdev->features)) * 8;
 125         for (i = 0; i < bits; i++)
 126                 if (ioread8(&in_features[i / 8]) & (BIT(i % 8)))
 127                         features |= BIT(i);
 128
 129         return features;
 130 }
 131
 132 static void vop_transport_features(struct virtio_device *vdev)
 133 {
 134         /*
 135          * Packed ring isn't enabled on virtio_vop for now,
 136          * because virtio_vop uses vring_new_virtqueue() which
 137          * creates virtio rings on preallocated memory.
 138          */
 139         __virtio_clear_bit(vdev, VIRTIO_F_RING_PACKED);
 140 }
 141
 142 static int vop_finalize_features(struct virtio_device *vdev)
 143 {
 144         unsigned int i, bits;
 145         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 146         u8 feature_len = ioread8(&desc->feature_len);
 147         /* Second half of bitmap is features we accept. */
 148         u8 __iomem *out_features =
 149                 _vop_vq_features(desc) + feature_len;
 150
 151         /* Give virtio_ring a chance to accept features. */
 152         vring_transport_features(vdev);
 153
 154         /* Give virtio_vop a chance to accept features. */
 155         vop_transport_features(vdev);
 156
 157         memset_io(out_features, 0, feature_len);
 158         bits = min_t(unsigned, feature_len,
 159                      sizeof(vdev->features)) * 8;
 160         for (i = 0; i < bits; i++) {
 161                 if (__virtio_test_bit(vdev, i))
 162                         iowrite8(ioread8(&out_features[i / 8]) | (1 << (i % 8)),
 163                                  &out_features[i / 8]);
 164         }
 165         return 0;
 166 }
 167
 168 /*
 169  * Reading and writing elements in config space
 170  */
 171 static void vop_get(struct virtio_device *vdev, unsigned int offset,
 172                     void *buf, unsigned len)
 173 {
 174         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 175
 176         if (offset + len > ioread8(&desc->config_len))
 177                 return;
 178         memcpy_fromio(buf, _vop_vq_configspace(desc) + offset, len);
 179 }
 180
 181 static void vop_set(struct virtio_device *vdev, unsigned int offset,
 182                     const void *buf, unsigned len)
 183 {
 184         struct mic_device_desc __iomem *desc = to_vopvdev(vdev)->desc;
 185
 186         if (offset + len > ioread8(&desc->config_len))
 187                 return;
 188         memcpy_toio(_vop_vq_configspace(desc) + offset, buf, len);
 189 }
 190
 191 /*
 192  * The operations to get and set the status word just access the status
 193  * field of the device descriptor. set_status also interrupts the host
 194  * to tell about status changes.
 195  */
 196 static u8 vop_get_status(struct virtio_device *vdev)
 197 {
 198         return ioread8(&to_vopvdev(vdev)->desc->status);
 199 }
 200
 201 static void vop_set_status(struct virtio_device *dev, u8 status)
 202 {
 203         struct _vop_vdev *vdev = to_vopvdev(dev);
 204         struct vop_device *vpdev = vdev->vpdev;
 205
 206         if (!status)
 207                 return;
 208         iowrite8(status, &vdev->desc->status);
 209         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 210 }
 211
 212 /* Inform host on a virtio device reset and wait for ack from host */
 213 static void vop_reset_inform_host(struct virtio_device *dev)
 214 {
 215         struct _vop_vdev *vdev = to_vopvdev(dev);
 216         struct mic_device_ctrl __iomem *dc = vdev->dc;
 217         struct vop_device *vpdev = vdev->vpdev;
 218         int retry;
 219
 220         iowrite8(0, &dc->host_ack);
 221         iowrite8(1, &dc->vdev_reset);
 222         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 223
 224         /* Wait till host completes all card accesses and acks the reset */
 225         for (retry = 100; retry--;) {
 226                 if (ioread8(&dc->host_ack))
 227                         break;
 228                 msleep(100);
 229         };
 230
 231         dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
 232
 233         /* Reset status to 0 in case we timed out */
 234         iowrite8(0, &vdev->desc->status);
 235 }
 236
 237 static void vop_reset(struct virtio_device *dev)
 238 {
 239         struct _vop_vdev *vdev = to_vopvdev(dev);
 240
 241         dev_dbg(_vop_dev(vdev), "%s: virtio id %d\n",
 242                 __func__, dev->id.device);
 243
 244         vop_reset_inform_host(dev);
 245         complete_all(&vdev->reset_done);
 246 }
 247
 248 /*
 249  * The virtio_ring code calls this API when it wants to notify the Host.
 250  */
 251 static bool vop_notify(struct virtqueue *vq)
 252 {
 253         struct _vop_vdev *vdev = vq->priv;
 254         struct vop_device *vpdev = vdev->vpdev;
 255
 256         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 257         return true;
 258 }
 259
 260 static void vop_del_vq(struct virtqueue *vq, int n)
 261 {
 262         struct _vop_vdev *vdev = to_vopvdev(vq->vdev);
 263         struct vring *vr = (struct vring *)(vq + 1);
 264         struct vop_device *vpdev = vdev->vpdev;
 265
 266         dma_unmap_single(&vpdev->dev, vdev->used[n],
 267                          vdev->used_size[n], DMA_BIDIRECTIONAL);
 268         free_pages((unsigned long)vr->used, get_order(vdev->used_size[n]));
 269         vring_del_virtqueue(vq);
 270         vpdev->hw_ops->iounmap(vpdev, vdev->vr[n]);
 271         vdev->vr[n] = NULL;
 272 }
 273
 274 static void vop_del_vqs(struct virtio_device *dev)
 275 {
 276         struct _vop_vdev *vdev = to_vopvdev(dev);
 277         struct virtqueue *vq, *n;
 278         int idx = 0;
 279
 280         dev_dbg(_vop_dev(vdev), "%s\n", __func__);
 281
 282         list_for_each_entry_safe(vq, n, &dev->vqs, list)
 283                 vop_del_vq(vq, idx++);
 284 }
 285
 286 /*
 287  * This routine will assign vring's allocated in host/io memory. Code in
 288  * virtio_ring.c however continues to access this io memory as if it were local
 289  * memory without io accessors.
 290  */
 291 static struct virtqueue *vop_find_vq(struct virtio_device *dev,
 292                                      unsigned index,
 293                                      void (*callback)(struct virtqueue *vq),
 294                                      const char *name, bool ctx)
 295 {
 296         struct _vop_vdev *vdev = to_vopvdev(dev);
 297         struct vop_device *vpdev = vdev->vpdev;
 298         struct mic_vqconfig __iomem *vqconfig;
 299         struct mic_vqconfig config;
 300         struct virtqueue *vq;
 301         void __iomem *va;
 302         struct _mic_vring_info __iomem *info;
 303         void *used;
 304         int vr_size, _vr_size, err, magic;
 305         struct vring *vr;
 306         u8 type = ioread8(&vdev->desc->type);
 307
 308         if (index >= ioread8(&vdev->desc->num_vq))
 309                 return ERR_PTR(-ENOENT);
 310
 311         if (!name)
 312                 return ERR_PTR(-ENOENT);
 313
 314         /* First assign the vring's allocated in host memory */
 315         vqconfig = _vop_vq_config(vdev->desc) + index;
 316         memcpy_fromio(&config, vqconfig, sizeof(config));
 317         _vr_size = vring_size(le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN);
 318         vr_size = PAGE_ALIGN(_vr_size + sizeof(struct _mic_vring_info));
 319         va = vpdev->hw_ops->ioremap(vpdev, le64_to_cpu(config.address),
 320                         vr_size);
 321         if (!va)
 322                 return ERR_PTR(-ENOMEM);
 323         vdev->vr[index] = va;
 324         memset_io(va, 0x0, _vr_size);
 325         vq = vring_new_virtqueue(
 326                                 index,
 327                                 le16_to_cpu(config.num), MIC_VIRTIO_RING_ALIGN,
 328                                 dev,
 329                                 false,
 330                                 ctx,
 331                                 (void __force *)va, vop_notify, callback, name);
 332         if (!vq) {
 333                 err = -ENOMEM;
 334                 goto unmap;
 335         }
 336         info = va + _vr_size;
 337         magic = ioread32(&info->magic);
 338
 339         if (WARN(magic != MIC_MAGIC + type + index, "magic mismatch")) {
 340                 err = -EIO;
 341                 goto unmap;
 342         }
 343
 344         /* Allocate and reassign used ring now */
 345         vdev->used_size[index] = PAGE_ALIGN(sizeof(__u16) * 3 +
 346                                              sizeof(struct vring_used_elem) *
 347                                              le16_to_cpu(config.num));
 348         used = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
 349                                         get_order(vdev->used_size[index]));
 350         if (!used) {
 351                 err = -ENOMEM;
 352                 dev_err(_vop_dev(vdev), "%s %d err %d\n",
 353                         __func__, __LINE__, err);
 354                 goto del_vq;
 355         }
 356         vdev->used[index] = dma_map_single(&vpdev->dev, used,
 357                                             vdev->used_size[index],
 358                                             DMA_BIDIRECTIONAL);
 359         if (dma_mapping_error(&vpdev->dev, vdev->used[index])) {
 360                 err = -ENOMEM;
 361                 dev_err(_vop_dev(vdev), "%s %d err %d\n",
 362                         __func__, __LINE__, err);
 363                 goto free_used;
 364         }
 365         writeq(vdev->used[index], &vqconfig->used_address);
 366         /*
 367          * To reassign the used ring here we are directly accessing
 368          * struct vring_virtqueue which is a private data structure
 369          * in virtio_ring.c. At the minimum, a BUILD_BUG_ON() in
 370          * vring_new_virtqueue() would ensure that
 371          *  (&vq->vring == (struct vring *) (&vq->vq + 1));
 372          */
 373         vr = (struct vring *)(vq + 1);
 374         vr->used = used;
 375
 376         vq->priv = vdev;
 377         return vq;
 378 free_used:
 379         free_pages((unsigned long)used,
 380                    get_order(vdev->used_size[index]));
 381 del_vq:
 382         vring_del_virtqueue(vq);
 383 unmap:
 384         vpdev->hw_ops->iounmap(vpdev, vdev->vr[index]);
 385         return ERR_PTR(err);
 386 }
 387
 388 static int vop_find_vqs(struct virtio_device *dev, unsigned nvqs,
 389                         struct virtqueue *vqs[],
 390                         vq_callback_t *callbacks[],
 391                         const char * const names[], const bool *ctx,
 392                         struct irq_affinity *desc)
 393 {
 394         struct _vop_vdev *vdev = to_vopvdev(dev);
 395         struct vop_device *vpdev = vdev->vpdev;
 396         struct mic_device_ctrl __iomem *dc = vdev->dc;
 397         int i, err, retry, queue_idx = 0;
 398
 399         /* We must have this many virtqueues. */
 400         if (nvqs > ioread8(&vdev->desc->num_vq))
 401                 return -ENOENT;
 402
 403         for (i = 0; i < nvqs; ++i) {
 404                 if (!names[i]) {
 405                         vqs[i] = NULL;
 406                         continue;
 407                 }
 408
 409                 dev_dbg(_vop_dev(vdev), "%s: %d: %s\n",
 410                         __func__, i, names[i]);
 411                 vqs[i] = vop_find_vq(dev, queue_idx++, callbacks[i], names[i],
 412                                      ctx ? ctx[i] : false);
 413                 if (IS_ERR(vqs[i])) {
 414                         err = PTR_ERR(vqs[i]);
 415                         goto error;
 416                 }
 417         }
 418
 419         iowrite8(1, &dc->used_address_updated);
 420         /*
 421          * Send an interrupt to the host to inform it that used
 422          * rings have been re-assigned.
 423          */
 424         vpdev->hw_ops->send_intr(vpdev, vdev->c2h_vdev_db);
 425         for (retry = 100; --retry;) {
 426                 if (!ioread8(&dc->used_address_updated))
 427                         break;
 428                 msleep(100);
 429         };
 430
 431         dev_dbg(_vop_dev(vdev), "%s: retry: %d\n", __func__, retry);
 432         if (!retry) {
 433                 err = -ENODEV;
 434                 goto error;
 435         }
 436
 437         return 0;
 438 error:
 439         vop_del_vqs(dev);
 440         return err;
 441 }
 442
 443 /*
 444  * The config ops structure as defined by virtio config
 445  */
 446 static struct virtio_config_ops vop_vq_config_ops = {
 447         .get_features = vop_get_features,
 448         .finalize_features = vop_finalize_features,
 449         .get = vop_get,
 450         .set = vop_set,
 451         .get_status = vop_get_status,
 452         .set_status = vop_set_status,
 453         .reset = vop_reset,
 454         .find_vqs = vop_find_vqs,
 455         .del_vqs = vop_del_vqs,
 456 };
 457
 458 static irqreturn_t vop_virtio_intr_handler(int irq, void *data)
 459 {
 460         struct _vop_vdev *vdev = data;
 461         struct vop_device *vpdev = vdev->vpdev;
 462         struct virtqueue *vq;
 463
 464         vpdev->hw_ops->ack_interrupt(vpdev, vdev->h2c_vdev_db);
 465         list_for_each_entry(vq, &vdev->vdev.vqs, list)
 466                 vring_interrupt(0, vq);
 467
 468         return IRQ_HANDLED;
 469 }
 470
 471 static void vop_virtio_release_dev(struct device *_d)
 472 {
 473         struct virtio_device *vdev =
 474                         container_of(_d, struct virtio_device, dev);
 475         struct _vop_vdev *vop_vdev =
 476                         container_of(vdev, struct _vop_vdev, vdev);
 477
 478         kfree(vop_vdev);
 479 }
 480
 481 /*
 482  * adds a new device and register it with virtio
 483  * appropriate drivers are loaded by the device model
 484  */
 485 static int _vop_add_device(struct mic_device_desc __iomem *d,
 486                            unsigned int offset, struct vop_device *vpdev,
 487                            int dnode)
 488 {
 489         struct _vop_vdev *vdev, *reg_dev = NULL;
 490         int ret;
 491         u8 type = ioread8(&d->type);
 492
 493         vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);
 494         if (!vdev)
 495                 return -ENOMEM;
 496
 497         vdev->vpdev = vpdev;
 498         vdev->vdev.dev.parent = &vpdev->dev;
 499         vdev->vdev.dev.release = vop_virtio_release_dev;
 500         vdev->vdev.id.device = type;
 501         vdev->vdev.config = &vop_vq_config_ops;
 502         vdev->desc = d;
 503         vdev->dc = (void __iomem *)d + _vop_aligned_desc_size(d);
 504         vdev->dnode = dnode;
 505         vdev->vdev.priv = (void *)(u64)dnode;
 506         init_completion(&vdev->reset_done);
 507
 508         vdev->h2c_vdev_db = vpdev->hw_ops->next_db(vpdev);
 509         vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,
 510                         vop_virtio_intr_handler, "virtio intr",
 511                         vdev, vdev->h2c_vdev_db);
 512         if (IS_ERR(vdev->virtio_cookie)) {
 513                 ret = PTR_ERR(vdev->virtio_cookie);
 514                 goto kfree;
 515         }
 516         iowrite8((u8)vdev->h2c_vdev_db, &vdev->dc->h2c_vdev_db);
 517         vdev->c2h_vdev_db = ioread8(&vdev->dc->c2h_vdev_db);
 518
 519         ret = register_virtio_device(&vdev->vdev);
 520         reg_dev = vdev;
 521         if (ret) {
 522                 dev_err(_vop_dev(vdev),
 523                         "Failed to register vop device %u type %u\n",
 524                         offset, type);
 525                 goto free_irq;
 526         }
 527         writeq((u64)vdev, &vdev->dc->vdev);
 528         dev_dbg(_vop_dev(vdev), "%s: registered vop device %u type %u vdev %p\n",
 529                 __func__, offset, type, vdev);
 530
 531         return 0;
 532
 533 free_irq:
 534         vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
 535 kfree:
 536         if (reg_dev)
 537                 put_device(&vdev->vdev.dev);
 538         else
 539                 kfree(vdev);
 540         return ret;
 541 }
 542
 543 /*
 544  * match for a vop device with a specific desc pointer
 545  */
 546 static int vop_match_desc(struct device *dev, void *data)
 547 {
 548         struct virtio_device *_dev = dev_to_virtio(dev);
 549         struct _vop_vdev *vdev = to_vopvdev(_dev);
 550
 551         return vdev->desc == (void __iomem *)data;
 552 }
 553
 554 static void _vop_handle_config_change(struct mic_device_desc __iomem *d,
 555                                       unsigned int offset,
 556                                       struct vop_device *vpdev)
 557 {
 558         struct mic_device_ctrl __iomem *dc
 559                 = (void __iomem *)d + _vop_aligned_desc_size(d);
 560         struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
 561
 562         if (ioread8(&dc->config_change) != MIC_VIRTIO_PARAM_CONFIG_CHANGED)
 563                 return;
 564
 565         dev_dbg(&vpdev->dev, "%s %d\n", __func__, __LINE__);
 566         virtio_config_changed(&vdev->vdev);
 567         iowrite8(1, &dc->guest_ack);
 568 }
 569
 570 /*
 571  * removes a virtio device if a hot remove event has been
 572  * requested by the host.
 573  */
 574 static int _vop_remove_device(struct mic_device_desc __iomem *d,
 575                               unsigned int offset, struct vop_device *vpdev)
 576 {
 577         struct mic_device_ctrl __iomem *dc
 578                 = (void __iomem *)d + _vop_aligned_desc_size(d);
 579         struct _vop_vdev *vdev = (struct _vop_vdev *)readq(&dc->vdev);
 580         u8 status;
 581         int ret = -1;
 582
 583         if (ioread8(&dc->config_change) == MIC_VIRTIO_PARAM_DEV_REMOVE) {
 584                 dev_dbg(&vpdev->dev,
 585                         "%s %d config_change %d type %d vdev %p\n",
 586                         __func__, __LINE__,
 587                         ioread8(&dc->config_change), ioread8(&d->type), vdev);
 588                 status = ioread8(&d->status);
 589                 reinit_completion(&vdev->reset_done);
 590                 unregister_virtio_device(&vdev->vdev);
 591                 vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);
 592                 iowrite8(-1, &dc->h2c_vdev_db);
 593                 if (status & VIRTIO_CONFIG_S_DRIVER_OK)
 594                         wait_for_completion(&vdev->reset_done);
 595                 put_device(&vdev->vdev.dev);
 596                 iowrite8(1, &dc->guest_ack);
 597                 dev_dbg(&vpdev->dev, "%s %d guest_ack %d\n",
 598                         __func__, __LINE__, ioread8(&dc->guest_ack));
 599                 iowrite8(-1, &d->type);
 600                 ret = 0;
 601         }
 602         return ret;
 603 }
 604
 605 #define REMOVE_DEVICES true
 606
 607 static void _vop_scan_devices(void __iomem *dp, struct vop_device *vpdev,
 608                               bool remove, int dnode)
 609 {
 610         s8 type;
 611         unsigned int i;
 612         struct mic_device_desc __iomem *d;
 613         struct mic_device_ctrl __iomem *dc;
 614         struct device *dev;
 615         int ret;
 616
 617         for (i = sizeof(struct mic_bootparam);
 618                         i < MIC_DP_SIZE; i += _vop_total_desc_size(d)) {
 619                 d = dp + i;
 620                 dc = (void __iomem *)d + _vop_aligned_desc_size(d);
 621                 /*
 622                  * This read barrier is paired with the corresponding write
 623                  * barrier on the host which is inserted before adding or
 624                  * removing a virtio device descriptor, by updating the type.
 625                  */
 626                 rmb();
 627                 type = ioread8(&d->type);
 628
 629                 /* end of list */
 630                 if (type == 0)
 631                         break;
 632
 633                 if (type == -1)
 634                         continue;
 635
 636                 /* device already exists */
 637                 dev = device_find_child(&vpdev->dev, (void __force *)d,
 638                                         vop_match_desc);
 639                 if (dev) {
 640                         if (remove)
 641                                 iowrite8(MIC_VIRTIO_PARAM_DEV_REMOVE,
 642                                          &dc->config_change);
 643                         put_device(dev);
 644                         _vop_handle_config_change(d, i, vpdev);
 645                         ret = _vop_remove_device(d, i, vpdev);
 646                         if (remove) {
 647                                 iowrite8(0, &dc->config_change);
 648                                 iowrite8(0, &dc->guest_ack);
 649                         }
 650                         continue;
 651                 }
 652
 653                 /* new device */
 654                 dev_dbg(&vpdev->dev, "%s %d Adding new virtio device %p\n",
 655                         __func__, __LINE__, d);
 656                 if (!remove)
 657                         _vop_add_device(d, i, vpdev, dnode);
 658         }
 659 }
 660
 661 static void vop_scan_devices(struct vop_info *vi,
 662                              struct vop_device *vpdev, bool remove)
 663 {
 664         void __iomem *dp = vpdev->hw_ops->get_remote_dp(vpdev);
 665
 666         if (!dp)
 667                 return;
 668         mutex_lock(&vi->vop_mutex);
 669         _vop_scan_devices(dp, vpdev, remove, vpdev->dnode);
 670         mutex_unlock(&vi->vop_mutex);
 671 }
 672
 673 /*
 674  * vop_hotplug_device tries to find changes in the device page.
 675  */
 676 static void vop_hotplug_devices(struct work_struct *work)
 677 {
 678         struct vop_info *vi = container_of(work, struct vop_info,
 679                                              hotplug_work);
 680
 681         vop_scan_devices(vi, vi->vpdev, !REMOVE_DEVICES);
 682 }
 683
 684 /*
 685  * Interrupt handler for hot plug/config changes etc.
 686  */
 687 static irqreturn_t vop_extint_handler(int irq, void *data)
 688 {
 689         struct vop_info *vi = data;
 690         struct mic_bootparam __iomem *bp;
 691         struct vop_device *vpdev = vi->vpdev;
 692
 693         bp = vpdev->hw_ops->get_remote_dp(vpdev);
 694         dev_dbg(&vpdev->dev, "%s %d hotplug work\n",
 695                 __func__, __LINE__);
 696         vpdev->hw_ops->ack_interrupt(vpdev, ioread8(&bp->h2c_config_db));
 697         schedule_work(&vi->hotplug_work);
 698         return IRQ_HANDLED;
 699 }
 700
 701 static int vop_driver_probe(struct vop_device *vpdev)
 702 {
 703         struct vop_info *vi;
 704         int rc;
 705
 706         vi = kzalloc(sizeof(*vi), GFP_KERNEL);
 707         if (!vi) {
 708                 rc = -ENOMEM;
 709                 goto exit;
 710         }
 711         dev_set_drvdata(&vpdev->dev, vi);
 712         vi->vpdev = vpdev;
 713
 714         mutex_init(&vi->vop_mutex);
 715         INIT_WORK(&vi->hotplug_work, vop_hotplug_devices);
 716         if (vpdev->dnode) {
 717                 rc = vop_host_init(vi);
 718                 if (rc < 0)
 719                         goto free;
 720         } else {
 721                 struct mic_bootparam __iomem *bootparam;
 722
 723                 vop_scan_devices(vi, vpdev, !REMOVE_DEVICES);
 724
 725                 vi->h2c_config_db = vpdev->hw_ops->next_db(vpdev);
 726                 vi->cookie = vpdev->hw_ops->request_irq(vpdev,
 727                                                         vop_extint_handler,
 728                                                         "virtio_config_intr",
 729                                                         vi, vi->h2c_config_db);
 730                 if (IS_ERR(vi->cookie)) {
 731                         rc = PTR_ERR(vi->cookie);
 732                         goto free;
 733                 }
 734                 bootparam = vpdev->hw_ops->get_remote_dp(vpdev);
 735                 iowrite8(vi->h2c_config_db, &bootparam->h2c_config_db);
 736         }
 737         vop_init_debugfs(vi);
 738         return 0;
 739 free:
 740         kfree(vi);
 741 exit:
 742         return rc;
 743 }
 744
 745 static void vop_driver_remove(struct vop_device *vpdev)
 746 {
 747         struct vop_info *vi = dev_get_drvdata(&vpdev->dev);
 748
 749         if (vpdev->dnode) {
 750                 vop_host_uninit(vi);
 751         } else {
 752                 struct mic_bootparam __iomem *bootparam =
 753                         vpdev->hw_ops->get_remote_dp(vpdev);
 754                 if (bootparam)
 755                         iowrite8(-1, &bootparam->h2c_config_db);
 756                 vpdev->hw_ops->free_irq(vpdev, vi->cookie, vi);
 757                 flush_work(&vi->hotplug_work);
 758                 vop_scan_devices(vi, vpdev, REMOVE_DEVICES);
 759         }
 760         vop_exit_debugfs(vi);
 761         kfree(vi);
 762 }
 763
 764 static struct vop_device_id id_table[] = {
 765         { VOP_DEV_TRNSP, VOP_DEV_ANY_ID },
 766         { 0 },
 767 };
 768
 769 static struct vop_driver vop_driver = {
 770         .driver.name =  KBUILD_MODNAME,
 771         .driver.owner = THIS_MODULE,
 772         .id_table = id_table,
 773         .probe = vop_driver_probe,
 774         .remove = vop_driver_remove,
 775 };
 776
 777 module_vop_driver(vop_driver);
 778
 779 MODULE_DEVICE_TABLE(mbus, id_table);
 780 MODULE_AUTHOR("Intel Corporation");
 781 MODULE_DESCRIPTION("Intel(R) Virtio Over PCIe (VOP) driver");
 782 MODULE_LICENSE("GPL v2");