drivers/s390/cio/vfio_ccw_drv.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * VFIO based Physical Subchannel device driver
   4  *
   5  * Copyright IBM Corp. 2017
   6  * Copyright Red Hat, Inc. 2019
   7  *
   8  * Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
   9  *            Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
  10  *            Cornelia Huck <cohuck@redhat.com>
  11  */
  12
  13 #include <linux/module.h>
  14 #include <linux/init.h>
  15 #include <linux/device.h>
  16 #include <linux/slab.h>
  17 #include <linux/uuid.h>
  18 #include <linux/mdev.h>
  19
  20 #include <asm/isc.h>
  21
  22 #include "chp.h"
  23 #include "ioasm.h"
  24 #include "css.h"
  25 #include "vfio_ccw_private.h"
  26
  27 struct workqueue_struct *vfio_ccw_work_q;
  28 static struct kmem_cache *vfio_ccw_io_region;
  29 static struct kmem_cache *vfio_ccw_cmd_region;
  30 static struct kmem_cache *vfio_ccw_schib_region;
  31 static struct kmem_cache *vfio_ccw_crw_region;
  32
  33 debug_info_t *vfio_ccw_debug_msg_id;
  34 debug_info_t *vfio_ccw_debug_trace_id;
  35
  36 /*
  37  * Helpers
  38  */
  39 int vfio_ccw_sch_quiesce(struct subchannel *sch)
  40 {
  41         struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
  42         DECLARE_COMPLETION_ONSTACK(completion);
  43         int iretry, ret = 0;
  44
  45         spin_lock_irq(sch->lock);
  46         if (!sch->schib.pmcw.ena)
  47                 goto out_unlock;
  48         ret = cio_disable_subchannel(sch);
  49         if (ret != -EBUSY)
  50                 goto out_unlock;
  51
  52         iretry = 255;
  53         do {
  54
  55                 ret = cio_cancel_halt_clear(sch, &iretry);
  56
  57                 if (ret == -EIO) {
  58                         pr_err("vfio_ccw: could not quiesce subchannel 0.%x.%04x!\n",
  59                                sch->schid.ssid, sch->schid.sch_no);
  60                         break;
  61                 }
  62
  63                 /*
  64                  * Flush all I/O and wait for
  65                  * cancel/halt/clear completion.
  66                  */
  67                 private->completion = &completion;
  68                 spin_unlock_irq(sch->lock);
  69
  70                 if (ret == -EBUSY)
  71                         wait_for_completion_timeout(&completion, 3*HZ);
  72
  73                 private->completion = NULL;
  74                 flush_workqueue(vfio_ccw_work_q);
  75                 spin_lock_irq(sch->lock);
  76                 ret = cio_disable_subchannel(sch);
  77         } while (ret == -EBUSY);
  78 out_unlock:
  79         private->state = VFIO_CCW_STATE_NOT_OPER;
  80         spin_unlock_irq(sch->lock);
  81         return ret;
  82 }
  83
  84 static void vfio_ccw_sch_io_todo(struct work_struct *work)
  85 {
  86         struct vfio_ccw_private *private;
  87         struct irb *irb;
  88         bool is_final;
  89         bool cp_is_finished = false;
  90
  91         private = container_of(work, struct vfio_ccw_private, io_work);
  92         irb = &private->irb;
  93
  94         is_final = !(scsw_actl(&irb->scsw) &
  95                      (SCSW_ACTL_DEVACT | SCSW_ACTL_SCHACT));
  96         if (scsw_is_solicited(&irb->scsw)) {
  97                 cp_update_scsw(&private->cp, &irb->scsw);
  98                 if (is_final && private->state == VFIO_CCW_STATE_CP_PENDING) {
  99                         cp_free(&private->cp);
 100                         cp_is_finished = true;
 101                 }
 102         }
 103         mutex_lock(&private->io_mutex);
 104         memcpy(private->io_region->irb_area, irb, sizeof(*irb));
 105         mutex_unlock(&private->io_mutex);
 106
 107         /*
 108          * Reset to IDLE only if processing of a channel program
 109          * has finished. Do not overwrite a possible processing
 110          * state if the final interrupt was for HSCH or CSCH.
 111          */
 112         if (private->mdev && cp_is_finished)
 113                 private->state = VFIO_CCW_STATE_IDLE;
 114
 115         if (private->io_trigger)
 116                 eventfd_signal(private->io_trigger, 1);
 117 }
 118
 119 static void vfio_ccw_crw_todo(struct work_struct *work)
 120 {
 121         struct vfio_ccw_private *private;
 122
 123         private = container_of(work, struct vfio_ccw_private, crw_work);
 124
 125         if (!list_empty(&private->crw) && private->crw_trigger)
 126                 eventfd_signal(private->crw_trigger, 1);
 127 }
 128
 129 /*
 130  * Css driver callbacks
 131  */
 132 static void vfio_ccw_sch_irq(struct subchannel *sch)
 133 {
 134         struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
 135
 136         inc_irq_stat(IRQIO_CIO);
 137         vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_INTERRUPT);
 138 }
 139
 140 static void vfio_ccw_free_regions(struct vfio_ccw_private *private)
 141 {
 142         if (private->crw_region)
 143                 kmem_cache_free(vfio_ccw_crw_region, private->crw_region);
 144         if (private->schib_region)
 145                 kmem_cache_free(vfio_ccw_schib_region, private->schib_region);
 146         if (private->cmd_region)
 147                 kmem_cache_free(vfio_ccw_cmd_region, private->cmd_region);
 148         if (private->io_region)
 149                 kmem_cache_free(vfio_ccw_io_region, private->io_region);
 150 }
 151
 152 static int vfio_ccw_sch_probe(struct subchannel *sch)
 153 {
 154         struct pmcw *pmcw = &sch->schib.pmcw;
 155         struct vfio_ccw_private *private;
 156         int ret = -ENOMEM;
 157
 158         if (pmcw->qf) {
 159                 dev_warn(&sch->dev, "vfio: ccw: does not support QDIO: %s\n",
 160                          dev_name(&sch->dev));
 161                 return -ENODEV;
 162         }
 163
 164         private = kzalloc(sizeof(*private), GFP_KERNEL | GFP_DMA);
 165         if (!private)
 166                 return -ENOMEM;
 167
 168         private->cp.guest_cp = kcalloc(CCWCHAIN_LEN_MAX, sizeof(struct ccw1),
 169                                        GFP_KERNEL);
 170         if (!private->cp.guest_cp)
 171                 goto out_free;
 172
 173         private->io_region = kmem_cache_zalloc(vfio_ccw_io_region,
 174                                                GFP_KERNEL | GFP_DMA);
 175         if (!private->io_region)
 176                 goto out_free;
 177
 178         private->cmd_region = kmem_cache_zalloc(vfio_ccw_cmd_region,
 179                                                 GFP_KERNEL | GFP_DMA);
 180         if (!private->cmd_region)
 181                 goto out_free;
 182
 183         private->schib_region = kmem_cache_zalloc(vfio_ccw_schib_region,
 184                                                   GFP_KERNEL | GFP_DMA);
 185
 186         if (!private->schib_region)
 187                 goto out_free;
 188
 189         private->crw_region = kmem_cache_zalloc(vfio_ccw_crw_region,
 190                                                 GFP_KERNEL | GFP_DMA);
 191
 192         if (!private->crw_region)
 193                 goto out_free;
 194
 195         private->sch = sch;
 196         dev_set_drvdata(&sch->dev, private);
 197         mutex_init(&private->io_mutex);
 198
 199         spin_lock_irq(sch->lock);
 200         private->state = VFIO_CCW_STATE_NOT_OPER;
 201         sch->isc = VFIO_CCW_ISC;
 202         ret = cio_enable_subchannel(sch, (u32)(unsigned long)sch);
 203         spin_unlock_irq(sch->lock);
 204         if (ret)
 205                 goto out_free;
 206
 207         INIT_LIST_HEAD(&private->crw);
 208         INIT_WORK(&private->io_work, vfio_ccw_sch_io_todo);
 209         INIT_WORK(&private->crw_work, vfio_ccw_crw_todo);
 210         atomic_set(&private->avail, 1);
 211         private->state = VFIO_CCW_STATE_STANDBY;
 212
 213         ret = vfio_ccw_mdev_reg(sch);
 214         if (ret)
 215                 goto out_disable;
 216
 217         if (dev_get_uevent_suppress(&sch->dev)) {
 218                 dev_set_uevent_suppress(&sch->dev, 0);
 219                 kobject_uevent(&sch->dev.kobj, KOBJ_ADD);
 220         }
 221
 222         VFIO_CCW_MSG_EVENT(4, "bound to subchannel %x.%x.%04x\n",
 223                            sch->schid.cssid, sch->schid.ssid,
 224                            sch->schid.sch_no);
 225         return 0;
 226
 227 out_disable:
 228         cio_disable_subchannel(sch);
 229 out_free:
 230         dev_set_drvdata(&sch->dev, NULL);
 231         vfio_ccw_free_regions(private);
 232         kfree(private->cp.guest_cp);
 233         kfree(private);
 234         return ret;
 235 }
 236
 237 static void vfio_ccw_sch_remove(struct subchannel *sch)
 238 {
 239         struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
 240         struct vfio_ccw_crw *crw, *temp;
 241
 242         vfio_ccw_sch_quiesce(sch);
 243
 244         list_for_each_entry_safe(crw, temp, &private->crw, next) {
 245                 list_del(&crw->next);
 246                 kfree(crw);
 247         }
 248
 249         vfio_ccw_mdev_unreg(sch);
 250
 251         dev_set_drvdata(&sch->dev, NULL);
 252
 253         vfio_ccw_free_regions(private);
 254         kfree(private->cp.guest_cp);
 255         kfree(private);
 256
 257         VFIO_CCW_MSG_EVENT(4, "unbound from subchannel %x.%x.%04x\n",
 258                            sch->schid.cssid, sch->schid.ssid,
 259                            sch->schid.sch_no);
 260 }
 261
 262 static void vfio_ccw_sch_shutdown(struct subchannel *sch)
 263 {
 264         vfio_ccw_sch_quiesce(sch);
 265 }
 266
 267 /**
 268  * vfio_ccw_sch_event - process subchannel event
 269  * @sch: subchannel
 270  * @process: non-zero if function is called in process context
 271  *
 272  * An unspecified event occurred for this subchannel. Adjust data according
 273  * to the current operational state of the subchannel. Return zero when the
 274  * event has been handled sufficiently or -EAGAIN when this function should
 275  * be called again in process context.
 276  */
 277 static int vfio_ccw_sch_event(struct subchannel *sch, int process)
 278 {
 279         struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
 280         unsigned long flags;
 281         int rc = -EAGAIN;
 282
 283         spin_lock_irqsave(sch->lock, flags);
 284         if (!device_is_registered(&sch->dev))
 285                 goto out_unlock;
 286
 287         if (work_pending(&sch->todo_work))
 288                 goto out_unlock;
 289
 290         if (cio_update_schib(sch)) {
 291                 vfio_ccw_fsm_event(private, VFIO_CCW_EVENT_NOT_OPER);
 292                 rc = 0;
 293                 goto out_unlock;
 294         }
 295
 296         private = dev_get_drvdata(&sch->dev);
 297         if (private->state == VFIO_CCW_STATE_NOT_OPER) {
 298                 private->state = private->mdev ? VFIO_CCW_STATE_IDLE :
 299                                  VFIO_CCW_STATE_STANDBY;
 300         }
 301         rc = 0;
 302
 303 out_unlock:
 304         spin_unlock_irqrestore(sch->lock, flags);
 305
 306         return rc;
 307 }
 308
 309 static void vfio_ccw_queue_crw(struct vfio_ccw_private *private,
 310                                unsigned int rsc,
 311                                unsigned int erc,
 312                                unsigned int rsid)
 313 {
 314         struct vfio_ccw_crw *crw;
 315
 316         /*
 317          * If unable to allocate a CRW, just drop the event and
 318          * carry on.  The guest will either see a later one or
 319          * learn when it issues its own store subchannel.
 320          */
 321         crw = kzalloc(sizeof(*crw), GFP_ATOMIC);
 322         if (!crw)
 323                 return;
 324
 325         /*
 326          * Build the CRW based on the inputs given to us.
 327          */
 328         crw->crw.rsc = rsc;
 329         crw->crw.erc = erc;
 330         crw->crw.rsid = rsid;
 331
 332         list_add_tail(&crw->next, &private->crw);
 333         queue_work(vfio_ccw_work_q, &private->crw_work);
 334 }
 335
 336 static int vfio_ccw_chp_event(struct subchannel *sch,
 337                               struct chp_link *link, int event)
 338 {
 339         struct vfio_ccw_private *private = dev_get_drvdata(&sch->dev);
 340         int mask = chp_ssd_get_mask(&sch->ssd_info, link);
 341         int retry = 255;
 342
 343         if (!private || !mask)
 344                 return 0;
 345
 346         trace_vfio_ccw_chp_event(private->sch->schid, mask, event);
 347         VFIO_CCW_MSG_EVENT(2, "%pUl (%x.%x.%04x): mask=0x%x event=%d\n",
 348                            mdev_uuid(private->mdev), sch->schid.cssid,
 349                            sch->schid.ssid, sch->schid.sch_no,
 350                            mask, event);
 351
 352         if (cio_update_schib(sch))
 353                 return -ENODEV;
 354
 355         switch (event) {
 356         case CHP_VARY_OFF:
 357                 /* Path logically turned off */
 358                 sch->opm &= ~mask;
 359                 sch->lpm &= ~mask;
 360                 if (sch->schib.pmcw.lpum & mask)
 361                         cio_cancel_halt_clear(sch, &retry);
 362                 break;
 363         case CHP_OFFLINE:
 364                 /* Path is gone */
 365                 if (sch->schib.pmcw.lpum & mask)
 366                         cio_cancel_halt_clear(sch, &retry);
 367                 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_PERRN,
 368                                    link->chpid.id);
 369                 break;
 370         case CHP_VARY_ON:
 371                 /* Path logically turned on */
 372                 sch->opm |= mask;
 373                 sch->lpm |= mask;
 374                 break;
 375         case CHP_ONLINE:
 376                 /* Path became available */
 377                 sch->lpm |= mask & sch->opm;
 378                 vfio_ccw_queue_crw(private, CRW_RSC_CPATH, CRW_ERC_INIT,
 379                                    link->chpid.id);
 380                 break;
 381         }
 382
 383         return 0;
 384 }
 385
 386 static struct css_device_id vfio_ccw_sch_ids[] = {
 387         { .match_flags = 0x1, .type = SUBCHANNEL_TYPE_IO, },
 388         { /* end of list */ },
 389 };
 390 MODULE_DEVICE_TABLE(css, vfio_ccw_sch_ids);
 391
 392 static struct css_driver vfio_ccw_sch_driver = {
 393         .drv = {
 394                 .name = "vfio_ccw",
 395                 .owner = THIS_MODULE,
 396         },
 397         .subchannel_type = vfio_ccw_sch_ids,
 398         .irq = vfio_ccw_sch_irq,
 399         .probe = vfio_ccw_sch_probe,
 400         .remove = vfio_ccw_sch_remove,
 401         .shutdown = vfio_ccw_sch_shutdown,
 402         .sch_event = vfio_ccw_sch_event,
 403         .chp_event = vfio_ccw_chp_event,
 404 };
 405
 406 static int __init vfio_ccw_debug_init(void)
 407 {
 408         vfio_ccw_debug_msg_id = debug_register("vfio_ccw_msg", 16, 1,
 409                                                11 * sizeof(long));
 410         if (!vfio_ccw_debug_msg_id)
 411                 goto out_unregister;
 412         debug_register_view(vfio_ccw_debug_msg_id, &debug_sprintf_view);
 413         debug_set_level(vfio_ccw_debug_msg_id, 2);
 414         vfio_ccw_debug_trace_id = debug_register("vfio_ccw_trace", 16, 1, 16);
 415         if (!vfio_ccw_debug_trace_id)
 416                 goto out_unregister;
 417         debug_register_view(vfio_ccw_debug_trace_id, &debug_hex_ascii_view);
 418         debug_set_level(vfio_ccw_debug_trace_id, 2);
 419         return 0;
 420
 421 out_unregister:
 422         debug_unregister(vfio_ccw_debug_msg_id);
 423         debug_unregister(vfio_ccw_debug_trace_id);
 424         return -1;
 425 }
 426
 427 static void vfio_ccw_debug_exit(void)
 428 {
 429         debug_unregister(vfio_ccw_debug_msg_id);
 430         debug_unregister(vfio_ccw_debug_trace_id);
 431 }
 432
 433 static void vfio_ccw_destroy_regions(void)
 434 {
 435         kmem_cache_destroy(vfio_ccw_crw_region);
 436         kmem_cache_destroy(vfio_ccw_schib_region);
 437         kmem_cache_destroy(vfio_ccw_cmd_region);
 438         kmem_cache_destroy(vfio_ccw_io_region);
 439 }
 440
 441 static int __init vfio_ccw_sch_init(void)
 442 {
 443         int ret;
 444
 445         ret = vfio_ccw_debug_init();
 446         if (ret)
 447                 return ret;
 448
 449         vfio_ccw_work_q = create_singlethread_workqueue("vfio-ccw");
 450         if (!vfio_ccw_work_q) {
 451                 ret = -ENOMEM;
 452                 goto out_err;
 453         }
 454
 455         vfio_ccw_io_region = kmem_cache_create_usercopy("vfio_ccw_io_region",
 456                                         sizeof(struct ccw_io_region), 0,
 457                                         SLAB_ACCOUNT, 0,
 458                                         sizeof(struct ccw_io_region), NULL);
 459         if (!vfio_ccw_io_region) {
 460                 ret = -ENOMEM;
 461                 goto out_err;
 462         }
 463
 464         vfio_ccw_cmd_region = kmem_cache_create_usercopy("vfio_ccw_cmd_region",
 465                                         sizeof(struct ccw_cmd_region), 0,
 466                                         SLAB_ACCOUNT, 0,
 467                                         sizeof(struct ccw_cmd_region), NULL);
 468         if (!vfio_ccw_cmd_region) {
 469                 ret = -ENOMEM;
 470                 goto out_err;
 471         }
 472
 473         vfio_ccw_schib_region = kmem_cache_create_usercopy("vfio_ccw_schib_region",
 474                                         sizeof(struct ccw_schib_region), 0,
 475                                         SLAB_ACCOUNT, 0,
 476                                         sizeof(struct ccw_schib_region), NULL);
 477
 478         if (!vfio_ccw_schib_region) {
 479                 ret = -ENOMEM;
 480                 goto out_err;
 481         }
 482
 483         vfio_ccw_crw_region = kmem_cache_create_usercopy("vfio_ccw_crw_region",
 484                                         sizeof(struct ccw_crw_region), 0,
 485                                         SLAB_ACCOUNT, 0,
 486                                         sizeof(struct ccw_crw_region), NULL);
 487
 488         if (!vfio_ccw_crw_region) {
 489                 ret = -ENOMEM;
 490                 goto out_err;
 491         }
 492
 493         isc_register(VFIO_CCW_ISC);
 494         ret = css_driver_register(&vfio_ccw_sch_driver);
 495         if (ret) {
 496                 isc_unregister(VFIO_CCW_ISC);
 497                 goto out_err;
 498         }
 499
 500         return ret;
 501
 502 out_err:
 503         vfio_ccw_destroy_regions();
 504         destroy_workqueue(vfio_ccw_work_q);
 505         vfio_ccw_debug_exit();
 506         return ret;
 507 }
 508
 509 static void __exit vfio_ccw_sch_exit(void)
 510 {
 511         css_driver_unregister(&vfio_ccw_sch_driver);
 512         isc_unregister(VFIO_CCW_ISC);
 513         vfio_ccw_destroy_regions();
 514         destroy_workqueue(vfio_ccw_work_q);
 515         vfio_ccw_debug_exit();
 516 }
 517 module_init(vfio_ccw_sch_init);
 518 module_exit(vfio_ccw_sch_exit);
 519
 520 MODULE_LICENSE("GPL v2");