drivers/s390/block/scm_blk.c

   1 /*
   2  * Block driver for s390 storage class memory.
   3  *
   4  * Copyright IBM Corp. 2012
   5  * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
   6  */
   7
   8 #define KMSG_COMPONENT "scm_block"
   9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/interrupt.h>
  12 #include <linux/spinlock.h>
  13 #include <linux/mempool.h>
  14 #include <linux/module.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/blk-mq.h>
  17 #include <linux/genhd.h>
  18 #include <linux/slab.h>
  19 #include <linux/list.h>
  20 #include <asm/eadm.h>
  21 #include "scm_blk.h"
  22
  23 debug_info_t *scm_debug;
  24 static int scm_major;
  25 static mempool_t *aidaw_pool;
  26 static DEFINE_SPINLOCK(list_lock);
  27 static LIST_HEAD(inactive_requests);
  28 static unsigned int nr_requests = 64;
  29 static unsigned int nr_requests_per_io = 8;
  30 static atomic_t nr_devices = ATOMIC_INIT(0);
  31 module_param(nr_requests, uint, S_IRUGO);
  32 MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
  33
  34 module_param(nr_requests_per_io, uint, S_IRUGO);
  35 MODULE_PARM_DESC(nr_requests_per_io, "Number of requests per IO.");
  36
  37 MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
  38 MODULE_LICENSE("GPL");
  39 MODULE_ALIAS("scm:scmdev*");
  40
  41 static void __scm_free_rq(struct scm_request *scmrq)
  42 {
  43         struct aob_rq_header *aobrq = to_aobrq(scmrq);
  44
  45         free_page((unsigned long) scmrq->aob);
  46         kfree(scmrq->request);
  47         kfree(aobrq);
  48 }
  49
  50 static void scm_free_rqs(void)
  51 {
  52         struct list_head *iter, *safe;
  53         struct scm_request *scmrq;
  54
  55         spin_lock_irq(&list_lock);
  56         list_for_each_safe(iter, safe, &inactive_requests) {
  57                 scmrq = list_entry(iter, struct scm_request, list);
  58                 list_del(&scmrq->list);
  59                 __scm_free_rq(scmrq);
  60         }
  61         spin_unlock_irq(&list_lock);
  62
  63         mempool_destroy(aidaw_pool);
  64 }
  65
  66 static int __scm_alloc_rq(void)
  67 {
  68         struct aob_rq_header *aobrq;
  69         struct scm_request *scmrq;
  70
  71         aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
  72         if (!aobrq)
  73                 return -ENOMEM;
  74
  75         scmrq = (void *) aobrq->data;
  76         scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
  77         if (!scmrq->aob)
  78                 goto free;
  79
  80         scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]),
  81                                  GFP_KERNEL);
  82         if (!scmrq->request)
  83                 goto free;
  84
  85         INIT_LIST_HEAD(&scmrq->list);
  86         spin_lock_irq(&list_lock);
  87         list_add(&scmrq->list, &inactive_requests);
  88         spin_unlock_irq(&list_lock);
  89
  90         return 0;
  91 free:
  92         __scm_free_rq(scmrq);
  93         return -ENOMEM;
  94 }
  95
  96 static int scm_alloc_rqs(unsigned int nrqs)
  97 {
  98         int ret = 0;
  99
 100         aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0);
 101         if (!aidaw_pool)
 102                 return -ENOMEM;
 103
 104         while (nrqs-- && !ret)
 105                 ret = __scm_alloc_rq();
 106
 107         return ret;
 108 }
 109
 110 static struct scm_request *scm_request_fetch(void)
 111 {
 112         struct scm_request *scmrq = NULL;
 113
 114         spin_lock_irq(&list_lock);
 115         if (list_empty(&inactive_requests))
 116                 goto out;
 117         scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
 118         list_del(&scmrq->list);
 119 out:
 120         spin_unlock_irq(&list_lock);
 121         return scmrq;
 122 }
 123
 124 static void scm_request_done(struct scm_request *scmrq)
 125 {
 126         unsigned long flags;
 127         struct msb *msb;
 128         u64 aidaw;
 129         int i;
 130
 131         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
 132                 msb = &scmrq->aob->msb[i];
 133                 aidaw = msb->data_addr;
 134
 135                 if ((msb->flags & MSB_FLAG_IDA) && aidaw &&
 136                     IS_ALIGNED(aidaw, PAGE_SIZE))
 137                         mempool_free(virt_to_page(aidaw), aidaw_pool);
 138         }
 139
 140         spin_lock_irqsave(&list_lock, flags);
 141         list_add(&scmrq->list, &inactive_requests);
 142         spin_unlock_irqrestore(&list_lock, flags);
 143 }
 144
 145 static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
 146 {
 147         return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
 148 }
 149
 150 static inline struct aidaw *scm_aidaw_alloc(void)
 151 {
 152         struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC);
 153
 154         return page ? page_address(page) : NULL;
 155 }
 156
 157 static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw)
 158 {
 159         unsigned long _aidaw = (unsigned long) aidaw;
 160         unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw;
 161
 162         return (bytes / sizeof(*aidaw)) * PAGE_SIZE;
 163 }
 164
 165 struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)
 166 {
 167         struct aidaw *aidaw;
 168
 169         if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)
 170                 return scmrq->next_aidaw;
 171
 172         aidaw = scm_aidaw_alloc();
 173         if (aidaw)
 174                 memset(aidaw, 0, PAGE_SIZE);
 175         return aidaw;
 176 }
 177
 178 static int scm_request_prepare(struct scm_request *scmrq)
 179 {
 180         struct scm_blk_dev *bdev = scmrq->bdev;
 181         struct scm_device *scmdev = bdev->gendisk->private_data;
 182         int pos = scmrq->aob->request.msb_count;
 183         struct msb *msb = &scmrq->aob->msb[pos];
 184         struct request *req = scmrq->request[pos];
 185         struct req_iterator iter;
 186         struct aidaw *aidaw;
 187         struct bio_vec bv;
 188
 189         aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));
 190         if (!aidaw)
 191                 return -ENOMEM;
 192
 193         msb->bs = MSB_BS_4K;
 194         scmrq->aob->request.msb_count++;
 195         msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9);
 196         msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE;
 197         msb->flags |= MSB_FLAG_IDA;
 198         msb->data_addr = (u64) aidaw;
 199
 200         rq_for_each_segment(bv, req, iter) {
 201                 WARN_ON(bv.bv_offset);
 202                 msb->blk_count += bv.bv_len >> 12;
 203                 aidaw->data_addr = (u64) page_address(bv.bv_page);
 204                 aidaw++;
 205         }
 206
 207         scmrq->next_aidaw = aidaw;
 208         return 0;
 209 }
 210
 211 static inline void scm_request_set(struct scm_request *scmrq,
 212                                    struct request *req)
 213 {
 214         scmrq->request[scmrq->aob->request.msb_count] = req;
 215 }
 216
 217 static inline void scm_request_init(struct scm_blk_dev *bdev,
 218                                     struct scm_request *scmrq)
 219 {
 220         struct aob_rq_header *aobrq = to_aobrq(scmrq);
 221         struct aob *aob = scmrq->aob;
 222
 223         memset(scmrq->request, 0,
 224                nr_requests_per_io * sizeof(scmrq->request[0]));
 225         memset(aob, 0, sizeof(*aob));
 226         aobrq->scmdev = bdev->scmdev;
 227         aob->request.cmd_code = ARQB_CMD_MOVE;
 228         aob->request.data = (u64) aobrq;
 229         scmrq->bdev = bdev;
 230         scmrq->retries = 4;
 231         scmrq->error = BLK_STS_OK;
 232         /* We don't use all msbs - place aidaws at the end of the aob page. */
 233         scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];
 234 }
 235
 236 static void scm_request_requeue(struct scm_request *scmrq)
 237 {
 238         struct scm_blk_dev *bdev = scmrq->bdev;
 239         int i;
 240
 241         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)
 242                 blk_mq_requeue_request(scmrq->request[i], false);
 243
 244         atomic_dec(&bdev->queued_reqs);
 245         scm_request_done(scmrq);
 246         blk_mq_kick_requeue_list(bdev->rq);
 247 }
 248
 249 static void scm_request_finish(struct scm_request *scmrq)
 250 {
 251         struct scm_blk_dev *bdev = scmrq->bdev;
 252         blk_status_t *error;
 253         int i;
 254
 255         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
 256                 error = blk_mq_rq_to_pdu(scmrq->request[i]);
 257                 *error = scmrq->error;
 258                 blk_mq_complete_request(scmrq->request[i]);
 259         }
 260
 261         atomic_dec(&bdev->queued_reqs);
 262         scm_request_done(scmrq);
 263 }
 264
 265 static void scm_request_start(struct scm_request *scmrq)
 266 {
 267         struct scm_blk_dev *bdev = scmrq->bdev;
 268
 269         atomic_inc(&bdev->queued_reqs);
 270         if (eadm_start_aob(scmrq->aob)) {
 271                 SCM_LOG(5, "no subchannel");
 272                 scm_request_requeue(scmrq);
 273         }
 274 }
 275
 276 struct scm_queue {
 277         struct scm_request *scmrq;
 278         spinlock_t lock;
 279 };
 280
 281 static blk_status_t scm_blk_request(struct blk_mq_hw_ctx *hctx,
 282                            const struct blk_mq_queue_data *qd)
 283 {
 284         struct scm_device *scmdev = hctx->queue->queuedata;
 285         struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
 286         struct scm_queue *sq = hctx->driver_data;
 287         struct request *req = qd->rq;
 288         struct scm_request *scmrq;
 289
 290         spin_lock(&sq->lock);
 291         if (!scm_permit_request(bdev, req)) {
 292                 spin_unlock(&sq->lock);
 293                 return BLK_STS_RESOURCE;
 294         }
 295
 296         scmrq = sq->scmrq;
 297         if (!scmrq) {
 298                 scmrq = scm_request_fetch();
 299                 if (!scmrq) {
 300                         SCM_LOG(5, "no request");
 301                         spin_unlock(&sq->lock);
 302                         return BLK_STS_RESOURCE;
 303                 }
 304                 scm_request_init(bdev, scmrq);
 305                 sq->scmrq = scmrq;
 306         }
 307         scm_request_set(scmrq, req);
 308
 309         if (scm_request_prepare(scmrq)) {
 310                 SCM_LOG(5, "aidaw alloc failed");
 311                 scm_request_set(scmrq, NULL);
 312
 313                 if (scmrq->aob->request.msb_count)
 314                         scm_request_start(scmrq);
 315
 316                 sq->scmrq = NULL;
 317                 spin_unlock(&sq->lock);
 318                 return BLK_STS_RESOURCE;
 319         }
 320         blk_mq_start_request(req);
 321
 322         if (qd->last || scmrq->aob->request.msb_count == nr_requests_per_io) {
 323                 scm_request_start(scmrq);
 324                 sq->scmrq = NULL;
 325         }
 326         spin_unlock(&sq->lock);
 327         return BLK_STS_OK;
 328 }
 329
 330 static int scm_blk_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 331                              unsigned int idx)
 332 {
 333         struct scm_queue *qd = kzalloc(sizeof(*qd), GFP_KERNEL);
 334
 335         if (!qd)
 336                 return -ENOMEM;
 337
 338         spin_lock_init(&qd->lock);
 339         hctx->driver_data = qd;
 340
 341         return 0;
 342 }
 343
 344 static void scm_blk_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int idx)
 345 {
 346         struct scm_queue *qd = hctx->driver_data;
 347
 348         WARN_ON(qd->scmrq);
 349         kfree(hctx->driver_data);
 350         hctx->driver_data = NULL;
 351 }
 352
 353 static void __scmrq_log_error(struct scm_request *scmrq)
 354 {
 355         struct aob *aob = scmrq->aob;
 356
 357         if (scmrq->error == BLK_STS_TIMEOUT)
 358                 SCM_LOG(1, "Request timeout");
 359         else {
 360                 SCM_LOG(1, "Request error");
 361                 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
 362         }
 363         if (scmrq->retries)
 364                 SCM_LOG(1, "Retry request");
 365         else
 366                 pr_err("An I/O operation to SCM failed with rc=%d\n",
 367                        scmrq->error);
 368 }
 369
 370 static void scm_blk_handle_error(struct scm_request *scmrq)
 371 {
 372         struct scm_blk_dev *bdev = scmrq->bdev;
 373         unsigned long flags;
 374
 375         if (scmrq->error != BLK_STS_IOERR)
 376                 goto restart;
 377
 378         /* For -EIO the response block is valid. */
 379         switch (scmrq->aob->response.eqc) {
 380         case EQC_WR_PROHIBIT:
 381                 spin_lock_irqsave(&bdev->lock, flags);
 382                 if (bdev->state != SCM_WR_PROHIBIT)
 383                         pr_info("%lx: Write access to the SCM increment is suspended\n",
 384                                 (unsigned long) bdev->scmdev->address);
 385                 bdev->state = SCM_WR_PROHIBIT;
 386                 spin_unlock_irqrestore(&bdev->lock, flags);
 387                 goto requeue;
 388         default:
 389                 break;
 390         }
 391
 392 restart:
 393         if (!eadm_start_aob(scmrq->aob))
 394                 return;
 395
 396 requeue:
 397         scm_request_requeue(scmrq);
 398 }
 399
 400 void scm_blk_irq(struct scm_device *scmdev, void *data, blk_status_t error)
 401 {
 402         struct scm_request *scmrq = data;
 403
 404         scmrq->error = error;
 405         if (error) {
 406                 __scmrq_log_error(scmrq);
 407                 if (scmrq->retries-- > 0) {
 408                         scm_blk_handle_error(scmrq);
 409                         return;
 410                 }
 411         }
 412
 413         scm_request_finish(scmrq);
 414 }
 415
 416 static void scm_blk_request_done(struct request *req)
 417 {
 418         blk_status_t *error = blk_mq_rq_to_pdu(req);
 419
 420         blk_mq_end_request(req, *error);
 421 }
 422
 423 static const struct block_device_operations scm_blk_devops = {
 424         .owner = THIS_MODULE,
 425 };
 426
 427 static const struct blk_mq_ops scm_mq_ops = {
 428         .queue_rq = scm_blk_request,
 429         .complete = scm_blk_request_done,
 430         .init_hctx = scm_blk_init_hctx,
 431         .exit_hctx = scm_blk_exit_hctx,
 432 };
 433
 434 int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
 435 {
 436         unsigned int devindex, nr_max_blk;
 437         struct request_queue *rq;
 438         int len, ret;
 439
 440         devindex = atomic_inc_return(&nr_devices) - 1;
 441         /* scma..scmz + scmaa..scmzz */
 442         if (devindex > 701) {
 443                 ret = -ENODEV;
 444                 goto out;
 445         }
 446
 447         bdev->scmdev = scmdev;
 448         bdev->state = SCM_OPER;
 449         spin_lock_init(&bdev->lock);
 450         atomic_set(&bdev->queued_reqs, 0);
 451
 452         bdev->tag_set.ops = &scm_mq_ops;
 453         bdev->tag_set.cmd_size = sizeof(blk_status_t);
 454         bdev->tag_set.nr_hw_queues = nr_requests;
 455         bdev->tag_set.queue_depth = nr_requests_per_io * nr_requests;
 456         bdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
 457
 458         ret = blk_mq_alloc_tag_set(&bdev->tag_set);
 459         if (ret)
 460                 goto out;
 461
 462         rq = blk_mq_init_queue(&bdev->tag_set);
 463         if (IS_ERR(rq)) {
 464                 ret = PTR_ERR(rq);
 465                 goto out_tag;
 466         }
 467         bdev->rq = rq;
 468         nr_max_blk = min(scmdev->nr_max_block,
 469                          (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
 470
 471         blk_queue_logical_block_size(rq, 1 << 12);
 472         blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
 473         blk_queue_max_segments(rq, nr_max_blk);
 474         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
 475         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq);
 476
 477         bdev->gendisk = alloc_disk(SCM_NR_PARTS);
 478         if (!bdev->gendisk) {
 479                 ret = -ENOMEM;
 480                 goto out_queue;
 481         }
 482         rq->queuedata = scmdev;
 483         bdev->gendisk->private_data = scmdev;
 484         bdev->gendisk->fops = &scm_blk_devops;
 485         bdev->gendisk->queue = rq;
 486         bdev->gendisk->major = scm_major;
 487         bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
 488
 489         len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
 490         if (devindex > 25) {
 491                 len += snprintf(bdev->gendisk->disk_name + len,
 492                                 DISK_NAME_LEN - len, "%c",
 493                                 'a' + (devindex / 26) - 1);
 494                 devindex = devindex % 26;
 495         }
 496         snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
 497                  'a' + devindex);
 498
 499         /* 512 byte sectors */
 500         set_capacity(bdev->gendisk, scmdev->size >> 9);
 501         device_add_disk(&scmdev->dev, bdev->gendisk);
 502         return 0;
 503
 504 out_queue:
 505         blk_cleanup_queue(rq);
 506 out_tag:
 507         blk_mq_free_tag_set(&bdev->tag_set);
 508 out:
 509         atomic_dec(&nr_devices);
 510         return ret;
 511 }
 512
 513 void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
 514 {
 515         del_gendisk(bdev->gendisk);
 516         blk_cleanup_queue(bdev->gendisk->queue);
 517         blk_mq_free_tag_set(&bdev->tag_set);
 518         put_disk(bdev->gendisk);
 519 }
 520
 521 void scm_blk_set_available(struct scm_blk_dev *bdev)
 522 {
 523         unsigned long flags;
 524
 525         spin_lock_irqsave(&bdev->lock, flags);
 526         if (bdev->state == SCM_WR_PROHIBIT)
 527                 pr_info("%lx: Write access to the SCM increment is restored\n",
 528                         (unsigned long) bdev->scmdev->address);
 529         bdev->state = SCM_OPER;
 530         spin_unlock_irqrestore(&bdev->lock, flags);
 531 }
 532
 533 static bool __init scm_blk_params_valid(void)
 534 {
 535         if (!nr_requests_per_io || nr_requests_per_io > 64)
 536                 return false;
 537
 538         return true;
 539 }
 540
 541 static int __init scm_blk_init(void)
 542 {
 543         int ret = -EINVAL;
 544
 545         if (!scm_blk_params_valid())
 546                 goto out;
 547
 548         ret = register_blkdev(0, "scm");
 549         if (ret < 0)
 550                 goto out;
 551
 552         scm_major = ret;
 553         ret = scm_alloc_rqs(nr_requests);
 554         if (ret)
 555                 goto out_free;
 556
 557         scm_debug = debug_register("scm_log", 16, 1, 16);
 558         if (!scm_debug) {
 559                 ret = -ENOMEM;
 560                 goto out_free;
 561         }
 562
 563         debug_register_view(scm_debug, &debug_hex_ascii_view);
 564         debug_set_level(scm_debug, 2);
 565
 566         ret = scm_drv_init();
 567         if (ret)
 568                 goto out_dbf;
 569
 570         return ret;
 571
 572 out_dbf:
 573         debug_unregister(scm_debug);
 574 out_free:
 575         scm_free_rqs();
 576         unregister_blkdev(scm_major, "scm");
 577 out:
 578         return ret;
 579 }
 580 module_init(scm_blk_init);
 581
 582 static void __exit scm_blk_cleanup(void)
 583 {
 584         scm_drv_cleanup();
 585         debug_unregister(scm_debug);
 586         scm_free_rqs();
 587         unregister_blkdev(scm_major, "scm");
 588 }
 589 module_exit(scm_blk_cleanup);