block/blk-sysfs.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Functions related to sysfs handling
   4  */
   5 #include <linux/kernel.h>
   6 #include <linux/slab.h>
   7 #include <linux/module.h>
   8 #include <linux/bio.h>
   9 #include <linux/blkdev.h>
  10 #include <linux/backing-dev.h>
  11 #include <linux/blktrace_api.h>
  12 #include <linux/blk-mq.h>
  13 #include <linux/blk-cgroup.h>
  14
  15 #include "blk.h"
  16 #include "blk-mq.h"
  17 #include "blk-mq-debugfs.h"
  18 #include "blk-wbt.h"
  19
  20 struct queue_sysfs_entry {
  21         struct attribute attr;
  22         ssize_t (*show)(struct request_queue *, char *);
  23         ssize_t (*store)(struct request_queue *, const char *, size_t);
  24 };
  25
  26 static ssize_t
  27 queue_var_show(unsigned long var, char *page)
  28 {
  29         return sprintf(page, "%lu\n", var);
  30 }
  31
  32 static ssize_t
  33 queue_var_store(unsigned long *var, const char *page, size_t count)
  34 {
  35         int err;
  36         unsigned long v;
  37
  38         err = kstrtoul(page, 10, &v);
  39         if (err || v > UINT_MAX)
  40                 return -EINVAL;
  41
  42         *var = v;
  43
  44         return count;
  45 }
  46
  47 static ssize_t queue_var_store64(s64 *var, const char *page)
  48 {
  49         int err;
  50         s64 v;
  51
  52         err = kstrtos64(page, 10, &v);
  53         if (err < 0)
  54                 return err;
  55
  56         *var = v;
  57         return 0;
  58 }
  59
  60 static ssize_t queue_requests_show(struct request_queue *q, char *page)
  61 {
  62         return queue_var_show(q->nr_requests, (page));
  63 }
  64
  65 static ssize_t
  66 queue_requests_store(struct request_queue *q, const char *page, size_t count)
  67 {
  68         unsigned long nr;
  69         int ret, err;
  70
  71         if (!queue_is_mq(q))
  72                 return -EINVAL;
  73
  74         ret = queue_var_store(&nr, page, count);
  75         if (ret < 0)
  76                 return ret;
  77
  78         if (nr < BLKDEV_MIN_RQ)
  79                 nr = BLKDEV_MIN_RQ;
  80
  81         err = blk_mq_update_nr_requests(q, nr);
  82         if (err)
  83                 return err;
  84
  85         return ret;
  86 }
  87
  88 static ssize_t queue_ra_show(struct request_queue *q, char *page)
  89 {
  90         unsigned long ra_kb = q->backing_dev_info->ra_pages <<
  91                                         (PAGE_SHIFT - 10);
  92
  93         return queue_var_show(ra_kb, (page));
  94 }
  95
  96 static ssize_t
  97 queue_ra_store(struct request_queue *q, const char *page, size_t count)
  98 {
  99         unsigned long ra_kb;
 100         ssize_t ret = queue_var_store(&ra_kb, page, count);
 101
 102         if (ret < 0)
 103                 return ret;
 104
 105         q->backing_dev_info->ra_pages = ra_kb >> (PAGE_SHIFT - 10);
 106
 107         return ret;
 108 }
 109
 110 static ssize_t queue_max_sectors_show(struct request_queue *q, char *page)
 111 {
 112         int max_sectors_kb = queue_max_sectors(q) >> 1;
 113
 114         return queue_var_show(max_sectors_kb, (page));
 115 }
 116
 117 static ssize_t queue_max_segments_show(struct request_queue *q, char *page)
 118 {
 119         return queue_var_show(queue_max_segments(q), (page));
 120 }
 121
 122 static ssize_t queue_max_discard_segments_show(struct request_queue *q,
 123                 char *page)
 124 {
 125         return queue_var_show(queue_max_discard_segments(q), (page));
 126 }
 127
 128 static ssize_t queue_max_integrity_segments_show(struct request_queue *q, char *page)
 129 {
 130         return queue_var_show(q->limits.max_integrity_segments, (page));
 131 }
 132
 133 static ssize_t queue_max_segment_size_show(struct request_queue *q, char *page)
 134 {
 135         if (blk_queue_cluster(q))
 136                 return queue_var_show(queue_max_segment_size(q), (page));
 137
 138         return queue_var_show(PAGE_SIZE, (page));
 139 }
 140
 141 static ssize_t queue_logical_block_size_show(struct request_queue *q, char *page)
 142 {
 143         return queue_var_show(queue_logical_block_size(q), page);
 144 }
 145
 146 static ssize_t queue_physical_block_size_show(struct request_queue *q, char *page)
 147 {
 148         return queue_var_show(queue_physical_block_size(q), page);
 149 }
 150
 151 static ssize_t queue_chunk_sectors_show(struct request_queue *q, char *page)
 152 {
 153         return queue_var_show(q->limits.chunk_sectors, page);
 154 }
 155
 156 static ssize_t queue_io_min_show(struct request_queue *q, char *page)
 157 {
 158         return queue_var_show(queue_io_min(q), page);
 159 }
 160
 161 static ssize_t queue_io_opt_show(struct request_queue *q, char *page)
 162 {
 163         return queue_var_show(queue_io_opt(q), page);
 164 }
 165
 166 static ssize_t queue_discard_granularity_show(struct request_queue *q, char *page)
 167 {
 168         return queue_var_show(q->limits.discard_granularity, page);
 169 }
 170
 171 static ssize_t queue_discard_max_hw_show(struct request_queue *q, char *page)
 172 {
 173
 174         return sprintf(page, "%llu\n",
 175                 (unsigned long long)q->limits.max_hw_discard_sectors << 9);
 176 }
 177
 178 static ssize_t queue_discard_max_show(struct request_queue *q, char *page)
 179 {
 180         return sprintf(page, "%llu\n",
 181                        (unsigned long long)q->limits.max_discard_sectors << 9);
 182 }
 183
 184 static ssize_t queue_discard_max_store(struct request_queue *q,
 185                                        const char *page, size_t count)
 186 {
 187         unsigned long max_discard;
 188         ssize_t ret = queue_var_store(&max_discard, page, count);
 189
 190         if (ret < 0)
 191                 return ret;
 192
 193         if (max_discard & (q->limits.discard_granularity - 1))
 194                 return -EINVAL;
 195
 196         max_discard >>= 9;
 197         if (max_discard > UINT_MAX)
 198                 return -EINVAL;
 199
 200         if (max_discard > q->limits.max_hw_discard_sectors)
 201                 max_discard = q->limits.max_hw_discard_sectors;
 202
 203         q->limits.max_discard_sectors = max_discard;
 204         return ret;
 205 }
 206
 207 static ssize_t queue_discard_zeroes_data_show(struct request_queue *q, char *page)
 208 {
 209         return queue_var_show(0, page);
 210 }
 211
 212 static ssize_t queue_write_same_max_show(struct request_queue *q, char *page)
 213 {
 214         return sprintf(page, "%llu\n",
 215                 (unsigned long long)q->limits.max_write_same_sectors << 9);
 216 }
 217
 218 static ssize_t queue_write_zeroes_max_show(struct request_queue *q, char *page)
 219 {
 220         return sprintf(page, "%llu\n",
 221                 (unsigned long long)q->limits.max_write_zeroes_sectors << 9);
 222 }
 223
 224 static ssize_t
 225 queue_max_sectors_store(struct request_queue *q, const char *page, size_t count)
 226 {
 227         unsigned long max_sectors_kb,
 228                 max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1,
 229                         page_kb = 1 << (PAGE_SHIFT - 10);
 230         ssize_t ret = queue_var_store(&max_sectors_kb, page, count);
 231
 232         if (ret < 0)
 233                 return ret;
 234
 235         max_hw_sectors_kb = min_not_zero(max_hw_sectors_kb, (unsigned long)
 236                                          q->limits.max_dev_sectors >> 1);
 237
 238         if (max_sectors_kb > max_hw_sectors_kb || max_sectors_kb < page_kb)
 239                 return -EINVAL;
 240
 241         spin_lock_irq(&q->queue_lock);
 242         q->limits.max_sectors = max_sectors_kb << 1;
 243         q->backing_dev_info->io_pages = max_sectors_kb >> (PAGE_SHIFT - 10);
 244         spin_unlock_irq(&q->queue_lock);
 245
 246         return ret;
 247 }
 248
 249 static ssize_t queue_max_hw_sectors_show(struct request_queue *q, char *page)
 250 {
 251         int max_hw_sectors_kb = queue_max_hw_sectors(q) >> 1;
 252
 253         return queue_var_show(max_hw_sectors_kb, (page));
 254 }
 255
 256 #define QUEUE_SYSFS_BIT_FNS(name, flag, neg)                            \
 257 static ssize_t                                                          \
 258 queue_show_##name(struct request_queue *q, char *page)                  \
 259 {                                                                       \
 260         int bit;                                                        \
 261         bit = test_bit(QUEUE_FLAG_##flag, &q->queue_flags);             \
 262         return queue_var_show(neg ? !bit : bit, page);                  \
 263 }                                                                       \
 264 static ssize_t                                                          \
 265 queue_store_##name(struct request_queue *q, const char *page, size_t count) \
 266 {                                                                       \
 267         unsigned long val;                                              \
 268         ssize_t ret;                                                    \
 269         ret = queue_var_store(&val, page, count);                       \
 270         if (ret < 0)                                                    \
 271                  return ret;                                            \
 272         if (neg)                                                        \
 273                 val = !val;                                             \
 274                                                                         \
 275         if (val)                                                        \
 276                 blk_queue_flag_set(QUEUE_FLAG_##flag, q);               \
 277         else                                                            \
 278                 blk_queue_flag_clear(QUEUE_FLAG_##flag, q);             \
 279         return ret;                                                     \
 280 }
 281
 282 QUEUE_SYSFS_BIT_FNS(nonrot, NONROT, 1);
 283 QUEUE_SYSFS_BIT_FNS(random, ADD_RANDOM, 0);
 284 QUEUE_SYSFS_BIT_FNS(iostats, IO_STAT, 0);
 285 #undef QUEUE_SYSFS_BIT_FNS
 286
 287 static ssize_t queue_zoned_show(struct request_queue *q, char *page)
 288 {
 289         switch (blk_queue_zoned_model(q)) {
 290         case BLK_ZONED_HA:
 291                 return sprintf(page, "host-aware\n");
 292         case BLK_ZONED_HM:
 293                 return sprintf(page, "host-managed\n");
 294         default:
 295                 return sprintf(page, "none\n");
 296         }
 297 }
 298
 299 static ssize_t queue_nr_zones_show(struct request_queue *q, char *page)
 300 {
 301         return queue_var_show(blk_queue_nr_zones(q), page);
 302 }
 303
 304 static ssize_t queue_nomerges_show(struct request_queue *q, char *page)
 305 {
 306         return queue_var_show((blk_queue_nomerges(q) << 1) |
 307                                blk_queue_noxmerges(q), page);
 308 }
 309
 310 static ssize_t queue_nomerges_store(struct request_queue *q, const char *page,
 311                                     size_t count)
 312 {
 313         unsigned long nm;
 314         ssize_t ret = queue_var_store(&nm, page, count);
 315
 316         if (ret < 0)
 317                 return ret;
 318
 319         blk_queue_flag_clear(QUEUE_FLAG_NOMERGES, q);
 320         blk_queue_flag_clear(QUEUE_FLAG_NOXMERGES, q);
 321         if (nm == 2)
 322                 blk_queue_flag_set(QUEUE_FLAG_NOMERGES, q);
 323         else if (nm)
 324                 blk_queue_flag_set(QUEUE_FLAG_NOXMERGES, q);
 325
 326         return ret;
 327 }
 328
 329 static ssize_t queue_rq_affinity_show(struct request_queue *q, char *page)
 330 {
 331         bool set = test_bit(QUEUE_FLAG_SAME_COMP, &q->queue_flags);
 332         bool force = test_bit(QUEUE_FLAG_SAME_FORCE, &q->queue_flags);
 333
 334         return queue_var_show(set << force, page);
 335 }
 336
 337 static ssize_t
 338 queue_rq_affinity_store(struct request_queue *q, const char *page, size_t count)
 339 {
 340         ssize_t ret = -EINVAL;
 341 #ifdef CONFIG_SMP
 342         unsigned long val;
 343
 344         ret = queue_var_store(&val, page, count);
 345         if (ret < 0)
 346                 return ret;
 347
 348         if (val == 2) {
 349                 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
 350                 blk_queue_flag_set(QUEUE_FLAG_SAME_FORCE, q);
 351         } else if (val == 1) {
 352                 blk_queue_flag_set(QUEUE_FLAG_SAME_COMP, q);
 353                 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
 354         } else if (val == 0) {
 355                 blk_queue_flag_clear(QUEUE_FLAG_SAME_COMP, q);
 356                 blk_queue_flag_clear(QUEUE_FLAG_SAME_FORCE, q);
 357         }
 358 #endif
 359         return ret;
 360 }
 361
 362 static ssize_t queue_poll_delay_show(struct request_queue *q, char *page)
 363 {
 364         int val;
 365
 366         if (q->poll_nsec == -1)
 367                 val = -1;
 368         else
 369                 val = q->poll_nsec / 1000;
 370
 371         return sprintf(page, "%d\n", val);
 372 }
 373
 374 static ssize_t queue_poll_delay_store(struct request_queue *q, const char *page,
 375                                 size_t count)
 376 {
 377         int err, val;
 378
 379         if (!q->mq_ops || !q->mq_ops->poll)
 380                 return -EINVAL;
 381
 382         err = kstrtoint(page, 10, &val);
 383         if (err < 0)
 384                 return err;
 385
 386         if (val == -1)
 387                 q->poll_nsec = -1;
 388         else
 389                 q->poll_nsec = val * 1000;
 390
 391         return count;
 392 }
 393
 394 static ssize_t queue_poll_show(struct request_queue *q, char *page)
 395 {
 396         return queue_var_show(test_bit(QUEUE_FLAG_POLL, &q->queue_flags), page);
 397 }
 398
 399 static ssize_t queue_poll_store(struct request_queue *q, const char *page,
 400                                 size_t count)
 401 {
 402         unsigned long poll_on;
 403         ssize_t ret;
 404
 405         if (!q->tag_set || q->tag_set->nr_maps <= HCTX_TYPE_POLL)
 406                 return -EINVAL;
 407
 408         ret = queue_var_store(&poll_on, page, count);
 409         if (ret < 0)
 410                 return ret;
 411
 412         if (poll_on)
 413                 blk_queue_flag_set(QUEUE_FLAG_POLL, q);
 414         else
 415                 blk_queue_flag_clear(QUEUE_FLAG_POLL, q);
 416
 417         return ret;
 418 }
 419
 420 static ssize_t queue_io_timeout_show(struct request_queue *q, char *page)
 421 {
 422         return sprintf(page, "%u\n", jiffies_to_msecs(q->rq_timeout));
 423 }
 424
 425 static ssize_t queue_io_timeout_store(struct request_queue *q, const char *page,
 426                                   size_t count)
 427 {
 428         unsigned int val;
 429         int err;
 430
 431         err = kstrtou32(page, 10, &val);
 432         if (err || val == 0)
 433                 return -EINVAL;
 434
 435         blk_queue_rq_timeout(q, msecs_to_jiffies(val));
 436
 437         return count;
 438 }
 439
 440 static ssize_t queue_wb_lat_show(struct request_queue *q, char *page)
 441 {
 442         if (!wbt_rq_qos(q))
 443                 return -EINVAL;
 444
 445         return sprintf(page, "%llu\n", div_u64(wbt_get_min_lat(q), 1000));
 446 }
 447
 448 static ssize_t queue_wb_lat_store(struct request_queue *q, const char *page,
 449                                   size_t count)
 450 {
 451         struct rq_qos *rqos;
 452         ssize_t ret;
 453         s64 val;
 454
 455         ret = queue_var_store64(&val, page);
 456         if (ret < 0)
 457                 return ret;
 458         if (val < -1)
 459                 return -EINVAL;
 460
 461         rqos = wbt_rq_qos(q);
 462         if (!rqos) {
 463                 ret = wbt_init(q);
 464                 if (ret)
 465                         return ret;
 466         }
 467
 468         if (val == -1)
 469                 val = wbt_default_latency_nsec(q);
 470         else if (val >= 0)
 471                 val *= 1000ULL;
 472
 473         /*
 474          * Ensure that the queue is idled, in case the latency update
 475          * ends up either enabling or disabling wbt completely. We can't
 476          * have IO inflight if that happens.
 477          */
 478         blk_mq_freeze_queue(q);
 479         blk_mq_quiesce_queue(q);
 480
 481         wbt_set_min_lat(q, val);
 482         wbt_update_limits(q);
 483
 484         blk_mq_unquiesce_queue(q);
 485         blk_mq_unfreeze_queue(q);
 486
 487         return count;
 488 }
 489
 490 static ssize_t queue_wc_show(struct request_queue *q, char *page)
 491 {
 492         if (test_bit(QUEUE_FLAG_WC, &q->queue_flags))
 493                 return sprintf(page, "write back\n");
 494
 495         return sprintf(page, "write through\n");
 496 }
 497
 498 static ssize_t queue_wc_store(struct request_queue *q, const char *page,
 499                               size_t count)
 500 {
 501         int set = -1;
 502
 503         if (!strncmp(page, "write back", 10))
 504                 set = 1;
 505         else if (!strncmp(page, "write through", 13) ||
 506                  !strncmp(page, "none", 4))
 507                 set = 0;
 508
 509         if (set == -1)
 510                 return -EINVAL;
 511
 512         if (set)
 513                 blk_queue_flag_set(QUEUE_FLAG_WC, q);
 514         else
 515                 blk_queue_flag_clear(QUEUE_FLAG_WC, q);
 516
 517         return count;
 518 }
 519
 520 static ssize_t queue_fua_show(struct request_queue *q, char *page)
 521 {
 522         return sprintf(page, "%u\n", test_bit(QUEUE_FLAG_FUA, &q->queue_flags));
 523 }
 524
 525 static ssize_t queue_dax_show(struct request_queue *q, char *page)
 526 {
 527         return queue_var_show(blk_queue_dax(q), page);
 528 }
 529
 530 static struct queue_sysfs_entry queue_requests_entry = {
 531         .attr = {.name = "nr_requests", .mode = 0644 },
 532         .show = queue_requests_show,
 533         .store = queue_requests_store,
 534 };
 535
 536 static struct queue_sysfs_entry queue_ra_entry = {
 537         .attr = {.name = "read_ahead_kb", .mode = 0644 },
 538         .show = queue_ra_show,
 539         .store = queue_ra_store,
 540 };
 541
 542 static struct queue_sysfs_entry queue_max_sectors_entry = {
 543         .attr = {.name = "max_sectors_kb", .mode = 0644 },
 544         .show = queue_max_sectors_show,
 545         .store = queue_max_sectors_store,
 546 };
 547
 548 static struct queue_sysfs_entry queue_max_hw_sectors_entry = {
 549         .attr = {.name = "max_hw_sectors_kb", .mode = 0444 },
 550         .show = queue_max_hw_sectors_show,
 551 };
 552
 553 static struct queue_sysfs_entry queue_max_segments_entry = {
 554         .attr = {.name = "max_segments", .mode = 0444 },
 555         .show = queue_max_segments_show,
 556 };
 557
 558 static struct queue_sysfs_entry queue_max_discard_segments_entry = {
 559         .attr = {.name = "max_discard_segments", .mode = 0444 },
 560         .show = queue_max_discard_segments_show,
 561 };
 562
 563 static struct queue_sysfs_entry queue_max_integrity_segments_entry = {
 564         .attr = {.name = "max_integrity_segments", .mode = 0444 },
 565         .show = queue_max_integrity_segments_show,
 566 };
 567
 568 static struct queue_sysfs_entry queue_max_segment_size_entry = {
 569         .attr = {.name = "max_segment_size", .mode = 0444 },
 570         .show = queue_max_segment_size_show,
 571 };
 572
 573 static struct queue_sysfs_entry queue_iosched_entry = {
 574         .attr = {.name = "scheduler", .mode = 0644 },
 575         .show = elv_iosched_show,
 576         .store = elv_iosched_store,
 577 };
 578
 579 static struct queue_sysfs_entry queue_hw_sector_size_entry = {
 580         .attr = {.name = "hw_sector_size", .mode = 0444 },
 581         .show = queue_logical_block_size_show,
 582 };
 583
 584 static struct queue_sysfs_entry queue_logical_block_size_entry = {
 585         .attr = {.name = "logical_block_size", .mode = 0444 },
 586         .show = queue_logical_block_size_show,
 587 };
 588
 589 static struct queue_sysfs_entry queue_physical_block_size_entry = {
 590         .attr = {.name = "physical_block_size", .mode = 0444 },
 591         .show = queue_physical_block_size_show,
 592 };
 593
 594 static struct queue_sysfs_entry queue_chunk_sectors_entry = {
 595         .attr = {.name = "chunk_sectors", .mode = 0444 },
 596         .show = queue_chunk_sectors_show,
 597 };
 598
 599 static struct queue_sysfs_entry queue_io_min_entry = {
 600         .attr = {.name = "minimum_io_size", .mode = 0444 },
 601         .show = queue_io_min_show,
 602 };
 603
 604 static struct queue_sysfs_entry queue_io_opt_entry = {
 605         .attr = {.name = "optimal_io_size", .mode = 0444 },
 606         .show = queue_io_opt_show,
 607 };
 608
 609 static struct queue_sysfs_entry queue_discard_granularity_entry = {
 610         .attr = {.name = "discard_granularity", .mode = 0444 },
 611         .show = queue_discard_granularity_show,
 612 };
 613
 614 static struct queue_sysfs_entry queue_discard_max_hw_entry = {
 615         .attr = {.name = "discard_max_hw_bytes", .mode = 0444 },
 616         .show = queue_discard_max_hw_show,
 617 };
 618
 619 static struct queue_sysfs_entry queue_discard_max_entry = {
 620         .attr = {.name = "discard_max_bytes", .mode = 0644 },
 621         .show = queue_discard_max_show,
 622         .store = queue_discard_max_store,
 623 };
 624
 625 static struct queue_sysfs_entry queue_discard_zeroes_data_entry = {
 626         .attr = {.name = "discard_zeroes_data", .mode = 0444 },
 627         .show = queue_discard_zeroes_data_show,
 628 };
 629
 630 static struct queue_sysfs_entry queue_write_same_max_entry = {
 631         .attr = {.name = "write_same_max_bytes", .mode = 0444 },
 632         .show = queue_write_same_max_show,
 633 };
 634
 635 static struct queue_sysfs_entry queue_write_zeroes_max_entry = {
 636         .attr = {.name = "write_zeroes_max_bytes", .mode = 0444 },
 637         .show = queue_write_zeroes_max_show,
 638 };
 639
 640 static struct queue_sysfs_entry queue_nonrot_entry = {
 641         .attr = {.name = "rotational", .mode = 0644 },
 642         .show = queue_show_nonrot,
 643         .store = queue_store_nonrot,
 644 };
 645
 646 static struct queue_sysfs_entry queue_zoned_entry = {
 647         .attr = {.name = "zoned", .mode = 0444 },
 648         .show = queue_zoned_show,
 649 };
 650
 651 static struct queue_sysfs_entry queue_nr_zones_entry = {
 652         .attr = {.name = "nr_zones", .mode = 0444 },
 653         .show = queue_nr_zones_show,
 654 };
 655
 656 static struct queue_sysfs_entry queue_nomerges_entry = {
 657         .attr = {.name = "nomerges", .mode = 0644 },
 658         .show = queue_nomerges_show,
 659         .store = queue_nomerges_store,
 660 };
 661
 662 static struct queue_sysfs_entry queue_rq_affinity_entry = {
 663         .attr = {.name = "rq_affinity", .mode = 0644 },
 664         .show = queue_rq_affinity_show,
 665         .store = queue_rq_affinity_store,
 666 };
 667
 668 static struct queue_sysfs_entry queue_iostats_entry = {
 669         .attr = {.name = "iostats", .mode = 0644 },
 670         .show = queue_show_iostats,
 671         .store = queue_store_iostats,
 672 };
 673
 674 static struct queue_sysfs_entry queue_random_entry = {
 675         .attr = {.name = "add_random", .mode = 0644 },
 676         .show = queue_show_random,
 677         .store = queue_store_random,
 678 };
 679
 680 static struct queue_sysfs_entry queue_poll_entry = {
 681         .attr = {.name = "io_poll", .mode = 0644 },
 682         .show = queue_poll_show,
 683         .store = queue_poll_store,
 684 };
 685
 686 static struct queue_sysfs_entry queue_poll_delay_entry = {
 687         .attr = {.name = "io_poll_delay", .mode = 0644 },
 688         .show = queue_poll_delay_show,
 689         .store = queue_poll_delay_store,
 690 };
 691
 692 static struct queue_sysfs_entry queue_wc_entry = {
 693         .attr = {.name = "write_cache", .mode = 0644 },
 694         .show = queue_wc_show,
 695         .store = queue_wc_store,
 696 };
 697
 698 static struct queue_sysfs_entry queue_fua_entry = {
 699         .attr = {.name = "fua", .mode = 0444 },
 700         .show = queue_fua_show,
 701 };
 702
 703 static struct queue_sysfs_entry queue_dax_entry = {
 704         .attr = {.name = "dax", .mode = 0444 },
 705         .show = queue_dax_show,
 706 };
 707
 708 static struct queue_sysfs_entry queue_io_timeout_entry = {
 709         .attr = {.name = "io_timeout", .mode = 0644 },
 710         .show = queue_io_timeout_show,
 711         .store = queue_io_timeout_store,
 712 };
 713
 714 static struct queue_sysfs_entry queue_wb_lat_entry = {
 715         .attr = {.name = "wbt_lat_usec", .mode = 0644 },
 716         .show = queue_wb_lat_show,
 717         .store = queue_wb_lat_store,
 718 };
 719
 720 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
 721 static struct queue_sysfs_entry throtl_sample_time_entry = {
 722         .attr = {.name = "throttle_sample_time", .mode = 0644 },
 723         .show = blk_throtl_sample_time_show,
 724         .store = blk_throtl_sample_time_store,
 725 };
 726 #endif
 727
 728 static struct attribute *default_attrs[] = {
 729         &queue_requests_entry.attr,
 730         &queue_ra_entry.attr,
 731         &queue_max_hw_sectors_entry.attr,
 732         &queue_max_sectors_entry.attr,
 733         &queue_max_segments_entry.attr,
 734         &queue_max_discard_segments_entry.attr,
 735         &queue_max_integrity_segments_entry.attr,
 736         &queue_max_segment_size_entry.attr,
 737         &queue_iosched_entry.attr,
 738         &queue_hw_sector_size_entry.attr,
 739         &queue_logical_block_size_entry.attr,
 740         &queue_physical_block_size_entry.attr,
 741         &queue_chunk_sectors_entry.attr,
 742         &queue_io_min_entry.attr,
 743         &queue_io_opt_entry.attr,
 744         &queue_discard_granularity_entry.attr,
 745         &queue_discard_max_entry.attr,
 746         &queue_discard_max_hw_entry.attr,
 747         &queue_discard_zeroes_data_entry.attr,
 748         &queue_write_same_max_entry.attr,
 749         &queue_write_zeroes_max_entry.attr,
 750         &queue_nonrot_entry.attr,
 751         &queue_zoned_entry.attr,
 752         &queue_nr_zones_entry.attr,
 753         &queue_nomerges_entry.attr,
 754         &queue_rq_affinity_entry.attr,
 755         &queue_iostats_entry.attr,
 756         &queue_random_entry.attr,
 757         &queue_poll_entry.attr,
 758         &queue_wc_entry.attr,
 759         &queue_fua_entry.attr,
 760         &queue_dax_entry.attr,
 761         &queue_wb_lat_entry.attr,
 762         &queue_poll_delay_entry.attr,
 763         &queue_io_timeout_entry.attr,
 764 #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
 765         &throtl_sample_time_entry.attr,
 766 #endif
 767         NULL,
 768 };
 769
 770 #define to_queue(atr) container_of((atr), struct queue_sysfs_entry, attr)
 771
 772 static ssize_t
 773 queue_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
 774 {
 775         struct queue_sysfs_entry *entry = to_queue(attr);
 776         struct request_queue *q =
 777                 container_of(kobj, struct request_queue, kobj);
 778         ssize_t res;
 779
 780         if (!entry->show)
 781                 return -EIO;
 782         mutex_lock(&q->sysfs_lock);
 783         if (blk_queue_dying(q)) {
 784                 mutex_unlock(&q->sysfs_lock);
 785                 return -ENOENT;
 786         }
 787         res = entry->show(q, page);
 788         mutex_unlock(&q->sysfs_lock);
 789         return res;
 790 }
 791
 792 static ssize_t
 793 queue_attr_store(struct kobject *kobj, struct attribute *attr,
 794                     const char *page, size_t length)
 795 {
 796         struct queue_sysfs_entry *entry = to_queue(attr);
 797         struct request_queue *q;
 798         ssize_t res;
 799
 800         if (!entry->store)
 801                 return -EIO;
 802
 803         q = container_of(kobj, struct request_queue, kobj);
 804         mutex_lock(&q->sysfs_lock);
 805         if (blk_queue_dying(q)) {
 806                 mutex_unlock(&q->sysfs_lock);
 807                 return -ENOENT;
 808         }
 809         res = entry->store(q, page, length);
 810         mutex_unlock(&q->sysfs_lock);
 811         return res;
 812 }
 813
 814 static void blk_free_queue_rcu(struct rcu_head *rcu_head)
 815 {
 816         struct request_queue *q = container_of(rcu_head, struct request_queue,
 817                                                rcu_head);
 818         kmem_cache_free(blk_requestq_cachep, q);
 819 }
 820
 821 /**
 822  * __blk_release_queue - release a request queue when it is no longer needed
 823  * @work: pointer to the release_work member of the request queue to be released
 824  *
 825  * Description:
 826  *     blk_release_queue is the counterpart of blk_init_queue(). It should be
 827  *     called when a request queue is being released; typically when a block
 828  *     device is being de-registered. Its primary task it to free the queue
 829  *     itself.
 830  *
 831  * Notes:
 832  *     The low level driver must have finished any outstanding requests first
 833  *     via blk_cleanup_queue().
 834  *
 835  *     Although blk_release_queue() may be called with preemption disabled,
 836  *     __blk_release_queue() may sleep.
 837  */
 838 static void __blk_release_queue(struct work_struct *work)
 839 {
 840         struct request_queue *q = container_of(work, typeof(*q), release_work);
 841
 842         if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
 843                 blk_stat_remove_callback(q, q->poll_cb);
 844         blk_stat_free_callback(q->poll_cb);
 845
 846         if (!blk_queue_dead(q)) {
 847                 /*
 848                  * Last reference was dropped without having called
 849                  * blk_cleanup_queue().
 850                  */
 851                 WARN_ONCE(blk_queue_init_done(q),
 852                           "request queue %p has been registered but blk_cleanup_queue() has not been called for that queue\n",
 853                           q);
 854                 blk_exit_queue(q);
 855         }
 856
 857         WARN(blk_queue_root_blkg(q),
 858              "request queue %p is being released but it has not yet been removed from the blkcg controller\n",
 859              q);
 860
 861         blk_free_queue_stats(q->stats);
 862
 863         blk_queue_free_zone_bitmaps(q);
 864
 865         if (queue_is_mq(q))
 866                 blk_mq_release(q);
 867
 868         blk_trace_shutdown(q);
 869
 870         if (queue_is_mq(q))
 871                 blk_mq_debugfs_unregister(q);
 872
 873         bioset_exit(&q->bio_split);
 874
 875         ida_simple_remove(&blk_queue_ida, q->id);
 876         call_rcu(&q->rcu_head, blk_free_queue_rcu);
 877 }
 878
 879 static void blk_release_queue(struct kobject *kobj)
 880 {
 881         struct request_queue *q =
 882                 container_of(kobj, struct request_queue, kobj);
 883
 884         INIT_WORK(&q->release_work, __blk_release_queue);
 885         schedule_work(&q->release_work);
 886 }
 887
 888 static const struct sysfs_ops queue_sysfs_ops = {
 889         .show   = queue_attr_show,
 890         .store  = queue_attr_store,
 891 };
 892
 893 struct kobj_type blk_queue_ktype = {
 894         .sysfs_ops      = &queue_sysfs_ops,
 895         .default_attrs  = default_attrs,
 896         .release        = blk_release_queue,
 897 };
 898
 899 /**
 900  * blk_register_queue - register a block layer queue with sysfs
 901  * @disk: Disk of which the request queue should be registered with sysfs.
 902  */
 903 int blk_register_queue(struct gendisk *disk)
 904 {
 905         int ret;
 906         struct device *dev = disk_to_dev(disk);
 907         struct request_queue *q = disk->queue;
 908
 909         if (WARN_ON(!q))
 910                 return -ENXIO;
 911
 912         WARN_ONCE(test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags),
 913                   "%s is registering an already registered queue\n",
 914                   kobject_name(&dev->kobj));
 915         blk_queue_flag_set(QUEUE_FLAG_REGISTERED, q);
 916
 917         /*
 918          * SCSI probing may synchronously create and destroy a lot of
 919          * request_queues for non-existent devices.  Shutting down a fully
 920          * functional queue takes measureable wallclock time as RCU grace
 921          * periods are involved.  To avoid excessive latency in these
 922          * cases, a request_queue starts out in a degraded mode which is
 923          * faster to shut down and is made fully functional here as
 924          * request_queues for non-existent devices never get registered.
 925          */
 926         if (!blk_queue_init_done(q)) {
 927                 blk_queue_flag_set(QUEUE_FLAG_INIT_DONE, q);
 928                 percpu_ref_switch_to_percpu(&q->q_usage_counter);
 929         }
 930
 931         ret = blk_trace_init_sysfs(dev);
 932         if (ret)
 933                 return ret;
 934
 935         /* Prevent changes through sysfs until registration is completed. */
 936         mutex_lock(&q->sysfs_lock);
 937
 938         ret = kobject_add(&q->kobj, kobject_get(&dev->kobj), "%s", "queue");
 939         if (ret < 0) {
 940                 blk_trace_remove_sysfs(dev);
 941                 goto unlock;
 942         }
 943
 944         if (queue_is_mq(q)) {
 945                 __blk_mq_register_dev(dev, q);
 946                 blk_mq_debugfs_register(q);
 947         }
 948
 949         kobject_uevent(&q->kobj, KOBJ_ADD);
 950
 951         wbt_enable_default(q);
 952
 953         blk_throtl_register_queue(q);
 954
 955         if (q->elevator) {
 956                 ret = elv_register_queue(q);
 957                 if (ret) {
 958                         mutex_unlock(&q->sysfs_lock);
 959                         kobject_uevent(&q->kobj, KOBJ_REMOVE);
 960                         kobject_del(&q->kobj);
 961                         blk_trace_remove_sysfs(dev);
 962                         kobject_put(&dev->kobj);
 963                         return ret;
 964                 }
 965         }
 966         ret = 0;
 967 unlock:
 968         mutex_unlock(&q->sysfs_lock);
 969         return ret;
 970 }
 971 EXPORT_SYMBOL_GPL(blk_register_queue);
 972
 973 /**
 974  * blk_unregister_queue - counterpart of blk_register_queue()
 975  * @disk: Disk of which the request queue should be unregistered from sysfs.
 976  *
 977  * Note: the caller is responsible for guaranteeing that this function is called
 978  * after blk_register_queue() has finished.
 979  */
 980 void blk_unregister_queue(struct gendisk *disk)
 981 {
 982         struct request_queue *q = disk->queue;
 983
 984         if (WARN_ON(!q))
 985                 return;
 986
 987         /* Return early if disk->queue was never registered. */
 988         if (!test_bit(QUEUE_FLAG_REGISTERED, &q->queue_flags))
 989                 return;
 990
 991         /*
 992          * Since sysfs_remove_dir() prevents adding new directory entries
 993          * before removal of existing entries starts, protect against
 994          * concurrent elv_iosched_store() calls.
 995          */
 996         mutex_lock(&q->sysfs_lock);
 997
 998         blk_queue_flag_clear(QUEUE_FLAG_REGISTERED, q);
 999
1000         /*
1001          * Remove the sysfs attributes before unregistering the queue data
1002          * structures that can be modified through sysfs.
1003          */
1004         if (queue_is_mq(q))
1005                 blk_mq_unregister_dev(disk_to_dev(disk), q);
1006         mutex_unlock(&q->sysfs_lock);
1007
1008         kobject_uevent(&q->kobj, KOBJ_REMOVE);
1009         kobject_del(&q->kobj);
1010         blk_trace_remove_sysfs(disk_to_dev(disk));
1011
1012         mutex_lock(&q->sysfs_lock);
1013         if (q->elevator)
1014                 elv_unregister_queue(q);
1015         mutex_unlock(&q->sysfs_lock);
1016
1017         kobject_put(&disk_to_dev(disk)->kobj);
1018 }