block/blk-stat.c

   1 /*
   2  * Block stat tracking code
   3  *
   4  * Copyright (C) 2016 Jens Axboe
   5  */
   6 #include <linux/kernel.h>
   7 #include <linux/blk-mq.h>
   8
   9 #include "blk-stat.h"
  10 #include "blk-mq.h"
  11
  12 static void blk_stat_flush_batch(struct blk_rq_stat *stat)
  13 {
  14         const s32 nr_batch = READ_ONCE(stat->nr_batch);
  15         const s32 nr_samples = READ_ONCE(stat->nr_samples);
  16
  17         if (!nr_batch)
  18                 return;
  19         if (!nr_samples)
  20                 stat->mean = div64_s64(stat->batch, nr_batch);
  21         else {
  22                 stat->mean = div64_s64((stat->mean * nr_samples) +
  23                                         stat->batch,
  24                                         nr_batch + nr_samples);
  25         }
  26
  27         stat->nr_samples += nr_batch;
  28         stat->nr_batch = stat->batch = 0;
  29 }
  30
  31 static void blk_stat_sum(struct blk_rq_stat *dst, struct blk_rq_stat *src)
  32 {
  33         blk_stat_flush_batch(src);
  34
  35         if (!src->nr_samples)
  36                 return;
  37
  38         dst->min = min(dst->min, src->min);
  39         dst->max = max(dst->max, src->max);
  40
  41         if (!dst->nr_samples)
  42                 dst->mean = src->mean;
  43         else {
  44                 dst->mean = div64_s64((src->mean * src->nr_samples) +
  45                                         (dst->mean * dst->nr_samples),
  46                                         dst->nr_samples + src->nr_samples);
  47         }
  48         dst->nr_samples += src->nr_samples;
  49 }
  50
  51 static void blk_mq_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
  52 {
  53         struct blk_mq_hw_ctx *hctx;
  54         struct blk_mq_ctx *ctx;
  55         uint64_t latest = 0;
  56         int i, j, nr;
  57
  58         blk_stat_init(&dst[BLK_STAT_READ]);
  59         blk_stat_init(&dst[BLK_STAT_WRITE]);
  60
  61         nr = 0;
  62         do {
  63                 uint64_t newest = 0;
  64
  65                 queue_for_each_hw_ctx(q, hctx, i) {
  66                         hctx_for_each_ctx(hctx, ctx, j) {
  67                                 blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
  68                                 blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
  69
  70                                 if (!ctx->stat[BLK_STAT_READ].nr_samples &&
  71                                     !ctx->stat[BLK_STAT_WRITE].nr_samples)
  72                                         continue;
  73                                 if (ctx->stat[BLK_STAT_READ].time > newest)
  74                                         newest = ctx->stat[BLK_STAT_READ].time;
  75                                 if (ctx->stat[BLK_STAT_WRITE].time > newest)
  76                                         newest = ctx->stat[BLK_STAT_WRITE].time;
  77                         }
  78                 }
  79
  80                 /*
  81                  * No samples
  82                  */
  83                 if (!newest)
  84                         break;
  85
  86                 if (newest > latest)
  87                         latest = newest;
  88
  89                 queue_for_each_hw_ctx(q, hctx, i) {
  90                         hctx_for_each_ctx(hctx, ctx, j) {
  91                                 if (ctx->stat[BLK_STAT_READ].time == newest) {
  92                                         blk_stat_sum(&dst[BLK_STAT_READ],
  93                                                      &ctx->stat[BLK_STAT_READ]);
  94                                         nr++;
  95                                 }
  96                                 if (ctx->stat[BLK_STAT_WRITE].time == newest) {
  97                                         blk_stat_sum(&dst[BLK_STAT_WRITE],
  98                                                      &ctx->stat[BLK_STAT_WRITE]);
  99                                         nr++;
 100                                 }
 101                         }
 102                 }
 103                 /*
 104                  * If we race on finding an entry, just loop back again.
 105                  * Should be very rare.
 106                  */
 107         } while (!nr);
 108
 109         dst[BLK_STAT_READ].time = dst[BLK_STAT_WRITE].time = latest;
 110 }
 111
 112 void blk_queue_stat_get(struct request_queue *q, struct blk_rq_stat *dst)
 113 {
 114         if (q->mq_ops)
 115                 blk_mq_stat_get(q, dst);
 116         else {
 117                 blk_stat_flush_batch(&q->rq_stats[BLK_STAT_READ]);
 118                 blk_stat_flush_batch(&q->rq_stats[BLK_STAT_WRITE]);
 119                 memcpy(&dst[BLK_STAT_READ], &q->rq_stats[BLK_STAT_READ],
 120                                 sizeof(struct blk_rq_stat));
 121                 memcpy(&dst[BLK_STAT_WRITE], &q->rq_stats[BLK_STAT_WRITE],
 122                                 sizeof(struct blk_rq_stat));
 123         }
 124 }
 125
 126 void blk_hctx_stat_get(struct blk_mq_hw_ctx *hctx, struct blk_rq_stat *dst)
 127 {
 128         struct blk_mq_ctx *ctx;
 129         unsigned int i, nr;
 130
 131         nr = 0;
 132         do {
 133                 uint64_t newest = 0;
 134
 135                 hctx_for_each_ctx(hctx, ctx, i) {
 136                         blk_stat_flush_batch(&ctx->stat[BLK_STAT_READ]);
 137                         blk_stat_flush_batch(&ctx->stat[BLK_STAT_WRITE]);
 138
 139                         if (!ctx->stat[BLK_STAT_READ].nr_samples &&
 140                             !ctx->stat[BLK_STAT_WRITE].nr_samples)
 141                                 continue;
 142
 143                         if (ctx->stat[BLK_STAT_READ].time > newest)
 144                                 newest = ctx->stat[BLK_STAT_READ].time;
 145                         if (ctx->stat[BLK_STAT_WRITE].time > newest)
 146                                 newest = ctx->stat[BLK_STAT_WRITE].time;
 147                 }
 148
 149                 if (!newest)
 150                         break;
 151
 152                 hctx_for_each_ctx(hctx, ctx, i) {
 153                         if (ctx->stat[BLK_STAT_READ].time == newest) {
 154                                 blk_stat_sum(&dst[BLK_STAT_READ],
 155                                                 &ctx->stat[BLK_STAT_READ]);
 156                                 nr++;
 157                         }
 158                         if (ctx->stat[BLK_STAT_WRITE].time == newest) {
 159                                 blk_stat_sum(&dst[BLK_STAT_WRITE],
 160                                                 &ctx->stat[BLK_STAT_WRITE]);
 161                                 nr++;
 162                         }
 163                 }
 164                 /*
 165                  * If we race on finding an entry, just loop back again.
 166                  * Should be very rare, as the window is only updated
 167                  * occasionally
 168                  */
 169         } while (!nr);
 170 }
 171
 172 static void __blk_stat_init(struct blk_rq_stat *stat, s64 time_now)
 173 {
 174         stat->min = -1ULL;
 175         stat->max = stat->nr_samples = stat->mean = 0;
 176         stat->batch = stat->nr_batch = 0;
 177         stat->time = time_now & BLK_STAT_NSEC_MASK;
 178 }
 179
 180 void blk_stat_init(struct blk_rq_stat *stat)
 181 {
 182         __blk_stat_init(stat, ktime_to_ns(ktime_get()));
 183 }
 184
 185 static bool __blk_stat_is_current(struct blk_rq_stat *stat, s64 now)
 186 {
 187         return (now & BLK_STAT_NSEC_MASK) == (stat->time & BLK_STAT_NSEC_MASK);
 188 }
 189
 190 bool blk_stat_is_current(struct blk_rq_stat *stat)
 191 {
 192         return __blk_stat_is_current(stat, ktime_to_ns(ktime_get()));
 193 }
 194
 195 void blk_stat_add(struct blk_rq_stat *stat, struct request *rq)
 196 {
 197         s64 now, value;
 198
 199         now = __blk_stat_time(ktime_to_ns(ktime_get()));
 200         if (now < blk_stat_time(&rq->issue_stat))
 201                 return;
 202
 203         if (!__blk_stat_is_current(stat, now))
 204                 __blk_stat_init(stat, now);
 205
 206         value = now - blk_stat_time(&rq->issue_stat);
 207         if (value > stat->max)
 208                 stat->max = value;
 209         if (value < stat->min)
 210                 stat->min = value;
 211
 212         if (stat->batch + value < stat->batch ||
 213             stat->nr_batch + 1 == BLK_RQ_STAT_BATCH)
 214                 blk_stat_flush_batch(stat);
 215
 216         stat->batch += value;
 217         stat->nr_batch++;
 218 }
 219
 220 void blk_stat_clear(struct request_queue *q)
 221 {
 222         if (q->mq_ops) {
 223                 struct blk_mq_hw_ctx *hctx;
 224                 struct blk_mq_ctx *ctx;
 225                 int i, j;
 226
 227                 queue_for_each_hw_ctx(q, hctx, i) {
 228                         hctx_for_each_ctx(hctx, ctx, j) {
 229                                 blk_stat_init(&ctx->stat[BLK_STAT_READ]);
 230                                 blk_stat_init(&ctx->stat[BLK_STAT_WRITE]);
 231                         }
 232                 }
 233         } else {
 234                 blk_stat_init(&q->rq_stats[BLK_STAT_READ]);
 235                 blk_stat_init(&q->rq_stats[BLK_STAT_WRITE]);
 236         }
 237 }
 238
 239 void blk_stat_set_issue_time(struct blk_issue_stat *stat)
 240 {
 241         stat->time = (stat->time & BLK_STAT_MASK) |
 242                         (ktime_to_ns(ktime_get()) & BLK_STAT_TIME_MASK);
 243 }
 244
 245 /*
 246  * Enable stat tracking, return whether it was enabled
 247  */
 248 bool blk_stat_enable(struct request_queue *q)
 249 {
 250         if (!test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
 251                 set_bit(QUEUE_FLAG_STATS, &q->queue_flags);
 252                 return false;
 253         }
 254
 255         return true;
 256 }