blk_mq_run_hw_queue(hctx, true);
}
-static int sched_rq_cmp(void *priv, struct list_head *a, struct list_head *b)
+static int sched_rq_cmp(void *priv, const struct list_head *a,
+ const struct list_head *b)
{
struct request *rqa = container_of(a, struct request, queuelist);
struct request *rqb = container_of(b, struct request, queuelist);
do {
struct request *rq;
+ int budget_token;
if (e->type->ops.has_work && !e->type->ops.has_work(hctx))
break;
break;
}
- if (!blk_mq_get_dispatch_budget(q))
+ budget_token = blk_mq_get_dispatch_budget(q);
+ if (budget_token < 0)
break;
rq = e->type->ops.dispatch_request(hctx);
if (!rq) {
- blk_mq_put_dispatch_budget(q);
+ blk_mq_put_dispatch_budget(q, budget_token);
/*
* We're releasing without dispatching. Holding the
* budget could have blocked any "hctx"s with the
break;
}
+ blk_mq_set_rq_budget_token(rq, budget_token);
+
/*
* Now this rq owns the budget which has to be released
* if this rq won't be queued to driver via .queue_rq()
struct request *rq;
do {
+ int budget_token;
+
if (!list_empty_careful(&hctx->dispatch)) {
ret = -EAGAIN;
break;
if (!sbitmap_any_bit_set(&hctx->ctx_map))
break;
- if (!blk_mq_get_dispatch_budget(q))
+ budget_token = blk_mq_get_dispatch_budget(q);
+ if (budget_token < 0)
break;
rq = blk_mq_dequeue_from_ctx(hctx, ctx);
if (!rq) {
- blk_mq_put_dispatch_budget(q);
+ blk_mq_put_dispatch_budget(q, budget_token);
/*
* We're releasing without dispatching. Holding the
* budget could have blocked any "hctx"s with the
break;
}
+ blk_mq_set_rq_budget_token(rq, budget_token);
+
/*
* Now this rq owns the budget which has to be released
* if this rq won't be queued to driver via .queue_rq()