1 /******************************************************************************
3 * Back-end of the driver for virtual block devices. This portion of the
4 * driver exports a 'unified' block-device interface that can be accessed
5 * by any operating system that implements a compatible front end. A
6 * reference front-end implementation can be found in:
7 * drivers/block/xen-blkfront.c
9 * Copyright (c) 2003-2004, Keir Fraser & Steve Hand
10 * Copyright (c) 2005, Christopher Clark
12 * This program is free software; you can redistribute it and/or
13 * modify it under the terms of the GNU General Public License version 2
14 * as published by the Free Software Foundation; or, when distributed
15 * separately from the Linux kernel or incorporated into other
16 * software packages, subject to the following license:
18 * Permission is hereby granted, free of charge, to any person obtaining a copy
19 * of this source file (the "Software"), to deal in the Software without
20 * restriction, including without limitation the rights to use, copy, modify,
21 * merge, publish, distribute, sublicense, and/or sell copies of the Software,
22 * and to permit persons to whom the Software is furnished to do so, subject to
23 * the following conditions:
25 * The above copyright notice and this permission notice shall be included in
26 * all copies or substantial portions of the Software.
28 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
29 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
30 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
31 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
32 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
33 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
37 #include <linux/spinlock.h>
38 #include <linux/kthread.h>
39 #include <linux/list.h>
40 #include <linux/delay.h>
41 #include <linux/freezer.h>
42 #include <linux/bitmap.h>
44 #include <xen/events.h>
47 #include <asm/xen/hypervisor.h>
48 #include <asm/xen/hypercall.h>
49 #include <xen/balloon.h>
53 * Maximum number of unused free pages to keep in the internal buffer.
54 * Setting this to a value too low will reduce memory used in each backend,
55 * but can have a performance penalty.
57 * A sane value is xen_blkif_reqs * BLKIF_MAX_SEGMENTS_PER_REQUEST, but can
58 * be set to a lower value that might degrade performance on some intensive
62 static int xen_blkif_max_buffer_pages = 1024;
63 module_param_named(max_buffer_pages, xen_blkif_max_buffer_pages, int, 0644);
64 MODULE_PARM_DESC(max_buffer_pages,
65 "Maximum number of free pages to keep in each block backend buffer");
68 * Maximum number of grants to map persistently in blkback. For maximum
69 * performance this should be the total numbers of grants that can be used
70 * to fill the ring, but since this might become too high, specially with
71 * the use of indirect descriptors, we set it to a value that provides good
72 * performance without using too much memory.
74 * When the list of persistent grants is full we clean it up using a LRU
78 static int xen_blkif_max_pgrants = 1056;
79 module_param_named(max_persistent_grants, xen_blkif_max_pgrants, int, 0644);
80 MODULE_PARM_DESC(max_persistent_grants,
81 "Maximum number of grants to map persistently");
84 * The LRU mechanism to clean the lists of persistent grants needs to
85 * be executed periodically. The time interval between consecutive executions
86 * of the purge mechanism is set in ms.
88 #define LRU_INTERVAL 100
91 * When the persistent grants list is full we will remove unused grants
92 * from the list. The percent number of grants to be removed at each LRU
95 #define LRU_PERCENT_CLEAN 5
97 /* Run-time switchable: /sys/module/blkback/parameters/ */
98 static unsigned int log_stats;
99 module_param(log_stats, int, 0644);
101 #define BLKBACK_INVALID_HANDLE (~0)
103 /* Number of free pages to remove on each call to free_xenballooned_pages */
104 #define NUM_BATCH_FREE_PAGES 10
106 static inline int get_free_page(struct xen_blkif *blkif, struct page **page)
110 spin_lock_irqsave(&blkif->free_pages_lock, flags);
111 if (list_empty(&blkif->free_pages)) {
112 BUG_ON(blkif->free_pages_num != 0);
113 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
114 return alloc_xenballooned_pages(1, page, false);
116 BUG_ON(blkif->free_pages_num == 0);
117 page[0] = list_first_entry(&blkif->free_pages, struct page, lru);
118 list_del(&page[0]->lru);
119 blkif->free_pages_num--;
120 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
125 static inline void put_free_pages(struct xen_blkif *blkif, struct page **page,
131 spin_lock_irqsave(&blkif->free_pages_lock, flags);
132 for (i = 0; i < num; i++)
133 list_add(&page[i]->lru, &blkif->free_pages);
134 blkif->free_pages_num += num;
135 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
138 static inline void shrink_free_pagepool(struct xen_blkif *blkif, int num)
140 /* Remove requested pages in batches of NUM_BATCH_FREE_PAGES */
141 struct page *page[NUM_BATCH_FREE_PAGES];
142 unsigned int num_pages = 0;
145 spin_lock_irqsave(&blkif->free_pages_lock, flags);
146 while (blkif->free_pages_num > num) {
147 BUG_ON(list_empty(&blkif->free_pages));
148 page[num_pages] = list_first_entry(&blkif->free_pages,
150 list_del(&page[num_pages]->lru);
151 blkif->free_pages_num--;
152 if (++num_pages == NUM_BATCH_FREE_PAGES) {
153 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
154 free_xenballooned_pages(num_pages, page);
155 spin_lock_irqsave(&blkif->free_pages_lock, flags);
159 spin_unlock_irqrestore(&blkif->free_pages_lock, flags);
161 free_xenballooned_pages(num_pages, page);
164 #define vaddr(page) ((unsigned long)pfn_to_kaddr(page_to_pfn(page)))
166 static int do_block_io_op(struct xen_blkif *blkif);
167 static int dispatch_rw_block_io(struct xen_blkif *blkif,
168 struct blkif_request *req,
169 struct pending_req *pending_req);
170 static void make_response(struct xen_blkif *blkif, u64 id,
171 unsigned short op, int st);
173 #define foreach_grant_safe(pos, n, rbtree, node) \
174 for ((pos) = container_of(rb_first((rbtree)), typeof(*(pos)), node), \
175 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL; \
176 &(pos)->node != NULL; \
177 (pos) = container_of(n, typeof(*(pos)), node), \
178 (n) = (&(pos)->node != NULL) ? rb_next(&(pos)->node) : NULL)
182 * We don't need locking around the persistent grant helpers
183 * because blkback uses a single-thread for each backed, so we
184 * can be sure that this functions will never be called recursively.
186 * The only exception to that is put_persistent_grant, that can be called
187 * from interrupt context (by xen_blkbk_unmap), so we have to use atomic
188 * bit operations to modify the flags of a persistent grant and to count
189 * the number of used grants.
191 static int add_persistent_gnt(struct xen_blkif *blkif,
192 struct persistent_gnt *persistent_gnt)
194 struct rb_node **new = NULL, *parent = NULL;
195 struct persistent_gnt *this;
197 if (blkif->persistent_gnt_c >= xen_blkif_max_pgrants) {
198 if (!blkif->vbd.overflow_max_grants)
199 blkif->vbd.overflow_max_grants = 1;
202 /* Figure out where to put new node */
203 new = &blkif->persistent_gnts.rb_node;
205 this = container_of(*new, struct persistent_gnt, node);
208 if (persistent_gnt->gnt < this->gnt)
209 new = &((*new)->rb_left);
210 else if (persistent_gnt->gnt > this->gnt)
211 new = &((*new)->rb_right);
213 pr_alert_ratelimited(DRV_PFX " trying to add a gref that's already in the tree\n");
218 bitmap_zero(persistent_gnt->flags, PERSISTENT_GNT_FLAGS_SIZE);
219 set_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags);
220 /* Add new node and rebalance tree. */
221 rb_link_node(&(persistent_gnt->node), parent, new);
222 rb_insert_color(&(persistent_gnt->node), &blkif->persistent_gnts);
223 blkif->persistent_gnt_c++;
224 atomic_inc(&blkif->persistent_gnt_in_use);
228 static struct persistent_gnt *get_persistent_gnt(struct xen_blkif *blkif,
231 struct persistent_gnt *data;
232 struct rb_node *node = NULL;
234 node = blkif->persistent_gnts.rb_node;
236 data = container_of(node, struct persistent_gnt, node);
238 if (gref < data->gnt)
239 node = node->rb_left;
240 else if (gref > data->gnt)
241 node = node->rb_right;
243 if(test_bit(PERSISTENT_GNT_ACTIVE, data->flags)) {
244 pr_alert_ratelimited(DRV_PFX " requesting a grant already in use\n");
247 set_bit(PERSISTENT_GNT_ACTIVE, data->flags);
248 atomic_inc(&blkif->persistent_gnt_in_use);
255 static void put_persistent_gnt(struct xen_blkif *blkif,
256 struct persistent_gnt *persistent_gnt)
258 if(!test_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags))
259 pr_alert_ratelimited(DRV_PFX " freeing a grant already unused");
260 set_bit(PERSISTENT_GNT_WAS_ACTIVE, persistent_gnt->flags);
261 clear_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags);
262 atomic_dec(&blkif->persistent_gnt_in_use);
265 static void free_persistent_gnts(struct xen_blkif *blkif, struct rb_root *root,
268 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
269 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
270 struct persistent_gnt *persistent_gnt;
273 int segs_to_unmap = 0;
275 foreach_grant_safe(persistent_gnt, n, root, node) {
276 BUG_ON(persistent_gnt->handle ==
277 BLKBACK_INVALID_HANDLE);
278 gnttab_set_unmap_op(&unmap[segs_to_unmap],
279 (unsigned long) pfn_to_kaddr(page_to_pfn(
280 persistent_gnt->page)),
282 persistent_gnt->handle);
284 pages[segs_to_unmap] = persistent_gnt->page;
286 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST ||
287 !rb_next(&persistent_gnt->node)) {
288 ret = gnttab_unmap_refs(unmap, NULL, pages,
291 put_free_pages(blkif, pages, segs_to_unmap);
295 rb_erase(&persistent_gnt->node, root);
296 kfree(persistent_gnt);
302 static void unmap_purged_grants(struct work_struct *work)
304 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
305 struct page *pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
306 struct persistent_gnt *persistent_gnt;
307 int ret, segs_to_unmap = 0;
308 struct xen_blkif *blkif = container_of(work, typeof(*blkif), persistent_purge_work);
310 while(!list_empty(&blkif->persistent_purge_list)) {
311 persistent_gnt = list_first_entry(&blkif->persistent_purge_list,
312 struct persistent_gnt,
314 list_del(&persistent_gnt->remove_node);
316 gnttab_set_unmap_op(&unmap[segs_to_unmap],
317 vaddr(persistent_gnt->page),
319 persistent_gnt->handle);
321 pages[segs_to_unmap] = persistent_gnt->page;
323 if (++segs_to_unmap == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
324 ret = gnttab_unmap_refs(unmap, NULL, pages,
327 put_free_pages(blkif, pages, segs_to_unmap);
330 kfree(persistent_gnt);
332 if (segs_to_unmap > 0) {
333 ret = gnttab_unmap_refs(unmap, NULL, pages, segs_to_unmap);
335 put_free_pages(blkif, pages, segs_to_unmap);
339 static void purge_persistent_gnt(struct xen_blkif *blkif)
341 struct persistent_gnt *persistent_gnt;
343 unsigned int num_clean, total;
344 bool scan_used = false;
345 struct rb_root *root;
347 if (blkif->persistent_gnt_c < xen_blkif_max_pgrants ||
348 (blkif->persistent_gnt_c == xen_blkif_max_pgrants &&
349 !blkif->vbd.overflow_max_grants)) {
353 if (work_pending(&blkif->persistent_purge_work)) {
354 pr_alert_ratelimited(DRV_PFX "Scheduled work from previous purge is still pending, cannot purge list\n");
358 num_clean = (xen_blkif_max_pgrants / 100) * LRU_PERCENT_CLEAN;
359 num_clean = blkif->persistent_gnt_c - xen_blkif_max_pgrants + num_clean;
360 num_clean = min(blkif->persistent_gnt_c, num_clean);
362 (blkif->persistent_gnt_c -
363 atomic_read(&blkif->persistent_gnt_in_use)))
367 * At this point, we can assure that there will be no calls
368 * to get_persistent_grant (because we are executing this code from
369 * xen_blkif_schedule), there can only be calls to put_persistent_gnt,
370 * which means that the number of currently used grants will go down,
371 * but never up, so we will always be able to remove the requested
377 pr_debug(DRV_PFX "Going to purge %u persistent grants\n", num_clean);
379 INIT_LIST_HEAD(&blkif->persistent_purge_list);
380 root = &blkif->persistent_gnts;
382 foreach_grant_safe(persistent_gnt, n, root, node) {
383 BUG_ON(persistent_gnt->handle ==
384 BLKBACK_INVALID_HANDLE);
386 if (test_bit(PERSISTENT_GNT_ACTIVE, persistent_gnt->flags))
389 (test_bit(PERSISTENT_GNT_WAS_ACTIVE, persistent_gnt->flags)))
392 rb_erase(&persistent_gnt->node, root);
393 list_add(&persistent_gnt->remove_node,
394 &blkif->persistent_purge_list);
395 if (--num_clean == 0)
399 * If we get here it means we also need to start cleaning
400 * grants that were used since last purge in order to cope
401 * with the requested num
404 pr_debug(DRV_PFX "Still missing %u purged frames\n", num_clean);
409 /* Remove the "used" flag from all the persistent grants */
410 foreach_grant_safe(persistent_gnt, n, root, node) {
411 BUG_ON(persistent_gnt->handle ==
412 BLKBACK_INVALID_HANDLE);
413 clear_bit(PERSISTENT_GNT_WAS_ACTIVE, persistent_gnt->flags);
415 blkif->persistent_gnt_c -= (total - num_clean);
416 blkif->vbd.overflow_max_grants = 0;
418 /* We can defer this work */
419 INIT_WORK(&blkif->persistent_purge_work, unmap_purged_grants);
420 schedule_work(&blkif->persistent_purge_work);
421 pr_debug(DRV_PFX "Purged %u/%u\n", (total - num_clean), total);
426 * Retrieve from the 'pending_reqs' a free pending_req structure to be used.
428 static struct pending_req *alloc_req(struct xen_blkif *blkif)
430 struct pending_req *req = NULL;
433 spin_lock_irqsave(&blkif->pending_free_lock, flags);
434 if (!list_empty(&blkif->pending_free)) {
435 req = list_entry(blkif->pending_free.next, struct pending_req,
437 list_del(&req->free_list);
439 spin_unlock_irqrestore(&blkif->pending_free_lock, flags);
444 * Return the 'pending_req' structure back to the freepool. We also
445 * wake up the thread if it was waiting for a free page.
447 static void free_req(struct xen_blkif *blkif, struct pending_req *req)
452 spin_lock_irqsave(&blkif->pending_free_lock, flags);
453 was_empty = list_empty(&blkif->pending_free);
454 list_add(&req->free_list, &blkif->pending_free);
455 spin_unlock_irqrestore(&blkif->pending_free_lock, flags);
457 wake_up(&blkif->pending_free_wq);
461 * Routines for managing virtual block devices (vbds).
463 static int xen_vbd_translate(struct phys_req *req, struct xen_blkif *blkif,
466 struct xen_vbd *vbd = &blkif->vbd;
469 if ((operation != READ) && vbd->readonly)
472 if (likely(req->nr_sects)) {
473 blkif_sector_t end = req->sector_number + req->nr_sects;
475 if (unlikely(end < req->sector_number))
477 if (unlikely(end > vbd_sz(vbd)))
481 req->dev = vbd->pdevice;
482 req->bdev = vbd->bdev;
489 static void xen_vbd_resize(struct xen_blkif *blkif)
491 struct xen_vbd *vbd = &blkif->vbd;
492 struct xenbus_transaction xbt;
494 struct xenbus_device *dev = xen_blkbk_xenbus(blkif->be);
495 unsigned long long new_size = vbd_sz(vbd);
497 pr_info(DRV_PFX "VBD Resize: Domid: %d, Device: (%d, %d)\n",
498 blkif->domid, MAJOR(vbd->pdevice), MINOR(vbd->pdevice));
499 pr_info(DRV_PFX "VBD Resize: new size %llu\n", new_size);
500 vbd->size = new_size;
502 err = xenbus_transaction_start(&xbt);
504 pr_warn(DRV_PFX "Error starting transaction");
507 err = xenbus_printf(xbt, dev->nodename, "sectors", "%llu",
508 (unsigned long long)vbd_sz(vbd));
510 pr_warn(DRV_PFX "Error writing new size");
514 * Write the current state; we will use this to synchronize
515 * the front-end. If the current state is "connected" the
516 * front-end will get the new size information online.
518 err = xenbus_printf(xbt, dev->nodename, "state", "%d", dev->state);
520 pr_warn(DRV_PFX "Error writing the state");
524 err = xenbus_transaction_end(xbt, 0);
528 pr_warn(DRV_PFX "Error ending transaction");
531 xenbus_transaction_end(xbt, 1);
535 * Notification from the guest OS.
537 static void blkif_notify_work(struct xen_blkif *blkif)
539 blkif->waiting_reqs = 1;
543 irqreturn_t xen_blkif_be_int(int irq, void *dev_id)
545 blkif_notify_work(dev_id);
550 * SCHEDULER FUNCTIONS
553 static void print_stats(struct xen_blkif *blkif)
555 pr_info("xen-blkback (%s): oo %3llu | rd %4llu | wr %4llu | f %4llu"
556 " | ds %4llu | pg: %4u/%4d\n",
557 current->comm, blkif->st_oo_req,
558 blkif->st_rd_req, blkif->st_wr_req,
559 blkif->st_f_req, blkif->st_ds_req,
560 blkif->persistent_gnt_c,
561 xen_blkif_max_pgrants);
562 blkif->st_print = jiffies + msecs_to_jiffies(10 * 1000);
563 blkif->st_rd_req = 0;
564 blkif->st_wr_req = 0;
565 blkif->st_oo_req = 0;
566 blkif->st_ds_req = 0;
569 int xen_blkif_schedule(void *arg)
571 struct xen_blkif *blkif = arg;
572 struct xen_vbd *vbd = &blkif->vbd;
573 unsigned long timeout;
575 xen_blkif_get(blkif);
577 while (!kthread_should_stop()) {
580 if (unlikely(vbd->size != vbd_sz(vbd)))
581 xen_vbd_resize(blkif);
583 timeout = msecs_to_jiffies(LRU_INTERVAL);
585 timeout = wait_event_interruptible_timeout(
587 blkif->waiting_reqs || kthread_should_stop(),
591 timeout = wait_event_interruptible_timeout(
592 blkif->pending_free_wq,
593 !list_empty(&blkif->pending_free) ||
594 kthread_should_stop(),
599 blkif->waiting_reqs = 0;
600 smp_mb(); /* clear flag *before* checking for work */
602 if (do_block_io_op(blkif))
603 blkif->waiting_reqs = 1;
606 if (blkif->vbd.feature_gnt_persistent &&
607 time_after(jiffies, blkif->next_lru)) {
608 purge_persistent_gnt(blkif);
609 blkif->next_lru = jiffies + msecs_to_jiffies(LRU_INTERVAL);
612 /* Shrink if we have more than xen_blkif_max_buffer_pages */
613 shrink_free_pagepool(blkif, xen_blkif_max_buffer_pages);
615 if (log_stats && time_after(jiffies, blkif->st_print))
619 /* Since we are shutting down remove all pages from the buffer */
620 shrink_free_pagepool(blkif, 0 /* All */);
622 /* Free all persistent grant pages */
623 if (!RB_EMPTY_ROOT(&blkif->persistent_gnts))
624 free_persistent_gnts(blkif, &blkif->persistent_gnts,
625 blkif->persistent_gnt_c);
627 BUG_ON(!RB_EMPTY_ROOT(&blkif->persistent_gnts));
628 blkif->persistent_gnt_c = 0;
633 blkif->xenblkd = NULL;
634 xen_blkif_put(blkif);
640 * Unmap the grant references, and also remove the M2P over-rides
641 * used in the 'pending_req'.
643 static void xen_blkbk_unmap(struct xen_blkif *blkif,
644 struct grant_page *pages[],
647 struct gnttab_unmap_grant_ref unmap[BLKIF_MAX_SEGMENTS_PER_REQUEST];
648 struct page *unmap_pages[BLKIF_MAX_SEGMENTS_PER_REQUEST];
649 unsigned int i, invcount = 0;
652 for (i = 0; i < num; i++) {
653 if (pages[i]->persistent_gnt != NULL) {
654 put_persistent_gnt(blkif, pages[i]->persistent_gnt);
657 if (pages[i]->handle == BLKBACK_INVALID_HANDLE)
659 unmap_pages[invcount] = pages[i]->page;
660 gnttab_set_unmap_op(&unmap[invcount], vaddr(pages[i]->page),
661 GNTMAP_host_map, pages[i]->handle);
662 pages[i]->handle = BLKBACK_INVALID_HANDLE;
663 if (++invcount == BLKIF_MAX_SEGMENTS_PER_REQUEST) {
664 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages,
667 put_free_pages(blkif, unmap_pages, invcount);
672 ret = gnttab_unmap_refs(unmap, NULL, unmap_pages, invcount);
674 put_free_pages(blkif, unmap_pages, invcount);
678 static int xen_blkbk_map(struct xen_blkif *blkif,
679 struct grant_page *pages[],
682 struct gnttab_map_grant_ref map[BLKIF_MAX_SEGMENTS_PER_REQUEST];
683 struct page *pages_to_gnt[BLKIF_MAX_SEGMENTS_PER_REQUEST];
684 struct persistent_gnt *persistent_gnt = NULL;
685 phys_addr_t addr = 0;
686 int i, seg_idx, new_map_idx;
689 int last_map = 0, map_until = 0;
690 int use_persistent_gnts;
692 use_persistent_gnts = (blkif->vbd.feature_gnt_persistent);
695 * Fill out preq.nr_sects with proper amount of sectors, and setup
696 * assign map[..] with the PFN of the page in our domain with the
697 * corresponding grant reference for each page.
700 for (i = map_until; i < num; i++) {
703 if (use_persistent_gnts)
704 persistent_gnt = get_persistent_gnt(
708 if (persistent_gnt) {
710 * We are using persistent grants and
711 * the grant is already mapped
713 pages[i]->page = persistent_gnt->page;
714 pages[i]->persistent_gnt = persistent_gnt;
716 if (get_free_page(blkif, &pages[i]->page))
718 addr = vaddr(pages[i]->page);
719 pages_to_gnt[segs_to_map] = pages[i]->page;
720 pages[i]->persistent_gnt = NULL;
721 flags = GNTMAP_host_map;
722 if (!use_persistent_gnts && ro)
723 flags |= GNTMAP_readonly;
724 gnttab_set_map_op(&map[segs_to_map++], addr,
725 flags, pages[i]->gref,
729 if (segs_to_map == BLKIF_MAX_SEGMENTS_PER_REQUEST)
734 ret = gnttab_map_refs(map, NULL, pages_to_gnt, segs_to_map);
739 * Now swizzle the MFN in our domain with the MFN from the other domain
740 * so that when we access vaddr(pending_req,i) it has the contents of
741 * the page from the other domain.
743 for (seg_idx = last_map, new_map_idx = 0; seg_idx < map_until; seg_idx++) {
744 if (!pages[seg_idx]->persistent_gnt) {
745 /* This is a newly mapped grant */
746 BUG_ON(new_map_idx >= segs_to_map);
747 if (unlikely(map[new_map_idx].status != 0)) {
748 pr_debug(DRV_PFX "invalid buffer -- could not remap it\n");
749 pages[seg_idx]->handle = BLKBACK_INVALID_HANDLE;
753 pages[seg_idx]->handle = map[new_map_idx].handle;
757 if (use_persistent_gnts &&
758 blkif->persistent_gnt_c < xen_blkif_max_pgrants) {
760 * We are using persistent grants, the grant is
761 * not mapped but we might have room for it.
763 persistent_gnt = kmalloc(sizeof(struct persistent_gnt),
765 if (!persistent_gnt) {
767 * If we don't have enough memory to
768 * allocate the persistent_gnt struct
769 * map this grant non-persistenly
773 persistent_gnt->gnt = map[new_map_idx].ref;
774 persistent_gnt->handle = map[new_map_idx].handle;
775 persistent_gnt->page = pages[seg_idx]->page;
776 if (add_persistent_gnt(blkif,
778 kfree(persistent_gnt);
779 persistent_gnt = NULL;
782 pages[seg_idx]->persistent_gnt = persistent_gnt;
783 pr_debug(DRV_PFX " grant %u added to the tree of persistent grants, using %u/%u\n",
784 persistent_gnt->gnt, blkif->persistent_gnt_c,
785 xen_blkif_max_pgrants);
788 if (use_persistent_gnts && !blkif->vbd.overflow_max_grants) {
789 blkif->vbd.overflow_max_grants = 1;
790 pr_debug(DRV_PFX " domain %u, device %#x is using maximum number of persistent grants\n",
791 blkif->domid, blkif->vbd.handle);
794 * We could not map this grant persistently, so use it as
795 * a non-persistent grant.
801 last_map = map_until;
802 if (map_until != num)
808 pr_alert(DRV_PFX "%s: out of memory\n", __func__);
809 put_free_pages(blkif, pages_to_gnt, segs_to_map);
813 static int xen_blkbk_map_seg(struct pending_req *pending_req)
817 rc = xen_blkbk_map(pending_req->blkif, pending_req->segments,
818 pending_req->nr_pages,
819 (pending_req->operation != BLKIF_OP_READ));
824 static int xen_blkbk_parse_indirect(struct blkif_request *req,
825 struct pending_req *pending_req,
826 struct seg_buf seg[],
827 struct phys_req *preq)
829 struct grant_page **pages = pending_req->indirect_pages;
830 struct xen_blkif *blkif = pending_req->blkif;
831 int indirect_grefs, rc, n, nseg, i;
832 struct blkif_request_segment_aligned *segments = NULL;
834 nseg = pending_req->nr_pages;
835 indirect_grefs = INDIRECT_PAGES(nseg);
836 BUG_ON(indirect_grefs > BLKIF_MAX_INDIRECT_PAGES_PER_REQUEST);
838 for (i = 0; i < indirect_grefs; i++)
839 pages[i]->gref = req->u.indirect.indirect_grefs[i];
841 rc = xen_blkbk_map(blkif, pages, indirect_grefs, true);
845 for (n = 0, i = 0; n < nseg; n++) {
846 if ((n % SEGS_PER_INDIRECT_FRAME) == 0) {
847 /* Map indirect segments */
849 kunmap_atomic(segments);
850 segments = kmap_atomic(pages[n/SEGS_PER_INDIRECT_FRAME]->page);
852 i = n % SEGS_PER_INDIRECT_FRAME;
853 pending_req->segments[n]->gref = segments[i].gref;
854 seg[n].nsec = segments[i].last_sect -
855 segments[i].first_sect + 1;
856 seg[n].offset = (segments[i].first_sect << 9);
857 if ((segments[i].last_sect >= (PAGE_SIZE >> 9)) ||
858 (segments[i].last_sect < segments[i].first_sect)) {
862 preq->nr_sects += seg[n].nsec;
867 kunmap_atomic(segments);
868 xen_blkbk_unmap(blkif, pages, indirect_grefs);
872 static int dispatch_discard_io(struct xen_blkif *blkif,
873 struct blkif_request *req)
876 int status = BLKIF_RSP_OKAY;
877 struct block_device *bdev = blkif->vbd.bdev;
878 unsigned long secure;
882 xen_blkif_get(blkif);
883 secure = (blkif->vbd.discard_secure &&
884 (req->u.discard.flag & BLKIF_DISCARD_SECURE)) ?
885 BLKDEV_DISCARD_SECURE : 0;
887 err = blkdev_issue_discard(bdev, req->u.discard.sector_number,
888 req->u.discard.nr_sectors,
891 if (err == -EOPNOTSUPP) {
892 pr_debug(DRV_PFX "discard op failed, not supported\n");
893 status = BLKIF_RSP_EOPNOTSUPP;
895 status = BLKIF_RSP_ERROR;
897 make_response(blkif, req->u.discard.id, req->operation, status);
898 xen_blkif_put(blkif);
902 static int dispatch_other_io(struct xen_blkif *blkif,
903 struct blkif_request *req,
904 struct pending_req *pending_req)
906 free_req(blkif, pending_req);
907 make_response(blkif, req->u.other.id, req->operation,
908 BLKIF_RSP_EOPNOTSUPP);
912 static void xen_blk_drain_io(struct xen_blkif *blkif)
914 atomic_set(&blkif->drain, 1);
916 /* The initial value is one, and one refcnt taken at the
917 * start of the xen_blkif_schedule thread. */
918 if (atomic_read(&blkif->refcnt) <= 2)
920 wait_for_completion_interruptible_timeout(
921 &blkif->drain_complete, HZ);
923 if (!atomic_read(&blkif->drain))
925 } while (!kthread_should_stop());
926 atomic_set(&blkif->drain, 0);
930 * Completion callback on the bio's. Called as bh->b_end_io()
933 static void __end_block_io_op(struct pending_req *pending_req, int error)
935 /* An error fails the entire request. */
936 if ((pending_req->operation == BLKIF_OP_FLUSH_DISKCACHE) &&
937 (error == -EOPNOTSUPP)) {
938 pr_debug(DRV_PFX "flush diskcache op failed, not supported\n");
939 xen_blkbk_flush_diskcache(XBT_NIL, pending_req->blkif->be, 0);
940 pending_req->status = BLKIF_RSP_EOPNOTSUPP;
941 } else if ((pending_req->operation == BLKIF_OP_WRITE_BARRIER) &&
942 (error == -EOPNOTSUPP)) {
943 pr_debug(DRV_PFX "write barrier op failed, not supported\n");
944 xen_blkbk_barrier(XBT_NIL, pending_req->blkif->be, 0);
945 pending_req->status = BLKIF_RSP_EOPNOTSUPP;
947 pr_debug(DRV_PFX "Buffer not up-to-date at end of operation,"
948 " error=%d\n", error);
949 pending_req->status = BLKIF_RSP_ERROR;
953 * If all of the bio's have completed it is time to unmap
954 * the grant references associated with 'request' and provide
955 * the proper response on the ring.
957 if (atomic_dec_and_test(&pending_req->pendcnt)) {
958 xen_blkbk_unmap(pending_req->blkif,
959 pending_req->segments,
960 pending_req->nr_pages);
961 make_response(pending_req->blkif, pending_req->id,
962 pending_req->operation, pending_req->status);
963 xen_blkif_put(pending_req->blkif);
964 if (atomic_read(&pending_req->blkif->refcnt) <= 2) {
965 if (atomic_read(&pending_req->blkif->drain))
966 complete(&pending_req->blkif->drain_complete);
968 free_req(pending_req->blkif, pending_req);
975 static void end_block_io_op(struct bio *bio, int error)
977 __end_block_io_op(bio->bi_private, error);
984 * Function to copy the from the ring buffer the 'struct blkif_request'
985 * (which has the sectors we want, number of them, grant references, etc),
986 * and transmute it to the block API to hand it over to the proper block disk.
989 __do_block_io_op(struct xen_blkif *blkif)
991 union blkif_back_rings *blk_rings = &blkif->blk_rings;
992 struct blkif_request req;
993 struct pending_req *pending_req;
997 rc = blk_rings->common.req_cons;
998 rp = blk_rings->common.sring->req_prod;
999 rmb(); /* Ensure we see queued requests up to 'rp'. */
1003 if (RING_REQUEST_CONS_OVERFLOW(&blk_rings->common, rc))
1006 if (kthread_should_stop()) {
1011 pending_req = alloc_req(blkif);
1012 if (NULL == pending_req) {
1018 switch (blkif->blk_protocol) {
1019 case BLKIF_PROTOCOL_NATIVE:
1020 memcpy(&req, RING_GET_REQUEST(&blk_rings->native, rc), sizeof(req));
1022 case BLKIF_PROTOCOL_X86_32:
1023 blkif_get_x86_32_req(&req, RING_GET_REQUEST(&blk_rings->x86_32, rc));
1025 case BLKIF_PROTOCOL_X86_64:
1026 blkif_get_x86_64_req(&req, RING_GET_REQUEST(&blk_rings->x86_64, rc));
1031 blk_rings->common.req_cons = ++rc; /* before make_response() */
1033 /* Apply all sanity checks to /private copy/ of request. */
1036 switch (req.operation) {
1038 case BLKIF_OP_WRITE:
1039 case BLKIF_OP_WRITE_BARRIER:
1040 case BLKIF_OP_FLUSH_DISKCACHE:
1041 case BLKIF_OP_INDIRECT:
1042 if (dispatch_rw_block_io(blkif, &req, pending_req))
1045 case BLKIF_OP_DISCARD:
1046 free_req(blkif, pending_req);
1047 if (dispatch_discard_io(blkif, &req))
1051 if (dispatch_other_io(blkif, &req, pending_req))
1056 /* Yield point for this unbounded loop. */
1064 do_block_io_op(struct xen_blkif *blkif)
1066 union blkif_back_rings *blk_rings = &blkif->blk_rings;
1070 more_to_do = __do_block_io_op(blkif);
1074 RING_FINAL_CHECK_FOR_REQUESTS(&blk_rings->common, more_to_do);
1075 } while (more_to_do);
1080 * Transmutation of the 'struct blkif_request' to a proper 'struct bio'
1081 * and call the 'submit_bio' to pass it to the underlying storage.
1083 static int dispatch_rw_block_io(struct xen_blkif *blkif,
1084 struct blkif_request *req,
1085 struct pending_req *pending_req)
1087 struct phys_req preq;
1088 struct seg_buf *seg = pending_req->seg;
1090 struct bio *bio = NULL;
1091 struct bio **biolist = pending_req->biolist;
1094 struct blk_plug plug;
1096 struct grant_page **pages = pending_req->segments;
1097 unsigned short req_operation;
1099 req_operation = req->operation == BLKIF_OP_INDIRECT ?
1100 req->u.indirect.indirect_op : req->operation;
1101 if ((req->operation == BLKIF_OP_INDIRECT) &&
1102 (req_operation != BLKIF_OP_READ) &&
1103 (req_operation != BLKIF_OP_WRITE)) {
1104 pr_debug(DRV_PFX "Invalid indirect operation (%u)\n",
1109 switch (req_operation) {
1114 case BLKIF_OP_WRITE:
1116 operation = WRITE_ODIRECT;
1118 case BLKIF_OP_WRITE_BARRIER:
1120 case BLKIF_OP_FLUSH_DISKCACHE:
1122 operation = WRITE_FLUSH;
1125 operation = 0; /* make gcc happy */
1130 /* Check that the number of segments is sane. */
1131 nseg = req->operation == BLKIF_OP_INDIRECT ?
1132 req->u.indirect.nr_segments : req->u.rw.nr_segments;
1134 if (unlikely(nseg == 0 && operation != WRITE_FLUSH) ||
1135 unlikely((req->operation != BLKIF_OP_INDIRECT) &&
1136 (nseg > BLKIF_MAX_SEGMENTS_PER_REQUEST)) ||
1137 unlikely((req->operation == BLKIF_OP_INDIRECT) &&
1138 (nseg > MAX_INDIRECT_SEGMENTS))) {
1139 pr_debug(DRV_PFX "Bad number of segments in request (%d)\n",
1141 /* Haven't submitted any bio's yet. */
1147 pending_req->blkif = blkif;
1148 pending_req->id = req->u.rw.id;
1149 pending_req->operation = req_operation;
1150 pending_req->status = BLKIF_RSP_OKAY;
1151 pending_req->nr_pages = nseg;
1153 if (req->operation != BLKIF_OP_INDIRECT) {
1154 preq.dev = req->u.rw.handle;
1155 preq.sector_number = req->u.rw.sector_number;
1156 for (i = 0; i < nseg; i++) {
1157 pages[i]->gref = req->u.rw.seg[i].gref;
1158 seg[i].nsec = req->u.rw.seg[i].last_sect -
1159 req->u.rw.seg[i].first_sect + 1;
1160 seg[i].offset = (req->u.rw.seg[i].first_sect << 9);
1161 if ((req->u.rw.seg[i].last_sect >= (PAGE_SIZE >> 9)) ||
1162 (req->u.rw.seg[i].last_sect <
1163 req->u.rw.seg[i].first_sect))
1165 preq.nr_sects += seg[i].nsec;
1168 preq.dev = req->u.indirect.handle;
1169 preq.sector_number = req->u.indirect.sector_number;
1170 if (xen_blkbk_parse_indirect(req, pending_req, seg, &preq))
1174 if (xen_vbd_translate(&preq, blkif, operation) != 0) {
1175 pr_debug(DRV_PFX "access denied: %s of [%llu,%llu] on dev=%04x\n",
1176 operation == READ ? "read" : "write",
1178 preq.sector_number + preq.nr_sects,
1179 blkif->vbd.pdevice);
1184 * This check _MUST_ be done after xen_vbd_translate as the preq.bdev
1187 for (i = 0; i < nseg; i++) {
1188 if (((int)preq.sector_number|(int)seg[i].nsec) &
1189 ((bdev_logical_block_size(preq.bdev) >> 9) - 1)) {
1190 pr_debug(DRV_PFX "Misaligned I/O request from domain %d",
1196 /* Wait on all outstanding I/O's and once that has been completed
1197 * issue the WRITE_FLUSH.
1200 xen_blk_drain_io(pending_req->blkif);
1203 * If we have failed at this point, we need to undo the M2P override,
1204 * set gnttab_set_unmap_op on all of the grant references and perform
1205 * the hypercall to unmap the grants - that is all done in
1208 if (xen_blkbk_map_seg(pending_req))
1212 * This corresponding xen_blkif_put is done in __end_block_io_op, or
1213 * below (in "!bio") if we are handling a BLKIF_OP_DISCARD.
1215 xen_blkif_get(blkif);
1217 for (i = 0; i < nseg; i++) {
1218 while ((bio == NULL) ||
1222 seg[i].offset) == 0)) {
1224 bio = bio_alloc(GFP_KERNEL, nseg-i);
1225 if (unlikely(bio == NULL))
1228 biolist[nbio++] = bio;
1229 bio->bi_bdev = preq.bdev;
1230 bio->bi_private = pending_req;
1231 bio->bi_end_io = end_block_io_op;
1232 bio->bi_sector = preq.sector_number;
1235 preq.sector_number += seg[i].nsec;
1238 /* This will be hit if the operation was a flush or discard. */
1240 BUG_ON(operation != WRITE_FLUSH);
1242 bio = bio_alloc(GFP_KERNEL, 0);
1243 if (unlikely(bio == NULL))
1246 biolist[nbio++] = bio;
1247 bio->bi_bdev = preq.bdev;
1248 bio->bi_private = pending_req;
1249 bio->bi_end_io = end_block_io_op;
1252 atomic_set(&pending_req->pendcnt, nbio);
1253 blk_start_plug(&plug);
1255 for (i = 0; i < nbio; i++)
1256 submit_bio(operation, biolist[i]);
1258 /* Let the I/Os go.. */
1259 blk_finish_plug(&plug);
1261 if (operation == READ)
1262 blkif->st_rd_sect += preq.nr_sects;
1263 else if (operation & WRITE)
1264 blkif->st_wr_sect += preq.nr_sects;
1269 xen_blkbk_unmap(blkif, pending_req->segments,
1270 pending_req->nr_pages);
1272 /* Haven't submitted any bio's yet. */
1273 make_response(blkif, req->u.rw.id, req_operation, BLKIF_RSP_ERROR);
1274 free_req(blkif, pending_req);
1275 msleep(1); /* back off a bit */
1279 for (i = 0; i < nbio; i++)
1280 bio_put(biolist[i]);
1281 atomic_set(&pending_req->pendcnt, 1);
1282 __end_block_io_op(pending_req, -EINVAL);
1283 msleep(1); /* back off a bit */
1290 * Put a response on the ring on how the operation fared.
1292 static void make_response(struct xen_blkif *blkif, u64 id,
1293 unsigned short op, int st)
1295 struct blkif_response resp;
1296 unsigned long flags;
1297 union blkif_back_rings *blk_rings = &blkif->blk_rings;
1301 resp.operation = op;
1304 spin_lock_irqsave(&blkif->blk_ring_lock, flags);
1305 /* Place on the response ring for the relevant domain. */
1306 switch (blkif->blk_protocol) {
1307 case BLKIF_PROTOCOL_NATIVE:
1308 memcpy(RING_GET_RESPONSE(&blk_rings->native, blk_rings->native.rsp_prod_pvt),
1309 &resp, sizeof(resp));
1311 case BLKIF_PROTOCOL_X86_32:
1312 memcpy(RING_GET_RESPONSE(&blk_rings->x86_32, blk_rings->x86_32.rsp_prod_pvt),
1313 &resp, sizeof(resp));
1315 case BLKIF_PROTOCOL_X86_64:
1316 memcpy(RING_GET_RESPONSE(&blk_rings->x86_64, blk_rings->x86_64.rsp_prod_pvt),
1317 &resp, sizeof(resp));
1322 blk_rings->common.rsp_prod_pvt++;
1323 RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&blk_rings->common, notify);
1324 spin_unlock_irqrestore(&blkif->blk_ring_lock, flags);
1326 notify_remote_via_irq(blkif->irq);
1329 static int __init xen_blkif_init(void)
1336 rc = xen_blkif_interface_init();
1340 rc = xen_blkif_xenbus_init();
1348 module_init(xen_blkif_init);
1350 MODULE_LICENSE("Dual BSD/GPL");
1351 MODULE_ALIAS("xen-backend:vbd");