fs/xfs/linux-2.6/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2005 Silicon Graphics, Inc.
   3  * All Rights Reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it would be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write the Free Software Foundation,
  16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 #include <linux/stddef.h>
  19 #include <linux/errno.h>
  20 #include <linux/slab.h>
  21 #include <linux/pagemap.h>
  22 #include <linux/init.h>
  23 #include <linux/vmalloc.h>
  24 #include <linux/bio.h>
  25 #include <linux/sysctl.h>
  26 #include <linux/proc_fs.h>
  27 #include <linux/workqueue.h>
  28 #include <linux/percpu.h>
  29 #include <linux/blkdev.h>
  30 #include <linux/hash.h>
  31 #include <linux/kthread.h>
  32 #include "xfs_linux.h"
  33
  34 STATIC kmem_zone_t *xfs_buf_zone;
  35 STATIC kmem_shaker_t xfs_buf_shake;
  36 STATIC int xfsbufd(void *);
  37 STATIC int xfsbufd_wakeup(int, gfp_t);
  38 STATIC void xfs_buf_delwri_queue(xfs_buf_t *, int);
  39
  40 STATIC struct workqueue_struct *xfslogd_workqueue;
  41 struct workqueue_struct *xfsdatad_workqueue;
  42
  43 #ifdef XFS_BUF_TRACE
  44 void
  45 xfs_buf_trace(
  46         xfs_buf_t       *bp,
  47         char            *id,
  48         void            *data,
  49         void            *ra)
  50 {
  51         ktrace_enter(xfs_buf_trace_buf,
  52                 bp, id,
  53                 (void *)(unsigned long)bp->b_flags,
  54                 (void *)(unsigned long)bp->b_hold.counter,
  55                 (void *)(unsigned long)bp->b_sema.count.counter,
  56                 (void *)current,
  57                 data, ra,
  58                 (void *)(unsigned long)((bp->b_file_offset>>32) & 0xffffffff),
  59                 (void *)(unsigned long)(bp->b_file_offset & 0xffffffff),
  60                 (void *)(unsigned long)bp->b_buffer_length,
  61                 NULL, NULL, NULL, NULL, NULL);
  62 }
  63 ktrace_t *xfs_buf_trace_buf;
  64 #define XFS_BUF_TRACE_SIZE      4096
  65 #define XB_TRACE(bp, id, data)  \
  66         xfs_buf_trace(bp, id, (void *)data, (void *)__builtin_return_address(0))
  67 #else
  68 #define XB_TRACE(bp, id, data)  do { } while (0)
  69 #endif
  70
  71 #ifdef XFS_BUF_LOCK_TRACKING
  72 # define XB_SET_OWNER(bp)       ((bp)->b_last_holder = current->pid)
  73 # define XB_CLEAR_OWNER(bp)     ((bp)->b_last_holder = -1)
  74 # define XB_GET_OWNER(bp)       ((bp)->b_last_holder)
  75 #else
  76 # define XB_SET_OWNER(bp)       do { } while (0)
  77 # define XB_CLEAR_OWNER(bp)     do { } while (0)
  78 # define XB_GET_OWNER(bp)       do { } while (0)
  79 #endif
  80
  81 #define xb_to_gfp(flags) \
  82         ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
  83           ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
  84
  85 #define xb_to_km(flags) \
  86          (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
  87
  88 #define xfs_buf_allocate(flags) \
  89         kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
  90 #define xfs_buf_deallocate(bp) \
  91         kmem_zone_free(xfs_buf_zone, (bp));
  92
  93 /*
  94  *      Page Region interfaces.
  95  *
  96  *      For pages in filesystems where the blocksize is smaller than the
  97  *      pagesize, we use the page->private field (long) to hold a bitmap
  98  *      of uptodate regions within the page.
  99  *
 100  *      Each such region is "bytes per page / bits per long" bytes long.
 101  *
 102  *      NBPPR == number-of-bytes-per-page-region
 103  *      BTOPR == bytes-to-page-region (rounded up)
 104  *      BTOPRT == bytes-to-page-region-truncated (rounded down)
 105  */
 106 #if (BITS_PER_LONG == 32)
 107 #define PRSHIFT         (PAGE_CACHE_SHIFT - 5)  /* (32 == 1<<5) */
 108 #elif (BITS_PER_LONG == 64)
 109 #define PRSHIFT         (PAGE_CACHE_SHIFT - 6)  /* (64 == 1<<6) */
 110 #else
 111 #error BITS_PER_LONG must be 32 or 64
 112 #endif
 113 #define NBPPR           (PAGE_CACHE_SIZE/BITS_PER_LONG)
 114 #define BTOPR(b)        (((unsigned int)(b) + (NBPPR - 1)) >> PRSHIFT)
 115 #define BTOPRT(b)       (((unsigned int)(b) >> PRSHIFT))
 116
 117 STATIC unsigned long
 118 page_region_mask(
 119         size_t          offset,
 120         size_t          length)
 121 {
 122         unsigned long   mask;
 123         int             first, final;
 124
 125         first = BTOPR(offset);
 126         final = BTOPRT(offset + length - 1);
 127         first = min(first, final);
 128
 129         mask = ~0UL;
 130         mask <<= BITS_PER_LONG - (final - first);
 131         mask >>= BITS_PER_LONG - (final);
 132
 133         ASSERT(offset + length <= PAGE_CACHE_SIZE);
 134         ASSERT((final - first) < BITS_PER_LONG && (final - first) >= 0);
 135
 136         return mask;
 137 }
 138
 139 STATIC inline void
 140 set_page_region(
 141         struct page     *page,
 142         size_t          offset,
 143         size_t          length)
 144 {
 145         set_page_private(page,
 146                 page_private(page) | page_region_mask(offset, length));
 147         if (page_private(page) == ~0UL)
 148                 SetPageUptodate(page);
 149 }
 150
 151 STATIC inline int
 152 test_page_region(
 153         struct page     *page,
 154         size_t          offset,
 155         size_t          length)
 156 {
 157         unsigned long   mask = page_region_mask(offset, length);
 158
 159         return (mask && (page_private(page) & mask) == mask);
 160 }
 161
 162 /*
 163  *      Mapping of multi-page buffers into contiguous virtual space
 164  */
 165
 166 typedef struct a_list {
 167         void            *vm_addr;
 168         struct a_list   *next;
 169 } a_list_t;
 170
 171 STATIC a_list_t         *as_free_head;
 172 STATIC int              as_list_len;
 173 STATIC DEFINE_SPINLOCK(as_lock);
 174
 175 /*
 176  *      Try to batch vunmaps because they are costly.
 177  */
 178 STATIC void
 179 free_address(
 180         void            *addr)
 181 {
 182         a_list_t        *aentry;
 183
 184         aentry = kmalloc(sizeof(a_list_t), GFP_ATOMIC & ~__GFP_HIGH);
 185         if (likely(aentry)) {
 186                 spin_lock(&as_lock);
 187                 aentry->next = as_free_head;
 188                 aentry->vm_addr = addr;
 189                 as_free_head = aentry;
 190                 as_list_len++;
 191                 spin_unlock(&as_lock);
 192         } else {
 193                 vunmap(addr);
 194         }
 195 }
 196
 197 STATIC void
 198 purge_addresses(void)
 199 {
 200         a_list_t        *aentry, *old;
 201
 202         if (as_free_head == NULL)
 203                 return;
 204
 205         spin_lock(&as_lock);
 206         aentry = as_free_head;
 207         as_free_head = NULL;
 208         as_list_len = 0;
 209         spin_unlock(&as_lock);
 210
 211         while ((old = aentry) != NULL) {
 212                 vunmap(aentry->vm_addr);
 213                 aentry = aentry->next;
 214                 kfree(old);
 215         }
 216 }
 217
 218 /*
 219  *      Internal xfs_buf_t object manipulation
 220  */
 221
 222 STATIC void
 223 _xfs_buf_initialize(
 224         xfs_buf_t               *bp,
 225         xfs_buftarg_t           *target,
 226         xfs_off_t               range_base,
 227         size_t                  range_length,
 228         xfs_buf_flags_t         flags)
 229 {
 230         /*
 231          * We don't want certain flags to appear in b_flags.
 232          */
 233         flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
 234
 235         memset(bp, 0, sizeof(xfs_buf_t));
 236         atomic_set(&bp->b_hold, 1);
 237         init_MUTEX_LOCKED(&bp->b_iodonesema);
 238         INIT_LIST_HEAD(&bp->b_list);
 239         INIT_LIST_HEAD(&bp->b_hash_list);
 240         init_MUTEX_LOCKED(&bp->b_sema); /* held, no waiters */
 241         XB_SET_OWNER(bp);
 242         bp->b_target = target;
 243         bp->b_file_offset = range_base;
 244         /*
 245          * Set buffer_length and count_desired to the same value initially.
 246          * I/O routines should use count_desired, which will be the same in
 247          * most cases but may be reset (e.g. XFS recovery).
 248          */
 249         bp->b_buffer_length = bp->b_count_desired = range_length;
 250         bp->b_flags = flags;
 251         bp->b_bn = XFS_BUF_DADDR_NULL;
 252         atomic_set(&bp->b_pin_count, 0);
 253         init_waitqueue_head(&bp->b_waiters);
 254
 255         XFS_STATS_INC(xb_create);
 256         XB_TRACE(bp, "initialize", target);
 257 }
 258
 259 /*
 260  *      Allocate a page array capable of holding a specified number
 261  *      of pages, and point the page buf at it.
 262  */
 263 STATIC int
 264 _xfs_buf_get_pages(
 265         xfs_buf_t               *bp,
 266         int                     page_count,
 267         xfs_buf_flags_t         flags)
 268 {
 269         /* Make sure that we have a page list */
 270         if (bp->b_pages == NULL) {
 271                 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
 272                 bp->b_page_count = page_count;
 273                 if (page_count <= XB_PAGES) {
 274                         bp->b_pages = bp->b_page_array;
 275                 } else {
 276                         bp->b_pages = kmem_alloc(sizeof(struct page *) *
 277                                         page_count, xb_to_km(flags));
 278                         if (bp->b_pages == NULL)
 279                                 return -ENOMEM;
 280                 }
 281                 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
 282         }
 283         return 0;
 284 }
 285
 286 /*
 287  *      Frees b_pages if it was allocated.
 288  */
 289 STATIC void
 290 _xfs_buf_free_pages(
 291         xfs_buf_t       *bp)
 292 {
 293         if (bp->b_pages != bp->b_page_array) {
 294                 kmem_free(bp->b_pages,
 295                           bp->b_page_count * sizeof(struct page *));
 296         }
 297 }
 298
 299 /*
 300  *      Releases the specified buffer.
 301  *
 302  *      The modification state of any associated pages is left unchanged.
 303  *      The buffer most not be on any hash - use xfs_buf_rele instead for
 304  *      hashed and refcounted buffers
 305  */
 306 void
 307 xfs_buf_free(
 308         xfs_buf_t               *bp)
 309 {
 310         XB_TRACE(bp, "free", 0);
 311
 312         ASSERT(list_empty(&bp->b_hash_list));
 313
 314         if (bp->b_flags & _XBF_PAGE_CACHE) {
 315                 uint            i;
 316
 317                 if ((bp->b_flags & XBF_MAPPED) && (bp->b_page_count > 1))
 318                         free_address(bp->b_addr - bp->b_offset);
 319
 320                 for (i = 0; i < bp->b_page_count; i++)
 321                         page_cache_release(bp->b_pages[i]);
 322                 _xfs_buf_free_pages(bp);
 323         } else if (bp->b_flags & _XBF_KMEM_ALLOC) {
 324                  /*
 325                   * XXX(hch): bp->b_count_desired might be incorrect (see
 326                   * xfs_buf_associate_memory for details), but fortunately
 327                   * the Linux version of kmem_free ignores the len argument..
 328                   */
 329                 kmem_free(bp->b_addr, bp->b_count_desired);
 330                 _xfs_buf_free_pages(bp);
 331         }
 332
 333         xfs_buf_deallocate(bp);
 334 }
 335
 336 /*
 337  *      Finds all pages for buffer in question and builds it's page list.
 338  */
 339 STATIC int
 340 _xfs_buf_lookup_pages(
 341         xfs_buf_t               *bp,
 342         uint                    flags)
 343 {
 344         struct address_space    *mapping = bp->b_target->bt_mapping;
 345         size_t                  blocksize = bp->b_target->bt_bsize;
 346         size_t                  size = bp->b_count_desired;
 347         size_t                  nbytes, offset;
 348         gfp_t                   gfp_mask = xb_to_gfp(flags);
 349         unsigned short          page_count, i;
 350         pgoff_t                 first;
 351         xfs_off_t               end;
 352         int                     error;
 353
 354         end = bp->b_file_offset + bp->b_buffer_length;
 355         page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
 356
 357         error = _xfs_buf_get_pages(bp, page_count, flags);
 358         if (unlikely(error))
 359                 return error;
 360         bp->b_flags |= _XBF_PAGE_CACHE;
 361
 362         offset = bp->b_offset;
 363         first = bp->b_file_offset >> PAGE_CACHE_SHIFT;
 364
 365         for (i = 0; i < bp->b_page_count; i++) {
 366                 struct page     *page;
 367                 uint            retries = 0;
 368
 369               retry:
 370                 page = find_or_create_page(mapping, first + i, gfp_mask);
 371                 if (unlikely(page == NULL)) {
 372                         if (flags & XBF_READ_AHEAD) {
 373                                 bp->b_page_count = i;
 374                                 for (i = 0; i < bp->b_page_count; i++)
 375                                         unlock_page(bp->b_pages[i]);
 376                                 return -ENOMEM;
 377                         }
 378
 379                         /*
 380                          * This could deadlock.
 381                          *
 382                          * But until all the XFS lowlevel code is revamped to
 383                          * handle buffer allocation failures we can't do much.
 384                          */
 385                         if (!(++retries % 100))
 386                                 printk(KERN_ERR
 387                                         "XFS: possible memory allocation "
 388                                         "deadlock in %s (mode:0x%x)\n",
 389                                         __FUNCTION__, gfp_mask);
 390
 391                         XFS_STATS_INC(xb_page_retries);
 392                         xfsbufd_wakeup(0, gfp_mask);
 393                         blk_congestion_wait(WRITE, HZ/50);
 394                         goto retry;
 395                 }
 396
 397                 XFS_STATS_INC(xb_page_found);
 398
 399                 nbytes = min_t(size_t, size, PAGE_CACHE_SIZE - offset);
 400                 size -= nbytes;
 401
 402                 if (!PageUptodate(page)) {
 403                         page_count--;
 404                         if (blocksize >= PAGE_CACHE_SIZE) {
 405                                 if (flags & XBF_READ)
 406                                         bp->b_locked = 1;
 407                         } else if (!PagePrivate(page)) {
 408                                 if (test_page_region(page, offset, nbytes))
 409                                         page_count++;
 410                         }
 411                 }
 412
 413                 bp->b_pages[i] = page;
 414                 offset = 0;
 415         }
 416
 417         if (!bp->b_locked) {
 418                 for (i = 0; i < bp->b_page_count; i++)
 419                         unlock_page(bp->b_pages[i]);
 420         }
 421
 422         if (page_count == bp->b_page_count)
 423                 bp->b_flags |= XBF_DONE;
 424
 425         XB_TRACE(bp, "lookup_pages", (long)page_count);
 426         return error;
 427 }
 428
 429 /*
 430  *      Map buffer into kernel address-space if nessecary.
 431  */
 432 STATIC int
 433 _xfs_buf_map_pages(
 434         xfs_buf_t               *bp,
 435         uint                    flags)
 436 {
 437         /* A single page buffer is always mappable */
 438         if (bp->b_page_count == 1) {
 439                 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
 440                 bp->b_flags |= XBF_MAPPED;
 441         } else if (flags & XBF_MAPPED) {
 442                 if (as_list_len > 64)
 443                         purge_addresses();
 444                 bp->b_addr = vmap(bp->b_pages, bp->b_page_count,
 445                                         VM_MAP, PAGE_KERNEL);
 446                 if (unlikely(bp->b_addr == NULL))
 447                         return -ENOMEM;
 448                 bp->b_addr += bp->b_offset;
 449                 bp->b_flags |= XBF_MAPPED;
 450         }
 451
 452         return 0;
 453 }
 454
 455 /*
 456  *      Finding and Reading Buffers
 457  */
 458
 459 /*
 460  *      Look up, and creates if absent, a lockable buffer for
 461  *      a given range of an inode.  The buffer is returned
 462  *      locked.  If other overlapping buffers exist, they are
 463  *      released before the new buffer is created and locked,
 464  *      which may imply that this call will block until those buffers
 465  *      are unlocked.  No I/O is implied by this call.
 466  */
 467 xfs_buf_t *
 468 _xfs_buf_find(
 469         xfs_buftarg_t           *btp,   /* block device target          */
 470         xfs_off_t               ioff,   /* starting offset of range     */
 471         size_t                  isize,  /* length of range              */
 472         xfs_buf_flags_t         flags,
 473         xfs_buf_t               *new_bp)
 474 {
 475         xfs_off_t               range_base;
 476         size_t                  range_length;
 477         xfs_bufhash_t           *hash;
 478         xfs_buf_t               *bp, *n;
 479
 480         range_base = (ioff << BBSHIFT);
 481         range_length = (isize << BBSHIFT);
 482
 483         /* Check for IOs smaller than the sector size / not sector aligned */
 484         ASSERT(!(range_length < (1 << btp->bt_sshift)));
 485         ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
 486
 487         hash = &btp->bt_hash[hash_long((unsigned long)ioff, btp->bt_hashshift)];
 488
 489         spin_lock(&hash->bh_lock);
 490
 491         list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
 492                 ASSERT(btp == bp->b_target);
 493                 if (bp->b_file_offset == range_base &&
 494                     bp->b_buffer_length == range_length) {
 495                         /*
 496                          * If we look at something, bring it to the
 497                          * front of the list for next time.
 498                          */
 499                         atomic_inc(&bp->b_hold);
 500                         list_move(&bp->b_hash_list, &hash->bh_list);
 501                         goto found;
 502                 }
 503         }
 504
 505         /* No match found */
 506         if (new_bp) {
 507                 _xfs_buf_initialize(new_bp, btp, range_base,
 508                                 range_length, flags);
 509                 new_bp->b_hash = hash;
 510                 list_add(&new_bp->b_hash_list, &hash->bh_list);
 511         } else {
 512                 XFS_STATS_INC(xb_miss_locked);
 513         }
 514
 515         spin_unlock(&hash->bh_lock);
 516         return new_bp;
 517
 518 found:
 519         spin_unlock(&hash->bh_lock);
 520
 521         /* Attempt to get the semaphore without sleeping,
 522          * if this does not work then we need to drop the
 523          * spinlock and do a hard attempt on the semaphore.
 524          */
 525         if (down_trylock(&bp->b_sema)) {
 526                 if (!(flags & XBF_TRYLOCK)) {
 527                         /* wait for buffer ownership */
 528                         XB_TRACE(bp, "get_lock", 0);
 529                         xfs_buf_lock(bp);
 530                         XFS_STATS_INC(xb_get_locked_waited);
 531                 } else {
 532                         /* We asked for a trylock and failed, no need
 533                          * to look at file offset and length here, we
 534                          * know that this buffer at least overlaps our
 535                          * buffer and is locked, therefore our buffer
 536                          * either does not exist, or is this buffer.
 537                          */
 538                         xfs_buf_rele(bp);
 539                         XFS_STATS_INC(xb_busy_locked);
 540                         return NULL;
 541                 }
 542         } else {
 543                 /* trylock worked */
 544                 XB_SET_OWNER(bp);
 545         }
 546
 547         if (bp->b_flags & XBF_STALE) {
 548                 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
 549                 bp->b_flags &= XBF_MAPPED;
 550         }
 551         XB_TRACE(bp, "got_lock", 0);
 552         XFS_STATS_INC(xb_get_locked);
 553         return bp;
 554 }
 555
 556 /*
 557  *      Assembles a buffer covering the specified range.
 558  *      Storage in memory for all portions of the buffer will be allocated,
 559  *      although backing storage may not be.
 560  */
 561 xfs_buf_t *
 562 xfs_buf_get_flags(
 563         xfs_buftarg_t           *target,/* target for buffer            */
 564         xfs_off_t               ioff,   /* starting offset of range     */
 565         size_t                  isize,  /* length of range              */
 566         xfs_buf_flags_t         flags)
 567 {
 568         xfs_buf_t               *bp, *new_bp;
 569         int                     error = 0, i;
 570
 571         new_bp = xfs_buf_allocate(flags);
 572         if (unlikely(!new_bp))
 573                 return NULL;
 574
 575         bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
 576         if (bp == new_bp) {
 577                 error = _xfs_buf_lookup_pages(bp, flags);
 578                 if (error)
 579                         goto no_buffer;
 580         } else {
 581                 xfs_buf_deallocate(new_bp);
 582                 if (unlikely(bp == NULL))
 583                         return NULL;
 584         }
 585
 586         for (i = 0; i < bp->b_page_count; i++)
 587                 mark_page_accessed(bp->b_pages[i]);
 588
 589         if (!(bp->b_flags & XBF_MAPPED)) {
 590                 error = _xfs_buf_map_pages(bp, flags);
 591                 if (unlikely(error)) {
 592                         printk(KERN_WARNING "%s: failed to map pages\n",
 593                                         __FUNCTION__);
 594                         goto no_buffer;
 595                 }
 596         }
 597
 598         XFS_STATS_INC(xb_get);
 599
 600         /*
 601          * Always fill in the block number now, the mapped cases can do
 602          * their own overlay of this later.
 603          */
 604         bp->b_bn = ioff;
 605         bp->b_count_desired = bp->b_buffer_length;
 606
 607         XB_TRACE(bp, "get", (unsigned long)flags);
 608         return bp;
 609
 610  no_buffer:
 611         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 612                 xfs_buf_unlock(bp);
 613         xfs_buf_rele(bp);
 614         return NULL;
 615 }
 616
 617 xfs_buf_t *
 618 xfs_buf_read_flags(
 619         xfs_buftarg_t           *target,
 620         xfs_off_t               ioff,
 621         size_t                  isize,
 622         xfs_buf_flags_t         flags)
 623 {
 624         xfs_buf_t               *bp;
 625
 626         flags |= XBF_READ;
 627
 628         bp = xfs_buf_get_flags(target, ioff, isize, flags);
 629         if (bp) {
 630                 if (!XFS_BUF_ISDONE(bp)) {
 631                         XB_TRACE(bp, "read", (unsigned long)flags);
 632                         XFS_STATS_INC(xb_get_read);
 633                         xfs_buf_iostart(bp, flags);
 634                 } else if (flags & XBF_ASYNC) {
 635                         XB_TRACE(bp, "read_async", (unsigned long)flags);
 636                         /*
 637                          * Read ahead call which is already satisfied,
 638                          * drop the buffer
 639                          */
 640                         goto no_buffer;
 641                 } else {
 642                         XB_TRACE(bp, "read_done", (unsigned long)flags);
 643                         /* We do not want read in the flags */
 644                         bp->b_flags &= ~XBF_READ;
 645                 }
 646         }
 647
 648         return bp;
 649
 650  no_buffer:
 651         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 652                 xfs_buf_unlock(bp);
 653         xfs_buf_rele(bp);
 654         return NULL;
 655 }
 656
 657 /*
 658  *      If we are not low on memory then do the readahead in a deadlock
 659  *      safe manner.
 660  */
 661 void
 662 xfs_buf_readahead(
 663         xfs_buftarg_t           *target,
 664         xfs_off_t               ioff,
 665         size_t                  isize,
 666         xfs_buf_flags_t         flags)
 667 {
 668         struct backing_dev_info *bdi;
 669
 670         bdi = target->bt_mapping->backing_dev_info;
 671         if (bdi_read_congested(bdi))
 672                 return;
 673
 674         flags |= (XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD);
 675         xfs_buf_read_flags(target, ioff, isize, flags);
 676 }
 677
 678 xfs_buf_t *
 679 xfs_buf_get_empty(
 680         size_t                  len,
 681         xfs_buftarg_t           *target)
 682 {
 683         xfs_buf_t               *bp;
 684
 685         bp = xfs_buf_allocate(0);
 686         if (bp)
 687                 _xfs_buf_initialize(bp, target, 0, len, 0);
 688         return bp;
 689 }
 690
 691 static inline struct page *
 692 mem_to_page(
 693         void                    *addr)
 694 {
 695         if (((unsigned long)addr < VMALLOC_START) ||
 696             ((unsigned long)addr >= VMALLOC_END)) {
 697                 return virt_to_page(addr);
 698         } else {
 699                 return vmalloc_to_page(addr);
 700         }
 701 }
 702
 703 int
 704 xfs_buf_associate_memory(
 705         xfs_buf_t               *bp,
 706         void                    *mem,
 707         size_t                  len)
 708 {
 709         int                     rval;
 710         int                     i = 0;
 711         size_t                  ptr;
 712         size_t                  end, end_cur;
 713         off_t                   offset;
 714         int                     page_count;
 715
 716         page_count = PAGE_CACHE_ALIGN(len) >> PAGE_CACHE_SHIFT;
 717         offset = (off_t) mem - ((off_t)mem & PAGE_CACHE_MASK);
 718         if (offset && (len > PAGE_CACHE_SIZE))
 719                 page_count++;
 720
 721         /* Free any previous set of page pointers */
 722         if (bp->b_pages)
 723                 _xfs_buf_free_pages(bp);
 724
 725         bp->b_pages = NULL;
 726         bp->b_addr = mem;
 727
 728         rval = _xfs_buf_get_pages(bp, page_count, 0);
 729         if (rval)
 730                 return rval;
 731
 732         bp->b_offset = offset;
 733         ptr = (size_t) mem & PAGE_CACHE_MASK;
 734         end = PAGE_CACHE_ALIGN((size_t) mem + len);
 735         end_cur = end;
 736         /* set up first page */
 737         bp->b_pages[0] = mem_to_page(mem);
 738
 739         ptr += PAGE_CACHE_SIZE;
 740         bp->b_page_count = ++i;
 741         while (ptr < end) {
 742                 bp->b_pages[i] = mem_to_page((void *)ptr);
 743                 bp->b_page_count = ++i;
 744                 ptr += PAGE_CACHE_SIZE;
 745         }
 746         bp->b_locked = 0;
 747
 748         bp->b_count_desired = bp->b_buffer_length = len;
 749         bp->b_flags |= XBF_MAPPED;
 750
 751         return 0;
 752 }
 753
 754 xfs_buf_t *
 755 xfs_buf_get_noaddr(
 756         size_t                  len,
 757         xfs_buftarg_t           *target)
 758 {
 759         size_t                  malloc_len = len;
 760         xfs_buf_t               *bp;
 761         void                    *data;
 762         int                     error;
 763
 764         bp = xfs_buf_allocate(0);
 765         if (unlikely(bp == NULL))
 766                 goto fail;
 767         _xfs_buf_initialize(bp, target, 0, len, 0);
 768
 769  try_again:
 770         data = kmem_alloc(malloc_len, KM_SLEEP | KM_MAYFAIL);
 771         if (unlikely(data == NULL))
 772                 goto fail_free_buf;
 773
 774         /* check whether alignment matches.. */
 775         if ((__psunsigned_t)data !=
 776             ((__psunsigned_t)data & ~target->bt_smask)) {
 777                 /* .. else double the size and try again */
 778                 kmem_free(data, malloc_len);
 779                 malloc_len <<= 1;
 780                 goto try_again;
 781         }
 782
 783         error = xfs_buf_associate_memory(bp, data, len);
 784         if (error)
 785                 goto fail_free_mem;
 786         bp->b_flags |= _XBF_KMEM_ALLOC;
 787
 788         xfs_buf_unlock(bp);
 789
 790         XB_TRACE(bp, "no_daddr", data);
 791         return bp;
 792  fail_free_mem:
 793         kmem_free(data, malloc_len);
 794  fail_free_buf:
 795         xfs_buf_free(bp);
 796  fail:
 797         return NULL;
 798 }
 799
 800 /*
 801  *      Increment reference count on buffer, to hold the buffer concurrently
 802  *      with another thread which may release (free) the buffer asynchronously.
 803  *      Must hold the buffer already to call this function.
 804  */
 805 void
 806 xfs_buf_hold(
 807         xfs_buf_t               *bp)
 808 {
 809         atomic_inc(&bp->b_hold);
 810         XB_TRACE(bp, "hold", 0);
 811 }
 812
 813 /*
 814  *      Releases a hold on the specified buffer.  If the
 815  *      the hold count is 1, calls xfs_buf_free.
 816  */
 817 void
 818 xfs_buf_rele(
 819         xfs_buf_t               *bp)
 820 {
 821         xfs_bufhash_t           *hash = bp->b_hash;
 822
 823         XB_TRACE(bp, "rele", bp->b_relse);
 824
 825         if (atomic_dec_and_lock(&bp->b_hold, &hash->bh_lock)) {
 826                 if (bp->b_relse) {
 827                         atomic_inc(&bp->b_hold);
 828                         spin_unlock(&hash->bh_lock);
 829                         (*(bp->b_relse)) (bp);
 830                 } else if (bp->b_flags & XBF_FS_MANAGED) {
 831                         spin_unlock(&hash->bh_lock);
 832                 } else {
 833                         ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
 834                         list_del_init(&bp->b_hash_list);
 835                         spin_unlock(&hash->bh_lock);
 836                         xfs_buf_free(bp);
 837                 }
 838         } else {
 839                 /*
 840                  * Catch reference count leaks
 841                  */
 842                 ASSERT(atomic_read(&bp->b_hold) >= 0);
 843         }
 844 }
 845
 846
 847 /*
 848  *      Mutual exclusion on buffers.  Locking model:
 849  *
 850  *      Buffers associated with inodes for which buffer locking
 851  *      is not enabled are not protected by semaphores, and are
 852  *      assumed to be exclusively owned by the caller.  There is a
 853  *      spinlock in the buffer, used by the caller when concurrent
 854  *      access is possible.
 855  */
 856
 857 /*
 858  *      Locks a buffer object, if it is not already locked.
 859  *      Note that this in no way locks the underlying pages, so it is only
 860  *      useful for synchronizing concurrent use of buffer objects, not for
 861  *      synchronizing independent access to the underlying pages.
 862  */
 863 int
 864 xfs_buf_cond_lock(
 865         xfs_buf_t               *bp)
 866 {
 867         int                     locked;
 868
 869         locked = down_trylock(&bp->b_sema) == 0;
 870         if (locked) {
 871                 XB_SET_OWNER(bp);
 872         }
 873         XB_TRACE(bp, "cond_lock", (long)locked);
 874         return locked ? 0 : -EBUSY;
 875 }
 876
 877 #if defined(DEBUG) || defined(XFS_BLI_TRACE)
 878 int
 879 xfs_buf_lock_value(
 880         xfs_buf_t               *bp)
 881 {
 882         return atomic_read(&bp->b_sema.count);
 883 }
 884 #endif
 885
 886 /*
 887  *      Locks a buffer object.
 888  *      Note that this in no way locks the underlying pages, so it is only
 889  *      useful for synchronizing concurrent use of buffer objects, not for
 890  *      synchronizing independent access to the underlying pages.
 891  */
 892 void
 893 xfs_buf_lock(
 894         xfs_buf_t               *bp)
 895 {
 896         XB_TRACE(bp, "lock", 0);
 897         if (atomic_read(&bp->b_io_remaining))
 898                 blk_run_address_space(bp->b_target->bt_mapping);
 899         down(&bp->b_sema);
 900         XB_SET_OWNER(bp);
 901         XB_TRACE(bp, "locked", 0);
 902 }
 903
 904 /*
 905  *      Releases the lock on the buffer object.
 906  *      If the buffer is marked delwri but is not queued, do so before we
 907  *      unlock the buffer as we need to set flags correctly.  We also need to
 908  *      take a reference for the delwri queue because the unlocker is going to
 909  *      drop their's and they don't know we just queued it.
 910  */
 911 void
 912 xfs_buf_unlock(
 913         xfs_buf_t               *bp)
 914 {
 915         if ((bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)) == XBF_DELWRI) {
 916                 atomic_inc(&bp->b_hold);
 917                 bp->b_flags |= XBF_ASYNC;
 918                 xfs_buf_delwri_queue(bp, 0);
 919         }
 920
 921         XB_CLEAR_OWNER(bp);
 922         up(&bp->b_sema);
 923         XB_TRACE(bp, "unlock", 0);
 924 }
 925
 926
 927 /*
 928  *      Pinning Buffer Storage in Memory
 929  *      Ensure that no attempt to force a buffer to disk will succeed.
 930  */
 931 void
 932 xfs_buf_pin(
 933         xfs_buf_t               *bp)
 934 {
 935         atomic_inc(&bp->b_pin_count);
 936         XB_TRACE(bp, "pin", (long)bp->b_pin_count.counter);
 937 }
 938
 939 void
 940 xfs_buf_unpin(
 941         xfs_buf_t               *bp)
 942 {
 943         if (atomic_dec_and_test(&bp->b_pin_count))
 944                 wake_up_all(&bp->b_waiters);
 945         XB_TRACE(bp, "unpin", (long)bp->b_pin_count.counter);
 946 }
 947
 948 int
 949 xfs_buf_ispin(
 950         xfs_buf_t               *bp)
 951 {
 952         return atomic_read(&bp->b_pin_count);
 953 }
 954
 955 STATIC void
 956 xfs_buf_wait_unpin(
 957         xfs_buf_t               *bp)
 958 {
 959         DECLARE_WAITQUEUE       (wait, current);
 960
 961         if (atomic_read(&bp->b_pin_count) == 0)
 962                 return;
 963
 964         add_wait_queue(&bp->b_waiters, &wait);
 965         for (;;) {
 966                 set_current_state(TASK_UNINTERRUPTIBLE);
 967                 if (atomic_read(&bp->b_pin_count) == 0)
 968                         break;
 969                 if (atomic_read(&bp->b_io_remaining))
 970                         blk_run_address_space(bp->b_target->bt_mapping);
 971                 schedule();
 972         }
 973         remove_wait_queue(&bp->b_waiters, &wait);
 974         set_current_state(TASK_RUNNING);
 975 }
 976
 977 /*
 978  *      Buffer Utility Routines
 979  */
 980
 981 STATIC void
 982 xfs_buf_iodone_work(
 983         void                    *v)
 984 {
 985         xfs_buf_t               *bp = (xfs_buf_t *)v;
 986
 987         if (bp->b_iodone)
 988                 (*(bp->b_iodone))(bp);
 989         else if (bp->b_flags & XBF_ASYNC)
 990                 xfs_buf_relse(bp);
 991 }
 992
 993 void
 994 xfs_buf_ioend(
 995         xfs_buf_t               *bp,
 996         int                     schedule)
 997 {
 998         bp->b_flags &= ~(XBF_READ | XBF_WRITE);
 999         if (bp->b_error == 0)
1000                 bp->b_flags |= XBF_DONE;
1001
1002         XB_TRACE(bp, "iodone", bp->b_iodone);
1003
1004         if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
1005                 if (schedule) {
1006                         INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work, bp);
1007                         queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1008                 } else {
1009                         xfs_buf_iodone_work(bp);
1010                 }
1011         } else {
1012                 up(&bp->b_iodonesema);
1013         }
1014 }
1015
1016 void
1017 xfs_buf_ioerror(
1018         xfs_buf_t               *bp,
1019         int                     error)
1020 {
1021         ASSERT(error >= 0 && error <= 0xffff);
1022         bp->b_error = (unsigned short)error;
1023         XB_TRACE(bp, "ioerror", (unsigned long)error);
1024 }
1025
1026 /*
1027  *      Initiate I/O on a buffer, based on the flags supplied.
1028  *      The b_iodone routine in the buffer supplied will only be called
1029  *      when all of the subsidiary I/O requests, if any, have been completed.
1030  */
1031 int
1032 xfs_buf_iostart(
1033         xfs_buf_t               *bp,
1034         xfs_buf_flags_t         flags)
1035 {
1036         int                     status = 0;
1037
1038         XB_TRACE(bp, "iostart", (unsigned long)flags);
1039
1040         if (flags & XBF_DELWRI) {
1041                 bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC);
1042                 bp->b_flags |= flags & (XBF_DELWRI | XBF_ASYNC);
1043                 xfs_buf_delwri_queue(bp, 1);
1044                 return status;
1045         }
1046
1047         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_ASYNC | XBF_DELWRI | \
1048                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1049         bp->b_flags |= flags & (XBF_READ | XBF_WRITE | XBF_ASYNC | \
1050                         XBF_READ_AHEAD | _XBF_RUN_QUEUES);
1051
1052         BUG_ON(bp->b_bn == XFS_BUF_DADDR_NULL);
1053
1054         /* For writes allow an alternate strategy routine to precede
1055          * the actual I/O request (which may not be issued at all in
1056          * a shutdown situation, for example).
1057          */
1058         status = (flags & XBF_WRITE) ?
1059                 xfs_buf_iostrategy(bp) : xfs_buf_iorequest(bp);
1060
1061         /* Wait for I/O if we are not an async request.
1062          * Note: async I/O request completion will release the buffer,
1063          * and that can already be done by this point.  So using the
1064          * buffer pointer from here on, after async I/O, is invalid.
1065          */
1066         if (!status && !(flags & XBF_ASYNC))
1067                 status = xfs_buf_iowait(bp);
1068
1069         return status;
1070 }
1071
1072 STATIC __inline__ int
1073 _xfs_buf_iolocked(
1074         xfs_buf_t               *bp)
1075 {
1076         ASSERT(bp->b_flags & (XBF_READ | XBF_WRITE));
1077         if (bp->b_flags & XBF_READ)
1078                 return bp->b_locked;
1079         return 0;
1080 }
1081
1082 STATIC __inline__ void
1083 _xfs_buf_ioend(
1084         xfs_buf_t               *bp,
1085         int                     schedule)
1086 {
1087         if (atomic_dec_and_test(&bp->b_io_remaining) == 1) {
1088                 bp->b_locked = 0;
1089                 xfs_buf_ioend(bp, schedule);
1090         }
1091 }
1092
1093 STATIC int
1094 xfs_buf_bio_end_io(
1095         struct bio              *bio,
1096         unsigned int            bytes_done,
1097         int                     error)
1098 {
1099         xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1100         unsigned int            blocksize = bp->b_target->bt_bsize;
1101         struct bio_vec          *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1102
1103         if (bio->bi_size)
1104                 return 1;
1105
1106         if (!test_bit(BIO_UPTODATE, &bio->bi_flags))
1107                 bp->b_error = EIO;
1108
1109         do {
1110                 struct page     *page = bvec->bv_page;
1111
1112                 if (unlikely(bp->b_error)) {
1113                         if (bp->b_flags & XBF_READ)
1114                                 ClearPageUptodate(page);
1115                         SetPageError(page);
1116                 } else if (blocksize >= PAGE_CACHE_SIZE) {
1117                         SetPageUptodate(page);
1118                 } else if (!PagePrivate(page) &&
1119                                 (bp->b_flags & _XBF_PAGE_CACHE)) {
1120                         set_page_region(page, bvec->bv_offset, bvec->bv_len);
1121                 }
1122
1123                 if (--bvec >= bio->bi_io_vec)
1124                         prefetchw(&bvec->bv_page->flags);
1125
1126                 if (_xfs_buf_iolocked(bp)) {
1127                         unlock_page(page);
1128                 }
1129         } while (bvec >= bio->bi_io_vec);
1130
1131         _xfs_buf_ioend(bp, 1);
1132         bio_put(bio);
1133         return 0;
1134 }
1135
1136 STATIC void
1137 _xfs_buf_ioapply(
1138         xfs_buf_t               *bp)
1139 {
1140         int                     i, rw, map_i, total_nr_pages, nr_pages;
1141         struct bio              *bio;
1142         int                     offset = bp->b_offset;
1143         int                     size = bp->b_count_desired;
1144         sector_t                sector = bp->b_bn;
1145         unsigned int            blocksize = bp->b_target->bt_bsize;
1146         int                     locking = _xfs_buf_iolocked(bp);
1147
1148         total_nr_pages = bp->b_page_count;
1149         map_i = 0;
1150
1151         if (bp->b_flags & _XBF_RUN_QUEUES) {
1152                 bp->b_flags &= ~_XBF_RUN_QUEUES;
1153                 rw = (bp->b_flags & XBF_READ) ? READ_SYNC : WRITE_SYNC;
1154         } else {
1155                 rw = (bp->b_flags & XBF_READ) ? READ : WRITE;
1156         }
1157
1158         if (bp->b_flags & XBF_ORDERED) {
1159                 ASSERT(!(bp->b_flags & XBF_READ));
1160                 rw = WRITE_BARRIER;
1161         }
1162
1163         /* Special code path for reading a sub page size buffer in --
1164          * we populate up the whole page, and hence the other metadata
1165          * in the same page.  This optimization is only valid when the
1166          * filesystem block size is not smaller than the page size.
1167          */
1168         if ((bp->b_buffer_length < PAGE_CACHE_SIZE) &&
1169             (bp->b_flags & XBF_READ) && locking &&
1170             (blocksize >= PAGE_CACHE_SIZE)) {
1171                 bio = bio_alloc(GFP_NOIO, 1);
1172
1173                 bio->bi_bdev = bp->b_target->bt_bdev;
1174                 bio->bi_sector = sector - (offset >> BBSHIFT);
1175                 bio->bi_end_io = xfs_buf_bio_end_io;
1176                 bio->bi_private = bp;
1177
1178                 bio_add_page(bio, bp->b_pages[0], PAGE_CACHE_SIZE, 0);
1179                 size = 0;
1180
1181                 atomic_inc(&bp->b_io_remaining);
1182
1183                 goto submit_io;
1184         }
1185
1186         /* Lock down the pages which we need to for the request */
1187         if (locking && (bp->b_flags & XBF_WRITE) && (bp->b_locked == 0)) {
1188                 for (i = 0; size; i++) {
1189                         int             nbytes = PAGE_CACHE_SIZE - offset;
1190                         struct page     *page = bp->b_pages[i];
1191
1192                         if (nbytes > size)
1193                                 nbytes = size;
1194
1195                         lock_page(page);
1196
1197                         size -= nbytes;
1198                         offset = 0;
1199                 }
1200                 offset = bp->b_offset;
1201                 size = bp->b_count_desired;
1202         }
1203
1204 next_chunk:
1205         atomic_inc(&bp->b_io_remaining);
1206         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1207         if (nr_pages > total_nr_pages)
1208                 nr_pages = total_nr_pages;
1209
1210         bio = bio_alloc(GFP_NOIO, nr_pages);
1211         bio->bi_bdev = bp->b_target->bt_bdev;
1212         bio->bi_sector = sector;
1213         bio->bi_end_io = xfs_buf_bio_end_io;
1214         bio->bi_private = bp;
1215
1216         for (; size && nr_pages; nr_pages--, map_i++) {
1217                 int     rbytes, nbytes = PAGE_CACHE_SIZE - offset;
1218
1219                 if (nbytes > size)
1220                         nbytes = size;
1221
1222                 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1223                 if (rbytes < nbytes)
1224                         break;
1225
1226                 offset = 0;
1227                 sector += nbytes >> BBSHIFT;
1228                 size -= nbytes;
1229                 total_nr_pages--;
1230         }
1231
1232 submit_io:
1233         if (likely(bio->bi_size)) {
1234                 submit_bio(rw, bio);
1235                 if (size)
1236                         goto next_chunk;
1237         } else {
1238                 bio_put(bio);
1239                 xfs_buf_ioerror(bp, EIO);
1240         }
1241 }
1242
1243 int
1244 xfs_buf_iorequest(
1245         xfs_buf_t               *bp)
1246 {
1247         XB_TRACE(bp, "iorequest", 0);
1248
1249         if (bp->b_flags & XBF_DELWRI) {
1250                 xfs_buf_delwri_queue(bp, 1);
1251                 return 0;
1252         }
1253
1254         if (bp->b_flags & XBF_WRITE) {
1255                 xfs_buf_wait_unpin(bp);
1256         }
1257
1258         xfs_buf_hold(bp);
1259
1260         /* Set the count to 1 initially, this will stop an I/O
1261          * completion callout which happens before we have started
1262          * all the I/O from calling xfs_buf_ioend too early.
1263          */
1264         atomic_set(&bp->b_io_remaining, 1);
1265         _xfs_buf_ioapply(bp);
1266         _xfs_buf_ioend(bp, 0);
1267
1268         xfs_buf_rele(bp);
1269         return 0;
1270 }
1271
1272 /*
1273  *      Waits for I/O to complete on the buffer supplied.
1274  *      It returns immediately if no I/O is pending.
1275  *      It returns the I/O error code, if any, or 0 if there was no error.
1276  */
1277 int
1278 xfs_buf_iowait(
1279         xfs_buf_t               *bp)
1280 {
1281         XB_TRACE(bp, "iowait", 0);
1282         if (atomic_read(&bp->b_io_remaining))
1283                 blk_run_address_space(bp->b_target->bt_mapping);
1284         down(&bp->b_iodonesema);
1285         XB_TRACE(bp, "iowaited", (long)bp->b_error);
1286         return bp->b_error;
1287 }
1288
1289 xfs_caddr_t
1290 xfs_buf_offset(
1291         xfs_buf_t               *bp,
1292         size_t                  offset)
1293 {
1294         struct page             *page;
1295
1296         if (bp->b_flags & XBF_MAPPED)
1297                 return XFS_BUF_PTR(bp) + offset;
1298
1299         offset += bp->b_offset;
1300         page = bp->b_pages[offset >> PAGE_CACHE_SHIFT];
1301         return (xfs_caddr_t)page_address(page) + (offset & (PAGE_CACHE_SIZE-1));
1302 }
1303
1304 /*
1305  *      Move data into or out of a buffer.
1306  */
1307 void
1308 xfs_buf_iomove(
1309         xfs_buf_t               *bp,    /* buffer to process            */
1310         size_t                  boff,   /* starting buffer offset       */
1311         size_t                  bsize,  /* length to copy               */
1312         caddr_t                 data,   /* data address                 */
1313         xfs_buf_rw_t            mode)   /* read/write/zero flag         */
1314 {
1315         size_t                  bend, cpoff, csize;
1316         struct page             *page;
1317
1318         bend = boff + bsize;
1319         while (boff < bend) {
1320                 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1321                 cpoff = xfs_buf_poff(boff + bp->b_offset);
1322                 csize = min_t(size_t,
1323                               PAGE_CACHE_SIZE-cpoff, bp->b_count_desired-boff);
1324
1325                 ASSERT(((csize + cpoff) <= PAGE_CACHE_SIZE));
1326
1327                 switch (mode) {
1328                 case XBRW_ZERO:
1329                         memset(page_address(page) + cpoff, 0, csize);
1330                         break;
1331                 case XBRW_READ:
1332                         memcpy(data, page_address(page) + cpoff, csize);
1333                         break;
1334                 case XBRW_WRITE:
1335                         memcpy(page_address(page) + cpoff, data, csize);
1336                 }
1337
1338                 boff += csize;
1339                 data += csize;
1340         }
1341 }
1342
1343 /*
1344  *      Handling of buffer targets (buftargs).
1345  */
1346
1347 /*
1348  *      Wait for any bufs with callbacks that have been submitted but
1349  *      have not yet returned... walk the hash list for the target.
1350  */
1351 void
1352 xfs_wait_buftarg(
1353         xfs_buftarg_t   *btp)
1354 {
1355         xfs_buf_t       *bp, *n;
1356         xfs_bufhash_t   *hash;
1357         uint            i;
1358
1359         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1360                 hash = &btp->bt_hash[i];
1361 again:
1362                 spin_lock(&hash->bh_lock);
1363                 list_for_each_entry_safe(bp, n, &hash->bh_list, b_hash_list) {
1364                         ASSERT(btp == bp->b_target);
1365                         if (!(bp->b_flags & XBF_FS_MANAGED)) {
1366                                 spin_unlock(&hash->bh_lock);
1367                                 /*
1368                                  * Catch superblock reference count leaks
1369                                  * immediately
1370                                  */
1371                                 BUG_ON(bp->b_bn == 0);
1372                                 delay(100);
1373                                 goto again;
1374                         }
1375                 }
1376                 spin_unlock(&hash->bh_lock);
1377         }
1378 }
1379
1380 /*
1381  *      Allocate buffer hash table for a given target.
1382  *      For devices containing metadata (i.e. not the log/realtime devices)
1383  *      we need to allocate a much larger hash table.
1384  */
1385 STATIC void
1386 xfs_alloc_bufhash(
1387         xfs_buftarg_t           *btp,
1388         int                     external)
1389 {
1390         unsigned int            i;
1391
1392         btp->bt_hashshift = external ? 3 : 8;   /* 8 or 256 buckets */
1393         btp->bt_hashmask = (1 << btp->bt_hashshift) - 1;
1394         btp->bt_hash = kmem_zalloc((1 << btp->bt_hashshift) *
1395                                         sizeof(xfs_bufhash_t), KM_SLEEP);
1396         for (i = 0; i < (1 << btp->bt_hashshift); i++) {
1397                 spin_lock_init(&btp->bt_hash[i].bh_lock);
1398                 INIT_LIST_HEAD(&btp->bt_hash[i].bh_list);
1399         }
1400 }
1401
1402 STATIC void
1403 xfs_free_bufhash(
1404         xfs_buftarg_t           *btp)
1405 {
1406         kmem_free(btp->bt_hash, (1<<btp->bt_hashshift) * sizeof(xfs_bufhash_t));
1407         btp->bt_hash = NULL;
1408 }
1409
1410 /*
1411  *      buftarg list for delwrite queue processing
1412  */
1413 STATIC LIST_HEAD(xfs_buftarg_list);
1414 STATIC DEFINE_SPINLOCK(xfs_buftarg_lock);
1415
1416 STATIC void
1417 xfs_register_buftarg(
1418         xfs_buftarg_t           *btp)
1419 {
1420         spin_lock(&xfs_buftarg_lock);
1421         list_add(&btp->bt_list, &xfs_buftarg_list);
1422         spin_unlock(&xfs_buftarg_lock);
1423 }
1424
1425 STATIC void
1426 xfs_unregister_buftarg(
1427         xfs_buftarg_t           *btp)
1428 {
1429         spin_lock(&xfs_buftarg_lock);
1430         list_del(&btp->bt_list);
1431         spin_unlock(&xfs_buftarg_lock);
1432 }
1433
1434 void
1435 xfs_free_buftarg(
1436         xfs_buftarg_t           *btp,
1437         int                     external)
1438 {
1439         xfs_flush_buftarg(btp, 1);
1440         if (external)
1441                 xfs_blkdev_put(btp->bt_bdev);
1442         xfs_free_bufhash(btp);
1443         iput(btp->bt_mapping->host);
1444
1445         /* Unregister the buftarg first so that we don't get a
1446          * wakeup finding a non-existent task
1447          */
1448         xfs_unregister_buftarg(btp);
1449         kthread_stop(btp->bt_task);
1450
1451         kmem_free(btp, sizeof(*btp));
1452 }
1453
1454 STATIC int
1455 xfs_setsize_buftarg_flags(
1456         xfs_buftarg_t           *btp,
1457         unsigned int            blocksize,
1458         unsigned int            sectorsize,
1459         int                     verbose)
1460 {
1461         btp->bt_bsize = blocksize;
1462         btp->bt_sshift = ffs(sectorsize) - 1;
1463         btp->bt_smask = sectorsize - 1;
1464
1465         if (set_blocksize(btp->bt_bdev, sectorsize)) {
1466                 printk(KERN_WARNING
1467                         "XFS: Cannot set_blocksize to %u on device %s\n",
1468                         sectorsize, XFS_BUFTARG_NAME(btp));
1469                 return EINVAL;
1470         }
1471
1472         if (verbose &&
1473             (PAGE_CACHE_SIZE / BITS_PER_LONG) > sectorsize) {
1474                 printk(KERN_WARNING
1475                         "XFS: %u byte sectors in use on device %s.  "
1476                         "This is suboptimal; %u or greater is ideal.\n",
1477                         sectorsize, XFS_BUFTARG_NAME(btp),
1478                         (unsigned int)PAGE_CACHE_SIZE / BITS_PER_LONG);
1479         }
1480
1481         return 0;
1482 }
1483
1484 /*
1485  *      When allocating the initial buffer target we have not yet
1486  *      read in the superblock, so don't know what sized sectors
1487  *      are being used is at this early stage.  Play safe.
1488  */
1489 STATIC int
1490 xfs_setsize_buftarg_early(
1491         xfs_buftarg_t           *btp,
1492         struct block_device     *bdev)
1493 {
1494         return xfs_setsize_buftarg_flags(btp,
1495                         PAGE_CACHE_SIZE, bdev_hardsect_size(bdev), 0);
1496 }
1497
1498 int
1499 xfs_setsize_buftarg(
1500         xfs_buftarg_t           *btp,
1501         unsigned int            blocksize,
1502         unsigned int            sectorsize)
1503 {
1504         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1505 }
1506
1507 STATIC int
1508 xfs_mapping_buftarg(
1509         xfs_buftarg_t           *btp,
1510         struct block_device     *bdev)
1511 {
1512         struct backing_dev_info *bdi;
1513         struct inode            *inode;
1514         struct address_space    *mapping;
1515         static struct address_space_operations mapping_aops = {
1516                 .sync_page = block_sync_page,
1517         };
1518
1519         inode = new_inode(bdev->bd_inode->i_sb);
1520         if (!inode) {
1521                 printk(KERN_WARNING
1522                         "XFS: Cannot allocate mapping inode for device %s\n",
1523                         XFS_BUFTARG_NAME(btp));
1524                 return ENOMEM;
1525         }
1526         inode->i_mode = S_IFBLK;
1527         inode->i_bdev = bdev;
1528         inode->i_rdev = bdev->bd_dev;
1529         bdi = blk_get_backing_dev_info(bdev);
1530         if (!bdi)
1531                 bdi = &default_backing_dev_info;
1532         mapping = &inode->i_data;
1533         mapping->a_ops = &mapping_aops;
1534         mapping->backing_dev_info = bdi;
1535         mapping_set_gfp_mask(mapping, GFP_NOFS);
1536         btp->bt_mapping = mapping;
1537         return 0;
1538 }
1539
1540 STATIC int
1541 xfs_alloc_delwrite_queue(
1542         xfs_buftarg_t           *btp)
1543 {
1544         int     error = 0;
1545
1546         INIT_LIST_HEAD(&btp->bt_list);
1547         INIT_LIST_HEAD(&btp->bt_delwrite_queue);
1548         spinlock_init(&btp->bt_delwrite_lock, "delwri_lock");
1549         btp->bt_flags = 0;
1550         btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd");
1551         if (IS_ERR(btp->bt_task)) {
1552                 error = PTR_ERR(btp->bt_task);
1553                 goto out_error;
1554         }
1555         xfs_register_buftarg(btp);
1556 out_error:
1557         return error;
1558 }
1559
1560 xfs_buftarg_t *
1561 xfs_alloc_buftarg(
1562         struct block_device     *bdev,
1563         int                     external)
1564 {
1565         xfs_buftarg_t           *btp;
1566
1567         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1568
1569         btp->bt_dev =  bdev->bd_dev;
1570         btp->bt_bdev = bdev;
1571         if (xfs_setsize_buftarg_early(btp, bdev))
1572                 goto error;
1573         if (xfs_mapping_buftarg(btp, bdev))
1574                 goto error;
1575         if (xfs_alloc_delwrite_queue(btp))
1576                 goto error;
1577         xfs_alloc_bufhash(btp, external);
1578         return btp;
1579
1580 error:
1581         kmem_free(btp, sizeof(*btp));
1582         return NULL;
1583 }
1584
1585
1586 /*
1587  *      Delayed write buffer handling
1588  */
1589 STATIC void
1590 xfs_buf_delwri_queue(
1591         xfs_buf_t               *bp,
1592         int                     unlock)
1593 {
1594         struct list_head        *dwq = &bp->b_target->bt_delwrite_queue;
1595         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1596
1597         XB_TRACE(bp, "delwri_q", (long)unlock);
1598         ASSERT((bp->b_flags&(XBF_DELWRI|XBF_ASYNC)) == (XBF_DELWRI|XBF_ASYNC));
1599
1600         spin_lock(dwlk);
1601         /* If already in the queue, dequeue and place at tail */
1602         if (!list_empty(&bp->b_list)) {
1603                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1604                 if (unlock)
1605                         atomic_dec(&bp->b_hold);
1606                 list_del(&bp->b_list);
1607         }
1608
1609         bp->b_flags |= _XBF_DELWRI_Q;
1610         list_add_tail(&bp->b_list, dwq);
1611         bp->b_queuetime = jiffies;
1612         spin_unlock(dwlk);
1613
1614         if (unlock)
1615                 xfs_buf_unlock(bp);
1616 }
1617
1618 void
1619 xfs_buf_delwri_dequeue(
1620         xfs_buf_t               *bp)
1621 {
1622         spinlock_t              *dwlk = &bp->b_target->bt_delwrite_lock;
1623         int                     dequeued = 0;
1624
1625         spin_lock(dwlk);
1626         if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1627                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1628                 list_del_init(&bp->b_list);
1629                 dequeued = 1;
1630         }
1631         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1632         spin_unlock(dwlk);
1633
1634         if (dequeued)
1635                 xfs_buf_rele(bp);
1636
1637         XB_TRACE(bp, "delwri_dq", (long)dequeued);
1638 }
1639
1640 STATIC void
1641 xfs_buf_runall_queues(
1642         struct workqueue_struct *queue)
1643 {
1644         flush_workqueue(queue);
1645 }
1646
1647 STATIC int
1648 xfsbufd_wakeup(
1649         int                     priority,
1650         gfp_t                   mask)
1651 {
1652         xfs_buftarg_t           *btp;
1653
1654         spin_lock(&xfs_buftarg_lock);
1655         list_for_each_entry(btp, &xfs_buftarg_list, bt_list) {
1656                 if (test_bit(XBT_FORCE_SLEEP, &btp->bt_flags))
1657                         continue;
1658                 set_bit(XBT_FORCE_FLUSH, &btp->bt_flags);
1659                 wake_up_process(btp->bt_task);
1660         }
1661         spin_unlock(&xfs_buftarg_lock);
1662         return 0;
1663 }
1664
1665 STATIC int
1666 xfsbufd(
1667         void                    *data)
1668 {
1669         struct list_head        tmp;
1670         unsigned long           age;
1671         xfs_buftarg_t           *target = (xfs_buftarg_t *)data;
1672         xfs_buf_t               *bp, *n;
1673         struct list_head        *dwq = &target->bt_delwrite_queue;
1674         spinlock_t              *dwlk = &target->bt_delwrite_lock;
1675
1676         current->flags |= PF_MEMALLOC;
1677
1678         INIT_LIST_HEAD(&tmp);
1679         do {
1680                 if (unlikely(freezing(current))) {
1681                         set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1682                         refrigerator();
1683                 } else {
1684                         clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1685                 }
1686
1687                 schedule_timeout_interruptible(
1688                         xfs_buf_timer_centisecs * msecs_to_jiffies(10));
1689
1690                 age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1691                 spin_lock(dwlk);
1692                 list_for_each_entry_safe(bp, n, dwq, b_list) {
1693                         XB_TRACE(bp, "walkq1", (long)xfs_buf_ispin(bp));
1694                         ASSERT(bp->b_flags & XBF_DELWRI);
1695
1696                         if (!xfs_buf_ispin(bp) && !xfs_buf_cond_lock(bp)) {
1697                                 if (!test_bit(XBT_FORCE_FLUSH,
1698                                                 &target->bt_flags) &&
1699                                     time_before(jiffies,
1700                                                 bp->b_queuetime + age)) {
1701                                         xfs_buf_unlock(bp);
1702                                         break;
1703                                 }
1704
1705                                 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1706                                 bp->b_flags |= XBF_WRITE;
1707                                 list_move(&bp->b_list, &tmp);
1708                         }
1709                 }
1710                 spin_unlock(dwlk);
1711
1712                 while (!list_empty(&tmp)) {
1713                         bp = list_entry(tmp.next, xfs_buf_t, b_list);
1714                         ASSERT(target == bp->b_target);
1715
1716                         list_del_init(&bp->b_list);
1717                         xfs_buf_iostrategy(bp);
1718
1719                         blk_run_address_space(target->bt_mapping);
1720                 }
1721
1722                 if (as_list_len > 0)
1723                         purge_addresses();
1724
1725                 clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1726         } while (!kthread_should_stop());
1727
1728         return 0;
1729 }
1730
1731 /*
1732  *      Go through all incore buffers, and release buffers if they belong to
1733  *      the given device. This is used in filesystem error handling to
1734  *      preserve the consistency of its metadata.
1735  */
1736 int
1737 xfs_flush_buftarg(
1738         xfs_buftarg_t           *target,
1739         int                     wait)
1740 {
1741         struct list_head        tmp;
1742         xfs_buf_t               *bp, *n;
1743         int                     pincount = 0;
1744         struct list_head        *dwq = &target->bt_delwrite_queue;
1745         spinlock_t              *dwlk = &target->bt_delwrite_lock;
1746
1747         xfs_buf_runall_queues(xfsdatad_workqueue);
1748         xfs_buf_runall_queues(xfslogd_workqueue);
1749
1750         INIT_LIST_HEAD(&tmp);
1751         spin_lock(dwlk);
1752         list_for_each_entry_safe(bp, n, dwq, b_list) {
1753                 ASSERT(bp->b_target == target);
1754                 ASSERT(bp->b_flags & (XBF_DELWRI | _XBF_DELWRI_Q));
1755                 XB_TRACE(bp, "walkq2", (long)xfs_buf_ispin(bp));
1756                 if (xfs_buf_ispin(bp)) {
1757                         pincount++;
1758                         continue;
1759                 }
1760
1761                 list_move(&bp->b_list, &tmp);
1762         }
1763         spin_unlock(dwlk);
1764
1765         /*
1766          * Dropped the delayed write list lock, now walk the temporary list
1767          */
1768         list_for_each_entry_safe(bp, n, &tmp, b_list) {
1769                 xfs_buf_lock(bp);
1770                 bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1771                 bp->b_flags |= XBF_WRITE;
1772                 if (wait)
1773                         bp->b_flags &= ~XBF_ASYNC;
1774                 else
1775                         list_del_init(&bp->b_list);
1776
1777                 xfs_buf_iostrategy(bp);
1778         }
1779
1780         /*
1781          * Remaining list items must be flushed before returning
1782          */
1783         while (!list_empty(&tmp)) {
1784                 bp = list_entry(tmp.next, xfs_buf_t, b_list);
1785
1786                 list_del_init(&bp->b_list);
1787                 xfs_iowait(bp);
1788                 xfs_buf_relse(bp);
1789         }
1790
1791         if (wait)
1792                 blk_run_address_space(target->bt_mapping);
1793
1794         return pincount;
1795 }
1796
1797 int __init
1798 xfs_buf_init(void)
1799 {
1800         int             error = -ENOMEM;
1801
1802 #ifdef XFS_BUF_TRACE
1803         xfs_buf_trace_buf = ktrace_alloc(XFS_BUF_TRACE_SIZE, KM_SLEEP);
1804 #endif
1805
1806         xfs_buf_zone = kmem_zone_init(sizeof(xfs_buf_t), "xfs_buf");
1807         if (!xfs_buf_zone)
1808                 goto out_free_trace_buf;
1809
1810         xfslogd_workqueue = create_workqueue("xfslogd");
1811         if (!xfslogd_workqueue)
1812                 goto out_free_buf_zone;
1813
1814         xfsdatad_workqueue = create_workqueue("xfsdatad");
1815         if (!xfsdatad_workqueue)
1816                 goto out_destroy_xfslogd_workqueue;
1817
1818         xfs_buf_shake = kmem_shake_register(xfsbufd_wakeup);
1819         if (!xfs_buf_shake)
1820                 goto out_destroy_xfsdatad_workqueue;
1821
1822         return 0;
1823
1824  out_destroy_xfsdatad_workqueue:
1825         destroy_workqueue(xfsdatad_workqueue);
1826  out_destroy_xfslogd_workqueue:
1827         destroy_workqueue(xfslogd_workqueue);
1828  out_free_buf_zone:
1829         kmem_zone_destroy(xfs_buf_zone);
1830  out_free_trace_buf:
1831 #ifdef XFS_BUF_TRACE
1832         ktrace_free(xfs_buf_trace_buf);
1833 #endif
1834         return error;
1835 }
1836
1837 void
1838 xfs_buf_terminate(void)
1839 {
1840         kmem_shake_deregister(xfs_buf_shake);
1841         destroy_workqueue(xfsdatad_workqueue);
1842         destroy_workqueue(xfslogd_workqueue);
1843         kmem_zone_destroy(xfs_buf_zone);
1844 #ifdef XFS_BUF_TRACE
1845         ktrace_free(xfs_buf_trace_buf);
1846 #endif
1847 }