fs: Add helper to clean bdev aliases under a bh and use it
[sfrench/cifs-2.6.git] / fs / buffer.c
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/syscalls.h>
23 #include <linux/fs.h>
24 #include <linux/iomap.h>
25 #include <linux/mm.h>
26 #include <linux/percpu.h>
27 #include <linux/slab.h>
28 #include <linux/capability.h>
29 #include <linux/blkdev.h>
30 #include <linux/file.h>
31 #include <linux/quotaops.h>
32 #include <linux/highmem.h>
33 #include <linux/export.h>
34 #include <linux/backing-dev.h>
35 #include <linux/writeback.h>
36 #include <linux/hash.h>
37 #include <linux/suspend.h>
38 #include <linux/buffer_head.h>
39 #include <linux/task_io_accounting_ops.h>
40 #include <linux/bio.h>
41 #include <linux/notifier.h>
42 #include <linux/cpu.h>
43 #include <linux/bitops.h>
44 #include <linux/mpage.h>
45 #include <linux/bit_spinlock.h>
46 #include <linux/pagevec.h>
47 #include <trace/events/block.h>
48
49 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
50 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
51                          unsigned long bio_flags,
52                          struct writeback_control *wbc);
53
54 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
55
56 void init_buffer(struct buffer_head *bh, bh_end_io_t *handler, void *private)
57 {
58         bh->b_end_io = handler;
59         bh->b_private = private;
60 }
61 EXPORT_SYMBOL(init_buffer);
62
63 inline void touch_buffer(struct buffer_head *bh)
64 {
65         trace_block_touch_buffer(bh);
66         mark_page_accessed(bh->b_page);
67 }
68 EXPORT_SYMBOL(touch_buffer);
69
70 void __lock_buffer(struct buffer_head *bh)
71 {
72         wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
73 }
74 EXPORT_SYMBOL(__lock_buffer);
75
76 void unlock_buffer(struct buffer_head *bh)
77 {
78         clear_bit_unlock(BH_Lock, &bh->b_state);
79         smp_mb__after_atomic();
80         wake_up_bit(&bh->b_state, BH_Lock);
81 }
82 EXPORT_SYMBOL(unlock_buffer);
83
84 /*
85  * Returns if the page has dirty or writeback buffers. If all the buffers
86  * are unlocked and clean then the PageDirty information is stale. If
87  * any of the pages are locked, it is assumed they are locked for IO.
88  */
89 void buffer_check_dirty_writeback(struct page *page,
90                                      bool *dirty, bool *writeback)
91 {
92         struct buffer_head *head, *bh;
93         *dirty = false;
94         *writeback = false;
95
96         BUG_ON(!PageLocked(page));
97
98         if (!page_has_buffers(page))
99                 return;
100
101         if (PageWriteback(page))
102                 *writeback = true;
103
104         head = page_buffers(page);
105         bh = head;
106         do {
107                 if (buffer_locked(bh))
108                         *writeback = true;
109
110                 if (buffer_dirty(bh))
111                         *dirty = true;
112
113                 bh = bh->b_this_page;
114         } while (bh != head);
115 }
116 EXPORT_SYMBOL(buffer_check_dirty_writeback);
117
118 /*
119  * Block until a buffer comes unlocked.  This doesn't stop it
120  * from becoming locked again - you have to lock it yourself
121  * if you want to preserve its state.
122  */
123 void __wait_on_buffer(struct buffer_head * bh)
124 {
125         wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
126 }
127 EXPORT_SYMBOL(__wait_on_buffer);
128
129 static void
130 __clear_page_buffers(struct page *page)
131 {
132         ClearPagePrivate(page);
133         set_page_private(page, 0);
134         put_page(page);
135 }
136
137 static void buffer_io_error(struct buffer_head *bh, char *msg)
138 {
139         if (!test_bit(BH_Quiet, &bh->b_state))
140                 printk_ratelimited(KERN_ERR
141                         "Buffer I/O error on dev %pg, logical block %llu%s\n",
142                         bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
143 }
144
145 /*
146  * End-of-IO handler helper function which does not touch the bh after
147  * unlocking it.
148  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
149  * a race there is benign: unlock_buffer() only use the bh's address for
150  * hashing after unlocking the buffer, so it doesn't actually touch the bh
151  * itself.
152  */
153 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
154 {
155         if (uptodate) {
156                 set_buffer_uptodate(bh);
157         } else {
158                 /* This happens, due to failed read-ahead attempts. */
159                 clear_buffer_uptodate(bh);
160         }
161         unlock_buffer(bh);
162 }
163
164 /*
165  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
166  * unlock the buffer. This is what ll_rw_block uses too.
167  */
168 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
169 {
170         __end_buffer_read_notouch(bh, uptodate);
171         put_bh(bh);
172 }
173 EXPORT_SYMBOL(end_buffer_read_sync);
174
175 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
176 {
177         if (uptodate) {
178                 set_buffer_uptodate(bh);
179         } else {
180                 buffer_io_error(bh, ", lost sync page write");
181                 set_buffer_write_io_error(bh);
182                 clear_buffer_uptodate(bh);
183         }
184         unlock_buffer(bh);
185         put_bh(bh);
186 }
187 EXPORT_SYMBOL(end_buffer_write_sync);
188
189 /*
190  * Various filesystems appear to want __find_get_block to be non-blocking.
191  * But it's the page lock which protects the buffers.  To get around this,
192  * we get exclusion from try_to_free_buffers with the blockdev mapping's
193  * private_lock.
194  *
195  * Hack idea: for the blockdev mapping, i_bufferlist_lock contention
196  * may be quite high.  This code could TryLock the page, and if that
197  * succeeds, there is no need to take private_lock. (But if
198  * private_lock is contended then so is mapping->tree_lock).
199  */
200 static struct buffer_head *
201 __find_get_block_slow(struct block_device *bdev, sector_t block)
202 {
203         struct inode *bd_inode = bdev->bd_inode;
204         struct address_space *bd_mapping = bd_inode->i_mapping;
205         struct buffer_head *ret = NULL;
206         pgoff_t index;
207         struct buffer_head *bh;
208         struct buffer_head *head;
209         struct page *page;
210         int all_mapped = 1;
211
212         index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
213         page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
214         if (!page)
215                 goto out;
216
217         spin_lock(&bd_mapping->private_lock);
218         if (!page_has_buffers(page))
219                 goto out_unlock;
220         head = page_buffers(page);
221         bh = head;
222         do {
223                 if (!buffer_mapped(bh))
224                         all_mapped = 0;
225                 else if (bh->b_blocknr == block) {
226                         ret = bh;
227                         get_bh(bh);
228                         goto out_unlock;
229                 }
230                 bh = bh->b_this_page;
231         } while (bh != head);
232
233         /* we might be here because some of the buffers on this page are
234          * not mapped.  This is due to various races between
235          * file io on the block device and getblk.  It gets dealt with
236          * elsewhere, don't buffer_error if we had some unmapped buffers
237          */
238         if (all_mapped) {
239                 printk("__find_get_block_slow() failed. "
240                         "block=%llu, b_blocknr=%llu\n",
241                         (unsigned long long)block,
242                         (unsigned long long)bh->b_blocknr);
243                 printk("b_state=0x%08lx, b_size=%zu\n",
244                         bh->b_state, bh->b_size);
245                 printk("device %pg blocksize: %d\n", bdev,
246                         1 << bd_inode->i_blkbits);
247         }
248 out_unlock:
249         spin_unlock(&bd_mapping->private_lock);
250         put_page(page);
251 out:
252         return ret;
253 }
254
255 /*
256  * Kick the writeback threads then try to free up some ZONE_NORMAL memory.
257  */
258 static void free_more_memory(void)
259 {
260         struct zoneref *z;
261         int nid;
262
263         wakeup_flusher_threads(1024, WB_REASON_FREE_MORE_MEM);
264         yield();
265
266         for_each_online_node(nid) {
267
268                 z = first_zones_zonelist(node_zonelist(nid, GFP_NOFS),
269                                                 gfp_zone(GFP_NOFS), NULL);
270                 if (z->zone)
271                         try_to_free_pages(node_zonelist(nid, GFP_NOFS), 0,
272                                                 GFP_NOFS, NULL);
273         }
274 }
275
276 /*
277  * I/O completion handler for block_read_full_page() - pages
278  * which come unlocked at the end of I/O.
279  */
280 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
281 {
282         unsigned long flags;
283         struct buffer_head *first;
284         struct buffer_head *tmp;
285         struct page *page;
286         int page_uptodate = 1;
287
288         BUG_ON(!buffer_async_read(bh));
289
290         page = bh->b_page;
291         if (uptodate) {
292                 set_buffer_uptodate(bh);
293         } else {
294                 clear_buffer_uptodate(bh);
295                 buffer_io_error(bh, ", async page read");
296                 SetPageError(page);
297         }
298
299         /*
300          * Be _very_ careful from here on. Bad things can happen if
301          * two buffer heads end IO at almost the same time and both
302          * decide that the page is now completely done.
303          */
304         first = page_buffers(page);
305         local_irq_save(flags);
306         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
307         clear_buffer_async_read(bh);
308         unlock_buffer(bh);
309         tmp = bh;
310         do {
311                 if (!buffer_uptodate(tmp))
312                         page_uptodate = 0;
313                 if (buffer_async_read(tmp)) {
314                         BUG_ON(!buffer_locked(tmp));
315                         goto still_busy;
316                 }
317                 tmp = tmp->b_this_page;
318         } while (tmp != bh);
319         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
320         local_irq_restore(flags);
321
322         /*
323          * If none of the buffers had errors and they are all
324          * uptodate then we can set the page uptodate.
325          */
326         if (page_uptodate && !PageError(page))
327                 SetPageUptodate(page);
328         unlock_page(page);
329         return;
330
331 still_busy:
332         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
333         local_irq_restore(flags);
334         return;
335 }
336
337 /*
338  * Completion handler for block_write_full_page() - pages which are unlocked
339  * during I/O, and which have PageWriteback cleared upon I/O completion.
340  */
341 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
342 {
343         unsigned long flags;
344         struct buffer_head *first;
345         struct buffer_head *tmp;
346         struct page *page;
347
348         BUG_ON(!buffer_async_write(bh));
349
350         page = bh->b_page;
351         if (uptodate) {
352                 set_buffer_uptodate(bh);
353         } else {
354                 buffer_io_error(bh, ", lost async page write");
355                 mapping_set_error(page->mapping, -EIO);
356                 set_buffer_write_io_error(bh);
357                 clear_buffer_uptodate(bh);
358                 SetPageError(page);
359         }
360
361         first = page_buffers(page);
362         local_irq_save(flags);
363         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
364
365         clear_buffer_async_write(bh);
366         unlock_buffer(bh);
367         tmp = bh->b_this_page;
368         while (tmp != bh) {
369                 if (buffer_async_write(tmp)) {
370                         BUG_ON(!buffer_locked(tmp));
371                         goto still_busy;
372                 }
373                 tmp = tmp->b_this_page;
374         }
375         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
376         local_irq_restore(flags);
377         end_page_writeback(page);
378         return;
379
380 still_busy:
381         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
382         local_irq_restore(flags);
383         return;
384 }
385 EXPORT_SYMBOL(end_buffer_async_write);
386
387 /*
388  * If a page's buffers are under async readin (end_buffer_async_read
389  * completion) then there is a possibility that another thread of
390  * control could lock one of the buffers after it has completed
391  * but while some of the other buffers have not completed.  This
392  * locked buffer would confuse end_buffer_async_read() into not unlocking
393  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
394  * that this buffer is not under async I/O.
395  *
396  * The page comes unlocked when it has no locked buffer_async buffers
397  * left.
398  *
399  * PageLocked prevents anyone starting new async I/O reads any of
400  * the buffers.
401  *
402  * PageWriteback is used to prevent simultaneous writeout of the same
403  * page.
404  *
405  * PageLocked prevents anyone from starting writeback of a page which is
406  * under read I/O (PageWriteback is only ever set against a locked page).
407  */
408 static void mark_buffer_async_read(struct buffer_head *bh)
409 {
410         bh->b_end_io = end_buffer_async_read;
411         set_buffer_async_read(bh);
412 }
413
414 static void mark_buffer_async_write_endio(struct buffer_head *bh,
415                                           bh_end_io_t *handler)
416 {
417         bh->b_end_io = handler;
418         set_buffer_async_write(bh);
419 }
420
421 void mark_buffer_async_write(struct buffer_head *bh)
422 {
423         mark_buffer_async_write_endio(bh, end_buffer_async_write);
424 }
425 EXPORT_SYMBOL(mark_buffer_async_write);
426
427
428 /*
429  * fs/buffer.c contains helper functions for buffer-backed address space's
430  * fsync functions.  A common requirement for buffer-based filesystems is
431  * that certain data from the backing blockdev needs to be written out for
432  * a successful fsync().  For example, ext2 indirect blocks need to be
433  * written back and waited upon before fsync() returns.
434  *
435  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
436  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
437  * management of a list of dependent buffers at ->i_mapping->private_list.
438  *
439  * Locking is a little subtle: try_to_free_buffers() will remove buffers
440  * from their controlling inode's queue when they are being freed.  But
441  * try_to_free_buffers() will be operating against the *blockdev* mapping
442  * at the time, not against the S_ISREG file which depends on those buffers.
443  * So the locking for private_list is via the private_lock in the address_space
444  * which backs the buffers.  Which is different from the address_space 
445  * against which the buffers are listed.  So for a particular address_space,
446  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
447  * mapping->private_list will always be protected by the backing blockdev's
448  * ->private_lock.
449  *
450  * Which introduces a requirement: all buffers on an address_space's
451  * ->private_list must be from the same address_space: the blockdev's.
452  *
453  * address_spaces which do not place buffers at ->private_list via these
454  * utility functions are free to use private_lock and private_list for
455  * whatever they want.  The only requirement is that list_empty(private_list)
456  * be true at clear_inode() time.
457  *
458  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
459  * filesystems should do that.  invalidate_inode_buffers() should just go
460  * BUG_ON(!list_empty).
461  *
462  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
463  * take an address_space, not an inode.  And it should be called
464  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
465  * queued up.
466  *
467  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
468  * list if it is already on a list.  Because if the buffer is on a list,
469  * it *must* already be on the right one.  If not, the filesystem is being
470  * silly.  This will save a ton of locking.  But first we have to ensure
471  * that buffers are taken *off* the old inode's list when they are freed
472  * (presumably in truncate).  That requires careful auditing of all
473  * filesystems (do it inside bforget()).  It could also be done by bringing
474  * b_inode back.
475  */
476
477 /*
478  * The buffer's backing address_space's private_lock must be held
479  */
480 static void __remove_assoc_queue(struct buffer_head *bh)
481 {
482         list_del_init(&bh->b_assoc_buffers);
483         WARN_ON(!bh->b_assoc_map);
484         if (buffer_write_io_error(bh))
485                 set_bit(AS_EIO, &bh->b_assoc_map->flags);
486         bh->b_assoc_map = NULL;
487 }
488
489 int inode_has_buffers(struct inode *inode)
490 {
491         return !list_empty(&inode->i_data.private_list);
492 }
493
494 /*
495  * osync is designed to support O_SYNC io.  It waits synchronously for
496  * all already-submitted IO to complete, but does not queue any new
497  * writes to the disk.
498  *
499  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
500  * you dirty the buffers, and then use osync_inode_buffers to wait for
501  * completion.  Any other dirty buffers which are not yet queued for
502  * write will not be flushed to disk by the osync.
503  */
504 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
505 {
506         struct buffer_head *bh;
507         struct list_head *p;
508         int err = 0;
509
510         spin_lock(lock);
511 repeat:
512         list_for_each_prev(p, list) {
513                 bh = BH_ENTRY(p);
514                 if (buffer_locked(bh)) {
515                         get_bh(bh);
516                         spin_unlock(lock);
517                         wait_on_buffer(bh);
518                         if (!buffer_uptodate(bh))
519                                 err = -EIO;
520                         brelse(bh);
521                         spin_lock(lock);
522                         goto repeat;
523                 }
524         }
525         spin_unlock(lock);
526         return err;
527 }
528
529 static void do_thaw_one(struct super_block *sb, void *unused)
530 {
531         while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
532                 printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
533 }
534
535 static void do_thaw_all(struct work_struct *work)
536 {
537         iterate_supers(do_thaw_one, NULL);
538         kfree(work);
539         printk(KERN_WARNING "Emergency Thaw complete\n");
540 }
541
542 /**
543  * emergency_thaw_all -- forcibly thaw every frozen filesystem
544  *
545  * Used for emergency unfreeze of all filesystems via SysRq
546  */
547 void emergency_thaw_all(void)
548 {
549         struct work_struct *work;
550
551         work = kmalloc(sizeof(*work), GFP_ATOMIC);
552         if (work) {
553                 INIT_WORK(work, do_thaw_all);
554                 schedule_work(work);
555         }
556 }
557
558 /**
559  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
560  * @mapping: the mapping which wants those buffers written
561  *
562  * Starts I/O against the buffers at mapping->private_list, and waits upon
563  * that I/O.
564  *
565  * Basically, this is a convenience function for fsync().
566  * @mapping is a file or directory which needs those buffers to be written for
567  * a successful fsync().
568  */
569 int sync_mapping_buffers(struct address_space *mapping)
570 {
571         struct address_space *buffer_mapping = mapping->private_data;
572
573         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
574                 return 0;
575
576         return fsync_buffers_list(&buffer_mapping->private_lock,
577                                         &mapping->private_list);
578 }
579 EXPORT_SYMBOL(sync_mapping_buffers);
580
581 /*
582  * Called when we've recently written block `bblock', and it is known that
583  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
584  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
585  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
586  */
587 void write_boundary_block(struct block_device *bdev,
588                         sector_t bblock, unsigned blocksize)
589 {
590         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
591         if (bh) {
592                 if (buffer_dirty(bh))
593                         ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
594                 put_bh(bh);
595         }
596 }
597
598 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
599 {
600         struct address_space *mapping = inode->i_mapping;
601         struct address_space *buffer_mapping = bh->b_page->mapping;
602
603         mark_buffer_dirty(bh);
604         if (!mapping->private_data) {
605                 mapping->private_data = buffer_mapping;
606         } else {
607                 BUG_ON(mapping->private_data != buffer_mapping);
608         }
609         if (!bh->b_assoc_map) {
610                 spin_lock(&buffer_mapping->private_lock);
611                 list_move_tail(&bh->b_assoc_buffers,
612                                 &mapping->private_list);
613                 bh->b_assoc_map = mapping;
614                 spin_unlock(&buffer_mapping->private_lock);
615         }
616 }
617 EXPORT_SYMBOL(mark_buffer_dirty_inode);
618
619 /*
620  * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
621  * dirty.
622  *
623  * If warn is true, then emit a warning if the page is not uptodate and has
624  * not been truncated.
625  *
626  * The caller must hold lock_page_memcg().
627  */
628 static void __set_page_dirty(struct page *page, struct address_space *mapping,
629                              int warn)
630 {
631         unsigned long flags;
632
633         spin_lock_irqsave(&mapping->tree_lock, flags);
634         if (page->mapping) {    /* Race with truncate? */
635                 WARN_ON_ONCE(warn && !PageUptodate(page));
636                 account_page_dirtied(page, mapping);
637                 radix_tree_tag_set(&mapping->page_tree,
638                                 page_index(page), PAGECACHE_TAG_DIRTY);
639         }
640         spin_unlock_irqrestore(&mapping->tree_lock, flags);
641 }
642
643 /*
644  * Add a page to the dirty page list.
645  *
646  * It is a sad fact of life that this function is called from several places
647  * deeply under spinlocking.  It may not sleep.
648  *
649  * If the page has buffers, the uptodate buffers are set dirty, to preserve
650  * dirty-state coherency between the page and the buffers.  It the page does
651  * not have buffers then when they are later attached they will all be set
652  * dirty.
653  *
654  * The buffers are dirtied before the page is dirtied.  There's a small race
655  * window in which a writepage caller may see the page cleanness but not the
656  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
657  * before the buffers, a concurrent writepage caller could clear the page dirty
658  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
659  * page on the dirty page list.
660  *
661  * We use private_lock to lock against try_to_free_buffers while using the
662  * page's buffer list.  Also use this to protect against clean buffers being
663  * added to the page after it was set dirty.
664  *
665  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
666  * address_space though.
667  */
668 int __set_page_dirty_buffers(struct page *page)
669 {
670         int newly_dirty;
671         struct address_space *mapping = page_mapping(page);
672
673         if (unlikely(!mapping))
674                 return !TestSetPageDirty(page);
675
676         spin_lock(&mapping->private_lock);
677         if (page_has_buffers(page)) {
678                 struct buffer_head *head = page_buffers(page);
679                 struct buffer_head *bh = head;
680
681                 do {
682                         set_buffer_dirty(bh);
683                         bh = bh->b_this_page;
684                 } while (bh != head);
685         }
686         /*
687          * Lock out page->mem_cgroup migration to keep PageDirty
688          * synchronized with per-memcg dirty page counters.
689          */
690         lock_page_memcg(page);
691         newly_dirty = !TestSetPageDirty(page);
692         spin_unlock(&mapping->private_lock);
693
694         if (newly_dirty)
695                 __set_page_dirty(page, mapping, 1);
696
697         unlock_page_memcg(page);
698
699         if (newly_dirty)
700                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
701
702         return newly_dirty;
703 }
704 EXPORT_SYMBOL(__set_page_dirty_buffers);
705
706 /*
707  * Write out and wait upon a list of buffers.
708  *
709  * We have conflicting pressures: we want to make sure that all
710  * initially dirty buffers get waited on, but that any subsequently
711  * dirtied buffers don't.  After all, we don't want fsync to last
712  * forever if somebody is actively writing to the file.
713  *
714  * Do this in two main stages: first we copy dirty buffers to a
715  * temporary inode list, queueing the writes as we go.  Then we clean
716  * up, waiting for those writes to complete.
717  * 
718  * During this second stage, any subsequent updates to the file may end
719  * up refiling the buffer on the original inode's dirty list again, so
720  * there is a chance we will end up with a buffer queued for write but
721  * not yet completed on that list.  So, as a final cleanup we go through
722  * the osync code to catch these locked, dirty buffers without requeuing
723  * any newly dirty buffers for write.
724  */
725 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
726 {
727         struct buffer_head *bh;
728         struct list_head tmp;
729         struct address_space *mapping;
730         int err = 0, err2;
731         struct blk_plug plug;
732
733         INIT_LIST_HEAD(&tmp);
734         blk_start_plug(&plug);
735
736         spin_lock(lock);
737         while (!list_empty(list)) {
738                 bh = BH_ENTRY(list->next);
739                 mapping = bh->b_assoc_map;
740                 __remove_assoc_queue(bh);
741                 /* Avoid race with mark_buffer_dirty_inode() which does
742                  * a lockless check and we rely on seeing the dirty bit */
743                 smp_mb();
744                 if (buffer_dirty(bh) || buffer_locked(bh)) {
745                         list_add(&bh->b_assoc_buffers, &tmp);
746                         bh->b_assoc_map = mapping;
747                         if (buffer_dirty(bh)) {
748                                 get_bh(bh);
749                                 spin_unlock(lock);
750                                 /*
751                                  * Ensure any pending I/O completes so that
752                                  * write_dirty_buffer() actually writes the
753                                  * current contents - it is a noop if I/O is
754                                  * still in flight on potentially older
755                                  * contents.
756                                  */
757                                 write_dirty_buffer(bh, REQ_SYNC);
758
759                                 /*
760                                  * Kick off IO for the previous mapping. Note
761                                  * that we will not run the very last mapping,
762                                  * wait_on_buffer() will do that for us
763                                  * through sync_buffer().
764                                  */
765                                 brelse(bh);
766                                 spin_lock(lock);
767                         }
768                 }
769         }
770
771         spin_unlock(lock);
772         blk_finish_plug(&plug);
773         spin_lock(lock);
774
775         while (!list_empty(&tmp)) {
776                 bh = BH_ENTRY(tmp.prev);
777                 get_bh(bh);
778                 mapping = bh->b_assoc_map;
779                 __remove_assoc_queue(bh);
780                 /* Avoid race with mark_buffer_dirty_inode() which does
781                  * a lockless check and we rely on seeing the dirty bit */
782                 smp_mb();
783                 if (buffer_dirty(bh)) {
784                         list_add(&bh->b_assoc_buffers,
785                                  &mapping->private_list);
786                         bh->b_assoc_map = mapping;
787                 }
788                 spin_unlock(lock);
789                 wait_on_buffer(bh);
790                 if (!buffer_uptodate(bh))
791                         err = -EIO;
792                 brelse(bh);
793                 spin_lock(lock);
794         }
795         
796         spin_unlock(lock);
797         err2 = osync_buffers_list(lock, list);
798         if (err)
799                 return err;
800         else
801                 return err2;
802 }
803
804 /*
805  * Invalidate any and all dirty buffers on a given inode.  We are
806  * probably unmounting the fs, but that doesn't mean we have already
807  * done a sync().  Just drop the buffers from the inode list.
808  *
809  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
810  * assumes that all the buffers are against the blockdev.  Not true
811  * for reiserfs.
812  */
813 void invalidate_inode_buffers(struct inode *inode)
814 {
815         if (inode_has_buffers(inode)) {
816                 struct address_space *mapping = &inode->i_data;
817                 struct list_head *list = &mapping->private_list;
818                 struct address_space *buffer_mapping = mapping->private_data;
819
820                 spin_lock(&buffer_mapping->private_lock);
821                 while (!list_empty(list))
822                         __remove_assoc_queue(BH_ENTRY(list->next));
823                 spin_unlock(&buffer_mapping->private_lock);
824         }
825 }
826 EXPORT_SYMBOL(invalidate_inode_buffers);
827
828 /*
829  * Remove any clean buffers from the inode's buffer list.  This is called
830  * when we're trying to free the inode itself.  Those buffers can pin it.
831  *
832  * Returns true if all buffers were removed.
833  */
834 int remove_inode_buffers(struct inode *inode)
835 {
836         int ret = 1;
837
838         if (inode_has_buffers(inode)) {
839                 struct address_space *mapping = &inode->i_data;
840                 struct list_head *list = &mapping->private_list;
841                 struct address_space *buffer_mapping = mapping->private_data;
842
843                 spin_lock(&buffer_mapping->private_lock);
844                 while (!list_empty(list)) {
845                         struct buffer_head *bh = BH_ENTRY(list->next);
846                         if (buffer_dirty(bh)) {
847                                 ret = 0;
848                                 break;
849                         }
850                         __remove_assoc_queue(bh);
851                 }
852                 spin_unlock(&buffer_mapping->private_lock);
853         }
854         return ret;
855 }
856
857 /*
858  * Create the appropriate buffers when given a page for data area and
859  * the size of each buffer.. Use the bh->b_this_page linked list to
860  * follow the buffers created.  Return NULL if unable to create more
861  * buffers.
862  *
863  * The retry flag is used to differentiate async IO (paging, swapping)
864  * which may not fail from ordinary buffer allocations.
865  */
866 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
867                 int retry)
868 {
869         struct buffer_head *bh, *head;
870         long offset;
871
872 try_again:
873         head = NULL;
874         offset = PAGE_SIZE;
875         while ((offset -= size) >= 0) {
876                 bh = alloc_buffer_head(GFP_NOFS);
877                 if (!bh)
878                         goto no_grow;
879
880                 bh->b_this_page = head;
881                 bh->b_blocknr = -1;
882                 head = bh;
883
884                 bh->b_size = size;
885
886                 /* Link the buffer to its page */
887                 set_bh_page(bh, page, offset);
888         }
889         return head;
890 /*
891  * In case anything failed, we just free everything we got.
892  */
893 no_grow:
894         if (head) {
895                 do {
896                         bh = head;
897                         head = head->b_this_page;
898                         free_buffer_head(bh);
899                 } while (head);
900         }
901
902         /*
903          * Return failure for non-async IO requests.  Async IO requests
904          * are not allowed to fail, so we have to wait until buffer heads
905          * become available.  But we don't want tasks sleeping with 
906          * partially complete buffers, so all were released above.
907          */
908         if (!retry)
909                 return NULL;
910
911         /* We're _really_ low on memory. Now we just
912          * wait for old buffer heads to become free due to
913          * finishing IO.  Since this is an async request and
914          * the reserve list is empty, we're sure there are 
915          * async buffer heads in use.
916          */
917         free_more_memory();
918         goto try_again;
919 }
920 EXPORT_SYMBOL_GPL(alloc_page_buffers);
921
922 static inline void
923 link_dev_buffers(struct page *page, struct buffer_head *head)
924 {
925         struct buffer_head *bh, *tail;
926
927         bh = head;
928         do {
929                 tail = bh;
930                 bh = bh->b_this_page;
931         } while (bh);
932         tail->b_this_page = head;
933         attach_page_buffers(page, head);
934 }
935
936 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
937 {
938         sector_t retval = ~((sector_t)0);
939         loff_t sz = i_size_read(bdev->bd_inode);
940
941         if (sz) {
942                 unsigned int sizebits = blksize_bits(size);
943                 retval = (sz >> sizebits);
944         }
945         return retval;
946 }
947
948 /*
949  * Initialise the state of a blockdev page's buffers.
950  */ 
951 static sector_t
952 init_page_buffers(struct page *page, struct block_device *bdev,
953                         sector_t block, int size)
954 {
955         struct buffer_head *head = page_buffers(page);
956         struct buffer_head *bh = head;
957         int uptodate = PageUptodate(page);
958         sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
959
960         do {
961                 if (!buffer_mapped(bh)) {
962                         init_buffer(bh, NULL, NULL);
963                         bh->b_bdev = bdev;
964                         bh->b_blocknr = block;
965                         if (uptodate)
966                                 set_buffer_uptodate(bh);
967                         if (block < end_block)
968                                 set_buffer_mapped(bh);
969                 }
970                 block++;
971                 bh = bh->b_this_page;
972         } while (bh != head);
973
974         /*
975          * Caller needs to validate requested block against end of device.
976          */
977         return end_block;
978 }
979
980 /*
981  * Create the page-cache page that contains the requested block.
982  *
983  * This is used purely for blockdev mappings.
984  */
985 static int
986 grow_dev_page(struct block_device *bdev, sector_t block,
987               pgoff_t index, int size, int sizebits, gfp_t gfp)
988 {
989         struct inode *inode = bdev->bd_inode;
990         struct page *page;
991         struct buffer_head *bh;
992         sector_t end_block;
993         int ret = 0;            /* Will call free_more_memory() */
994         gfp_t gfp_mask;
995
996         gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
997
998         /*
999          * XXX: __getblk_slow() can not really deal with failure and
1000          * will endlessly loop on improvised global reclaim.  Prefer
1001          * looping in the allocator rather than here, at least that
1002          * code knows what it's doing.
1003          */
1004         gfp_mask |= __GFP_NOFAIL;
1005
1006         page = find_or_create_page(inode->i_mapping, index, gfp_mask);
1007         if (!page)
1008                 return ret;
1009
1010         BUG_ON(!PageLocked(page));
1011
1012         if (page_has_buffers(page)) {
1013                 bh = page_buffers(page);
1014                 if (bh->b_size == size) {
1015                         end_block = init_page_buffers(page, bdev,
1016                                                 (sector_t)index << sizebits,
1017                                                 size);
1018                         goto done;
1019                 }
1020                 if (!try_to_free_buffers(page))
1021                         goto failed;
1022         }
1023
1024         /*
1025          * Allocate some buffers for this page
1026          */
1027         bh = alloc_page_buffers(page, size, 0);
1028         if (!bh)
1029                 goto failed;
1030
1031         /*
1032          * Link the page to the buffers and initialise them.  Take the
1033          * lock to be atomic wrt __find_get_block(), which does not
1034          * run under the page lock.
1035          */
1036         spin_lock(&inode->i_mapping->private_lock);
1037         link_dev_buffers(page, bh);
1038         end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
1039                         size);
1040         spin_unlock(&inode->i_mapping->private_lock);
1041 done:
1042         ret = (block < end_block) ? 1 : -ENXIO;
1043 failed:
1044         unlock_page(page);
1045         put_page(page);
1046         return ret;
1047 }
1048
1049 /*
1050  * Create buffers for the specified block device block's page.  If
1051  * that page was dirty, the buffers are set dirty also.
1052  */
1053 static int
1054 grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
1055 {
1056         pgoff_t index;
1057         int sizebits;
1058
1059         sizebits = -1;
1060         do {
1061                 sizebits++;
1062         } while ((size << sizebits) < PAGE_SIZE);
1063
1064         index = block >> sizebits;
1065
1066         /*
1067          * Check for a block which wants to lie outside our maximum possible
1068          * pagecache index.  (this comparison is done using sector_t types).
1069          */
1070         if (unlikely(index != block >> sizebits)) {
1071                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1072                         "device %pg\n",
1073                         __func__, (unsigned long long)block,
1074                         bdev);
1075                 return -EIO;
1076         }
1077
1078         /* Create a page with the proper size buffers.. */
1079         return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1080 }
1081
1082 static struct buffer_head *
1083 __getblk_slow(struct block_device *bdev, sector_t block,
1084              unsigned size, gfp_t gfp)
1085 {
1086         /* Size must be multiple of hard sectorsize */
1087         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1088                         (size < 512 || size > PAGE_SIZE))) {
1089                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1090                                         size);
1091                 printk(KERN_ERR "logical block size: %d\n",
1092                                         bdev_logical_block_size(bdev));
1093
1094                 dump_stack();
1095                 return NULL;
1096         }
1097
1098         for (;;) {
1099                 struct buffer_head *bh;
1100                 int ret;
1101
1102                 bh = __find_get_block(bdev, block, size);
1103                 if (bh)
1104                         return bh;
1105
1106                 ret = grow_buffers(bdev, block, size, gfp);
1107                 if (ret < 0)
1108                         return NULL;
1109                 if (ret == 0)
1110                         free_more_memory();
1111         }
1112 }
1113
1114 /*
1115  * The relationship between dirty buffers and dirty pages:
1116  *
1117  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1118  * the page is tagged dirty in its radix tree.
1119  *
1120  * At all times, the dirtiness of the buffers represents the dirtiness of
1121  * subsections of the page.  If the page has buffers, the page dirty bit is
1122  * merely a hint about the true dirty state.
1123  *
1124  * When a page is set dirty in its entirety, all its buffers are marked dirty
1125  * (if the page has buffers).
1126  *
1127  * When a buffer is marked dirty, its page is dirtied, but the page's other
1128  * buffers are not.
1129  *
1130  * Also.  When blockdev buffers are explicitly read with bread(), they
1131  * individually become uptodate.  But their backing page remains not
1132  * uptodate - even if all of its buffers are uptodate.  A subsequent
1133  * block_read_full_page() against that page will discover all the uptodate
1134  * buffers, will set the page uptodate and will perform no I/O.
1135  */
1136
1137 /**
1138  * mark_buffer_dirty - mark a buffer_head as needing writeout
1139  * @bh: the buffer_head to mark dirty
1140  *
1141  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1142  * backing page dirty, then tag the page as dirty in its address_space's radix
1143  * tree and then attach the address_space's inode to its superblock's dirty
1144  * inode list.
1145  *
1146  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1147  * mapping->tree_lock and mapping->host->i_lock.
1148  */
1149 void mark_buffer_dirty(struct buffer_head *bh)
1150 {
1151         WARN_ON_ONCE(!buffer_uptodate(bh));
1152
1153         trace_block_dirty_buffer(bh);
1154
1155         /*
1156          * Very *carefully* optimize the it-is-already-dirty case.
1157          *
1158          * Don't let the final "is it dirty" escape to before we
1159          * perhaps modified the buffer.
1160          */
1161         if (buffer_dirty(bh)) {
1162                 smp_mb();
1163                 if (buffer_dirty(bh))
1164                         return;
1165         }
1166
1167         if (!test_set_buffer_dirty(bh)) {
1168                 struct page *page = bh->b_page;
1169                 struct address_space *mapping = NULL;
1170
1171                 lock_page_memcg(page);
1172                 if (!TestSetPageDirty(page)) {
1173                         mapping = page_mapping(page);
1174                         if (mapping)
1175                                 __set_page_dirty(page, mapping, 0);
1176                 }
1177                 unlock_page_memcg(page);
1178                 if (mapping)
1179                         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1180         }
1181 }
1182 EXPORT_SYMBOL(mark_buffer_dirty);
1183
1184 /*
1185  * Decrement a buffer_head's reference count.  If all buffers against a page
1186  * have zero reference count, are clean and unlocked, and if the page is clean
1187  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1188  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1189  * a page but it ends up not being freed, and buffers may later be reattached).
1190  */
1191 void __brelse(struct buffer_head * buf)
1192 {
1193         if (atomic_read(&buf->b_count)) {
1194                 put_bh(buf);
1195                 return;
1196         }
1197         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1198 }
1199 EXPORT_SYMBOL(__brelse);
1200
1201 /*
1202  * bforget() is like brelse(), except it discards any
1203  * potentially dirty data.
1204  */
1205 void __bforget(struct buffer_head *bh)
1206 {
1207         clear_buffer_dirty(bh);
1208         if (bh->b_assoc_map) {
1209                 struct address_space *buffer_mapping = bh->b_page->mapping;
1210
1211                 spin_lock(&buffer_mapping->private_lock);
1212                 list_del_init(&bh->b_assoc_buffers);
1213                 bh->b_assoc_map = NULL;
1214                 spin_unlock(&buffer_mapping->private_lock);
1215         }
1216         __brelse(bh);
1217 }
1218 EXPORT_SYMBOL(__bforget);
1219
1220 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1221 {
1222         lock_buffer(bh);
1223         if (buffer_uptodate(bh)) {
1224                 unlock_buffer(bh);
1225                 return bh;
1226         } else {
1227                 get_bh(bh);
1228                 bh->b_end_io = end_buffer_read_sync;
1229                 submit_bh(REQ_OP_READ, 0, bh);
1230                 wait_on_buffer(bh);
1231                 if (buffer_uptodate(bh))
1232                         return bh;
1233         }
1234         brelse(bh);
1235         return NULL;
1236 }
1237
1238 /*
1239  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1240  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1241  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1242  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1243  * CPU's LRUs at the same time.
1244  *
1245  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1246  * sb_find_get_block().
1247  *
1248  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1249  * a local interrupt disable for that.
1250  */
1251
1252 #define BH_LRU_SIZE     16
1253
1254 struct bh_lru {
1255         struct buffer_head *bhs[BH_LRU_SIZE];
1256 };
1257
1258 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1259
1260 #ifdef CONFIG_SMP
1261 #define bh_lru_lock()   local_irq_disable()
1262 #define bh_lru_unlock() local_irq_enable()
1263 #else
1264 #define bh_lru_lock()   preempt_disable()
1265 #define bh_lru_unlock() preempt_enable()
1266 #endif
1267
1268 static inline void check_irqs_on(void)
1269 {
1270 #ifdef irqs_disabled
1271         BUG_ON(irqs_disabled());
1272 #endif
1273 }
1274
1275 /*
1276  * The LRU management algorithm is dopey-but-simple.  Sorry.
1277  */
1278 static void bh_lru_install(struct buffer_head *bh)
1279 {
1280         struct buffer_head *evictee = NULL;
1281
1282         check_irqs_on();
1283         bh_lru_lock();
1284         if (__this_cpu_read(bh_lrus.bhs[0]) != bh) {
1285                 struct buffer_head *bhs[BH_LRU_SIZE];
1286                 int in;
1287                 int out = 0;
1288
1289                 get_bh(bh);
1290                 bhs[out++] = bh;
1291                 for (in = 0; in < BH_LRU_SIZE; in++) {
1292                         struct buffer_head *bh2 =
1293                                 __this_cpu_read(bh_lrus.bhs[in]);
1294
1295                         if (bh2 == bh) {
1296                                 __brelse(bh2);
1297                         } else {
1298                                 if (out >= BH_LRU_SIZE) {
1299                                         BUG_ON(evictee != NULL);
1300                                         evictee = bh2;
1301                                 } else {
1302                                         bhs[out++] = bh2;
1303                                 }
1304                         }
1305                 }
1306                 while (out < BH_LRU_SIZE)
1307                         bhs[out++] = NULL;
1308                 memcpy(this_cpu_ptr(&bh_lrus.bhs), bhs, sizeof(bhs));
1309         }
1310         bh_lru_unlock();
1311
1312         if (evictee)
1313                 __brelse(evictee);
1314 }
1315
1316 /*
1317  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1318  */
1319 static struct buffer_head *
1320 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1321 {
1322         struct buffer_head *ret = NULL;
1323         unsigned int i;
1324
1325         check_irqs_on();
1326         bh_lru_lock();
1327         for (i = 0; i < BH_LRU_SIZE; i++) {
1328                 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1329
1330                 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1331                     bh->b_size == size) {
1332                         if (i) {
1333                                 while (i) {
1334                                         __this_cpu_write(bh_lrus.bhs[i],
1335                                                 __this_cpu_read(bh_lrus.bhs[i - 1]));
1336                                         i--;
1337                                 }
1338                                 __this_cpu_write(bh_lrus.bhs[0], bh);
1339                         }
1340                         get_bh(bh);
1341                         ret = bh;
1342                         break;
1343                 }
1344         }
1345         bh_lru_unlock();
1346         return ret;
1347 }
1348
1349 /*
1350  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1351  * it in the LRU and mark it as accessed.  If it is not present then return
1352  * NULL
1353  */
1354 struct buffer_head *
1355 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1356 {
1357         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1358
1359         if (bh == NULL) {
1360                 /* __find_get_block_slow will mark the page accessed */
1361                 bh = __find_get_block_slow(bdev, block);
1362                 if (bh)
1363                         bh_lru_install(bh);
1364         } else
1365                 touch_buffer(bh);
1366
1367         return bh;
1368 }
1369 EXPORT_SYMBOL(__find_get_block);
1370
1371 /*
1372  * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1373  * which corresponds to the passed block_device, block and size. The
1374  * returned buffer has its reference count incremented.
1375  *
1376  * __getblk_gfp() will lock up the machine if grow_dev_page's
1377  * try_to_free_buffers() attempt is failing.  FIXME, perhaps?
1378  */
1379 struct buffer_head *
1380 __getblk_gfp(struct block_device *bdev, sector_t block,
1381              unsigned size, gfp_t gfp)
1382 {
1383         struct buffer_head *bh = __find_get_block(bdev, block, size);
1384
1385         might_sleep();
1386         if (bh == NULL)
1387                 bh = __getblk_slow(bdev, block, size, gfp);
1388         return bh;
1389 }
1390 EXPORT_SYMBOL(__getblk_gfp);
1391
1392 /*
1393  * Do async read-ahead on a buffer..
1394  */
1395 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1396 {
1397         struct buffer_head *bh = __getblk(bdev, block, size);
1398         if (likely(bh)) {
1399                 ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1400                 brelse(bh);
1401         }
1402 }
1403 EXPORT_SYMBOL(__breadahead);
1404
1405 /**
1406  *  __bread_gfp() - reads a specified block and returns the bh
1407  *  @bdev: the block_device to read from
1408  *  @block: number of block
1409  *  @size: size (in bytes) to read
1410  *  @gfp: page allocation flag
1411  *
1412  *  Reads a specified block, and returns buffer head that contains it.
1413  *  The page cache can be allocated from non-movable area
1414  *  not to prevent page migration if you set gfp to zero.
1415  *  It returns NULL if the block was unreadable.
1416  */
1417 struct buffer_head *
1418 __bread_gfp(struct block_device *bdev, sector_t block,
1419                    unsigned size, gfp_t gfp)
1420 {
1421         struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1422
1423         if (likely(bh) && !buffer_uptodate(bh))
1424                 bh = __bread_slow(bh);
1425         return bh;
1426 }
1427 EXPORT_SYMBOL(__bread_gfp);
1428
1429 /*
1430  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1431  * This doesn't race because it runs in each cpu either in irq
1432  * or with preempt disabled.
1433  */
1434 static void invalidate_bh_lru(void *arg)
1435 {
1436         struct bh_lru *b = &get_cpu_var(bh_lrus);
1437         int i;
1438
1439         for (i = 0; i < BH_LRU_SIZE; i++) {
1440                 brelse(b->bhs[i]);
1441                 b->bhs[i] = NULL;
1442         }
1443         put_cpu_var(bh_lrus);
1444 }
1445
1446 static bool has_bh_in_lru(int cpu, void *dummy)
1447 {
1448         struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1449         int i;
1450         
1451         for (i = 0; i < BH_LRU_SIZE; i++) {
1452                 if (b->bhs[i])
1453                         return 1;
1454         }
1455
1456         return 0;
1457 }
1458
1459 void invalidate_bh_lrus(void)
1460 {
1461         on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
1462 }
1463 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1464
1465 void set_bh_page(struct buffer_head *bh,
1466                 struct page *page, unsigned long offset)
1467 {
1468         bh->b_page = page;
1469         BUG_ON(offset >= PAGE_SIZE);
1470         if (PageHighMem(page))
1471                 /*
1472                  * This catches illegal uses and preserves the offset:
1473                  */
1474                 bh->b_data = (char *)(0 + offset);
1475         else
1476                 bh->b_data = page_address(page) + offset;
1477 }
1478 EXPORT_SYMBOL(set_bh_page);
1479
1480 /*
1481  * Called when truncating a buffer on a page completely.
1482  */
1483
1484 /* Bits that are cleared during an invalidate */
1485 #define BUFFER_FLAGS_DISCARD \
1486         (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1487          1 << BH_Delay | 1 << BH_Unwritten)
1488
1489 static void discard_buffer(struct buffer_head * bh)
1490 {
1491         unsigned long b_state, b_state_old;
1492
1493         lock_buffer(bh);
1494         clear_buffer_dirty(bh);
1495         bh->b_bdev = NULL;
1496         b_state = bh->b_state;
1497         for (;;) {
1498                 b_state_old = cmpxchg(&bh->b_state, b_state,
1499                                       (b_state & ~BUFFER_FLAGS_DISCARD));
1500                 if (b_state_old == b_state)
1501                         break;
1502                 b_state = b_state_old;
1503         }
1504         unlock_buffer(bh);
1505 }
1506
1507 /**
1508  * block_invalidatepage - invalidate part or all of a buffer-backed page
1509  *
1510  * @page: the page which is affected
1511  * @offset: start of the range to invalidate
1512  * @length: length of the range to invalidate
1513  *
1514  * block_invalidatepage() is called when all or part of the page has become
1515  * invalidated by a truncate operation.
1516  *
1517  * block_invalidatepage() does not have to release all buffers, but it must
1518  * ensure that no dirty buffer is left outside @offset and that no I/O
1519  * is underway against any of the blocks which are outside the truncation
1520  * point.  Because the caller is about to free (and possibly reuse) those
1521  * blocks on-disk.
1522  */
1523 void block_invalidatepage(struct page *page, unsigned int offset,
1524                           unsigned int length)
1525 {
1526         struct buffer_head *head, *bh, *next;
1527         unsigned int curr_off = 0;
1528         unsigned int stop = length + offset;
1529
1530         BUG_ON(!PageLocked(page));
1531         if (!page_has_buffers(page))
1532                 goto out;
1533
1534         /*
1535          * Check for overflow
1536          */
1537         BUG_ON(stop > PAGE_SIZE || stop < length);
1538
1539         head = page_buffers(page);
1540         bh = head;
1541         do {
1542                 unsigned int next_off = curr_off + bh->b_size;
1543                 next = bh->b_this_page;
1544
1545                 /*
1546                  * Are we still fully in range ?
1547                  */
1548                 if (next_off > stop)
1549                         goto out;
1550
1551                 /*
1552                  * is this block fully invalidated?
1553                  */
1554                 if (offset <= curr_off)
1555                         discard_buffer(bh);
1556                 curr_off = next_off;
1557                 bh = next;
1558         } while (bh != head);
1559
1560         /*
1561          * We release buffers only if the entire page is being invalidated.
1562          * The get_block cached value has been unconditionally invalidated,
1563          * so real IO is not possible anymore.
1564          */
1565         if (offset == 0)
1566                 try_to_release_page(page, 0);
1567 out:
1568         return;
1569 }
1570 EXPORT_SYMBOL(block_invalidatepage);
1571
1572
1573 /*
1574  * We attach and possibly dirty the buffers atomically wrt
1575  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1576  * is already excluded via the page lock.
1577  */
1578 void create_empty_buffers(struct page *page,
1579                         unsigned long blocksize, unsigned long b_state)
1580 {
1581         struct buffer_head *bh, *head, *tail;
1582
1583         head = alloc_page_buffers(page, blocksize, 1);
1584         bh = head;
1585         do {
1586                 bh->b_state |= b_state;
1587                 tail = bh;
1588                 bh = bh->b_this_page;
1589         } while (bh);
1590         tail->b_this_page = head;
1591
1592         spin_lock(&page->mapping->private_lock);
1593         if (PageUptodate(page) || PageDirty(page)) {
1594                 bh = head;
1595                 do {
1596                         if (PageDirty(page))
1597                                 set_buffer_dirty(bh);
1598                         if (PageUptodate(page))
1599                                 set_buffer_uptodate(bh);
1600                         bh = bh->b_this_page;
1601                 } while (bh != head);
1602         }
1603         attach_page_buffers(page, head);
1604         spin_unlock(&page->mapping->private_lock);
1605 }
1606 EXPORT_SYMBOL(create_empty_buffers);
1607
1608 /*
1609  * We are taking a block for data and we don't want any output from any
1610  * buffer-cache aliases starting from return from that function and
1611  * until the moment when something will explicitly mark the buffer
1612  * dirty (hopefully that will not happen until we will free that block ;-)
1613  * We don't even need to mark it not-uptodate - nobody can expect
1614  * anything from a newly allocated buffer anyway. We used to used
1615  * unmap_buffer() for such invalidation, but that was wrong. We definitely
1616  * don't want to mark the alias unmapped, for example - it would confuse
1617  * anyone who might pick it with bread() afterwards...
1618  *
1619  * Also..  Note that bforget() doesn't lock the buffer.  So there can
1620  * be writeout I/O going on against recently-freed buffers.  We don't
1621  * wait on that I/O in bforget() - it's more efficient to wait on the I/O
1622  * only if we really need to.  That happens here.
1623  */
1624 void unmap_underlying_metadata(struct block_device *bdev, sector_t block)
1625 {
1626         struct buffer_head *old_bh;
1627
1628         might_sleep();
1629
1630         old_bh = __find_get_block_slow(bdev, block);
1631         if (old_bh) {
1632                 clear_buffer_dirty(old_bh);
1633                 wait_on_buffer(old_bh);
1634                 clear_buffer_req(old_bh);
1635                 __brelse(old_bh);
1636         }
1637 }
1638 EXPORT_SYMBOL(unmap_underlying_metadata);
1639
1640 /**
1641  * clean_bdev_aliases: clean a range of buffers in block device
1642  * @bdev: Block device to clean buffers in
1643  * @block: Start of a range of blocks to clean
1644  * @len: Number of blocks to clean
1645  *
1646  * We are taking a range of blocks for data and we don't want writeback of any
1647  * buffer-cache aliases starting from return from this function and until the
1648  * moment when something will explicitly mark the buffer dirty (hopefully that
1649  * will not happen until we will free that block ;-) We don't even need to mark
1650  * it not-uptodate - nobody can expect anything from a newly allocated buffer
1651  * anyway. We used to use unmap_buffer() for such invalidation, but that was
1652  * wrong. We definitely don't want to mark the alias unmapped, for example - it
1653  * would confuse anyone who might pick it with bread() afterwards...
1654  *
1655  * Also..  Note that bforget() doesn't lock the buffer.  So there can be
1656  * writeout I/O going on against recently-freed buffers.  We don't wait on that
1657  * I/O in bforget() - it's more efficient to wait on the I/O only if we really
1658  * need to.  That happens here.
1659  */
1660 void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1661 {
1662         struct inode *bd_inode = bdev->bd_inode;
1663         struct address_space *bd_mapping = bd_inode->i_mapping;
1664         struct pagevec pvec;
1665         pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1666         pgoff_t end;
1667         int i;
1668         struct buffer_head *bh;
1669         struct buffer_head *head;
1670
1671         end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1672         pagevec_init(&pvec, 0);
1673         while (index <= end && pagevec_lookup(&pvec, bd_mapping, index,
1674                         min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
1675                 for (i = 0; i < pagevec_count(&pvec); i++) {
1676                         struct page *page = pvec.pages[i];
1677
1678                         index = page->index;
1679                         if (index > end)
1680                                 break;
1681                         if (!page_has_buffers(page))
1682                                 continue;
1683                         /*
1684                          * We use page lock instead of bd_mapping->private_lock
1685                          * to pin buffers here since we can afford to sleep and
1686                          * it scales better than a global spinlock lock.
1687                          */
1688                         lock_page(page);
1689                         /* Recheck when the page is locked which pins bhs */
1690                         if (!page_has_buffers(page))
1691                                 goto unlock_page;
1692                         head = page_buffers(page);
1693                         bh = head;
1694                         do {
1695                                 if (!buffer_mapped(bh))
1696                                         goto next;
1697                                 if (bh->b_blocknr >= block + len)
1698                                         break;
1699                                 clear_buffer_dirty(bh);
1700                                 wait_on_buffer(bh);
1701                                 clear_buffer_req(bh);
1702 next:
1703                                 bh = bh->b_this_page;
1704                         } while (bh != head);
1705 unlock_page:
1706                         unlock_page(page);
1707                 }
1708                 pagevec_release(&pvec);
1709                 cond_resched();
1710                 index++;
1711         }
1712 }
1713 EXPORT_SYMBOL(clean_bdev_aliases);
1714
1715 /*
1716  * Size is a power-of-two in the range 512..PAGE_SIZE,
1717  * and the case we care about most is PAGE_SIZE.
1718  *
1719  * So this *could* possibly be written with those
1720  * constraints in mind (relevant mostly if some
1721  * architecture has a slow bit-scan instruction)
1722  */
1723 static inline int block_size_bits(unsigned int blocksize)
1724 {
1725         return ilog2(blocksize);
1726 }
1727
1728 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1729 {
1730         BUG_ON(!PageLocked(page));
1731
1732         if (!page_has_buffers(page))
1733                 create_empty_buffers(page, 1 << ACCESS_ONCE(inode->i_blkbits), b_state);
1734         return page_buffers(page);
1735 }
1736
1737 /*
1738  * NOTE! All mapped/uptodate combinations are valid:
1739  *
1740  *      Mapped  Uptodate        Meaning
1741  *
1742  *      No      No              "unknown" - must do get_block()
1743  *      No      Yes             "hole" - zero-filled
1744  *      Yes     No              "allocated" - allocated on disk, not read in
1745  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1746  *
1747  * "Dirty" is valid only with the last case (mapped+uptodate).
1748  */
1749
1750 /*
1751  * While block_write_full_page is writing back the dirty buffers under
1752  * the page lock, whoever dirtied the buffers may decide to clean them
1753  * again at any time.  We handle that by only looking at the buffer
1754  * state inside lock_buffer().
1755  *
1756  * If block_write_full_page() is called for regular writeback
1757  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1758  * locked buffer.   This only can happen if someone has written the buffer
1759  * directly, with submit_bh().  At the address_space level PageWriteback
1760  * prevents this contention from occurring.
1761  *
1762  * If block_write_full_page() is called with wbc->sync_mode ==
1763  * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1764  * causes the writes to be flagged as synchronous writes.
1765  */
1766 int __block_write_full_page(struct inode *inode, struct page *page,
1767                         get_block_t *get_block, struct writeback_control *wbc,
1768                         bh_end_io_t *handler)
1769 {
1770         int err;
1771         sector_t block;
1772         sector_t last_block;
1773         struct buffer_head *bh, *head;
1774         unsigned int blocksize, bbits;
1775         int nr_underway = 0;
1776         int write_flags = wbc_to_write_flags(wbc);
1777
1778         head = create_page_buffers(page, inode,
1779                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1780
1781         /*
1782          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1783          * here, and the (potentially unmapped) buffers may become dirty at
1784          * any time.  If a buffer becomes dirty here after we've inspected it
1785          * then we just miss that fact, and the page stays dirty.
1786          *
1787          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1788          * handle that here by just cleaning them.
1789          */
1790
1791         bh = head;
1792         blocksize = bh->b_size;
1793         bbits = block_size_bits(blocksize);
1794
1795         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1796         last_block = (i_size_read(inode) - 1) >> bbits;
1797
1798         /*
1799          * Get all the dirty buffers mapped to disk addresses and
1800          * handle any aliases from the underlying blockdev's mapping.
1801          */
1802         do {
1803                 if (block > last_block) {
1804                         /*
1805                          * mapped buffers outside i_size will occur, because
1806                          * this page can be outside i_size when there is a
1807                          * truncate in progress.
1808                          */
1809                         /*
1810                          * The buffer was zeroed by block_write_full_page()
1811                          */
1812                         clear_buffer_dirty(bh);
1813                         set_buffer_uptodate(bh);
1814                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1815                            buffer_dirty(bh)) {
1816                         WARN_ON(bh->b_size != blocksize);
1817                         err = get_block(inode, block, bh, 1);
1818                         if (err)
1819                                 goto recover;
1820                         clear_buffer_delay(bh);
1821                         if (buffer_new(bh)) {
1822                                 /* blockdev mappings never come here */
1823                                 clear_buffer_new(bh);
1824                                 clean_bdev_bh_alias(bh);
1825                         }
1826                 }
1827                 bh = bh->b_this_page;
1828                 block++;
1829         } while (bh != head);
1830
1831         do {
1832                 if (!buffer_mapped(bh))
1833                         continue;
1834                 /*
1835                  * If it's a fully non-blocking write attempt and we cannot
1836                  * lock the buffer then redirty the page.  Note that this can
1837                  * potentially cause a busy-wait loop from writeback threads
1838                  * and kswapd activity, but those code paths have their own
1839                  * higher-level throttling.
1840                  */
1841                 if (wbc->sync_mode != WB_SYNC_NONE) {
1842                         lock_buffer(bh);
1843                 } else if (!trylock_buffer(bh)) {
1844                         redirty_page_for_writepage(wbc, page);
1845                         continue;
1846                 }
1847                 if (test_clear_buffer_dirty(bh)) {
1848                         mark_buffer_async_write_endio(bh, handler);
1849                 } else {
1850                         unlock_buffer(bh);
1851                 }
1852         } while ((bh = bh->b_this_page) != head);
1853
1854         /*
1855          * The page and its buffers are protected by PageWriteback(), so we can
1856          * drop the bh refcounts early.
1857          */
1858         BUG_ON(PageWriteback(page));
1859         set_page_writeback(page);
1860
1861         do {
1862                 struct buffer_head *next = bh->b_this_page;
1863                 if (buffer_async_write(bh)) {
1864                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, 0, wbc);
1865                         nr_underway++;
1866                 }
1867                 bh = next;
1868         } while (bh != head);
1869         unlock_page(page);
1870
1871         err = 0;
1872 done:
1873         if (nr_underway == 0) {
1874                 /*
1875                  * The page was marked dirty, but the buffers were
1876                  * clean.  Someone wrote them back by hand with
1877                  * ll_rw_block/submit_bh.  A rare case.
1878                  */
1879                 end_page_writeback(page);
1880
1881                 /*
1882                  * The page and buffer_heads can be released at any time from
1883                  * here on.
1884                  */
1885         }
1886         return err;
1887
1888 recover:
1889         /*
1890          * ENOSPC, or some other error.  We may already have added some
1891          * blocks to the file, so we need to write these out to avoid
1892          * exposing stale data.
1893          * The page is currently locked and not marked for writeback
1894          */
1895         bh = head;
1896         /* Recovery: lock and submit the mapped buffers */
1897         do {
1898                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1899                     !buffer_delay(bh)) {
1900                         lock_buffer(bh);
1901                         mark_buffer_async_write_endio(bh, handler);
1902                 } else {
1903                         /*
1904                          * The buffer may have been set dirty during
1905                          * attachment to a dirty page.
1906                          */
1907                         clear_buffer_dirty(bh);
1908                 }
1909         } while ((bh = bh->b_this_page) != head);
1910         SetPageError(page);
1911         BUG_ON(PageWriteback(page));
1912         mapping_set_error(page->mapping, err);
1913         set_page_writeback(page);
1914         do {
1915                 struct buffer_head *next = bh->b_this_page;
1916                 if (buffer_async_write(bh)) {
1917                         clear_buffer_dirty(bh);
1918                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh, 0, wbc);
1919                         nr_underway++;
1920                 }
1921                 bh = next;
1922         } while (bh != head);
1923         unlock_page(page);
1924         goto done;
1925 }
1926 EXPORT_SYMBOL(__block_write_full_page);
1927
1928 /*
1929  * If a page has any new buffers, zero them out here, and mark them uptodate
1930  * and dirty so they'll be written out (in order to prevent uninitialised
1931  * block data from leaking). And clear the new bit.
1932  */
1933 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1934 {
1935         unsigned int block_start, block_end;
1936         struct buffer_head *head, *bh;
1937
1938         BUG_ON(!PageLocked(page));
1939         if (!page_has_buffers(page))
1940                 return;
1941
1942         bh = head = page_buffers(page);
1943         block_start = 0;
1944         do {
1945                 block_end = block_start + bh->b_size;
1946
1947                 if (buffer_new(bh)) {
1948                         if (block_end > from && block_start < to) {
1949                                 if (!PageUptodate(page)) {
1950                                         unsigned start, size;
1951
1952                                         start = max(from, block_start);
1953                                         size = min(to, block_end) - start;
1954
1955                                         zero_user(page, start, size);
1956                                         set_buffer_uptodate(bh);
1957                                 }
1958
1959                                 clear_buffer_new(bh);
1960                                 mark_buffer_dirty(bh);
1961                         }
1962                 }
1963
1964                 block_start = block_end;
1965                 bh = bh->b_this_page;
1966         } while (bh != head);
1967 }
1968 EXPORT_SYMBOL(page_zero_new_buffers);
1969
1970 static void
1971 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
1972                 struct iomap *iomap)
1973 {
1974         loff_t offset = block << inode->i_blkbits;
1975
1976         bh->b_bdev = iomap->bdev;
1977
1978         /*
1979          * Block points to offset in file we need to map, iomap contains
1980          * the offset at which the map starts. If the map ends before the
1981          * current block, then do not map the buffer and let the caller
1982          * handle it.
1983          */
1984         BUG_ON(offset >= iomap->offset + iomap->length);
1985
1986         switch (iomap->type) {
1987         case IOMAP_HOLE:
1988                 /*
1989                  * If the buffer is not up to date or beyond the current EOF,
1990                  * we need to mark it as new to ensure sub-block zeroing is
1991                  * executed if necessary.
1992                  */
1993                 if (!buffer_uptodate(bh) ||
1994                     (offset >= i_size_read(inode)))
1995                         set_buffer_new(bh);
1996                 break;
1997         case IOMAP_DELALLOC:
1998                 if (!buffer_uptodate(bh) ||
1999                     (offset >= i_size_read(inode)))
2000                         set_buffer_new(bh);
2001                 set_buffer_uptodate(bh);
2002                 set_buffer_mapped(bh);
2003                 set_buffer_delay(bh);
2004                 break;
2005         case IOMAP_UNWRITTEN:
2006                 /*
2007                  * For unwritten regions, we always need to ensure that
2008                  * sub-block writes cause the regions in the block we are not
2009                  * writing to are zeroed. Set the buffer as new to ensure this.
2010                  */
2011                 set_buffer_new(bh);
2012                 set_buffer_unwritten(bh);
2013                 /* FALLTHRU */
2014         case IOMAP_MAPPED:
2015                 if (offset >= i_size_read(inode))
2016                         set_buffer_new(bh);
2017                 bh->b_blocknr = (iomap->blkno >> (inode->i_blkbits - 9)) +
2018                                 ((offset - iomap->offset) >> inode->i_blkbits);
2019                 set_buffer_mapped(bh);
2020                 break;
2021         }
2022 }
2023
2024 int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
2025                 get_block_t *get_block, struct iomap *iomap)
2026 {
2027         unsigned from = pos & (PAGE_SIZE - 1);
2028         unsigned to = from + len;
2029         struct inode *inode = page->mapping->host;
2030         unsigned block_start, block_end;
2031         sector_t block;
2032         int err = 0;
2033         unsigned blocksize, bbits;
2034         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
2035
2036         BUG_ON(!PageLocked(page));
2037         BUG_ON(from > PAGE_SIZE);
2038         BUG_ON(to > PAGE_SIZE);
2039         BUG_ON(from > to);
2040
2041         head = create_page_buffers(page, inode, 0);
2042         blocksize = head->b_size;
2043         bbits = block_size_bits(blocksize);
2044
2045         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
2046
2047         for(bh = head, block_start = 0; bh != head || !block_start;
2048             block++, block_start=block_end, bh = bh->b_this_page) {
2049                 block_end = block_start + blocksize;
2050                 if (block_end <= from || block_start >= to) {
2051                         if (PageUptodate(page)) {
2052                                 if (!buffer_uptodate(bh))
2053                                         set_buffer_uptodate(bh);
2054                         }
2055                         continue;
2056                 }
2057                 if (buffer_new(bh))
2058                         clear_buffer_new(bh);
2059                 if (!buffer_mapped(bh)) {
2060                         WARN_ON(bh->b_size != blocksize);
2061                         if (get_block) {
2062                                 err = get_block(inode, block, bh, 1);
2063                                 if (err)
2064                                         break;
2065                         } else {
2066                                 iomap_to_bh(inode, block, bh, iomap);
2067                         }
2068
2069                         if (buffer_new(bh)) {
2070                                 clean_bdev_bh_alias(bh);
2071                                 if (PageUptodate(page)) {
2072                                         clear_buffer_new(bh);
2073                                         set_buffer_uptodate(bh);
2074                                         mark_buffer_dirty(bh);
2075                                         continue;
2076                                 }
2077                                 if (block_end > to || block_start < from)
2078                                         zero_user_segments(page,
2079                                                 to, block_end,
2080                                                 block_start, from);
2081                                 continue;
2082                         }
2083                 }
2084                 if (PageUptodate(page)) {
2085                         if (!buffer_uptodate(bh))
2086                                 set_buffer_uptodate(bh);
2087                         continue; 
2088                 }
2089                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
2090                     !buffer_unwritten(bh) &&
2091                      (block_start < from || block_end > to)) {
2092                         ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2093                         *wait_bh++=bh;
2094                 }
2095         }
2096         /*
2097          * If we issued read requests - let them complete.
2098          */
2099         while(wait_bh > wait) {
2100                 wait_on_buffer(*--wait_bh);
2101                 if (!buffer_uptodate(*wait_bh))
2102                         err = -EIO;
2103         }
2104         if (unlikely(err))
2105                 page_zero_new_buffers(page, from, to);
2106         return err;
2107 }
2108
2109 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2110                 get_block_t *get_block)
2111 {
2112         return __block_write_begin_int(page, pos, len, get_block, NULL);
2113 }
2114 EXPORT_SYMBOL(__block_write_begin);
2115
2116 static int __block_commit_write(struct inode *inode, struct page *page,
2117                 unsigned from, unsigned to)
2118 {
2119         unsigned block_start, block_end;
2120         int partial = 0;
2121         unsigned blocksize;
2122         struct buffer_head *bh, *head;
2123
2124         bh = head = page_buffers(page);
2125         blocksize = bh->b_size;
2126
2127         block_start = 0;
2128         do {
2129                 block_end = block_start + blocksize;
2130                 if (block_end <= from || block_start >= to) {
2131                         if (!buffer_uptodate(bh))
2132                                 partial = 1;
2133                 } else {
2134                         set_buffer_uptodate(bh);
2135                         mark_buffer_dirty(bh);
2136                 }
2137                 clear_buffer_new(bh);
2138
2139                 block_start = block_end;
2140                 bh = bh->b_this_page;
2141         } while (bh != head);
2142
2143         /*
2144          * If this is a partial write which happened to make all buffers
2145          * uptodate then we can optimize away a bogus readpage() for
2146          * the next read(). Here we 'discover' whether the page went
2147          * uptodate as a result of this (potentially partial) write.
2148          */
2149         if (!partial)
2150                 SetPageUptodate(page);
2151         return 0;
2152 }
2153
2154 /*
2155  * block_write_begin takes care of the basic task of block allocation and
2156  * bringing partial write blocks uptodate first.
2157  *
2158  * The filesystem needs to handle block truncation upon failure.
2159  */
2160 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2161                 unsigned flags, struct page **pagep, get_block_t *get_block)
2162 {
2163         pgoff_t index = pos >> PAGE_SHIFT;
2164         struct page *page;
2165         int status;
2166
2167         page = grab_cache_page_write_begin(mapping, index, flags);
2168         if (!page)
2169                 return -ENOMEM;
2170
2171         status = __block_write_begin(page, pos, len, get_block);
2172         if (unlikely(status)) {
2173                 unlock_page(page);
2174                 put_page(page);
2175                 page = NULL;
2176         }
2177
2178         *pagep = page;
2179         return status;
2180 }
2181 EXPORT_SYMBOL(block_write_begin);
2182
2183 int block_write_end(struct file *file, struct address_space *mapping,
2184                         loff_t pos, unsigned len, unsigned copied,
2185                         struct page *page, void *fsdata)
2186 {
2187         struct inode *inode = mapping->host;
2188         unsigned start;
2189
2190         start = pos & (PAGE_SIZE - 1);
2191
2192         if (unlikely(copied < len)) {
2193                 /*
2194                  * The buffers that were written will now be uptodate, so we
2195                  * don't have to worry about a readpage reading them and
2196                  * overwriting a partial write. However if we have encountered
2197                  * a short write and only partially written into a buffer, it
2198                  * will not be marked uptodate, so a readpage might come in and
2199                  * destroy our partial write.
2200                  *
2201                  * Do the simplest thing, and just treat any short write to a
2202                  * non uptodate page as a zero-length write, and force the
2203                  * caller to redo the whole thing.
2204                  */
2205                 if (!PageUptodate(page))
2206                         copied = 0;
2207
2208                 page_zero_new_buffers(page, start+copied, start+len);
2209         }
2210         flush_dcache_page(page);
2211
2212         /* This could be a short (even 0-length) commit */
2213         __block_commit_write(inode, page, start, start+copied);
2214
2215         return copied;
2216 }
2217 EXPORT_SYMBOL(block_write_end);
2218
2219 int generic_write_end(struct file *file, struct address_space *mapping,
2220                         loff_t pos, unsigned len, unsigned copied,
2221                         struct page *page, void *fsdata)
2222 {
2223         struct inode *inode = mapping->host;
2224         loff_t old_size = inode->i_size;
2225         int i_size_changed = 0;
2226
2227         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2228
2229         /*
2230          * No need to use i_size_read() here, the i_size
2231          * cannot change under us because we hold i_mutex.
2232          *
2233          * But it's important to update i_size while still holding page lock:
2234          * page writeout could otherwise come in and zero beyond i_size.
2235          */
2236         if (pos+copied > inode->i_size) {
2237                 i_size_write(inode, pos+copied);
2238                 i_size_changed = 1;
2239         }
2240
2241         unlock_page(page);
2242         put_page(page);
2243
2244         if (old_size < pos)
2245                 pagecache_isize_extended(inode, old_size, pos);
2246         /*
2247          * Don't mark the inode dirty under page lock. First, it unnecessarily
2248          * makes the holding time of page lock longer. Second, it forces lock
2249          * ordering of page lock and transaction start for journaling
2250          * filesystems.
2251          */
2252         if (i_size_changed)
2253                 mark_inode_dirty(inode);
2254
2255         return copied;
2256 }
2257 EXPORT_SYMBOL(generic_write_end);
2258
2259 /*
2260  * block_is_partially_uptodate checks whether buffers within a page are
2261  * uptodate or not.
2262  *
2263  * Returns true if all buffers which correspond to a file portion
2264  * we want to read are uptodate.
2265  */
2266 int block_is_partially_uptodate(struct page *page, unsigned long from,
2267                                         unsigned long count)
2268 {
2269         unsigned block_start, block_end, blocksize;
2270         unsigned to;
2271         struct buffer_head *bh, *head;
2272         int ret = 1;
2273
2274         if (!page_has_buffers(page))
2275                 return 0;
2276
2277         head = page_buffers(page);
2278         blocksize = head->b_size;
2279         to = min_t(unsigned, PAGE_SIZE - from, count);
2280         to = from + to;
2281         if (from < blocksize && to > PAGE_SIZE - blocksize)
2282                 return 0;
2283
2284         bh = head;
2285         block_start = 0;
2286         do {
2287                 block_end = block_start + blocksize;
2288                 if (block_end > from && block_start < to) {
2289                         if (!buffer_uptodate(bh)) {
2290                                 ret = 0;
2291                                 break;
2292                         }
2293                         if (block_end >= to)
2294                                 break;
2295                 }
2296                 block_start = block_end;
2297                 bh = bh->b_this_page;
2298         } while (bh != head);
2299
2300         return ret;
2301 }
2302 EXPORT_SYMBOL(block_is_partially_uptodate);
2303
2304 /*
2305  * Generic "read page" function for block devices that have the normal
2306  * get_block functionality. This is most of the block device filesystems.
2307  * Reads the page asynchronously --- the unlock_buffer() and
2308  * set/clear_buffer_uptodate() functions propagate buffer state into the
2309  * page struct once IO has completed.
2310  */
2311 int block_read_full_page(struct page *page, get_block_t *get_block)
2312 {
2313         struct inode *inode = page->mapping->host;
2314         sector_t iblock, lblock;
2315         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2316         unsigned int blocksize, bbits;
2317         int nr, i;
2318         int fully_mapped = 1;
2319
2320         head = create_page_buffers(page, inode, 0);
2321         blocksize = head->b_size;
2322         bbits = block_size_bits(blocksize);
2323
2324         iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
2325         lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2326         bh = head;
2327         nr = 0;
2328         i = 0;
2329
2330         do {
2331                 if (buffer_uptodate(bh))
2332                         continue;
2333
2334                 if (!buffer_mapped(bh)) {
2335                         int err = 0;
2336
2337                         fully_mapped = 0;
2338                         if (iblock < lblock) {
2339                                 WARN_ON(bh->b_size != blocksize);
2340                                 err = get_block(inode, iblock, bh, 0);
2341                                 if (err)
2342                                         SetPageError(page);
2343                         }
2344                         if (!buffer_mapped(bh)) {
2345                                 zero_user(page, i * blocksize, blocksize);
2346                                 if (!err)
2347                                         set_buffer_uptodate(bh);
2348                                 continue;
2349                         }
2350                         /*
2351                          * get_block() might have updated the buffer
2352                          * synchronously
2353                          */
2354                         if (buffer_uptodate(bh))
2355                                 continue;
2356                 }
2357                 arr[nr++] = bh;
2358         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2359
2360         if (fully_mapped)
2361                 SetPageMappedToDisk(page);
2362
2363         if (!nr) {
2364                 /*
2365                  * All buffers are uptodate - we can set the page uptodate
2366                  * as well. But not if get_block() returned an error.
2367                  */
2368                 if (!PageError(page))
2369                         SetPageUptodate(page);
2370                 unlock_page(page);
2371                 return 0;
2372         }
2373
2374         /* Stage two: lock the buffers */
2375         for (i = 0; i < nr; i++) {
2376                 bh = arr[i];
2377                 lock_buffer(bh);
2378                 mark_buffer_async_read(bh);
2379         }
2380
2381         /*
2382          * Stage 3: start the IO.  Check for uptodateness
2383          * inside the buffer lock in case another process reading
2384          * the underlying blockdev brought it uptodate (the sct fix).
2385          */
2386         for (i = 0; i < nr; i++) {
2387                 bh = arr[i];
2388                 if (buffer_uptodate(bh))
2389                         end_buffer_async_read(bh, 1);
2390                 else
2391                         submit_bh(REQ_OP_READ, 0, bh);
2392         }
2393         return 0;
2394 }
2395 EXPORT_SYMBOL(block_read_full_page);
2396
2397 /* utility function for filesystems that need to do work on expanding
2398  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2399  * deal with the hole.  
2400  */
2401 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2402 {
2403         struct address_space *mapping = inode->i_mapping;
2404         struct page *page;
2405         void *fsdata;
2406         int err;
2407
2408         err = inode_newsize_ok(inode, size);
2409         if (err)
2410                 goto out;
2411
2412         err = pagecache_write_begin(NULL, mapping, size, 0,
2413                                 AOP_FLAG_UNINTERRUPTIBLE|AOP_FLAG_CONT_EXPAND,
2414                                 &page, &fsdata);
2415         if (err)
2416                 goto out;
2417
2418         err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2419         BUG_ON(err > 0);
2420
2421 out:
2422         return err;
2423 }
2424 EXPORT_SYMBOL(generic_cont_expand_simple);
2425
2426 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2427                             loff_t pos, loff_t *bytes)
2428 {
2429         struct inode *inode = mapping->host;
2430         unsigned blocksize = 1 << inode->i_blkbits;
2431         struct page *page;
2432         void *fsdata;
2433         pgoff_t index, curidx;
2434         loff_t curpos;
2435         unsigned zerofrom, offset, len;
2436         int err = 0;
2437
2438         index = pos >> PAGE_SHIFT;
2439         offset = pos & ~PAGE_MASK;
2440
2441         while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2442                 zerofrom = curpos & ~PAGE_MASK;
2443                 if (zerofrom & (blocksize-1)) {
2444                         *bytes |= (blocksize-1);
2445                         (*bytes)++;
2446                 }
2447                 len = PAGE_SIZE - zerofrom;
2448
2449                 err = pagecache_write_begin(file, mapping, curpos, len,
2450                                                 AOP_FLAG_UNINTERRUPTIBLE,
2451                                                 &page, &fsdata);
2452                 if (err)
2453                         goto out;
2454                 zero_user(page, zerofrom, len);
2455                 err = pagecache_write_end(file, mapping, curpos, len, len,
2456                                                 page, fsdata);
2457                 if (err < 0)
2458                         goto out;
2459                 BUG_ON(err != len);
2460                 err = 0;
2461
2462                 balance_dirty_pages_ratelimited(mapping);
2463
2464                 if (unlikely(fatal_signal_pending(current))) {
2465                         err = -EINTR;
2466                         goto out;
2467                 }
2468         }
2469
2470         /* page covers the boundary, find the boundary offset */
2471         if (index == curidx) {
2472                 zerofrom = curpos & ~PAGE_MASK;
2473                 /* if we will expand the thing last block will be filled */
2474                 if (offset <= zerofrom) {
2475                         goto out;
2476                 }
2477                 if (zerofrom & (blocksize-1)) {
2478                         *bytes |= (blocksize-1);
2479                         (*bytes)++;
2480                 }
2481                 len = offset - zerofrom;
2482
2483                 err = pagecache_write_begin(file, mapping, curpos, len,
2484                                                 AOP_FLAG_UNINTERRUPTIBLE,
2485                                                 &page, &fsdata);
2486                 if (err)
2487                         goto out;
2488                 zero_user(page, zerofrom, len);
2489                 err = pagecache_write_end(file, mapping, curpos, len, len,
2490                                                 page, fsdata);
2491                 if (err < 0)
2492                         goto out;
2493                 BUG_ON(err != len);
2494                 err = 0;
2495         }
2496 out:
2497         return err;
2498 }
2499
2500 /*
2501  * For moronic filesystems that do not allow holes in file.
2502  * We may have to extend the file.
2503  */
2504 int cont_write_begin(struct file *file, struct address_space *mapping,
2505                         loff_t pos, unsigned len, unsigned flags,
2506                         struct page **pagep, void **fsdata,
2507                         get_block_t *get_block, loff_t *bytes)
2508 {
2509         struct inode *inode = mapping->host;
2510         unsigned blocksize = 1 << inode->i_blkbits;
2511         unsigned zerofrom;
2512         int err;
2513
2514         err = cont_expand_zero(file, mapping, pos, bytes);
2515         if (err)
2516                 return err;
2517
2518         zerofrom = *bytes & ~PAGE_MASK;
2519         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2520                 *bytes |= (blocksize-1);
2521                 (*bytes)++;
2522         }
2523
2524         return block_write_begin(mapping, pos, len, flags, pagep, get_block);
2525 }
2526 EXPORT_SYMBOL(cont_write_begin);
2527
2528 int block_commit_write(struct page *page, unsigned from, unsigned to)
2529 {
2530         struct inode *inode = page->mapping->host;
2531         __block_commit_write(inode,page,from,to);
2532         return 0;
2533 }
2534 EXPORT_SYMBOL(block_commit_write);
2535
2536 /*
2537  * block_page_mkwrite() is not allowed to change the file size as it gets
2538  * called from a page fault handler when a page is first dirtied. Hence we must
2539  * be careful to check for EOF conditions here. We set the page up correctly
2540  * for a written page which means we get ENOSPC checking when writing into
2541  * holes and correct delalloc and unwritten extent mapping on filesystems that
2542  * support these features.
2543  *
2544  * We are not allowed to take the i_mutex here so we have to play games to
2545  * protect against truncate races as the page could now be beyond EOF.  Because
2546  * truncate writes the inode size before removing pages, once we have the
2547  * page lock we can determine safely if the page is beyond EOF. If it is not
2548  * beyond EOF, then the page is guaranteed safe against truncation until we
2549  * unlock the page.
2550  *
2551  * Direct callers of this function should protect against filesystem freezing
2552  * using sb_start_pagefault() - sb_end_pagefault() functions.
2553  */
2554 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2555                          get_block_t get_block)
2556 {
2557         struct page *page = vmf->page;
2558         struct inode *inode = file_inode(vma->vm_file);
2559         unsigned long end;
2560         loff_t size;
2561         int ret;
2562
2563         lock_page(page);
2564         size = i_size_read(inode);
2565         if ((page->mapping != inode->i_mapping) ||
2566             (page_offset(page) > size)) {
2567                 /* We overload EFAULT to mean page got truncated */
2568                 ret = -EFAULT;
2569                 goto out_unlock;
2570         }
2571
2572         /* page is wholly or partially inside EOF */
2573         if (((page->index + 1) << PAGE_SHIFT) > size)
2574                 end = size & ~PAGE_MASK;
2575         else
2576                 end = PAGE_SIZE;
2577
2578         ret = __block_write_begin(page, 0, end, get_block);
2579         if (!ret)
2580                 ret = block_commit_write(page, 0, end);
2581
2582         if (unlikely(ret < 0))
2583                 goto out_unlock;
2584         set_page_dirty(page);
2585         wait_for_stable_page(page);
2586         return 0;
2587 out_unlock:
2588         unlock_page(page);
2589         return ret;
2590 }
2591 EXPORT_SYMBOL(block_page_mkwrite);
2592
2593 /*
2594  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2595  * immediately, while under the page lock.  So it needs a special end_io
2596  * handler which does not touch the bh after unlocking it.
2597  */
2598 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2599 {
2600         __end_buffer_read_notouch(bh, uptodate);
2601 }
2602
2603 /*
2604  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2605  * the page (converting it to circular linked list and taking care of page
2606  * dirty races).
2607  */
2608 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2609 {
2610         struct buffer_head *bh;
2611
2612         BUG_ON(!PageLocked(page));
2613
2614         spin_lock(&page->mapping->private_lock);
2615         bh = head;
2616         do {
2617                 if (PageDirty(page))
2618                         set_buffer_dirty(bh);
2619                 if (!bh->b_this_page)
2620                         bh->b_this_page = head;
2621                 bh = bh->b_this_page;
2622         } while (bh != head);
2623         attach_page_buffers(page, head);
2624         spin_unlock(&page->mapping->private_lock);
2625 }
2626
2627 /*
2628  * On entry, the page is fully not uptodate.
2629  * On exit the page is fully uptodate in the areas outside (from,to)
2630  * The filesystem needs to handle block truncation upon failure.
2631  */
2632 int nobh_write_begin(struct address_space *mapping,
2633                         loff_t pos, unsigned len, unsigned flags,
2634                         struct page **pagep, void **fsdata,
2635                         get_block_t *get_block)
2636 {
2637         struct inode *inode = mapping->host;
2638         const unsigned blkbits = inode->i_blkbits;
2639         const unsigned blocksize = 1 << blkbits;
2640         struct buffer_head *head, *bh;
2641         struct page *page;
2642         pgoff_t index;
2643         unsigned from, to;
2644         unsigned block_in_page;
2645         unsigned block_start, block_end;
2646         sector_t block_in_file;
2647         int nr_reads = 0;
2648         int ret = 0;
2649         int is_mapped_to_disk = 1;
2650
2651         index = pos >> PAGE_SHIFT;
2652         from = pos & (PAGE_SIZE - 1);
2653         to = from + len;
2654
2655         page = grab_cache_page_write_begin(mapping, index, flags);
2656         if (!page)
2657                 return -ENOMEM;
2658         *pagep = page;
2659         *fsdata = NULL;
2660
2661         if (page_has_buffers(page)) {
2662                 ret = __block_write_begin(page, pos, len, get_block);
2663                 if (unlikely(ret))
2664                         goto out_release;
2665                 return ret;
2666         }
2667
2668         if (PageMappedToDisk(page))
2669                 return 0;
2670
2671         /*
2672          * Allocate buffers so that we can keep track of state, and potentially
2673          * attach them to the page if an error occurs. In the common case of
2674          * no error, they will just be freed again without ever being attached
2675          * to the page (which is all OK, because we're under the page lock).
2676          *
2677          * Be careful: the buffer linked list is a NULL terminated one, rather
2678          * than the circular one we're used to.
2679          */
2680         head = alloc_page_buffers(page, blocksize, 0);
2681         if (!head) {
2682                 ret = -ENOMEM;
2683                 goto out_release;
2684         }
2685
2686         block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
2687
2688         /*
2689          * We loop across all blocks in the page, whether or not they are
2690          * part of the affected region.  This is so we can discover if the
2691          * page is fully mapped-to-disk.
2692          */
2693         for (block_start = 0, block_in_page = 0, bh = head;
2694                   block_start < PAGE_SIZE;
2695                   block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2696                 int create;
2697
2698                 block_end = block_start + blocksize;
2699                 bh->b_state = 0;
2700                 create = 1;
2701                 if (block_start >= to)
2702                         create = 0;
2703                 ret = get_block(inode, block_in_file + block_in_page,
2704                                         bh, create);
2705                 if (ret)
2706                         goto failed;
2707                 if (!buffer_mapped(bh))
2708                         is_mapped_to_disk = 0;
2709                 if (buffer_new(bh))
2710                         clean_bdev_bh_alias(bh);
2711                 if (PageUptodate(page)) {
2712                         set_buffer_uptodate(bh);
2713                         continue;
2714                 }
2715                 if (buffer_new(bh) || !buffer_mapped(bh)) {
2716                         zero_user_segments(page, block_start, from,
2717                                                         to, block_end);
2718                         continue;
2719                 }
2720                 if (buffer_uptodate(bh))
2721                         continue;       /* reiserfs does this */
2722                 if (block_start < from || block_end > to) {
2723                         lock_buffer(bh);
2724                         bh->b_end_io = end_buffer_read_nobh;
2725                         submit_bh(REQ_OP_READ, 0, bh);
2726                         nr_reads++;
2727                 }
2728         }
2729
2730         if (nr_reads) {
2731                 /*
2732                  * The page is locked, so these buffers are protected from
2733                  * any VM or truncate activity.  Hence we don't need to care
2734                  * for the buffer_head refcounts.
2735                  */
2736                 for (bh = head; bh; bh = bh->b_this_page) {
2737                         wait_on_buffer(bh);
2738                         if (!buffer_uptodate(bh))
2739                                 ret = -EIO;
2740                 }
2741                 if (ret)
2742                         goto failed;
2743         }
2744
2745         if (is_mapped_to_disk)
2746                 SetPageMappedToDisk(page);
2747
2748         *fsdata = head; /* to be released by nobh_write_end */
2749
2750         return 0;
2751
2752 failed:
2753         BUG_ON(!ret);
2754         /*
2755          * Error recovery is a bit difficult. We need to zero out blocks that
2756          * were newly allocated, and dirty them to ensure they get written out.
2757          * Buffers need to be attached to the page at this point, otherwise
2758          * the handling of potential IO errors during writeout would be hard
2759          * (could try doing synchronous writeout, but what if that fails too?)
2760          */
2761         attach_nobh_buffers(page, head);
2762         page_zero_new_buffers(page, from, to);
2763
2764 out_release:
2765         unlock_page(page);
2766         put_page(page);
2767         *pagep = NULL;
2768
2769         return ret;
2770 }
2771 EXPORT_SYMBOL(nobh_write_begin);
2772
2773 int nobh_write_end(struct file *file, struct address_space *mapping,
2774                         loff_t pos, unsigned len, unsigned copied,
2775                         struct page *page, void *fsdata)
2776 {
2777         struct inode *inode = page->mapping->host;
2778         struct buffer_head *head = fsdata;
2779         struct buffer_head *bh;
2780         BUG_ON(fsdata != NULL && page_has_buffers(page));
2781
2782         if (unlikely(copied < len) && head)
2783                 attach_nobh_buffers(page, head);
2784         if (page_has_buffers(page))
2785                 return generic_write_end(file, mapping, pos, len,
2786                                         copied, page, fsdata);
2787
2788         SetPageUptodate(page);
2789         set_page_dirty(page);
2790         if (pos+copied > inode->i_size) {
2791                 i_size_write(inode, pos+copied);
2792                 mark_inode_dirty(inode);
2793         }
2794
2795         unlock_page(page);
2796         put_page(page);
2797
2798         while (head) {
2799                 bh = head;
2800                 head = head->b_this_page;
2801                 free_buffer_head(bh);
2802         }
2803
2804         return copied;
2805 }
2806 EXPORT_SYMBOL(nobh_write_end);
2807
2808 /*
2809  * nobh_writepage() - based on block_full_write_page() except
2810  * that it tries to operate without attaching bufferheads to
2811  * the page.
2812  */
2813 int nobh_writepage(struct page *page, get_block_t *get_block,
2814                         struct writeback_control *wbc)
2815 {
2816         struct inode * const inode = page->mapping->host;
2817         loff_t i_size = i_size_read(inode);
2818         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2819         unsigned offset;
2820         int ret;
2821
2822         /* Is the page fully inside i_size? */
2823         if (page->index < end_index)
2824                 goto out;
2825
2826         /* Is the page fully outside i_size? (truncate in progress) */
2827         offset = i_size & (PAGE_SIZE-1);
2828         if (page->index >= end_index+1 || !offset) {
2829                 /*
2830                  * The page may have dirty, unmapped buffers.  For example,
2831                  * they may have been added in ext3_writepage().  Make them
2832                  * freeable here, so the page does not leak.
2833                  */
2834 #if 0
2835                 /* Not really sure about this  - do we need this ? */
2836                 if (page->mapping->a_ops->invalidatepage)
2837                         page->mapping->a_ops->invalidatepage(page, offset);
2838 #endif
2839                 unlock_page(page);
2840                 return 0; /* don't care */
2841         }
2842
2843         /*
2844          * The page straddles i_size.  It must be zeroed out on each and every
2845          * writepage invocation because it may be mmapped.  "A file is mapped
2846          * in multiples of the page size.  For a file that is not a multiple of
2847          * the  page size, the remaining memory is zeroed when mapped, and
2848          * writes to that region are not written out to the file."
2849          */
2850         zero_user_segment(page, offset, PAGE_SIZE);
2851 out:
2852         ret = mpage_writepage(page, get_block, wbc);
2853         if (ret == -EAGAIN)
2854                 ret = __block_write_full_page(inode, page, get_block, wbc,
2855                                               end_buffer_async_write);
2856         return ret;
2857 }
2858 EXPORT_SYMBOL(nobh_writepage);
2859
2860 int nobh_truncate_page(struct address_space *mapping,
2861                         loff_t from, get_block_t *get_block)
2862 {
2863         pgoff_t index = from >> PAGE_SHIFT;
2864         unsigned offset = from & (PAGE_SIZE-1);
2865         unsigned blocksize;
2866         sector_t iblock;
2867         unsigned length, pos;
2868         struct inode *inode = mapping->host;
2869         struct page *page;
2870         struct buffer_head map_bh;
2871         int err;
2872
2873         blocksize = 1 << inode->i_blkbits;
2874         length = offset & (blocksize - 1);
2875
2876         /* Block boundary? Nothing to do */
2877         if (!length)
2878                 return 0;
2879
2880         length = blocksize - length;
2881         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2882
2883         page = grab_cache_page(mapping, index);
2884         err = -ENOMEM;
2885         if (!page)
2886                 goto out;
2887
2888         if (page_has_buffers(page)) {
2889 has_buffers:
2890                 unlock_page(page);
2891                 put_page(page);
2892                 return block_truncate_page(mapping, from, get_block);
2893         }
2894
2895         /* Find the buffer that contains "offset" */
2896         pos = blocksize;
2897         while (offset >= pos) {
2898                 iblock++;
2899                 pos += blocksize;
2900         }
2901
2902         map_bh.b_size = blocksize;
2903         map_bh.b_state = 0;
2904         err = get_block(inode, iblock, &map_bh, 0);
2905         if (err)
2906                 goto unlock;
2907         /* unmapped? It's a hole - nothing to do */
2908         if (!buffer_mapped(&map_bh))
2909                 goto unlock;
2910
2911         /* Ok, it's mapped. Make sure it's up-to-date */
2912         if (!PageUptodate(page)) {
2913                 err = mapping->a_ops->readpage(NULL, page);
2914                 if (err) {
2915                         put_page(page);
2916                         goto out;
2917                 }
2918                 lock_page(page);
2919                 if (!PageUptodate(page)) {
2920                         err = -EIO;
2921                         goto unlock;
2922                 }
2923                 if (page_has_buffers(page))
2924                         goto has_buffers;
2925         }
2926         zero_user(page, offset, length);
2927         set_page_dirty(page);
2928         err = 0;
2929
2930 unlock:
2931         unlock_page(page);
2932         put_page(page);
2933 out:
2934         return err;
2935 }
2936 EXPORT_SYMBOL(nobh_truncate_page);
2937
2938 int block_truncate_page(struct address_space *mapping,
2939                         loff_t from, get_block_t *get_block)
2940 {
2941         pgoff_t index = from >> PAGE_SHIFT;
2942         unsigned offset = from & (PAGE_SIZE-1);
2943         unsigned blocksize;
2944         sector_t iblock;
2945         unsigned length, pos;
2946         struct inode *inode = mapping->host;
2947         struct page *page;
2948         struct buffer_head *bh;
2949         int err;
2950
2951         blocksize = 1 << inode->i_blkbits;
2952         length = offset & (blocksize - 1);
2953
2954         /* Block boundary? Nothing to do */
2955         if (!length)
2956                 return 0;
2957
2958         length = blocksize - length;
2959         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2960         
2961         page = grab_cache_page(mapping, index);
2962         err = -ENOMEM;
2963         if (!page)
2964                 goto out;
2965
2966         if (!page_has_buffers(page))
2967                 create_empty_buffers(page, blocksize, 0);
2968
2969         /* Find the buffer that contains "offset" */
2970         bh = page_buffers(page);
2971         pos = blocksize;
2972         while (offset >= pos) {
2973                 bh = bh->b_this_page;
2974                 iblock++;
2975                 pos += blocksize;
2976         }
2977
2978         err = 0;
2979         if (!buffer_mapped(bh)) {
2980                 WARN_ON(bh->b_size != blocksize);
2981                 err = get_block(inode, iblock, bh, 0);
2982                 if (err)
2983                         goto unlock;
2984                 /* unmapped? It's a hole - nothing to do */
2985                 if (!buffer_mapped(bh))
2986                         goto unlock;
2987         }
2988
2989         /* Ok, it's mapped. Make sure it's up-to-date */
2990         if (PageUptodate(page))
2991                 set_buffer_uptodate(bh);
2992
2993         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2994                 err = -EIO;
2995                 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2996                 wait_on_buffer(bh);
2997                 /* Uhhuh. Read error. Complain and punt. */
2998                 if (!buffer_uptodate(bh))
2999                         goto unlock;
3000         }
3001
3002         zero_user(page, offset, length);
3003         mark_buffer_dirty(bh);
3004         err = 0;
3005
3006 unlock:
3007         unlock_page(page);
3008         put_page(page);
3009 out:
3010         return err;
3011 }
3012 EXPORT_SYMBOL(block_truncate_page);
3013
3014 /*
3015  * The generic ->writepage function for buffer-backed address_spaces
3016  */
3017 int block_write_full_page(struct page *page, get_block_t *get_block,
3018                         struct writeback_control *wbc)
3019 {
3020         struct inode * const inode = page->mapping->host;
3021         loff_t i_size = i_size_read(inode);
3022         const pgoff_t end_index = i_size >> PAGE_SHIFT;
3023         unsigned offset;
3024
3025         /* Is the page fully inside i_size? */
3026         if (page->index < end_index)
3027                 return __block_write_full_page(inode, page, get_block, wbc,
3028                                                end_buffer_async_write);
3029
3030         /* Is the page fully outside i_size? (truncate in progress) */
3031         offset = i_size & (PAGE_SIZE-1);
3032         if (page->index >= end_index+1 || !offset) {
3033                 /*
3034                  * The page may have dirty, unmapped buffers.  For example,
3035                  * they may have been added in ext3_writepage().  Make them
3036                  * freeable here, so the page does not leak.
3037                  */
3038                 do_invalidatepage(page, 0, PAGE_SIZE);
3039                 unlock_page(page);
3040                 return 0; /* don't care */
3041         }
3042
3043         /*
3044          * The page straddles i_size.  It must be zeroed out on each and every
3045          * writepage invocation because it may be mmapped.  "A file is mapped
3046          * in multiples of the page size.  For a file that is not a multiple of
3047          * the  page size, the remaining memory is zeroed when mapped, and
3048          * writes to that region are not written out to the file."
3049          */
3050         zero_user_segment(page, offset, PAGE_SIZE);
3051         return __block_write_full_page(inode, page, get_block, wbc,
3052                                                         end_buffer_async_write);
3053 }
3054 EXPORT_SYMBOL(block_write_full_page);
3055
3056 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
3057                             get_block_t *get_block)
3058 {
3059         struct buffer_head tmp;
3060         struct inode *inode = mapping->host;
3061         tmp.b_state = 0;
3062         tmp.b_blocknr = 0;
3063         tmp.b_size = 1 << inode->i_blkbits;
3064         get_block(inode, block, &tmp, 0);
3065         return tmp.b_blocknr;
3066 }
3067 EXPORT_SYMBOL(generic_block_bmap);
3068
3069 static void end_bio_bh_io_sync(struct bio *bio)
3070 {
3071         struct buffer_head *bh = bio->bi_private;
3072
3073         if (unlikely(bio_flagged(bio, BIO_QUIET)))
3074                 set_bit(BH_Quiet, &bh->b_state);
3075
3076         bh->b_end_io(bh, !bio->bi_error);
3077         bio_put(bio);
3078 }
3079
3080 /*
3081  * This allows us to do IO even on the odd last sectors
3082  * of a device, even if the block size is some multiple
3083  * of the physical sector size.
3084  *
3085  * We'll just truncate the bio to the size of the device,
3086  * and clear the end of the buffer head manually.
3087  *
3088  * Truly out-of-range accesses will turn into actual IO
3089  * errors, this only handles the "we need to be able to
3090  * do IO at the final sector" case.
3091  */
3092 void guard_bio_eod(int op, struct bio *bio)
3093 {
3094         sector_t maxsector;
3095         struct bio_vec *bvec = &bio->bi_io_vec[bio->bi_vcnt - 1];
3096         unsigned truncated_bytes;
3097
3098         maxsector = i_size_read(bio->bi_bdev->bd_inode) >> 9;
3099         if (!maxsector)
3100                 return;
3101
3102         /*
3103          * If the *whole* IO is past the end of the device,
3104          * let it through, and the IO layer will turn it into
3105          * an EIO.
3106          */
3107         if (unlikely(bio->bi_iter.bi_sector >= maxsector))
3108                 return;
3109
3110         maxsector -= bio->bi_iter.bi_sector;
3111         if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
3112                 return;
3113
3114         /* Uhhuh. We've got a bio that straddles the device size! */
3115         truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
3116
3117         /* Truncate the bio.. */
3118         bio->bi_iter.bi_size -= truncated_bytes;
3119         bvec->bv_len -= truncated_bytes;
3120
3121         /* ..and clear the end of the buffer for reads */
3122         if (op == REQ_OP_READ) {
3123                 zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
3124                                 truncated_bytes);
3125         }
3126 }
3127
3128 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
3129                          unsigned long bio_flags, struct writeback_control *wbc)
3130 {
3131         struct bio *bio;
3132
3133         BUG_ON(!buffer_locked(bh));
3134         BUG_ON(!buffer_mapped(bh));
3135         BUG_ON(!bh->b_end_io);
3136         BUG_ON(buffer_delay(bh));
3137         BUG_ON(buffer_unwritten(bh));
3138
3139         /*
3140          * Only clear out a write error when rewriting
3141          */
3142         if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
3143                 clear_buffer_write_io_error(bh);
3144
3145         /*
3146          * from here on down, it's all bio -- do the initial mapping,
3147          * submit_bio -> generic_make_request may further map this bio around
3148          */
3149         bio = bio_alloc(GFP_NOIO, 1);
3150
3151         if (wbc) {
3152                 wbc_init_bio(wbc, bio);
3153                 wbc_account_io(wbc, bh->b_page, bh->b_size);
3154         }
3155
3156         bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3157         bio->bi_bdev = bh->b_bdev;
3158
3159         bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
3160         BUG_ON(bio->bi_iter.bi_size != bh->b_size);
3161
3162         bio->bi_end_io = end_bio_bh_io_sync;
3163         bio->bi_private = bh;
3164         bio->bi_flags |= bio_flags;
3165
3166         /* Take care of bh's that straddle the end of the device */
3167         guard_bio_eod(op, bio);
3168
3169         if (buffer_meta(bh))
3170                 op_flags |= REQ_META;
3171         if (buffer_prio(bh))
3172                 op_flags |= REQ_PRIO;
3173         bio_set_op_attrs(bio, op, op_flags);
3174
3175         submit_bio(bio);
3176         return 0;
3177 }
3178
3179 int _submit_bh(int op, int op_flags, struct buffer_head *bh,
3180                unsigned long bio_flags)
3181 {
3182         return submit_bh_wbc(op, op_flags, bh, bio_flags, NULL);
3183 }
3184 EXPORT_SYMBOL_GPL(_submit_bh);
3185
3186 int submit_bh(int op, int op_flags,  struct buffer_head *bh)
3187 {
3188         return submit_bh_wbc(op, op_flags, bh, 0, NULL);
3189 }
3190 EXPORT_SYMBOL(submit_bh);
3191
3192 /**
3193  * ll_rw_block: low-level access to block devices (DEPRECATED)
3194  * @op: whether to %READ or %WRITE
3195  * @op_flags: req_flag_bits
3196  * @nr: number of &struct buffer_heads in the array
3197  * @bhs: array of pointers to &struct buffer_head
3198  *
3199  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3200  * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
3201  * @op_flags contains flags modifying the detailed I/O behavior, most notably
3202  * %REQ_RAHEAD.
3203  *
3204  * This function drops any buffer that it cannot get a lock on (with the
3205  * BH_Lock state bit), any buffer that appears to be clean when doing a write
3206  * request, and any buffer that appears to be up-to-date when doing read
3207  * request.  Further it marks as clean buffers that are processed for
3208  * writing (the buffer cache won't assume that they are actually clean
3209  * until the buffer gets unlocked).
3210  *
3211  * ll_rw_block sets b_end_io to simple completion handler that marks
3212  * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3213  * any waiters. 
3214  *
3215  * All of the buffers must be for the same device, and must also be a
3216  * multiple of the current approved size for the device.
3217  */
3218 void ll_rw_block(int op, int op_flags,  int nr, struct buffer_head *bhs[])
3219 {
3220         int i;
3221
3222         for (i = 0; i < nr; i++) {
3223                 struct buffer_head *bh = bhs[i];
3224
3225                 if (!trylock_buffer(bh))
3226                         continue;
3227                 if (op == WRITE) {
3228                         if (test_clear_buffer_dirty(bh)) {
3229                                 bh->b_end_io = end_buffer_write_sync;
3230                                 get_bh(bh);
3231                                 submit_bh(op, op_flags, bh);
3232                                 continue;
3233                         }
3234                 } else {
3235                         if (!buffer_uptodate(bh)) {
3236                                 bh->b_end_io = end_buffer_read_sync;
3237                                 get_bh(bh);
3238                                 submit_bh(op, op_flags, bh);
3239                                 continue;
3240                         }
3241                 }
3242                 unlock_buffer(bh);
3243         }
3244 }
3245 EXPORT_SYMBOL(ll_rw_block);
3246
3247 void write_dirty_buffer(struct buffer_head *bh, int op_flags)
3248 {
3249         lock_buffer(bh);
3250         if (!test_clear_buffer_dirty(bh)) {
3251                 unlock_buffer(bh);
3252                 return;
3253         }
3254         bh->b_end_io = end_buffer_write_sync;
3255         get_bh(bh);
3256         submit_bh(REQ_OP_WRITE, op_flags, bh);
3257 }
3258 EXPORT_SYMBOL(write_dirty_buffer);
3259
3260 /*
3261  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3262  * and then start new I/O and then wait upon it.  The caller must have a ref on
3263  * the buffer_head.
3264  */
3265 int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
3266 {
3267         int ret = 0;
3268
3269         WARN_ON(atomic_read(&bh->b_count) < 1);
3270         lock_buffer(bh);
3271         if (test_clear_buffer_dirty(bh)) {
3272                 get_bh(bh);
3273                 bh->b_end_io = end_buffer_write_sync;
3274                 ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
3275                 wait_on_buffer(bh);
3276                 if (!ret && !buffer_uptodate(bh))
3277                         ret = -EIO;
3278         } else {
3279                 unlock_buffer(bh);
3280         }
3281         return ret;
3282 }
3283 EXPORT_SYMBOL(__sync_dirty_buffer);
3284
3285 int sync_dirty_buffer(struct buffer_head *bh)
3286 {
3287         return __sync_dirty_buffer(bh, REQ_SYNC);
3288 }
3289 EXPORT_SYMBOL(sync_dirty_buffer);
3290
3291 /*
3292  * try_to_free_buffers() checks if all the buffers on this particular page
3293  * are unused, and releases them if so.
3294  *
3295  * Exclusion against try_to_free_buffers may be obtained by either
3296  * locking the page or by holding its mapping's private_lock.
3297  *
3298  * If the page is dirty but all the buffers are clean then we need to
3299  * be sure to mark the page clean as well.  This is because the page
3300  * may be against a block device, and a later reattachment of buffers
3301  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3302  * filesystem data on the same device.
3303  *
3304  * The same applies to regular filesystem pages: if all the buffers are
3305  * clean then we set the page clean and proceed.  To do that, we require
3306  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3307  * private_lock.
3308  *
3309  * try_to_free_buffers() is non-blocking.
3310  */
3311 static inline int buffer_busy(struct buffer_head *bh)
3312 {
3313         return atomic_read(&bh->b_count) |
3314                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3315 }
3316
3317 static int
3318 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3319 {
3320         struct buffer_head *head = page_buffers(page);
3321         struct buffer_head *bh;
3322
3323         bh = head;
3324         do {
3325                 if (buffer_write_io_error(bh) && page->mapping)
3326                         mapping_set_error(page->mapping, -EIO);
3327                 if (buffer_busy(bh))
3328                         goto failed;
3329                 bh = bh->b_this_page;
3330         } while (bh != head);
3331
3332         do {
3333                 struct buffer_head *next = bh->b_this_page;
3334
3335                 if (bh->b_assoc_map)
3336                         __remove_assoc_queue(bh);
3337                 bh = next;
3338         } while (bh != head);
3339         *buffers_to_free = head;
3340         __clear_page_buffers(page);
3341         return 1;
3342 failed:
3343         return 0;
3344 }
3345
3346 int try_to_free_buffers(struct page *page)
3347 {
3348         struct address_space * const mapping = page->mapping;
3349         struct buffer_head *buffers_to_free = NULL;
3350         int ret = 0;
3351
3352         BUG_ON(!PageLocked(page));
3353         if (PageWriteback(page))
3354                 return 0;
3355
3356         if (mapping == NULL) {          /* can this still happen? */
3357                 ret = drop_buffers(page, &buffers_to_free);
3358                 goto out;
3359         }
3360
3361         spin_lock(&mapping->private_lock);
3362         ret = drop_buffers(page, &buffers_to_free);
3363
3364         /*
3365          * If the filesystem writes its buffers by hand (eg ext3)
3366          * then we can have clean buffers against a dirty page.  We
3367          * clean the page here; otherwise the VM will never notice
3368          * that the filesystem did any IO at all.
3369          *
3370          * Also, during truncate, discard_buffer will have marked all
3371          * the page's buffers clean.  We discover that here and clean
3372          * the page also.
3373          *
3374          * private_lock must be held over this entire operation in order
3375          * to synchronise against __set_page_dirty_buffers and prevent the
3376          * dirty bit from being lost.
3377          */
3378         if (ret)
3379                 cancel_dirty_page(page);
3380         spin_unlock(&mapping->private_lock);
3381 out:
3382         if (buffers_to_free) {
3383                 struct buffer_head *bh = buffers_to_free;
3384
3385                 do {
3386                         struct buffer_head *next = bh->b_this_page;
3387                         free_buffer_head(bh);
3388                         bh = next;
3389                 } while (bh != buffers_to_free);
3390         }
3391         return ret;
3392 }
3393 EXPORT_SYMBOL(try_to_free_buffers);
3394
3395 /*
3396  * There are no bdflush tunables left.  But distributions are
3397  * still running obsolete flush daemons, so we terminate them here.
3398  *
3399  * Use of bdflush() is deprecated and will be removed in a future kernel.
3400  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3401  */
3402 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3403 {
3404         static int msg_count;
3405
3406         if (!capable(CAP_SYS_ADMIN))
3407                 return -EPERM;
3408
3409         if (msg_count < 5) {
3410                 msg_count++;
3411                 printk(KERN_INFO
3412                         "warning: process `%s' used the obsolete bdflush"
3413                         " system call\n", current->comm);
3414                 printk(KERN_INFO "Fix your initscripts?\n");
3415         }
3416
3417         if (func == 1)
3418                 do_exit(0);
3419         return 0;
3420 }
3421
3422 /*
3423  * Buffer-head allocation
3424  */
3425 static struct kmem_cache *bh_cachep __read_mostly;
3426
3427 /*
3428  * Once the number of bh's in the machine exceeds this level, we start
3429  * stripping them in writeback.
3430  */
3431 static unsigned long max_buffer_heads;
3432
3433 int buffer_heads_over_limit;
3434
3435 struct bh_accounting {
3436         int nr;                 /* Number of live bh's */
3437         int ratelimit;          /* Limit cacheline bouncing */
3438 };
3439
3440 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3441
3442 static void recalc_bh_state(void)
3443 {
3444         int i;
3445         int tot = 0;
3446
3447         if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3448                 return;
3449         __this_cpu_write(bh_accounting.ratelimit, 0);
3450         for_each_online_cpu(i)
3451                 tot += per_cpu(bh_accounting, i).nr;
3452         buffer_heads_over_limit = (tot > max_buffer_heads);
3453 }
3454
3455 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3456 {
3457         struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3458         if (ret) {
3459                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3460                 preempt_disable();
3461                 __this_cpu_inc(bh_accounting.nr);
3462                 recalc_bh_state();
3463                 preempt_enable();
3464         }
3465         return ret;
3466 }
3467 EXPORT_SYMBOL(alloc_buffer_head);
3468
3469 void free_buffer_head(struct buffer_head *bh)
3470 {
3471         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3472         kmem_cache_free(bh_cachep, bh);
3473         preempt_disable();
3474         __this_cpu_dec(bh_accounting.nr);
3475         recalc_bh_state();
3476         preempt_enable();
3477 }
3478 EXPORT_SYMBOL(free_buffer_head);
3479
3480 static void buffer_exit_cpu(int cpu)
3481 {
3482         int i;
3483         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3484
3485         for (i = 0; i < BH_LRU_SIZE; i++) {
3486                 brelse(b->bhs[i]);
3487                 b->bhs[i] = NULL;
3488         }
3489         this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3490         per_cpu(bh_accounting, cpu).nr = 0;
3491 }
3492
3493 static int buffer_cpu_notify(struct notifier_block *self,
3494                               unsigned long action, void *hcpu)
3495 {
3496         if (action == CPU_DEAD || action == CPU_DEAD_FROZEN)
3497                 buffer_exit_cpu((unsigned long)hcpu);
3498         return NOTIFY_OK;
3499 }
3500
3501 /**
3502  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3503  * @bh: struct buffer_head
3504  *
3505  * Return true if the buffer is up-to-date and false,
3506  * with the buffer locked, if not.
3507  */
3508 int bh_uptodate_or_lock(struct buffer_head *bh)
3509 {
3510         if (!buffer_uptodate(bh)) {
3511                 lock_buffer(bh);
3512                 if (!buffer_uptodate(bh))
3513                         return 0;
3514                 unlock_buffer(bh);
3515         }
3516         return 1;
3517 }
3518 EXPORT_SYMBOL(bh_uptodate_or_lock);
3519
3520 /**
3521  * bh_submit_read - Submit a locked buffer for reading
3522  * @bh: struct buffer_head
3523  *
3524  * Returns zero on success and -EIO on error.
3525  */
3526 int bh_submit_read(struct buffer_head *bh)
3527 {
3528         BUG_ON(!buffer_locked(bh));
3529
3530         if (buffer_uptodate(bh)) {
3531                 unlock_buffer(bh);
3532                 return 0;
3533         }
3534
3535         get_bh(bh);
3536         bh->b_end_io = end_buffer_read_sync;
3537         submit_bh(REQ_OP_READ, 0, bh);
3538         wait_on_buffer(bh);
3539         if (buffer_uptodate(bh))
3540                 return 0;
3541         return -EIO;
3542 }
3543 EXPORT_SYMBOL(bh_submit_read);
3544
3545 void __init buffer_init(void)
3546 {
3547         unsigned long nrpages;
3548
3549         bh_cachep = kmem_cache_create("buffer_head",
3550                         sizeof(struct buffer_head), 0,
3551                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3552                                 SLAB_MEM_SPREAD),
3553                                 NULL);
3554
3555         /*
3556          * Limit the bh occupancy to 10% of ZONE_NORMAL
3557          */
3558         nrpages = (nr_free_buffer_pages() * 10) / 100;
3559         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3560         hotcpu_notifier(buffer_cpu_notify, 0);
3561 }