aba2a948b235f4ef1fdcc1287821cd3b13a0a198
[sfrench/cifs-2.6.git] / fs / buffer.c
1 /*
2  *  linux/fs/buffer.c
3  *
4  *  Copyright (C) 1991, 1992, 2002  Linus Torvalds
5  */
6
7 /*
8  * Start bdflush() with kernel_thread not syscall - Paul Gortmaker, 12/95
9  *
10  * Removed a lot of unnecessary code and simplified things now that
11  * the buffer cache isn't our primary cache - Andrew Tridgell 12/96
12  *
13  * Speed up hash, lru, and free list operations.  Use gfp() for allocating
14  * hash table, use SLAB cache for buffer heads. SMP threading.  -DaveM
15  *
16  * Added 32k buffer block sizes - these are required older ARM systems. - RMK
17  *
18  * async buffer flushing, 1999 Andrea Arcangeli <andrea@suse.de>
19  */
20
21 #include <linux/kernel.h>
22 #include <linux/sched/signal.h>
23 #include <linux/syscalls.h>
24 #include <linux/fs.h>
25 #include <linux/iomap.h>
26 #include <linux/mm.h>
27 #include <linux/percpu.h>
28 #include <linux/slab.h>
29 #include <linux/capability.h>
30 #include <linux/blkdev.h>
31 #include <linux/file.h>
32 #include <linux/quotaops.h>
33 #include <linux/highmem.h>
34 #include <linux/export.h>
35 #include <linux/backing-dev.h>
36 #include <linux/writeback.h>
37 #include <linux/hash.h>
38 #include <linux/suspend.h>
39 #include <linux/buffer_head.h>
40 #include <linux/task_io_accounting_ops.h>
41 #include <linux/bio.h>
42 #include <linux/notifier.h>
43 #include <linux/cpu.h>
44 #include <linux/bitops.h>
45 #include <linux/mpage.h>
46 #include <linux/bit_spinlock.h>
47 #include <linux/pagevec.h>
48 #include <trace/events/block.h>
49
50 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list);
51 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
52                          enum rw_hint hint, struct writeback_control *wbc);
53
54 #define BH_ENTRY(list) list_entry((list), struct buffer_head, b_assoc_buffers)
55
56 inline void touch_buffer(struct buffer_head *bh)
57 {
58         trace_block_touch_buffer(bh);
59         mark_page_accessed(bh->b_page);
60 }
61 EXPORT_SYMBOL(touch_buffer);
62
63 void __lock_buffer(struct buffer_head *bh)
64 {
65         wait_on_bit_lock_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
66 }
67 EXPORT_SYMBOL(__lock_buffer);
68
69 void unlock_buffer(struct buffer_head *bh)
70 {
71         clear_bit_unlock(BH_Lock, &bh->b_state);
72         smp_mb__after_atomic();
73         wake_up_bit(&bh->b_state, BH_Lock);
74 }
75 EXPORT_SYMBOL(unlock_buffer);
76
77 /*
78  * Returns if the page has dirty or writeback buffers. If all the buffers
79  * are unlocked and clean then the PageDirty information is stale. If
80  * any of the pages are locked, it is assumed they are locked for IO.
81  */
82 void buffer_check_dirty_writeback(struct page *page,
83                                      bool *dirty, bool *writeback)
84 {
85         struct buffer_head *head, *bh;
86         *dirty = false;
87         *writeback = false;
88
89         BUG_ON(!PageLocked(page));
90
91         if (!page_has_buffers(page))
92                 return;
93
94         if (PageWriteback(page))
95                 *writeback = true;
96
97         head = page_buffers(page);
98         bh = head;
99         do {
100                 if (buffer_locked(bh))
101                         *writeback = true;
102
103                 if (buffer_dirty(bh))
104                         *dirty = true;
105
106                 bh = bh->b_this_page;
107         } while (bh != head);
108 }
109 EXPORT_SYMBOL(buffer_check_dirty_writeback);
110
111 /*
112  * Block until a buffer comes unlocked.  This doesn't stop it
113  * from becoming locked again - you have to lock it yourself
114  * if you want to preserve its state.
115  */
116 void __wait_on_buffer(struct buffer_head * bh)
117 {
118         wait_on_bit_io(&bh->b_state, BH_Lock, TASK_UNINTERRUPTIBLE);
119 }
120 EXPORT_SYMBOL(__wait_on_buffer);
121
122 static void
123 __clear_page_buffers(struct page *page)
124 {
125         ClearPagePrivate(page);
126         set_page_private(page, 0);
127         put_page(page);
128 }
129
130 static void buffer_io_error(struct buffer_head *bh, char *msg)
131 {
132         if (!test_bit(BH_Quiet, &bh->b_state))
133                 printk_ratelimited(KERN_ERR
134                         "Buffer I/O error on dev %pg, logical block %llu%s\n",
135                         bh->b_bdev, (unsigned long long)bh->b_blocknr, msg);
136 }
137
138 /*
139  * End-of-IO handler helper function which does not touch the bh after
140  * unlocking it.
141  * Note: unlock_buffer() sort-of does touch the bh after unlocking it, but
142  * a race there is benign: unlock_buffer() only use the bh's address for
143  * hashing after unlocking the buffer, so it doesn't actually touch the bh
144  * itself.
145  */
146 static void __end_buffer_read_notouch(struct buffer_head *bh, int uptodate)
147 {
148         if (uptodate) {
149                 set_buffer_uptodate(bh);
150         } else {
151                 /* This happens, due to failed read-ahead attempts. */
152                 clear_buffer_uptodate(bh);
153         }
154         unlock_buffer(bh);
155 }
156
157 /*
158  * Default synchronous end-of-IO handler..  Just mark it up-to-date and
159  * unlock the buffer. This is what ll_rw_block uses too.
160  */
161 void end_buffer_read_sync(struct buffer_head *bh, int uptodate)
162 {
163         __end_buffer_read_notouch(bh, uptodate);
164         put_bh(bh);
165 }
166 EXPORT_SYMBOL(end_buffer_read_sync);
167
168 void end_buffer_write_sync(struct buffer_head *bh, int uptodate)
169 {
170         if (uptodate) {
171                 set_buffer_uptodate(bh);
172         } else {
173                 buffer_io_error(bh, ", lost sync page write");
174                 mark_buffer_write_io_error(bh);
175                 clear_buffer_uptodate(bh);
176         }
177         unlock_buffer(bh);
178         put_bh(bh);
179 }
180 EXPORT_SYMBOL(end_buffer_write_sync);
181
182 /*
183  * Various filesystems appear to want __find_get_block to be non-blocking.
184  * But it's the page lock which protects the buffers.  To get around this,
185  * we get exclusion from try_to_free_buffers with the blockdev mapping's
186  * private_lock.
187  *
188  * Hack idea: for the blockdev mapping, private_lock contention
189  * may be quite high.  This code could TryLock the page, and if that
190  * succeeds, there is no need to take private_lock.
191  */
192 static struct buffer_head *
193 __find_get_block_slow(struct block_device *bdev, sector_t block)
194 {
195         struct inode *bd_inode = bdev->bd_inode;
196         struct address_space *bd_mapping = bd_inode->i_mapping;
197         struct buffer_head *ret = NULL;
198         pgoff_t index;
199         struct buffer_head *bh;
200         struct buffer_head *head;
201         struct page *page;
202         int all_mapped = 1;
203
204         index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
205         page = find_get_page_flags(bd_mapping, index, FGP_ACCESSED);
206         if (!page)
207                 goto out;
208
209         spin_lock(&bd_mapping->private_lock);
210         if (!page_has_buffers(page))
211                 goto out_unlock;
212         head = page_buffers(page);
213         bh = head;
214         do {
215                 if (!buffer_mapped(bh))
216                         all_mapped = 0;
217                 else if (bh->b_blocknr == block) {
218                         ret = bh;
219                         get_bh(bh);
220                         goto out_unlock;
221                 }
222                 bh = bh->b_this_page;
223         } while (bh != head);
224
225         /* we might be here because some of the buffers on this page are
226          * not mapped.  This is due to various races between
227          * file io on the block device and getblk.  It gets dealt with
228          * elsewhere, don't buffer_error if we had some unmapped buffers
229          */
230         if (all_mapped) {
231                 printk("__find_get_block_slow() failed. "
232                         "block=%llu, b_blocknr=%llu\n",
233                         (unsigned long long)block,
234                         (unsigned long long)bh->b_blocknr);
235                 printk("b_state=0x%08lx, b_size=%zu\n",
236                         bh->b_state, bh->b_size);
237                 printk("device %pg blocksize: %d\n", bdev,
238                         1 << bd_inode->i_blkbits);
239         }
240 out_unlock:
241         spin_unlock(&bd_mapping->private_lock);
242         put_page(page);
243 out:
244         return ret;
245 }
246
247 /*
248  * I/O completion handler for block_read_full_page() - pages
249  * which come unlocked at the end of I/O.
250  */
251 static void end_buffer_async_read(struct buffer_head *bh, int uptodate)
252 {
253         unsigned long flags;
254         struct buffer_head *first;
255         struct buffer_head *tmp;
256         struct page *page;
257         int page_uptodate = 1;
258
259         BUG_ON(!buffer_async_read(bh));
260
261         page = bh->b_page;
262         if (uptodate) {
263                 set_buffer_uptodate(bh);
264         } else {
265                 clear_buffer_uptodate(bh);
266                 buffer_io_error(bh, ", async page read");
267                 SetPageError(page);
268         }
269
270         /*
271          * Be _very_ careful from here on. Bad things can happen if
272          * two buffer heads end IO at almost the same time and both
273          * decide that the page is now completely done.
274          */
275         first = page_buffers(page);
276         local_irq_save(flags);
277         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
278         clear_buffer_async_read(bh);
279         unlock_buffer(bh);
280         tmp = bh;
281         do {
282                 if (!buffer_uptodate(tmp))
283                         page_uptodate = 0;
284                 if (buffer_async_read(tmp)) {
285                         BUG_ON(!buffer_locked(tmp));
286                         goto still_busy;
287                 }
288                 tmp = tmp->b_this_page;
289         } while (tmp != bh);
290         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
291         local_irq_restore(flags);
292
293         /*
294          * If none of the buffers had errors and they are all
295          * uptodate then we can set the page uptodate.
296          */
297         if (page_uptodate && !PageError(page))
298                 SetPageUptodate(page);
299         unlock_page(page);
300         return;
301
302 still_busy:
303         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
304         local_irq_restore(flags);
305         return;
306 }
307
308 /*
309  * Completion handler for block_write_full_page() - pages which are unlocked
310  * during I/O, and which have PageWriteback cleared upon I/O completion.
311  */
312 void end_buffer_async_write(struct buffer_head *bh, int uptodate)
313 {
314         unsigned long flags;
315         struct buffer_head *first;
316         struct buffer_head *tmp;
317         struct page *page;
318
319         BUG_ON(!buffer_async_write(bh));
320
321         page = bh->b_page;
322         if (uptodate) {
323                 set_buffer_uptodate(bh);
324         } else {
325                 buffer_io_error(bh, ", lost async page write");
326                 mark_buffer_write_io_error(bh);
327                 clear_buffer_uptodate(bh);
328                 SetPageError(page);
329         }
330
331         first = page_buffers(page);
332         local_irq_save(flags);
333         bit_spin_lock(BH_Uptodate_Lock, &first->b_state);
334
335         clear_buffer_async_write(bh);
336         unlock_buffer(bh);
337         tmp = bh->b_this_page;
338         while (tmp != bh) {
339                 if (buffer_async_write(tmp)) {
340                         BUG_ON(!buffer_locked(tmp));
341                         goto still_busy;
342                 }
343                 tmp = tmp->b_this_page;
344         }
345         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
346         local_irq_restore(flags);
347         end_page_writeback(page);
348         return;
349
350 still_busy:
351         bit_spin_unlock(BH_Uptodate_Lock, &first->b_state);
352         local_irq_restore(flags);
353         return;
354 }
355 EXPORT_SYMBOL(end_buffer_async_write);
356
357 /*
358  * If a page's buffers are under async readin (end_buffer_async_read
359  * completion) then there is a possibility that another thread of
360  * control could lock one of the buffers after it has completed
361  * but while some of the other buffers have not completed.  This
362  * locked buffer would confuse end_buffer_async_read() into not unlocking
363  * the page.  So the absence of BH_Async_Read tells end_buffer_async_read()
364  * that this buffer is not under async I/O.
365  *
366  * The page comes unlocked when it has no locked buffer_async buffers
367  * left.
368  *
369  * PageLocked prevents anyone starting new async I/O reads any of
370  * the buffers.
371  *
372  * PageWriteback is used to prevent simultaneous writeout of the same
373  * page.
374  *
375  * PageLocked prevents anyone from starting writeback of a page which is
376  * under read I/O (PageWriteback is only ever set against a locked page).
377  */
378 static void mark_buffer_async_read(struct buffer_head *bh)
379 {
380         bh->b_end_io = end_buffer_async_read;
381         set_buffer_async_read(bh);
382 }
383
384 static void mark_buffer_async_write_endio(struct buffer_head *bh,
385                                           bh_end_io_t *handler)
386 {
387         bh->b_end_io = handler;
388         set_buffer_async_write(bh);
389 }
390
391 void mark_buffer_async_write(struct buffer_head *bh)
392 {
393         mark_buffer_async_write_endio(bh, end_buffer_async_write);
394 }
395 EXPORT_SYMBOL(mark_buffer_async_write);
396
397
398 /*
399  * fs/buffer.c contains helper functions for buffer-backed address space's
400  * fsync functions.  A common requirement for buffer-based filesystems is
401  * that certain data from the backing blockdev needs to be written out for
402  * a successful fsync().  For example, ext2 indirect blocks need to be
403  * written back and waited upon before fsync() returns.
404  *
405  * The functions mark_buffer_inode_dirty(), fsync_inode_buffers(),
406  * inode_has_buffers() and invalidate_inode_buffers() are provided for the
407  * management of a list of dependent buffers at ->i_mapping->private_list.
408  *
409  * Locking is a little subtle: try_to_free_buffers() will remove buffers
410  * from their controlling inode's queue when they are being freed.  But
411  * try_to_free_buffers() will be operating against the *blockdev* mapping
412  * at the time, not against the S_ISREG file which depends on those buffers.
413  * So the locking for private_list is via the private_lock in the address_space
414  * which backs the buffers.  Which is different from the address_space 
415  * against which the buffers are listed.  So for a particular address_space,
416  * mapping->private_lock does *not* protect mapping->private_list!  In fact,
417  * mapping->private_list will always be protected by the backing blockdev's
418  * ->private_lock.
419  *
420  * Which introduces a requirement: all buffers on an address_space's
421  * ->private_list must be from the same address_space: the blockdev's.
422  *
423  * address_spaces which do not place buffers at ->private_list via these
424  * utility functions are free to use private_lock and private_list for
425  * whatever they want.  The only requirement is that list_empty(private_list)
426  * be true at clear_inode() time.
427  *
428  * FIXME: clear_inode should not call invalidate_inode_buffers().  The
429  * filesystems should do that.  invalidate_inode_buffers() should just go
430  * BUG_ON(!list_empty).
431  *
432  * FIXME: mark_buffer_dirty_inode() is a data-plane operation.  It should
433  * take an address_space, not an inode.  And it should be called
434  * mark_buffer_dirty_fsync() to clearly define why those buffers are being
435  * queued up.
436  *
437  * FIXME: mark_buffer_dirty_inode() doesn't need to add the buffer to the
438  * list if it is already on a list.  Because if the buffer is on a list,
439  * it *must* already be on the right one.  If not, the filesystem is being
440  * silly.  This will save a ton of locking.  But first we have to ensure
441  * that buffers are taken *off* the old inode's list when they are freed
442  * (presumably in truncate).  That requires careful auditing of all
443  * filesystems (do it inside bforget()).  It could also be done by bringing
444  * b_inode back.
445  */
446
447 /*
448  * The buffer's backing address_space's private_lock must be held
449  */
450 static void __remove_assoc_queue(struct buffer_head *bh)
451 {
452         list_del_init(&bh->b_assoc_buffers);
453         WARN_ON(!bh->b_assoc_map);
454         bh->b_assoc_map = NULL;
455 }
456
457 int inode_has_buffers(struct inode *inode)
458 {
459         return !list_empty(&inode->i_data.private_list);
460 }
461
462 /*
463  * osync is designed to support O_SYNC io.  It waits synchronously for
464  * all already-submitted IO to complete, but does not queue any new
465  * writes to the disk.
466  *
467  * To do O_SYNC writes, just queue the buffer writes with ll_rw_block as
468  * you dirty the buffers, and then use osync_inode_buffers to wait for
469  * completion.  Any other dirty buffers which are not yet queued for
470  * write will not be flushed to disk by the osync.
471  */
472 static int osync_buffers_list(spinlock_t *lock, struct list_head *list)
473 {
474         struct buffer_head *bh;
475         struct list_head *p;
476         int err = 0;
477
478         spin_lock(lock);
479 repeat:
480         list_for_each_prev(p, list) {
481                 bh = BH_ENTRY(p);
482                 if (buffer_locked(bh)) {
483                         get_bh(bh);
484                         spin_unlock(lock);
485                         wait_on_buffer(bh);
486                         if (!buffer_uptodate(bh))
487                                 err = -EIO;
488                         brelse(bh);
489                         spin_lock(lock);
490                         goto repeat;
491                 }
492         }
493         spin_unlock(lock);
494         return err;
495 }
496
497 void emergency_thaw_bdev(struct super_block *sb)
498 {
499         while (sb->s_bdev && !thaw_bdev(sb->s_bdev, sb))
500                 printk(KERN_WARNING "Emergency Thaw on %pg\n", sb->s_bdev);
501 }
502
503 /**
504  * sync_mapping_buffers - write out & wait upon a mapping's "associated" buffers
505  * @mapping: the mapping which wants those buffers written
506  *
507  * Starts I/O against the buffers at mapping->private_list, and waits upon
508  * that I/O.
509  *
510  * Basically, this is a convenience function for fsync().
511  * @mapping is a file or directory which needs those buffers to be written for
512  * a successful fsync().
513  */
514 int sync_mapping_buffers(struct address_space *mapping)
515 {
516         struct address_space *buffer_mapping = mapping->private_data;
517
518         if (buffer_mapping == NULL || list_empty(&mapping->private_list))
519                 return 0;
520
521         return fsync_buffers_list(&buffer_mapping->private_lock,
522                                         &mapping->private_list);
523 }
524 EXPORT_SYMBOL(sync_mapping_buffers);
525
526 /*
527  * Called when we've recently written block `bblock', and it is known that
528  * `bblock' was for a buffer_boundary() buffer.  This means that the block at
529  * `bblock + 1' is probably a dirty indirect block.  Hunt it down and, if it's
530  * dirty, schedule it for IO.  So that indirects merge nicely with their data.
531  */
532 void write_boundary_block(struct block_device *bdev,
533                         sector_t bblock, unsigned blocksize)
534 {
535         struct buffer_head *bh = __find_get_block(bdev, bblock + 1, blocksize);
536         if (bh) {
537                 if (buffer_dirty(bh))
538                         ll_rw_block(REQ_OP_WRITE, 0, 1, &bh);
539                 put_bh(bh);
540         }
541 }
542
543 void mark_buffer_dirty_inode(struct buffer_head *bh, struct inode *inode)
544 {
545         struct address_space *mapping = inode->i_mapping;
546         struct address_space *buffer_mapping = bh->b_page->mapping;
547
548         mark_buffer_dirty(bh);
549         if (!mapping->private_data) {
550                 mapping->private_data = buffer_mapping;
551         } else {
552                 BUG_ON(mapping->private_data != buffer_mapping);
553         }
554         if (!bh->b_assoc_map) {
555                 spin_lock(&buffer_mapping->private_lock);
556                 list_move_tail(&bh->b_assoc_buffers,
557                                 &mapping->private_list);
558                 bh->b_assoc_map = mapping;
559                 spin_unlock(&buffer_mapping->private_lock);
560         }
561 }
562 EXPORT_SYMBOL(mark_buffer_dirty_inode);
563
564 /*
565  * Mark the page dirty, and set it dirty in the radix tree, and mark the inode
566  * dirty.
567  *
568  * If warn is true, then emit a warning if the page is not uptodate and has
569  * not been truncated.
570  *
571  * The caller must hold lock_page_memcg().
572  */
573 void __set_page_dirty(struct page *page, struct address_space *mapping,
574                              int warn)
575 {
576         unsigned long flags;
577
578         xa_lock_irqsave(&mapping->i_pages, flags);
579         if (page->mapping) {    /* Race with truncate? */
580                 WARN_ON_ONCE(warn && !PageUptodate(page));
581                 account_page_dirtied(page, mapping);
582                 radix_tree_tag_set(&mapping->i_pages,
583                                 page_index(page), PAGECACHE_TAG_DIRTY);
584         }
585         xa_unlock_irqrestore(&mapping->i_pages, flags);
586 }
587 EXPORT_SYMBOL_GPL(__set_page_dirty);
588
589 /*
590  * Add a page to the dirty page list.
591  *
592  * It is a sad fact of life that this function is called from several places
593  * deeply under spinlocking.  It may not sleep.
594  *
595  * If the page has buffers, the uptodate buffers are set dirty, to preserve
596  * dirty-state coherency between the page and the buffers.  It the page does
597  * not have buffers then when they are later attached they will all be set
598  * dirty.
599  *
600  * The buffers are dirtied before the page is dirtied.  There's a small race
601  * window in which a writepage caller may see the page cleanness but not the
602  * buffer dirtiness.  That's fine.  If this code were to set the page dirty
603  * before the buffers, a concurrent writepage caller could clear the page dirty
604  * bit, see a bunch of clean buffers and we'd end up with dirty buffers/clean
605  * page on the dirty page list.
606  *
607  * We use private_lock to lock against try_to_free_buffers while using the
608  * page's buffer list.  Also use this to protect against clean buffers being
609  * added to the page after it was set dirty.
610  *
611  * FIXME: may need to call ->reservepage here as well.  That's rather up to the
612  * address_space though.
613  */
614 int __set_page_dirty_buffers(struct page *page)
615 {
616         int newly_dirty;
617         struct address_space *mapping = page_mapping(page);
618
619         if (unlikely(!mapping))
620                 return !TestSetPageDirty(page);
621
622         spin_lock(&mapping->private_lock);
623         if (page_has_buffers(page)) {
624                 struct buffer_head *head = page_buffers(page);
625                 struct buffer_head *bh = head;
626
627                 do {
628                         set_buffer_dirty(bh);
629                         bh = bh->b_this_page;
630                 } while (bh != head);
631         }
632         /*
633          * Lock out page->mem_cgroup migration to keep PageDirty
634          * synchronized with per-memcg dirty page counters.
635          */
636         lock_page_memcg(page);
637         newly_dirty = !TestSetPageDirty(page);
638         spin_unlock(&mapping->private_lock);
639
640         if (newly_dirty)
641                 __set_page_dirty(page, mapping, 1);
642
643         unlock_page_memcg(page);
644
645         if (newly_dirty)
646                 __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
647
648         return newly_dirty;
649 }
650 EXPORT_SYMBOL(__set_page_dirty_buffers);
651
652 /*
653  * Write out and wait upon a list of buffers.
654  *
655  * We have conflicting pressures: we want to make sure that all
656  * initially dirty buffers get waited on, but that any subsequently
657  * dirtied buffers don't.  After all, we don't want fsync to last
658  * forever if somebody is actively writing to the file.
659  *
660  * Do this in two main stages: first we copy dirty buffers to a
661  * temporary inode list, queueing the writes as we go.  Then we clean
662  * up, waiting for those writes to complete.
663  * 
664  * During this second stage, any subsequent updates to the file may end
665  * up refiling the buffer on the original inode's dirty list again, so
666  * there is a chance we will end up with a buffer queued for write but
667  * not yet completed on that list.  So, as a final cleanup we go through
668  * the osync code to catch these locked, dirty buffers without requeuing
669  * any newly dirty buffers for write.
670  */
671 static int fsync_buffers_list(spinlock_t *lock, struct list_head *list)
672 {
673         struct buffer_head *bh;
674         struct list_head tmp;
675         struct address_space *mapping;
676         int err = 0, err2;
677         struct blk_plug plug;
678
679         INIT_LIST_HEAD(&tmp);
680         blk_start_plug(&plug);
681
682         spin_lock(lock);
683         while (!list_empty(list)) {
684                 bh = BH_ENTRY(list->next);
685                 mapping = bh->b_assoc_map;
686                 __remove_assoc_queue(bh);
687                 /* Avoid race with mark_buffer_dirty_inode() which does
688                  * a lockless check and we rely on seeing the dirty bit */
689                 smp_mb();
690                 if (buffer_dirty(bh) || buffer_locked(bh)) {
691                         list_add(&bh->b_assoc_buffers, &tmp);
692                         bh->b_assoc_map = mapping;
693                         if (buffer_dirty(bh)) {
694                                 get_bh(bh);
695                                 spin_unlock(lock);
696                                 /*
697                                  * Ensure any pending I/O completes so that
698                                  * write_dirty_buffer() actually writes the
699                                  * current contents - it is a noop if I/O is
700                                  * still in flight on potentially older
701                                  * contents.
702                                  */
703                                 write_dirty_buffer(bh, REQ_SYNC);
704
705                                 /*
706                                  * Kick off IO for the previous mapping. Note
707                                  * that we will not run the very last mapping,
708                                  * wait_on_buffer() will do that for us
709                                  * through sync_buffer().
710                                  */
711                                 brelse(bh);
712                                 spin_lock(lock);
713                         }
714                 }
715         }
716
717         spin_unlock(lock);
718         blk_finish_plug(&plug);
719         spin_lock(lock);
720
721         while (!list_empty(&tmp)) {
722                 bh = BH_ENTRY(tmp.prev);
723                 get_bh(bh);
724                 mapping = bh->b_assoc_map;
725                 __remove_assoc_queue(bh);
726                 /* Avoid race with mark_buffer_dirty_inode() which does
727                  * a lockless check and we rely on seeing the dirty bit */
728                 smp_mb();
729                 if (buffer_dirty(bh)) {
730                         list_add(&bh->b_assoc_buffers,
731                                  &mapping->private_list);
732                         bh->b_assoc_map = mapping;
733                 }
734                 spin_unlock(lock);
735                 wait_on_buffer(bh);
736                 if (!buffer_uptodate(bh))
737                         err = -EIO;
738                 brelse(bh);
739                 spin_lock(lock);
740         }
741         
742         spin_unlock(lock);
743         err2 = osync_buffers_list(lock, list);
744         if (err)
745                 return err;
746         else
747                 return err2;
748 }
749
750 /*
751  * Invalidate any and all dirty buffers on a given inode.  We are
752  * probably unmounting the fs, but that doesn't mean we have already
753  * done a sync().  Just drop the buffers from the inode list.
754  *
755  * NOTE: we take the inode's blockdev's mapping's private_lock.  Which
756  * assumes that all the buffers are against the blockdev.  Not true
757  * for reiserfs.
758  */
759 void invalidate_inode_buffers(struct inode *inode)
760 {
761         if (inode_has_buffers(inode)) {
762                 struct address_space *mapping = &inode->i_data;
763                 struct list_head *list = &mapping->private_list;
764                 struct address_space *buffer_mapping = mapping->private_data;
765
766                 spin_lock(&buffer_mapping->private_lock);
767                 while (!list_empty(list))
768                         __remove_assoc_queue(BH_ENTRY(list->next));
769                 spin_unlock(&buffer_mapping->private_lock);
770         }
771 }
772 EXPORT_SYMBOL(invalidate_inode_buffers);
773
774 /*
775  * Remove any clean buffers from the inode's buffer list.  This is called
776  * when we're trying to free the inode itself.  Those buffers can pin it.
777  *
778  * Returns true if all buffers were removed.
779  */
780 int remove_inode_buffers(struct inode *inode)
781 {
782         int ret = 1;
783
784         if (inode_has_buffers(inode)) {
785                 struct address_space *mapping = &inode->i_data;
786                 struct list_head *list = &mapping->private_list;
787                 struct address_space *buffer_mapping = mapping->private_data;
788
789                 spin_lock(&buffer_mapping->private_lock);
790                 while (!list_empty(list)) {
791                         struct buffer_head *bh = BH_ENTRY(list->next);
792                         if (buffer_dirty(bh)) {
793                                 ret = 0;
794                                 break;
795                         }
796                         __remove_assoc_queue(bh);
797                 }
798                 spin_unlock(&buffer_mapping->private_lock);
799         }
800         return ret;
801 }
802
803 /*
804  * Create the appropriate buffers when given a page for data area and
805  * the size of each buffer.. Use the bh->b_this_page linked list to
806  * follow the buffers created.  Return NULL if unable to create more
807  * buffers.
808  *
809  * The retry flag is used to differentiate async IO (paging, swapping)
810  * which may not fail from ordinary buffer allocations.
811  */
812 struct buffer_head *alloc_page_buffers(struct page *page, unsigned long size,
813                 bool retry)
814 {
815         struct buffer_head *bh, *head;
816         gfp_t gfp = GFP_NOFS;
817         long offset;
818
819         if (retry)
820                 gfp |= __GFP_NOFAIL;
821
822         head = NULL;
823         offset = PAGE_SIZE;
824         while ((offset -= size) >= 0) {
825                 bh = alloc_buffer_head(gfp);
826                 if (!bh)
827                         goto no_grow;
828
829                 bh->b_this_page = head;
830                 bh->b_blocknr = -1;
831                 head = bh;
832
833                 bh->b_size = size;
834
835                 /* Link the buffer to its page */
836                 set_bh_page(bh, page, offset);
837         }
838         return head;
839 /*
840  * In case anything failed, we just free everything we got.
841  */
842 no_grow:
843         if (head) {
844                 do {
845                         bh = head;
846                         head = head->b_this_page;
847                         free_buffer_head(bh);
848                 } while (head);
849         }
850
851         return NULL;
852 }
853 EXPORT_SYMBOL_GPL(alloc_page_buffers);
854
855 static inline void
856 link_dev_buffers(struct page *page, struct buffer_head *head)
857 {
858         struct buffer_head *bh, *tail;
859
860         bh = head;
861         do {
862                 tail = bh;
863                 bh = bh->b_this_page;
864         } while (bh);
865         tail->b_this_page = head;
866         attach_page_buffers(page, head);
867 }
868
869 static sector_t blkdev_max_block(struct block_device *bdev, unsigned int size)
870 {
871         sector_t retval = ~((sector_t)0);
872         loff_t sz = i_size_read(bdev->bd_inode);
873
874         if (sz) {
875                 unsigned int sizebits = blksize_bits(size);
876                 retval = (sz >> sizebits);
877         }
878         return retval;
879 }
880
881 /*
882  * Initialise the state of a blockdev page's buffers.
883  */ 
884 static sector_t
885 init_page_buffers(struct page *page, struct block_device *bdev,
886                         sector_t block, int size)
887 {
888         struct buffer_head *head = page_buffers(page);
889         struct buffer_head *bh = head;
890         int uptodate = PageUptodate(page);
891         sector_t end_block = blkdev_max_block(I_BDEV(bdev->bd_inode), size);
892
893         do {
894                 if (!buffer_mapped(bh)) {
895                         bh->b_end_io = NULL;
896                         bh->b_private = NULL;
897                         bh->b_bdev = bdev;
898                         bh->b_blocknr = block;
899                         if (uptodate)
900                                 set_buffer_uptodate(bh);
901                         if (block < end_block)
902                                 set_buffer_mapped(bh);
903                 }
904                 block++;
905                 bh = bh->b_this_page;
906         } while (bh != head);
907
908         /*
909          * Caller needs to validate requested block against end of device.
910          */
911         return end_block;
912 }
913
914 /*
915  * Create the page-cache page that contains the requested block.
916  *
917  * This is used purely for blockdev mappings.
918  */
919 static int
920 grow_dev_page(struct block_device *bdev, sector_t block,
921               pgoff_t index, int size, int sizebits, gfp_t gfp)
922 {
923         struct inode *inode = bdev->bd_inode;
924         struct page *page;
925         struct buffer_head *bh;
926         sector_t end_block;
927         int ret = 0;            /* Will call free_more_memory() */
928         gfp_t gfp_mask;
929
930         gfp_mask = mapping_gfp_constraint(inode->i_mapping, ~__GFP_FS) | gfp;
931
932         /*
933          * XXX: __getblk_slow() can not really deal with failure and
934          * will endlessly loop on improvised global reclaim.  Prefer
935          * looping in the allocator rather than here, at least that
936          * code knows what it's doing.
937          */
938         gfp_mask |= __GFP_NOFAIL;
939
940         page = find_or_create_page(inode->i_mapping, index, gfp_mask);
941
942         BUG_ON(!PageLocked(page));
943
944         if (page_has_buffers(page)) {
945                 bh = page_buffers(page);
946                 if (bh->b_size == size) {
947                         end_block = init_page_buffers(page, bdev,
948                                                 (sector_t)index << sizebits,
949                                                 size);
950                         goto done;
951                 }
952                 if (!try_to_free_buffers(page))
953                         goto failed;
954         }
955
956         /*
957          * Allocate some buffers for this page
958          */
959         bh = alloc_page_buffers(page, size, true);
960
961         /*
962          * Link the page to the buffers and initialise them.  Take the
963          * lock to be atomic wrt __find_get_block(), which does not
964          * run under the page lock.
965          */
966         spin_lock(&inode->i_mapping->private_lock);
967         link_dev_buffers(page, bh);
968         end_block = init_page_buffers(page, bdev, (sector_t)index << sizebits,
969                         size);
970         spin_unlock(&inode->i_mapping->private_lock);
971 done:
972         ret = (block < end_block) ? 1 : -ENXIO;
973 failed:
974         unlock_page(page);
975         put_page(page);
976         return ret;
977 }
978
979 /*
980  * Create buffers for the specified block device block's page.  If
981  * that page was dirty, the buffers are set dirty also.
982  */
983 static int
984 grow_buffers(struct block_device *bdev, sector_t block, int size, gfp_t gfp)
985 {
986         pgoff_t index;
987         int sizebits;
988
989         sizebits = -1;
990         do {
991                 sizebits++;
992         } while ((size << sizebits) < PAGE_SIZE);
993
994         index = block >> sizebits;
995
996         /*
997          * Check for a block which wants to lie outside our maximum possible
998          * pagecache index.  (this comparison is done using sector_t types).
999          */
1000         if (unlikely(index != block >> sizebits)) {
1001                 printk(KERN_ERR "%s: requested out-of-range block %llu for "
1002                         "device %pg\n",
1003                         __func__, (unsigned long long)block,
1004                         bdev);
1005                 return -EIO;
1006         }
1007
1008         /* Create a page with the proper size buffers.. */
1009         return grow_dev_page(bdev, block, index, size, sizebits, gfp);
1010 }
1011
1012 static struct buffer_head *
1013 __getblk_slow(struct block_device *bdev, sector_t block,
1014              unsigned size, gfp_t gfp)
1015 {
1016         /* Size must be multiple of hard sectorsize */
1017         if (unlikely(size & (bdev_logical_block_size(bdev)-1) ||
1018                         (size < 512 || size > PAGE_SIZE))) {
1019                 printk(KERN_ERR "getblk(): invalid block size %d requested\n",
1020                                         size);
1021                 printk(KERN_ERR "logical block size: %d\n",
1022                                         bdev_logical_block_size(bdev));
1023
1024                 dump_stack();
1025                 return NULL;
1026         }
1027
1028         for (;;) {
1029                 struct buffer_head *bh;
1030                 int ret;
1031
1032                 bh = __find_get_block(bdev, block, size);
1033                 if (bh)
1034                         return bh;
1035
1036                 ret = grow_buffers(bdev, block, size, gfp);
1037                 if (ret < 0)
1038                         return NULL;
1039         }
1040 }
1041
1042 /*
1043  * The relationship between dirty buffers and dirty pages:
1044  *
1045  * Whenever a page has any dirty buffers, the page's dirty bit is set, and
1046  * the page is tagged dirty in its radix tree.
1047  *
1048  * At all times, the dirtiness of the buffers represents the dirtiness of
1049  * subsections of the page.  If the page has buffers, the page dirty bit is
1050  * merely a hint about the true dirty state.
1051  *
1052  * When a page is set dirty in its entirety, all its buffers are marked dirty
1053  * (if the page has buffers).
1054  *
1055  * When a buffer is marked dirty, its page is dirtied, but the page's other
1056  * buffers are not.
1057  *
1058  * Also.  When blockdev buffers are explicitly read with bread(), they
1059  * individually become uptodate.  But their backing page remains not
1060  * uptodate - even if all of its buffers are uptodate.  A subsequent
1061  * block_read_full_page() against that page will discover all the uptodate
1062  * buffers, will set the page uptodate and will perform no I/O.
1063  */
1064
1065 /**
1066  * mark_buffer_dirty - mark a buffer_head as needing writeout
1067  * @bh: the buffer_head to mark dirty
1068  *
1069  * mark_buffer_dirty() will set the dirty bit against the buffer, then set its
1070  * backing page dirty, then tag the page as dirty in its address_space's radix
1071  * tree and then attach the address_space's inode to its superblock's dirty
1072  * inode list.
1073  *
1074  * mark_buffer_dirty() is atomic.  It takes bh->b_page->mapping->private_lock,
1075  * i_pages lock and mapping->host->i_lock.
1076  */
1077 void mark_buffer_dirty(struct buffer_head *bh)
1078 {
1079         WARN_ON_ONCE(!buffer_uptodate(bh));
1080
1081         trace_block_dirty_buffer(bh);
1082
1083         /*
1084          * Very *carefully* optimize the it-is-already-dirty case.
1085          *
1086          * Don't let the final "is it dirty" escape to before we
1087          * perhaps modified the buffer.
1088          */
1089         if (buffer_dirty(bh)) {
1090                 smp_mb();
1091                 if (buffer_dirty(bh))
1092                         return;
1093         }
1094
1095         if (!test_set_buffer_dirty(bh)) {
1096                 struct page *page = bh->b_page;
1097                 struct address_space *mapping = NULL;
1098
1099                 lock_page_memcg(page);
1100                 if (!TestSetPageDirty(page)) {
1101                         mapping = page_mapping(page);
1102                         if (mapping)
1103                                 __set_page_dirty(page, mapping, 0);
1104                 }
1105                 unlock_page_memcg(page);
1106                 if (mapping)
1107                         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
1108         }
1109 }
1110 EXPORT_SYMBOL(mark_buffer_dirty);
1111
1112 void mark_buffer_write_io_error(struct buffer_head *bh)
1113 {
1114         set_buffer_write_io_error(bh);
1115         /* FIXME: do we need to set this in both places? */
1116         if (bh->b_page && bh->b_page->mapping)
1117                 mapping_set_error(bh->b_page->mapping, -EIO);
1118         if (bh->b_assoc_map)
1119                 mapping_set_error(bh->b_assoc_map, -EIO);
1120 }
1121 EXPORT_SYMBOL(mark_buffer_write_io_error);
1122
1123 /*
1124  * Decrement a buffer_head's reference count.  If all buffers against a page
1125  * have zero reference count, are clean and unlocked, and if the page is clean
1126  * and unlocked then try_to_free_buffers() may strip the buffers from the page
1127  * in preparation for freeing it (sometimes, rarely, buffers are removed from
1128  * a page but it ends up not being freed, and buffers may later be reattached).
1129  */
1130 void __brelse(struct buffer_head * buf)
1131 {
1132         if (atomic_read(&buf->b_count)) {
1133                 put_bh(buf);
1134                 return;
1135         }
1136         WARN(1, KERN_ERR "VFS: brelse: Trying to free free buffer\n");
1137 }
1138 EXPORT_SYMBOL(__brelse);
1139
1140 /*
1141  * bforget() is like brelse(), except it discards any
1142  * potentially dirty data.
1143  */
1144 void __bforget(struct buffer_head *bh)
1145 {
1146         clear_buffer_dirty(bh);
1147         if (bh->b_assoc_map) {
1148                 struct address_space *buffer_mapping = bh->b_page->mapping;
1149
1150                 spin_lock(&buffer_mapping->private_lock);
1151                 list_del_init(&bh->b_assoc_buffers);
1152                 bh->b_assoc_map = NULL;
1153                 spin_unlock(&buffer_mapping->private_lock);
1154         }
1155         __brelse(bh);
1156 }
1157 EXPORT_SYMBOL(__bforget);
1158
1159 static struct buffer_head *__bread_slow(struct buffer_head *bh)
1160 {
1161         lock_buffer(bh);
1162         if (buffer_uptodate(bh)) {
1163                 unlock_buffer(bh);
1164                 return bh;
1165         } else {
1166                 get_bh(bh);
1167                 bh->b_end_io = end_buffer_read_sync;
1168                 submit_bh(REQ_OP_READ, 0, bh);
1169                 wait_on_buffer(bh);
1170                 if (buffer_uptodate(bh))
1171                         return bh;
1172         }
1173         brelse(bh);
1174         return NULL;
1175 }
1176
1177 /*
1178  * Per-cpu buffer LRU implementation.  To reduce the cost of __find_get_block().
1179  * The bhs[] array is sorted - newest buffer is at bhs[0].  Buffers have their
1180  * refcount elevated by one when they're in an LRU.  A buffer can only appear
1181  * once in a particular CPU's LRU.  A single buffer can be present in multiple
1182  * CPU's LRUs at the same time.
1183  *
1184  * This is a transparent caching front-end to sb_bread(), sb_getblk() and
1185  * sb_find_get_block().
1186  *
1187  * The LRUs themselves only need locking against invalidate_bh_lrus.  We use
1188  * a local interrupt disable for that.
1189  */
1190
1191 #define BH_LRU_SIZE     16
1192
1193 struct bh_lru {
1194         struct buffer_head *bhs[BH_LRU_SIZE];
1195 };
1196
1197 static DEFINE_PER_CPU(struct bh_lru, bh_lrus) = {{ NULL }};
1198
1199 #ifdef CONFIG_SMP
1200 #define bh_lru_lock()   local_irq_disable()
1201 #define bh_lru_unlock() local_irq_enable()
1202 #else
1203 #define bh_lru_lock()   preempt_disable()
1204 #define bh_lru_unlock() preempt_enable()
1205 #endif
1206
1207 static inline void check_irqs_on(void)
1208 {
1209 #ifdef irqs_disabled
1210         BUG_ON(irqs_disabled());
1211 #endif
1212 }
1213
1214 /*
1215  * Install a buffer_head into this cpu's LRU.  If not already in the LRU, it is
1216  * inserted at the front, and the buffer_head at the back if any is evicted.
1217  * Or, if already in the LRU it is moved to the front.
1218  */
1219 static void bh_lru_install(struct buffer_head *bh)
1220 {
1221         struct buffer_head *evictee = bh;
1222         struct bh_lru *b;
1223         int i;
1224
1225         check_irqs_on();
1226         bh_lru_lock();
1227
1228         b = this_cpu_ptr(&bh_lrus);
1229         for (i = 0; i < BH_LRU_SIZE; i++) {
1230                 swap(evictee, b->bhs[i]);
1231                 if (evictee == bh) {
1232                         bh_lru_unlock();
1233                         return;
1234                 }
1235         }
1236
1237         get_bh(bh);
1238         bh_lru_unlock();
1239         brelse(evictee);
1240 }
1241
1242 /*
1243  * Look up the bh in this cpu's LRU.  If it's there, move it to the head.
1244  */
1245 static struct buffer_head *
1246 lookup_bh_lru(struct block_device *bdev, sector_t block, unsigned size)
1247 {
1248         struct buffer_head *ret = NULL;
1249         unsigned int i;
1250
1251         check_irqs_on();
1252         bh_lru_lock();
1253         for (i = 0; i < BH_LRU_SIZE; i++) {
1254                 struct buffer_head *bh = __this_cpu_read(bh_lrus.bhs[i]);
1255
1256                 if (bh && bh->b_blocknr == block && bh->b_bdev == bdev &&
1257                     bh->b_size == size) {
1258                         if (i) {
1259                                 while (i) {
1260                                         __this_cpu_write(bh_lrus.bhs[i],
1261                                                 __this_cpu_read(bh_lrus.bhs[i - 1]));
1262                                         i--;
1263                                 }
1264                                 __this_cpu_write(bh_lrus.bhs[0], bh);
1265                         }
1266                         get_bh(bh);
1267                         ret = bh;
1268                         break;
1269                 }
1270         }
1271         bh_lru_unlock();
1272         return ret;
1273 }
1274
1275 /*
1276  * Perform a pagecache lookup for the matching buffer.  If it's there, refresh
1277  * it in the LRU and mark it as accessed.  If it is not present then return
1278  * NULL
1279  */
1280 struct buffer_head *
1281 __find_get_block(struct block_device *bdev, sector_t block, unsigned size)
1282 {
1283         struct buffer_head *bh = lookup_bh_lru(bdev, block, size);
1284
1285         if (bh == NULL) {
1286                 /* __find_get_block_slow will mark the page accessed */
1287                 bh = __find_get_block_slow(bdev, block);
1288                 if (bh)
1289                         bh_lru_install(bh);
1290         } else
1291                 touch_buffer(bh);
1292
1293         return bh;
1294 }
1295 EXPORT_SYMBOL(__find_get_block);
1296
1297 /*
1298  * __getblk_gfp() will locate (and, if necessary, create) the buffer_head
1299  * which corresponds to the passed block_device, block and size. The
1300  * returned buffer has its reference count incremented.
1301  *
1302  * __getblk_gfp() will lock up the machine if grow_dev_page's
1303  * try_to_free_buffers() attempt is failing.  FIXME, perhaps?
1304  */
1305 struct buffer_head *
1306 __getblk_gfp(struct block_device *bdev, sector_t block,
1307              unsigned size, gfp_t gfp)
1308 {
1309         struct buffer_head *bh = __find_get_block(bdev, block, size);
1310
1311         might_sleep();
1312         if (bh == NULL)
1313                 bh = __getblk_slow(bdev, block, size, gfp);
1314         return bh;
1315 }
1316 EXPORT_SYMBOL(__getblk_gfp);
1317
1318 /*
1319  * Do async read-ahead on a buffer..
1320  */
1321 void __breadahead(struct block_device *bdev, sector_t block, unsigned size)
1322 {
1323         struct buffer_head *bh = __getblk(bdev, block, size);
1324         if (likely(bh)) {
1325                 ll_rw_block(REQ_OP_READ, REQ_RAHEAD, 1, &bh);
1326                 brelse(bh);
1327         }
1328 }
1329 EXPORT_SYMBOL(__breadahead);
1330
1331 /**
1332  *  __bread_gfp() - reads a specified block and returns the bh
1333  *  @bdev: the block_device to read from
1334  *  @block: number of block
1335  *  @size: size (in bytes) to read
1336  *  @gfp: page allocation flag
1337  *
1338  *  Reads a specified block, and returns buffer head that contains it.
1339  *  The page cache can be allocated from non-movable area
1340  *  not to prevent page migration if you set gfp to zero.
1341  *  It returns NULL if the block was unreadable.
1342  */
1343 struct buffer_head *
1344 __bread_gfp(struct block_device *bdev, sector_t block,
1345                    unsigned size, gfp_t gfp)
1346 {
1347         struct buffer_head *bh = __getblk_gfp(bdev, block, size, gfp);
1348
1349         if (likely(bh) && !buffer_uptodate(bh))
1350                 bh = __bread_slow(bh);
1351         return bh;
1352 }
1353 EXPORT_SYMBOL(__bread_gfp);
1354
1355 /*
1356  * invalidate_bh_lrus() is called rarely - but not only at unmount.
1357  * This doesn't race because it runs in each cpu either in irq
1358  * or with preempt disabled.
1359  */
1360 static void invalidate_bh_lru(void *arg)
1361 {
1362         struct bh_lru *b = &get_cpu_var(bh_lrus);
1363         int i;
1364
1365         for (i = 0; i < BH_LRU_SIZE; i++) {
1366                 brelse(b->bhs[i]);
1367                 b->bhs[i] = NULL;
1368         }
1369         put_cpu_var(bh_lrus);
1370 }
1371
1372 static bool has_bh_in_lru(int cpu, void *dummy)
1373 {
1374         struct bh_lru *b = per_cpu_ptr(&bh_lrus, cpu);
1375         int i;
1376         
1377         for (i = 0; i < BH_LRU_SIZE; i++) {
1378                 if (b->bhs[i])
1379                         return 1;
1380         }
1381
1382         return 0;
1383 }
1384
1385 void invalidate_bh_lrus(void)
1386 {
1387         on_each_cpu_cond(has_bh_in_lru, invalidate_bh_lru, NULL, 1, GFP_KERNEL);
1388 }
1389 EXPORT_SYMBOL_GPL(invalidate_bh_lrus);
1390
1391 void set_bh_page(struct buffer_head *bh,
1392                 struct page *page, unsigned long offset)
1393 {
1394         bh->b_page = page;
1395         BUG_ON(offset >= PAGE_SIZE);
1396         if (PageHighMem(page))
1397                 /*
1398                  * This catches illegal uses and preserves the offset:
1399                  */
1400                 bh->b_data = (char *)(0 + offset);
1401         else
1402                 bh->b_data = page_address(page) + offset;
1403 }
1404 EXPORT_SYMBOL(set_bh_page);
1405
1406 /*
1407  * Called when truncating a buffer on a page completely.
1408  */
1409
1410 /* Bits that are cleared during an invalidate */
1411 #define BUFFER_FLAGS_DISCARD \
1412         (1 << BH_Mapped | 1 << BH_New | 1 << BH_Req | \
1413          1 << BH_Delay | 1 << BH_Unwritten)
1414
1415 static void discard_buffer(struct buffer_head * bh)
1416 {
1417         unsigned long b_state, b_state_old;
1418
1419         lock_buffer(bh);
1420         clear_buffer_dirty(bh);
1421         bh->b_bdev = NULL;
1422         b_state = bh->b_state;
1423         for (;;) {
1424                 b_state_old = cmpxchg(&bh->b_state, b_state,
1425                                       (b_state & ~BUFFER_FLAGS_DISCARD));
1426                 if (b_state_old == b_state)
1427                         break;
1428                 b_state = b_state_old;
1429         }
1430         unlock_buffer(bh);
1431 }
1432
1433 /**
1434  * block_invalidatepage - invalidate part or all of a buffer-backed page
1435  *
1436  * @page: the page which is affected
1437  * @offset: start of the range to invalidate
1438  * @length: length of the range to invalidate
1439  *
1440  * block_invalidatepage() is called when all or part of the page has become
1441  * invalidated by a truncate operation.
1442  *
1443  * block_invalidatepage() does not have to release all buffers, but it must
1444  * ensure that no dirty buffer is left outside @offset and that no I/O
1445  * is underway against any of the blocks which are outside the truncation
1446  * point.  Because the caller is about to free (and possibly reuse) those
1447  * blocks on-disk.
1448  */
1449 void block_invalidatepage(struct page *page, unsigned int offset,
1450                           unsigned int length)
1451 {
1452         struct buffer_head *head, *bh, *next;
1453         unsigned int curr_off = 0;
1454         unsigned int stop = length + offset;
1455
1456         BUG_ON(!PageLocked(page));
1457         if (!page_has_buffers(page))
1458                 goto out;
1459
1460         /*
1461          * Check for overflow
1462          */
1463         BUG_ON(stop > PAGE_SIZE || stop < length);
1464
1465         head = page_buffers(page);
1466         bh = head;
1467         do {
1468                 unsigned int next_off = curr_off + bh->b_size;
1469                 next = bh->b_this_page;
1470
1471                 /*
1472                  * Are we still fully in range ?
1473                  */
1474                 if (next_off > stop)
1475                         goto out;
1476
1477                 /*
1478                  * is this block fully invalidated?
1479                  */
1480                 if (offset <= curr_off)
1481                         discard_buffer(bh);
1482                 curr_off = next_off;
1483                 bh = next;
1484         } while (bh != head);
1485
1486         /*
1487          * We release buffers only if the entire page is being invalidated.
1488          * The get_block cached value has been unconditionally invalidated,
1489          * so real IO is not possible anymore.
1490          */
1491         if (length == PAGE_SIZE)
1492                 try_to_release_page(page, 0);
1493 out:
1494         return;
1495 }
1496 EXPORT_SYMBOL(block_invalidatepage);
1497
1498
1499 /*
1500  * We attach and possibly dirty the buffers atomically wrt
1501  * __set_page_dirty_buffers() via private_lock.  try_to_free_buffers
1502  * is already excluded via the page lock.
1503  */
1504 void create_empty_buffers(struct page *page,
1505                         unsigned long blocksize, unsigned long b_state)
1506 {
1507         struct buffer_head *bh, *head, *tail;
1508
1509         head = alloc_page_buffers(page, blocksize, true);
1510         bh = head;
1511         do {
1512                 bh->b_state |= b_state;
1513                 tail = bh;
1514                 bh = bh->b_this_page;
1515         } while (bh);
1516         tail->b_this_page = head;
1517
1518         spin_lock(&page->mapping->private_lock);
1519         if (PageUptodate(page) || PageDirty(page)) {
1520                 bh = head;
1521                 do {
1522                         if (PageDirty(page))
1523                                 set_buffer_dirty(bh);
1524                         if (PageUptodate(page))
1525                                 set_buffer_uptodate(bh);
1526                         bh = bh->b_this_page;
1527                 } while (bh != head);
1528         }
1529         attach_page_buffers(page, head);
1530         spin_unlock(&page->mapping->private_lock);
1531 }
1532 EXPORT_SYMBOL(create_empty_buffers);
1533
1534 /**
1535  * clean_bdev_aliases: clean a range of buffers in block device
1536  * @bdev: Block device to clean buffers in
1537  * @block: Start of a range of blocks to clean
1538  * @len: Number of blocks to clean
1539  *
1540  * We are taking a range of blocks for data and we don't want writeback of any
1541  * buffer-cache aliases starting from return from this function and until the
1542  * moment when something will explicitly mark the buffer dirty (hopefully that
1543  * will not happen until we will free that block ;-) We don't even need to mark
1544  * it not-uptodate - nobody can expect anything from a newly allocated buffer
1545  * anyway. We used to use unmap_buffer() for such invalidation, but that was
1546  * wrong. We definitely don't want to mark the alias unmapped, for example - it
1547  * would confuse anyone who might pick it with bread() afterwards...
1548  *
1549  * Also..  Note that bforget() doesn't lock the buffer.  So there can be
1550  * writeout I/O going on against recently-freed buffers.  We don't wait on that
1551  * I/O in bforget() - it's more efficient to wait on the I/O only if we really
1552  * need to.  That happens here.
1553  */
1554 void clean_bdev_aliases(struct block_device *bdev, sector_t block, sector_t len)
1555 {
1556         struct inode *bd_inode = bdev->bd_inode;
1557         struct address_space *bd_mapping = bd_inode->i_mapping;
1558         struct pagevec pvec;
1559         pgoff_t index = block >> (PAGE_SHIFT - bd_inode->i_blkbits);
1560         pgoff_t end;
1561         int i, count;
1562         struct buffer_head *bh;
1563         struct buffer_head *head;
1564
1565         end = (block + len - 1) >> (PAGE_SHIFT - bd_inode->i_blkbits);
1566         pagevec_init(&pvec);
1567         while (pagevec_lookup_range(&pvec, bd_mapping, &index, end)) {
1568                 count = pagevec_count(&pvec);
1569                 for (i = 0; i < count; i++) {
1570                         struct page *page = pvec.pages[i];
1571
1572                         if (!page_has_buffers(page))
1573                                 continue;
1574                         /*
1575                          * We use page lock instead of bd_mapping->private_lock
1576                          * to pin buffers here since we can afford to sleep and
1577                          * it scales better than a global spinlock lock.
1578                          */
1579                         lock_page(page);
1580                         /* Recheck when the page is locked which pins bhs */
1581                         if (!page_has_buffers(page))
1582                                 goto unlock_page;
1583                         head = page_buffers(page);
1584                         bh = head;
1585                         do {
1586                                 if (!buffer_mapped(bh) || (bh->b_blocknr < block))
1587                                         goto next;
1588                                 if (bh->b_blocknr >= block + len)
1589                                         break;
1590                                 clear_buffer_dirty(bh);
1591                                 wait_on_buffer(bh);
1592                                 clear_buffer_req(bh);
1593 next:
1594                                 bh = bh->b_this_page;
1595                         } while (bh != head);
1596 unlock_page:
1597                         unlock_page(page);
1598                 }
1599                 pagevec_release(&pvec);
1600                 cond_resched();
1601                 /* End of range already reached? */
1602                 if (index > end || !index)
1603                         break;
1604         }
1605 }
1606 EXPORT_SYMBOL(clean_bdev_aliases);
1607
1608 /*
1609  * Size is a power-of-two in the range 512..PAGE_SIZE,
1610  * and the case we care about most is PAGE_SIZE.
1611  *
1612  * So this *could* possibly be written with those
1613  * constraints in mind (relevant mostly if some
1614  * architecture has a slow bit-scan instruction)
1615  */
1616 static inline int block_size_bits(unsigned int blocksize)
1617 {
1618         return ilog2(blocksize);
1619 }
1620
1621 static struct buffer_head *create_page_buffers(struct page *page, struct inode *inode, unsigned int b_state)
1622 {
1623         BUG_ON(!PageLocked(page));
1624
1625         if (!page_has_buffers(page))
1626                 create_empty_buffers(page, 1 << READ_ONCE(inode->i_blkbits),
1627                                      b_state);
1628         return page_buffers(page);
1629 }
1630
1631 /*
1632  * NOTE! All mapped/uptodate combinations are valid:
1633  *
1634  *      Mapped  Uptodate        Meaning
1635  *
1636  *      No      No              "unknown" - must do get_block()
1637  *      No      Yes             "hole" - zero-filled
1638  *      Yes     No              "allocated" - allocated on disk, not read in
1639  *      Yes     Yes             "valid" - allocated and up-to-date in memory.
1640  *
1641  * "Dirty" is valid only with the last case (mapped+uptodate).
1642  */
1643
1644 /*
1645  * While block_write_full_page is writing back the dirty buffers under
1646  * the page lock, whoever dirtied the buffers may decide to clean them
1647  * again at any time.  We handle that by only looking at the buffer
1648  * state inside lock_buffer().
1649  *
1650  * If block_write_full_page() is called for regular writeback
1651  * (wbc->sync_mode == WB_SYNC_NONE) then it will redirty a page which has a
1652  * locked buffer.   This only can happen if someone has written the buffer
1653  * directly, with submit_bh().  At the address_space level PageWriteback
1654  * prevents this contention from occurring.
1655  *
1656  * If block_write_full_page() is called with wbc->sync_mode ==
1657  * WB_SYNC_ALL, the writes are posted using REQ_SYNC; this
1658  * causes the writes to be flagged as synchronous writes.
1659  */
1660 int __block_write_full_page(struct inode *inode, struct page *page,
1661                         get_block_t *get_block, struct writeback_control *wbc,
1662                         bh_end_io_t *handler)
1663 {
1664         int err;
1665         sector_t block;
1666         sector_t last_block;
1667         struct buffer_head *bh, *head;
1668         unsigned int blocksize, bbits;
1669         int nr_underway = 0;
1670         int write_flags = wbc_to_write_flags(wbc);
1671
1672         head = create_page_buffers(page, inode,
1673                                         (1 << BH_Dirty)|(1 << BH_Uptodate));
1674
1675         /*
1676          * Be very careful.  We have no exclusion from __set_page_dirty_buffers
1677          * here, and the (potentially unmapped) buffers may become dirty at
1678          * any time.  If a buffer becomes dirty here after we've inspected it
1679          * then we just miss that fact, and the page stays dirty.
1680          *
1681          * Buffers outside i_size may be dirtied by __set_page_dirty_buffers;
1682          * handle that here by just cleaning them.
1683          */
1684
1685         bh = head;
1686         blocksize = bh->b_size;
1687         bbits = block_size_bits(blocksize);
1688
1689         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1690         last_block = (i_size_read(inode) - 1) >> bbits;
1691
1692         /*
1693          * Get all the dirty buffers mapped to disk addresses and
1694          * handle any aliases from the underlying blockdev's mapping.
1695          */
1696         do {
1697                 if (block > last_block) {
1698                         /*
1699                          * mapped buffers outside i_size will occur, because
1700                          * this page can be outside i_size when there is a
1701                          * truncate in progress.
1702                          */
1703                         /*
1704                          * The buffer was zeroed by block_write_full_page()
1705                          */
1706                         clear_buffer_dirty(bh);
1707                         set_buffer_uptodate(bh);
1708                 } else if ((!buffer_mapped(bh) || buffer_delay(bh)) &&
1709                            buffer_dirty(bh)) {
1710                         WARN_ON(bh->b_size != blocksize);
1711                         err = get_block(inode, block, bh, 1);
1712                         if (err)
1713                                 goto recover;
1714                         clear_buffer_delay(bh);
1715                         if (buffer_new(bh)) {
1716                                 /* blockdev mappings never come here */
1717                                 clear_buffer_new(bh);
1718                                 clean_bdev_bh_alias(bh);
1719                         }
1720                 }
1721                 bh = bh->b_this_page;
1722                 block++;
1723         } while (bh != head);
1724
1725         do {
1726                 if (!buffer_mapped(bh))
1727                         continue;
1728                 /*
1729                  * If it's a fully non-blocking write attempt and we cannot
1730                  * lock the buffer then redirty the page.  Note that this can
1731                  * potentially cause a busy-wait loop from writeback threads
1732                  * and kswapd activity, but those code paths have their own
1733                  * higher-level throttling.
1734                  */
1735                 if (wbc->sync_mode != WB_SYNC_NONE) {
1736                         lock_buffer(bh);
1737                 } else if (!trylock_buffer(bh)) {
1738                         redirty_page_for_writepage(wbc, page);
1739                         continue;
1740                 }
1741                 if (test_clear_buffer_dirty(bh)) {
1742                         mark_buffer_async_write_endio(bh, handler);
1743                 } else {
1744                         unlock_buffer(bh);
1745                 }
1746         } while ((bh = bh->b_this_page) != head);
1747
1748         /*
1749          * The page and its buffers are protected by PageWriteback(), so we can
1750          * drop the bh refcounts early.
1751          */
1752         BUG_ON(PageWriteback(page));
1753         set_page_writeback(page);
1754
1755         do {
1756                 struct buffer_head *next = bh->b_this_page;
1757                 if (buffer_async_write(bh)) {
1758                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1759                                         inode->i_write_hint, wbc);
1760                         nr_underway++;
1761                 }
1762                 bh = next;
1763         } while (bh != head);
1764         unlock_page(page);
1765
1766         err = 0;
1767 done:
1768         if (nr_underway == 0) {
1769                 /*
1770                  * The page was marked dirty, but the buffers were
1771                  * clean.  Someone wrote them back by hand with
1772                  * ll_rw_block/submit_bh.  A rare case.
1773                  */
1774                 end_page_writeback(page);
1775
1776                 /*
1777                  * The page and buffer_heads can be released at any time from
1778                  * here on.
1779                  */
1780         }
1781         return err;
1782
1783 recover:
1784         /*
1785          * ENOSPC, or some other error.  We may already have added some
1786          * blocks to the file, so we need to write these out to avoid
1787          * exposing stale data.
1788          * The page is currently locked and not marked for writeback
1789          */
1790         bh = head;
1791         /* Recovery: lock and submit the mapped buffers */
1792         do {
1793                 if (buffer_mapped(bh) && buffer_dirty(bh) &&
1794                     !buffer_delay(bh)) {
1795                         lock_buffer(bh);
1796                         mark_buffer_async_write_endio(bh, handler);
1797                 } else {
1798                         /*
1799                          * The buffer may have been set dirty during
1800                          * attachment to a dirty page.
1801                          */
1802                         clear_buffer_dirty(bh);
1803                 }
1804         } while ((bh = bh->b_this_page) != head);
1805         SetPageError(page);
1806         BUG_ON(PageWriteback(page));
1807         mapping_set_error(page->mapping, err);
1808         set_page_writeback(page);
1809         do {
1810                 struct buffer_head *next = bh->b_this_page;
1811                 if (buffer_async_write(bh)) {
1812                         clear_buffer_dirty(bh);
1813                         submit_bh_wbc(REQ_OP_WRITE, write_flags, bh,
1814                                         inode->i_write_hint, wbc);
1815                         nr_underway++;
1816                 }
1817                 bh = next;
1818         } while (bh != head);
1819         unlock_page(page);
1820         goto done;
1821 }
1822 EXPORT_SYMBOL(__block_write_full_page);
1823
1824 /*
1825  * If a page has any new buffers, zero them out here, and mark them uptodate
1826  * and dirty so they'll be written out (in order to prevent uninitialised
1827  * block data from leaking). And clear the new bit.
1828  */
1829 void page_zero_new_buffers(struct page *page, unsigned from, unsigned to)
1830 {
1831         unsigned int block_start, block_end;
1832         struct buffer_head *head, *bh;
1833
1834         BUG_ON(!PageLocked(page));
1835         if (!page_has_buffers(page))
1836                 return;
1837
1838         bh = head = page_buffers(page);
1839         block_start = 0;
1840         do {
1841                 block_end = block_start + bh->b_size;
1842
1843                 if (buffer_new(bh)) {
1844                         if (block_end > from && block_start < to) {
1845                                 if (!PageUptodate(page)) {
1846                                         unsigned start, size;
1847
1848                                         start = max(from, block_start);
1849                                         size = min(to, block_end) - start;
1850
1851                                         zero_user(page, start, size);
1852                                         set_buffer_uptodate(bh);
1853                                 }
1854
1855                                 clear_buffer_new(bh);
1856                                 mark_buffer_dirty(bh);
1857                         }
1858                 }
1859
1860                 block_start = block_end;
1861                 bh = bh->b_this_page;
1862         } while (bh != head);
1863 }
1864 EXPORT_SYMBOL(page_zero_new_buffers);
1865
1866 static void
1867 iomap_to_bh(struct inode *inode, sector_t block, struct buffer_head *bh,
1868                 struct iomap *iomap)
1869 {
1870         loff_t offset = block << inode->i_blkbits;
1871
1872         bh->b_bdev = iomap->bdev;
1873
1874         /*
1875          * Block points to offset in file we need to map, iomap contains
1876          * the offset at which the map starts. If the map ends before the
1877          * current block, then do not map the buffer and let the caller
1878          * handle it.
1879          */
1880         BUG_ON(offset >= iomap->offset + iomap->length);
1881
1882         switch (iomap->type) {
1883         case IOMAP_HOLE:
1884                 /*
1885                  * If the buffer is not up to date or beyond the current EOF,
1886                  * we need to mark it as new to ensure sub-block zeroing is
1887                  * executed if necessary.
1888                  */
1889                 if (!buffer_uptodate(bh) ||
1890                     (offset >= i_size_read(inode)))
1891                         set_buffer_new(bh);
1892                 break;
1893         case IOMAP_DELALLOC:
1894                 if (!buffer_uptodate(bh) ||
1895                     (offset >= i_size_read(inode)))
1896                         set_buffer_new(bh);
1897                 set_buffer_uptodate(bh);
1898                 set_buffer_mapped(bh);
1899                 set_buffer_delay(bh);
1900                 break;
1901         case IOMAP_UNWRITTEN:
1902                 /*
1903                  * For unwritten regions, we always need to ensure that
1904                  * sub-block writes cause the regions in the block we are not
1905                  * writing to are zeroed. Set the buffer as new to ensure this.
1906                  */
1907                 set_buffer_new(bh);
1908                 set_buffer_unwritten(bh);
1909                 /* FALLTHRU */
1910         case IOMAP_MAPPED:
1911                 if (offset >= i_size_read(inode))
1912                         set_buffer_new(bh);
1913                 bh->b_blocknr = (iomap->addr + offset - iomap->offset) >>
1914                                 inode->i_blkbits;
1915                 set_buffer_mapped(bh);
1916                 break;
1917         }
1918 }
1919
1920 int __block_write_begin_int(struct page *page, loff_t pos, unsigned len,
1921                 get_block_t *get_block, struct iomap *iomap)
1922 {
1923         unsigned from = pos & (PAGE_SIZE - 1);
1924         unsigned to = from + len;
1925         struct inode *inode = page->mapping->host;
1926         unsigned block_start, block_end;
1927         sector_t block;
1928         int err = 0;
1929         unsigned blocksize, bbits;
1930         struct buffer_head *bh, *head, *wait[2], **wait_bh=wait;
1931
1932         BUG_ON(!PageLocked(page));
1933         BUG_ON(from > PAGE_SIZE);
1934         BUG_ON(to > PAGE_SIZE);
1935         BUG_ON(from > to);
1936
1937         head = create_page_buffers(page, inode, 0);
1938         blocksize = head->b_size;
1939         bbits = block_size_bits(blocksize);
1940
1941         block = (sector_t)page->index << (PAGE_SHIFT - bbits);
1942
1943         for(bh = head, block_start = 0; bh != head || !block_start;
1944             block++, block_start=block_end, bh = bh->b_this_page) {
1945                 block_end = block_start + blocksize;
1946                 if (block_end <= from || block_start >= to) {
1947                         if (PageUptodate(page)) {
1948                                 if (!buffer_uptodate(bh))
1949                                         set_buffer_uptodate(bh);
1950                         }
1951                         continue;
1952                 }
1953                 if (buffer_new(bh))
1954                         clear_buffer_new(bh);
1955                 if (!buffer_mapped(bh)) {
1956                         WARN_ON(bh->b_size != blocksize);
1957                         if (get_block) {
1958                                 err = get_block(inode, block, bh, 1);
1959                                 if (err)
1960                                         break;
1961                         } else {
1962                                 iomap_to_bh(inode, block, bh, iomap);
1963                         }
1964
1965                         if (buffer_new(bh)) {
1966                                 clean_bdev_bh_alias(bh);
1967                                 if (PageUptodate(page)) {
1968                                         clear_buffer_new(bh);
1969                                         set_buffer_uptodate(bh);
1970                                         mark_buffer_dirty(bh);
1971                                         continue;
1972                                 }
1973                                 if (block_end > to || block_start < from)
1974                                         zero_user_segments(page,
1975                                                 to, block_end,
1976                                                 block_start, from);
1977                                 continue;
1978                         }
1979                 }
1980                 if (PageUptodate(page)) {
1981                         if (!buffer_uptodate(bh))
1982                                 set_buffer_uptodate(bh);
1983                         continue; 
1984                 }
1985                 if (!buffer_uptodate(bh) && !buffer_delay(bh) &&
1986                     !buffer_unwritten(bh) &&
1987                      (block_start < from || block_end > to)) {
1988                         ll_rw_block(REQ_OP_READ, 0, 1, &bh);
1989                         *wait_bh++=bh;
1990                 }
1991         }
1992         /*
1993          * If we issued read requests - let them complete.
1994          */
1995         while(wait_bh > wait) {
1996                 wait_on_buffer(*--wait_bh);
1997                 if (!buffer_uptodate(*wait_bh))
1998                         err = -EIO;
1999         }
2000         if (unlikely(err))
2001                 page_zero_new_buffers(page, from, to);
2002         return err;
2003 }
2004
2005 int __block_write_begin(struct page *page, loff_t pos, unsigned len,
2006                 get_block_t *get_block)
2007 {
2008         return __block_write_begin_int(page, pos, len, get_block, NULL);
2009 }
2010 EXPORT_SYMBOL(__block_write_begin);
2011
2012 static int __block_commit_write(struct inode *inode, struct page *page,
2013                 unsigned from, unsigned to)
2014 {
2015         unsigned block_start, block_end;
2016         int partial = 0;
2017         unsigned blocksize;
2018         struct buffer_head *bh, *head;
2019
2020         bh = head = page_buffers(page);
2021         blocksize = bh->b_size;
2022
2023         block_start = 0;
2024         do {
2025                 block_end = block_start + blocksize;
2026                 if (block_end <= from || block_start >= to) {
2027                         if (!buffer_uptodate(bh))
2028                                 partial = 1;
2029                 } else {
2030                         set_buffer_uptodate(bh);
2031                         mark_buffer_dirty(bh);
2032                 }
2033                 clear_buffer_new(bh);
2034
2035                 block_start = block_end;
2036                 bh = bh->b_this_page;
2037         } while (bh != head);
2038
2039         /*
2040          * If this is a partial write which happened to make all buffers
2041          * uptodate then we can optimize away a bogus readpage() for
2042          * the next read(). Here we 'discover' whether the page went
2043          * uptodate as a result of this (potentially partial) write.
2044          */
2045         if (!partial)
2046                 SetPageUptodate(page);
2047         return 0;
2048 }
2049
2050 /*
2051  * block_write_begin takes care of the basic task of block allocation and
2052  * bringing partial write blocks uptodate first.
2053  *
2054  * The filesystem needs to handle block truncation upon failure.
2055  */
2056 int block_write_begin(struct address_space *mapping, loff_t pos, unsigned len,
2057                 unsigned flags, struct page **pagep, get_block_t *get_block)
2058 {
2059         pgoff_t index = pos >> PAGE_SHIFT;
2060         struct page *page;
2061         int status;
2062
2063         page = grab_cache_page_write_begin(mapping, index, flags);
2064         if (!page)
2065                 return -ENOMEM;
2066
2067         status = __block_write_begin(page, pos, len, get_block);
2068         if (unlikely(status)) {
2069                 unlock_page(page);
2070                 put_page(page);
2071                 page = NULL;
2072         }
2073
2074         *pagep = page;
2075         return status;
2076 }
2077 EXPORT_SYMBOL(block_write_begin);
2078
2079 int __generic_write_end(struct inode *inode, loff_t pos, unsigned copied,
2080                 struct page *page)
2081 {
2082         loff_t old_size = inode->i_size;
2083         bool i_size_changed = false;
2084
2085         /*
2086          * No need to use i_size_read() here, the i_size cannot change under us
2087          * because we hold i_rwsem.
2088          *
2089          * But it's important to update i_size while still holding page lock:
2090          * page writeout could otherwise come in and zero beyond i_size.
2091          */
2092         if (pos + copied > inode->i_size) {
2093                 i_size_write(inode, pos + copied);
2094                 i_size_changed = true;
2095         }
2096
2097         unlock_page(page);
2098         put_page(page);
2099
2100         if (old_size < pos)
2101                 pagecache_isize_extended(inode, old_size, pos);
2102         /*
2103          * Don't mark the inode dirty under page lock. First, it unnecessarily
2104          * makes the holding time of page lock longer. Second, it forces lock
2105          * ordering of page lock and transaction start for journaling
2106          * filesystems.
2107          */
2108         if (i_size_changed)
2109                 mark_inode_dirty(inode);
2110         return copied;
2111 }
2112
2113 int block_write_end(struct file *file, struct address_space *mapping,
2114                         loff_t pos, unsigned len, unsigned copied,
2115                         struct page *page, void *fsdata)
2116 {
2117         struct inode *inode = mapping->host;
2118         unsigned start;
2119
2120         start = pos & (PAGE_SIZE - 1);
2121
2122         if (unlikely(copied < len)) {
2123                 /*
2124                  * The buffers that were written will now be uptodate, so we
2125                  * don't have to worry about a readpage reading them and
2126                  * overwriting a partial write. However if we have encountered
2127                  * a short write and only partially written into a buffer, it
2128                  * will not be marked uptodate, so a readpage might come in and
2129                  * destroy our partial write.
2130                  *
2131                  * Do the simplest thing, and just treat any short write to a
2132                  * non uptodate page as a zero-length write, and force the
2133                  * caller to redo the whole thing.
2134                  */
2135                 if (!PageUptodate(page))
2136                         copied = 0;
2137
2138                 page_zero_new_buffers(page, start+copied, start+len);
2139         }
2140         flush_dcache_page(page);
2141
2142         /* This could be a short (even 0-length) commit */
2143         __block_commit_write(inode, page, start, start+copied);
2144
2145         return copied;
2146 }
2147 EXPORT_SYMBOL(block_write_end);
2148
2149 int generic_write_end(struct file *file, struct address_space *mapping,
2150                         loff_t pos, unsigned len, unsigned copied,
2151                         struct page *page, void *fsdata)
2152 {
2153         copied = block_write_end(file, mapping, pos, len, copied, page, fsdata);
2154         return __generic_write_end(mapping->host, pos, copied, page);
2155 }
2156 EXPORT_SYMBOL(generic_write_end);
2157
2158 /*
2159  * block_is_partially_uptodate checks whether buffers within a page are
2160  * uptodate or not.
2161  *
2162  * Returns true if all buffers which correspond to a file portion
2163  * we want to read are uptodate.
2164  */
2165 int block_is_partially_uptodate(struct page *page, unsigned long from,
2166                                         unsigned long count)
2167 {
2168         unsigned block_start, block_end, blocksize;
2169         unsigned to;
2170         struct buffer_head *bh, *head;
2171         int ret = 1;
2172
2173         if (!page_has_buffers(page))
2174                 return 0;
2175
2176         head = page_buffers(page);
2177         blocksize = head->b_size;
2178         to = min_t(unsigned, PAGE_SIZE - from, count);
2179         to = from + to;
2180         if (from < blocksize && to > PAGE_SIZE - blocksize)
2181                 return 0;
2182
2183         bh = head;
2184         block_start = 0;
2185         do {
2186                 block_end = block_start + blocksize;
2187                 if (block_end > from && block_start < to) {
2188                         if (!buffer_uptodate(bh)) {
2189                                 ret = 0;
2190                                 break;
2191                         }
2192                         if (block_end >= to)
2193                                 break;
2194                 }
2195                 block_start = block_end;
2196                 bh = bh->b_this_page;
2197         } while (bh != head);
2198
2199         return ret;
2200 }
2201 EXPORT_SYMBOL(block_is_partially_uptodate);
2202
2203 /*
2204  * Generic "read page" function for block devices that have the normal
2205  * get_block functionality. This is most of the block device filesystems.
2206  * Reads the page asynchronously --- the unlock_buffer() and
2207  * set/clear_buffer_uptodate() functions propagate buffer state into the
2208  * page struct once IO has completed.
2209  */
2210 int block_read_full_page(struct page *page, get_block_t *get_block)
2211 {
2212         struct inode *inode = page->mapping->host;
2213         sector_t iblock, lblock;
2214         struct buffer_head *bh, *head, *arr[MAX_BUF_PER_PAGE];
2215         unsigned int blocksize, bbits;
2216         int nr, i;
2217         int fully_mapped = 1;
2218
2219         head = create_page_buffers(page, inode, 0);
2220         blocksize = head->b_size;
2221         bbits = block_size_bits(blocksize);
2222
2223         iblock = (sector_t)page->index << (PAGE_SHIFT - bbits);
2224         lblock = (i_size_read(inode)+blocksize-1) >> bbits;
2225         bh = head;
2226         nr = 0;
2227         i = 0;
2228
2229         do {
2230                 if (buffer_uptodate(bh))
2231                         continue;
2232
2233                 if (!buffer_mapped(bh)) {
2234                         int err = 0;
2235
2236                         fully_mapped = 0;
2237                         if (iblock < lblock) {
2238                                 WARN_ON(bh->b_size != blocksize);
2239                                 err = get_block(inode, iblock, bh, 0);
2240                                 if (err)
2241                                         SetPageError(page);
2242                         }
2243                         if (!buffer_mapped(bh)) {
2244                                 zero_user(page, i * blocksize, blocksize);
2245                                 if (!err)
2246                                         set_buffer_uptodate(bh);
2247                                 continue;
2248                         }
2249                         /*
2250                          * get_block() might have updated the buffer
2251                          * synchronously
2252                          */
2253                         if (buffer_uptodate(bh))
2254                                 continue;
2255                 }
2256                 arr[nr++] = bh;
2257         } while (i++, iblock++, (bh = bh->b_this_page) != head);
2258
2259         if (fully_mapped)
2260                 SetPageMappedToDisk(page);
2261
2262         if (!nr) {
2263                 /*
2264                  * All buffers are uptodate - we can set the page uptodate
2265                  * as well. But not if get_block() returned an error.
2266                  */
2267                 if (!PageError(page))
2268                         SetPageUptodate(page);
2269                 unlock_page(page);
2270                 return 0;
2271         }
2272
2273         /* Stage two: lock the buffers */
2274         for (i = 0; i < nr; i++) {
2275                 bh = arr[i];
2276                 lock_buffer(bh);
2277                 mark_buffer_async_read(bh);
2278         }
2279
2280         /*
2281          * Stage 3: start the IO.  Check for uptodateness
2282          * inside the buffer lock in case another process reading
2283          * the underlying blockdev brought it uptodate (the sct fix).
2284          */
2285         for (i = 0; i < nr; i++) {
2286                 bh = arr[i];
2287                 if (buffer_uptodate(bh))
2288                         end_buffer_async_read(bh, 1);
2289                 else
2290                         submit_bh(REQ_OP_READ, 0, bh);
2291         }
2292         return 0;
2293 }
2294 EXPORT_SYMBOL(block_read_full_page);
2295
2296 /* utility function for filesystems that need to do work on expanding
2297  * truncates.  Uses filesystem pagecache writes to allow the filesystem to
2298  * deal with the hole.  
2299  */
2300 int generic_cont_expand_simple(struct inode *inode, loff_t size)
2301 {
2302         struct address_space *mapping = inode->i_mapping;
2303         struct page *page;
2304         void *fsdata;
2305         int err;
2306
2307         err = inode_newsize_ok(inode, size);
2308         if (err)
2309                 goto out;
2310
2311         err = pagecache_write_begin(NULL, mapping, size, 0,
2312                                     AOP_FLAG_CONT_EXPAND, &page, &fsdata);
2313         if (err)
2314                 goto out;
2315
2316         err = pagecache_write_end(NULL, mapping, size, 0, 0, page, fsdata);
2317         BUG_ON(err > 0);
2318
2319 out:
2320         return err;
2321 }
2322 EXPORT_SYMBOL(generic_cont_expand_simple);
2323
2324 static int cont_expand_zero(struct file *file, struct address_space *mapping,
2325                             loff_t pos, loff_t *bytes)
2326 {
2327         struct inode *inode = mapping->host;
2328         unsigned int blocksize = i_blocksize(inode);
2329         struct page *page;
2330         void *fsdata;
2331         pgoff_t index, curidx;
2332         loff_t curpos;
2333         unsigned zerofrom, offset, len;
2334         int err = 0;
2335
2336         index = pos >> PAGE_SHIFT;
2337         offset = pos & ~PAGE_MASK;
2338
2339         while (index > (curidx = (curpos = *bytes)>>PAGE_SHIFT)) {
2340                 zerofrom = curpos & ~PAGE_MASK;
2341                 if (zerofrom & (blocksize-1)) {
2342                         *bytes |= (blocksize-1);
2343                         (*bytes)++;
2344                 }
2345                 len = PAGE_SIZE - zerofrom;
2346
2347                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2348                                             &page, &fsdata);
2349                 if (err)
2350                         goto out;
2351                 zero_user(page, zerofrom, len);
2352                 err = pagecache_write_end(file, mapping, curpos, len, len,
2353                                                 page, fsdata);
2354                 if (err < 0)
2355                         goto out;
2356                 BUG_ON(err != len);
2357                 err = 0;
2358
2359                 balance_dirty_pages_ratelimited(mapping);
2360
2361                 if (unlikely(fatal_signal_pending(current))) {
2362                         err = -EINTR;
2363                         goto out;
2364                 }
2365         }
2366
2367         /* page covers the boundary, find the boundary offset */
2368         if (index == curidx) {
2369                 zerofrom = curpos & ~PAGE_MASK;
2370                 /* if we will expand the thing last block will be filled */
2371                 if (offset <= zerofrom) {
2372                         goto out;
2373                 }
2374                 if (zerofrom & (blocksize-1)) {
2375                         *bytes |= (blocksize-1);
2376                         (*bytes)++;
2377                 }
2378                 len = offset - zerofrom;
2379
2380                 err = pagecache_write_begin(file, mapping, curpos, len, 0,
2381                                             &page, &fsdata);
2382                 if (err)
2383                         goto out;
2384                 zero_user(page, zerofrom, len);
2385                 err = pagecache_write_end(file, mapping, curpos, len, len,
2386                                                 page, fsdata);
2387                 if (err < 0)
2388                         goto out;
2389                 BUG_ON(err != len);
2390                 err = 0;
2391         }
2392 out:
2393         return err;
2394 }
2395
2396 /*
2397  * For moronic filesystems that do not allow holes in file.
2398  * We may have to extend the file.
2399  */
2400 int cont_write_begin(struct file *file, struct address_space *mapping,
2401                         loff_t pos, unsigned len, unsigned flags,
2402                         struct page **pagep, void **fsdata,
2403                         get_block_t *get_block, loff_t *bytes)
2404 {
2405         struct inode *inode = mapping->host;
2406         unsigned int blocksize = i_blocksize(inode);
2407         unsigned int zerofrom;
2408         int err;
2409
2410         err = cont_expand_zero(file, mapping, pos, bytes);
2411         if (err)
2412                 return err;
2413
2414         zerofrom = *bytes & ~PAGE_MASK;
2415         if (pos+len > *bytes && zerofrom & (blocksize-1)) {
2416                 *bytes |= (blocksize-1);
2417                 (*bytes)++;
2418         }
2419
2420         return block_write_begin(mapping, pos, len, flags, pagep, get_block);
2421 }
2422 EXPORT_SYMBOL(cont_write_begin);
2423
2424 int block_commit_write(struct page *page, unsigned from, unsigned to)
2425 {
2426         struct inode *inode = page->mapping->host;
2427         __block_commit_write(inode,page,from,to);
2428         return 0;
2429 }
2430 EXPORT_SYMBOL(block_commit_write);
2431
2432 /*
2433  * block_page_mkwrite() is not allowed to change the file size as it gets
2434  * called from a page fault handler when a page is first dirtied. Hence we must
2435  * be careful to check for EOF conditions here. We set the page up correctly
2436  * for a written page which means we get ENOSPC checking when writing into
2437  * holes and correct delalloc and unwritten extent mapping on filesystems that
2438  * support these features.
2439  *
2440  * We are not allowed to take the i_mutex here so we have to play games to
2441  * protect against truncate races as the page could now be beyond EOF.  Because
2442  * truncate writes the inode size before removing pages, once we have the
2443  * page lock we can determine safely if the page is beyond EOF. If it is not
2444  * beyond EOF, then the page is guaranteed safe against truncation until we
2445  * unlock the page.
2446  *
2447  * Direct callers of this function should protect against filesystem freezing
2448  * using sb_start_pagefault() - sb_end_pagefault() functions.
2449  */
2450 int block_page_mkwrite(struct vm_area_struct *vma, struct vm_fault *vmf,
2451                          get_block_t get_block)
2452 {
2453         struct page *page = vmf->page;
2454         struct inode *inode = file_inode(vma->vm_file);
2455         unsigned long end;
2456         loff_t size;
2457         int ret;
2458
2459         lock_page(page);
2460         size = i_size_read(inode);
2461         if ((page->mapping != inode->i_mapping) ||
2462             (page_offset(page) > size)) {
2463                 /* We overload EFAULT to mean page got truncated */
2464                 ret = -EFAULT;
2465                 goto out_unlock;
2466         }
2467
2468         /* page is wholly or partially inside EOF */
2469         if (((page->index + 1) << PAGE_SHIFT) > size)
2470                 end = size & ~PAGE_MASK;
2471         else
2472                 end = PAGE_SIZE;
2473
2474         ret = __block_write_begin(page, 0, end, get_block);
2475         if (!ret)
2476                 ret = block_commit_write(page, 0, end);
2477
2478         if (unlikely(ret < 0))
2479                 goto out_unlock;
2480         set_page_dirty(page);
2481         wait_for_stable_page(page);
2482         return 0;
2483 out_unlock:
2484         unlock_page(page);
2485         return ret;
2486 }
2487 EXPORT_SYMBOL(block_page_mkwrite);
2488
2489 /*
2490  * nobh_write_begin()'s prereads are special: the buffer_heads are freed
2491  * immediately, while under the page lock.  So it needs a special end_io
2492  * handler which does not touch the bh after unlocking it.
2493  */
2494 static void end_buffer_read_nobh(struct buffer_head *bh, int uptodate)
2495 {
2496         __end_buffer_read_notouch(bh, uptodate);
2497 }
2498
2499 /*
2500  * Attach the singly-linked list of buffers created by nobh_write_begin, to
2501  * the page (converting it to circular linked list and taking care of page
2502  * dirty races).
2503  */
2504 static void attach_nobh_buffers(struct page *page, struct buffer_head *head)
2505 {
2506         struct buffer_head *bh;
2507
2508         BUG_ON(!PageLocked(page));
2509
2510         spin_lock(&page->mapping->private_lock);
2511         bh = head;
2512         do {
2513                 if (PageDirty(page))
2514                         set_buffer_dirty(bh);
2515                 if (!bh->b_this_page)
2516                         bh->b_this_page = head;
2517                 bh = bh->b_this_page;
2518         } while (bh != head);
2519         attach_page_buffers(page, head);
2520         spin_unlock(&page->mapping->private_lock);
2521 }
2522
2523 /*
2524  * On entry, the page is fully not uptodate.
2525  * On exit the page is fully uptodate in the areas outside (from,to)
2526  * The filesystem needs to handle block truncation upon failure.
2527  */
2528 int nobh_write_begin(struct address_space *mapping,
2529                         loff_t pos, unsigned len, unsigned flags,
2530                         struct page **pagep, void **fsdata,
2531                         get_block_t *get_block)
2532 {
2533         struct inode *inode = mapping->host;
2534         const unsigned blkbits = inode->i_blkbits;
2535         const unsigned blocksize = 1 << blkbits;
2536         struct buffer_head *head, *bh;
2537         struct page *page;
2538         pgoff_t index;
2539         unsigned from, to;
2540         unsigned block_in_page;
2541         unsigned block_start, block_end;
2542         sector_t block_in_file;
2543         int nr_reads = 0;
2544         int ret = 0;
2545         int is_mapped_to_disk = 1;
2546
2547         index = pos >> PAGE_SHIFT;
2548         from = pos & (PAGE_SIZE - 1);
2549         to = from + len;
2550
2551         page = grab_cache_page_write_begin(mapping, index, flags);
2552         if (!page)
2553                 return -ENOMEM;
2554         *pagep = page;
2555         *fsdata = NULL;
2556
2557         if (page_has_buffers(page)) {
2558                 ret = __block_write_begin(page, pos, len, get_block);
2559                 if (unlikely(ret))
2560                         goto out_release;
2561                 return ret;
2562         }
2563
2564         if (PageMappedToDisk(page))
2565                 return 0;
2566
2567         /*
2568          * Allocate buffers so that we can keep track of state, and potentially
2569          * attach them to the page if an error occurs. In the common case of
2570          * no error, they will just be freed again without ever being attached
2571          * to the page (which is all OK, because we're under the page lock).
2572          *
2573          * Be careful: the buffer linked list is a NULL terminated one, rather
2574          * than the circular one we're used to.
2575          */
2576         head = alloc_page_buffers(page, blocksize, false);
2577         if (!head) {
2578                 ret = -ENOMEM;
2579                 goto out_release;
2580         }
2581
2582         block_in_file = (sector_t)page->index << (PAGE_SHIFT - blkbits);
2583
2584         /*
2585          * We loop across all blocks in the page, whether or not they are
2586          * part of the affected region.  This is so we can discover if the
2587          * page is fully mapped-to-disk.
2588          */
2589         for (block_start = 0, block_in_page = 0, bh = head;
2590                   block_start < PAGE_SIZE;
2591                   block_in_page++, block_start += blocksize, bh = bh->b_this_page) {
2592                 int create;
2593
2594                 block_end = block_start + blocksize;
2595                 bh->b_state = 0;
2596                 create = 1;
2597                 if (block_start >= to)
2598                         create = 0;
2599                 ret = get_block(inode, block_in_file + block_in_page,
2600                                         bh, create);
2601                 if (ret)
2602                         goto failed;
2603                 if (!buffer_mapped(bh))
2604                         is_mapped_to_disk = 0;
2605                 if (buffer_new(bh))
2606                         clean_bdev_bh_alias(bh);
2607                 if (PageUptodate(page)) {
2608                         set_buffer_uptodate(bh);
2609                         continue;
2610                 }
2611                 if (buffer_new(bh) || !buffer_mapped(bh)) {
2612                         zero_user_segments(page, block_start, from,
2613                                                         to, block_end);
2614                         continue;
2615                 }
2616                 if (buffer_uptodate(bh))
2617                         continue;       /* reiserfs does this */
2618                 if (block_start < from || block_end > to) {
2619                         lock_buffer(bh);
2620                         bh->b_end_io = end_buffer_read_nobh;
2621                         submit_bh(REQ_OP_READ, 0, bh);
2622                         nr_reads++;
2623                 }
2624         }
2625
2626         if (nr_reads) {
2627                 /*
2628                  * The page is locked, so these buffers are protected from
2629                  * any VM or truncate activity.  Hence we don't need to care
2630                  * for the buffer_head refcounts.
2631                  */
2632                 for (bh = head; bh; bh = bh->b_this_page) {
2633                         wait_on_buffer(bh);
2634                         if (!buffer_uptodate(bh))
2635                                 ret = -EIO;
2636                 }
2637                 if (ret)
2638                         goto failed;
2639         }
2640
2641         if (is_mapped_to_disk)
2642                 SetPageMappedToDisk(page);
2643
2644         *fsdata = head; /* to be released by nobh_write_end */
2645
2646         return 0;
2647
2648 failed:
2649         BUG_ON(!ret);
2650         /*
2651          * Error recovery is a bit difficult. We need to zero out blocks that
2652          * were newly allocated, and dirty them to ensure they get written out.
2653          * Buffers need to be attached to the page at this point, otherwise
2654          * the handling of potential IO errors during writeout would be hard
2655          * (could try doing synchronous writeout, but what if that fails too?)
2656          */
2657         attach_nobh_buffers(page, head);
2658         page_zero_new_buffers(page, from, to);
2659
2660 out_release:
2661         unlock_page(page);
2662         put_page(page);
2663         *pagep = NULL;
2664
2665         return ret;
2666 }
2667 EXPORT_SYMBOL(nobh_write_begin);
2668
2669 int nobh_write_end(struct file *file, struct address_space *mapping,
2670                         loff_t pos, unsigned len, unsigned copied,
2671                         struct page *page, void *fsdata)
2672 {
2673         struct inode *inode = page->mapping->host;
2674         struct buffer_head *head = fsdata;
2675         struct buffer_head *bh;
2676         BUG_ON(fsdata != NULL && page_has_buffers(page));
2677
2678         if (unlikely(copied < len) && head)
2679                 attach_nobh_buffers(page, head);
2680         if (page_has_buffers(page))
2681                 return generic_write_end(file, mapping, pos, len,
2682                                         copied, page, fsdata);
2683
2684         SetPageUptodate(page);
2685         set_page_dirty(page);
2686         if (pos+copied > inode->i_size) {
2687                 i_size_write(inode, pos+copied);
2688                 mark_inode_dirty(inode);
2689         }
2690
2691         unlock_page(page);
2692         put_page(page);
2693
2694         while (head) {
2695                 bh = head;
2696                 head = head->b_this_page;
2697                 free_buffer_head(bh);
2698         }
2699
2700         return copied;
2701 }
2702 EXPORT_SYMBOL(nobh_write_end);
2703
2704 /*
2705  * nobh_writepage() - based on block_full_write_page() except
2706  * that it tries to operate without attaching bufferheads to
2707  * the page.
2708  */
2709 int nobh_writepage(struct page *page, get_block_t *get_block,
2710                         struct writeback_control *wbc)
2711 {
2712         struct inode * const inode = page->mapping->host;
2713         loff_t i_size = i_size_read(inode);
2714         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2715         unsigned offset;
2716         int ret;
2717
2718         /* Is the page fully inside i_size? */
2719         if (page->index < end_index)
2720                 goto out;
2721
2722         /* Is the page fully outside i_size? (truncate in progress) */
2723         offset = i_size & (PAGE_SIZE-1);
2724         if (page->index >= end_index+1 || !offset) {
2725                 /*
2726                  * The page may have dirty, unmapped buffers.  For example,
2727                  * they may have been added in ext3_writepage().  Make them
2728                  * freeable here, so the page does not leak.
2729                  */
2730 #if 0
2731                 /* Not really sure about this  - do we need this ? */
2732                 if (page->mapping->a_ops->invalidatepage)
2733                         page->mapping->a_ops->invalidatepage(page, offset);
2734 #endif
2735                 unlock_page(page);
2736                 return 0; /* don't care */
2737         }
2738
2739         /*
2740          * The page straddles i_size.  It must be zeroed out on each and every
2741          * writepage invocation because it may be mmapped.  "A file is mapped
2742          * in multiples of the page size.  For a file that is not a multiple of
2743          * the  page size, the remaining memory is zeroed when mapped, and
2744          * writes to that region are not written out to the file."
2745          */
2746         zero_user_segment(page, offset, PAGE_SIZE);
2747 out:
2748         ret = mpage_writepage(page, get_block, wbc);
2749         if (ret == -EAGAIN)
2750                 ret = __block_write_full_page(inode, page, get_block, wbc,
2751                                               end_buffer_async_write);
2752         return ret;
2753 }
2754 EXPORT_SYMBOL(nobh_writepage);
2755
2756 int nobh_truncate_page(struct address_space *mapping,
2757                         loff_t from, get_block_t *get_block)
2758 {
2759         pgoff_t index = from >> PAGE_SHIFT;
2760         unsigned offset = from & (PAGE_SIZE-1);
2761         unsigned blocksize;
2762         sector_t iblock;
2763         unsigned length, pos;
2764         struct inode *inode = mapping->host;
2765         struct page *page;
2766         struct buffer_head map_bh;
2767         int err;
2768
2769         blocksize = i_blocksize(inode);
2770         length = offset & (blocksize - 1);
2771
2772         /* Block boundary? Nothing to do */
2773         if (!length)
2774                 return 0;
2775
2776         length = blocksize - length;
2777         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2778
2779         page = grab_cache_page(mapping, index);
2780         err = -ENOMEM;
2781         if (!page)
2782                 goto out;
2783
2784         if (page_has_buffers(page)) {
2785 has_buffers:
2786                 unlock_page(page);
2787                 put_page(page);
2788                 return block_truncate_page(mapping, from, get_block);
2789         }
2790
2791         /* Find the buffer that contains "offset" */
2792         pos = blocksize;
2793         while (offset >= pos) {
2794                 iblock++;
2795                 pos += blocksize;
2796         }
2797
2798         map_bh.b_size = blocksize;
2799         map_bh.b_state = 0;
2800         err = get_block(inode, iblock, &map_bh, 0);
2801         if (err)
2802                 goto unlock;
2803         /* unmapped? It's a hole - nothing to do */
2804         if (!buffer_mapped(&map_bh))
2805                 goto unlock;
2806
2807         /* Ok, it's mapped. Make sure it's up-to-date */
2808         if (!PageUptodate(page)) {
2809                 err = mapping->a_ops->readpage(NULL, page);
2810                 if (err) {
2811                         put_page(page);
2812                         goto out;
2813                 }
2814                 lock_page(page);
2815                 if (!PageUptodate(page)) {
2816                         err = -EIO;
2817                         goto unlock;
2818                 }
2819                 if (page_has_buffers(page))
2820                         goto has_buffers;
2821         }
2822         zero_user(page, offset, length);
2823         set_page_dirty(page);
2824         err = 0;
2825
2826 unlock:
2827         unlock_page(page);
2828         put_page(page);
2829 out:
2830         return err;
2831 }
2832 EXPORT_SYMBOL(nobh_truncate_page);
2833
2834 int block_truncate_page(struct address_space *mapping,
2835                         loff_t from, get_block_t *get_block)
2836 {
2837         pgoff_t index = from >> PAGE_SHIFT;
2838         unsigned offset = from & (PAGE_SIZE-1);
2839         unsigned blocksize;
2840         sector_t iblock;
2841         unsigned length, pos;
2842         struct inode *inode = mapping->host;
2843         struct page *page;
2844         struct buffer_head *bh;
2845         int err;
2846
2847         blocksize = i_blocksize(inode);
2848         length = offset & (blocksize - 1);
2849
2850         /* Block boundary? Nothing to do */
2851         if (!length)
2852                 return 0;
2853
2854         length = blocksize - length;
2855         iblock = (sector_t)index << (PAGE_SHIFT - inode->i_blkbits);
2856         
2857         page = grab_cache_page(mapping, index);
2858         err = -ENOMEM;
2859         if (!page)
2860                 goto out;
2861
2862         if (!page_has_buffers(page))
2863                 create_empty_buffers(page, blocksize, 0);
2864
2865         /* Find the buffer that contains "offset" */
2866         bh = page_buffers(page);
2867         pos = blocksize;
2868         while (offset >= pos) {
2869                 bh = bh->b_this_page;
2870                 iblock++;
2871                 pos += blocksize;
2872         }
2873
2874         err = 0;
2875         if (!buffer_mapped(bh)) {
2876                 WARN_ON(bh->b_size != blocksize);
2877                 err = get_block(inode, iblock, bh, 0);
2878                 if (err)
2879                         goto unlock;
2880                 /* unmapped? It's a hole - nothing to do */
2881                 if (!buffer_mapped(bh))
2882                         goto unlock;
2883         }
2884
2885         /* Ok, it's mapped. Make sure it's up-to-date */
2886         if (PageUptodate(page))
2887                 set_buffer_uptodate(bh);
2888
2889         if (!buffer_uptodate(bh) && !buffer_delay(bh) && !buffer_unwritten(bh)) {
2890                 err = -EIO;
2891                 ll_rw_block(REQ_OP_READ, 0, 1, &bh);
2892                 wait_on_buffer(bh);
2893                 /* Uhhuh. Read error. Complain and punt. */
2894                 if (!buffer_uptodate(bh))
2895                         goto unlock;
2896         }
2897
2898         zero_user(page, offset, length);
2899         mark_buffer_dirty(bh);
2900         err = 0;
2901
2902 unlock:
2903         unlock_page(page);
2904         put_page(page);
2905 out:
2906         return err;
2907 }
2908 EXPORT_SYMBOL(block_truncate_page);
2909
2910 /*
2911  * The generic ->writepage function for buffer-backed address_spaces
2912  */
2913 int block_write_full_page(struct page *page, get_block_t *get_block,
2914                         struct writeback_control *wbc)
2915 {
2916         struct inode * const inode = page->mapping->host;
2917         loff_t i_size = i_size_read(inode);
2918         const pgoff_t end_index = i_size >> PAGE_SHIFT;
2919         unsigned offset;
2920
2921         /* Is the page fully inside i_size? */
2922         if (page->index < end_index)
2923                 return __block_write_full_page(inode, page, get_block, wbc,
2924                                                end_buffer_async_write);
2925
2926         /* Is the page fully outside i_size? (truncate in progress) */
2927         offset = i_size & (PAGE_SIZE-1);
2928         if (page->index >= end_index+1 || !offset) {
2929                 /*
2930                  * The page may have dirty, unmapped buffers.  For example,
2931                  * they may have been added in ext3_writepage().  Make them
2932                  * freeable here, so the page does not leak.
2933                  */
2934                 do_invalidatepage(page, 0, PAGE_SIZE);
2935                 unlock_page(page);
2936                 return 0; /* don't care */
2937         }
2938
2939         /*
2940          * The page straddles i_size.  It must be zeroed out on each and every
2941          * writepage invocation because it may be mmapped.  "A file is mapped
2942          * in multiples of the page size.  For a file that is not a multiple of
2943          * the  page size, the remaining memory is zeroed when mapped, and
2944          * writes to that region are not written out to the file."
2945          */
2946         zero_user_segment(page, offset, PAGE_SIZE);
2947         return __block_write_full_page(inode, page, get_block, wbc,
2948                                                         end_buffer_async_write);
2949 }
2950 EXPORT_SYMBOL(block_write_full_page);
2951
2952 sector_t generic_block_bmap(struct address_space *mapping, sector_t block,
2953                             get_block_t *get_block)
2954 {
2955         struct inode *inode = mapping->host;
2956         struct buffer_head tmp = {
2957                 .b_size = i_blocksize(inode),
2958         };
2959
2960         get_block(inode, block, &tmp, 0);
2961         return tmp.b_blocknr;
2962 }
2963 EXPORT_SYMBOL(generic_block_bmap);
2964
2965 static void end_bio_bh_io_sync(struct bio *bio)
2966 {
2967         struct buffer_head *bh = bio->bi_private;
2968
2969         if (unlikely(bio_flagged(bio, BIO_QUIET)))
2970                 set_bit(BH_Quiet, &bh->b_state);
2971
2972         bh->b_end_io(bh, !bio->bi_status);
2973         bio_put(bio);
2974 }
2975
2976 /*
2977  * This allows us to do IO even on the odd last sectors
2978  * of a device, even if the block size is some multiple
2979  * of the physical sector size.
2980  *
2981  * We'll just truncate the bio to the size of the device,
2982  * and clear the end of the buffer head manually.
2983  *
2984  * Truly out-of-range accesses will turn into actual IO
2985  * errors, this only handles the "we need to be able to
2986  * do IO at the final sector" case.
2987  */
2988 void guard_bio_eod(int op, struct bio *bio)
2989 {
2990         sector_t maxsector;
2991         struct bio_vec *bvec = bio_last_bvec_all(bio);
2992         unsigned truncated_bytes;
2993         struct hd_struct *part;
2994
2995         rcu_read_lock();
2996         part = __disk_get_part(bio->bi_disk, bio->bi_partno);
2997         if (part)
2998                 maxsector = part_nr_sects_read(part);
2999         else
3000                 maxsector = get_capacity(bio->bi_disk);
3001         rcu_read_unlock();
3002
3003         if (!maxsector)
3004                 return;
3005
3006         /*
3007          * If the *whole* IO is past the end of the device,
3008          * let it through, and the IO layer will turn it into
3009          * an EIO.
3010          */
3011         if (unlikely(bio->bi_iter.bi_sector >= maxsector))
3012                 return;
3013
3014         maxsector -= bio->bi_iter.bi_sector;
3015         if (likely((bio->bi_iter.bi_size >> 9) <= maxsector))
3016                 return;
3017
3018         /* Uhhuh. We've got a bio that straddles the device size! */
3019         truncated_bytes = bio->bi_iter.bi_size - (maxsector << 9);
3020
3021         /* Truncate the bio.. */
3022         bio->bi_iter.bi_size -= truncated_bytes;
3023         bvec->bv_len -= truncated_bytes;
3024
3025         /* ..and clear the end of the buffer for reads */
3026         if (op == REQ_OP_READ) {
3027                 zero_user(bvec->bv_page, bvec->bv_offset + bvec->bv_len,
3028                                 truncated_bytes);
3029         }
3030 }
3031
3032 static int submit_bh_wbc(int op, int op_flags, struct buffer_head *bh,
3033                          enum rw_hint write_hint, struct writeback_control *wbc)
3034 {
3035         struct bio *bio;
3036
3037         BUG_ON(!buffer_locked(bh));
3038         BUG_ON(!buffer_mapped(bh));
3039         BUG_ON(!bh->b_end_io);
3040         BUG_ON(buffer_delay(bh));
3041         BUG_ON(buffer_unwritten(bh));
3042
3043         /*
3044          * Only clear out a write error when rewriting
3045          */
3046         if (test_set_buffer_req(bh) && (op == REQ_OP_WRITE))
3047                 clear_buffer_write_io_error(bh);
3048
3049         /*
3050          * from here on down, it's all bio -- do the initial mapping,
3051          * submit_bio -> generic_make_request may further map this bio around
3052          */
3053         bio = bio_alloc(GFP_NOIO, 1);
3054
3055         if (wbc) {
3056                 wbc_init_bio(wbc, bio);
3057                 wbc_account_io(wbc, bh->b_page, bh->b_size);
3058         }
3059
3060         bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
3061         bio_set_dev(bio, bh->b_bdev);
3062         bio->bi_write_hint = write_hint;
3063
3064         bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh));
3065         BUG_ON(bio->bi_iter.bi_size != bh->b_size);
3066
3067         bio->bi_end_io = end_bio_bh_io_sync;
3068         bio->bi_private = bh;
3069
3070         /* Take care of bh's that straddle the end of the device */
3071         guard_bio_eod(op, bio);
3072
3073         if (buffer_meta(bh))
3074                 op_flags |= REQ_META;
3075         if (buffer_prio(bh))
3076                 op_flags |= REQ_PRIO;
3077         bio_set_op_attrs(bio, op, op_flags);
3078
3079         submit_bio(bio);
3080         return 0;
3081 }
3082
3083 int submit_bh(int op, int op_flags, struct buffer_head *bh)
3084 {
3085         return submit_bh_wbc(op, op_flags, bh, 0, NULL);
3086 }
3087 EXPORT_SYMBOL(submit_bh);
3088
3089 /**
3090  * ll_rw_block: low-level access to block devices (DEPRECATED)
3091  * @op: whether to %READ or %WRITE
3092  * @op_flags: req_flag_bits
3093  * @nr: number of &struct buffer_heads in the array
3094  * @bhs: array of pointers to &struct buffer_head
3095  *
3096  * ll_rw_block() takes an array of pointers to &struct buffer_heads, and
3097  * requests an I/O operation on them, either a %REQ_OP_READ or a %REQ_OP_WRITE.
3098  * @op_flags contains flags modifying the detailed I/O behavior, most notably
3099  * %REQ_RAHEAD.
3100  *
3101  * This function drops any buffer that it cannot get a lock on (with the
3102  * BH_Lock state bit), any buffer that appears to be clean when doing a write
3103  * request, and any buffer that appears to be up-to-date when doing read
3104  * request.  Further it marks as clean buffers that are processed for
3105  * writing (the buffer cache won't assume that they are actually clean
3106  * until the buffer gets unlocked).
3107  *
3108  * ll_rw_block sets b_end_io to simple completion handler that marks
3109  * the buffer up-to-date (if appropriate), unlocks the buffer and wakes
3110  * any waiters. 
3111  *
3112  * All of the buffers must be for the same device, and must also be a
3113  * multiple of the current approved size for the device.
3114  */
3115 void ll_rw_block(int op, int op_flags,  int nr, struct buffer_head *bhs[])
3116 {
3117         int i;
3118
3119         for (i = 0; i < nr; i++) {
3120                 struct buffer_head *bh = bhs[i];
3121
3122                 if (!trylock_buffer(bh))
3123                         continue;
3124                 if (op == WRITE) {
3125                         if (test_clear_buffer_dirty(bh)) {
3126                                 bh->b_end_io = end_buffer_write_sync;
3127                                 get_bh(bh);
3128                                 submit_bh(op, op_flags, bh);
3129                                 continue;
3130                         }
3131                 } else {
3132                         if (!buffer_uptodate(bh)) {
3133                                 bh->b_end_io = end_buffer_read_sync;
3134                                 get_bh(bh);
3135                                 submit_bh(op, op_flags, bh);
3136                                 continue;
3137                         }
3138                 }
3139                 unlock_buffer(bh);
3140         }
3141 }
3142 EXPORT_SYMBOL(ll_rw_block);
3143
3144 void write_dirty_buffer(struct buffer_head *bh, int op_flags)
3145 {
3146         lock_buffer(bh);
3147         if (!test_clear_buffer_dirty(bh)) {
3148                 unlock_buffer(bh);
3149                 return;
3150         }
3151         bh->b_end_io = end_buffer_write_sync;
3152         get_bh(bh);
3153         submit_bh(REQ_OP_WRITE, op_flags, bh);
3154 }
3155 EXPORT_SYMBOL(write_dirty_buffer);
3156
3157 /*
3158  * For a data-integrity writeout, we need to wait upon any in-progress I/O
3159  * and then start new I/O and then wait upon it.  The caller must have a ref on
3160  * the buffer_head.
3161  */
3162 int __sync_dirty_buffer(struct buffer_head *bh, int op_flags)
3163 {
3164         int ret = 0;
3165
3166         WARN_ON(atomic_read(&bh->b_count) < 1);
3167         lock_buffer(bh);
3168         if (test_clear_buffer_dirty(bh)) {
3169                 get_bh(bh);
3170                 bh->b_end_io = end_buffer_write_sync;
3171                 ret = submit_bh(REQ_OP_WRITE, op_flags, bh);
3172                 wait_on_buffer(bh);
3173                 if (!ret && !buffer_uptodate(bh))
3174                         ret = -EIO;
3175         } else {
3176                 unlock_buffer(bh);
3177         }
3178         return ret;
3179 }
3180 EXPORT_SYMBOL(__sync_dirty_buffer);
3181
3182 int sync_dirty_buffer(struct buffer_head *bh)
3183 {
3184         return __sync_dirty_buffer(bh, REQ_SYNC);
3185 }
3186 EXPORT_SYMBOL(sync_dirty_buffer);
3187
3188 /*
3189  * try_to_free_buffers() checks if all the buffers on this particular page
3190  * are unused, and releases them if so.
3191  *
3192  * Exclusion against try_to_free_buffers may be obtained by either
3193  * locking the page or by holding its mapping's private_lock.
3194  *
3195  * If the page is dirty but all the buffers are clean then we need to
3196  * be sure to mark the page clean as well.  This is because the page
3197  * may be against a block device, and a later reattachment of buffers
3198  * to a dirty page will set *all* buffers dirty.  Which would corrupt
3199  * filesystem data on the same device.
3200  *
3201  * The same applies to regular filesystem pages: if all the buffers are
3202  * clean then we set the page clean and proceed.  To do that, we require
3203  * total exclusion from __set_page_dirty_buffers().  That is obtained with
3204  * private_lock.
3205  *
3206  * try_to_free_buffers() is non-blocking.
3207  */
3208 static inline int buffer_busy(struct buffer_head *bh)
3209 {
3210         return atomic_read(&bh->b_count) |
3211                 (bh->b_state & ((1 << BH_Dirty) | (1 << BH_Lock)));
3212 }
3213
3214 static int
3215 drop_buffers(struct page *page, struct buffer_head **buffers_to_free)
3216 {
3217         struct buffer_head *head = page_buffers(page);
3218         struct buffer_head *bh;
3219
3220         bh = head;
3221         do {
3222                 if (buffer_busy(bh))
3223                         goto failed;
3224                 bh = bh->b_this_page;
3225         } while (bh != head);
3226
3227         do {
3228                 struct buffer_head *next = bh->b_this_page;
3229
3230                 if (bh->b_assoc_map)
3231                         __remove_assoc_queue(bh);
3232                 bh = next;
3233         } while (bh != head);
3234         *buffers_to_free = head;
3235         __clear_page_buffers(page);
3236         return 1;
3237 failed:
3238         return 0;
3239 }
3240
3241 int try_to_free_buffers(struct page *page)
3242 {
3243         struct address_space * const mapping = page->mapping;
3244         struct buffer_head *buffers_to_free = NULL;
3245         int ret = 0;
3246
3247         BUG_ON(!PageLocked(page));
3248         if (PageWriteback(page))
3249                 return 0;
3250
3251         if (mapping == NULL) {          /* can this still happen? */
3252                 ret = drop_buffers(page, &buffers_to_free);
3253                 goto out;
3254         }
3255
3256         spin_lock(&mapping->private_lock);
3257         ret = drop_buffers(page, &buffers_to_free);
3258
3259         /*
3260          * If the filesystem writes its buffers by hand (eg ext3)
3261          * then we can have clean buffers against a dirty page.  We
3262          * clean the page here; otherwise the VM will never notice
3263          * that the filesystem did any IO at all.
3264          *
3265          * Also, during truncate, discard_buffer will have marked all
3266          * the page's buffers clean.  We discover that here and clean
3267          * the page also.
3268          *
3269          * private_lock must be held over this entire operation in order
3270          * to synchronise against __set_page_dirty_buffers and prevent the
3271          * dirty bit from being lost.
3272          */
3273         if (ret)
3274                 cancel_dirty_page(page);
3275         spin_unlock(&mapping->private_lock);
3276 out:
3277         if (buffers_to_free) {
3278                 struct buffer_head *bh = buffers_to_free;
3279
3280                 do {
3281                         struct buffer_head *next = bh->b_this_page;
3282                         free_buffer_head(bh);
3283                         bh = next;
3284                 } while (bh != buffers_to_free);
3285         }
3286         return ret;
3287 }
3288 EXPORT_SYMBOL(try_to_free_buffers);
3289
3290 /*
3291  * There are no bdflush tunables left.  But distributions are
3292  * still running obsolete flush daemons, so we terminate them here.
3293  *
3294  * Use of bdflush() is deprecated and will be removed in a future kernel.
3295  * The `flush-X' kernel threads fully replace bdflush daemons and this call.
3296  */
3297 SYSCALL_DEFINE2(bdflush, int, func, long, data)
3298 {
3299         static int msg_count;
3300
3301         if (!capable(CAP_SYS_ADMIN))
3302                 return -EPERM;
3303
3304         if (msg_count < 5) {
3305                 msg_count++;
3306                 printk(KERN_INFO
3307                         "warning: process `%s' used the obsolete bdflush"
3308                         " system call\n", current->comm);
3309                 printk(KERN_INFO "Fix your initscripts?\n");
3310         }
3311
3312         if (func == 1)
3313                 do_exit(0);
3314         return 0;
3315 }
3316
3317 /*
3318  * Buffer-head allocation
3319  */
3320 static struct kmem_cache *bh_cachep __read_mostly;
3321
3322 /*
3323  * Once the number of bh's in the machine exceeds this level, we start
3324  * stripping them in writeback.
3325  */
3326 static unsigned long max_buffer_heads;
3327
3328 int buffer_heads_over_limit;
3329
3330 struct bh_accounting {
3331         int nr;                 /* Number of live bh's */
3332         int ratelimit;          /* Limit cacheline bouncing */
3333 };
3334
3335 static DEFINE_PER_CPU(struct bh_accounting, bh_accounting) = {0, 0};
3336
3337 static void recalc_bh_state(void)
3338 {
3339         int i;
3340         int tot = 0;
3341
3342         if (__this_cpu_inc_return(bh_accounting.ratelimit) - 1 < 4096)
3343                 return;
3344         __this_cpu_write(bh_accounting.ratelimit, 0);
3345         for_each_online_cpu(i)
3346                 tot += per_cpu(bh_accounting, i).nr;
3347         buffer_heads_over_limit = (tot > max_buffer_heads);
3348 }
3349
3350 struct buffer_head *alloc_buffer_head(gfp_t gfp_flags)
3351 {
3352         struct buffer_head *ret = kmem_cache_zalloc(bh_cachep, gfp_flags);
3353         if (ret) {
3354                 INIT_LIST_HEAD(&ret->b_assoc_buffers);
3355                 preempt_disable();
3356                 __this_cpu_inc(bh_accounting.nr);
3357                 recalc_bh_state();
3358                 preempt_enable();
3359         }
3360         return ret;
3361 }
3362 EXPORT_SYMBOL(alloc_buffer_head);
3363
3364 void free_buffer_head(struct buffer_head *bh)
3365 {
3366         BUG_ON(!list_empty(&bh->b_assoc_buffers));
3367         kmem_cache_free(bh_cachep, bh);
3368         preempt_disable();
3369         __this_cpu_dec(bh_accounting.nr);
3370         recalc_bh_state();
3371         preempt_enable();
3372 }
3373 EXPORT_SYMBOL(free_buffer_head);
3374
3375 static int buffer_exit_cpu_dead(unsigned int cpu)
3376 {
3377         int i;
3378         struct bh_lru *b = &per_cpu(bh_lrus, cpu);
3379
3380         for (i = 0; i < BH_LRU_SIZE; i++) {
3381                 brelse(b->bhs[i]);
3382                 b->bhs[i] = NULL;
3383         }
3384         this_cpu_add(bh_accounting.nr, per_cpu(bh_accounting, cpu).nr);
3385         per_cpu(bh_accounting, cpu).nr = 0;
3386         return 0;
3387 }
3388
3389 /**
3390  * bh_uptodate_or_lock - Test whether the buffer is uptodate
3391  * @bh: struct buffer_head
3392  *
3393  * Return true if the buffer is up-to-date and false,
3394  * with the buffer locked, if not.
3395  */
3396 int bh_uptodate_or_lock(struct buffer_head *bh)
3397 {
3398         if (!buffer_uptodate(bh)) {
3399                 lock_buffer(bh);
3400                 if (!buffer_uptodate(bh))
3401                         return 0;
3402                 unlock_buffer(bh);
3403         }
3404         return 1;
3405 }
3406 EXPORT_SYMBOL(bh_uptodate_or_lock);
3407
3408 /**
3409  * bh_submit_read - Submit a locked buffer for reading
3410  * @bh: struct buffer_head
3411  *
3412  * Returns zero on success and -EIO on error.
3413  */
3414 int bh_submit_read(struct buffer_head *bh)
3415 {
3416         BUG_ON(!buffer_locked(bh));
3417
3418         if (buffer_uptodate(bh)) {
3419                 unlock_buffer(bh);
3420                 return 0;
3421         }
3422
3423         get_bh(bh);
3424         bh->b_end_io = end_buffer_read_sync;
3425         submit_bh(REQ_OP_READ, 0, bh);
3426         wait_on_buffer(bh);
3427         if (buffer_uptodate(bh))
3428                 return 0;
3429         return -EIO;
3430 }
3431 EXPORT_SYMBOL(bh_submit_read);
3432
3433 void __init buffer_init(void)
3434 {
3435         unsigned long nrpages;
3436         int ret;
3437
3438         bh_cachep = kmem_cache_create("buffer_head",
3439                         sizeof(struct buffer_head), 0,
3440                                 (SLAB_RECLAIM_ACCOUNT|SLAB_PANIC|
3441                                 SLAB_MEM_SPREAD),
3442                                 NULL);
3443
3444         /*
3445          * Limit the bh occupancy to 10% of ZONE_NORMAL
3446          */
3447         nrpages = (nr_free_buffer_pages() * 10) / 100;
3448         max_buffer_heads = nrpages * (PAGE_SIZE / sizeof(struct buffer_head));
3449         ret = cpuhp_setup_state_nocalls(CPUHP_FS_BUFF_DEAD, "fs/buffer:dead",
3450                                         NULL, buffer_exit_cpu_dead);
3451         WARN_ON(ret < 0);
3452 }