fs/f2fs/data.c

   1 /*
   2  * fs/f2fs/data.c
   3  *
   4  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   5  *             http://www.samsung.com/
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  */
  11 #include <linux/fs.h>
  12 #include <linux/f2fs_fs.h>
  13 #include <linux/buffer_head.h>
  14 #include <linux/mpage.h>
  15 #include <linux/writeback.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/pagevec.h>
  18 #include <linux/blkdev.h>
  19 #include <linux/bio.h>
  20 #include <linux/prefetch.h>
  21 #include <linux/uio.h>
  22 #include <linux/mm.h>
  23 #include <linux/memcontrol.h>
  24 #include <linux/cleancache.h>
  25 #include <linux/sched/signal.h>
  26
  27 #include "f2fs.h"
  28 #include "node.h"
  29 #include "segment.h"
  30 #include "trace.h"
  31 #include <trace/events/f2fs.h>
  32
  33 static bool __is_cp_guaranteed(struct page *page)
  34 {
  35         struct address_space *mapping = page->mapping;
  36         struct inode *inode;
  37         struct f2fs_sb_info *sbi;
  38
  39         if (!mapping)
  40                 return false;
  41
  42         inode = mapping->host;
  43         sbi = F2FS_I_SB(inode);
  44
  45         if (inode->i_ino == F2FS_META_INO(sbi) ||
  46                         inode->i_ino ==  F2FS_NODE_INO(sbi) ||
  47                         S_ISDIR(inode->i_mode) ||
  48                         is_cold_data(page))
  49                 return true;
  50         return false;
  51 }
  52
  53 static void f2fs_read_end_io(struct bio *bio)
  54 {
  55         struct bio_vec *bvec;
  56         int i;
  57
  58 #ifdef CONFIG_F2FS_FAULT_INJECTION
  59         if (time_to_inject(F2FS_P_SB(bio->bi_io_vec->bv_page), FAULT_IO)) {
  60                 f2fs_show_injection_info(FAULT_IO);
  61                 bio->bi_status = BLK_STS_IOERR;
  62         }
  63 #endif
  64
  65         if (f2fs_bio_encrypted(bio)) {
  66                 if (bio->bi_status) {
  67                         fscrypt_release_ctx(bio->bi_private);
  68                 } else {
  69                         fscrypt_decrypt_bio_pages(bio->bi_private, bio);
  70                         return;
  71                 }
  72         }
  73
  74         bio_for_each_segment_all(bvec, bio, i) {
  75                 struct page *page = bvec->bv_page;
  76
  77                 if (!bio->bi_status) {
  78                         if (!PageUptodate(page))
  79                                 SetPageUptodate(page);
  80                 } else {
  81                         ClearPageUptodate(page);
  82                         SetPageError(page);
  83                 }
  84                 unlock_page(page);
  85         }
  86         bio_put(bio);
  87 }
  88
  89 static void f2fs_write_end_io(struct bio *bio)
  90 {
  91         struct f2fs_sb_info *sbi = bio->bi_private;
  92         struct bio_vec *bvec;
  93         int i;
  94
  95         bio_for_each_segment_all(bvec, bio, i) {
  96                 struct page *page = bvec->bv_page;
  97                 enum count_type type = WB_DATA_TYPE(page);
  98
  99                 if (IS_DUMMY_WRITTEN_PAGE(page)) {
 100                         set_page_private(page, (unsigned long)NULL);
 101                         ClearPagePrivate(page);
 102                         unlock_page(page);
 103                         mempool_free(page, sbi->write_io_dummy);
 104
 105                         if (unlikely(bio->bi_status))
 106                                 f2fs_stop_checkpoint(sbi, true);
 107                         continue;
 108                 }
 109
 110                 fscrypt_pullback_bio_page(&page, true);
 111
 112                 if (unlikely(bio->bi_status)) {
 113                         mapping_set_error(page->mapping, -EIO);
 114                         f2fs_stop_checkpoint(sbi, true);
 115                 }
 116                 dec_page_count(sbi, type);
 117                 clear_cold_data(page);
 118                 end_page_writeback(page);
 119         }
 120         if (!get_pages(sbi, F2FS_WB_CP_DATA) &&
 121                                 wq_has_sleeper(&sbi->cp_wait))
 122                 wake_up(&sbi->cp_wait);
 123
 124         bio_put(bio);
 125 }
 126
 127 /*
 128  * Return true, if pre_bio's bdev is same as its target device.
 129  */
 130 struct block_device *f2fs_target_device(struct f2fs_sb_info *sbi,
 131                                 block_t blk_addr, struct bio *bio)
 132 {
 133         struct block_device *bdev = sbi->sb->s_bdev;
 134         int i;
 135
 136         for (i = 0; i < sbi->s_ndevs; i++) {
 137                 if (FDEV(i).start_blk <= blk_addr &&
 138                                         FDEV(i).end_blk >= blk_addr) {
 139                         blk_addr -= FDEV(i).start_blk;
 140                         bdev = FDEV(i).bdev;
 141                         break;
 142                 }
 143         }
 144         if (bio) {
 145                 bio->bi_bdev = bdev;
 146                 bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(blk_addr);
 147         }
 148         return bdev;
 149 }
 150
 151 int f2fs_target_device_index(struct f2fs_sb_info *sbi, block_t blkaddr)
 152 {
 153         int i;
 154
 155         for (i = 0; i < sbi->s_ndevs; i++)
 156                 if (FDEV(i).start_blk <= blkaddr && FDEV(i).end_blk >= blkaddr)
 157                         return i;
 158         return 0;
 159 }
 160
 161 static bool __same_bdev(struct f2fs_sb_info *sbi,
 162                                 block_t blk_addr, struct bio *bio)
 163 {
 164         return f2fs_target_device(sbi, blk_addr, NULL) == bio->bi_bdev;
 165 }
 166
 167 /*
 168  * Low-level block read/write IO operations.
 169  */
 170 static struct bio *__bio_alloc(struct f2fs_sb_info *sbi, block_t blk_addr,
 171                                 int npages, bool is_read)
 172 {
 173         struct bio *bio;
 174
 175         bio = f2fs_bio_alloc(npages);
 176
 177         f2fs_target_device(sbi, blk_addr, bio);
 178         bio->bi_end_io = is_read ? f2fs_read_end_io : f2fs_write_end_io;
 179         bio->bi_private = is_read ? NULL : sbi;
 180
 181         return bio;
 182 }
 183
 184 static inline void __submit_bio(struct f2fs_sb_info *sbi,
 185                                 struct bio *bio, enum page_type type)
 186 {
 187         if (!is_read_io(bio_op(bio))) {
 188                 unsigned int start;
 189
 190                 if (f2fs_sb_mounted_blkzoned(sbi->sb) &&
 191                         current->plug && (type == DATA || type == NODE))
 192                         blk_finish_plug(current->plug);
 193
 194                 if (type != DATA && type != NODE)
 195                         goto submit_io;
 196
 197                 start = bio->bi_iter.bi_size >> F2FS_BLKSIZE_BITS;
 198                 start %= F2FS_IO_SIZE(sbi);
 199
 200                 if (start == 0)
 201                         goto submit_io;
 202
 203                 /* fill dummy pages */
 204                 for (; start < F2FS_IO_SIZE(sbi); start++) {
 205                         struct page *page =
 206                                 mempool_alloc(sbi->write_io_dummy,
 207                                         GFP_NOIO | __GFP_ZERO | __GFP_NOFAIL);
 208                         f2fs_bug_on(sbi, !page);
 209
 210                         SetPagePrivate(page);
 211                         set_page_private(page, (unsigned long)DUMMY_WRITTEN_PAGE);
 212                         lock_page(page);
 213                         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE)
 214                                 f2fs_bug_on(sbi, 1);
 215                 }
 216                 /*
 217                  * In the NODE case, we lose next block address chain. So, we
 218                  * need to do checkpoint in f2fs_sync_file.
 219                  */
 220                 if (type == NODE)
 221                         set_sbi_flag(sbi, SBI_NEED_CP);
 222         }
 223 submit_io:
 224         if (is_read_io(bio_op(bio)))
 225                 trace_f2fs_submit_read_bio(sbi->sb, type, bio);
 226         else
 227                 trace_f2fs_submit_write_bio(sbi->sb, type, bio);
 228         submit_bio(bio);
 229 }
 230
 231 static void __submit_merged_bio(struct f2fs_bio_info *io)
 232 {
 233         struct f2fs_io_info *fio = &io->fio;
 234
 235         if (!io->bio)
 236                 return;
 237
 238         bio_set_op_attrs(io->bio, fio->op, fio->op_flags);
 239
 240         if (is_read_io(fio->op))
 241                 trace_f2fs_prepare_read_bio(io->sbi->sb, fio->type, io->bio);
 242         else
 243                 trace_f2fs_prepare_write_bio(io->sbi->sb, fio->type, io->bio);
 244
 245         __submit_bio(io->sbi, io->bio, fio->type);
 246         io->bio = NULL;
 247 }
 248
 249 static bool __has_merged_page(struct f2fs_bio_info *io,
 250                                 struct inode *inode, nid_t ino, pgoff_t idx)
 251 {
 252         struct bio_vec *bvec;
 253         struct page *target;
 254         int i;
 255
 256         if (!io->bio)
 257                 return false;
 258
 259         if (!inode && !ino)
 260                 return true;
 261
 262         bio_for_each_segment_all(bvec, io->bio, i) {
 263
 264                 if (bvec->bv_page->mapping)
 265                         target = bvec->bv_page;
 266                 else
 267                         target = fscrypt_control_page(bvec->bv_page);
 268
 269                 if (idx != target->index)
 270                         continue;
 271
 272                 if (inode && inode == target->mapping->host)
 273                         return true;
 274                 if (ino && ino == ino_of_node(target))
 275                         return true;
 276         }
 277
 278         return false;
 279 }
 280
 281 static bool has_merged_page(struct f2fs_sb_info *sbi, struct inode *inode,
 282                                 nid_t ino, pgoff_t idx, enum page_type type)
 283 {
 284         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 285         enum temp_type temp;
 286         struct f2fs_bio_info *io;
 287         bool ret = false;
 288
 289         for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
 290                 io = sbi->write_io[btype] + temp;
 291
 292                 down_read(&io->io_rwsem);
 293                 ret = __has_merged_page(io, inode, ino, idx);
 294                 up_read(&io->io_rwsem);
 295
 296                 /* TODO: use HOT temp only for meta pages now. */
 297                 if (ret || btype == META)
 298                         break;
 299         }
 300         return ret;
 301 }
 302
 303 static void __f2fs_submit_merged_write(struct f2fs_sb_info *sbi,
 304                                 enum page_type type, enum temp_type temp)
 305 {
 306         enum page_type btype = PAGE_TYPE_OF_BIO(type);
 307         struct f2fs_bio_info *io = sbi->write_io[btype] + temp;
 308
 309         down_write(&io->io_rwsem);
 310
 311         /* change META to META_FLUSH in the checkpoint procedure */
 312         if (type >= META_FLUSH) {
 313                 io->fio.type = META_FLUSH;
 314                 io->fio.op = REQ_OP_WRITE;
 315                 io->fio.op_flags = REQ_META | REQ_PRIO | REQ_SYNC;
 316                 if (!test_opt(sbi, NOBARRIER))
 317                         io->fio.op_flags |= REQ_PREFLUSH | REQ_FUA;
 318         }
 319         __submit_merged_bio(io);
 320         up_write(&io->io_rwsem);
 321 }
 322
 323 static void __submit_merged_write_cond(struct f2fs_sb_info *sbi,
 324                                 struct inode *inode, nid_t ino, pgoff_t idx,
 325                                 enum page_type type, bool force)
 326 {
 327         enum temp_type temp;
 328
 329         if (!force && !has_merged_page(sbi, inode, ino, idx, type))
 330                 return;
 331
 332         for (temp = HOT; temp < NR_TEMP_TYPE; temp++) {
 333
 334                 __f2fs_submit_merged_write(sbi, type, temp);
 335
 336                 /* TODO: use HOT temp only for meta pages now. */
 337                 if (type >= META)
 338                         break;
 339         }
 340 }
 341
 342 void f2fs_submit_merged_write(struct f2fs_sb_info *sbi, enum page_type type)
 343 {
 344         __submit_merged_write_cond(sbi, NULL, 0, 0, type, true);
 345 }
 346
 347 void f2fs_submit_merged_write_cond(struct f2fs_sb_info *sbi,
 348                                 struct inode *inode, nid_t ino, pgoff_t idx,
 349                                 enum page_type type)
 350 {
 351         __submit_merged_write_cond(sbi, inode, ino, idx, type, false);
 352 }
 353
 354 void f2fs_flush_merged_writes(struct f2fs_sb_info *sbi)
 355 {
 356         f2fs_submit_merged_write(sbi, DATA);
 357         f2fs_submit_merged_write(sbi, NODE);
 358         f2fs_submit_merged_write(sbi, META);
 359 }
 360
 361 /*
 362  * Fill the locked page with data located in the block address.
 363  * A caller needs to unlock the page on failure.
 364  */
 365 int f2fs_submit_page_bio(struct f2fs_io_info *fio)
 366 {
 367         struct bio *bio;
 368         struct page *page = fio->encrypted_page ?
 369                         fio->encrypted_page : fio->page;
 370
 371         trace_f2fs_submit_page_bio(page, fio);
 372         f2fs_trace_ios(fio, 0);
 373
 374         /* Allocate a new bio */
 375         bio = __bio_alloc(fio->sbi, fio->new_blkaddr, 1, is_read_io(fio->op));
 376
 377         if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
 378                 bio_put(bio);
 379                 return -EFAULT;
 380         }
 381         bio_set_op_attrs(bio, fio->op, fio->op_flags);
 382
 383         __submit_bio(fio->sbi, bio, fio->type);
 384
 385         if (!is_read_io(fio->op))
 386                 inc_page_count(fio->sbi, WB_DATA_TYPE(fio->page));
 387         return 0;
 388 }
 389
 390 int f2fs_submit_page_write(struct f2fs_io_info *fio)
 391 {
 392         struct f2fs_sb_info *sbi = fio->sbi;
 393         enum page_type btype = PAGE_TYPE_OF_BIO(fio->type);
 394         struct f2fs_bio_info *io = sbi->write_io[btype] + fio->temp;
 395         struct page *bio_page;
 396         int err = 0;
 397
 398         f2fs_bug_on(sbi, is_read_io(fio->op));
 399
 400         down_write(&io->io_rwsem);
 401 next:
 402         if (fio->in_list) {
 403                 spin_lock(&io->io_lock);
 404                 if (list_empty(&io->io_list)) {
 405                         spin_unlock(&io->io_lock);
 406                         goto out_fail;
 407                 }
 408                 fio = list_first_entry(&io->io_list,
 409                                                 struct f2fs_io_info, list);
 410                 list_del(&fio->list);
 411                 spin_unlock(&io->io_lock);
 412         }
 413
 414         if (fio->old_blkaddr != NEW_ADDR)
 415                 verify_block_addr(sbi, fio->old_blkaddr);
 416         verify_block_addr(sbi, fio->new_blkaddr);
 417
 418         bio_page = fio->encrypted_page ? fio->encrypted_page : fio->page;
 419
 420         /* set submitted = 1 as a return value */
 421         fio->submitted = 1;
 422
 423         inc_page_count(sbi, WB_DATA_TYPE(bio_page));
 424
 425         if (io->bio && (io->last_block_in_bio != fio->new_blkaddr - 1 ||
 426             (io->fio.op != fio->op || io->fio.op_flags != fio->op_flags) ||
 427                         !__same_bdev(sbi, fio->new_blkaddr, io->bio)))
 428                 __submit_merged_bio(io);
 429 alloc_new:
 430         if (io->bio == NULL) {
 431                 if ((fio->type == DATA || fio->type == NODE) &&
 432                                 fio->new_blkaddr & F2FS_IO_SIZE_MASK(sbi)) {
 433                         err = -EAGAIN;
 434                         dec_page_count(sbi, WB_DATA_TYPE(bio_page));
 435                         goto out_fail;
 436                 }
 437                 io->bio = __bio_alloc(sbi, fio->new_blkaddr,
 438                                                 BIO_MAX_PAGES, false);
 439                 io->fio = *fio;
 440         }
 441
 442         if (bio_add_page(io->bio, bio_page, PAGE_SIZE, 0) < PAGE_SIZE) {
 443                 __submit_merged_bio(io);
 444                 goto alloc_new;
 445         }
 446
 447         io->last_block_in_bio = fio->new_blkaddr;
 448         f2fs_trace_ios(fio, 0);
 449
 450         trace_f2fs_submit_page_write(fio->page, fio);
 451
 452         if (fio->in_list)
 453                 goto next;
 454 out_fail:
 455         up_write(&io->io_rwsem);
 456         return err;
 457 }
 458
 459 static void __set_data_blkaddr(struct dnode_of_data *dn)
 460 {
 461         struct f2fs_node *rn = F2FS_NODE(dn->node_page);
 462         __le32 *addr_array;
 463
 464         /* Get physical address of data block */
 465         addr_array = blkaddr_in_node(rn);
 466         addr_array[dn->ofs_in_node] = cpu_to_le32(dn->data_blkaddr);
 467 }
 468
 469 /*
 470  * Lock ordering for the change of data block address:
 471  * ->data_page
 472  *  ->node_page
 473  *    update block addresses in the node page
 474  */
 475 void set_data_blkaddr(struct dnode_of_data *dn)
 476 {
 477         f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
 478         __set_data_blkaddr(dn);
 479         if (set_page_dirty(dn->node_page))
 480                 dn->node_changed = true;
 481 }
 482
 483 void f2fs_update_data_blkaddr(struct dnode_of_data *dn, block_t blkaddr)
 484 {
 485         dn->data_blkaddr = blkaddr;
 486         set_data_blkaddr(dn);
 487         f2fs_update_extent_cache(dn);
 488 }
 489
 490 /* dn->ofs_in_node will be returned with up-to-date last block pointer */
 491 int reserve_new_blocks(struct dnode_of_data *dn, blkcnt_t count)
 492 {
 493         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 494         int err;
 495
 496         if (!count)
 497                 return 0;
 498
 499         if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
 500                 return -EPERM;
 501         if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
 502                 return err;
 503
 504         trace_f2fs_reserve_new_blocks(dn->inode, dn->nid,
 505                                                 dn->ofs_in_node, count);
 506
 507         f2fs_wait_on_page_writeback(dn->node_page, NODE, true);
 508
 509         for (; count > 0; dn->ofs_in_node++) {
 510                 block_t blkaddr =
 511                         datablock_addr(dn->node_page, dn->ofs_in_node);
 512                 if (blkaddr == NULL_ADDR) {
 513                         dn->data_blkaddr = NEW_ADDR;
 514                         __set_data_blkaddr(dn);
 515                         count--;
 516                 }
 517         }
 518
 519         if (set_page_dirty(dn->node_page))
 520                 dn->node_changed = true;
 521         return 0;
 522 }
 523
 524 /* Should keep dn->ofs_in_node unchanged */
 525 int reserve_new_block(struct dnode_of_data *dn)
 526 {
 527         unsigned int ofs_in_node = dn->ofs_in_node;
 528         int ret;
 529
 530         ret = reserve_new_blocks(dn, 1);
 531         dn->ofs_in_node = ofs_in_node;
 532         return ret;
 533 }
 534
 535 int f2fs_reserve_block(struct dnode_of_data *dn, pgoff_t index)
 536 {
 537         bool need_put = dn->inode_page ? false : true;
 538         int err;
 539
 540         err = get_dnode_of_data(dn, index, ALLOC_NODE);
 541         if (err)
 542                 return err;
 543
 544         if (dn->data_blkaddr == NULL_ADDR)
 545                 err = reserve_new_block(dn);
 546         if (err || need_put)
 547                 f2fs_put_dnode(dn);
 548         return err;
 549 }
 550
 551 int f2fs_get_block(struct dnode_of_data *dn, pgoff_t index)
 552 {
 553         struct extent_info ei  = {0,0,0};
 554         struct inode *inode = dn->inode;
 555
 556         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 557                 dn->data_blkaddr = ei.blk + index - ei.fofs;
 558                 return 0;
 559         }
 560
 561         return f2fs_reserve_block(dn, index);
 562 }
 563
 564 struct page *get_read_data_page(struct inode *inode, pgoff_t index,
 565                                                 int op_flags, bool for_write)
 566 {
 567         struct address_space *mapping = inode->i_mapping;
 568         struct dnode_of_data dn;
 569         struct page *page;
 570         struct extent_info ei = {0,0,0};
 571         int err;
 572         struct f2fs_io_info fio = {
 573                 .sbi = F2FS_I_SB(inode),
 574                 .type = DATA,
 575                 .op = REQ_OP_READ,
 576                 .op_flags = op_flags,
 577                 .encrypted_page = NULL,
 578         };
 579
 580         if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
 581                 return read_mapping_page(mapping, index, NULL);
 582
 583         page = f2fs_grab_cache_page(mapping, index, for_write);
 584         if (!page)
 585                 return ERR_PTR(-ENOMEM);
 586
 587         if (f2fs_lookup_extent_cache(inode, index, &ei)) {
 588                 dn.data_blkaddr = ei.blk + index - ei.fofs;
 589                 goto got_it;
 590         }
 591
 592         set_new_dnode(&dn, inode, NULL, NULL, 0);
 593         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
 594         if (err)
 595                 goto put_err;
 596         f2fs_put_dnode(&dn);
 597
 598         if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
 599                 err = -ENOENT;
 600                 goto put_err;
 601         }
 602 got_it:
 603         if (PageUptodate(page)) {
 604                 unlock_page(page);
 605                 return page;
 606         }
 607
 608         /*
 609          * A new dentry page is allocated but not able to be written, since its
 610          * new inode page couldn't be allocated due to -ENOSPC.
 611          * In such the case, its blkaddr can be remained as NEW_ADDR.
 612          * see, f2fs_add_link -> get_new_data_page -> init_inode_metadata.
 613          */
 614         if (dn.data_blkaddr == NEW_ADDR) {
 615                 zero_user_segment(page, 0, PAGE_SIZE);
 616                 if (!PageUptodate(page))
 617                         SetPageUptodate(page);
 618                 unlock_page(page);
 619                 return page;
 620         }
 621
 622         fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
 623         fio.page = page;
 624         err = f2fs_submit_page_bio(&fio);
 625         if (err)
 626                 goto put_err;
 627         return page;
 628
 629 put_err:
 630         f2fs_put_page(page, 1);
 631         return ERR_PTR(err);
 632 }
 633
 634 struct page *find_data_page(struct inode *inode, pgoff_t index)
 635 {
 636         struct address_space *mapping = inode->i_mapping;
 637         struct page *page;
 638
 639         page = find_get_page(mapping, index);
 640         if (page && PageUptodate(page))
 641                 return page;
 642         f2fs_put_page(page, 0);
 643
 644         page = get_read_data_page(inode, index, 0, false);
 645         if (IS_ERR(page))
 646                 return page;
 647
 648         if (PageUptodate(page))
 649                 return page;
 650
 651         wait_on_page_locked(page);
 652         if (unlikely(!PageUptodate(page))) {
 653                 f2fs_put_page(page, 0);
 654                 return ERR_PTR(-EIO);
 655         }
 656         return page;
 657 }
 658
 659 /*
 660  * If it tries to access a hole, return an error.
 661  * Because, the callers, functions in dir.c and GC, should be able to know
 662  * whether this page exists or not.
 663  */
 664 struct page *get_lock_data_page(struct inode *inode, pgoff_t index,
 665                                                         bool for_write)
 666 {
 667         struct address_space *mapping = inode->i_mapping;
 668         struct page *page;
 669 repeat:
 670         page = get_read_data_page(inode, index, 0, for_write);
 671         if (IS_ERR(page))
 672                 return page;
 673
 674         /* wait for read completion */
 675         lock_page(page);
 676         if (unlikely(page->mapping != mapping)) {
 677                 f2fs_put_page(page, 1);
 678                 goto repeat;
 679         }
 680         if (unlikely(!PageUptodate(page))) {
 681                 f2fs_put_page(page, 1);
 682                 return ERR_PTR(-EIO);
 683         }
 684         return page;
 685 }
 686
 687 /*
 688  * Caller ensures that this data page is never allocated.
 689  * A new zero-filled data page is allocated in the page cache.
 690  *
 691  * Also, caller should grab and release a rwsem by calling f2fs_lock_op() and
 692  * f2fs_unlock_op().
 693  * Note that, ipage is set only by make_empty_dir, and if any error occur,
 694  * ipage should be released by this function.
 695  */
 696 struct page *get_new_data_page(struct inode *inode,
 697                 struct page *ipage, pgoff_t index, bool new_i_size)
 698 {
 699         struct address_space *mapping = inode->i_mapping;
 700         struct page *page;
 701         struct dnode_of_data dn;
 702         int err;
 703
 704         page = f2fs_grab_cache_page(mapping, index, true);
 705         if (!page) {
 706                 /*
 707                  * before exiting, we should make sure ipage will be released
 708                  * if any error occur.
 709                  */
 710                 f2fs_put_page(ipage, 1);
 711                 return ERR_PTR(-ENOMEM);
 712         }
 713
 714         set_new_dnode(&dn, inode, ipage, NULL, 0);
 715         err = f2fs_reserve_block(&dn, index);
 716         if (err) {
 717                 f2fs_put_page(page, 1);
 718                 return ERR_PTR(err);
 719         }
 720         if (!ipage)
 721                 f2fs_put_dnode(&dn);
 722
 723         if (PageUptodate(page))
 724                 goto got_it;
 725
 726         if (dn.data_blkaddr == NEW_ADDR) {
 727                 zero_user_segment(page, 0, PAGE_SIZE);
 728                 if (!PageUptodate(page))
 729                         SetPageUptodate(page);
 730         } else {
 731                 f2fs_put_page(page, 1);
 732
 733                 /* if ipage exists, blkaddr should be NEW_ADDR */
 734                 f2fs_bug_on(F2FS_I_SB(inode), ipage);
 735                 page = get_lock_data_page(inode, index, true);
 736                 if (IS_ERR(page))
 737                         return page;
 738         }
 739 got_it:
 740         if (new_i_size && i_size_read(inode) <
 741                                 ((loff_t)(index + 1) << PAGE_SHIFT))
 742                 f2fs_i_size_write(inode, ((loff_t)(index + 1) << PAGE_SHIFT));
 743         return page;
 744 }
 745
 746 static int __allocate_data_block(struct dnode_of_data *dn)
 747 {
 748         struct f2fs_sb_info *sbi = F2FS_I_SB(dn->inode);
 749         struct f2fs_summary sum;
 750         struct node_info ni;
 751         pgoff_t fofs;
 752         blkcnt_t count = 1;
 753         int err;
 754
 755         if (unlikely(is_inode_flag_set(dn->inode, FI_NO_ALLOC)))
 756                 return -EPERM;
 757
 758         dn->data_blkaddr = datablock_addr(dn->node_page, dn->ofs_in_node);
 759         if (dn->data_blkaddr == NEW_ADDR)
 760                 goto alloc;
 761
 762         if (unlikely((err = inc_valid_block_count(sbi, dn->inode, &count))))
 763                 return err;
 764
 765 alloc:
 766         get_node_info(sbi, dn->nid, &ni);
 767         set_summary(&sum, dn->nid, dn->ofs_in_node, ni.version);
 768
 769         allocate_data_block(sbi, NULL, dn->data_blkaddr, &dn->data_blkaddr,
 770                                         &sum, CURSEG_WARM_DATA, NULL, false);
 771         set_data_blkaddr(dn);
 772
 773         /* update i_size */
 774         fofs = start_bidx_of_node(ofs_of_node(dn->node_page), dn->inode) +
 775                                                         dn->ofs_in_node;
 776         if (i_size_read(dn->inode) < ((loff_t)(fofs + 1) << PAGE_SHIFT))
 777                 f2fs_i_size_write(dn->inode,
 778                                 ((loff_t)(fofs + 1) << PAGE_SHIFT));
 779         return 0;
 780 }
 781
 782 static inline bool __force_buffered_io(struct inode *inode, int rw)
 783 {
 784         return ((f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) ||
 785                         (rw == WRITE && test_opt(F2FS_I_SB(inode), LFS)) ||
 786                         F2FS_I_SB(inode)->s_ndevs);
 787 }
 788
 789 int f2fs_preallocate_blocks(struct kiocb *iocb, struct iov_iter *from)
 790 {
 791         struct inode *inode = file_inode(iocb->ki_filp);
 792         struct f2fs_map_blocks map;
 793         int err = 0;
 794
 795         if (is_inode_flag_set(inode, FI_NO_PREALLOC))
 796                 return 0;
 797
 798         map.m_lblk = F2FS_BLK_ALIGN(iocb->ki_pos);
 799         map.m_len = F2FS_BYTES_TO_BLK(iocb->ki_pos + iov_iter_count(from));
 800         if (map.m_len > map.m_lblk)
 801                 map.m_len -= map.m_lblk;
 802         else
 803                 map.m_len = 0;
 804
 805         map.m_next_pgofs = NULL;
 806
 807         if (iocb->ki_flags & IOCB_DIRECT) {
 808                 err = f2fs_convert_inline_inode(inode);
 809                 if (err)
 810                         return err;
 811                 return f2fs_map_blocks(inode, &map, 1,
 812                         __force_buffered_io(inode, WRITE) ?
 813                                 F2FS_GET_BLOCK_PRE_AIO :
 814                                 F2FS_GET_BLOCK_PRE_DIO);
 815         }
 816         if (iocb->ki_pos + iov_iter_count(from) > MAX_INLINE_DATA) {
 817                 err = f2fs_convert_inline_inode(inode);
 818                 if (err)
 819                         return err;
 820         }
 821         if (!f2fs_has_inline_data(inode))
 822                 return f2fs_map_blocks(inode, &map, 1, F2FS_GET_BLOCK_PRE_AIO);
 823         return err;
 824 }
 825
 826 static inline void __do_map_lock(struct f2fs_sb_info *sbi, int flag, bool lock)
 827 {
 828         if (flag == F2FS_GET_BLOCK_PRE_AIO) {
 829                 if (lock)
 830                         down_read(&sbi->node_change);
 831                 else
 832                         up_read(&sbi->node_change);
 833         } else {
 834                 if (lock)
 835                         f2fs_lock_op(sbi);
 836                 else
 837                         f2fs_unlock_op(sbi);
 838         }
 839 }
 840
 841 /*
 842  * f2fs_map_blocks() now supported readahead/bmap/rw direct_IO with
 843  * f2fs_map_blocks structure.
 844  * If original data blocks are allocated, then give them to blockdev.
 845  * Otherwise,
 846  *     a. preallocate requested block addresses
 847  *     b. do not use extent cache for better performance
 848  *     c. give the block addresses to blockdev
 849  */
 850 int f2fs_map_blocks(struct inode *inode, struct f2fs_map_blocks *map,
 851                                                 int create, int flag)
 852 {
 853         unsigned int maxblocks = map->m_len;
 854         struct dnode_of_data dn;
 855         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 856         int mode = create ? ALLOC_NODE : LOOKUP_NODE;
 857         pgoff_t pgofs, end_offset, end;
 858         int err = 0, ofs = 1;
 859         unsigned int ofs_in_node, last_ofs_in_node;
 860         blkcnt_t prealloc;
 861         struct extent_info ei = {0,0,0};
 862         block_t blkaddr;
 863
 864         if (!maxblocks)
 865                 return 0;
 866
 867         map->m_len = 0;
 868         map->m_flags = 0;
 869
 870         /* it only supports block size == page size */
 871         pgofs = (pgoff_t)map->m_lblk;
 872         end = pgofs + maxblocks;
 873
 874         if (!create && f2fs_lookup_extent_cache(inode, pgofs, &ei)) {
 875                 map->m_pblk = ei.blk + pgofs - ei.fofs;
 876                 map->m_len = min((pgoff_t)maxblocks, ei.fofs + ei.len - pgofs);
 877                 map->m_flags = F2FS_MAP_MAPPED;
 878                 goto out;
 879         }
 880
 881 next_dnode:
 882         if (create)
 883                 __do_map_lock(sbi, flag, true);
 884
 885         /* When reading holes, we need its node page */
 886         set_new_dnode(&dn, inode, NULL, NULL, 0);
 887         err = get_dnode_of_data(&dn, pgofs, mode);
 888         if (err) {
 889                 if (flag == F2FS_GET_BLOCK_BMAP)
 890                         map->m_pblk = 0;
 891                 if (err == -ENOENT) {
 892                         err = 0;
 893                         if (map->m_next_pgofs)
 894                                 *map->m_next_pgofs =
 895                                         get_next_page_offset(&dn, pgofs);
 896                 }
 897                 goto unlock_out;
 898         }
 899
 900         prealloc = 0;
 901         last_ofs_in_node = ofs_in_node = dn.ofs_in_node;
 902         end_offset = ADDRS_PER_PAGE(dn.node_page, inode);
 903
 904 next_block:
 905         blkaddr = datablock_addr(dn.node_page, dn.ofs_in_node);
 906
 907         if (blkaddr == NEW_ADDR || blkaddr == NULL_ADDR) {
 908                 if (create) {
 909                         if (unlikely(f2fs_cp_error(sbi))) {
 910                                 err = -EIO;
 911                                 goto sync_out;
 912                         }
 913                         if (flag == F2FS_GET_BLOCK_PRE_AIO) {
 914                                 if (blkaddr == NULL_ADDR) {
 915                                         prealloc++;
 916                                         last_ofs_in_node = dn.ofs_in_node;
 917                                 }
 918                         } else {
 919                                 err = __allocate_data_block(&dn);
 920                                 if (!err)
 921                                         set_inode_flag(inode, FI_APPEND_WRITE);
 922                         }
 923                         if (err)
 924                                 goto sync_out;
 925                         map->m_flags |= F2FS_MAP_NEW;
 926                         blkaddr = dn.data_blkaddr;
 927                 } else {
 928                         if (flag == F2FS_GET_BLOCK_BMAP) {
 929                                 map->m_pblk = 0;
 930                                 goto sync_out;
 931                         }
 932                         if (flag == F2FS_GET_BLOCK_FIEMAP &&
 933                                                 blkaddr == NULL_ADDR) {
 934                                 if (map->m_next_pgofs)
 935                                         *map->m_next_pgofs = pgofs + 1;
 936                         }
 937                         if (flag != F2FS_GET_BLOCK_FIEMAP ||
 938                                                 blkaddr != NEW_ADDR)
 939                                 goto sync_out;
 940                 }
 941         }
 942
 943         if (flag == F2FS_GET_BLOCK_PRE_AIO)
 944                 goto skip;
 945
 946         if (map->m_len == 0) {
 947                 /* preallocated unwritten block should be mapped for fiemap. */
 948                 if (blkaddr == NEW_ADDR)
 949                         map->m_flags |= F2FS_MAP_UNWRITTEN;
 950                 map->m_flags |= F2FS_MAP_MAPPED;
 951
 952                 map->m_pblk = blkaddr;
 953                 map->m_len = 1;
 954         } else if ((map->m_pblk != NEW_ADDR &&
 955                         blkaddr == (map->m_pblk + ofs)) ||
 956                         (map->m_pblk == NEW_ADDR && blkaddr == NEW_ADDR) ||
 957                         flag == F2FS_GET_BLOCK_PRE_DIO) {
 958                 ofs++;
 959                 map->m_len++;
 960         } else {
 961                 goto sync_out;
 962         }
 963
 964 skip:
 965         dn.ofs_in_node++;
 966         pgofs++;
 967
 968         /* preallocate blocks in batch for one dnode page */
 969         if (flag == F2FS_GET_BLOCK_PRE_AIO &&
 970                         (pgofs == end || dn.ofs_in_node == end_offset)) {
 971
 972                 dn.ofs_in_node = ofs_in_node;
 973                 err = reserve_new_blocks(&dn, prealloc);
 974                 if (err)
 975                         goto sync_out;
 976
 977                 map->m_len += dn.ofs_in_node - ofs_in_node;
 978                 if (prealloc && dn.ofs_in_node != last_ofs_in_node + 1) {
 979                         err = -ENOSPC;
 980                         goto sync_out;
 981                 }
 982                 dn.ofs_in_node = end_offset;
 983         }
 984
 985         if (pgofs >= end)
 986                 goto sync_out;
 987         else if (dn.ofs_in_node < end_offset)
 988                 goto next_block;
 989
 990         f2fs_put_dnode(&dn);
 991
 992         if (create) {
 993                 __do_map_lock(sbi, flag, false);
 994                 f2fs_balance_fs(sbi, dn.node_changed);
 995         }
 996         goto next_dnode;
 997
 998 sync_out:
 999         f2fs_put_dnode(&dn);
1000 unlock_out:
1001         if (create) {
1002                 __do_map_lock(sbi, flag, false);
1003                 f2fs_balance_fs(sbi, dn.node_changed);
1004         }
1005 out:
1006         trace_f2fs_map_blocks(inode, map, err);
1007         return err;
1008 }
1009
1010 static int __get_data_block(struct inode *inode, sector_t iblock,
1011                         struct buffer_head *bh, int create, int flag,
1012                         pgoff_t *next_pgofs)
1013 {
1014         struct f2fs_map_blocks map;
1015         int err;
1016
1017         map.m_lblk = iblock;
1018         map.m_len = bh->b_size >> inode->i_blkbits;
1019         map.m_next_pgofs = next_pgofs;
1020
1021         err = f2fs_map_blocks(inode, &map, create, flag);
1022         if (!err) {
1023                 map_bh(bh, inode->i_sb, map.m_pblk);
1024                 bh->b_state = (bh->b_state & ~F2FS_MAP_FLAGS) | map.m_flags;
1025                 bh->b_size = (u64)map.m_len << inode->i_blkbits;
1026         }
1027         return err;
1028 }
1029
1030 static int get_data_block(struct inode *inode, sector_t iblock,
1031                         struct buffer_head *bh_result, int create, int flag,
1032                         pgoff_t *next_pgofs)
1033 {
1034         return __get_data_block(inode, iblock, bh_result, create,
1035                                                         flag, next_pgofs);
1036 }
1037
1038 static int get_data_block_dio(struct inode *inode, sector_t iblock,
1039                         struct buffer_head *bh_result, int create)
1040 {
1041         return __get_data_block(inode, iblock, bh_result, create,
1042                                                 F2FS_GET_BLOCK_DIO, NULL);
1043 }
1044
1045 static int get_data_block_bmap(struct inode *inode, sector_t iblock,
1046                         struct buffer_head *bh_result, int create)
1047 {
1048         /* Block number less than F2FS MAX BLOCKS */
1049         if (unlikely(iblock >= F2FS_I_SB(inode)->max_file_blocks))
1050                 return -EFBIG;
1051
1052         return __get_data_block(inode, iblock, bh_result, create,
1053                                                 F2FS_GET_BLOCK_BMAP, NULL);
1054 }
1055
1056 static inline sector_t logical_to_blk(struct inode *inode, loff_t offset)
1057 {
1058         return (offset >> inode->i_blkbits);
1059 }
1060
1061 static inline loff_t blk_to_logical(struct inode *inode, sector_t blk)
1062 {
1063         return (blk << inode->i_blkbits);
1064 }
1065
1066 int f2fs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
1067                 u64 start, u64 len)
1068 {
1069         struct buffer_head map_bh;
1070         sector_t start_blk, last_blk;
1071         pgoff_t next_pgofs;
1072         u64 logical = 0, phys = 0, size = 0;
1073         u32 flags = 0;
1074         int ret = 0;
1075
1076         ret = fiemap_check_flags(fieinfo, FIEMAP_FLAG_SYNC);
1077         if (ret)
1078                 return ret;
1079
1080         if (f2fs_has_inline_data(inode)) {
1081                 ret = f2fs_inline_data_fiemap(inode, fieinfo, start, len);
1082                 if (ret != -EAGAIN)
1083                         return ret;
1084         }
1085
1086         inode_lock(inode);
1087
1088         if (logical_to_blk(inode, len) == 0)
1089                 len = blk_to_logical(inode, 1);
1090
1091         start_blk = logical_to_blk(inode, start);
1092         last_blk = logical_to_blk(inode, start + len - 1);
1093
1094 next:
1095         memset(&map_bh, 0, sizeof(struct buffer_head));
1096         map_bh.b_size = len;
1097
1098         ret = get_data_block(inode, start_blk, &map_bh, 0,
1099                                         F2FS_GET_BLOCK_FIEMAP, &next_pgofs);
1100         if (ret)
1101                 goto out;
1102
1103         /* HOLE */
1104         if (!buffer_mapped(&map_bh)) {
1105                 start_blk = next_pgofs;
1106
1107                 if (blk_to_logical(inode, start_blk) < blk_to_logical(inode,
1108                                         F2FS_I_SB(inode)->max_file_blocks))
1109                         goto prep_next;
1110
1111                 flags |= FIEMAP_EXTENT_LAST;
1112         }
1113
1114         if (size) {
1115                 if (f2fs_encrypted_inode(inode))
1116                         flags |= FIEMAP_EXTENT_DATA_ENCRYPTED;
1117
1118                 ret = fiemap_fill_next_extent(fieinfo, logical,
1119                                 phys, size, flags);
1120         }
1121
1122         if (start_blk > last_blk || ret)
1123                 goto out;
1124
1125         logical = blk_to_logical(inode, start_blk);
1126         phys = blk_to_logical(inode, map_bh.b_blocknr);
1127         size = map_bh.b_size;
1128         flags = 0;
1129         if (buffer_unwritten(&map_bh))
1130                 flags = FIEMAP_EXTENT_UNWRITTEN;
1131
1132         start_blk += logical_to_blk(inode, size);
1133
1134 prep_next:
1135         cond_resched();
1136         if (fatal_signal_pending(current))
1137                 ret = -EINTR;
1138         else
1139                 goto next;
1140 out:
1141         if (ret == 1)
1142                 ret = 0;
1143
1144         inode_unlock(inode);
1145         return ret;
1146 }
1147
1148 static struct bio *f2fs_grab_bio(struct inode *inode, block_t blkaddr,
1149                                  unsigned nr_pages)
1150 {
1151         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1152         struct fscrypt_ctx *ctx = NULL;
1153         struct bio *bio;
1154
1155         if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode)) {
1156                 ctx = fscrypt_get_ctx(inode, GFP_NOFS);
1157                 if (IS_ERR(ctx))
1158                         return ERR_CAST(ctx);
1159
1160                 /* wait the page to be moved by cleaning */
1161                 f2fs_wait_on_encrypted_page_writeback(sbi, blkaddr);
1162         }
1163
1164         bio = bio_alloc(GFP_KERNEL, min_t(int, nr_pages, BIO_MAX_PAGES));
1165         if (!bio) {
1166                 if (ctx)
1167                         fscrypt_release_ctx(ctx);
1168                 return ERR_PTR(-ENOMEM);
1169         }
1170         f2fs_target_device(sbi, blkaddr, bio);
1171         bio->bi_end_io = f2fs_read_end_io;
1172         bio->bi_private = ctx;
1173
1174         return bio;
1175 }
1176
1177 /*
1178  * This function was originally taken from fs/mpage.c, and customized for f2fs.
1179  * Major change was from block_size == page_size in f2fs by default.
1180  */
1181 static int f2fs_mpage_readpages(struct address_space *mapping,
1182                         struct list_head *pages, struct page *page,
1183                         unsigned nr_pages)
1184 {
1185         struct bio *bio = NULL;
1186         unsigned page_idx;
1187         sector_t last_block_in_bio = 0;
1188         struct inode *inode = mapping->host;
1189         const unsigned blkbits = inode->i_blkbits;
1190         const unsigned blocksize = 1 << blkbits;
1191         sector_t block_in_file;
1192         sector_t last_block;
1193         sector_t last_block_in_file;
1194         sector_t block_nr;
1195         struct f2fs_map_blocks map;
1196
1197         map.m_pblk = 0;
1198         map.m_lblk = 0;
1199         map.m_len = 0;
1200         map.m_flags = 0;
1201         map.m_next_pgofs = NULL;
1202
1203         for (page_idx = 0; nr_pages; page_idx++, nr_pages--) {
1204
1205                 if (pages) {
1206                         page = list_last_entry(pages, struct page, lru);
1207
1208                         prefetchw(&page->flags);
1209                         list_del(&page->lru);
1210                         if (add_to_page_cache_lru(page, mapping,
1211                                                   page->index,
1212                                                   readahead_gfp_mask(mapping)))
1213                                 goto next_page;
1214                 }
1215
1216                 block_in_file = (sector_t)page->index;
1217                 last_block = block_in_file + nr_pages;
1218                 last_block_in_file = (i_size_read(inode) + blocksize - 1) >>
1219                                                                 blkbits;
1220                 if (last_block > last_block_in_file)
1221                         last_block = last_block_in_file;
1222
1223                 /*
1224                  * Map blocks using the previous result first.
1225                  */
1226                 if ((map.m_flags & F2FS_MAP_MAPPED) &&
1227                                 block_in_file > map.m_lblk &&
1228                                 block_in_file < (map.m_lblk + map.m_len))
1229                         goto got_it;
1230
1231                 /*
1232                  * Then do more f2fs_map_blocks() calls until we are
1233                  * done with this page.
1234                  */
1235                 map.m_flags = 0;
1236
1237                 if (block_in_file < last_block) {
1238                         map.m_lblk = block_in_file;
1239                         map.m_len = last_block - block_in_file;
1240
1241                         if (f2fs_map_blocks(inode, &map, 0,
1242                                                 F2FS_GET_BLOCK_READ))
1243                                 goto set_error_page;
1244                 }
1245 got_it:
1246                 if ((map.m_flags & F2FS_MAP_MAPPED)) {
1247                         block_nr = map.m_pblk + block_in_file - map.m_lblk;
1248                         SetPageMappedToDisk(page);
1249
1250                         if (!PageUptodate(page) && !cleancache_get_page(page)) {
1251                                 SetPageUptodate(page);
1252                                 goto confused;
1253                         }
1254                 } else {
1255                         zero_user_segment(page, 0, PAGE_SIZE);
1256                         if (!PageUptodate(page))
1257                                 SetPageUptodate(page);
1258                         unlock_page(page);
1259                         goto next_page;
1260                 }
1261
1262                 /*
1263                  * This page will go to BIO.  Do we need to send this
1264                  * BIO off first?
1265                  */
1266                 if (bio && (last_block_in_bio != block_nr - 1 ||
1267                         !__same_bdev(F2FS_I_SB(inode), block_nr, bio))) {
1268 submit_and_realloc:
1269                         __submit_bio(F2FS_I_SB(inode), bio, DATA);
1270                         bio = NULL;
1271                 }
1272                 if (bio == NULL) {
1273                         bio = f2fs_grab_bio(inode, block_nr, nr_pages);
1274                         if (IS_ERR(bio)) {
1275                                 bio = NULL;
1276                                 goto set_error_page;
1277                         }
1278                         bio_set_op_attrs(bio, REQ_OP_READ, 0);
1279                 }
1280
1281                 if (bio_add_page(bio, page, blocksize, 0) < blocksize)
1282                         goto submit_and_realloc;
1283
1284                 last_block_in_bio = block_nr;
1285                 goto next_page;
1286 set_error_page:
1287                 SetPageError(page);
1288                 zero_user_segment(page, 0, PAGE_SIZE);
1289                 unlock_page(page);
1290                 goto next_page;
1291 confused:
1292                 if (bio) {
1293                         __submit_bio(F2FS_I_SB(inode), bio, DATA);
1294                         bio = NULL;
1295                 }
1296                 unlock_page(page);
1297 next_page:
1298                 if (pages)
1299                         put_page(page);
1300         }
1301         BUG_ON(pages && !list_empty(pages));
1302         if (bio)
1303                 __submit_bio(F2FS_I_SB(inode), bio, DATA);
1304         return 0;
1305 }
1306
1307 static int f2fs_read_data_page(struct file *file, struct page *page)
1308 {
1309         struct inode *inode = page->mapping->host;
1310         int ret = -EAGAIN;
1311
1312         trace_f2fs_readpage(page, DATA);
1313
1314         /* If the file has inline data, try to read it directly */
1315         if (f2fs_has_inline_data(inode))
1316                 ret = f2fs_read_inline_data(inode, page);
1317         if (ret == -EAGAIN)
1318                 ret = f2fs_mpage_readpages(page->mapping, NULL, page, 1);
1319         return ret;
1320 }
1321
1322 static int f2fs_read_data_pages(struct file *file,
1323                         struct address_space *mapping,
1324                         struct list_head *pages, unsigned nr_pages)
1325 {
1326         struct inode *inode = file->f_mapping->host;
1327         struct page *page = list_last_entry(pages, struct page, lru);
1328
1329         trace_f2fs_readpages(inode, page, nr_pages);
1330
1331         /* If the file has inline data, skip readpages */
1332         if (f2fs_has_inline_data(inode))
1333                 return 0;
1334
1335         return f2fs_mpage_readpages(mapping, pages, NULL, nr_pages);
1336 }
1337
1338 static int encrypt_one_page(struct f2fs_io_info *fio)
1339 {
1340         struct inode *inode = fio->page->mapping->host;
1341         gfp_t gfp_flags = GFP_NOFS;
1342
1343         if (!f2fs_encrypted_inode(inode) || !S_ISREG(inode->i_mode))
1344                 return 0;
1345
1346         /* wait for GCed encrypted page writeback */
1347         f2fs_wait_on_encrypted_page_writeback(fio->sbi, fio->old_blkaddr);
1348
1349 retry_encrypt:
1350         fio->encrypted_page = fscrypt_encrypt_page(inode, fio->page,
1351                         PAGE_SIZE, 0, fio->page->index, gfp_flags);
1352         if (!IS_ERR(fio->encrypted_page))
1353                 return 0;
1354
1355         /* flush pending IOs and wait for a while in the ENOMEM case */
1356         if (PTR_ERR(fio->encrypted_page) == -ENOMEM) {
1357                 f2fs_flush_merged_writes(fio->sbi);
1358                 congestion_wait(BLK_RW_ASYNC, HZ/50);
1359                 gfp_flags |= __GFP_NOFAIL;
1360                 goto retry_encrypt;
1361         }
1362         return PTR_ERR(fio->encrypted_page);
1363 }
1364
1365 static inline bool need_inplace_update(struct f2fs_io_info *fio)
1366 {
1367         struct inode *inode = fio->page->mapping->host;
1368
1369         if (S_ISDIR(inode->i_mode) || f2fs_is_atomic_file(inode))
1370                 return false;
1371         if (is_cold_data(fio->page))
1372                 return false;
1373         if (IS_ATOMIC_WRITTEN_PAGE(fio->page))
1374                 return false;
1375
1376         return need_inplace_update_policy(inode, fio);
1377 }
1378
1379 static inline bool valid_ipu_blkaddr(struct f2fs_io_info *fio)
1380 {
1381         if (fio->old_blkaddr == NEW_ADDR)
1382                 return false;
1383         if (fio->old_blkaddr == NULL_ADDR)
1384                 return false;
1385         return true;
1386 }
1387
1388 int do_write_data_page(struct f2fs_io_info *fio)
1389 {
1390         struct page *page = fio->page;
1391         struct inode *inode = page->mapping->host;
1392         struct dnode_of_data dn;
1393         struct extent_info ei = {0,0,0};
1394         bool ipu_force = false;
1395         int err = 0;
1396
1397         set_new_dnode(&dn, inode, NULL, NULL, 0);
1398         if (need_inplace_update(fio) &&
1399                         f2fs_lookup_extent_cache(inode, page->index, &ei)) {
1400                 fio->old_blkaddr = ei.blk + page->index - ei.fofs;
1401
1402                 if (valid_ipu_blkaddr(fio)) {
1403                         ipu_force = true;
1404                         fio->need_lock = LOCK_DONE;
1405                         goto got_it;
1406                 }
1407         }
1408
1409         /* Deadlock due to between page->lock and f2fs_lock_op */
1410         if (fio->need_lock == LOCK_REQ && !f2fs_trylock_op(fio->sbi))
1411                 return -EAGAIN;
1412
1413         err = get_dnode_of_data(&dn, page->index, LOOKUP_NODE);
1414         if (err)
1415                 goto out;
1416
1417         fio->old_blkaddr = dn.data_blkaddr;
1418
1419         /* This page is already truncated */
1420         if (fio->old_blkaddr == NULL_ADDR) {
1421                 ClearPageUptodate(page);
1422                 goto out_writepage;
1423         }
1424 got_it:
1425         /*
1426          * If current allocation needs SSR,
1427          * it had better in-place writes for updated data.
1428          */
1429         if (ipu_force || (valid_ipu_blkaddr(fio) && need_inplace_update(fio))) {
1430                 err = encrypt_one_page(fio);
1431                 if (err)
1432                         goto out_writepage;
1433
1434                 set_page_writeback(page);
1435                 f2fs_put_dnode(&dn);
1436                 if (fio->need_lock == LOCK_REQ)
1437                         f2fs_unlock_op(fio->sbi);
1438                 err = rewrite_data_page(fio);
1439                 trace_f2fs_do_write_data_page(fio->page, IPU);
1440                 set_inode_flag(inode, FI_UPDATE_WRITE);
1441                 return err;
1442         }
1443
1444         if (fio->need_lock == LOCK_RETRY) {
1445                 if (!f2fs_trylock_op(fio->sbi)) {
1446                         err = -EAGAIN;
1447                         goto out_writepage;
1448                 }
1449                 fio->need_lock = LOCK_REQ;
1450         }
1451
1452         err = encrypt_one_page(fio);
1453         if (err)
1454                 goto out_writepage;
1455
1456         set_page_writeback(page);
1457
1458         /* LFS mode write path */
1459         write_data_page(&dn, fio);
1460         trace_f2fs_do_write_data_page(page, OPU);
1461         set_inode_flag(inode, FI_APPEND_WRITE);
1462         if (page->index == 0)
1463                 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
1464 out_writepage:
1465         f2fs_put_dnode(&dn);
1466 out:
1467         if (fio->need_lock == LOCK_REQ)
1468                 f2fs_unlock_op(fio->sbi);
1469         return err;
1470 }
1471
1472 static int __write_data_page(struct page *page, bool *submitted,
1473                                 struct writeback_control *wbc)
1474 {
1475         struct inode *inode = page->mapping->host;
1476         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1477         loff_t i_size = i_size_read(inode);
1478         const pgoff_t end_index = ((unsigned long long) i_size)
1479                                                         >> PAGE_SHIFT;
1480         loff_t psize = (page->index + 1) << PAGE_SHIFT;
1481         unsigned offset = 0;
1482         bool need_balance_fs = false;
1483         int err = 0;
1484         struct f2fs_io_info fio = {
1485                 .sbi = sbi,
1486                 .type = DATA,
1487                 .op = REQ_OP_WRITE,
1488                 .op_flags = wbc_to_write_flags(wbc),
1489                 .old_blkaddr = NULL_ADDR,
1490                 .page = page,
1491                 .encrypted_page = NULL,
1492                 .submitted = false,
1493                 .need_lock = LOCK_RETRY,
1494         };
1495
1496         trace_f2fs_writepage(page, DATA);
1497
1498         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1499                 goto redirty_out;
1500
1501         if (page->index < end_index)
1502                 goto write;
1503
1504         /*
1505          * If the offset is out-of-range of file size,
1506          * this page does not have to be written to disk.
1507          */
1508         offset = i_size & (PAGE_SIZE - 1);
1509         if ((page->index >= end_index + 1) || !offset)
1510                 goto out;
1511
1512         zero_user_segment(page, offset, PAGE_SIZE);
1513 write:
1514         if (f2fs_is_drop_cache(inode))
1515                 goto out;
1516         /* we should not write 0'th page having journal header */
1517         if (f2fs_is_volatile_file(inode) && (!page->index ||
1518                         (!wbc->for_reclaim &&
1519                         available_free_memory(sbi, BASE_CHECK))))
1520                 goto redirty_out;
1521
1522         /* we should bypass data pages to proceed the kworkder jobs */
1523         if (unlikely(f2fs_cp_error(sbi))) {
1524                 mapping_set_error(page->mapping, -EIO);
1525                 goto out;
1526         }
1527
1528         /* Dentry blocks are controlled by checkpoint */
1529         if (S_ISDIR(inode->i_mode)) {
1530                 fio.need_lock = LOCK_DONE;
1531                 err = do_write_data_page(&fio);
1532                 goto done;
1533         }
1534
1535         if (!wbc->for_reclaim)
1536                 need_balance_fs = true;
1537         else if (has_not_enough_free_secs(sbi, 0, 0))
1538                 goto redirty_out;
1539         else
1540                 set_inode_flag(inode, FI_HOT_DATA);
1541
1542         err = -EAGAIN;
1543         if (f2fs_has_inline_data(inode)) {
1544                 err = f2fs_write_inline_data(inode, page);
1545                 if (!err)
1546                         goto out;
1547         }
1548
1549         if (err == -EAGAIN) {
1550                 err = do_write_data_page(&fio);
1551                 if (err == -EAGAIN) {
1552                         fio.need_lock = LOCK_REQ;
1553                         err = do_write_data_page(&fio);
1554                 }
1555         }
1556         if (F2FS_I(inode)->last_disk_size < psize)
1557                 F2FS_I(inode)->last_disk_size = psize;
1558
1559 done:
1560         if (err && err != -ENOENT)
1561                 goto redirty_out;
1562
1563 out:
1564         inode_dec_dirty_pages(inode);
1565         if (err)
1566                 ClearPageUptodate(page);
1567
1568         if (wbc->for_reclaim) {
1569                 f2fs_submit_merged_write_cond(sbi, inode, 0, page->index, DATA);
1570                 clear_inode_flag(inode, FI_HOT_DATA);
1571                 remove_dirty_inode(inode);
1572                 submitted = NULL;
1573         }
1574
1575         unlock_page(page);
1576         if (!S_ISDIR(inode->i_mode))
1577                 f2fs_balance_fs(sbi, need_balance_fs);
1578
1579         if (unlikely(f2fs_cp_error(sbi))) {
1580                 f2fs_submit_merged_write(sbi, DATA);
1581                 submitted = NULL;
1582         }
1583
1584         if (submitted)
1585                 *submitted = fio.submitted;
1586
1587         return 0;
1588
1589 redirty_out:
1590         redirty_page_for_writepage(wbc, page);
1591         if (!err)
1592                 return AOP_WRITEPAGE_ACTIVATE;
1593         unlock_page(page);
1594         return err;
1595 }
1596
1597 static int f2fs_write_data_page(struct page *page,
1598                                         struct writeback_control *wbc)
1599 {
1600         return __write_data_page(page, NULL, wbc);
1601 }
1602
1603 /*
1604  * This function was copied from write_cche_pages from mm/page-writeback.c.
1605  * The major change is making write step of cold data page separately from
1606  * warm/hot data page.
1607  */
1608 static int f2fs_write_cache_pages(struct address_space *mapping,
1609                                         struct writeback_control *wbc)
1610 {
1611         int ret = 0;
1612         int done = 0;
1613         struct pagevec pvec;
1614         int nr_pages;
1615         pgoff_t uninitialized_var(writeback_index);
1616         pgoff_t index;
1617         pgoff_t end;            /* Inclusive */
1618         pgoff_t done_index;
1619         pgoff_t last_idx = ULONG_MAX;
1620         int cycled;
1621         int range_whole = 0;
1622         int tag;
1623
1624         pagevec_init(&pvec, 0);
1625
1626         if (get_dirty_pages(mapping->host) <=
1627                                 SM_I(F2FS_M_SB(mapping))->min_hot_blocks)
1628                 set_inode_flag(mapping->host, FI_HOT_DATA);
1629         else
1630                 clear_inode_flag(mapping->host, FI_HOT_DATA);
1631
1632         if (wbc->range_cyclic) {
1633                 writeback_index = mapping->writeback_index; /* prev offset */
1634                 index = writeback_index;
1635                 if (index == 0)
1636                         cycled = 1;
1637                 else
1638                         cycled = 0;
1639                 end = -1;
1640         } else {
1641                 index = wbc->range_start >> PAGE_SHIFT;
1642                 end = wbc->range_end >> PAGE_SHIFT;
1643                 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
1644                         range_whole = 1;
1645                 cycled = 1; /* ignore range_cyclic tests */
1646         }
1647         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1648                 tag = PAGECACHE_TAG_TOWRITE;
1649         else
1650                 tag = PAGECACHE_TAG_DIRTY;
1651 retry:
1652         if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
1653                 tag_pages_for_writeback(mapping, index, end);
1654         done_index = index;
1655         while (!done && (index <= end)) {
1656                 int i;
1657
1658                 nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
1659                               min(end - index, (pgoff_t)PAGEVEC_SIZE - 1) + 1);
1660                 if (nr_pages == 0)
1661                         break;
1662
1663                 for (i = 0; i < nr_pages; i++) {
1664                         struct page *page = pvec.pages[i];
1665                         bool submitted = false;
1666
1667                         if (page->index > end) {
1668                                 done = 1;
1669                                 break;
1670                         }
1671
1672                         done_index = page->index;
1673 retry_write:
1674                         lock_page(page);
1675
1676                         if (unlikely(page->mapping != mapping)) {
1677 continue_unlock:
1678                                 unlock_page(page);
1679                                 continue;
1680                         }
1681
1682                         if (!PageDirty(page)) {
1683                                 /* someone wrote it for us */
1684                                 goto continue_unlock;
1685                         }
1686
1687                         if (PageWriteback(page)) {
1688                                 if (wbc->sync_mode != WB_SYNC_NONE)
1689                                         f2fs_wait_on_page_writeback(page,
1690                                                                 DATA, true);
1691                                 else
1692                                         goto continue_unlock;
1693                         }
1694
1695                         BUG_ON(PageWriteback(page));
1696                         if (!clear_page_dirty_for_io(page))
1697                                 goto continue_unlock;
1698
1699                         ret = __write_data_page(page, &submitted, wbc);
1700                         if (unlikely(ret)) {
1701                                 /*
1702                                  * keep nr_to_write, since vfs uses this to
1703                                  * get # of written pages.
1704                                  */
1705                                 if (ret == AOP_WRITEPAGE_ACTIVATE) {
1706                                         unlock_page(page);
1707                                         ret = 0;
1708                                         continue;
1709                                 } else if (ret == -EAGAIN) {
1710                                         ret = 0;
1711                                         if (wbc->sync_mode == WB_SYNC_ALL) {
1712                                                 cond_resched();
1713                                                 congestion_wait(BLK_RW_ASYNC,
1714                                                                         HZ/50);
1715                                                 goto retry_write;
1716                                         }
1717                                         continue;
1718                                 }
1719                                 done_index = page->index + 1;
1720                                 done = 1;
1721                                 break;
1722                         } else if (submitted) {
1723                                 last_idx = page->index;
1724                         }
1725
1726                         /* give a priority to WB_SYNC threads */
1727                         if ((atomic_read(&F2FS_M_SB(mapping)->wb_sync_req) ||
1728                                         --wbc->nr_to_write <= 0) &&
1729                                         wbc->sync_mode == WB_SYNC_NONE) {
1730                                 done = 1;
1731                                 break;
1732                         }
1733                 }
1734                 pagevec_release(&pvec);
1735                 cond_resched();
1736         }
1737
1738         if (!cycled && !done) {
1739                 cycled = 1;
1740                 index = 0;
1741                 end = writeback_index - 1;
1742                 goto retry;
1743         }
1744         if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
1745                 mapping->writeback_index = done_index;
1746
1747         if (last_idx != ULONG_MAX)
1748                 f2fs_submit_merged_write_cond(F2FS_M_SB(mapping), mapping->host,
1749                                                 0, last_idx, DATA);
1750
1751         return ret;
1752 }
1753
1754 static int f2fs_write_data_pages(struct address_space *mapping,
1755                             struct writeback_control *wbc)
1756 {
1757         struct inode *inode = mapping->host;
1758         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1759         struct blk_plug plug;
1760         int ret;
1761
1762         /* deal with chardevs and other special file */
1763         if (!mapping->a_ops->writepage)
1764                 return 0;
1765
1766         /* skip writing if there is no dirty page in this inode */
1767         if (!get_dirty_pages(inode) && wbc->sync_mode == WB_SYNC_NONE)
1768                 return 0;
1769
1770         /* during POR, we don't need to trigger writepage at all. */
1771         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
1772                 goto skip_write;
1773
1774         if (S_ISDIR(inode->i_mode) && wbc->sync_mode == WB_SYNC_NONE &&
1775                         get_dirty_pages(inode) < nr_pages_to_skip(sbi, DATA) &&
1776                         available_free_memory(sbi, DIRTY_DENTS))
1777                 goto skip_write;
1778
1779         /* skip writing during file defragment */
1780         if (is_inode_flag_set(inode, FI_DO_DEFRAG))
1781                 goto skip_write;
1782
1783         trace_f2fs_writepages(mapping->host, wbc, DATA);
1784
1785         /* to avoid spliting IOs due to mixed WB_SYNC_ALL and WB_SYNC_NONE */
1786         if (wbc->sync_mode == WB_SYNC_ALL)
1787                 atomic_inc(&sbi->wb_sync_req);
1788         else if (atomic_read(&sbi->wb_sync_req))
1789                 goto skip_write;
1790
1791         blk_start_plug(&plug);
1792         ret = f2fs_write_cache_pages(mapping, wbc);
1793         blk_finish_plug(&plug);
1794
1795         if (wbc->sync_mode == WB_SYNC_ALL)
1796                 atomic_dec(&sbi->wb_sync_req);
1797         /*
1798          * if some pages were truncated, we cannot guarantee its mapping->host
1799          * to detect pending bios.
1800          */
1801
1802         remove_dirty_inode(inode);
1803         return ret;
1804
1805 skip_write:
1806         wbc->pages_skipped += get_dirty_pages(inode);
1807         trace_f2fs_writepages(mapping->host, wbc, DATA);
1808         return 0;
1809 }
1810
1811 static void f2fs_write_failed(struct address_space *mapping, loff_t to)
1812 {
1813         struct inode *inode = mapping->host;
1814         loff_t i_size = i_size_read(inode);
1815
1816         if (to > i_size) {
1817                 down_write(&F2FS_I(inode)->i_mmap_sem);
1818                 truncate_pagecache(inode, i_size);
1819                 truncate_blocks(inode, i_size, true);
1820                 up_write(&F2FS_I(inode)->i_mmap_sem);
1821         }
1822 }
1823
1824 static int prepare_write_begin(struct f2fs_sb_info *sbi,
1825                         struct page *page, loff_t pos, unsigned len,
1826                         block_t *blk_addr, bool *node_changed)
1827 {
1828         struct inode *inode = page->mapping->host;
1829         pgoff_t index = page->index;
1830         struct dnode_of_data dn;
1831         struct page *ipage;
1832         bool locked = false;
1833         struct extent_info ei = {0,0,0};
1834         int err = 0;
1835
1836         /*
1837          * we already allocated all the blocks, so we don't need to get
1838          * the block addresses when there is no need to fill the page.
1839          */
1840         if (!f2fs_has_inline_data(inode) && len == PAGE_SIZE &&
1841                         !is_inode_flag_set(inode, FI_NO_PREALLOC))
1842                 return 0;
1843
1844         if (f2fs_has_inline_data(inode) ||
1845                         (pos & PAGE_MASK) >= i_size_read(inode)) {
1846                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, true);
1847                 locked = true;
1848         }
1849 restart:
1850         /* check inline_data */
1851         ipage = get_node_page(sbi, inode->i_ino);
1852         if (IS_ERR(ipage)) {
1853                 err = PTR_ERR(ipage);
1854                 goto unlock_out;
1855         }
1856
1857         set_new_dnode(&dn, inode, ipage, ipage, 0);
1858
1859         if (f2fs_has_inline_data(inode)) {
1860                 if (pos + len <= MAX_INLINE_DATA) {
1861                         read_inline_data(page, ipage);
1862                         set_inode_flag(inode, FI_DATA_EXIST);
1863                         if (inode->i_nlink)
1864                                 set_inline_node(ipage);
1865                 } else {
1866                         err = f2fs_convert_inline_page(&dn, page);
1867                         if (err)
1868                                 goto out;
1869                         if (dn.data_blkaddr == NULL_ADDR)
1870                                 err = f2fs_get_block(&dn, index);
1871                 }
1872         } else if (locked) {
1873                 err = f2fs_get_block(&dn, index);
1874         } else {
1875                 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
1876                         dn.data_blkaddr = ei.blk + index - ei.fofs;
1877                 } else {
1878                         /* hole case */
1879                         err = get_dnode_of_data(&dn, index, LOOKUP_NODE);
1880                         if (err || dn.data_blkaddr == NULL_ADDR) {
1881                                 f2fs_put_dnode(&dn);
1882                                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO,
1883                                                                 true);
1884                                 locked = true;
1885                                 goto restart;
1886                         }
1887                 }
1888         }
1889
1890         /* convert_inline_page can make node_changed */
1891         *blk_addr = dn.data_blkaddr;
1892         *node_changed = dn.node_changed;
1893 out:
1894         f2fs_put_dnode(&dn);
1895 unlock_out:
1896         if (locked)
1897                 __do_map_lock(sbi, F2FS_GET_BLOCK_PRE_AIO, false);
1898         return err;
1899 }
1900
1901 static int f2fs_write_begin(struct file *file, struct address_space *mapping,
1902                 loff_t pos, unsigned len, unsigned flags,
1903                 struct page **pagep, void **fsdata)
1904 {
1905         struct inode *inode = mapping->host;
1906         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1907         struct page *page = NULL;
1908         pgoff_t index = ((unsigned long long) pos) >> PAGE_SHIFT;
1909         bool need_balance = false;
1910         block_t blkaddr = NULL_ADDR;
1911         int err = 0;
1912
1913         trace_f2fs_write_begin(inode, pos, len, flags);
1914
1915         /*
1916          * We should check this at this moment to avoid deadlock on inode page
1917          * and #0 page. The locking rule for inline_data conversion should be:
1918          * lock_page(page #0) -> lock_page(inode_page)
1919          */
1920         if (index != 0) {
1921                 err = f2fs_convert_inline_inode(inode);
1922                 if (err)
1923                         goto fail;
1924         }
1925 repeat:
1926         /*
1927          * Do not use grab_cache_page_write_begin() to avoid deadlock due to
1928          * wait_for_stable_page. Will wait that below with our IO control.
1929          */
1930         page = pagecache_get_page(mapping, index,
1931                                 FGP_LOCK | FGP_WRITE | FGP_CREAT, GFP_NOFS);
1932         if (!page) {
1933                 err = -ENOMEM;
1934                 goto fail;
1935         }
1936
1937         *pagep = page;
1938
1939         err = prepare_write_begin(sbi, page, pos, len,
1940                                         &blkaddr, &need_balance);
1941         if (err)
1942                 goto fail;
1943
1944         if (need_balance && has_not_enough_free_secs(sbi, 0, 0)) {
1945                 unlock_page(page);
1946                 f2fs_balance_fs(sbi, true);
1947                 lock_page(page);
1948                 if (page->mapping != mapping) {
1949                         /* The page got truncated from under us */
1950                         f2fs_put_page(page, 1);
1951                         goto repeat;
1952                 }
1953         }
1954
1955         f2fs_wait_on_page_writeback(page, DATA, false);
1956
1957         /* wait for GCed encrypted page writeback */
1958         if (f2fs_encrypted_inode(inode) && S_ISREG(inode->i_mode))
1959                 f2fs_wait_on_encrypted_page_writeback(sbi, blkaddr);
1960
1961         if (len == PAGE_SIZE || PageUptodate(page))
1962                 return 0;
1963
1964         if (!(pos & (PAGE_SIZE - 1)) && (pos + len) >= i_size_read(inode)) {
1965                 zero_user_segment(page, len, PAGE_SIZE);
1966                 return 0;
1967         }
1968
1969         if (blkaddr == NEW_ADDR) {
1970                 zero_user_segment(page, 0, PAGE_SIZE);
1971                 SetPageUptodate(page);
1972         } else {
1973                 struct bio *bio;
1974
1975                 bio = f2fs_grab_bio(inode, blkaddr, 1);
1976                 if (IS_ERR(bio)) {
1977                         err = PTR_ERR(bio);
1978                         goto fail;
1979                 }
1980                 bio->bi_opf = REQ_OP_READ;
1981                 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
1982                         bio_put(bio);
1983                         err = -EFAULT;
1984                         goto fail;
1985                 }
1986
1987                 __submit_bio(sbi, bio, DATA);
1988
1989                 lock_page(page);
1990                 if (unlikely(page->mapping != mapping)) {
1991                         f2fs_put_page(page, 1);
1992                         goto repeat;
1993                 }
1994                 if (unlikely(!PageUptodate(page))) {
1995                         err = -EIO;
1996                         goto fail;
1997                 }
1998         }
1999         return 0;
2000
2001 fail:
2002         f2fs_put_page(page, 1);
2003         f2fs_write_failed(mapping, pos + len);
2004         return err;
2005 }
2006
2007 static int f2fs_write_end(struct file *file,
2008                         struct address_space *mapping,
2009                         loff_t pos, unsigned len, unsigned copied,
2010                         struct page *page, void *fsdata)
2011 {
2012         struct inode *inode = page->mapping->host;
2013
2014         trace_f2fs_write_end(inode, pos, len, copied);
2015
2016         /*
2017          * This should be come from len == PAGE_SIZE, and we expect copied
2018          * should be PAGE_SIZE. Otherwise, we treat it with zero copied and
2019          * let generic_perform_write() try to copy data again through copied=0.
2020          */
2021         if (!PageUptodate(page)) {
2022                 if (unlikely(copied != len))
2023                         copied = 0;
2024                 else
2025                         SetPageUptodate(page);
2026         }
2027         if (!copied)
2028                 goto unlock_out;
2029
2030         set_page_dirty(page);
2031
2032         if (pos + copied > i_size_read(inode))
2033                 f2fs_i_size_write(inode, pos + copied);
2034 unlock_out:
2035         f2fs_put_page(page, 1);
2036         f2fs_update_time(F2FS_I_SB(inode), REQ_TIME);
2037         return copied;
2038 }
2039
2040 static int check_direct_IO(struct inode *inode, struct iov_iter *iter,
2041                            loff_t offset)
2042 {
2043         unsigned blocksize_mask = inode->i_sb->s_blocksize - 1;
2044
2045         if (offset & blocksize_mask)
2046                 return -EINVAL;
2047
2048         if (iov_iter_alignment(iter) & blocksize_mask)
2049                 return -EINVAL;
2050
2051         return 0;
2052 }
2053
2054 static ssize_t f2fs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2055 {
2056         struct address_space *mapping = iocb->ki_filp->f_mapping;
2057         struct inode *inode = mapping->host;
2058         size_t count = iov_iter_count(iter);
2059         loff_t offset = iocb->ki_pos;
2060         int rw = iov_iter_rw(iter);
2061         int err;
2062
2063         err = check_direct_IO(inode, iter, offset);
2064         if (err)
2065                 return err;
2066
2067         if (__force_buffered_io(inode, rw))
2068                 return 0;
2069
2070         trace_f2fs_direct_IO_enter(inode, offset, count, rw);
2071
2072         down_read(&F2FS_I(inode)->dio_rwsem[rw]);
2073         err = blockdev_direct_IO(iocb, inode, iter, get_data_block_dio);
2074         up_read(&F2FS_I(inode)->dio_rwsem[rw]);
2075
2076         if (rw == WRITE) {
2077                 if (err > 0)
2078                         set_inode_flag(inode, FI_UPDATE_WRITE);
2079                 else if (err < 0)
2080                         f2fs_write_failed(mapping, offset + count);
2081         }
2082
2083         trace_f2fs_direct_IO_exit(inode, offset, count, rw, err);
2084
2085         return err;
2086 }
2087
2088 void f2fs_invalidate_page(struct page *page, unsigned int offset,
2089                                                         unsigned int length)
2090 {
2091         struct inode *inode = page->mapping->host;
2092         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
2093
2094         if (inode->i_ino >= F2FS_ROOT_INO(sbi) &&
2095                 (offset % PAGE_SIZE || length != PAGE_SIZE))
2096                 return;
2097
2098         if (PageDirty(page)) {
2099                 if (inode->i_ino == F2FS_META_INO(sbi)) {
2100                         dec_page_count(sbi, F2FS_DIRTY_META);
2101                 } else if (inode->i_ino == F2FS_NODE_INO(sbi)) {
2102                         dec_page_count(sbi, F2FS_DIRTY_NODES);
2103                 } else {
2104                         inode_dec_dirty_pages(inode);
2105                         remove_dirty_inode(inode);
2106                 }
2107         }
2108
2109         /* This is atomic written page, keep Private */
2110         if (IS_ATOMIC_WRITTEN_PAGE(page))
2111                 return drop_inmem_page(inode, page);
2112
2113         set_page_private(page, 0);
2114         ClearPagePrivate(page);
2115 }
2116
2117 int f2fs_release_page(struct page *page, gfp_t wait)
2118 {
2119         /* If this is dirty page, keep PagePrivate */
2120         if (PageDirty(page))
2121                 return 0;
2122
2123         /* This is atomic written page, keep Private */
2124         if (IS_ATOMIC_WRITTEN_PAGE(page))
2125                 return 0;
2126
2127         set_page_private(page, 0);
2128         ClearPagePrivate(page);
2129         return 1;
2130 }
2131
2132 /*
2133  * This was copied from __set_page_dirty_buffers which gives higher performance
2134  * in very high speed storages. (e.g., pmem)
2135  */
2136 void f2fs_set_page_dirty_nobuffers(struct page *page)
2137 {
2138         struct address_space *mapping = page->mapping;
2139         unsigned long flags;
2140
2141         if (unlikely(!mapping))
2142                 return;
2143
2144         spin_lock(&mapping->private_lock);
2145         lock_page_memcg(page);
2146         SetPageDirty(page);
2147         spin_unlock(&mapping->private_lock);
2148
2149         spin_lock_irqsave(&mapping->tree_lock, flags);
2150         WARN_ON_ONCE(!PageUptodate(page));
2151         account_page_dirtied(page, mapping);
2152         radix_tree_tag_set(&mapping->page_tree,
2153                         page_index(page), PAGECACHE_TAG_DIRTY);
2154         spin_unlock_irqrestore(&mapping->tree_lock, flags);
2155         unlock_page_memcg(page);
2156
2157         __mark_inode_dirty(mapping->host, I_DIRTY_PAGES);
2158         return;
2159 }
2160
2161 static int f2fs_set_data_page_dirty(struct page *page)
2162 {
2163         struct address_space *mapping = page->mapping;
2164         struct inode *inode = mapping->host;
2165
2166         trace_f2fs_set_page_dirty(page, DATA);
2167
2168         if (!PageUptodate(page))
2169                 SetPageUptodate(page);
2170
2171         if (f2fs_is_atomic_file(inode) && !f2fs_is_commit_atomic_write(inode)) {
2172                 if (!IS_ATOMIC_WRITTEN_PAGE(page)) {
2173                         register_inmem_page(inode, page);
2174                         return 1;
2175                 }
2176                 /*
2177                  * Previously, this page has been registered, we just
2178                  * return here.
2179                  */
2180                 return 0;
2181         }
2182
2183         if (!PageDirty(page)) {
2184                 f2fs_set_page_dirty_nobuffers(page);
2185                 update_dirty_page(inode, page);
2186                 return 1;
2187         }
2188         return 0;
2189 }
2190
2191 static sector_t f2fs_bmap(struct address_space *mapping, sector_t block)
2192 {
2193         struct inode *inode = mapping->host;
2194
2195         if (f2fs_has_inline_data(inode))
2196                 return 0;
2197
2198         /* make sure allocating whole blocks */
2199         if (mapping_tagged(mapping, PAGECACHE_TAG_DIRTY))
2200                 filemap_write_and_wait(mapping);
2201
2202         return generic_block_bmap(mapping, block, get_data_block_bmap);
2203 }
2204
2205 #ifdef CONFIG_MIGRATION
2206 #include <linux/migrate.h>
2207
2208 int f2fs_migrate_page(struct address_space *mapping,
2209                 struct page *newpage, struct page *page, enum migrate_mode mode)
2210 {
2211         int rc, extra_count;
2212         struct f2fs_inode_info *fi = F2FS_I(mapping->host);
2213         bool atomic_written = IS_ATOMIC_WRITTEN_PAGE(page);
2214
2215         BUG_ON(PageWriteback(page));
2216
2217         /* migrating an atomic written page is safe with the inmem_lock hold */
2218         if (atomic_written) {
2219                 if (mode != MIGRATE_SYNC)
2220                         return -EBUSY;
2221                 if (!mutex_trylock(&fi->inmem_lock))
2222                         return -EAGAIN;
2223         }
2224
2225         /*
2226          * A reference is expected if PagePrivate set when move mapping,
2227          * however F2FS breaks this for maintaining dirty page counts when
2228          * truncating pages. So here adjusting the 'extra_count' make it work.
2229          */
2230         extra_count = (atomic_written ? 1 : 0) - page_has_private(page);
2231         rc = migrate_page_move_mapping(mapping, newpage,
2232                                 page, NULL, mode, extra_count);
2233         if (rc != MIGRATEPAGE_SUCCESS) {
2234                 if (atomic_written)
2235                         mutex_unlock(&fi->inmem_lock);
2236                 return rc;
2237         }
2238
2239         if (atomic_written) {
2240                 struct inmem_pages *cur;
2241                 list_for_each_entry(cur, &fi->inmem_pages, list)
2242                         if (cur->page == page) {
2243                                 cur->page = newpage;
2244                                 break;
2245                         }
2246                 mutex_unlock(&fi->inmem_lock);
2247                 put_page(page);
2248                 get_page(newpage);
2249         }
2250
2251         if (PagePrivate(page))
2252                 SetPagePrivate(newpage);
2253         set_page_private(newpage, page_private(page));
2254
2255         migrate_page_copy(newpage, page);
2256
2257         return MIGRATEPAGE_SUCCESS;
2258 }
2259 #endif
2260
2261 const struct address_space_operations f2fs_dblock_aops = {
2262         .readpage       = f2fs_read_data_page,
2263         .readpages      = f2fs_read_data_pages,
2264         .writepage      = f2fs_write_data_page,
2265         .writepages     = f2fs_write_data_pages,
2266         .write_begin    = f2fs_write_begin,
2267         .write_end      = f2fs_write_end,
2268         .set_page_dirty = f2fs_set_data_page_dirty,
2269         .invalidatepage = f2fs_invalidate_page,
2270         .releasepage    = f2fs_release_page,
2271         .direct_IO      = f2fs_direct_IO,
2272         .bmap           = f2fs_bmap,
2273 #ifdef CONFIG_MIGRATION
2274         .migratepage    = f2fs_migrate_page,
2275 #endif
2276 };