fs/fuse/file.c

   1 /*
   2   FUSE: Filesystem in Userspace
   3   Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
   4
   5   This program can be distributed under the terms of the GNU GPL.
   6   See the file COPYING.
   7 */
   8
   9 #include "fuse_i.h"
  10
  11 #include <linux/pagemap.h>
  12 #include <linux/slab.h>
  13 #include <linux/kernel.h>
  14 #include <linux/sched.h>
  15 #include <linux/sched/signal.h>
  16 #include <linux/module.h>
  17 #include <linux/swap.h>
  18 #include <linux/falloc.h>
  19 #include <linux/uio.h>
  20 #include <linux/fs.h>
  21 #include <linux/file.h>
  22
  23 static int fuse_send_open(struct fuse_mount *fm, u64 nodeid,
  24                           unsigned int open_flags, int opcode,
  25                           struct fuse_open_out *outargp)
  26 {
  27         struct fuse_open_in inarg;
  28         FUSE_ARGS(args);
  29
  30         memset(&inarg, 0, sizeof(inarg));
  31         inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
  32         if (!fm->fc->atomic_o_trunc)
  33                 inarg.flags &= ~O_TRUNC;
  34
  35         if (fm->fc->handle_killpriv_v2 &&
  36             (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) {
  37                 inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID;
  38         }
  39
  40         args.opcode = opcode;
  41         args.nodeid = nodeid;
  42         args.in_numargs = 1;
  43         args.in_args[0].size = sizeof(inarg);
  44         args.in_args[0].value = &inarg;
  45         args.out_numargs = 1;
  46         args.out_args[0].size = sizeof(*outargp);
  47         args.out_args[0].value = outargp;
  48
  49         return fuse_simple_request(fm, &args);
  50 }
  51
  52 struct fuse_release_args {
  53         struct fuse_args args;
  54         struct fuse_release_in inarg;
  55         struct inode *inode;
  56 };
  57
  58 struct fuse_file *fuse_file_alloc(struct fuse_mount *fm)
  59 {
  60         struct fuse_file *ff;
  61
  62         ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT);
  63         if (unlikely(!ff))
  64                 return NULL;
  65
  66         ff->fm = fm;
  67         ff->release_args = kzalloc(sizeof(*ff->release_args),
  68                                    GFP_KERNEL_ACCOUNT);
  69         if (!ff->release_args) {
  70                 kfree(ff);
  71                 return NULL;
  72         }
  73
  74         INIT_LIST_HEAD(&ff->write_entry);
  75         mutex_init(&ff->readdir.lock);
  76         refcount_set(&ff->count, 1);
  77         RB_CLEAR_NODE(&ff->polled_node);
  78         init_waitqueue_head(&ff->poll_wait);
  79
  80         ff->kh = atomic64_inc_return(&fm->fc->khctr);
  81
  82         return ff;
  83 }
  84
  85 void fuse_file_free(struct fuse_file *ff)
  86 {
  87         kfree(ff->release_args);
  88         mutex_destroy(&ff->readdir.lock);
  89         kfree(ff);
  90 }
  91
  92 static struct fuse_file *fuse_file_get(struct fuse_file *ff)
  93 {
  94         refcount_inc(&ff->count);
  95         return ff;
  96 }
  97
  98 static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
  99                              int error)
 100 {
 101         struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
 102
 103         iput(ra->inode);
 104         kfree(ra);
 105 }
 106
 107 static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir)
 108 {
 109         if (refcount_dec_and_test(&ff->count)) {
 110                 struct fuse_args *args = &ff->release_args->args;
 111
 112                 if (isdir ? ff->fm->fc->no_opendir : ff->fm->fc->no_open) {
 113                         /* Do nothing when client does not implement 'open' */
 114                         fuse_release_end(ff->fm, args, 0);
 115                 } else if (sync) {
 116                         fuse_simple_request(ff->fm, args);
 117                         fuse_release_end(ff->fm, args, 0);
 118                 } else {
 119                         args->end = fuse_release_end;
 120                         if (fuse_simple_background(ff->fm, args,
 121                                                    GFP_KERNEL | __GFP_NOFAIL))
 122                                 fuse_release_end(ff->fm, args, -ENOTCONN);
 123                 }
 124                 kfree(ff);
 125         }
 126 }
 127
 128 struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid,
 129                                  unsigned int open_flags, bool isdir)
 130 {
 131         struct fuse_conn *fc = fm->fc;
 132         struct fuse_file *ff;
 133         int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
 134
 135         ff = fuse_file_alloc(fm);
 136         if (!ff)
 137                 return ERR_PTR(-ENOMEM);
 138
 139         ff->fh = 0;
 140         /* Default for no-open */
 141         ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
 142         if (isdir ? !fc->no_opendir : !fc->no_open) {
 143                 struct fuse_open_out outarg;
 144                 int err;
 145
 146                 err = fuse_send_open(fm, nodeid, open_flags, opcode, &outarg);
 147                 if (!err) {
 148                         ff->fh = outarg.fh;
 149                         ff->open_flags = outarg.open_flags;
 150
 151                 } else if (err != -ENOSYS) {
 152                         fuse_file_free(ff);
 153                         return ERR_PTR(err);
 154                 } else {
 155                         if (isdir)
 156                                 fc->no_opendir = 1;
 157                         else
 158                                 fc->no_open = 1;
 159                 }
 160         }
 161
 162         if (isdir)
 163                 ff->open_flags &= ~FOPEN_DIRECT_IO;
 164
 165         ff->nodeid = nodeid;
 166
 167         return ff;
 168 }
 169
 170 int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
 171                  bool isdir)
 172 {
 173         struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir);
 174
 175         if (!IS_ERR(ff))
 176                 file->private_data = ff;
 177
 178         return PTR_ERR_OR_ZERO(ff);
 179 }
 180 EXPORT_SYMBOL_GPL(fuse_do_open);
 181
 182 static void fuse_link_write_file(struct file *file)
 183 {
 184         struct inode *inode = file_inode(file);
 185         struct fuse_inode *fi = get_fuse_inode(inode);
 186         struct fuse_file *ff = file->private_data;
 187         /*
 188          * file may be written through mmap, so chain it onto the
 189          * inodes's write_file list
 190          */
 191         spin_lock(&fi->lock);
 192         if (list_empty(&ff->write_entry))
 193                 list_add(&ff->write_entry, &fi->write_files);
 194         spin_unlock(&fi->lock);
 195 }
 196
 197 void fuse_finish_open(struct inode *inode, struct file *file)
 198 {
 199         struct fuse_file *ff = file->private_data;
 200         struct fuse_conn *fc = get_fuse_conn(inode);
 201
 202         if (ff->open_flags & FOPEN_STREAM)
 203                 stream_open(inode, file);
 204         else if (ff->open_flags & FOPEN_NONSEEKABLE)
 205                 nonseekable_open(inode, file);
 206
 207         if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
 208                 struct fuse_inode *fi = get_fuse_inode(inode);
 209
 210                 spin_lock(&fi->lock);
 211                 fi->attr_version = atomic64_inc_return(&fc->attr_version);
 212                 i_size_write(inode, 0);
 213                 spin_unlock(&fi->lock);
 214                 file_update_time(file);
 215                 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
 216         }
 217         if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
 218                 fuse_link_write_file(file);
 219 }
 220
 221 int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
 222 {
 223         struct fuse_mount *fm = get_fuse_mount(inode);
 224         struct fuse_conn *fc = fm->fc;
 225         int err;
 226         bool is_wb_truncate = (file->f_flags & O_TRUNC) &&
 227                           fc->atomic_o_trunc &&
 228                           fc->writeback_cache;
 229         bool dax_truncate = (file->f_flags & O_TRUNC) &&
 230                           fc->atomic_o_trunc && FUSE_IS_DAX(inode);
 231
 232         if (fuse_is_bad(inode))
 233                 return -EIO;
 234
 235         err = generic_file_open(inode, file);
 236         if (err)
 237                 return err;
 238
 239         if (is_wb_truncate || dax_truncate)
 240                 inode_lock(inode);
 241
 242         if (dax_truncate) {
 243                 filemap_invalidate_lock(inode->i_mapping);
 244                 err = fuse_dax_break_layouts(inode, 0, 0);
 245                 if (err)
 246                         goto out_inode_unlock;
 247         }
 248
 249         if (is_wb_truncate || dax_truncate)
 250                 fuse_set_nowrite(inode);
 251
 252         err = fuse_do_open(fm, get_node_id(inode), file, isdir);
 253         if (!err)
 254                 fuse_finish_open(inode, file);
 255
 256         if (is_wb_truncate || dax_truncate)
 257                 fuse_release_nowrite(inode);
 258         if (!err) {
 259                 struct fuse_file *ff = file->private_data;
 260
 261                 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC))
 262                         truncate_pagecache(inode, 0);
 263                 else if (!(ff->open_flags & FOPEN_KEEP_CACHE))
 264                         invalidate_inode_pages2(inode->i_mapping);
 265         }
 266         if (dax_truncate)
 267                 filemap_invalidate_unlock(inode->i_mapping);
 268 out_inode_unlock:
 269         if (is_wb_truncate || dax_truncate)
 270                 inode_unlock(inode);
 271
 272         return err;
 273 }
 274
 275 static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
 276                                  unsigned int flags, int opcode)
 277 {
 278         struct fuse_conn *fc = ff->fm->fc;
 279         struct fuse_release_args *ra = ff->release_args;
 280
 281         /* Inode is NULL on error path of fuse_create_open() */
 282         if (likely(fi)) {
 283                 spin_lock(&fi->lock);
 284                 list_del(&ff->write_entry);
 285                 spin_unlock(&fi->lock);
 286         }
 287         spin_lock(&fc->lock);
 288         if (!RB_EMPTY_NODE(&ff->polled_node))
 289                 rb_erase(&ff->polled_node, &fc->polled_files);
 290         spin_unlock(&fc->lock);
 291
 292         wake_up_interruptible_all(&ff->poll_wait);
 293
 294         ra->inarg.fh = ff->fh;
 295         ra->inarg.flags = flags;
 296         ra->args.in_numargs = 1;
 297         ra->args.in_args[0].size = sizeof(struct fuse_release_in);
 298         ra->args.in_args[0].value = &ra->inarg;
 299         ra->args.opcode = opcode;
 300         ra->args.nodeid = ff->nodeid;
 301         ra->args.force = true;
 302         ra->args.nocreds = true;
 303 }
 304
 305 void fuse_file_release(struct inode *inode, struct fuse_file *ff,
 306                        unsigned int open_flags, fl_owner_t id, bool isdir)
 307 {
 308         struct fuse_inode *fi = get_fuse_inode(inode);
 309         struct fuse_release_args *ra = ff->release_args;
 310         int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
 311
 312         fuse_prepare_release(fi, ff, open_flags, opcode);
 313
 314         if (ff->flock) {
 315                 ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
 316                 ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id);
 317         }
 318         /* Hold inode until release is finished */
 319         ra->inode = igrab(inode);
 320
 321         /*
 322          * Normally this will send the RELEASE request, however if
 323          * some asynchronous READ or WRITE requests are outstanding,
 324          * the sending will be delayed.
 325          *
 326          * Make the release synchronous if this is a fuseblk mount,
 327          * synchronous RELEASE is allowed (and desirable) in this case
 328          * because the server can be trusted not to screw up.
 329          */
 330         fuse_file_put(ff, ff->fm->fc->destroy, isdir);
 331 }
 332
 333 void fuse_release_common(struct file *file, bool isdir)
 334 {
 335         fuse_file_release(file_inode(file), file->private_data, file->f_flags,
 336                           (fl_owner_t) file, isdir);
 337 }
 338
 339 static int fuse_open(struct inode *inode, struct file *file)
 340 {
 341         return fuse_open_common(inode, file, false);
 342 }
 343
 344 static int fuse_release(struct inode *inode, struct file *file)
 345 {
 346         struct fuse_conn *fc = get_fuse_conn(inode);
 347
 348         /*
 349          * Dirty pages might remain despite write_inode_now() call from
 350          * fuse_flush() due to writes racing with the close.
 351          */
 352         if (fc->writeback_cache)
 353                 write_inode_now(inode, 1);
 354
 355         fuse_release_common(file, false);
 356
 357         /* return value is ignored by VFS */
 358         return 0;
 359 }
 360
 361 void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff,
 362                        unsigned int flags)
 363 {
 364         WARN_ON(refcount_read(&ff->count) > 1);
 365         fuse_prepare_release(fi, ff, flags, FUSE_RELEASE);
 366         /*
 367          * iput(NULL) is a no-op and since the refcount is 1 and everything's
 368          * synchronous, we are fine with not doing igrab() here"
 369          */
 370         fuse_file_put(ff, true, false);
 371 }
 372 EXPORT_SYMBOL_GPL(fuse_sync_release);
 373
 374 /*
 375  * Scramble the ID space with XTEA, so that the value of the files_struct
 376  * pointer is not exposed to userspace.
 377  */
 378 u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
 379 {
 380         u32 *k = fc->scramble_key;
 381         u64 v = (unsigned long) id;
 382         u32 v0 = v;
 383         u32 v1 = v >> 32;
 384         u32 sum = 0;
 385         int i;
 386
 387         for (i = 0; i < 32; i++) {
 388                 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
 389                 sum += 0x9E3779B9;
 390                 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
 391         }
 392
 393         return (u64) v0 + ((u64) v1 << 32);
 394 }
 395
 396 struct fuse_writepage_args {
 397         struct fuse_io_args ia;
 398         struct rb_node writepages_entry;
 399         struct list_head queue_entry;
 400         struct fuse_writepage_args *next;
 401         struct inode *inode;
 402         struct fuse_sync_bucket *bucket;
 403 };
 404
 405 static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi,
 406                                             pgoff_t idx_from, pgoff_t idx_to)
 407 {
 408         struct rb_node *n;
 409
 410         n = fi->writepages.rb_node;
 411
 412         while (n) {
 413                 struct fuse_writepage_args *wpa;
 414                 pgoff_t curr_index;
 415
 416                 wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry);
 417                 WARN_ON(get_fuse_inode(wpa->inode) != fi);
 418                 curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;
 419                 if (idx_from >= curr_index + wpa->ia.ap.num_pages)
 420                         n = n->rb_right;
 421                 else if (idx_to < curr_index)
 422                         n = n->rb_left;
 423                 else
 424                         return wpa;
 425         }
 426         return NULL;
 427 }
 428
 429 /*
 430  * Check if any page in a range is under writeback
 431  *
 432  * This is currently done by walking the list of writepage requests
 433  * for the inode, which can be pretty inefficient.
 434  */
 435 static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
 436                                    pgoff_t idx_to)
 437 {
 438         struct fuse_inode *fi = get_fuse_inode(inode);
 439         bool found;
 440
 441         spin_lock(&fi->lock);
 442         found = fuse_find_writeback(fi, idx_from, idx_to);
 443         spin_unlock(&fi->lock);
 444
 445         return found;
 446 }
 447
 448 static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
 449 {
 450         return fuse_range_is_writeback(inode, index, index);
 451 }
 452
 453 /*
 454  * Wait for page writeback to be completed.
 455  *
 456  * Since fuse doesn't rely on the VM writeback tracking, this has to
 457  * use some other means.
 458  */
 459 static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
 460 {
 461         struct fuse_inode *fi = get_fuse_inode(inode);
 462
 463         wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
 464 }
 465
 466 /*
 467  * Wait for all pending writepages on the inode to finish.
 468  *
 469  * This is currently done by blocking further writes with FUSE_NOWRITE
 470  * and waiting for all sent writes to complete.
 471  *
 472  * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
 473  * could conflict with truncation.
 474  */
 475 static void fuse_sync_writes(struct inode *inode)
 476 {
 477         fuse_set_nowrite(inode);
 478         fuse_release_nowrite(inode);
 479 }
 480
 481 struct fuse_flush_args {
 482         struct fuse_args args;
 483         struct fuse_flush_in inarg;
 484         struct work_struct work;
 485         struct file *file;
 486 };
 487
 488 static int fuse_do_flush(struct fuse_flush_args *fa)
 489 {
 490         int err;
 491         struct inode *inode = file_inode(fa->file);
 492         struct fuse_mount *fm = get_fuse_mount(inode);
 493
 494         err = write_inode_now(inode, 1);
 495         if (err)
 496                 goto out;
 497
 498         inode_lock(inode);
 499         fuse_sync_writes(inode);
 500         inode_unlock(inode);
 501
 502         err = filemap_check_errors(fa->file->f_mapping);
 503         if (err)
 504                 goto out;
 505
 506         err = 0;
 507         if (fm->fc->no_flush)
 508                 goto inval_attr_out;
 509
 510         err = fuse_simple_request(fm, &fa->args);
 511         if (err == -ENOSYS) {
 512                 fm->fc->no_flush = 1;
 513                 err = 0;
 514         }
 515
 516 inval_attr_out:
 517         /*
 518          * In memory i_blocks is not maintained by fuse, if writeback cache is
 519          * enabled, i_blocks from cached attr may not be accurate.
 520          */
 521         if (!err && fm->fc->writeback_cache)
 522                 fuse_invalidate_attr_mask(inode, STATX_BLOCKS);
 523
 524 out:
 525         fput(fa->file);
 526         kfree(fa);
 527         return err;
 528 }
 529
 530 static void fuse_flush_async(struct work_struct *work)
 531 {
 532         struct fuse_flush_args *fa = container_of(work, typeof(*fa), work);
 533
 534         fuse_do_flush(fa);
 535 }
 536
 537 static int fuse_flush(struct file *file, fl_owner_t id)
 538 {
 539         struct fuse_flush_args *fa;
 540         struct inode *inode = file_inode(file);
 541         struct fuse_mount *fm = get_fuse_mount(inode);
 542         struct fuse_file *ff = file->private_data;
 543
 544         if (fuse_is_bad(inode))
 545                 return -EIO;
 546
 547         if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache)
 548                 return 0;
 549
 550         fa = kzalloc(sizeof(*fa), GFP_KERNEL);
 551         if (!fa)
 552                 return -ENOMEM;
 553
 554         fa->inarg.fh = ff->fh;
 555         fa->inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
 556         fa->args.opcode = FUSE_FLUSH;
 557         fa->args.nodeid = get_node_id(inode);
 558         fa->args.in_numargs = 1;
 559         fa->args.in_args[0].size = sizeof(fa->inarg);
 560         fa->args.in_args[0].value = &fa->inarg;
 561         fa->args.force = true;
 562         fa->file = get_file(file);
 563
 564         /* Don't wait if the task is exiting */
 565         if (current->flags & PF_EXITING) {
 566                 INIT_WORK(&fa->work, fuse_flush_async);
 567                 schedule_work(&fa->work);
 568                 return 0;
 569         }
 570
 571         return fuse_do_flush(fa);
 572 }
 573
 574 int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
 575                       int datasync, int opcode)
 576 {
 577         struct inode *inode = file->f_mapping->host;
 578         struct fuse_mount *fm = get_fuse_mount(inode);
 579         struct fuse_file *ff = file->private_data;
 580         FUSE_ARGS(args);
 581         struct fuse_fsync_in inarg;
 582
 583         memset(&inarg, 0, sizeof(inarg));
 584         inarg.fh = ff->fh;
 585         inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
 586         args.opcode = opcode;
 587         args.nodeid = get_node_id(inode);
 588         args.in_numargs = 1;
 589         args.in_args[0].size = sizeof(inarg);
 590         args.in_args[0].value = &inarg;
 591         return fuse_simple_request(fm, &args);
 592 }
 593
 594 static int fuse_fsync(struct file *file, loff_t start, loff_t end,
 595                       int datasync)
 596 {
 597         struct inode *inode = file->f_mapping->host;
 598         struct fuse_conn *fc = get_fuse_conn(inode);
 599         int err;
 600
 601         if (fuse_is_bad(inode))
 602                 return -EIO;
 603
 604         inode_lock(inode);
 605
 606         /*
 607          * Start writeback against all dirty pages of the inode, then
 608          * wait for all outstanding writes, before sending the FSYNC
 609          * request.
 610          */
 611         err = file_write_and_wait_range(file, start, end);
 612         if (err)
 613                 goto out;
 614
 615         fuse_sync_writes(inode);
 616
 617         /*
 618          * Due to implementation of fuse writeback
 619          * file_write_and_wait_range() does not catch errors.
 620          * We have to do this directly after fuse_sync_writes()
 621          */
 622         err = file_check_and_advance_wb_err(file);
 623         if (err)
 624                 goto out;
 625
 626         err = sync_inode_metadata(inode, 1);
 627         if (err)
 628                 goto out;
 629
 630         if (fc->no_fsync)
 631                 goto out;
 632
 633         err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
 634         if (err == -ENOSYS) {
 635                 fc->no_fsync = 1;
 636                 err = 0;
 637         }
 638 out:
 639         inode_unlock(inode);
 640
 641         return err;
 642 }
 643
 644 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 645                          size_t count, int opcode)
 646 {
 647         struct fuse_file *ff = file->private_data;
 648         struct fuse_args *args = &ia->ap.args;
 649
 650         ia->read.in.fh = ff->fh;
 651         ia->read.in.offset = pos;
 652         ia->read.in.size = count;
 653         ia->read.in.flags = file->f_flags;
 654         args->opcode = opcode;
 655         args->nodeid = ff->nodeid;
 656         args->in_numargs = 1;
 657         args->in_args[0].size = sizeof(ia->read.in);
 658         args->in_args[0].value = &ia->read.in;
 659         args->out_argvar = true;
 660         args->out_numargs = 1;
 661         args->out_args[0].size = count;
 662 }
 663
 664 static void fuse_release_user_pages(struct fuse_args_pages *ap,
 665                                     bool should_dirty)
 666 {
 667         unsigned int i;
 668
 669         for (i = 0; i < ap->num_pages; i++) {
 670                 if (should_dirty)
 671                         set_page_dirty_lock(ap->pages[i]);
 672                 put_page(ap->pages[i]);
 673         }
 674 }
 675
 676 static void fuse_io_release(struct kref *kref)
 677 {
 678         kfree(container_of(kref, struct fuse_io_priv, refcnt));
 679 }
 680
 681 static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
 682 {
 683         if (io->err)
 684                 return io->err;
 685
 686         if (io->bytes >= 0 && io->write)
 687                 return -EIO;
 688
 689         return io->bytes < 0 ? io->size : io->bytes;
 690 }
 691
 692 /*
 693  * In case of short read, the caller sets 'pos' to the position of
 694  * actual end of fuse request in IO request. Otherwise, if bytes_requested
 695  * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
 696  *
 697  * An example:
 698  * User requested DIO read of 64K. It was split into two 32K fuse requests,
 699  * both submitted asynchronously. The first of them was ACKed by userspace as
 700  * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
 701  * second request was ACKed as short, e.g. only 1K was read, resulting in
 702  * pos == 33K.
 703  *
 704  * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
 705  * will be equal to the length of the longest contiguous fragment of
 706  * transferred data starting from the beginning of IO request.
 707  */
 708 static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
 709 {
 710         int left;
 711
 712         spin_lock(&io->lock);
 713         if (err)
 714                 io->err = io->err ? : err;
 715         else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
 716                 io->bytes = pos;
 717
 718         left = --io->reqs;
 719         if (!left && io->blocking)
 720                 complete(io->done);
 721         spin_unlock(&io->lock);
 722
 723         if (!left && !io->blocking) {
 724                 ssize_t res = fuse_get_res_by_io(io);
 725
 726                 if (res >= 0) {
 727                         struct inode *inode = file_inode(io->iocb->ki_filp);
 728                         struct fuse_conn *fc = get_fuse_conn(inode);
 729                         struct fuse_inode *fi = get_fuse_inode(inode);
 730
 731                         spin_lock(&fi->lock);
 732                         fi->attr_version = atomic64_inc_return(&fc->attr_version);
 733                         spin_unlock(&fi->lock);
 734                 }
 735
 736                 io->iocb->ki_complete(io->iocb, res);
 737         }
 738
 739         kref_put(&io->refcnt, fuse_io_release);
 740 }
 741
 742 static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
 743                                           unsigned int npages)
 744 {
 745         struct fuse_io_args *ia;
 746
 747         ia = kzalloc(sizeof(*ia), GFP_KERNEL);
 748         if (ia) {
 749                 ia->io = io;
 750                 ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,
 751                                                 &ia->ap.descs);
 752                 if (!ia->ap.pages) {
 753                         kfree(ia);
 754                         ia = NULL;
 755                 }
 756         }
 757         return ia;
 758 }
 759
 760 static void fuse_io_free(struct fuse_io_args *ia)
 761 {
 762         kfree(ia->ap.pages);
 763         kfree(ia);
 764 }
 765
 766 static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
 767                                   int err)
 768 {
 769         struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 770         struct fuse_io_priv *io = ia->io;
 771         ssize_t pos = -1;
 772
 773         fuse_release_user_pages(&ia->ap, io->should_dirty);
 774
 775         if (err) {
 776                 /* Nothing */
 777         } else if (io->write) {
 778                 if (ia->write.out.size > ia->write.in.size) {
 779                         err = -EIO;
 780                 } else if (ia->write.in.size != ia->write.out.size) {
 781                         pos = ia->write.in.offset - io->offset +
 782                                 ia->write.out.size;
 783                 }
 784         } else {
 785                 u32 outsize = args->out_args[0].size;
 786
 787                 if (ia->read.in.size != outsize)
 788                         pos = ia->read.in.offset - io->offset + outsize;
 789         }
 790
 791         fuse_aio_complete(io, err, pos);
 792         fuse_io_free(ia);
 793 }
 794
 795 static ssize_t fuse_async_req_send(struct fuse_mount *fm,
 796                                    struct fuse_io_args *ia, size_t num_bytes)
 797 {
 798         ssize_t err;
 799         struct fuse_io_priv *io = ia->io;
 800
 801         spin_lock(&io->lock);
 802         kref_get(&io->refcnt);
 803         io->size += num_bytes;
 804         io->reqs++;
 805         spin_unlock(&io->lock);
 806
 807         ia->ap.args.end = fuse_aio_complete_req;
 808         ia->ap.args.may_block = io->should_dirty;
 809         err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
 810         if (err)
 811                 fuse_aio_complete_req(fm, &ia->ap.args, err);
 812
 813         return num_bytes;
 814 }
 815
 816 static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
 817                               fl_owner_t owner)
 818 {
 819         struct file *file = ia->io->iocb->ki_filp;
 820         struct fuse_file *ff = file->private_data;
 821         struct fuse_mount *fm = ff->fm;
 822
 823         fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 824         if (owner != NULL) {
 825                 ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
 826                 ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
 827         }
 828
 829         if (ia->io->async)
 830                 return fuse_async_req_send(fm, ia, count);
 831
 832         return fuse_simple_request(fm, &ia->ap.args);
 833 }
 834
 835 static void fuse_read_update_size(struct inode *inode, loff_t size,
 836                                   u64 attr_ver)
 837 {
 838         struct fuse_conn *fc = get_fuse_conn(inode);
 839         struct fuse_inode *fi = get_fuse_inode(inode);
 840
 841         spin_lock(&fi->lock);
 842         if (attr_ver >= fi->attr_version && size < inode->i_size &&
 843             !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
 844                 fi->attr_version = atomic64_inc_return(&fc->attr_version);
 845                 i_size_write(inode, size);
 846         }
 847         spin_unlock(&fi->lock);
 848 }
 849
 850 static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
 851                             struct fuse_args_pages *ap)
 852 {
 853         struct fuse_conn *fc = get_fuse_conn(inode);
 854
 855         /*
 856          * If writeback_cache is enabled, a short read means there's a hole in
 857          * the file.  Some data after the hole is in page cache, but has not
 858          * reached the client fs yet.  So the hole is not present there.
 859          */
 860         if (!fc->writeback_cache) {
 861                 loff_t pos = page_offset(ap->pages[0]) + num_read;
 862                 fuse_read_update_size(inode, pos, attr_ver);
 863         }
 864 }
 865
 866 static int fuse_do_readpage(struct file *file, struct page *page)
 867 {
 868         struct inode *inode = page->mapping->host;
 869         struct fuse_mount *fm = get_fuse_mount(inode);
 870         loff_t pos = page_offset(page);
 871         struct fuse_page_desc desc = { .length = PAGE_SIZE };
 872         struct fuse_io_args ia = {
 873                 .ap.args.page_zeroing = true,
 874                 .ap.args.out_pages = true,
 875                 .ap.num_pages = 1,
 876                 .ap.pages = &page,
 877                 .ap.descs = &desc,
 878         };
 879         ssize_t res;
 880         u64 attr_ver;
 881
 882         /*
 883          * Page writeback can extend beyond the lifetime of the
 884          * page-cache page, so make sure we read a properly synced
 885          * page.
 886          */
 887         fuse_wait_on_page_writeback(inode, page->index);
 888
 889         attr_ver = fuse_get_attr_version(fm->fc);
 890
 891         /* Don't overflow end offset */
 892         if (pos + (desc.length - 1) == LLONG_MAX)
 893                 desc.length--;
 894
 895         fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
 896         res = fuse_simple_request(fm, &ia.ap.args);
 897         if (res < 0)
 898                 return res;
 899         /*
 900          * Short read means EOF.  If file size is larger, truncate it
 901          */
 902         if (res < desc.length)
 903                 fuse_short_read(inode, attr_ver, res, &ia.ap);
 904
 905         SetPageUptodate(page);
 906
 907         return 0;
 908 }
 909
 910 static int fuse_read_folio(struct file *file, struct folio *folio)
 911 {
 912         struct page *page = &folio->page;
 913         struct inode *inode = page->mapping->host;
 914         int err;
 915
 916         err = -EIO;
 917         if (fuse_is_bad(inode))
 918                 goto out;
 919
 920         err = fuse_do_readpage(file, page);
 921         fuse_invalidate_atime(inode);
 922  out:
 923         unlock_page(page);
 924         return err;
 925 }
 926
 927 static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
 928                                int err)
 929 {
 930         int i;
 931         struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 932         struct fuse_args_pages *ap = &ia->ap;
 933         size_t count = ia->read.in.size;
 934         size_t num_read = args->out_args[0].size;
 935         struct address_space *mapping = NULL;
 936
 937         for (i = 0; mapping == NULL && i < ap->num_pages; i++)
 938                 mapping = ap->pages[i]->mapping;
 939
 940         if (mapping) {
 941                 struct inode *inode = mapping->host;
 942
 943                 /*
 944                  * Short read means EOF. If file size is larger, truncate it
 945                  */
 946                 if (!err && num_read < count)
 947                         fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
 948
 949                 fuse_invalidate_atime(inode);
 950         }
 951
 952         for (i = 0; i < ap->num_pages; i++) {
 953                 struct page *page = ap->pages[i];
 954
 955                 if (!err)
 956                         SetPageUptodate(page);
 957                 else
 958                         SetPageError(page);
 959                 unlock_page(page);
 960                 put_page(page);
 961         }
 962         if (ia->ff)
 963                 fuse_file_put(ia->ff, false, false);
 964
 965         fuse_io_free(ia);
 966 }
 967
 968 static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)
 969 {
 970         struct fuse_file *ff = file->private_data;
 971         struct fuse_mount *fm = ff->fm;
 972         struct fuse_args_pages *ap = &ia->ap;
 973         loff_t pos = page_offset(ap->pages[0]);
 974         size_t count = ap->num_pages << PAGE_SHIFT;
 975         ssize_t res;
 976         int err;
 977
 978         ap->args.out_pages = true;
 979         ap->args.page_zeroing = true;
 980         ap->args.page_replace = true;
 981
 982         /* Don't overflow end offset */
 983         if (pos + (count - 1) == LLONG_MAX) {
 984                 count--;
 985                 ap->descs[ap->num_pages - 1].length--;
 986         }
 987         WARN_ON((loff_t) (pos + count) < 0);
 988
 989         fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 990         ia->read.attr_ver = fuse_get_attr_version(fm->fc);
 991         if (fm->fc->async_read) {
 992                 ia->ff = fuse_file_get(ff);
 993                 ap->args.end = fuse_readpages_end;
 994                 err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
 995                 if (!err)
 996                         return;
 997         } else {
 998                 res = fuse_simple_request(fm, &ap->args);
 999                 err = res < 0 ? res : 0;
1000         }
1001         fuse_readpages_end(fm, &ap->args, err);
1002 }
1003
1004 static void fuse_readahead(struct readahead_control *rac)
1005 {
1006         struct inode *inode = rac->mapping->host;
1007         struct fuse_conn *fc = get_fuse_conn(inode);
1008         unsigned int i, max_pages, nr_pages = 0;
1009
1010         if (fuse_is_bad(inode))
1011                 return;
1012
1013         max_pages = min_t(unsigned int, fc->max_pages,
1014                         fc->max_read / PAGE_SIZE);
1015
1016         for (;;) {
1017                 struct fuse_io_args *ia;
1018                 struct fuse_args_pages *ap;
1019
1020                 if (fc->num_background >= fc->congestion_threshold &&
1021                     rac->ra->async_size >= readahead_count(rac))
1022                         /*
1023                          * Congested and only async pages left, so skip the
1024                          * rest.
1025                          */
1026                         break;
1027
1028                 nr_pages = readahead_count(rac) - nr_pages;
1029                 if (nr_pages > max_pages)
1030                         nr_pages = max_pages;
1031                 if (nr_pages == 0)
1032                         break;
1033                 ia = fuse_io_alloc(NULL, nr_pages);
1034                 if (!ia)
1035                         return;
1036                 ap = &ia->ap;
1037                 nr_pages = __readahead_batch(rac, ap->pages, nr_pages);
1038                 for (i = 0; i < nr_pages; i++) {
1039                         fuse_wait_on_page_writeback(inode,
1040                                                     readahead_index(rac) + i);
1041                         ap->descs[i].length = PAGE_SIZE;
1042                 }
1043                 ap->num_pages = nr_pages;
1044                 fuse_send_readpages(ia, rac->file);
1045         }
1046 }
1047
1048 static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
1049 {
1050         struct inode *inode = iocb->ki_filp->f_mapping->host;
1051         struct fuse_conn *fc = get_fuse_conn(inode);
1052
1053         /*
1054          * In auto invalidate mode, always update attributes on read.
1055          * Otherwise, only update if we attempt to read past EOF (to ensure
1056          * i_size is up to date).
1057          */
1058         if (fc->auto_inval_data ||
1059             (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
1060                 int err;
1061                 err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE);
1062                 if (err)
1063                         return err;
1064         }
1065
1066         return generic_file_read_iter(iocb, to);
1067 }
1068
1069 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
1070                                  loff_t pos, size_t count)
1071 {
1072         struct fuse_args *args = &ia->ap.args;
1073
1074         ia->write.in.fh = ff->fh;
1075         ia->write.in.offset = pos;
1076         ia->write.in.size = count;
1077         args->opcode = FUSE_WRITE;
1078         args->nodeid = ff->nodeid;
1079         args->in_numargs = 2;
1080         if (ff->fm->fc->minor < 9)
1081                 args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
1082         else
1083                 args->in_args[0].size = sizeof(ia->write.in);
1084         args->in_args[0].value = &ia->write.in;
1085         args->in_args[1].size = count;
1086         args->out_numargs = 1;
1087         args->out_args[0].size = sizeof(ia->write.out);
1088         args->out_args[0].value = &ia->write.out;
1089 }
1090
1091 static unsigned int fuse_write_flags(struct kiocb *iocb)
1092 {
1093         unsigned int flags = iocb->ki_filp->f_flags;
1094
1095         if (iocb_is_dsync(iocb))
1096                 flags |= O_DSYNC;
1097         if (iocb->ki_flags & IOCB_SYNC)
1098                 flags |= O_SYNC;
1099
1100         return flags;
1101 }
1102
1103 static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
1104                                size_t count, fl_owner_t owner)
1105 {
1106         struct kiocb *iocb = ia->io->iocb;
1107         struct file *file = iocb->ki_filp;
1108         struct fuse_file *ff = file->private_data;
1109         struct fuse_mount *fm = ff->fm;
1110         struct fuse_write_in *inarg = &ia->write.in;
1111         ssize_t err;
1112
1113         fuse_write_args_fill(ia, ff, pos, count);
1114         inarg->flags = fuse_write_flags(iocb);
1115         if (owner != NULL) {
1116                 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
1117                 inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
1118         }
1119
1120         if (ia->io->async)
1121                 return fuse_async_req_send(fm, ia, count);
1122
1123         err = fuse_simple_request(fm, &ia->ap.args);
1124         if (!err && ia->write.out.size > count)
1125                 err = -EIO;
1126
1127         return err ?: ia->write.out.size;
1128 }
1129
1130 bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written)
1131 {
1132         struct fuse_conn *fc = get_fuse_conn(inode);
1133         struct fuse_inode *fi = get_fuse_inode(inode);
1134         bool ret = false;
1135
1136         spin_lock(&fi->lock);
1137         fi->attr_version = atomic64_inc_return(&fc->attr_version);
1138         if (written > 0 && pos > inode->i_size) {
1139                 i_size_write(inode, pos);
1140                 ret = true;
1141         }
1142         spin_unlock(&fi->lock);
1143
1144         fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
1145
1146         return ret;
1147 }
1148
1149 static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
1150                                      struct kiocb *iocb, struct inode *inode,
1151                                      loff_t pos, size_t count)
1152 {
1153         struct fuse_args_pages *ap = &ia->ap;
1154         struct file *file = iocb->ki_filp;
1155         struct fuse_file *ff = file->private_data;
1156         struct fuse_mount *fm = ff->fm;
1157         unsigned int offset, i;
1158         bool short_write;
1159         int err;
1160
1161         for (i = 0; i < ap->num_pages; i++)
1162                 fuse_wait_on_page_writeback(inode, ap->pages[i]->index);
1163
1164         fuse_write_args_fill(ia, ff, pos, count);
1165         ia->write.in.flags = fuse_write_flags(iocb);
1166         if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID))
1167                 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
1168
1169         err = fuse_simple_request(fm, &ap->args);
1170         if (!err && ia->write.out.size > count)
1171                 err = -EIO;
1172
1173         short_write = ia->write.out.size < count;
1174         offset = ap->descs[0].offset;
1175         count = ia->write.out.size;
1176         for (i = 0; i < ap->num_pages; i++) {
1177                 struct page *page = ap->pages[i];
1178
1179                 if (err) {
1180                         ClearPageUptodate(page);
1181                 } else {
1182                         if (count >= PAGE_SIZE - offset)
1183                                 count -= PAGE_SIZE - offset;
1184                         else {
1185                                 if (short_write)
1186                                         ClearPageUptodate(page);
1187                                 count = 0;
1188                         }
1189                         offset = 0;
1190                 }
1191                 if (ia->write.page_locked && (i == ap->num_pages - 1))
1192                         unlock_page(page);
1193                 put_page(page);
1194         }
1195
1196         return err;
1197 }
1198
1199 static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
1200                                      struct address_space *mapping,
1201                                      struct iov_iter *ii, loff_t pos,
1202                                      unsigned int max_pages)
1203 {
1204         struct fuse_args_pages *ap = &ia->ap;
1205         struct fuse_conn *fc = get_fuse_conn(mapping->host);
1206         unsigned offset = pos & (PAGE_SIZE - 1);
1207         size_t count = 0;
1208         int err;
1209
1210         ap->args.in_pages = true;
1211         ap->descs[0].offset = offset;
1212
1213         do {
1214                 size_t tmp;
1215                 struct page *page;
1216                 pgoff_t index = pos >> PAGE_SHIFT;
1217                 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
1218                                      iov_iter_count(ii));
1219
1220                 bytes = min_t(size_t, bytes, fc->max_write - count);
1221
1222  again:
1223                 err = -EFAULT;
1224                 if (fault_in_iov_iter_readable(ii, bytes))
1225                         break;
1226
1227                 err = -ENOMEM;
1228                 page = grab_cache_page_write_begin(mapping, index);
1229                 if (!page)
1230                         break;
1231
1232                 if (mapping_writably_mapped(mapping))
1233                         flush_dcache_page(page);
1234
1235                 tmp = copy_page_from_iter_atomic(page, offset, bytes, ii);
1236                 flush_dcache_page(page);
1237
1238                 if (!tmp) {
1239                         unlock_page(page);
1240                         put_page(page);
1241                         goto again;
1242                 }
1243
1244                 err = 0;
1245                 ap->pages[ap->num_pages] = page;
1246                 ap->descs[ap->num_pages].length = tmp;
1247                 ap->num_pages++;
1248
1249                 count += tmp;
1250                 pos += tmp;
1251                 offset += tmp;
1252                 if (offset == PAGE_SIZE)
1253                         offset = 0;
1254
1255                 /* If we copied full page, mark it uptodate */
1256                 if (tmp == PAGE_SIZE)
1257                         SetPageUptodate(page);
1258
1259                 if (PageUptodate(page)) {
1260                         unlock_page(page);
1261                 } else {
1262                         ia->write.page_locked = true;
1263                         break;
1264                 }
1265                 if (!fc->big_writes)
1266                         break;
1267         } while (iov_iter_count(ii) && count < fc->max_write &&
1268                  ap->num_pages < max_pages && offset == 0);
1269
1270         return count > 0 ? count : err;
1271 }
1272
1273 static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
1274                                      unsigned int max_pages)
1275 {
1276         return min_t(unsigned int,
1277                      ((pos + len - 1) >> PAGE_SHIFT) -
1278                      (pos >> PAGE_SHIFT) + 1,
1279                      max_pages);
1280 }
1281
1282 static ssize_t fuse_perform_write(struct kiocb *iocb,
1283                                   struct address_space *mapping,
1284                                   struct iov_iter *ii, loff_t pos)
1285 {
1286         struct inode *inode = mapping->host;
1287         struct fuse_conn *fc = get_fuse_conn(inode);
1288         struct fuse_inode *fi = get_fuse_inode(inode);
1289         int err = 0;
1290         ssize_t res = 0;
1291
1292         if (inode->i_size < pos + iov_iter_count(ii))
1293                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1294
1295         do {
1296                 ssize_t count;
1297                 struct fuse_io_args ia = {};
1298                 struct fuse_args_pages *ap = &ia.ap;
1299                 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1300                                                       fc->max_pages);
1301
1302                 ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs);
1303                 if (!ap->pages) {
1304                         err = -ENOMEM;
1305                         break;
1306                 }
1307
1308                 count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);
1309                 if (count <= 0) {
1310                         err = count;
1311                 } else {
1312                         err = fuse_send_write_pages(&ia, iocb, inode,
1313                                                     pos, count);
1314                         if (!err) {
1315                                 size_t num_written = ia.write.out.size;
1316
1317                                 res += num_written;
1318                                 pos += num_written;
1319
1320                                 /* break out of the loop on short write */
1321                                 if (num_written != count)
1322                                         err = -EIO;
1323                         }
1324                 }
1325                 kfree(ap->pages);
1326         } while (!err && iov_iter_count(ii));
1327
1328         fuse_write_update_attr(inode, pos, res);
1329         clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1330
1331         return res > 0 ? res : err;
1332 }
1333
1334 static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
1335 {
1336         struct file *file = iocb->ki_filp;
1337         struct address_space *mapping = file->f_mapping;
1338         ssize_t written = 0;
1339         ssize_t written_buffered = 0;
1340         struct inode *inode = mapping->host;
1341         ssize_t err;
1342         struct fuse_conn *fc = get_fuse_conn(inode);
1343         loff_t endbyte = 0;
1344
1345         if (fc->writeback_cache) {
1346                 /* Update size (EOF optimization) and mode (SUID clearing) */
1347                 err = fuse_update_attributes(mapping->host, file,
1348                                              STATX_SIZE | STATX_MODE);
1349                 if (err)
1350                         return err;
1351
1352                 if (fc->handle_killpriv_v2 &&
1353                     setattr_should_drop_suidgid(&init_user_ns, file_inode(file))) {
1354                         goto writethrough;
1355                 }
1356
1357                 return generic_file_write_iter(iocb, from);
1358         }
1359
1360 writethrough:
1361         inode_lock(inode);
1362
1363         /* We can write back this queue in page reclaim */
1364         current->backing_dev_info = inode_to_bdi(inode);
1365
1366         err = generic_write_checks(iocb, from);
1367         if (err <= 0)
1368                 goto out;
1369
1370         err = file_remove_privs(file);
1371         if (err)
1372                 goto out;
1373
1374         err = file_update_time(file);
1375         if (err)
1376                 goto out;
1377
1378         if (iocb->ki_flags & IOCB_DIRECT) {
1379                 loff_t pos = iocb->ki_pos;
1380                 written = generic_file_direct_write(iocb, from);
1381                 if (written < 0 || !iov_iter_count(from))
1382                         goto out;
1383
1384                 pos += written;
1385
1386                 written_buffered = fuse_perform_write(iocb, mapping, from, pos);
1387                 if (written_buffered < 0) {
1388                         err = written_buffered;
1389                         goto out;
1390                 }
1391                 endbyte = pos + written_buffered - 1;
1392
1393                 err = filemap_write_and_wait_range(file->f_mapping, pos,
1394                                                    endbyte);
1395                 if (err)
1396                         goto out;
1397
1398                 invalidate_mapping_pages(file->f_mapping,
1399                                          pos >> PAGE_SHIFT,
1400                                          endbyte >> PAGE_SHIFT);
1401
1402                 written += written_buffered;
1403                 iocb->ki_pos = pos + written_buffered;
1404         } else {
1405                 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);
1406                 if (written >= 0)
1407                         iocb->ki_pos += written;
1408         }
1409 out:
1410         current->backing_dev_info = NULL;
1411         inode_unlock(inode);
1412         if (written > 0)
1413                 written = generic_write_sync(iocb, written);
1414
1415         return written ? written : err;
1416 }
1417
1418 static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1419 {
1420         return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1421 }
1422
1423 static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1424                                         size_t max_size)
1425 {
1426         return min(iov_iter_single_seg_count(ii), max_size);
1427 }
1428
1429 static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
1430                                size_t *nbytesp, int write,
1431                                unsigned int max_pages)
1432 {
1433         size_t nbytes = 0;  /* # bytes already packed in req */
1434         ssize_t ret = 0;
1435
1436         /* Special case for kernel I/O: can copy directly into the buffer */
1437         if (iov_iter_is_kvec(ii)) {
1438                 unsigned long user_addr = fuse_get_user_addr(ii);
1439                 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1440
1441                 if (write)
1442                         ap->args.in_args[1].value = (void *) user_addr;
1443                 else
1444                         ap->args.out_args[0].value = (void *) user_addr;
1445
1446                 iov_iter_advance(ii, frag_size);
1447                 *nbytesp = frag_size;
1448                 return 0;
1449         }
1450
1451         while (nbytes < *nbytesp && ap->num_pages < max_pages) {
1452                 unsigned npages;
1453                 size_t start;
1454                 ret = iov_iter_get_pages2(ii, &ap->pages[ap->num_pages],
1455                                         *nbytesp - nbytes,
1456                                         max_pages - ap->num_pages,
1457                                         &start);
1458                 if (ret < 0)
1459                         break;
1460
1461                 nbytes += ret;
1462
1463                 ret += start;
1464                 npages = DIV_ROUND_UP(ret, PAGE_SIZE);
1465
1466                 ap->descs[ap->num_pages].offset = start;
1467                 fuse_page_descs_length_init(ap->descs, ap->num_pages, npages);
1468
1469                 ap->num_pages += npages;
1470                 ap->descs[ap->num_pages - 1].length -=
1471                         (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1472         }
1473
1474         ap->args.user_pages = true;
1475         if (write)
1476                 ap->args.in_pages = true;
1477         else
1478                 ap->args.out_pages = true;
1479
1480         *nbytesp = nbytes;
1481
1482         return ret < 0 ? ret : 0;
1483 }
1484
1485 ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1486                        loff_t *ppos, int flags)
1487 {
1488         int write = flags & FUSE_DIO_WRITE;
1489         int cuse = flags & FUSE_DIO_CUSE;
1490         struct file *file = io->iocb->ki_filp;
1491         struct inode *inode = file->f_mapping->host;
1492         struct fuse_file *ff = file->private_data;
1493         struct fuse_conn *fc = ff->fm->fc;
1494         size_t nmax = write ? fc->max_write : fc->max_read;
1495         loff_t pos = *ppos;
1496         size_t count = iov_iter_count(iter);
1497         pgoff_t idx_from = pos >> PAGE_SHIFT;
1498         pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
1499         ssize_t res = 0;
1500         int err = 0;
1501         struct fuse_io_args *ia;
1502         unsigned int max_pages;
1503
1504         max_pages = iov_iter_npages(iter, fc->max_pages);
1505         ia = fuse_io_alloc(io, max_pages);
1506         if (!ia)
1507                 return -ENOMEM;
1508
1509         if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1510                 if (!write)
1511                         inode_lock(inode);
1512                 fuse_sync_writes(inode);
1513                 if (!write)
1514                         inode_unlock(inode);
1515         }
1516
1517         io->should_dirty = !write && user_backed_iter(iter);
1518         while (count) {
1519                 ssize_t nres;
1520                 fl_owner_t owner = current->files;
1521                 size_t nbytes = min(count, nmax);
1522
1523                 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
1524                                           max_pages);
1525                 if (err && !nbytes)
1526                         break;
1527
1528                 if (write) {
1529                         if (!capable(CAP_FSETID))
1530                                 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
1531
1532                         nres = fuse_send_write(ia, pos, nbytes, owner);
1533                 } else {
1534                         nres = fuse_send_read(ia, pos, nbytes, owner);
1535                 }
1536
1537                 if (!io->async || nres < 0) {
1538                         fuse_release_user_pages(&ia->ap, io->should_dirty);
1539                         fuse_io_free(ia);
1540                 }
1541                 ia = NULL;
1542                 if (nres < 0) {
1543                         iov_iter_revert(iter, nbytes);
1544                         err = nres;
1545                         break;
1546                 }
1547                 WARN_ON(nres > nbytes);
1548
1549                 count -= nres;
1550                 res += nres;
1551                 pos += nres;
1552                 if (nres != nbytes) {
1553                         iov_iter_revert(iter, nbytes - nres);
1554                         break;
1555                 }
1556                 if (count) {
1557                         max_pages = iov_iter_npages(iter, fc->max_pages);
1558                         ia = fuse_io_alloc(io, max_pages);
1559                         if (!ia)
1560                                 break;
1561                 }
1562         }
1563         if (ia)
1564                 fuse_io_free(ia);
1565         if (res > 0)
1566                 *ppos = pos;
1567
1568         return res > 0 ? res : err;
1569 }
1570 EXPORT_SYMBOL_GPL(fuse_direct_io);
1571
1572 static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1573                                   struct iov_iter *iter,
1574                                   loff_t *ppos)
1575 {
1576         ssize_t res;
1577         struct inode *inode = file_inode(io->iocb->ki_filp);
1578
1579         res = fuse_direct_io(io, iter, ppos, 0);
1580
1581         fuse_invalidate_atime(inode);
1582
1583         return res;
1584 }
1585
1586 static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
1587
1588 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
1589 {
1590         ssize_t res;
1591
1592         if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1593                 res = fuse_direct_IO(iocb, to);
1594         } else {
1595                 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1596
1597                 res = __fuse_direct_read(&io, to, &iocb->ki_pos);
1598         }
1599
1600         return res;
1601 }
1602
1603 static bool fuse_direct_write_extending_i_size(struct kiocb *iocb,
1604                                                struct iov_iter *iter)
1605 {
1606         struct inode *inode = file_inode(iocb->ki_filp);
1607
1608         return iocb->ki_pos + iov_iter_count(iter) > i_size_read(inode);
1609 }
1610
1611 static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1612 {
1613         struct inode *inode = file_inode(iocb->ki_filp);
1614         struct file *file = iocb->ki_filp;
1615         struct fuse_file *ff = file->private_data;
1616         struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1617         ssize_t res;
1618         bool exclusive_lock =
1619                 !(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES) ||
1620                 iocb->ki_flags & IOCB_APPEND ||
1621                 fuse_direct_write_extending_i_size(iocb, from);
1622
1623         /*
1624          * Take exclusive lock if
1625          * - Parallel direct writes are disabled - a user space decision
1626          * - Parallel direct writes are enabled and i_size is being extended.
1627          *   This might not be needed at all, but needs further investigation.
1628          */
1629         if (exclusive_lock)
1630                 inode_lock(inode);
1631         else {
1632                 inode_lock_shared(inode);
1633
1634                 /* A race with truncate might have come up as the decision for
1635                  * the lock type was done without holding the lock, check again.
1636                  */
1637                 if (fuse_direct_write_extending_i_size(iocb, from)) {
1638                         inode_unlock_shared(inode);
1639                         inode_lock(inode);
1640                         exclusive_lock = true;
1641                 }
1642         }
1643
1644         res = generic_write_checks(iocb, from);
1645         if (res > 0) {
1646                 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1647                         res = fuse_direct_IO(iocb, from);
1648                 } else {
1649                         res = fuse_direct_io(&io, from, &iocb->ki_pos,
1650                                              FUSE_DIO_WRITE);
1651                         fuse_write_update_attr(inode, iocb->ki_pos, res);
1652                 }
1653         }
1654         if (exclusive_lock)
1655                 inode_unlock(inode);
1656         else
1657                 inode_unlock_shared(inode);
1658
1659         return res;
1660 }
1661
1662 static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1663 {
1664         struct file *file = iocb->ki_filp;
1665         struct fuse_file *ff = file->private_data;
1666         struct inode *inode = file_inode(file);
1667
1668         if (fuse_is_bad(inode))
1669                 return -EIO;
1670
1671         if (FUSE_IS_DAX(inode))
1672                 return fuse_dax_read_iter(iocb, to);
1673
1674         if (!(ff->open_flags & FOPEN_DIRECT_IO))
1675                 return fuse_cache_read_iter(iocb, to);
1676         else
1677                 return fuse_direct_read_iter(iocb, to);
1678 }
1679
1680 static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1681 {
1682         struct file *file = iocb->ki_filp;
1683         struct fuse_file *ff = file->private_data;
1684         struct inode *inode = file_inode(file);
1685
1686         if (fuse_is_bad(inode))
1687                 return -EIO;
1688
1689         if (FUSE_IS_DAX(inode))
1690                 return fuse_dax_write_iter(iocb, from);
1691
1692         if (!(ff->open_flags & FOPEN_DIRECT_IO))
1693                 return fuse_cache_write_iter(iocb, from);
1694         else
1695                 return fuse_direct_write_iter(iocb, from);
1696 }
1697
1698 static void fuse_writepage_free(struct fuse_writepage_args *wpa)
1699 {
1700         struct fuse_args_pages *ap = &wpa->ia.ap;
1701         int i;
1702
1703         if (wpa->bucket)
1704                 fuse_sync_bucket_dec(wpa->bucket);
1705
1706         for (i = 0; i < ap->num_pages; i++)
1707                 __free_page(ap->pages[i]);
1708
1709         if (wpa->ia.ff)
1710                 fuse_file_put(wpa->ia.ff, false, false);
1711
1712         kfree(ap->pages);
1713         kfree(wpa);
1714 }
1715
1716 static void fuse_writepage_finish(struct fuse_mount *fm,
1717                                   struct fuse_writepage_args *wpa)
1718 {
1719         struct fuse_args_pages *ap = &wpa->ia.ap;
1720         struct inode *inode = wpa->inode;
1721         struct fuse_inode *fi = get_fuse_inode(inode);
1722         struct backing_dev_info *bdi = inode_to_bdi(inode);
1723         int i;
1724
1725         for (i = 0; i < ap->num_pages; i++) {
1726                 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1727                 dec_node_page_state(ap->pages[i], NR_WRITEBACK_TEMP);
1728                 wb_writeout_inc(&bdi->wb);
1729         }
1730         wake_up(&fi->page_waitq);
1731 }
1732
1733 /* Called under fi->lock, may release and reacquire it */
1734 static void fuse_send_writepage(struct fuse_mount *fm,
1735                                 struct fuse_writepage_args *wpa, loff_t size)
1736 __releases(fi->lock)
1737 __acquires(fi->lock)
1738 {
1739         struct fuse_writepage_args *aux, *next;
1740         struct fuse_inode *fi = get_fuse_inode(wpa->inode);
1741         struct fuse_write_in *inarg = &wpa->ia.write.in;
1742         struct fuse_args *args = &wpa->ia.ap.args;
1743         __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;
1744         int err;
1745
1746         fi->writectr++;
1747         if (inarg->offset + data_size <= size) {
1748                 inarg->size = data_size;
1749         } else if (inarg->offset < size) {
1750                 inarg->size = size - inarg->offset;
1751         } else {
1752                 /* Got truncated off completely */
1753                 goto out_free;
1754         }
1755
1756         args->in_args[1].size = inarg->size;
1757         args->force = true;
1758         args->nocreds = true;
1759
1760         err = fuse_simple_background(fm, args, GFP_ATOMIC);
1761         if (err == -ENOMEM) {
1762                 spin_unlock(&fi->lock);
1763                 err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
1764                 spin_lock(&fi->lock);
1765         }
1766
1767         /* Fails on broken connection only */
1768         if (unlikely(err))
1769                 goto out_free;
1770
1771         return;
1772
1773  out_free:
1774         fi->writectr--;
1775         rb_erase(&wpa->writepages_entry, &fi->writepages);
1776         fuse_writepage_finish(fm, wpa);
1777         spin_unlock(&fi->lock);
1778
1779         /* After fuse_writepage_finish() aux request list is private */
1780         for (aux = wpa->next; aux; aux = next) {
1781                 next = aux->next;
1782                 aux->next = NULL;
1783                 fuse_writepage_free(aux);
1784         }
1785
1786         fuse_writepage_free(wpa);
1787         spin_lock(&fi->lock);
1788 }
1789
1790 /*
1791  * If fi->writectr is positive (no truncate or fsync going on) send
1792  * all queued writepage requests.
1793  *
1794  * Called with fi->lock
1795  */
1796 void fuse_flush_writepages(struct inode *inode)
1797 __releases(fi->lock)
1798 __acquires(fi->lock)
1799 {
1800         struct fuse_mount *fm = get_fuse_mount(inode);
1801         struct fuse_inode *fi = get_fuse_inode(inode);
1802         loff_t crop = i_size_read(inode);
1803         struct fuse_writepage_args *wpa;
1804
1805         while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1806                 wpa = list_entry(fi->queued_writes.next,
1807                                  struct fuse_writepage_args, queue_entry);
1808                 list_del_init(&wpa->queue_entry);
1809                 fuse_send_writepage(fm, wpa, crop);
1810         }
1811 }
1812
1813 static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root,
1814                                                 struct fuse_writepage_args *wpa)
1815 {
1816         pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;
1817         pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;
1818         struct rb_node **p = &root->rb_node;
1819         struct rb_node  *parent = NULL;
1820
1821         WARN_ON(!wpa->ia.ap.num_pages);
1822         while (*p) {
1823                 struct fuse_writepage_args *curr;
1824                 pgoff_t curr_index;
1825
1826                 parent = *p;
1827                 curr = rb_entry(parent, struct fuse_writepage_args,
1828                                 writepages_entry);
1829                 WARN_ON(curr->inode != wpa->inode);
1830                 curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;
1831
1832                 if (idx_from >= curr_index + curr->ia.ap.num_pages)
1833                         p = &(*p)->rb_right;
1834                 else if (idx_to < curr_index)
1835                         p = &(*p)->rb_left;
1836                 else
1837                         return curr;
1838         }
1839
1840         rb_link_node(&wpa->writepages_entry, parent, p);
1841         rb_insert_color(&wpa->writepages_entry, root);
1842         return NULL;
1843 }
1844
1845 static void tree_insert(struct rb_root *root, struct fuse_writepage_args *wpa)
1846 {
1847         WARN_ON(fuse_insert_writeback(root, wpa));
1848 }
1849
1850 static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
1851                                int error)
1852 {
1853         struct fuse_writepage_args *wpa =
1854                 container_of(args, typeof(*wpa), ia.ap.args);
1855         struct inode *inode = wpa->inode;
1856         struct fuse_inode *fi = get_fuse_inode(inode);
1857         struct fuse_conn *fc = get_fuse_conn(inode);
1858
1859         mapping_set_error(inode->i_mapping, error);
1860         /*
1861          * A writeback finished and this might have updated mtime/ctime on
1862          * server making local mtime/ctime stale.  Hence invalidate attrs.
1863          * Do this only if writeback_cache is not enabled.  If writeback_cache
1864          * is enabled, we trust local ctime/mtime.
1865          */
1866         if (!fc->writeback_cache)
1867                 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY);
1868         spin_lock(&fi->lock);
1869         rb_erase(&wpa->writepages_entry, &fi->writepages);
1870         while (wpa->next) {
1871                 struct fuse_mount *fm = get_fuse_mount(inode);
1872                 struct fuse_write_in *inarg = &wpa->ia.write.in;
1873                 struct fuse_writepage_args *next = wpa->next;
1874
1875                 wpa->next = next->next;
1876                 next->next = NULL;
1877                 next->ia.ff = fuse_file_get(wpa->ia.ff);
1878                 tree_insert(&fi->writepages, next);
1879
1880                 /*
1881                  * Skip fuse_flush_writepages() to make it easy to crop requests
1882                  * based on primary request size.
1883                  *
1884                  * 1st case (trivial): there are no concurrent activities using
1885                  * fuse_set/release_nowrite.  Then we're on safe side because
1886                  * fuse_flush_writepages() would call fuse_send_writepage()
1887                  * anyway.
1888                  *
1889                  * 2nd case: someone called fuse_set_nowrite and it is waiting
1890                  * now for completion of all in-flight requests.  This happens
1891                  * rarely and no more than once per page, so this should be
1892                  * okay.
1893                  *
1894                  * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle
1895                  * of fuse_set_nowrite..fuse_release_nowrite section.  The fact
1896                  * that fuse_set_nowrite returned implies that all in-flight
1897                  * requests were completed along with all of their secondary
1898                  * requests.  Further primary requests are blocked by negative
1899                  * writectr.  Hence there cannot be any in-flight requests and
1900                  * no invocations of fuse_writepage_end() while we're in
1901                  * fuse_set_nowrite..fuse_release_nowrite section.
1902                  */
1903                 fuse_send_writepage(fm, next, inarg->offset + inarg->size);
1904         }
1905         fi->writectr--;
1906         fuse_writepage_finish(fm, wpa);
1907         spin_unlock(&fi->lock);
1908         fuse_writepage_free(wpa);
1909 }
1910
1911 static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi)
1912 {
1913         struct fuse_file *ff;
1914
1915         spin_lock(&fi->lock);
1916         ff = list_first_entry_or_null(&fi->write_files, struct fuse_file,
1917                                       write_entry);
1918         if (ff)
1919                 fuse_file_get(ff);
1920         spin_unlock(&fi->lock);
1921
1922         return ff;
1923 }
1924
1925 static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi)
1926 {
1927         struct fuse_file *ff = __fuse_write_file_get(fi);
1928         WARN_ON(!ff);
1929         return ff;
1930 }
1931
1932 int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1933 {
1934         struct fuse_inode *fi = get_fuse_inode(inode);
1935         struct fuse_file *ff;
1936         int err;
1937
1938         /*
1939          * Inode is always written before the last reference is dropped and
1940          * hence this should not be reached from reclaim.
1941          *
1942          * Writing back the inode from reclaim can deadlock if the request
1943          * processing itself needs an allocation.  Allocations triggering
1944          * reclaim while serving a request can't be prevented, because it can
1945          * involve any number of unrelated userspace processes.
1946          */
1947         WARN_ON(wbc->for_reclaim);
1948
1949         ff = __fuse_write_file_get(fi);
1950         err = fuse_flush_times(inode, ff);
1951         if (ff)
1952                 fuse_file_put(ff, false, false);
1953
1954         return err;
1955 }
1956
1957 static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
1958 {
1959         struct fuse_writepage_args *wpa;
1960         struct fuse_args_pages *ap;
1961
1962         wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
1963         if (wpa) {
1964                 ap = &wpa->ia.ap;
1965                 ap->num_pages = 0;
1966                 ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs);
1967                 if (!ap->pages) {
1968                         kfree(wpa);
1969                         wpa = NULL;
1970                 }
1971         }
1972         return wpa;
1973
1974 }
1975
1976 static void fuse_writepage_add_to_bucket(struct fuse_conn *fc,
1977                                          struct fuse_writepage_args *wpa)
1978 {
1979         if (!fc->sync_fs)
1980                 return;
1981
1982         rcu_read_lock();
1983         /* Prevent resurrection of dead bucket in unlikely race with syncfs */
1984         do {
1985                 wpa->bucket = rcu_dereference(fc->curr_bucket);
1986         } while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count)));
1987         rcu_read_unlock();
1988 }
1989
1990 static int fuse_writepage_locked(struct page *page)
1991 {
1992         struct address_space *mapping = page->mapping;
1993         struct inode *inode = mapping->host;
1994         struct fuse_conn *fc = get_fuse_conn(inode);
1995         struct fuse_inode *fi = get_fuse_inode(inode);
1996         struct fuse_writepage_args *wpa;
1997         struct fuse_args_pages *ap;
1998         struct page *tmp_page;
1999         int error = -ENOMEM;
2000
2001         set_page_writeback(page);
2002
2003         wpa = fuse_writepage_args_alloc();
2004         if (!wpa)
2005                 goto err;
2006         ap = &wpa->ia.ap;
2007
2008         tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
2009         if (!tmp_page)
2010                 goto err_free;
2011
2012         error = -EIO;
2013         wpa->ia.ff = fuse_write_file_get(fi);
2014         if (!wpa->ia.ff)
2015                 goto err_nofile;
2016
2017         fuse_writepage_add_to_bucket(fc, wpa);
2018         fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);
2019
2020         copy_highpage(tmp_page, page);
2021         wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
2022         wpa->next = NULL;
2023         ap->args.in_pages = true;
2024         ap->num_pages = 1;
2025         ap->pages[0] = tmp_page;
2026         ap->descs[0].offset = 0;
2027         ap->descs[0].length = PAGE_SIZE;
2028         ap->args.end = fuse_writepage_end;
2029         wpa->inode = inode;
2030
2031         inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
2032         inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
2033
2034         spin_lock(&fi->lock);
2035         tree_insert(&fi->writepages, wpa);
2036         list_add_tail(&wpa->queue_entry, &fi->queued_writes);
2037         fuse_flush_writepages(inode);
2038         spin_unlock(&fi->lock);
2039
2040         end_page_writeback(page);
2041
2042         return 0;
2043
2044 err_nofile:
2045         __free_page(tmp_page);
2046 err_free:
2047         kfree(wpa);
2048 err:
2049         mapping_set_error(page->mapping, error);
2050         end_page_writeback(page);
2051         return error;
2052 }
2053
2054 static int fuse_writepage(struct page *page, struct writeback_control *wbc)
2055 {
2056         struct fuse_conn *fc = get_fuse_conn(page->mapping->host);
2057         int err;
2058
2059         if (fuse_page_is_writeback(page->mapping->host, page->index)) {
2060                 /*
2061                  * ->writepages() should be called for sync() and friends.  We
2062                  * should only get here on direct reclaim and then we are
2063                  * allowed to skip a page which is already in flight
2064                  */
2065                 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
2066
2067                 redirty_page_for_writepage(wbc, page);
2068                 unlock_page(page);
2069                 return 0;
2070         }
2071
2072         if (wbc->sync_mode == WB_SYNC_NONE &&
2073             fc->num_background >= fc->congestion_threshold)
2074                 return AOP_WRITEPAGE_ACTIVATE;
2075
2076         err = fuse_writepage_locked(page);
2077         unlock_page(page);
2078
2079         return err;
2080 }
2081
2082 struct fuse_fill_wb_data {
2083         struct fuse_writepage_args *wpa;
2084         struct fuse_file *ff;
2085         struct inode *inode;
2086         struct page **orig_pages;
2087         unsigned int max_pages;
2088 };
2089
2090 static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)
2091 {
2092         struct fuse_args_pages *ap = &data->wpa->ia.ap;
2093         struct fuse_conn *fc = get_fuse_conn(data->inode);
2094         struct page **pages;
2095         struct fuse_page_desc *descs;
2096         unsigned int npages = min_t(unsigned int,
2097                                     max_t(unsigned int, data->max_pages * 2,
2098                                           FUSE_DEFAULT_MAX_PAGES_PER_REQ),
2099                                     fc->max_pages);
2100         WARN_ON(npages <= data->max_pages);
2101
2102         pages = fuse_pages_alloc(npages, GFP_NOFS, &descs);
2103         if (!pages)
2104                 return false;
2105
2106         memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages);
2107         memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages);
2108         kfree(ap->pages);
2109         ap->pages = pages;
2110         ap->descs = descs;
2111         data->max_pages = npages;
2112
2113         return true;
2114 }
2115
2116 static void fuse_writepages_send(struct fuse_fill_wb_data *data)
2117 {
2118         struct fuse_writepage_args *wpa = data->wpa;
2119         struct inode *inode = data->inode;
2120         struct fuse_inode *fi = get_fuse_inode(inode);
2121         int num_pages = wpa->ia.ap.num_pages;
2122         int i;
2123
2124         wpa->ia.ff = fuse_file_get(data->ff);
2125         spin_lock(&fi->lock);
2126         list_add_tail(&wpa->queue_entry, &fi->queued_writes);
2127         fuse_flush_writepages(inode);
2128         spin_unlock(&fi->lock);
2129
2130         for (i = 0; i < num_pages; i++)
2131                 end_page_writeback(data->orig_pages[i]);
2132 }
2133
2134 /*
2135  * Check under fi->lock if the page is under writeback, and insert it onto the
2136  * rb_tree if not. Otherwise iterate auxiliary write requests, to see if there's
2137  * one already added for a page at this offset.  If there's none, then insert
2138  * this new request onto the auxiliary list, otherwise reuse the existing one by
2139  * swapping the new temp page with the old one.
2140  */
2141 static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa,
2142                                struct page *page)
2143 {
2144         struct fuse_inode *fi = get_fuse_inode(new_wpa->inode);
2145         struct fuse_writepage_args *tmp;
2146         struct fuse_writepage_args *old_wpa;
2147         struct fuse_args_pages *new_ap = &new_wpa->ia.ap;
2148
2149         WARN_ON(new_ap->num_pages != 0);
2150         new_ap->num_pages = 1;
2151
2152         spin_lock(&fi->lock);
2153         old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa);
2154         if (!old_wpa) {
2155                 spin_unlock(&fi->lock);
2156                 return true;
2157         }
2158
2159         for (tmp = old_wpa->next; tmp; tmp = tmp->next) {
2160                 pgoff_t curr_index;
2161
2162                 WARN_ON(tmp->inode != new_wpa->inode);
2163                 curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;
2164                 if (curr_index == page->index) {
2165                         WARN_ON(tmp->ia.ap.num_pages != 1);
2166                         swap(tmp->ia.ap.pages[0], new_ap->pages[0]);
2167                         break;
2168                 }
2169         }
2170
2171         if (!tmp) {
2172                 new_wpa->next = old_wpa->next;
2173                 old_wpa->next = new_wpa;
2174         }
2175
2176         spin_unlock(&fi->lock);
2177
2178         if (tmp) {
2179                 struct backing_dev_info *bdi = inode_to_bdi(new_wpa->inode);
2180
2181                 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
2182                 dec_node_page_state(new_ap->pages[0], NR_WRITEBACK_TEMP);
2183                 wb_writeout_inc(&bdi->wb);
2184                 fuse_writepage_free(new_wpa);
2185         }
2186
2187         return false;
2188 }
2189
2190 static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page,
2191                                      struct fuse_args_pages *ap,
2192                                      struct fuse_fill_wb_data *data)
2193 {
2194         WARN_ON(!ap->num_pages);
2195
2196         /*
2197          * Being under writeback is unlikely but possible.  For example direct
2198          * read to an mmaped fuse file will set the page dirty twice; once when
2199          * the pages are faulted with get_user_pages(), and then after the read
2200          * completed.
2201          */
2202         if (fuse_page_is_writeback(data->inode, page->index))
2203                 return true;
2204
2205         /* Reached max pages */
2206         if (ap->num_pages == fc->max_pages)
2207                 return true;
2208
2209         /* Reached max write bytes */
2210         if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write)
2211                 return true;
2212
2213         /* Discontinuity */
2214         if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index)
2215                 return true;
2216
2217         /* Need to grow the pages array?  If so, did the expansion fail? */
2218         if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data))
2219                 return true;
2220
2221         return false;
2222 }
2223
2224 static int fuse_writepages_fill(struct page *page,
2225                 struct writeback_control *wbc, void *_data)
2226 {
2227         struct fuse_fill_wb_data *data = _data;
2228         struct fuse_writepage_args *wpa = data->wpa;
2229         struct fuse_args_pages *ap = &wpa->ia.ap;
2230         struct inode *inode = data->inode;
2231         struct fuse_inode *fi = get_fuse_inode(inode);
2232         struct fuse_conn *fc = get_fuse_conn(inode);
2233         struct page *tmp_page;
2234         int err;
2235
2236         if (!data->ff) {
2237                 err = -EIO;
2238                 data->ff = fuse_write_file_get(fi);
2239                 if (!data->ff)
2240                         goto out_unlock;
2241         }
2242
2243         if (wpa && fuse_writepage_need_send(fc, page, ap, data)) {
2244                 fuse_writepages_send(data);
2245                 data->wpa = NULL;
2246         }
2247
2248         err = -ENOMEM;
2249         tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
2250         if (!tmp_page)
2251                 goto out_unlock;
2252
2253         /*
2254          * The page must not be redirtied until the writeout is completed
2255          * (i.e. userspace has sent a reply to the write request).  Otherwise
2256          * there could be more than one temporary page instance for each real
2257          * page.
2258          *
2259          * This is ensured by holding the page lock in page_mkwrite() while
2260          * checking fuse_page_is_writeback().  We already hold the page lock
2261          * since clear_page_dirty_for_io() and keep it held until we add the
2262          * request to the fi->writepages list and increment ap->num_pages.
2263          * After this fuse_page_is_writeback() will indicate that the page is
2264          * under writeback, so we can release the page lock.
2265          */
2266         if (data->wpa == NULL) {
2267                 err = -ENOMEM;
2268                 wpa = fuse_writepage_args_alloc();
2269                 if (!wpa) {
2270                         __free_page(tmp_page);
2271                         goto out_unlock;
2272                 }
2273                 fuse_writepage_add_to_bucket(fc, wpa);
2274
2275                 data->max_pages = 1;
2276
2277                 ap = &wpa->ia.ap;
2278                 fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0);
2279                 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
2280                 wpa->next = NULL;
2281                 ap->args.in_pages = true;
2282                 ap->args.end = fuse_writepage_end;
2283                 ap->num_pages = 0;
2284                 wpa->inode = inode;
2285         }
2286         set_page_writeback(page);
2287
2288         copy_highpage(tmp_page, page);
2289         ap->pages[ap->num_pages] = tmp_page;
2290         ap->descs[ap->num_pages].offset = 0;
2291         ap->descs[ap->num_pages].length = PAGE_SIZE;
2292         data->orig_pages[ap->num_pages] = page;
2293
2294         inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
2295         inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
2296
2297         err = 0;
2298         if (data->wpa) {
2299                 /*
2300                  * Protected by fi->lock against concurrent access by
2301                  * fuse_page_is_writeback().
2302                  */
2303                 spin_lock(&fi->lock);
2304                 ap->num_pages++;
2305                 spin_unlock(&fi->lock);
2306         } else if (fuse_writepage_add(wpa, page)) {
2307                 data->wpa = wpa;
2308         } else {
2309                 end_page_writeback(page);
2310         }
2311 out_unlock:
2312         unlock_page(page);
2313
2314         return err;
2315 }
2316
2317 static int fuse_writepages(struct address_space *mapping,
2318                            struct writeback_control *wbc)
2319 {
2320         struct inode *inode = mapping->host;
2321         struct fuse_conn *fc = get_fuse_conn(inode);
2322         struct fuse_fill_wb_data data;
2323         int err;
2324
2325         err = -EIO;
2326         if (fuse_is_bad(inode))
2327                 goto out;
2328
2329         if (wbc->sync_mode == WB_SYNC_NONE &&
2330             fc->num_background >= fc->congestion_threshold)
2331                 return 0;
2332
2333         data.inode = inode;
2334         data.wpa = NULL;
2335         data.ff = NULL;
2336
2337         err = -ENOMEM;
2338         data.orig_pages = kcalloc(fc->max_pages,
2339                                   sizeof(struct page *),
2340                                   GFP_NOFS);
2341         if (!data.orig_pages)
2342                 goto out;
2343
2344         err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
2345         if (data.wpa) {
2346                 WARN_ON(!data.wpa->ia.ap.num_pages);
2347                 fuse_writepages_send(&data);
2348         }
2349         if (data.ff)
2350                 fuse_file_put(data.ff, false, false);
2351
2352         kfree(data.orig_pages);
2353 out:
2354         return err;
2355 }
2356
2357 /*
2358  * It's worthy to make sure that space is reserved on disk for the write,
2359  * but how to implement it without killing performance need more thinking.
2360  */
2361 static int fuse_write_begin(struct file *file, struct address_space *mapping,
2362                 loff_t pos, unsigned len, struct page **pagep, void **fsdata)
2363 {
2364         pgoff_t index = pos >> PAGE_SHIFT;
2365         struct fuse_conn *fc = get_fuse_conn(file_inode(file));
2366         struct page *page;
2367         loff_t fsize;
2368         int err = -ENOMEM;
2369
2370         WARN_ON(!fc->writeback_cache);
2371
2372         page = grab_cache_page_write_begin(mapping, index);
2373         if (!page)
2374                 goto error;
2375
2376         fuse_wait_on_page_writeback(mapping->host, page->index);
2377
2378         if (PageUptodate(page) || len == PAGE_SIZE)
2379                 goto success;
2380         /*
2381          * Check if the start this page comes after the end of file, in which
2382          * case the readpage can be optimized away.
2383          */
2384         fsize = i_size_read(mapping->host);
2385         if (fsize <= (pos & PAGE_MASK)) {
2386                 size_t off = pos & ~PAGE_MASK;
2387                 if (off)
2388                         zero_user_segment(page, 0, off);
2389                 goto success;
2390         }
2391         err = fuse_do_readpage(file, page);
2392         if (err)
2393                 goto cleanup;
2394 success:
2395         *pagep = page;
2396         return 0;
2397
2398 cleanup:
2399         unlock_page(page);
2400         put_page(page);
2401 error:
2402         return err;
2403 }
2404
2405 static int fuse_write_end(struct file *file, struct address_space *mapping,
2406                 loff_t pos, unsigned len, unsigned copied,
2407                 struct page *page, void *fsdata)
2408 {
2409         struct inode *inode = page->mapping->host;
2410
2411         /* Haven't copied anything?  Skip zeroing, size extending, dirtying. */
2412         if (!copied)
2413                 goto unlock;
2414
2415         pos += copied;
2416         if (!PageUptodate(page)) {
2417                 /* Zero any unwritten bytes at the end of the page */
2418                 size_t endoff = pos & ~PAGE_MASK;
2419                 if (endoff)
2420                         zero_user_segment(page, endoff, PAGE_SIZE);
2421                 SetPageUptodate(page);
2422         }
2423
2424         if (pos > inode->i_size)
2425                 i_size_write(inode, pos);
2426
2427         set_page_dirty(page);
2428
2429 unlock:
2430         unlock_page(page);
2431         put_page(page);
2432
2433         return copied;
2434 }
2435
2436 static int fuse_launder_folio(struct folio *folio)
2437 {
2438         int err = 0;
2439         if (folio_clear_dirty_for_io(folio)) {
2440                 struct inode *inode = folio->mapping->host;
2441
2442                 /* Serialize with pending writeback for the same page */
2443                 fuse_wait_on_page_writeback(inode, folio->index);
2444                 err = fuse_writepage_locked(&folio->page);
2445                 if (!err)
2446                         fuse_wait_on_page_writeback(inode, folio->index);
2447         }
2448         return err;
2449 }
2450
2451 /*
2452  * Write back dirty data/metadata now (there may not be any suitable
2453  * open files later for data)
2454  */
2455 static void fuse_vma_close(struct vm_area_struct *vma)
2456 {
2457         int err;
2458
2459         err = write_inode_now(vma->vm_file->f_mapping->host, 1);
2460         mapping_set_error(vma->vm_file->f_mapping, err);
2461 }
2462
2463 /*
2464  * Wait for writeback against this page to complete before allowing it
2465  * to be marked dirty again, and hence written back again, possibly
2466  * before the previous writepage completed.
2467  *
2468  * Block here, instead of in ->writepage(), so that the userspace fs
2469  * can only block processes actually operating on the filesystem.
2470  *
2471  * Otherwise unprivileged userspace fs would be able to block
2472  * unrelated:
2473  *
2474  * - page migration
2475  * - sync(2)
2476  * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
2477  */
2478 static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
2479 {
2480         struct page *page = vmf->page;
2481         struct inode *inode = file_inode(vmf->vma->vm_file);
2482
2483         file_update_time(vmf->vma->vm_file);
2484         lock_page(page);
2485         if (page->mapping != inode->i_mapping) {
2486                 unlock_page(page);
2487                 return VM_FAULT_NOPAGE;
2488         }
2489
2490         fuse_wait_on_page_writeback(inode, page->index);
2491         return VM_FAULT_LOCKED;
2492 }
2493
2494 static const struct vm_operations_struct fuse_file_vm_ops = {
2495         .close          = fuse_vma_close,
2496         .fault          = filemap_fault,
2497         .map_pages      = filemap_map_pages,
2498         .page_mkwrite   = fuse_page_mkwrite,
2499 };
2500
2501 static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2502 {
2503         struct fuse_file *ff = file->private_data;
2504
2505         /* DAX mmap is superior to direct_io mmap */
2506         if (FUSE_IS_DAX(file_inode(file)))
2507                 return fuse_dax_mmap(file, vma);
2508
2509         if (ff->open_flags & FOPEN_DIRECT_IO) {
2510                 /* Can't provide the coherency needed for MAP_SHARED */
2511                 if (vma->vm_flags & VM_MAYSHARE)
2512                         return -ENODEV;
2513
2514                 invalidate_inode_pages2(file->f_mapping);
2515
2516                 return generic_file_mmap(file, vma);
2517         }
2518
2519         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2520                 fuse_link_write_file(file);
2521
2522         file_accessed(file);
2523         vma->vm_ops = &fuse_file_vm_ops;
2524         return 0;
2525 }
2526
2527 static int convert_fuse_file_lock(struct fuse_conn *fc,
2528                                   const struct fuse_file_lock *ffl,
2529                                   struct file_lock *fl)
2530 {
2531         switch (ffl->type) {
2532         case F_UNLCK:
2533                 break;
2534
2535         case F_RDLCK:
2536         case F_WRLCK:
2537                 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2538                     ffl->end < ffl->start)
2539                         return -EIO;
2540
2541                 fl->fl_start = ffl->start;
2542                 fl->fl_end = ffl->end;
2543
2544                 /*
2545                  * Convert pid into init's pid namespace.  The locks API will
2546                  * translate it into the caller's pid namespace.
2547                  */
2548                 rcu_read_lock();
2549                 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
2550                 rcu_read_unlock();
2551                 break;
2552
2553         default:
2554                 return -EIO;
2555         }
2556         fl->fl_type = ffl->type;
2557         return 0;
2558 }
2559
2560 static void fuse_lk_fill(struct fuse_args *args, struct file *file,
2561                          const struct file_lock *fl, int opcode, pid_t pid,
2562                          int flock, struct fuse_lk_in *inarg)
2563 {
2564         struct inode *inode = file_inode(file);
2565         struct fuse_conn *fc = get_fuse_conn(inode);
2566         struct fuse_file *ff = file->private_data;
2567
2568         memset(inarg, 0, sizeof(*inarg));
2569         inarg->fh = ff->fh;
2570         inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2571         inarg->lk.start = fl->fl_start;
2572         inarg->lk.end = fl->fl_end;
2573         inarg->lk.type = fl->fl_type;
2574         inarg->lk.pid = pid;
2575         if (flock)
2576                 inarg->lk_flags |= FUSE_LK_FLOCK;
2577         args->opcode = opcode;
2578         args->nodeid = get_node_id(inode);
2579         args->in_numargs = 1;
2580         args->in_args[0].size = sizeof(*inarg);
2581         args->in_args[0].value = inarg;
2582 }
2583
2584 static int fuse_getlk(struct file *file, struct file_lock *fl)
2585 {
2586         struct inode *inode = file_inode(file);
2587         struct fuse_mount *fm = get_fuse_mount(inode);
2588         FUSE_ARGS(args);
2589         struct fuse_lk_in inarg;
2590         struct fuse_lk_out outarg;
2591         int err;
2592
2593         fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
2594         args.out_numargs = 1;
2595         args.out_args[0].size = sizeof(outarg);
2596         args.out_args[0].value = &outarg;
2597         err = fuse_simple_request(fm, &args);
2598         if (!err)
2599                 err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
2600
2601         return err;
2602 }
2603
2604 static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2605 {
2606         struct inode *inode = file_inode(file);
2607         struct fuse_mount *fm = get_fuse_mount(inode);
2608         FUSE_ARGS(args);
2609         struct fuse_lk_in inarg;
2610         int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2611         struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL;
2612         pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
2613         int err;
2614
2615         if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2616                 /* NLM needs asynchronous locks, which we don't support yet */
2617                 return -ENOLCK;
2618         }
2619
2620         /* Unlock on close is handled by the flush method */
2621         if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX)
2622                 return 0;
2623
2624         fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
2625         err = fuse_simple_request(fm, &args);
2626
2627         /* locking is restartable */
2628         if (err == -EINTR)
2629                 err = -ERESTARTSYS;
2630
2631         return err;
2632 }
2633
2634 static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2635 {
2636         struct inode *inode = file_inode(file);
2637         struct fuse_conn *fc = get_fuse_conn(inode);
2638         int err;
2639
2640         if (cmd == F_CANCELLK) {
2641                 err = 0;
2642         } else if (cmd == F_GETLK) {
2643                 if (fc->no_lock) {
2644                         posix_test_lock(file, fl);
2645                         err = 0;
2646                 } else
2647                         err = fuse_getlk(file, fl);
2648         } else {
2649                 if (fc->no_lock)
2650                         err = posix_lock_file(file, fl, NULL);
2651                 else
2652                         err = fuse_setlk(file, fl, 0);
2653         }
2654         return err;
2655 }
2656
2657 static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2658 {
2659         struct inode *inode = file_inode(file);
2660         struct fuse_conn *fc = get_fuse_conn(inode);
2661         int err;
2662
2663         if (fc->no_flock) {
2664                 err = locks_lock_file_wait(file, fl);
2665         } else {
2666                 struct fuse_file *ff = file->private_data;
2667
2668                 /* emulate flock with POSIX locks */
2669                 ff->flock = true;
2670                 err = fuse_setlk(file, fl, 1);
2671         }
2672
2673         return err;
2674 }
2675
2676 static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2677 {
2678         struct inode *inode = mapping->host;
2679         struct fuse_mount *fm = get_fuse_mount(inode);
2680         FUSE_ARGS(args);
2681         struct fuse_bmap_in inarg;
2682         struct fuse_bmap_out outarg;
2683         int err;
2684
2685         if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
2686                 return 0;
2687
2688         memset(&inarg, 0, sizeof(inarg));
2689         inarg.block = block;
2690         inarg.blocksize = inode->i_sb->s_blocksize;
2691         args.opcode = FUSE_BMAP;
2692         args.nodeid = get_node_id(inode);
2693         args.in_numargs = 1;
2694         args.in_args[0].size = sizeof(inarg);
2695         args.in_args[0].value = &inarg;
2696         args.out_numargs = 1;
2697         args.out_args[0].size = sizeof(outarg);
2698         args.out_args[0].value = &outarg;
2699         err = fuse_simple_request(fm, &args);
2700         if (err == -ENOSYS)
2701                 fm->fc->no_bmap = 1;
2702
2703         return err ? 0 : outarg.block;
2704 }
2705
2706 static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2707 {
2708         struct inode *inode = file->f_mapping->host;
2709         struct fuse_mount *fm = get_fuse_mount(inode);
2710         struct fuse_file *ff = file->private_data;
2711         FUSE_ARGS(args);
2712         struct fuse_lseek_in inarg = {
2713                 .fh = ff->fh,
2714                 .offset = offset,
2715                 .whence = whence
2716         };
2717         struct fuse_lseek_out outarg;
2718         int err;
2719
2720         if (fm->fc->no_lseek)
2721                 goto fallback;
2722
2723         args.opcode = FUSE_LSEEK;
2724         args.nodeid = ff->nodeid;
2725         args.in_numargs = 1;
2726         args.in_args[0].size = sizeof(inarg);
2727         args.in_args[0].value = &inarg;
2728         args.out_numargs = 1;
2729         args.out_args[0].size = sizeof(outarg);
2730         args.out_args[0].value = &outarg;
2731         err = fuse_simple_request(fm, &args);
2732         if (err) {
2733                 if (err == -ENOSYS) {
2734                         fm->fc->no_lseek = 1;
2735                         goto fallback;
2736                 }
2737                 return err;
2738         }
2739
2740         return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2741
2742 fallback:
2743         err = fuse_update_attributes(inode, file, STATX_SIZE);
2744         if (!err)
2745                 return generic_file_llseek(file, offset, whence);
2746         else
2747                 return err;
2748 }
2749
2750 static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2751 {
2752         loff_t retval;
2753         struct inode *inode = file_inode(file);
2754
2755         switch (whence) {
2756         case SEEK_SET:
2757         case SEEK_CUR:
2758                  /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */
2759                 retval = generic_file_llseek(file, offset, whence);
2760                 break;
2761         case SEEK_END:
2762                 inode_lock(inode);
2763                 retval = fuse_update_attributes(inode, file, STATX_SIZE);
2764                 if (!retval)
2765                         retval = generic_file_llseek(file, offset, whence);
2766                 inode_unlock(inode);
2767                 break;
2768         case SEEK_HOLE:
2769         case SEEK_DATA:
2770                 inode_lock(inode);
2771                 retval = fuse_lseek(file, offset, whence);
2772                 inode_unlock(inode);
2773                 break;
2774         default:
2775                 retval = -EINVAL;
2776         }
2777
2778         return retval;
2779 }
2780
2781 /*
2782  * All files which have been polled are linked to RB tree
2783  * fuse_conn->polled_files which is indexed by kh.  Walk the tree and
2784  * find the matching one.
2785  */
2786 static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2787                                               struct rb_node **parent_out)
2788 {
2789         struct rb_node **link = &fc->polled_files.rb_node;
2790         struct rb_node *last = NULL;
2791
2792         while (*link) {
2793                 struct fuse_file *ff;
2794
2795                 last = *link;
2796                 ff = rb_entry(last, struct fuse_file, polled_node);
2797
2798                 if (kh < ff->kh)
2799                         link = &last->rb_left;
2800                 else if (kh > ff->kh)
2801                         link = &last->rb_right;
2802                 else
2803                         return link;
2804         }
2805
2806         if (parent_out)
2807                 *parent_out = last;
2808         return link;
2809 }
2810
2811 /*
2812  * The file is about to be polled.  Make sure it's on the polled_files
2813  * RB tree.  Note that files once added to the polled_files tree are
2814  * not removed before the file is released.  This is because a file
2815  * polled once is likely to be polled again.
2816  */
2817 static void fuse_register_polled_file(struct fuse_conn *fc,
2818                                       struct fuse_file *ff)
2819 {
2820         spin_lock(&fc->lock);
2821         if (RB_EMPTY_NODE(&ff->polled_node)) {
2822                 struct rb_node **link, *parent;
2823
2824                 link = fuse_find_polled_node(fc, ff->kh, &parent);
2825                 BUG_ON(*link);
2826                 rb_link_node(&ff->polled_node, parent, link);
2827                 rb_insert_color(&ff->polled_node, &fc->polled_files);
2828         }
2829         spin_unlock(&fc->lock);
2830 }
2831
2832 __poll_t fuse_file_poll(struct file *file, poll_table *wait)
2833 {
2834         struct fuse_file *ff = file->private_data;
2835         struct fuse_mount *fm = ff->fm;
2836         struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
2837         struct fuse_poll_out outarg;
2838         FUSE_ARGS(args);
2839         int err;
2840
2841         if (fm->fc->no_poll)
2842                 return DEFAULT_POLLMASK;
2843
2844         poll_wait(file, &ff->poll_wait, wait);
2845         inarg.events = mangle_poll(poll_requested_events(wait));
2846
2847         /*
2848          * Ask for notification iff there's someone waiting for it.
2849          * The client may ignore the flag and always notify.
2850          */
2851         if (waitqueue_active(&ff->poll_wait)) {
2852                 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
2853                 fuse_register_polled_file(fm->fc, ff);
2854         }
2855
2856         args.opcode = FUSE_POLL;
2857         args.nodeid = ff->nodeid;
2858         args.in_numargs = 1;
2859         args.in_args[0].size = sizeof(inarg);
2860         args.in_args[0].value = &inarg;
2861         args.out_numargs = 1;
2862         args.out_args[0].size = sizeof(outarg);
2863         args.out_args[0].value = &outarg;
2864         err = fuse_simple_request(fm, &args);
2865
2866         if (!err)
2867                 return demangle_poll(outarg.revents);
2868         if (err == -ENOSYS) {
2869                 fm->fc->no_poll = 1;
2870                 return DEFAULT_POLLMASK;
2871         }
2872         return EPOLLERR;
2873 }
2874 EXPORT_SYMBOL_GPL(fuse_file_poll);
2875
2876 /*
2877  * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
2878  * wakes up the poll waiters.
2879  */
2880 int fuse_notify_poll_wakeup(struct fuse_conn *fc,
2881                             struct fuse_notify_poll_wakeup_out *outarg)
2882 {
2883         u64 kh = outarg->kh;
2884         struct rb_node **link;
2885
2886         spin_lock(&fc->lock);
2887
2888         link = fuse_find_polled_node(fc, kh, NULL);
2889         if (*link) {
2890                 struct fuse_file *ff;
2891
2892                 ff = rb_entry(*link, struct fuse_file, polled_node);
2893                 wake_up_interruptible_sync(&ff->poll_wait);
2894         }
2895
2896         spin_unlock(&fc->lock);
2897         return 0;
2898 }
2899
2900 static void fuse_do_truncate(struct file *file)
2901 {
2902         struct inode *inode = file->f_mapping->host;
2903         struct iattr attr;
2904
2905         attr.ia_valid = ATTR_SIZE;
2906         attr.ia_size = i_size_read(inode);
2907
2908         attr.ia_file = file;
2909         attr.ia_valid |= ATTR_FILE;
2910
2911         fuse_do_setattr(file_dentry(file), &attr, file);
2912 }
2913
2914 static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
2915 {
2916         return round_up(off, fc->max_pages << PAGE_SHIFT);
2917 }
2918
2919 static ssize_t
2920 fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
2921 {
2922         DECLARE_COMPLETION_ONSTACK(wait);
2923         ssize_t ret = 0;
2924         struct file *file = iocb->ki_filp;
2925         struct fuse_file *ff = file->private_data;
2926         loff_t pos = 0;
2927         struct inode *inode;
2928         loff_t i_size;
2929         size_t count = iov_iter_count(iter), shortened = 0;
2930         loff_t offset = iocb->ki_pos;
2931         struct fuse_io_priv *io;
2932
2933         pos = offset;
2934         inode = file->f_mapping->host;
2935         i_size = i_size_read(inode);
2936
2937         if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
2938                 return 0;
2939
2940         io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
2941         if (!io)
2942                 return -ENOMEM;
2943         spin_lock_init(&io->lock);
2944         kref_init(&io->refcnt);
2945         io->reqs = 1;
2946         io->bytes = -1;
2947         io->size = 0;
2948         io->offset = offset;
2949         io->write = (iov_iter_rw(iter) == WRITE);
2950         io->err = 0;
2951         /*
2952          * By default, we want to optimize all I/Os with async request
2953          * submission to the client filesystem if supported.
2954          */
2955         io->async = ff->fm->fc->async_dio;
2956         io->iocb = iocb;
2957         io->blocking = is_sync_kiocb(iocb);
2958
2959         /* optimization for short read */
2960         if (io->async && !io->write && offset + count > i_size) {
2961                 iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
2962                 shortened = count - iov_iter_count(iter);
2963                 count -= shortened;
2964         }
2965
2966         /*
2967          * We cannot asynchronously extend the size of a file.
2968          * In such case the aio will behave exactly like sync io.
2969          */
2970         if ((offset + count > i_size) && io->write)
2971                 io->blocking = true;
2972
2973         if (io->async && io->blocking) {
2974                 /*
2975                  * Additional reference to keep io around after
2976                  * calling fuse_aio_complete()
2977                  */
2978                 kref_get(&io->refcnt);
2979                 io->done = &wait;
2980         }
2981
2982         if (iov_iter_rw(iter) == WRITE) {
2983                 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
2984                 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
2985         } else {
2986                 ret = __fuse_direct_read(io, iter, &pos);
2987         }
2988         iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
2989
2990         if (io->async) {
2991                 bool blocking = io->blocking;
2992
2993                 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
2994
2995                 /* we have a non-extending, async request, so return */
2996                 if (!blocking)
2997                         return -EIOCBQUEUED;
2998
2999                 wait_for_completion(&wait);
3000                 ret = fuse_get_res_by_io(io);
3001         }
3002
3003         kref_put(&io->refcnt, fuse_io_release);
3004
3005         if (iov_iter_rw(iter) == WRITE) {
3006                 fuse_write_update_attr(inode, pos, ret);
3007                 /* For extending writes we already hold exclusive lock */
3008                 if (ret < 0 && offset + count > i_size)
3009                         fuse_do_truncate(file);
3010         }
3011
3012         return ret;
3013 }
3014
3015 static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
3016 {
3017         int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);
3018
3019         if (!err)
3020                 fuse_sync_writes(inode);
3021
3022         return err;
3023 }
3024
3025 static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
3026                                 loff_t length)
3027 {
3028         struct fuse_file *ff = file->private_data;
3029         struct inode *inode = file_inode(file);
3030         struct fuse_inode *fi = get_fuse_inode(inode);
3031         struct fuse_mount *fm = ff->fm;
3032         FUSE_ARGS(args);
3033         struct fuse_fallocate_in inarg = {
3034                 .fh = ff->fh,
3035                 .offset = offset,
3036                 .length = length,
3037                 .mode = mode
3038         };
3039         int err;
3040         bool block_faults = FUSE_IS_DAX(inode) &&
3041                 (!(mode & FALLOC_FL_KEEP_SIZE) ||
3042                  (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)));
3043
3044         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3045                      FALLOC_FL_ZERO_RANGE))
3046                 return -EOPNOTSUPP;
3047
3048         if (fm->fc->no_fallocate)
3049                 return -EOPNOTSUPP;
3050
3051         inode_lock(inode);
3052         if (block_faults) {
3053                 filemap_invalidate_lock(inode->i_mapping);
3054                 err = fuse_dax_break_layouts(inode, 0, 0);
3055                 if (err)
3056                         goto out;
3057         }
3058
3059         if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) {
3060                 loff_t endbyte = offset + length - 1;
3061
3062                 err = fuse_writeback_range(inode, offset, endbyte);
3063                 if (err)
3064                         goto out;
3065         }
3066
3067         if (!(mode & FALLOC_FL_KEEP_SIZE) &&
3068             offset + length > i_size_read(inode)) {
3069                 err = inode_newsize_ok(inode, offset + length);
3070                 if (err)
3071                         goto out;
3072         }
3073
3074         err = file_modified(file);
3075         if (err)
3076                 goto out;
3077
3078         if (!(mode & FALLOC_FL_KEEP_SIZE))
3079                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3080
3081         args.opcode = FUSE_FALLOCATE;
3082         args.nodeid = ff->nodeid;
3083         args.in_numargs = 1;
3084         args.in_args[0].size = sizeof(inarg);
3085         args.in_args[0].value = &inarg;
3086         err = fuse_simple_request(fm, &args);
3087         if (err == -ENOSYS) {
3088                 fm->fc->no_fallocate = 1;
3089                 err = -EOPNOTSUPP;
3090         }
3091         if (err)
3092                 goto out;
3093
3094         /* we could have extended the file */
3095         if (!(mode & FALLOC_FL_KEEP_SIZE)) {
3096                 if (fuse_write_update_attr(inode, offset + length, length))
3097                         file_update_time(file);
3098         }
3099
3100         if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE))
3101                 truncate_pagecache_range(inode, offset, offset + length - 1);
3102
3103         fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE);
3104
3105 out:
3106         if (!(mode & FALLOC_FL_KEEP_SIZE))
3107                 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3108
3109         if (block_faults)
3110                 filemap_invalidate_unlock(inode->i_mapping);
3111
3112         inode_unlock(inode);
3113
3114         fuse_flush_time_update(inode);
3115
3116         return err;
3117 }
3118
3119 static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
3120                                       struct file *file_out, loff_t pos_out,
3121                                       size_t len, unsigned int flags)
3122 {
3123         struct fuse_file *ff_in = file_in->private_data;
3124         struct fuse_file *ff_out = file_out->private_data;
3125         struct inode *inode_in = file_inode(file_in);
3126         struct inode *inode_out = file_inode(file_out);
3127         struct fuse_inode *fi_out = get_fuse_inode(inode_out);
3128         struct fuse_mount *fm = ff_in->fm;
3129         struct fuse_conn *fc = fm->fc;
3130         FUSE_ARGS(args);
3131         struct fuse_copy_file_range_in inarg = {
3132                 .fh_in = ff_in->fh,
3133                 .off_in = pos_in,
3134                 .nodeid_out = ff_out->nodeid,
3135                 .fh_out = ff_out->fh,
3136                 .off_out = pos_out,
3137                 .len = len,
3138                 .flags = flags
3139         };
3140         struct fuse_write_out outarg;
3141         ssize_t err;
3142         /* mark unstable when write-back is not used, and file_out gets
3143          * extended */
3144         bool is_unstable = (!fc->writeback_cache) &&
3145                            ((pos_out + len) > inode_out->i_size);
3146
3147         if (fc->no_copy_file_range)
3148                 return -EOPNOTSUPP;
3149
3150         if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
3151                 return -EXDEV;
3152
3153         inode_lock(inode_in);
3154         err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
3155         inode_unlock(inode_in);
3156         if (err)
3157                 return err;
3158
3159         inode_lock(inode_out);
3160
3161         err = file_modified(file_out);
3162         if (err)
3163                 goto out;
3164
3165         /*
3166          * Write out dirty pages in the destination file before sending the COPY
3167          * request to userspace.  After the request is completed, truncate off
3168          * pages (including partial ones) from the cache that have been copied,
3169          * since these contain stale data at that point.
3170          *
3171          * This should be mostly correct, but if the COPY writes to partial
3172          * pages (at the start or end) and the parts not covered by the COPY are
3173          * written through a memory map after calling fuse_writeback_range(),
3174          * then these partial page modifications will be lost on truncation.
3175          *
3176          * It is unlikely that someone would rely on such mixed style
3177          * modifications.  Yet this does give less guarantees than if the
3178          * copying was performed with write(2).
3179          *
3180          * To fix this a mapping->invalidate_lock could be used to prevent new
3181          * faults while the copy is ongoing.
3182          */
3183         err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
3184         if (err)
3185                 goto out;
3186
3187         if (is_unstable)
3188                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3189
3190         args.opcode = FUSE_COPY_FILE_RANGE;
3191         args.nodeid = ff_in->nodeid;
3192         args.in_numargs = 1;
3193         args.in_args[0].size = sizeof(inarg);
3194         args.in_args[0].value = &inarg;
3195         args.out_numargs = 1;
3196         args.out_args[0].size = sizeof(outarg);
3197         args.out_args[0].value = &outarg;
3198         err = fuse_simple_request(fm, &args);
3199         if (err == -ENOSYS) {
3200                 fc->no_copy_file_range = 1;
3201                 err = -EOPNOTSUPP;
3202         }
3203         if (err)
3204                 goto out;
3205
3206         truncate_inode_pages_range(inode_out->i_mapping,
3207                                    ALIGN_DOWN(pos_out, PAGE_SIZE),
3208                                    ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1);
3209
3210         file_update_time(file_out);
3211         fuse_write_update_attr(inode_out, pos_out + outarg.size, outarg.size);
3212
3213         err = outarg.size;
3214 out:
3215         if (is_unstable)
3216                 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3217
3218         inode_unlock(inode_out);
3219         file_accessed(file_in);
3220
3221         fuse_flush_time_update(inode_out);
3222
3223         return err;
3224 }
3225
3226 static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
3227                                     struct file *dst_file, loff_t dst_off,
3228                                     size_t len, unsigned int flags)
3229 {
3230         ssize_t ret;
3231
3232         ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
3233                                      len, flags);
3234
3235         if (ret == -EOPNOTSUPP || ret == -EXDEV)
3236                 ret = generic_copy_file_range(src_file, src_off, dst_file,
3237                                               dst_off, len, flags);
3238         return ret;
3239 }
3240
3241 static const struct file_operations fuse_file_operations = {
3242         .llseek         = fuse_file_llseek,
3243         .read_iter      = fuse_file_read_iter,
3244         .write_iter     = fuse_file_write_iter,
3245         .mmap           = fuse_file_mmap,
3246         .open           = fuse_open,
3247         .flush          = fuse_flush,
3248         .release        = fuse_release,
3249         .fsync          = fuse_fsync,
3250         .lock           = fuse_file_lock,
3251         .get_unmapped_area = thp_get_unmapped_area,
3252         .flock          = fuse_file_flock,
3253         .splice_read    = generic_file_splice_read,
3254         .splice_write   = iter_file_splice_write,
3255         .unlocked_ioctl = fuse_file_ioctl,
3256         .compat_ioctl   = fuse_file_compat_ioctl,
3257         .poll           = fuse_file_poll,
3258         .fallocate      = fuse_file_fallocate,
3259         .copy_file_range = fuse_copy_file_range,
3260 };
3261
3262 static const struct address_space_operations fuse_file_aops  = {
3263         .read_folio     = fuse_read_folio,
3264         .readahead      = fuse_readahead,
3265         .writepage      = fuse_writepage,
3266         .writepages     = fuse_writepages,
3267         .launder_folio  = fuse_launder_folio,
3268         .dirty_folio    = filemap_dirty_folio,
3269         .bmap           = fuse_bmap,
3270         .direct_IO      = fuse_direct_IO,
3271         .write_begin    = fuse_write_begin,
3272         .write_end      = fuse_write_end,
3273 };
3274
3275 void fuse_init_file_inode(struct inode *inode, unsigned int flags)
3276 {
3277         struct fuse_inode *fi = get_fuse_inode(inode);
3278
3279         inode->i_fop = &fuse_file_operations;
3280         inode->i_data.a_ops = &fuse_file_aops;
3281
3282         INIT_LIST_HEAD(&fi->write_files);
3283         INIT_LIST_HEAD(&fi->queued_writes);
3284         fi->writectr = 0;
3285         init_waitqueue_head(&fi->page_waitq);
3286         fi->writepages = RB_ROOT;
3287
3288         if (IS_ENABLED(CONFIG_FUSE_DAX))
3289                 fuse_dax_inode_init(inode, flags);
3290 }