mm: rid swapoff of quadratic complexity
[sfrench/cifs-2.6.git] / mm / shmem.c
1 /*
2  * Resizable virtual memory filesystem for Linux.
3  *
4  * Copyright (C) 2000 Linus Torvalds.
5  *               2000 Transmeta Corp.
6  *               2000-2001 Christoph Rohland
7  *               2000-2001 SAP AG
8  *               2002 Red Hat Inc.
9  * Copyright (C) 2002-2011 Hugh Dickins.
10  * Copyright (C) 2011 Google Inc.
11  * Copyright (C) 2002-2005 VERITAS Software Corporation.
12  * Copyright (C) 2004 Andi Kleen, SuSE Labs
13  *
14  * Extended attribute support for tmpfs:
15  * Copyright (c) 2004, Luke Kenneth Casson Leighton <lkcl@lkcl.net>
16  * Copyright (c) 2004 Red Hat, Inc., James Morris <jmorris@redhat.com>
17  *
18  * tiny-shmem:
19  * Copyright (c) 2004, 2008 Matt Mackall <mpm@selenic.com>
20  *
21  * This file is released under the GPL.
22  */
23
24 #include <linux/fs.h>
25 #include <linux/init.h>
26 #include <linux/vfs.h>
27 #include <linux/mount.h>
28 #include <linux/ramfs.h>
29 #include <linux/pagemap.h>
30 #include <linux/file.h>
31 #include <linux/mm.h>
32 #include <linux/random.h>
33 #include <linux/sched/signal.h>
34 #include <linux/export.h>
35 #include <linux/swap.h>
36 #include <linux/uio.h>
37 #include <linux/khugepaged.h>
38 #include <linux/hugetlb.h>
39 #include <linux/frontswap.h>
40
41 #include <asm/tlbflush.h> /* for arch/microblaze update_mmu_cache() */
42
43 static struct vfsmount *shm_mnt;
44
45 #ifdef CONFIG_SHMEM
46 /*
47  * This virtual memory filesystem is heavily based on the ramfs. It
48  * extends ramfs by the ability to use swap and honor resource limits
49  * which makes it a completely usable filesystem.
50  */
51
52 #include <linux/xattr.h>
53 #include <linux/exportfs.h>
54 #include <linux/posix_acl.h>
55 #include <linux/posix_acl_xattr.h>
56 #include <linux/mman.h>
57 #include <linux/string.h>
58 #include <linux/slab.h>
59 #include <linux/backing-dev.h>
60 #include <linux/shmem_fs.h>
61 #include <linux/writeback.h>
62 #include <linux/blkdev.h>
63 #include <linux/pagevec.h>
64 #include <linux/percpu_counter.h>
65 #include <linux/falloc.h>
66 #include <linux/splice.h>
67 #include <linux/security.h>
68 #include <linux/swapops.h>
69 #include <linux/mempolicy.h>
70 #include <linux/namei.h>
71 #include <linux/ctype.h>
72 #include <linux/migrate.h>
73 #include <linux/highmem.h>
74 #include <linux/seq_file.h>
75 #include <linux/magic.h>
76 #include <linux/syscalls.h>
77 #include <linux/fcntl.h>
78 #include <uapi/linux/memfd.h>
79 #include <linux/userfaultfd_k.h>
80 #include <linux/rmap.h>
81 #include <linux/uuid.h>
82
83 #include <linux/uaccess.h>
84 #include <asm/pgtable.h>
85
86 #include "internal.h"
87
88 #define BLOCKS_PER_PAGE  (PAGE_SIZE/512)
89 #define VM_ACCT(size)    (PAGE_ALIGN(size) >> PAGE_SHIFT)
90
91 /* Pretend that each entry is of this size in directory's i_size */
92 #define BOGO_DIRENT_SIZE 20
93
94 /* Symlink up to this size is kmalloc'ed instead of using a swappable page */
95 #define SHORT_SYMLINK_LEN 128
96
97 /*
98  * shmem_fallocate communicates with shmem_fault or shmem_writepage via
99  * inode->i_private (with i_mutex making sure that it has only one user at
100  * a time): we would prefer not to enlarge the shmem inode just for that.
101  */
102 struct shmem_falloc {
103         wait_queue_head_t *waitq; /* faults into hole wait for punch to end */
104         pgoff_t start;          /* start of range currently being fallocated */
105         pgoff_t next;           /* the next page offset to be fallocated */
106         pgoff_t nr_falloced;    /* how many new pages have been fallocated */
107         pgoff_t nr_unswapped;   /* how often writepage refused to swap out */
108 };
109
110 #ifdef CONFIG_TMPFS
111 static unsigned long shmem_default_max_blocks(void)
112 {
113         return totalram_pages() / 2;
114 }
115
116 static unsigned long shmem_default_max_inodes(void)
117 {
118         unsigned long nr_pages = totalram_pages();
119
120         return min(nr_pages - totalhigh_pages(), nr_pages / 2);
121 }
122 #endif
123
124 static bool shmem_should_replace_page(struct page *page, gfp_t gfp);
125 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
126                                 struct shmem_inode_info *info, pgoff_t index);
127 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
128                              struct page **pagep, enum sgp_type sgp,
129                              gfp_t gfp, struct vm_area_struct *vma,
130                              vm_fault_t *fault_type);
131 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
132                 struct page **pagep, enum sgp_type sgp,
133                 gfp_t gfp, struct vm_area_struct *vma,
134                 struct vm_fault *vmf, vm_fault_t *fault_type);
135
136 int shmem_getpage(struct inode *inode, pgoff_t index,
137                 struct page **pagep, enum sgp_type sgp)
138 {
139         return shmem_getpage_gfp(inode, index, pagep, sgp,
140                 mapping_gfp_mask(inode->i_mapping), NULL, NULL, NULL);
141 }
142
143 static inline struct shmem_sb_info *SHMEM_SB(struct super_block *sb)
144 {
145         return sb->s_fs_info;
146 }
147
148 /*
149  * shmem_file_setup pre-accounts the whole fixed size of a VM object,
150  * for shared memory and for shared anonymous (/dev/zero) mappings
151  * (unless MAP_NORESERVE and sysctl_overcommit_memory <= 1),
152  * consistent with the pre-accounting of private mappings ...
153  */
154 static inline int shmem_acct_size(unsigned long flags, loff_t size)
155 {
156         return (flags & VM_NORESERVE) ?
157                 0 : security_vm_enough_memory_mm(current->mm, VM_ACCT(size));
158 }
159
160 static inline void shmem_unacct_size(unsigned long flags, loff_t size)
161 {
162         if (!(flags & VM_NORESERVE))
163                 vm_unacct_memory(VM_ACCT(size));
164 }
165
166 static inline int shmem_reacct_size(unsigned long flags,
167                 loff_t oldsize, loff_t newsize)
168 {
169         if (!(flags & VM_NORESERVE)) {
170                 if (VM_ACCT(newsize) > VM_ACCT(oldsize))
171                         return security_vm_enough_memory_mm(current->mm,
172                                         VM_ACCT(newsize) - VM_ACCT(oldsize));
173                 else if (VM_ACCT(newsize) < VM_ACCT(oldsize))
174                         vm_unacct_memory(VM_ACCT(oldsize) - VM_ACCT(newsize));
175         }
176         return 0;
177 }
178
179 /*
180  * ... whereas tmpfs objects are accounted incrementally as
181  * pages are allocated, in order to allow large sparse files.
182  * shmem_getpage reports shmem_acct_block failure as -ENOSPC not -ENOMEM,
183  * so that a failure on a sparse tmpfs mapping will give SIGBUS not OOM.
184  */
185 static inline int shmem_acct_block(unsigned long flags, long pages)
186 {
187         if (!(flags & VM_NORESERVE))
188                 return 0;
189
190         return security_vm_enough_memory_mm(current->mm,
191                         pages * VM_ACCT(PAGE_SIZE));
192 }
193
194 static inline void shmem_unacct_blocks(unsigned long flags, long pages)
195 {
196         if (flags & VM_NORESERVE)
197                 vm_unacct_memory(pages * VM_ACCT(PAGE_SIZE));
198 }
199
200 static inline bool shmem_inode_acct_block(struct inode *inode, long pages)
201 {
202         struct shmem_inode_info *info = SHMEM_I(inode);
203         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
204
205         if (shmem_acct_block(info->flags, pages))
206                 return false;
207
208         if (sbinfo->max_blocks) {
209                 if (percpu_counter_compare(&sbinfo->used_blocks,
210                                            sbinfo->max_blocks - pages) > 0)
211                         goto unacct;
212                 percpu_counter_add(&sbinfo->used_blocks, pages);
213         }
214
215         return true;
216
217 unacct:
218         shmem_unacct_blocks(info->flags, pages);
219         return false;
220 }
221
222 static inline void shmem_inode_unacct_blocks(struct inode *inode, long pages)
223 {
224         struct shmem_inode_info *info = SHMEM_I(inode);
225         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
226
227         if (sbinfo->max_blocks)
228                 percpu_counter_sub(&sbinfo->used_blocks, pages);
229         shmem_unacct_blocks(info->flags, pages);
230 }
231
232 static const struct super_operations shmem_ops;
233 static const struct address_space_operations shmem_aops;
234 static const struct file_operations shmem_file_operations;
235 static const struct inode_operations shmem_inode_operations;
236 static const struct inode_operations shmem_dir_inode_operations;
237 static const struct inode_operations shmem_special_inode_operations;
238 static const struct vm_operations_struct shmem_vm_ops;
239 static struct file_system_type shmem_fs_type;
240
241 bool vma_is_shmem(struct vm_area_struct *vma)
242 {
243         return vma->vm_ops == &shmem_vm_ops;
244 }
245
246 static LIST_HEAD(shmem_swaplist);
247 static DEFINE_MUTEX(shmem_swaplist_mutex);
248
249 static int shmem_reserve_inode(struct super_block *sb)
250 {
251         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
252         if (sbinfo->max_inodes) {
253                 spin_lock(&sbinfo->stat_lock);
254                 if (!sbinfo->free_inodes) {
255                         spin_unlock(&sbinfo->stat_lock);
256                         return -ENOSPC;
257                 }
258                 sbinfo->free_inodes--;
259                 spin_unlock(&sbinfo->stat_lock);
260         }
261         return 0;
262 }
263
264 static void shmem_free_inode(struct super_block *sb)
265 {
266         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
267         if (sbinfo->max_inodes) {
268                 spin_lock(&sbinfo->stat_lock);
269                 sbinfo->free_inodes++;
270                 spin_unlock(&sbinfo->stat_lock);
271         }
272 }
273
274 /**
275  * shmem_recalc_inode - recalculate the block usage of an inode
276  * @inode: inode to recalc
277  *
278  * We have to calculate the free blocks since the mm can drop
279  * undirtied hole pages behind our back.
280  *
281  * But normally   info->alloced == inode->i_mapping->nrpages + info->swapped
282  * So mm freed is info->alloced - (inode->i_mapping->nrpages + info->swapped)
283  *
284  * It has to be called with the spinlock held.
285  */
286 static void shmem_recalc_inode(struct inode *inode)
287 {
288         struct shmem_inode_info *info = SHMEM_I(inode);
289         long freed;
290
291         freed = info->alloced - info->swapped - inode->i_mapping->nrpages;
292         if (freed > 0) {
293                 info->alloced -= freed;
294                 inode->i_blocks -= freed * BLOCKS_PER_PAGE;
295                 shmem_inode_unacct_blocks(inode, freed);
296         }
297 }
298
299 bool shmem_charge(struct inode *inode, long pages)
300 {
301         struct shmem_inode_info *info = SHMEM_I(inode);
302         unsigned long flags;
303
304         if (!shmem_inode_acct_block(inode, pages))
305                 return false;
306
307         /* nrpages adjustment first, then shmem_recalc_inode() when balanced */
308         inode->i_mapping->nrpages += pages;
309
310         spin_lock_irqsave(&info->lock, flags);
311         info->alloced += pages;
312         inode->i_blocks += pages * BLOCKS_PER_PAGE;
313         shmem_recalc_inode(inode);
314         spin_unlock_irqrestore(&info->lock, flags);
315
316         return true;
317 }
318
319 void shmem_uncharge(struct inode *inode, long pages)
320 {
321         struct shmem_inode_info *info = SHMEM_I(inode);
322         unsigned long flags;
323
324         /* nrpages adjustment done by __delete_from_page_cache() or caller */
325
326         spin_lock_irqsave(&info->lock, flags);
327         info->alloced -= pages;
328         inode->i_blocks -= pages * BLOCKS_PER_PAGE;
329         shmem_recalc_inode(inode);
330         spin_unlock_irqrestore(&info->lock, flags);
331
332         shmem_inode_unacct_blocks(inode, pages);
333 }
334
335 /*
336  * Replace item expected in xarray by a new item, while holding xa_lock.
337  */
338 static int shmem_replace_entry(struct address_space *mapping,
339                         pgoff_t index, void *expected, void *replacement)
340 {
341         XA_STATE(xas, &mapping->i_pages, index);
342         void *item;
343
344         VM_BUG_ON(!expected);
345         VM_BUG_ON(!replacement);
346         item = xas_load(&xas);
347         if (item != expected)
348                 return -ENOENT;
349         xas_store(&xas, replacement);
350         return 0;
351 }
352
353 /*
354  * Sometimes, before we decide whether to proceed or to fail, we must check
355  * that an entry was not already brought back from swap by a racing thread.
356  *
357  * Checking page is not enough: by the time a SwapCache page is locked, it
358  * might be reused, and again be SwapCache, using the same swap as before.
359  */
360 static bool shmem_confirm_swap(struct address_space *mapping,
361                                pgoff_t index, swp_entry_t swap)
362 {
363         return xa_load(&mapping->i_pages, index) == swp_to_radix_entry(swap);
364 }
365
366 /*
367  * Definitions for "huge tmpfs": tmpfs mounted with the huge= option
368  *
369  * SHMEM_HUGE_NEVER:
370  *      disables huge pages for the mount;
371  * SHMEM_HUGE_ALWAYS:
372  *      enables huge pages for the mount;
373  * SHMEM_HUGE_WITHIN_SIZE:
374  *      only allocate huge pages if the page will be fully within i_size,
375  *      also respect fadvise()/madvise() hints;
376  * SHMEM_HUGE_ADVISE:
377  *      only allocate huge pages if requested with fadvise()/madvise();
378  */
379
380 #define SHMEM_HUGE_NEVER        0
381 #define SHMEM_HUGE_ALWAYS       1
382 #define SHMEM_HUGE_WITHIN_SIZE  2
383 #define SHMEM_HUGE_ADVISE       3
384
385 /*
386  * Special values.
387  * Only can be set via /sys/kernel/mm/transparent_hugepage/shmem_enabled:
388  *
389  * SHMEM_HUGE_DENY:
390  *      disables huge on shm_mnt and all mounts, for emergency use;
391  * SHMEM_HUGE_FORCE:
392  *      enables huge on shm_mnt and all mounts, w/o needing option, for testing;
393  *
394  */
395 #define SHMEM_HUGE_DENY         (-1)
396 #define SHMEM_HUGE_FORCE        (-2)
397
398 #ifdef CONFIG_TRANSPARENT_HUGE_PAGECACHE
399 /* ifdef here to avoid bloating shmem.o when not necessary */
400
401 static int shmem_huge __read_mostly;
402
403 #if defined(CONFIG_SYSFS) || defined(CONFIG_TMPFS)
404 static int shmem_parse_huge(const char *str)
405 {
406         if (!strcmp(str, "never"))
407                 return SHMEM_HUGE_NEVER;
408         if (!strcmp(str, "always"))
409                 return SHMEM_HUGE_ALWAYS;
410         if (!strcmp(str, "within_size"))
411                 return SHMEM_HUGE_WITHIN_SIZE;
412         if (!strcmp(str, "advise"))
413                 return SHMEM_HUGE_ADVISE;
414         if (!strcmp(str, "deny"))
415                 return SHMEM_HUGE_DENY;
416         if (!strcmp(str, "force"))
417                 return SHMEM_HUGE_FORCE;
418         return -EINVAL;
419 }
420
421 static const char *shmem_format_huge(int huge)
422 {
423         switch (huge) {
424         case SHMEM_HUGE_NEVER:
425                 return "never";
426         case SHMEM_HUGE_ALWAYS:
427                 return "always";
428         case SHMEM_HUGE_WITHIN_SIZE:
429                 return "within_size";
430         case SHMEM_HUGE_ADVISE:
431                 return "advise";
432         case SHMEM_HUGE_DENY:
433                 return "deny";
434         case SHMEM_HUGE_FORCE:
435                 return "force";
436         default:
437                 VM_BUG_ON(1);
438                 return "bad_val";
439         }
440 }
441 #endif
442
443 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
444                 struct shrink_control *sc, unsigned long nr_to_split)
445 {
446         LIST_HEAD(list), *pos, *next;
447         LIST_HEAD(to_remove);
448         struct inode *inode;
449         struct shmem_inode_info *info;
450         struct page *page;
451         unsigned long batch = sc ? sc->nr_to_scan : 128;
452         int removed = 0, split = 0;
453
454         if (list_empty(&sbinfo->shrinklist))
455                 return SHRINK_STOP;
456
457         spin_lock(&sbinfo->shrinklist_lock);
458         list_for_each_safe(pos, next, &sbinfo->shrinklist) {
459                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
460
461                 /* pin the inode */
462                 inode = igrab(&info->vfs_inode);
463
464                 /* inode is about to be evicted */
465                 if (!inode) {
466                         list_del_init(&info->shrinklist);
467                         removed++;
468                         goto next;
469                 }
470
471                 /* Check if there's anything to gain */
472                 if (round_up(inode->i_size, PAGE_SIZE) ==
473                                 round_up(inode->i_size, HPAGE_PMD_SIZE)) {
474                         list_move(&info->shrinklist, &to_remove);
475                         removed++;
476                         goto next;
477                 }
478
479                 list_move(&info->shrinklist, &list);
480 next:
481                 if (!--batch)
482                         break;
483         }
484         spin_unlock(&sbinfo->shrinklist_lock);
485
486         list_for_each_safe(pos, next, &to_remove) {
487                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
488                 inode = &info->vfs_inode;
489                 list_del_init(&info->shrinklist);
490                 iput(inode);
491         }
492
493         list_for_each_safe(pos, next, &list) {
494                 int ret;
495
496                 info = list_entry(pos, struct shmem_inode_info, shrinklist);
497                 inode = &info->vfs_inode;
498
499                 if (nr_to_split && split >= nr_to_split)
500                         goto leave;
501
502                 page = find_get_page(inode->i_mapping,
503                                 (inode->i_size & HPAGE_PMD_MASK) >> PAGE_SHIFT);
504                 if (!page)
505                         goto drop;
506
507                 /* No huge page at the end of the file: nothing to split */
508                 if (!PageTransHuge(page)) {
509                         put_page(page);
510                         goto drop;
511                 }
512
513                 /*
514                  * Leave the inode on the list if we failed to lock
515                  * the page at this time.
516                  *
517                  * Waiting for the lock may lead to deadlock in the
518                  * reclaim path.
519                  */
520                 if (!trylock_page(page)) {
521                         put_page(page);
522                         goto leave;
523                 }
524
525                 ret = split_huge_page(page);
526                 unlock_page(page);
527                 put_page(page);
528
529                 /* If split failed leave the inode on the list */
530                 if (ret)
531                         goto leave;
532
533                 split++;
534 drop:
535                 list_del_init(&info->shrinklist);
536                 removed++;
537 leave:
538                 iput(inode);
539         }
540
541         spin_lock(&sbinfo->shrinklist_lock);
542         list_splice_tail(&list, &sbinfo->shrinklist);
543         sbinfo->shrinklist_len -= removed;
544         spin_unlock(&sbinfo->shrinklist_lock);
545
546         return split;
547 }
548
549 static long shmem_unused_huge_scan(struct super_block *sb,
550                 struct shrink_control *sc)
551 {
552         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
553
554         if (!READ_ONCE(sbinfo->shrinklist_len))
555                 return SHRINK_STOP;
556
557         return shmem_unused_huge_shrink(sbinfo, sc, 0);
558 }
559
560 static long shmem_unused_huge_count(struct super_block *sb,
561                 struct shrink_control *sc)
562 {
563         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
564         return READ_ONCE(sbinfo->shrinklist_len);
565 }
566 #else /* !CONFIG_TRANSPARENT_HUGE_PAGECACHE */
567
568 #define shmem_huge SHMEM_HUGE_DENY
569
570 static unsigned long shmem_unused_huge_shrink(struct shmem_sb_info *sbinfo,
571                 struct shrink_control *sc, unsigned long nr_to_split)
572 {
573         return 0;
574 }
575 #endif /* CONFIG_TRANSPARENT_HUGE_PAGECACHE */
576
577 static inline bool is_huge_enabled(struct shmem_sb_info *sbinfo)
578 {
579         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
580             (shmem_huge == SHMEM_HUGE_FORCE || sbinfo->huge) &&
581             shmem_huge != SHMEM_HUGE_DENY)
582                 return true;
583         return false;
584 }
585
586 /*
587  * Like add_to_page_cache_locked, but error if expected item has gone.
588  */
589 static int shmem_add_to_page_cache(struct page *page,
590                                    struct address_space *mapping,
591                                    pgoff_t index, void *expected, gfp_t gfp)
592 {
593         XA_STATE_ORDER(xas, &mapping->i_pages, index, compound_order(page));
594         unsigned long i = 0;
595         unsigned long nr = 1UL << compound_order(page);
596
597         VM_BUG_ON_PAGE(PageTail(page), page);
598         VM_BUG_ON_PAGE(index != round_down(index, nr), page);
599         VM_BUG_ON_PAGE(!PageLocked(page), page);
600         VM_BUG_ON_PAGE(!PageSwapBacked(page), page);
601         VM_BUG_ON(expected && PageTransHuge(page));
602
603         page_ref_add(page, nr);
604         page->mapping = mapping;
605         page->index = index;
606
607         do {
608                 void *entry;
609                 xas_lock_irq(&xas);
610                 entry = xas_find_conflict(&xas);
611                 if (entry != expected)
612                         xas_set_err(&xas, -EEXIST);
613                 xas_create_range(&xas);
614                 if (xas_error(&xas))
615                         goto unlock;
616 next:
617                 xas_store(&xas, page + i);
618                 if (++i < nr) {
619                         xas_next(&xas);
620                         goto next;
621                 }
622                 if (PageTransHuge(page)) {
623                         count_vm_event(THP_FILE_ALLOC);
624                         __inc_node_page_state(page, NR_SHMEM_THPS);
625                 }
626                 mapping->nrpages += nr;
627                 __mod_node_page_state(page_pgdat(page), NR_FILE_PAGES, nr);
628                 __mod_node_page_state(page_pgdat(page), NR_SHMEM, nr);
629 unlock:
630                 xas_unlock_irq(&xas);
631         } while (xas_nomem(&xas, gfp));
632
633         if (xas_error(&xas)) {
634                 page->mapping = NULL;
635                 page_ref_sub(page, nr);
636                 return xas_error(&xas);
637         }
638
639         return 0;
640 }
641
642 /*
643  * Like delete_from_page_cache, but substitutes swap for page.
644  */
645 static void shmem_delete_from_page_cache(struct page *page, void *radswap)
646 {
647         struct address_space *mapping = page->mapping;
648         int error;
649
650         VM_BUG_ON_PAGE(PageCompound(page), page);
651
652         xa_lock_irq(&mapping->i_pages);
653         error = shmem_replace_entry(mapping, page->index, page, radswap);
654         page->mapping = NULL;
655         mapping->nrpages--;
656         __dec_node_page_state(page, NR_FILE_PAGES);
657         __dec_node_page_state(page, NR_SHMEM);
658         xa_unlock_irq(&mapping->i_pages);
659         put_page(page);
660         BUG_ON(error);
661 }
662
663 /*
664  * Remove swap entry from page cache, free the swap and its page cache.
665  */
666 static int shmem_free_swap(struct address_space *mapping,
667                            pgoff_t index, void *radswap)
668 {
669         void *old;
670
671         old = xa_cmpxchg_irq(&mapping->i_pages, index, radswap, NULL, 0);
672         if (old != radswap)
673                 return -ENOENT;
674         free_swap_and_cache(radix_to_swp_entry(radswap));
675         return 0;
676 }
677
678 /*
679  * Determine (in bytes) how many of the shmem object's pages mapped by the
680  * given offsets are swapped out.
681  *
682  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
683  * as long as the inode doesn't go away and racy results are not a problem.
684  */
685 unsigned long shmem_partial_swap_usage(struct address_space *mapping,
686                                                 pgoff_t start, pgoff_t end)
687 {
688         XA_STATE(xas, &mapping->i_pages, start);
689         struct page *page;
690         unsigned long swapped = 0;
691
692         rcu_read_lock();
693         xas_for_each(&xas, page, end - 1) {
694                 if (xas_retry(&xas, page))
695                         continue;
696                 if (xa_is_value(page))
697                         swapped++;
698
699                 if (need_resched()) {
700                         xas_pause(&xas);
701                         cond_resched_rcu();
702                 }
703         }
704
705         rcu_read_unlock();
706
707         return swapped << PAGE_SHIFT;
708 }
709
710 /*
711  * Determine (in bytes) how many of the shmem object's pages mapped by the
712  * given vma is swapped out.
713  *
714  * This is safe to call without i_mutex or the i_pages lock thanks to RCU,
715  * as long as the inode doesn't go away and racy results are not a problem.
716  */
717 unsigned long shmem_swap_usage(struct vm_area_struct *vma)
718 {
719         struct inode *inode = file_inode(vma->vm_file);
720         struct shmem_inode_info *info = SHMEM_I(inode);
721         struct address_space *mapping = inode->i_mapping;
722         unsigned long swapped;
723
724         /* Be careful as we don't hold info->lock */
725         swapped = READ_ONCE(info->swapped);
726
727         /*
728          * The easier cases are when the shmem object has nothing in swap, or
729          * the vma maps it whole. Then we can simply use the stats that we
730          * already track.
731          */
732         if (!swapped)
733                 return 0;
734
735         if (!vma->vm_pgoff && vma->vm_end - vma->vm_start >= inode->i_size)
736                 return swapped << PAGE_SHIFT;
737
738         /* Here comes the more involved part */
739         return shmem_partial_swap_usage(mapping,
740                         linear_page_index(vma, vma->vm_start),
741                         linear_page_index(vma, vma->vm_end));
742 }
743
744 /*
745  * SysV IPC SHM_UNLOCK restore Unevictable pages to their evictable lists.
746  */
747 void shmem_unlock_mapping(struct address_space *mapping)
748 {
749         struct pagevec pvec;
750         pgoff_t indices[PAGEVEC_SIZE];
751         pgoff_t index = 0;
752
753         pagevec_init(&pvec);
754         /*
755          * Minor point, but we might as well stop if someone else SHM_LOCKs it.
756          */
757         while (!mapping_unevictable(mapping)) {
758                 /*
759                  * Avoid pagevec_lookup(): find_get_pages() returns 0 as if it
760                  * has finished, if it hits a row of PAGEVEC_SIZE swap entries.
761                  */
762                 pvec.nr = find_get_entries(mapping, index,
763                                            PAGEVEC_SIZE, pvec.pages, indices);
764                 if (!pvec.nr)
765                         break;
766                 index = indices[pvec.nr - 1] + 1;
767                 pagevec_remove_exceptionals(&pvec);
768                 check_move_unevictable_pages(&pvec);
769                 pagevec_release(&pvec);
770                 cond_resched();
771         }
772 }
773
774 /*
775  * Remove range of pages and swap entries from page cache, and free them.
776  * If !unfalloc, truncate or punch hole; if unfalloc, undo failed fallocate.
777  */
778 static void shmem_undo_range(struct inode *inode, loff_t lstart, loff_t lend,
779                                                                  bool unfalloc)
780 {
781         struct address_space *mapping = inode->i_mapping;
782         struct shmem_inode_info *info = SHMEM_I(inode);
783         pgoff_t start = (lstart + PAGE_SIZE - 1) >> PAGE_SHIFT;
784         pgoff_t end = (lend + 1) >> PAGE_SHIFT;
785         unsigned int partial_start = lstart & (PAGE_SIZE - 1);
786         unsigned int partial_end = (lend + 1) & (PAGE_SIZE - 1);
787         struct pagevec pvec;
788         pgoff_t indices[PAGEVEC_SIZE];
789         long nr_swaps_freed = 0;
790         pgoff_t index;
791         int i;
792
793         if (lend == -1)
794                 end = -1;       /* unsigned, so actually very big */
795
796         pagevec_init(&pvec);
797         index = start;
798         while (index < end) {
799                 pvec.nr = find_get_entries(mapping, index,
800                         min(end - index, (pgoff_t)PAGEVEC_SIZE),
801                         pvec.pages, indices);
802                 if (!pvec.nr)
803                         break;
804                 for (i = 0; i < pagevec_count(&pvec); i++) {
805                         struct page *page = pvec.pages[i];
806
807                         index = indices[i];
808                         if (index >= end)
809                                 break;
810
811                         if (xa_is_value(page)) {
812                                 if (unfalloc)
813                                         continue;
814                                 nr_swaps_freed += !shmem_free_swap(mapping,
815                                                                 index, page);
816                                 continue;
817                         }
818
819                         VM_BUG_ON_PAGE(page_to_pgoff(page) != index, page);
820
821                         if (!trylock_page(page))
822                                 continue;
823
824                         if (PageTransTail(page)) {
825                                 /* Middle of THP: zero out the page */
826                                 clear_highpage(page);
827                                 unlock_page(page);
828                                 continue;
829                         } else if (PageTransHuge(page)) {
830                                 if (index == round_down(end, HPAGE_PMD_NR)) {
831                                         /*
832                                          * Range ends in the middle of THP:
833                                          * zero out the page
834                                          */
835                                         clear_highpage(page);
836                                         unlock_page(page);
837                                         continue;
838                                 }
839                                 index += HPAGE_PMD_NR - 1;
840                                 i += HPAGE_PMD_NR - 1;
841                         }
842
843                         if (!unfalloc || !PageUptodate(page)) {
844                                 VM_BUG_ON_PAGE(PageTail(page), page);
845                                 if (page_mapping(page) == mapping) {
846                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
847                                         truncate_inode_page(mapping, page);
848                                 }
849                         }
850                         unlock_page(page);
851                 }
852                 pagevec_remove_exceptionals(&pvec);
853                 pagevec_release(&pvec);
854                 cond_resched();
855                 index++;
856         }
857
858         if (partial_start) {
859                 struct page *page = NULL;
860                 shmem_getpage(inode, start - 1, &page, SGP_READ);
861                 if (page) {
862                         unsigned int top = PAGE_SIZE;
863                         if (start > end) {
864                                 top = partial_end;
865                                 partial_end = 0;
866                         }
867                         zero_user_segment(page, partial_start, top);
868                         set_page_dirty(page);
869                         unlock_page(page);
870                         put_page(page);
871                 }
872         }
873         if (partial_end) {
874                 struct page *page = NULL;
875                 shmem_getpage(inode, end, &page, SGP_READ);
876                 if (page) {
877                         zero_user_segment(page, 0, partial_end);
878                         set_page_dirty(page);
879                         unlock_page(page);
880                         put_page(page);
881                 }
882         }
883         if (start >= end)
884                 return;
885
886         index = start;
887         while (index < end) {
888                 cond_resched();
889
890                 pvec.nr = find_get_entries(mapping, index,
891                                 min(end - index, (pgoff_t)PAGEVEC_SIZE),
892                                 pvec.pages, indices);
893                 if (!pvec.nr) {
894                         /* If all gone or hole-punch or unfalloc, we're done */
895                         if (index == start || end != -1)
896                                 break;
897                         /* But if truncating, restart to make sure all gone */
898                         index = start;
899                         continue;
900                 }
901                 for (i = 0; i < pagevec_count(&pvec); i++) {
902                         struct page *page = pvec.pages[i];
903
904                         index = indices[i];
905                         if (index >= end)
906                                 break;
907
908                         if (xa_is_value(page)) {
909                                 if (unfalloc)
910                                         continue;
911                                 if (shmem_free_swap(mapping, index, page)) {
912                                         /* Swap was replaced by page: retry */
913                                         index--;
914                                         break;
915                                 }
916                                 nr_swaps_freed++;
917                                 continue;
918                         }
919
920                         lock_page(page);
921
922                         if (PageTransTail(page)) {
923                                 /* Middle of THP: zero out the page */
924                                 clear_highpage(page);
925                                 unlock_page(page);
926                                 /*
927                                  * Partial thp truncate due 'start' in middle
928                                  * of THP: don't need to look on these pages
929                                  * again on !pvec.nr restart.
930                                  */
931                                 if (index != round_down(end, HPAGE_PMD_NR))
932                                         start++;
933                                 continue;
934                         } else if (PageTransHuge(page)) {
935                                 if (index == round_down(end, HPAGE_PMD_NR)) {
936                                         /*
937                                          * Range ends in the middle of THP:
938                                          * zero out the page
939                                          */
940                                         clear_highpage(page);
941                                         unlock_page(page);
942                                         continue;
943                                 }
944                                 index += HPAGE_PMD_NR - 1;
945                                 i += HPAGE_PMD_NR - 1;
946                         }
947
948                         if (!unfalloc || !PageUptodate(page)) {
949                                 VM_BUG_ON_PAGE(PageTail(page), page);
950                                 if (page_mapping(page) == mapping) {
951                                         VM_BUG_ON_PAGE(PageWriteback(page), page);
952                                         truncate_inode_page(mapping, page);
953                                 } else {
954                                         /* Page was replaced by swap: retry */
955                                         unlock_page(page);
956                                         index--;
957                                         break;
958                                 }
959                         }
960                         unlock_page(page);
961                 }
962                 pagevec_remove_exceptionals(&pvec);
963                 pagevec_release(&pvec);
964                 index++;
965         }
966
967         spin_lock_irq(&info->lock);
968         info->swapped -= nr_swaps_freed;
969         shmem_recalc_inode(inode);
970         spin_unlock_irq(&info->lock);
971 }
972
973 void shmem_truncate_range(struct inode *inode, loff_t lstart, loff_t lend)
974 {
975         shmem_undo_range(inode, lstart, lend, false);
976         inode->i_ctime = inode->i_mtime = current_time(inode);
977 }
978 EXPORT_SYMBOL_GPL(shmem_truncate_range);
979
980 static int shmem_getattr(const struct path *path, struct kstat *stat,
981                          u32 request_mask, unsigned int query_flags)
982 {
983         struct inode *inode = path->dentry->d_inode;
984         struct shmem_inode_info *info = SHMEM_I(inode);
985         struct shmem_sb_info *sb_info = SHMEM_SB(inode->i_sb);
986
987         if (info->alloced - info->swapped != inode->i_mapping->nrpages) {
988                 spin_lock_irq(&info->lock);
989                 shmem_recalc_inode(inode);
990                 spin_unlock_irq(&info->lock);
991         }
992         generic_fillattr(inode, stat);
993
994         if (is_huge_enabled(sb_info))
995                 stat->blksize = HPAGE_PMD_SIZE;
996
997         return 0;
998 }
999
1000 static int shmem_setattr(struct dentry *dentry, struct iattr *attr)
1001 {
1002         struct inode *inode = d_inode(dentry);
1003         struct shmem_inode_info *info = SHMEM_I(inode);
1004         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1005         int error;
1006
1007         error = setattr_prepare(dentry, attr);
1008         if (error)
1009                 return error;
1010
1011         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
1012                 loff_t oldsize = inode->i_size;
1013                 loff_t newsize = attr->ia_size;
1014
1015                 /* protected by i_mutex */
1016                 if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
1017                     (newsize > oldsize && (info->seals & F_SEAL_GROW)))
1018                         return -EPERM;
1019
1020                 if (newsize != oldsize) {
1021                         error = shmem_reacct_size(SHMEM_I(inode)->flags,
1022                                         oldsize, newsize);
1023                         if (error)
1024                                 return error;
1025                         i_size_write(inode, newsize);
1026                         inode->i_ctime = inode->i_mtime = current_time(inode);
1027                 }
1028                 if (newsize <= oldsize) {
1029                         loff_t holebegin = round_up(newsize, PAGE_SIZE);
1030                         if (oldsize > holebegin)
1031                                 unmap_mapping_range(inode->i_mapping,
1032                                                         holebegin, 0, 1);
1033                         if (info->alloced)
1034                                 shmem_truncate_range(inode,
1035                                                         newsize, (loff_t)-1);
1036                         /* unmap again to remove racily COWed private pages */
1037                         if (oldsize > holebegin)
1038                                 unmap_mapping_range(inode->i_mapping,
1039                                                         holebegin, 0, 1);
1040
1041                         /*
1042                          * Part of the huge page can be beyond i_size: subject
1043                          * to shrink under memory pressure.
1044                          */
1045                         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE)) {
1046                                 spin_lock(&sbinfo->shrinklist_lock);
1047                                 /*
1048                                  * _careful to defend against unlocked access to
1049                                  * ->shrink_list in shmem_unused_huge_shrink()
1050                                  */
1051                                 if (list_empty_careful(&info->shrinklist)) {
1052                                         list_add_tail(&info->shrinklist,
1053                                                         &sbinfo->shrinklist);
1054                                         sbinfo->shrinklist_len++;
1055                                 }
1056                                 spin_unlock(&sbinfo->shrinklist_lock);
1057                         }
1058                 }
1059         }
1060
1061         setattr_copy(inode, attr);
1062         if (attr->ia_valid & ATTR_MODE)
1063                 error = posix_acl_chmod(inode, inode->i_mode);
1064         return error;
1065 }
1066
1067 static void shmem_evict_inode(struct inode *inode)
1068 {
1069         struct shmem_inode_info *info = SHMEM_I(inode);
1070         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
1071
1072         if (inode->i_mapping->a_ops == &shmem_aops) {
1073                 shmem_unacct_size(info->flags, inode->i_size);
1074                 inode->i_size = 0;
1075                 shmem_truncate_range(inode, 0, (loff_t)-1);
1076                 if (!list_empty(&info->shrinklist)) {
1077                         spin_lock(&sbinfo->shrinklist_lock);
1078                         if (!list_empty(&info->shrinklist)) {
1079                                 list_del_init(&info->shrinklist);
1080                                 sbinfo->shrinklist_len--;
1081                         }
1082                         spin_unlock(&sbinfo->shrinklist_lock);
1083                 }
1084                 if (!list_empty(&info->swaplist)) {
1085                         mutex_lock(&shmem_swaplist_mutex);
1086                         list_del_init(&info->swaplist);
1087                         mutex_unlock(&shmem_swaplist_mutex);
1088                 }
1089         }
1090
1091         simple_xattrs_free(&info->xattrs);
1092         WARN_ON(inode->i_blocks);
1093         shmem_free_inode(inode->i_sb);
1094         clear_inode(inode);
1095 }
1096
1097 extern struct swap_info_struct *swap_info[];
1098
1099 static int shmem_find_swap_entries(struct address_space *mapping,
1100                                    pgoff_t start, unsigned int nr_entries,
1101                                    struct page **entries, pgoff_t *indices,
1102                                    bool frontswap)
1103 {
1104         XA_STATE(xas, &mapping->i_pages, start);
1105         struct page *page;
1106         unsigned int ret = 0;
1107
1108         if (!nr_entries)
1109                 return 0;
1110
1111         rcu_read_lock();
1112         xas_for_each(&xas, page, ULONG_MAX) {
1113                 if (xas_retry(&xas, page))
1114                         continue;
1115
1116                 if (!xa_is_value(page))
1117                         continue;
1118
1119                 if (frontswap) {
1120                         swp_entry_t entry = radix_to_swp_entry(page);
1121
1122                         if (!frontswap_test(swap_info[swp_type(entry)],
1123                                             swp_offset(entry)))
1124                                 continue;
1125                 }
1126
1127                 indices[ret] = xas.xa_index;
1128                 entries[ret] = page;
1129
1130                 if (need_resched()) {
1131                         xas_pause(&xas);
1132                         cond_resched_rcu();
1133                 }
1134                 if (++ret == nr_entries)
1135                         break;
1136         }
1137         rcu_read_unlock();
1138
1139         return ret;
1140 }
1141
1142 /*
1143  * Move the swapped pages for an inode to page cache. Returns the count
1144  * of pages swapped in, or the error in case of failure.
1145  */
1146 static int shmem_unuse_swap_entries(struct inode *inode, struct pagevec pvec,
1147                                     pgoff_t *indices)
1148 {
1149         int i = 0;
1150         int ret = 0;
1151         int error = 0;
1152         struct address_space *mapping = inode->i_mapping;
1153
1154         for (i = 0; i < pvec.nr; i++) {
1155                 struct page *page = pvec.pages[i];
1156
1157                 if (!xa_is_value(page))
1158                         continue;
1159                 error = shmem_swapin_page(inode, indices[i],
1160                                           &page, SGP_CACHE,
1161                                           mapping_gfp_mask(mapping),
1162                                           NULL, NULL);
1163                 if (error == 0) {
1164                         unlock_page(page);
1165                         put_page(page);
1166                         ret++;
1167                 }
1168                 if (error == -ENOMEM)
1169                         break;
1170                 error = 0;
1171         }
1172         return error ? error : ret;
1173 }
1174
1175 /*
1176  * If swap found in inode, free it and move page from swapcache to filecache.
1177  */
1178 static int shmem_unuse_inode(struct inode *inode, unsigned int type,
1179                              bool frontswap, unsigned long *fs_pages_to_unuse)
1180 {
1181         struct address_space *mapping = inode->i_mapping;
1182         pgoff_t start = 0;
1183         struct pagevec pvec;
1184         pgoff_t indices[PAGEVEC_SIZE];
1185         bool frontswap_partial = (frontswap && *fs_pages_to_unuse > 0);
1186         int ret = 0;
1187
1188         pagevec_init(&pvec);
1189         do {
1190                 unsigned int nr_entries = PAGEVEC_SIZE;
1191
1192                 if (frontswap_partial && *fs_pages_to_unuse < PAGEVEC_SIZE)
1193                         nr_entries = *fs_pages_to_unuse;
1194
1195                 pvec.nr = shmem_find_swap_entries(mapping, start, nr_entries,
1196                                                   pvec.pages, indices,
1197                                                   frontswap);
1198                 if (pvec.nr == 0) {
1199                         ret = 0;
1200                         break;
1201                 }
1202
1203                 ret = shmem_unuse_swap_entries(inode, pvec, indices);
1204                 if (ret < 0)
1205                         break;
1206
1207                 if (frontswap_partial) {
1208                         *fs_pages_to_unuse -= ret;
1209                         if (*fs_pages_to_unuse == 0) {
1210                                 ret = FRONTSWAP_PAGES_UNUSED;
1211                                 break;
1212                         }
1213                 }
1214
1215                 start = indices[pvec.nr - 1];
1216         } while (true);
1217
1218         return ret;
1219 }
1220
1221 /*
1222  * Read all the shared memory data that resides in the swap
1223  * device 'type' back into memory, so the swap device can be
1224  * unused.
1225  */
1226 int shmem_unuse(unsigned int type, bool frontswap,
1227                 unsigned long *fs_pages_to_unuse)
1228 {
1229         struct shmem_inode_info *info, *next;
1230         struct inode *inode;
1231         struct inode *prev_inode = NULL;
1232         int error = 0;
1233
1234         if (list_empty(&shmem_swaplist))
1235                 return 0;
1236
1237         mutex_lock(&shmem_swaplist_mutex);
1238
1239         /*
1240          * The extra refcount on the inode is necessary to safely dereference
1241          * p->next after re-acquiring the lock. New shmem inodes with swap
1242          * get added to the end of the list and we will scan them all.
1243          */
1244         list_for_each_entry_safe(info, next, &shmem_swaplist, swaplist) {
1245                 if (!info->swapped) {
1246                         list_del_init(&info->swaplist);
1247                         continue;
1248                 }
1249
1250                 inode = igrab(&info->vfs_inode);
1251                 if (!inode)
1252                         continue;
1253
1254                 mutex_unlock(&shmem_swaplist_mutex);
1255                 if (prev_inode)
1256                         iput(prev_inode);
1257                 prev_inode = inode;
1258
1259                 error = shmem_unuse_inode(inode, type, frontswap,
1260                                           fs_pages_to_unuse);
1261                 cond_resched();
1262
1263                 mutex_lock(&shmem_swaplist_mutex);
1264                 next = list_next_entry(info, swaplist);
1265                 if (!info->swapped)
1266                         list_del_init(&info->swaplist);
1267                 if (error)
1268                         break;
1269         }
1270         mutex_unlock(&shmem_swaplist_mutex);
1271
1272         if (prev_inode)
1273                 iput(prev_inode);
1274
1275         return error;
1276 }
1277
1278 /*
1279  * Move the page from the page cache to the swap cache.
1280  */
1281 static int shmem_writepage(struct page *page, struct writeback_control *wbc)
1282 {
1283         struct shmem_inode_info *info;
1284         struct address_space *mapping;
1285         struct inode *inode;
1286         swp_entry_t swap;
1287         pgoff_t index;
1288
1289         VM_BUG_ON_PAGE(PageCompound(page), page);
1290         BUG_ON(!PageLocked(page));
1291         mapping = page->mapping;
1292         index = page->index;
1293         inode = mapping->host;
1294         info = SHMEM_I(inode);
1295         if (info->flags & VM_LOCKED)
1296                 goto redirty;
1297         if (!total_swap_pages)
1298                 goto redirty;
1299
1300         /*
1301          * Our capabilities prevent regular writeback or sync from ever calling
1302          * shmem_writepage; but a stacking filesystem might use ->writepage of
1303          * its underlying filesystem, in which case tmpfs should write out to
1304          * swap only in response to memory pressure, and not for the writeback
1305          * threads or sync.
1306          */
1307         if (!wbc->for_reclaim) {
1308                 WARN_ON_ONCE(1);        /* Still happens? Tell us about it! */
1309                 goto redirty;
1310         }
1311
1312         /*
1313          * This is somewhat ridiculous, but without plumbing a SWAP_MAP_FALLOC
1314          * value into swapfile.c, the only way we can correctly account for a
1315          * fallocated page arriving here is now to initialize it and write it.
1316          *
1317          * That's okay for a page already fallocated earlier, but if we have
1318          * not yet completed the fallocation, then (a) we want to keep track
1319          * of this page in case we have to undo it, and (b) it may not be a
1320          * good idea to continue anyway, once we're pushing into swap.  So
1321          * reactivate the page, and let shmem_fallocate() quit when too many.
1322          */
1323         if (!PageUptodate(page)) {
1324                 if (inode->i_private) {
1325                         struct shmem_falloc *shmem_falloc;
1326                         spin_lock(&inode->i_lock);
1327                         shmem_falloc = inode->i_private;
1328                         if (shmem_falloc &&
1329                             !shmem_falloc->waitq &&
1330                             index >= shmem_falloc->start &&
1331                             index < shmem_falloc->next)
1332                                 shmem_falloc->nr_unswapped++;
1333                         else
1334                                 shmem_falloc = NULL;
1335                         spin_unlock(&inode->i_lock);
1336                         if (shmem_falloc)
1337                                 goto redirty;
1338                 }
1339                 clear_highpage(page);
1340                 flush_dcache_page(page);
1341                 SetPageUptodate(page);
1342         }
1343
1344         swap = get_swap_page(page);
1345         if (!swap.val)
1346                 goto redirty;
1347
1348         /*
1349          * Add inode to shmem_unuse()'s list of swapped-out inodes,
1350          * if it's not already there.  Do it now before the page is
1351          * moved to swap cache, when its pagelock no longer protects
1352          * the inode from eviction.  But don't unlock the mutex until
1353          * we've incremented swapped, because shmem_unuse_inode() will
1354          * prune a !swapped inode from the swaplist under this mutex.
1355          */
1356         mutex_lock(&shmem_swaplist_mutex);
1357         if (list_empty(&info->swaplist))
1358                 list_add(&info->swaplist, &shmem_swaplist);
1359
1360         if (add_to_swap_cache(page, swap, GFP_ATOMIC) == 0) {
1361                 spin_lock_irq(&info->lock);
1362                 shmem_recalc_inode(inode);
1363                 info->swapped++;
1364                 spin_unlock_irq(&info->lock);
1365
1366                 swap_shmem_alloc(swap);
1367                 shmem_delete_from_page_cache(page, swp_to_radix_entry(swap));
1368
1369                 mutex_unlock(&shmem_swaplist_mutex);
1370                 BUG_ON(page_mapped(page));
1371                 swap_writepage(page, wbc);
1372                 return 0;
1373         }
1374
1375         mutex_unlock(&shmem_swaplist_mutex);
1376         put_swap_page(page, swap);
1377 redirty:
1378         set_page_dirty(page);
1379         if (wbc->for_reclaim)
1380                 return AOP_WRITEPAGE_ACTIVATE;  /* Return with page locked */
1381         unlock_page(page);
1382         return 0;
1383 }
1384
1385 #if defined(CONFIG_NUMA) && defined(CONFIG_TMPFS)
1386 static void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1387 {
1388         char buffer[64];
1389
1390         if (!mpol || mpol->mode == MPOL_DEFAULT)
1391                 return;         /* show nothing */
1392
1393         mpol_to_str(buffer, sizeof(buffer), mpol);
1394
1395         seq_printf(seq, ",mpol=%s", buffer);
1396 }
1397
1398 static struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1399 {
1400         struct mempolicy *mpol = NULL;
1401         if (sbinfo->mpol) {
1402                 spin_lock(&sbinfo->stat_lock);  /* prevent replace/use races */
1403                 mpol = sbinfo->mpol;
1404                 mpol_get(mpol);
1405                 spin_unlock(&sbinfo->stat_lock);
1406         }
1407         return mpol;
1408 }
1409 #else /* !CONFIG_NUMA || !CONFIG_TMPFS */
1410 static inline void shmem_show_mpol(struct seq_file *seq, struct mempolicy *mpol)
1411 {
1412 }
1413 static inline struct mempolicy *shmem_get_sbmpol(struct shmem_sb_info *sbinfo)
1414 {
1415         return NULL;
1416 }
1417 #endif /* CONFIG_NUMA && CONFIG_TMPFS */
1418 #ifndef CONFIG_NUMA
1419 #define vm_policy vm_private_data
1420 #endif
1421
1422 static void shmem_pseudo_vma_init(struct vm_area_struct *vma,
1423                 struct shmem_inode_info *info, pgoff_t index)
1424 {
1425         /* Create a pseudo vma that just contains the policy */
1426         vma_init(vma, NULL);
1427         /* Bias interleave by inode number to distribute better across nodes */
1428         vma->vm_pgoff = index + info->vfs_inode.i_ino;
1429         vma->vm_policy = mpol_shared_policy_lookup(&info->policy, index);
1430 }
1431
1432 static void shmem_pseudo_vma_destroy(struct vm_area_struct *vma)
1433 {
1434         /* Drop reference taken by mpol_shared_policy_lookup() */
1435         mpol_cond_put(vma->vm_policy);
1436 }
1437
1438 static struct page *shmem_swapin(swp_entry_t swap, gfp_t gfp,
1439                         struct shmem_inode_info *info, pgoff_t index)
1440 {
1441         struct vm_area_struct pvma;
1442         struct page *page;
1443         struct vm_fault vmf;
1444
1445         shmem_pseudo_vma_init(&pvma, info, index);
1446         vmf.vma = &pvma;
1447         vmf.address = 0;
1448         page = swap_cluster_readahead(swap, gfp, &vmf);
1449         shmem_pseudo_vma_destroy(&pvma);
1450
1451         return page;
1452 }
1453
1454 static struct page *shmem_alloc_hugepage(gfp_t gfp,
1455                 struct shmem_inode_info *info, pgoff_t index)
1456 {
1457         struct vm_area_struct pvma;
1458         struct address_space *mapping = info->vfs_inode.i_mapping;
1459         pgoff_t hindex;
1460         struct page *page;
1461
1462         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1463                 return NULL;
1464
1465         hindex = round_down(index, HPAGE_PMD_NR);
1466         if (xa_find(&mapping->i_pages, &hindex, hindex + HPAGE_PMD_NR - 1,
1467                                                                 XA_PRESENT))
1468                 return NULL;
1469
1470         shmem_pseudo_vma_init(&pvma, info, hindex);
1471         page = alloc_pages_vma(gfp | __GFP_COMP | __GFP_NORETRY | __GFP_NOWARN,
1472                         HPAGE_PMD_ORDER, &pvma, 0, numa_node_id(), true);
1473         shmem_pseudo_vma_destroy(&pvma);
1474         if (page)
1475                 prep_transhuge_page(page);
1476         return page;
1477 }
1478
1479 static struct page *shmem_alloc_page(gfp_t gfp,
1480                         struct shmem_inode_info *info, pgoff_t index)
1481 {
1482         struct vm_area_struct pvma;
1483         struct page *page;
1484
1485         shmem_pseudo_vma_init(&pvma, info, index);
1486         page = alloc_page_vma(gfp, &pvma, 0);
1487         shmem_pseudo_vma_destroy(&pvma);
1488
1489         return page;
1490 }
1491
1492 static struct page *shmem_alloc_and_acct_page(gfp_t gfp,
1493                 struct inode *inode,
1494                 pgoff_t index, bool huge)
1495 {
1496         struct shmem_inode_info *info = SHMEM_I(inode);
1497         struct page *page;
1498         int nr;
1499         int err = -ENOSPC;
1500
1501         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
1502                 huge = false;
1503         nr = huge ? HPAGE_PMD_NR : 1;
1504
1505         if (!shmem_inode_acct_block(inode, nr))
1506                 goto failed;
1507
1508         if (huge)
1509                 page = shmem_alloc_hugepage(gfp, info, index);
1510         else
1511                 page = shmem_alloc_page(gfp, info, index);
1512         if (page) {
1513                 __SetPageLocked(page);
1514                 __SetPageSwapBacked(page);
1515                 return page;
1516         }
1517
1518         err = -ENOMEM;
1519         shmem_inode_unacct_blocks(inode, nr);
1520 failed:
1521         return ERR_PTR(err);
1522 }
1523
1524 /*
1525  * When a page is moved from swapcache to shmem filecache (either by the
1526  * usual swapin of shmem_getpage_gfp(), or by the less common swapoff of
1527  * shmem_unuse_inode()), it may have been read in earlier from swap, in
1528  * ignorance of the mapping it belongs to.  If that mapping has special
1529  * constraints (like the gma500 GEM driver, which requires RAM below 4GB),
1530  * we may need to copy to a suitable page before moving to filecache.
1531  *
1532  * In a future release, this may well be extended to respect cpuset and
1533  * NUMA mempolicy, and applied also to anonymous pages in do_swap_page();
1534  * but for now it is a simple matter of zone.
1535  */
1536 static bool shmem_should_replace_page(struct page *page, gfp_t gfp)
1537 {
1538         return page_zonenum(page) > gfp_zone(gfp);
1539 }
1540
1541 static int shmem_replace_page(struct page **pagep, gfp_t gfp,
1542                                 struct shmem_inode_info *info, pgoff_t index)
1543 {
1544         struct page *oldpage, *newpage;
1545         struct address_space *swap_mapping;
1546         swp_entry_t entry;
1547         pgoff_t swap_index;
1548         int error;
1549
1550         oldpage = *pagep;
1551         entry.val = page_private(oldpage);
1552         swap_index = swp_offset(entry);
1553         swap_mapping = page_mapping(oldpage);
1554
1555         /*
1556          * We have arrived here because our zones are constrained, so don't
1557          * limit chance of success by further cpuset and node constraints.
1558          */
1559         gfp &= ~GFP_CONSTRAINT_MASK;
1560         newpage = shmem_alloc_page(gfp, info, index);
1561         if (!newpage)
1562                 return -ENOMEM;
1563
1564         get_page(newpage);
1565         copy_highpage(newpage, oldpage);
1566         flush_dcache_page(newpage);
1567
1568         __SetPageLocked(newpage);
1569         __SetPageSwapBacked(newpage);
1570         SetPageUptodate(newpage);
1571         set_page_private(newpage, entry.val);
1572         SetPageSwapCache(newpage);
1573
1574         /*
1575          * Our caller will very soon move newpage out of swapcache, but it's
1576          * a nice clean interface for us to replace oldpage by newpage there.
1577          */
1578         xa_lock_irq(&swap_mapping->i_pages);
1579         error = shmem_replace_entry(swap_mapping, swap_index, oldpage, newpage);
1580         if (!error) {
1581                 __inc_node_page_state(newpage, NR_FILE_PAGES);
1582                 __dec_node_page_state(oldpage, NR_FILE_PAGES);
1583         }
1584         xa_unlock_irq(&swap_mapping->i_pages);
1585
1586         if (unlikely(error)) {
1587                 /*
1588                  * Is this possible?  I think not, now that our callers check
1589                  * both PageSwapCache and page_private after getting page lock;
1590                  * but be defensive.  Reverse old to newpage for clear and free.
1591                  */
1592                 oldpage = newpage;
1593         } else {
1594                 mem_cgroup_migrate(oldpage, newpage);
1595                 lru_cache_add_anon(newpage);
1596                 *pagep = newpage;
1597         }
1598
1599         ClearPageSwapCache(oldpage);
1600         set_page_private(oldpage, 0);
1601
1602         unlock_page(oldpage);
1603         put_page(oldpage);
1604         put_page(oldpage);
1605         return error;
1606 }
1607
1608 /*
1609  * Swap in the page pointed to by *pagep.
1610  * Caller has to make sure that *pagep contains a valid swapped page.
1611  * Returns 0 and the page in pagep if success. On failure, returns the
1612  * the error code and NULL in *pagep.
1613  */
1614 static int shmem_swapin_page(struct inode *inode, pgoff_t index,
1615                              struct page **pagep, enum sgp_type sgp,
1616                              gfp_t gfp, struct vm_area_struct *vma,
1617                              vm_fault_t *fault_type)
1618 {
1619         struct address_space *mapping = inode->i_mapping;
1620         struct shmem_inode_info *info = SHMEM_I(inode);
1621         struct mm_struct *charge_mm = vma ? vma->vm_mm : current->mm;
1622         struct mem_cgroup *memcg;
1623         struct page *page;
1624         swp_entry_t swap;
1625         int error;
1626
1627         VM_BUG_ON(!*pagep || !xa_is_value(*pagep));
1628         swap = radix_to_swp_entry(*pagep);
1629         *pagep = NULL;
1630
1631         /* Look it up and read it in.. */
1632         page = lookup_swap_cache(swap, NULL, 0);
1633         if (!page) {
1634                 /* Or update major stats only when swapin succeeds?? */
1635                 if (fault_type) {
1636                         *fault_type |= VM_FAULT_MAJOR;
1637                         count_vm_event(PGMAJFAULT);
1638                         count_memcg_event_mm(charge_mm, PGMAJFAULT);
1639                 }
1640                 /* Here we actually start the io */
1641                 page = shmem_swapin(swap, gfp, info, index);
1642                 if (!page) {
1643                         error = -ENOMEM;
1644                         goto failed;
1645                 }
1646         }
1647
1648         /* We have to do this with page locked to prevent races */
1649         lock_page(page);
1650         if (!PageSwapCache(page) || page_private(page) != swap.val ||
1651             !shmem_confirm_swap(mapping, index, swap)) {
1652                 error = -EEXIST;
1653                 goto unlock;
1654         }
1655         if (!PageUptodate(page)) {
1656                 error = -EIO;
1657                 goto failed;
1658         }
1659         wait_on_page_writeback(page);
1660
1661         if (shmem_should_replace_page(page, gfp)) {
1662                 error = shmem_replace_page(&page, gfp, info, index);
1663                 if (error)
1664                         goto failed;
1665         }
1666
1667         error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg,
1668                                             false);
1669         if (!error) {
1670                 error = shmem_add_to_page_cache(page, mapping, index,
1671                                                 swp_to_radix_entry(swap), gfp);
1672                 /*
1673                  * We already confirmed swap under page lock, and make
1674                  * no memory allocation here, so usually no possibility
1675                  * of error; but free_swap_and_cache() only trylocks a
1676                  * page, so it is just possible that the entry has been
1677                  * truncated or holepunched since swap was confirmed.
1678                  * shmem_undo_range() will have done some of the
1679                  * unaccounting, now delete_from_swap_cache() will do
1680                  * the rest.
1681                  */
1682                 if (error) {
1683                         mem_cgroup_cancel_charge(page, memcg, false);
1684                         delete_from_swap_cache(page);
1685                 }
1686         }
1687         if (error)
1688                 goto failed;
1689
1690         mem_cgroup_commit_charge(page, memcg, true, false);
1691
1692         spin_lock_irq(&info->lock);
1693         info->swapped--;
1694         shmem_recalc_inode(inode);
1695         spin_unlock_irq(&info->lock);
1696
1697         if (sgp == SGP_WRITE)
1698                 mark_page_accessed(page);
1699
1700         delete_from_swap_cache(page);
1701         set_page_dirty(page);
1702         swap_free(swap);
1703
1704         *pagep = page;
1705         return 0;
1706 failed:
1707         if (!shmem_confirm_swap(mapping, index, swap))
1708                 error = -EEXIST;
1709 unlock:
1710         if (page) {
1711                 unlock_page(page);
1712                 put_page(page);
1713         }
1714
1715         return error;
1716 }
1717
1718 /*
1719  * shmem_getpage_gfp - find page in cache, or get from swap, or allocate
1720  *
1721  * If we allocate a new one we do not mark it dirty. That's up to the
1722  * vm. If we swap it in we mark it dirty since we also free the swap
1723  * entry since a page cannot live in both the swap and page cache.
1724  *
1725  * fault_mm and fault_type are only supplied by shmem_fault:
1726  * otherwise they are NULL.
1727  */
1728 static int shmem_getpage_gfp(struct inode *inode, pgoff_t index,
1729         struct page **pagep, enum sgp_type sgp, gfp_t gfp,
1730         struct vm_area_struct *vma, struct vm_fault *vmf,
1731                         vm_fault_t *fault_type)
1732 {
1733         struct address_space *mapping = inode->i_mapping;
1734         struct shmem_inode_info *info = SHMEM_I(inode);
1735         struct shmem_sb_info *sbinfo;
1736         struct mm_struct *charge_mm;
1737         struct mem_cgroup *memcg;
1738         struct page *page;
1739         enum sgp_type sgp_huge = sgp;
1740         pgoff_t hindex = index;
1741         int error;
1742         int once = 0;
1743         int alloced = 0;
1744
1745         if (index > (MAX_LFS_FILESIZE >> PAGE_SHIFT))
1746                 return -EFBIG;
1747         if (sgp == SGP_NOHUGE || sgp == SGP_HUGE)
1748                 sgp = SGP_CACHE;
1749 repeat:
1750         if (sgp <= SGP_CACHE &&
1751             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1752                 return -EINVAL;
1753         }
1754
1755         sbinfo = SHMEM_SB(inode->i_sb);
1756         charge_mm = vma ? vma->vm_mm : current->mm;
1757
1758         page = find_lock_entry(mapping, index);
1759         if (xa_is_value(page)) {
1760                 error = shmem_swapin_page(inode, index, &page,
1761                                           sgp, gfp, vma, fault_type);
1762                 if (error == -EEXIST)
1763                         goto repeat;
1764
1765                 *pagep = page;
1766                 return error;
1767         }
1768
1769         if (page && sgp == SGP_WRITE)
1770                 mark_page_accessed(page);
1771
1772         /* fallocated page? */
1773         if (page && !PageUptodate(page)) {
1774                 if (sgp != SGP_READ)
1775                         goto clear;
1776                 unlock_page(page);
1777                 put_page(page);
1778                 page = NULL;
1779         }
1780         if (page || sgp == SGP_READ) {
1781                 *pagep = page;
1782                 return 0;
1783         }
1784
1785         /*
1786          * Fast cache lookup did not find it:
1787          * bring it back from swap or allocate.
1788          */
1789
1790         if (vma && userfaultfd_missing(vma)) {
1791                 *fault_type = handle_userfault(vmf, VM_UFFD_MISSING);
1792                 return 0;
1793         }
1794
1795         /* shmem_symlink() */
1796         if (mapping->a_ops != &shmem_aops)
1797                 goto alloc_nohuge;
1798         if (shmem_huge == SHMEM_HUGE_DENY || sgp_huge == SGP_NOHUGE)
1799                 goto alloc_nohuge;
1800         if (shmem_huge == SHMEM_HUGE_FORCE)
1801                 goto alloc_huge;
1802         switch (sbinfo->huge) {
1803                 loff_t i_size;
1804                 pgoff_t off;
1805         case SHMEM_HUGE_NEVER:
1806                 goto alloc_nohuge;
1807         case SHMEM_HUGE_WITHIN_SIZE:
1808                 off = round_up(index, HPAGE_PMD_NR);
1809                 i_size = round_up(i_size_read(inode), PAGE_SIZE);
1810                 if (i_size >= HPAGE_PMD_SIZE &&
1811                     i_size >> PAGE_SHIFT >= off)
1812                         goto alloc_huge;
1813                 /* fallthrough */
1814         case SHMEM_HUGE_ADVISE:
1815                 if (sgp_huge == SGP_HUGE)
1816                         goto alloc_huge;
1817                 /* TODO: implement fadvise() hints */
1818                 goto alloc_nohuge;
1819         }
1820
1821 alloc_huge:
1822         page = shmem_alloc_and_acct_page(gfp, inode, index, true);
1823         if (IS_ERR(page)) {
1824 alloc_nohuge:
1825                 page = shmem_alloc_and_acct_page(gfp, inode,
1826                                                  index, false);
1827         }
1828         if (IS_ERR(page)) {
1829                 int retry = 5;
1830
1831                 error = PTR_ERR(page);
1832                 page = NULL;
1833                 if (error != -ENOSPC)
1834                         goto unlock;
1835                 /*
1836                  * Try to reclaim some space by splitting a huge page
1837                  * beyond i_size on the filesystem.
1838                  */
1839                 while (retry--) {
1840                         int ret;
1841
1842                         ret = shmem_unused_huge_shrink(sbinfo, NULL, 1);
1843                         if (ret == SHRINK_STOP)
1844                                 break;
1845                         if (ret)
1846                                 goto alloc_nohuge;
1847                 }
1848                 goto unlock;
1849         }
1850
1851         if (PageTransHuge(page))
1852                 hindex = round_down(index, HPAGE_PMD_NR);
1853         else
1854                 hindex = index;
1855
1856         if (sgp == SGP_WRITE)
1857                 __SetPageReferenced(page);
1858
1859         error = mem_cgroup_try_charge_delay(page, charge_mm, gfp, &memcg,
1860                                             PageTransHuge(page));
1861         if (error)
1862                 goto unacct;
1863         error = shmem_add_to_page_cache(page, mapping, hindex,
1864                                         NULL, gfp & GFP_RECLAIM_MASK);
1865         if (error) {
1866                 mem_cgroup_cancel_charge(page, memcg,
1867                                          PageTransHuge(page));
1868                 goto unacct;
1869         }
1870         mem_cgroup_commit_charge(page, memcg, false,
1871                                  PageTransHuge(page));
1872         lru_cache_add_anon(page);
1873
1874         spin_lock_irq(&info->lock);
1875         info->alloced += 1 << compound_order(page);
1876         inode->i_blocks += BLOCKS_PER_PAGE << compound_order(page);
1877         shmem_recalc_inode(inode);
1878         spin_unlock_irq(&info->lock);
1879         alloced = true;
1880
1881         if (PageTransHuge(page) &&
1882             DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE) <
1883                         hindex + HPAGE_PMD_NR - 1) {
1884                 /*
1885                  * Part of the huge page is beyond i_size: subject
1886                  * to shrink under memory pressure.
1887                  */
1888                 spin_lock(&sbinfo->shrinklist_lock);
1889                 /*
1890                  * _careful to defend against unlocked access to
1891                  * ->shrink_list in shmem_unused_huge_shrink()
1892                  */
1893                 if (list_empty_careful(&info->shrinklist)) {
1894                         list_add_tail(&info->shrinklist,
1895                                       &sbinfo->shrinklist);
1896                         sbinfo->shrinklist_len++;
1897                 }
1898                 spin_unlock(&sbinfo->shrinklist_lock);
1899         }
1900
1901         /*
1902          * Let SGP_FALLOC use the SGP_WRITE optimization on a new page.
1903          */
1904         if (sgp == SGP_FALLOC)
1905                 sgp = SGP_WRITE;
1906 clear:
1907         /*
1908          * Let SGP_WRITE caller clear ends if write does not fill page;
1909          * but SGP_FALLOC on a page fallocated earlier must initialize
1910          * it now, lest undo on failure cancel our earlier guarantee.
1911          */
1912         if (sgp != SGP_WRITE && !PageUptodate(page)) {
1913                 struct page *head = compound_head(page);
1914                 int i;
1915
1916                 for (i = 0; i < (1 << compound_order(head)); i++) {
1917                         clear_highpage(head + i);
1918                         flush_dcache_page(head + i);
1919                 }
1920                 SetPageUptodate(head);
1921         }
1922
1923         /* Perhaps the file has been truncated since we checked */
1924         if (sgp <= SGP_CACHE &&
1925             ((loff_t)index << PAGE_SHIFT) >= i_size_read(inode)) {
1926                 if (alloced) {
1927                         ClearPageDirty(page);
1928                         delete_from_page_cache(page);
1929                         spin_lock_irq(&info->lock);
1930                         shmem_recalc_inode(inode);
1931                         spin_unlock_irq(&info->lock);
1932                 }
1933                 error = -EINVAL;
1934                 goto unlock;
1935         }
1936         *pagep = page + index - hindex;
1937         return 0;
1938
1939         /*
1940          * Error recovery.
1941          */
1942 unacct:
1943         shmem_inode_unacct_blocks(inode, 1 << compound_order(page));
1944
1945         if (PageTransHuge(page)) {
1946                 unlock_page(page);
1947                 put_page(page);
1948                 goto alloc_nohuge;
1949         }
1950 unlock:
1951         if (page) {
1952                 unlock_page(page);
1953                 put_page(page);
1954         }
1955         if (error == -ENOSPC && !once++) {
1956                 spin_lock_irq(&info->lock);
1957                 shmem_recalc_inode(inode);
1958                 spin_unlock_irq(&info->lock);
1959                 goto repeat;
1960         }
1961         if (error == -EEXIST)
1962                 goto repeat;
1963         return error;
1964 }
1965
1966 /*
1967  * This is like autoremove_wake_function, but it removes the wait queue
1968  * entry unconditionally - even if something else had already woken the
1969  * target.
1970  */
1971 static int synchronous_wake_function(wait_queue_entry_t *wait, unsigned mode, int sync, void *key)
1972 {
1973         int ret = default_wake_function(wait, mode, sync, key);
1974         list_del_init(&wait->entry);
1975         return ret;
1976 }
1977
1978 static vm_fault_t shmem_fault(struct vm_fault *vmf)
1979 {
1980         struct vm_area_struct *vma = vmf->vma;
1981         struct inode *inode = file_inode(vma->vm_file);
1982         gfp_t gfp = mapping_gfp_mask(inode->i_mapping);
1983         enum sgp_type sgp;
1984         int err;
1985         vm_fault_t ret = VM_FAULT_LOCKED;
1986
1987         /*
1988          * Trinity finds that probing a hole which tmpfs is punching can
1989          * prevent the hole-punch from ever completing: which in turn
1990          * locks writers out with its hold on i_mutex.  So refrain from
1991          * faulting pages into the hole while it's being punched.  Although
1992          * shmem_undo_range() does remove the additions, it may be unable to
1993          * keep up, as each new page needs its own unmap_mapping_range() call,
1994          * and the i_mmap tree grows ever slower to scan if new vmas are added.
1995          *
1996          * It does not matter if we sometimes reach this check just before the
1997          * hole-punch begins, so that one fault then races with the punch:
1998          * we just need to make racing faults a rare case.
1999          *
2000          * The implementation below would be much simpler if we just used a
2001          * standard mutex or completion: but we cannot take i_mutex in fault,
2002          * and bloating every shmem inode for this unlikely case would be sad.
2003          */
2004         if (unlikely(inode->i_private)) {
2005                 struct shmem_falloc *shmem_falloc;
2006
2007                 spin_lock(&inode->i_lock);
2008                 shmem_falloc = inode->i_private;
2009                 if (shmem_falloc &&
2010                     shmem_falloc->waitq &&
2011                     vmf->pgoff >= shmem_falloc->start &&
2012                     vmf->pgoff < shmem_falloc->next) {
2013                         wait_queue_head_t *shmem_falloc_waitq;
2014                         DEFINE_WAIT_FUNC(shmem_fault_wait, synchronous_wake_function);
2015
2016                         ret = VM_FAULT_NOPAGE;
2017                         if ((vmf->flags & FAULT_FLAG_ALLOW_RETRY) &&
2018                            !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT)) {
2019                                 /* It's polite to up mmap_sem if we can */
2020                                 up_read(&vma->vm_mm->mmap_sem);
2021                                 ret = VM_FAULT_RETRY;
2022                         }
2023
2024                         shmem_falloc_waitq = shmem_falloc->waitq;
2025                         prepare_to_wait(shmem_falloc_waitq, &shmem_fault_wait,
2026                                         TASK_UNINTERRUPTIBLE);
2027                         spin_unlock(&inode->i_lock);
2028                         schedule();
2029
2030                         /*
2031                          * shmem_falloc_waitq points into the shmem_fallocate()
2032                          * stack of the hole-punching task: shmem_falloc_waitq
2033                          * is usually invalid by the time we reach here, but
2034                          * finish_wait() does not dereference it in that case;
2035                          * though i_lock needed lest racing with wake_up_all().
2036                          */
2037                         spin_lock(&inode->i_lock);
2038                         finish_wait(shmem_falloc_waitq, &shmem_fault_wait);
2039                         spin_unlock(&inode->i_lock);
2040                         return ret;
2041                 }
2042                 spin_unlock(&inode->i_lock);
2043         }
2044
2045         sgp = SGP_CACHE;
2046
2047         if ((vma->vm_flags & VM_NOHUGEPAGE) ||
2048             test_bit(MMF_DISABLE_THP, &vma->vm_mm->flags))
2049                 sgp = SGP_NOHUGE;
2050         else if (vma->vm_flags & VM_HUGEPAGE)
2051                 sgp = SGP_HUGE;
2052
2053         err = shmem_getpage_gfp(inode, vmf->pgoff, &vmf->page, sgp,
2054                                   gfp, vma, vmf, &ret);
2055         if (err)
2056                 return vmf_error(err);
2057         return ret;
2058 }
2059
2060 unsigned long shmem_get_unmapped_area(struct file *file,
2061                                       unsigned long uaddr, unsigned long len,
2062                                       unsigned long pgoff, unsigned long flags)
2063 {
2064         unsigned long (*get_area)(struct file *,
2065                 unsigned long, unsigned long, unsigned long, unsigned long);
2066         unsigned long addr;
2067         unsigned long offset;
2068         unsigned long inflated_len;
2069         unsigned long inflated_addr;
2070         unsigned long inflated_offset;
2071
2072         if (len > TASK_SIZE)
2073                 return -ENOMEM;
2074
2075         get_area = current->mm->get_unmapped_area;
2076         addr = get_area(file, uaddr, len, pgoff, flags);
2077
2078         if (!IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE))
2079                 return addr;
2080         if (IS_ERR_VALUE(addr))
2081                 return addr;
2082         if (addr & ~PAGE_MASK)
2083                 return addr;
2084         if (addr > TASK_SIZE - len)
2085                 return addr;
2086
2087         if (shmem_huge == SHMEM_HUGE_DENY)
2088                 return addr;
2089         if (len < HPAGE_PMD_SIZE)
2090                 return addr;
2091         if (flags & MAP_FIXED)
2092                 return addr;
2093         /*
2094          * Our priority is to support MAP_SHARED mapped hugely;
2095          * and support MAP_PRIVATE mapped hugely too, until it is COWed.
2096          * But if caller specified an address hint, respect that as before.
2097          */
2098         if (uaddr)
2099                 return addr;
2100
2101         if (shmem_huge != SHMEM_HUGE_FORCE) {
2102                 struct super_block *sb;
2103
2104                 if (file) {
2105                         VM_BUG_ON(file->f_op != &shmem_file_operations);
2106                         sb = file_inode(file)->i_sb;
2107                 } else {
2108                         /*
2109                          * Called directly from mm/mmap.c, or drivers/char/mem.c
2110                          * for "/dev/zero", to create a shared anonymous object.
2111                          */
2112                         if (IS_ERR(shm_mnt))
2113                                 return addr;
2114                         sb = shm_mnt->mnt_sb;
2115                 }
2116                 if (SHMEM_SB(sb)->huge == SHMEM_HUGE_NEVER)
2117                         return addr;
2118         }
2119
2120         offset = (pgoff << PAGE_SHIFT) & (HPAGE_PMD_SIZE-1);
2121         if (offset && offset + len < 2 * HPAGE_PMD_SIZE)
2122                 return addr;
2123         if ((addr & (HPAGE_PMD_SIZE-1)) == offset)
2124                 return addr;
2125
2126         inflated_len = len + HPAGE_PMD_SIZE - PAGE_SIZE;
2127         if (inflated_len > TASK_SIZE)
2128                 return addr;
2129         if (inflated_len < len)
2130                 return addr;
2131
2132         inflated_addr = get_area(NULL, 0, inflated_len, 0, flags);
2133         if (IS_ERR_VALUE(inflated_addr))
2134                 return addr;
2135         if (inflated_addr & ~PAGE_MASK)
2136                 return addr;
2137
2138         inflated_offset = inflated_addr & (HPAGE_PMD_SIZE-1);
2139         inflated_addr += offset - inflated_offset;
2140         if (inflated_offset > offset)
2141                 inflated_addr += HPAGE_PMD_SIZE;
2142
2143         if (inflated_addr > TASK_SIZE - len)
2144                 return addr;
2145         return inflated_addr;
2146 }
2147
2148 #ifdef CONFIG_NUMA
2149 static int shmem_set_policy(struct vm_area_struct *vma, struct mempolicy *mpol)
2150 {
2151         struct inode *inode = file_inode(vma->vm_file);
2152         return mpol_set_shared_policy(&SHMEM_I(inode)->policy, vma, mpol);
2153 }
2154
2155 static struct mempolicy *shmem_get_policy(struct vm_area_struct *vma,
2156                                           unsigned long addr)
2157 {
2158         struct inode *inode = file_inode(vma->vm_file);
2159         pgoff_t index;
2160
2161         index = ((addr - vma->vm_start) >> PAGE_SHIFT) + vma->vm_pgoff;
2162         return mpol_shared_policy_lookup(&SHMEM_I(inode)->policy, index);
2163 }
2164 #endif
2165
2166 int shmem_lock(struct file *file, int lock, struct user_struct *user)
2167 {
2168         struct inode *inode = file_inode(file);
2169         struct shmem_inode_info *info = SHMEM_I(inode);
2170         int retval = -ENOMEM;
2171
2172         spin_lock_irq(&info->lock);
2173         if (lock && !(info->flags & VM_LOCKED)) {
2174                 if (!user_shm_lock(inode->i_size, user))
2175                         goto out_nomem;
2176                 info->flags |= VM_LOCKED;
2177                 mapping_set_unevictable(file->f_mapping);
2178         }
2179         if (!lock && (info->flags & VM_LOCKED) && user) {
2180                 user_shm_unlock(inode->i_size, user);
2181                 info->flags &= ~VM_LOCKED;
2182                 mapping_clear_unevictable(file->f_mapping);
2183         }
2184         retval = 0;
2185
2186 out_nomem:
2187         spin_unlock_irq(&info->lock);
2188         return retval;
2189 }
2190
2191 static int shmem_mmap(struct file *file, struct vm_area_struct *vma)
2192 {
2193         file_accessed(file);
2194         vma->vm_ops = &shmem_vm_ops;
2195         if (IS_ENABLED(CONFIG_TRANSPARENT_HUGE_PAGECACHE) &&
2196                         ((vma->vm_start + ~HPAGE_PMD_MASK) & HPAGE_PMD_MASK) <
2197                         (vma->vm_end & HPAGE_PMD_MASK)) {
2198                 khugepaged_enter(vma, vma->vm_flags);
2199         }
2200         return 0;
2201 }
2202
2203 static struct inode *shmem_get_inode(struct super_block *sb, const struct inode *dir,
2204                                      umode_t mode, dev_t dev, unsigned long flags)
2205 {
2206         struct inode *inode;
2207         struct shmem_inode_info *info;
2208         struct shmem_sb_info *sbinfo = SHMEM_SB(sb);
2209
2210         if (shmem_reserve_inode(sb))
2211                 return NULL;
2212
2213         inode = new_inode(sb);
2214         if (inode) {
2215                 inode->i_ino = get_next_ino();
2216                 inode_init_owner(inode, dir, mode);
2217                 inode->i_blocks = 0;
2218                 inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
2219                 inode->i_generation = prandom_u32();
2220                 info = SHMEM_I(inode);
2221                 memset(info, 0, (char *)inode - (char *)info);
2222                 spin_lock_init(&info->lock);
2223                 info->seals = F_SEAL_SEAL;
2224                 info->flags = flags & VM_NORESERVE;
2225                 INIT_LIST_HEAD(&info->shrinklist);
2226                 INIT_LIST_HEAD(&info->swaplist);
2227                 simple_xattrs_init(&info->xattrs);
2228                 cache_no_acl(inode);
2229
2230                 switch (mode & S_IFMT) {
2231                 default:
2232                         inode->i_op = &shmem_special_inode_operations;
2233                         init_special_inode(inode, mode, dev);
2234                         break;
2235                 case S_IFREG:
2236                         inode->i_mapping->a_ops = &shmem_aops;
2237                         inode->i_op = &shmem_inode_operations;
2238                         inode->i_fop = &shmem_file_operations;
2239                         mpol_shared_policy_init(&info->policy,
2240                                                  shmem_get_sbmpol(sbinfo));
2241                         break;
2242                 case S_IFDIR:
2243                         inc_nlink(inode);
2244                         /* Some things misbehave if size == 0 on a directory */
2245                         inode->i_size = 2 * BOGO_DIRENT_SIZE;
2246                         inode->i_op = &shmem_dir_inode_operations;
2247                         inode->i_fop = &simple_dir_operations;
2248                         break;
2249                 case S_IFLNK:
2250                         /*
2251                          * Must not load anything in the rbtree,
2252                          * mpol_free_shared_policy will not be called.
2253                          */
2254                         mpol_shared_policy_init(&info->policy, NULL);
2255                         break;
2256                 }
2257
2258                 lockdep_annotate_inode_mutex_key(inode);
2259         } else
2260                 shmem_free_inode(sb);
2261         return inode;
2262 }
2263
2264 bool shmem_mapping(struct address_space *mapping)
2265 {
2266         return mapping->a_ops == &shmem_aops;
2267 }
2268
2269 static int shmem_mfill_atomic_pte(struct mm_struct *dst_mm,
2270                                   pmd_t *dst_pmd,
2271                                   struct vm_area_struct *dst_vma,
2272                                   unsigned long dst_addr,
2273                                   unsigned long src_addr,
2274                                   bool zeropage,
2275                                   struct page **pagep)
2276 {
2277         struct inode *inode = file_inode(dst_vma->vm_file);
2278         struct shmem_inode_info *info = SHMEM_I(inode);
2279         struct address_space *mapping = inode->i_mapping;
2280         gfp_t gfp = mapping_gfp_mask(mapping);
2281         pgoff_t pgoff = linear_page_index(dst_vma, dst_addr);
2282         struct mem_cgroup *memcg;
2283         spinlock_t *ptl;
2284         void *page_kaddr;
2285         struct page *page;
2286         pte_t _dst_pte, *dst_pte;
2287         int ret;
2288         pgoff_t offset, max_off;
2289
2290         ret = -ENOMEM;
2291         if (!shmem_inode_acct_block(inode, 1))
2292                 goto out;
2293
2294         if (!*pagep) {
2295                 page = shmem_alloc_page(gfp, info, pgoff);
2296                 if (!page)
2297                         goto out_unacct_blocks;
2298
2299                 if (!zeropage) {        /* mcopy_atomic */
2300                         page_kaddr = kmap_atomic(page);
2301                         ret = copy_from_user(page_kaddr,
2302                                              (const void __user *)src_addr,
2303                                              PAGE_SIZE);
2304                         kunmap_atomic(page_kaddr);
2305
2306                         /* fallback to copy_from_user outside mmap_sem */
2307                         if (unlikely(ret)) {
2308                                 *pagep = page;
2309                                 shmem_inode_unacct_blocks(inode, 1);
2310                                 /* don't free the page */
2311                                 return -ENOENT;
2312                         }
2313                 } else {                /* mfill_zeropage_atomic */
2314                         clear_highpage(page);
2315                 }
2316         } else {
2317                 page = *pagep;
2318                 *pagep = NULL;
2319         }
2320
2321         VM_BUG_ON(PageLocked(page) || PageSwapBacked(page));
2322         __SetPageLocked(page);
2323         __SetPageSwapBacked(page);
2324         __SetPageUptodate(page);
2325
2326         ret = -EFAULT;
2327         offset = linear_page_index(dst_vma, dst_addr);
2328         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2329         if (unlikely(offset >= max_off))
2330                 goto out_release;
2331
2332         ret = mem_cgroup_try_charge_delay(page, dst_mm, gfp, &memcg, false);
2333         if (ret)
2334                 goto out_release;
2335
2336         ret = shmem_add_to_page_cache(page, mapping, pgoff, NULL,
2337                                                 gfp & GFP_RECLAIM_MASK);
2338         if (ret)
2339                 goto out_release_uncharge;
2340
2341         mem_cgroup_commit_charge(page, memcg, false, false);
2342
2343         _dst_pte = mk_pte(page, dst_vma->vm_page_prot);
2344         if (dst_vma->vm_flags & VM_WRITE)
2345                 _dst_pte = pte_mkwrite(pte_mkdirty(_dst_pte));
2346         else {
2347                 /*
2348                  * We don't set the pte dirty if the vma has no
2349                  * VM_WRITE permission, so mark the page dirty or it
2350                  * could be freed from under us. We could do it
2351                  * unconditionally before unlock_page(), but doing it
2352                  * only if VM_WRITE is not set is faster.
2353                  */
2354                 set_page_dirty(page);
2355         }
2356
2357         dst_pte = pte_offset_map_lock(dst_mm, dst_pmd, dst_addr, &ptl);
2358
2359         ret = -EFAULT;
2360         max_off = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
2361         if (unlikely(offset >= max_off))
2362                 goto out_release_uncharge_unlock;
2363
2364         ret = -EEXIST;
2365         if (!pte_none(*dst_pte))
2366                 goto out_release_uncharge_unlock;
2367
2368         lru_cache_add_anon(page);
2369
2370         spin_lock(&info->lock);
2371         info->alloced++;
2372         inode->i_blocks += BLOCKS_PER_PAGE;
2373         shmem_recalc_inode(inode);
2374         spin_unlock(&info->lock);
2375
2376         inc_mm_counter(dst_mm, mm_counter_file(page));
2377         page_add_file_rmap(page, false);
2378         set_pte_at(dst_mm, dst_addr, dst_pte, _dst_pte);
2379
2380         /* No need to invalidate - it was non-present before */
2381         update_mmu_cache(dst_vma, dst_addr, dst_pte);
2382         pte_unmap_unlock(dst_pte, ptl);
2383         unlock_page(page);
2384         ret = 0;
2385 out:
2386         return ret;
2387 out_release_uncharge_unlock:
2388         pte_unmap_unlock(dst_pte, ptl);
2389         ClearPageDirty(page);
2390         delete_from_page_cache(page);
2391 out_release_uncharge:
2392         mem_cgroup_cancel_charge(page, memcg, false);
2393 out_release:
2394         unlock_page(page);
2395         put_page(page);
2396 out_unacct_blocks:
2397         shmem_inode_unacct_blocks(inode, 1);
2398         goto out;
2399 }
2400
2401 int shmem_mcopy_atomic_pte(struct mm_struct *dst_mm,
2402                            pmd_t *dst_pmd,
2403                            struct vm_area_struct *dst_vma,
2404                            unsigned long dst_addr,
2405                            unsigned long src_addr,
2406                            struct page **pagep)
2407 {
2408         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2409                                       dst_addr, src_addr, false, pagep);
2410 }
2411
2412 int shmem_mfill_zeropage_pte(struct mm_struct *dst_mm,
2413                              pmd_t *dst_pmd,
2414                              struct vm_area_struct *dst_vma,
2415                              unsigned long dst_addr)
2416 {
2417         struct page *page = NULL;
2418
2419         return shmem_mfill_atomic_pte(dst_mm, dst_pmd, dst_vma,
2420                                       dst_addr, 0, true, &page);
2421 }
2422
2423 #ifdef CONFIG_TMPFS
2424 static const struct inode_operations shmem_symlink_inode_operations;
2425 static const struct inode_operations shmem_short_symlink_operations;
2426
2427 #ifdef CONFIG_TMPFS_XATTR
2428 static int shmem_initxattrs(struct inode *, const struct xattr *, void *);
2429 #else
2430 #define shmem_initxattrs NULL
2431 #endif
2432
2433 static int
2434 shmem_write_begin(struct file *file, struct address_space *mapping,
2435                         loff_t pos, unsigned len, unsigned flags,
2436                         struct page **pagep, void **fsdata)
2437 {
2438         struct inode *inode = mapping->host;
2439         struct shmem_inode_info *info = SHMEM_I(inode);
2440         pgoff_t index = pos >> PAGE_SHIFT;
2441
2442         /* i_mutex is held by caller */
2443         if (unlikely(info->seals & (F_SEAL_WRITE | F_SEAL_GROW))) {
2444                 if (info->seals & F_SEAL_WRITE)
2445                         return -EPERM;
2446                 if ((info->seals & F_SEAL_GROW) && pos + len > inode->i_size)
2447                         return -EPERM;
2448         }
2449
2450         return shmem_getpage(inode, index, pagep, SGP_WRITE);
2451 }
2452
2453 static int
2454 shmem_write_end(struct file *file, struct address_space *mapping,
2455                         loff_t pos, unsigned len, unsigned copied,
2456                         struct page *page, void *fsdata)
2457 {
2458         struct inode *inode = mapping->host;
2459
2460         if (pos + copied > inode->i_size)
2461                 i_size_write(inode, pos + copied);
2462
2463         if (!PageUptodate(page)) {
2464                 struct page *head = compound_head(page);
2465                 if (PageTransCompound(page)) {
2466                         int i;
2467
2468                         for (i = 0; i < HPAGE_PMD_NR; i++) {
2469                                 if (head + i == page)
2470                                         continue;
2471                                 clear_highpage(head + i);
2472                                 flush_dcache_page(head + i);
2473                         }
2474                 }
2475                 if (copied < PAGE_SIZE) {
2476                         unsigned from = pos & (PAGE_SIZE - 1);
2477                         zero_user_segments(page, 0, from,
2478                                         from + copied, PAGE_SIZE);
2479                 }
2480                 SetPageUptodate(head);
2481         }
2482         set_page_dirty(page);
2483         unlock_page(page);
2484         put_page(page);
2485
2486         return copied;
2487 }
2488
2489 static ssize_t shmem_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
2490 {
2491         struct file *file = iocb->ki_filp;
2492         struct inode *inode = file_inode(file);
2493         struct address_space *mapping = inode->i_mapping;
2494         pgoff_t index;
2495         unsigned long offset;
2496         enum sgp_type sgp = SGP_READ;
2497         int error = 0;
2498         ssize_t retval = 0;
2499         loff_t *ppos = &iocb->ki_pos;
2500
2501         /*
2502          * Might this read be for a stacking filesystem?  Then when reading
2503          * holes of a sparse file, we actually need to allocate those pages,
2504          * and even mark them dirty, so it cannot exceed the max_blocks limit.
2505          */
2506         if (!iter_is_iovec(to))
2507                 sgp = SGP_CACHE;
2508
2509         index = *ppos >> PAGE_SHIFT;
2510         offset = *ppos & ~PAGE_MASK;
2511
2512         for (;;) {
2513                 struct page *page = NULL;
2514                 pgoff_t end_index;
2515                 unsigned long nr, ret;
2516                 loff_t i_size = i_size_read(inode);
2517
2518                 end_index = i_size >> PAGE_SHIFT;
2519                 if (index > end_index)
2520                         break;
2521                 if (index == end_index) {
2522                         nr = i_size & ~PAGE_MASK;
2523                         if (nr <= offset)
2524                                 break;
2525                 }
2526
2527                 error = shmem_getpage(inode, index, &page, sgp);
2528                 if (error) {
2529                         if (error == -EINVAL)
2530                                 error = 0;
2531                         break;
2532                 }
2533                 if (page) {
2534                         if (sgp == SGP_CACHE)
2535                                 set_page_dirty(page);
2536                         unlock_page(page);
2537                 }
2538
2539                 /*
2540                  * We must evaluate after, since reads (unlike writes)
2541                  * are called without i_mutex protection against truncate
2542                  */
2543                 nr = PAGE_SIZE;
2544                 i_size = i_size_read(inode);
2545                 end_index = i_size >> PAGE_SHIFT;
2546                 if (index == end_index) {
2547                         nr = i_size & ~PAGE_MASK;
2548                         if (nr <= offset) {
2549                                 if (page)
2550                                         put_page(page);
2551                                 break;
2552                         }
2553                 }
2554                 nr -= offset;
2555
2556                 if (page) {
2557                         /*
2558                          * If users can be writing to this page using arbitrary
2559                          * virtual addresses, take care about potential aliasing
2560                          * before reading the page on the kernel side.
2561                          */
2562                         if (mapping_writably_mapped(mapping))
2563                                 flush_dcache_page(page);
2564                         /*
2565                          * Mark the page accessed if we read the beginning.
2566                          */
2567                         if (!offset)
2568                                 mark_page_accessed(page);
2569                 } else {
2570                         page = ZERO_PAGE(0);
2571                         get_page(page);
2572                 }
2573
2574                 /*
2575                  * Ok, we have the page, and it's up-to-date, so
2576                  * now we can copy it to user space...
2577                  */
2578                 ret = copy_page_to_iter(page, offset, nr, to);
2579                 retval += ret;
2580                 offset += ret;
2581                 index += offset >> PAGE_SHIFT;
2582                 offset &= ~PAGE_MASK;
2583
2584                 put_page(page);
2585                 if (!iov_iter_count(to))
2586                         break;
2587                 if (ret < nr) {
2588                         error = -EFAULT;
2589                         break;
2590                 }
2591                 cond_resched();
2592         }
2593
2594         *ppos = ((loff_t) index << PAGE_SHIFT) + offset;
2595         file_accessed(file);
2596         return retval ? retval : error;
2597 }
2598
2599 /*
2600  * llseek SEEK_DATA or SEEK_HOLE through the page cache.
2601  */
2602 static pgoff_t shmem_seek_hole_data(struct address_space *mapping,
2603                                     pgoff_t index, pgoff_t end, int whence)
2604 {
2605         struct page *page;
2606         struct pagevec pvec;
2607         pgoff_t indices[PAGEVEC_SIZE];
2608         bool done = false;
2609         int i;
2610
2611         pagevec_init(&pvec);
2612         pvec.nr = 1;            /* start small: we may be there already */
2613         while (!done) {
2614                 pvec.nr = find_get_entries(mapping, index,
2615                                         pvec.nr, pvec.pages, indices);
2616                 if (!pvec.nr) {
2617                         if (whence == SEEK_DATA)
2618                                 index = end;
2619                         break;
2620                 }
2621                 for (i = 0; i < pvec.nr; i++, index++) {
2622                         if (index < indices[i]) {
2623                                 if (whence == SEEK_HOLE) {
2624                                         done = true;
2625                                         break;
2626                                 }
2627                                 index = indices[i];
2628                         }
2629                         page = pvec.pages[i];
2630                         if (page && !xa_is_value(page)) {
2631                                 if (!PageUptodate(page))
2632                                         page = NULL;
2633                         }
2634                         if (index >= end ||
2635                             (page && whence == SEEK_DATA) ||
2636                             (!page && whence == SEEK_HOLE)) {
2637                                 done = true;
2638                                 break;
2639                         }
2640                 }
2641                 pagevec_remove_exceptionals(&pvec);
2642                 pagevec_release(&pvec);
2643                 pvec.nr = PAGEVEC_SIZE;
2644                 cond_resched();
2645         }
2646         return index;
2647 }
2648
2649 static loff_t shmem_file_llseek(struct file *file, loff_t offset, int whence)
2650 {
2651         struct address_space *mapping = file->f_mapping;
2652         struct inode *inode = mapping->host;
2653         pgoff_t start, end;
2654         loff_t new_offset;
2655
2656         if (whence != SEEK_DATA && whence != SEEK_HOLE)
2657                 return generic_file_llseek_size(file, offset, whence,
2658                                         MAX_LFS_FILESIZE, i_size_read(inode));
2659         inode_lock(inode);
2660         /* We're holding i_mutex so we can access i_size directly */
2661
2662         if (offset < 0 || offset >= inode->i_size)
2663                 offset = -ENXIO;
2664         else {
2665                 start = offset >> PAGE_SHIFT;
2666                 end = (inode->i_size + PAGE_SIZE - 1) >> PAGE_SHIFT;
2667                 new_offset = shmem_seek_hole_data(mapping, start, end, whence);
2668                 new_offset <<= PAGE_SHIFT;
2669                 if (new_offset > offset) {
2670                         if (new_offset < inode->i_size)
2671                                 offset = new_offset;
2672                         else if (whence == SEEK_DATA)
2673                                 offset = -ENXIO;
2674                         else
2675                                 offset = inode->i_size;
2676                 }
2677         }
2678
2679         if (offset >= 0)
2680                 offset = vfs_setpos(file, offset, MAX_LFS_FILESIZE);
2681         inode_unlock(inode);
2682         return offset;
2683 }
2684
2685 static long shmem_fallocate(struct file *file, int mode, loff_t offset,
2686                                                          loff_t len)
2687 {
2688         struct inode *inode = file_inode(file);
2689         struct shmem_sb_info *sbinfo = SHMEM_SB(inode->i_sb);
2690         struct shmem_inode_info *info = SHMEM_I(inode);
2691         struct shmem_falloc shmem_falloc;
2692         pgoff_t start, index, end;
2693         int error;
2694
2695         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
2696                 return -EOPNOTSUPP;
2697
2698         inode_lock(inode);
2699
2700         if (mode & FALLOC_FL_PUNCH_HOLE) {
2701                 struct address_space *mapping = file->f_mapping;
2702                 loff_t unmap_start = round_up(offset, PAGE_SIZE);
2703                 loff_t unmap_end = round_down(offset + len, PAGE_SIZE) - 1;
2704                 DECLARE_WAIT_QUEUE_HEAD_ONSTACK(shmem_falloc_waitq);
2705
2706                 /* protected by i_mutex */
2707                 if (info->seals & F_SEAL_WRITE) {
2708                         error = -EPERM;
2709                         goto out;
2710                 }
2711
2712                 shmem_falloc.waitq = &shmem_falloc_waitq;
2713                 shmem_falloc.start = unmap_start >> PAGE_SHIFT;
2714                 shmem_falloc.next = (unmap_end + 1) >> PAGE_SHIFT;
2715                 spin_lock(&inode->i_lock);
2716                 inode->i_private = &shmem_falloc;
2717                 spin_unlock(&inode->i_lock);
2718
2719                 if ((u64)unmap_end > (u64)unmap_start)
2720                         unmap_mapping_range(mapping, unmap_start,
2721                                             1 + unmap_end - unmap_start, 0);
2722                 shmem_truncate_range(inode, offset, offset + len - 1);
2723                 /* No need to unmap again: hole-punching leaves COWed pages */
2724
2725                 spin_lock(&inode->i_lock);
2726                 inode->i_private = NULL;
2727                 wake_up_all(&shmem_falloc_waitq);
2728                 WARN_ON_ONCE(!list_empty(&shmem_falloc_waitq.head));
2729                 spin_unlock(&inode->i_lock);
2730                 error = 0;
2731                 goto out;
2732         }
2733
2734         /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
2735         error = inode_newsize_ok(inode, offset + len);
2736         if (error)
2737                 goto out;
2738
2739         if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
2740                 error = -EPERM;
2741                 goto out;
2742         }
2743
2744         start = offset >> PAGE_SHIFT;
2745         end = (offset + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
2746         /* Try to avoid a swapstorm if len is impossible to satisfy */
2747         if (sbinfo->max_blocks && end - start > sbinfo->max_blocks) {
2748                 error = -ENOSPC;
2749                 goto out;
2750         }
2751
2752         shmem_falloc.waitq = NULL;
2753         shmem_falloc.start = start;
2754         shmem_falloc.next  = start;
2755         shmem_falloc.nr_falloced = 0;
2756         shmem_falloc.nr_unswapped = 0;
2757         spin_lock(&inode->i_lock);
2758         inode->i_private = &shmem_falloc;
2759         spin_unlock(&inode->i_lock);
2760
2761         for (index = start; index < end; index++) {
2762                 struct page *page;
2763
2764                 /*
2765                  * Good, the fallocate(2) manpage permits EINTR: we may have
2766                  * been interrupted because we are using up too much memory.
2767                  */
2768                 if (signal_pending(current))
2769                         error = -EINTR;
2770                 else if (shmem_falloc.nr_unswapped > shmem_falloc.nr_falloced)
2771                         error = -ENOMEM;
2772                 else
2773                         error = shmem_getpage(inode, index, &page, SGP_FALLOC);
2774                 if (error) {
2775                         /* Remove the !PageUptodate pages we added */
2776                         if (index > start) {
2777                                 shmem_undo_range(inode,
2778                                     (loff_t)start << PAGE_SHIFT,
2779                                     ((loff_t)index << PAGE_SHIFT) - 1, true);
2780                         }
2781                         goto undone;
2782                 }
2783
2784                 /*
2785                  * Inform shmem_writepage() how far we have reached.
2786                  * No need for lock or barrier: we have the page lock.
2787                  */
2788                 shmem_falloc.next++;
2789                 if (!PageUptodate(page))
2790                         shmem_falloc.nr_falloced++;
2791
2792                 /*
2793                  * If !PageUptodate, leave it that way so that freeable pages
2794                  * can be recognized if we need to rollback on error later.
2795                  * But set_page_dirty so that memory pressure will swap rather
2796                  * than free the pages we are allocating (and SGP_CACHE pages
2797                  * might still be clean: we now need to mark those dirty too).
2798                  */
2799                 set_page_dirty(page);
2800                 unlock_page(page);
2801                 put_page(page);
2802                 cond_resched();
2803         }
2804
2805         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
2806                 i_size_write(inode, offset + len);
2807         inode->i_ctime = current_time(inode);
2808 undone:
2809         spin_lock(&inode->i_lock);
2810         inode->i_private = NULL;
2811         spin_unlock(&inode->i_lock);
2812 out:
2813         inode_unlock(inode);
2814         return error;
2815 }
2816
2817 static int shmem_statfs(struct dentry *dentry, struct kstatfs *buf)
2818 {
2819         struct shmem_sb_info *sbinfo = SHMEM_SB(dentry->d_sb);
2820
2821         buf->f_type = TMPFS_MAGIC;
2822         buf->f_bsize = PAGE_SIZE;
2823         buf->f_namelen = NAME_MAX;
2824         if (sbinfo->max_blocks) {
2825                 buf->f_blocks = sbinfo->max_blocks;
2826                 buf->f_bavail =
2827                 buf->f_bfree  = sbinfo->max_blocks -
2828                                 percpu_counter_sum(&sbinfo->used_blocks);
2829         }
2830         if (sbinfo->max_inodes) {
2831                 buf->f_files = sbinfo->max_inodes;
2832                 buf->f_ffree = sbinfo->free_inodes;
2833         }
2834         /* else leave those fields 0 like simple_statfs */
2835         return 0;
2836 }
2837
2838 /*
2839  * File creation. Allocate an inode, and we're done..
2840  */
2841 static int
2842 shmem_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev)
2843 {
2844         struct inode *inode;
2845         int error = -ENOSPC;
2846
2847         inode = shmem_get_inode(dir->i_sb, dir, mode, dev, VM_NORESERVE);
2848         if (inode) {
2849                 error = simple_acl_create(dir, inode);
2850                 if (error)
2851                         goto out_iput;
2852                 error = security_inode_init_security(inode, dir,
2853                                                      &dentry->d_name,
2854                                                      shmem_initxattrs, NULL);
2855                 if (error && error != -EOPNOTSUPP)
2856                         goto out_iput;
2857
2858                 error = 0;
2859                 dir->i_size += BOGO_DIRENT_SIZE;
2860                 dir->i_ctime = dir->i_mtime = current_time(dir);
2861                 d_instantiate(dentry, inode);
2862                 dget(dentry); /* Extra count - pin the dentry in core */
2863         }
2864         return error;
2865 out_iput:
2866         iput(inode);
2867         return error;
2868 }
2869
2870 static int
2871 shmem_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
2872 {
2873         struct inode *inode;
2874         int error = -ENOSPC;
2875
2876         inode = shmem_get_inode(dir->i_sb, dir, mode, 0, VM_NORESERVE);
2877         if (inode) {
2878                 error = security_inode_init_security(inode, dir,
2879                                                      NULL,
2880                                                      shmem_initxattrs, NULL);
2881                 if (error && error != -EOPNOTSUPP)
2882                         goto out_iput;
2883                 error = simple_acl_create(dir, inode);
2884                 if (error)
2885                         goto out_iput;
2886                 d_tmpfile(dentry, inode);
2887         }
2888         return error;
2889 out_iput:
2890         iput(inode);
2891         return error;
2892 }
2893
2894 static int shmem_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
2895 {
2896         int error;
2897
2898         if ((error = shmem_mknod(dir, dentry, mode | S_IFDIR, 0)))
2899                 return error;
2900         inc_nlink(dir);
2901         return 0;
2902 }
2903
2904 static int shmem_create(struct inode *dir, struct dentry *dentry, umode_t mode,
2905                 bool excl)
2906 {
2907         return shmem_mknod(dir, dentry, mode | S_IFREG, 0);
2908 }
2909
2910 /*
2911  * Link a file..
2912  */
2913 static int shmem_link(struct dentry *old_dentry, struct inode *dir, struct dentry *dentry)
2914 {
2915         struct inode *inode = d_inode(old_dentry);
2916         int ret = 0;
2917
2918         /*
2919          * No ordinary (disk based) filesystem counts links as inodes;
2920          * but each new link needs a new dentry, pinning lowmem, and
2921          * tmpfs dentries cannot be pruned until they are unlinked.
2922          * But if an O_TMPFILE file is linked into the tmpfs, the
2923          * first link must skip that, to get the accounting right.
2924          */
2925         if (inode->i_nlink) {
2926                 ret = shmem_reserve_inode(inode->i_sb);
2927                 if (ret)
2928                         goto out;
2929         }
2930
2931         dir->i_size += BOGO_DIRENT_SIZE;
2932         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2933         inc_nlink(inode);
2934         ihold(inode);   /* New dentry reference */
2935         dget(dentry);           /* Extra pinning count for the created dentry */
2936         d_instantiate(dentry, inode);
2937 out:
2938         return ret;
2939 }
2940
2941 static int shmem_unlink(struct inode *dir, struct dentry *dentry)
2942 {
2943         struct inode *inode = d_inode(dentry);
2944
2945         if (inode->i_nlink > 1 && !S_ISDIR(inode->i_mode))
2946                 shmem_free_inode(inode->i_sb);
2947
2948         dir->i_size -= BOGO_DIRENT_SIZE;
2949         inode->i_ctime = dir->i_ctime = dir->i_mtime = current_time(inode);
2950         drop_nlink(inode);
2951         dput(dentry);   /* Undo the count from "create" - this does all the work */
2952         return 0;
2953 }
2954
2955 static int shmem_rmdir(struct inode *dir, struct dentry *dentry)
2956 {
2957         if (!simple_empty(dentry))
2958                 return -ENOTEMPTY;
2959
2960         drop_nlink(d_inode(dentry));
2961         drop_nlink(dir);
2962         return shmem_unlink(dir, dentry);
2963 }
2964
2965 static int shmem_exchange(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry)
2966 {
2967         bool old_is_dir = d_is_dir(old_dentry);
2968         bool new_is_dir = d_is_dir(new_dentry);
2969
2970         if (old_dir != new_dir && old_is_dir != new_is_dir) {
2971                 if (old_is_dir) {
2972                         drop_nlink(old_dir);
2973                         inc_nlink(new_dir);
2974                 } else {
2975                         drop_nlink(new_dir);
2976                         inc_nlink(old_dir);
2977                 }
2978         }
2979         old_dir->i_ctime = old_dir->i_mtime =
2980         new_dir->i_ctime = new_dir->i_mtime =
2981         d_inode(old_dentry)->i_ctime =
2982         d_inode(new_dentry)->i_ctime = current_time(old_dir);
2983
2984         return 0;
2985 }
2986
2987 static int shmem_whiteout(struct inode *old_dir, struct dentry *old_dentry)
2988 {
2989         struct dentry *whiteout;
2990         int error;
2991
2992         whiteout = d_alloc(old_dentry->d_parent, &old_dentry->d_name);
2993         if (!whiteout)
2994                 return -ENOMEM;
2995
2996         error = shmem_mknod(old_dir, whiteout,
2997                             S_IFCHR | WHITEOUT_MODE, WHITEOUT_DEV);
2998         dput(whiteout);
2999         if (error)
3000                 return error;
3001
3002         /*
3003          * Cheat and hash the whiteout while the old dentry is still in
3004          * place, instead of playing games with FS_RENAME_DOES_D_MOVE.
3005          *
3006          * d_lookup() will consistently find one of them at this point,
3007          * not sure which one, but that isn't even important.
3008          */
3009         d_rehash(whiteout);
3010         return 0;
3011 }
3012
3013 /*
3014  * The VFS layer already does all the dentry stuff for rename,
3015  * we just have to decrement the usage count for the target if
3016  * it exists so that the VFS layer correctly free's it when it
3017  * gets overwritten.
3018  */
3019 static int shmem_rename2(struct inode *old_dir, struct dentry *old_dentry, struct inode *new_dir, struct dentry *new_dentry, unsigned int flags)
3020 {
3021         struct inode *inode = d_inode(old_dentry);
3022         int they_are_dirs = S_ISDIR(inode->i_mode);
3023
3024         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
3025                 return -EINVAL;
3026
3027         if (flags & RENAME_EXCHANGE)
3028                 return shmem_exchange(old_dir, old_dentry, new_dir, new_dentry);
3029
3030         if (!simple_empty(new_dentry))
3031                 return -ENOTEMPTY;
3032
3033         if (flags & RENAME_WHITEOUT) {
3034                 int error;
3035
3036                 error = shmem_whiteout(old_dir, old_dentry);
3037                 if (error)
3038                         return error;
3039         }
3040
3041         if (d_really_is_positive(new_dentry)) {
3042                 (void) shmem_unlink(new_dir, new_dentry);
3043                 if (they_are_dirs) {
3044                         drop_nlink(d_inode(new_dentry));
3045                         drop_nlink(old_dir);
3046                 }
3047         } else if (they_are_dirs) {
3048                 drop_nlink(old_dir);
3049                 inc_nlink(new_dir);
3050         }
3051
3052         old_dir->i_size -= BOGO_DIRENT_SIZE;
3053         new_dir->i_size += BOGO_DIRENT_SIZE;
3054         old_dir->i_ctime = old_dir->i_mtime =
3055         new_dir->i_ctime = new_dir->i_mtime =
3056         inode->i_ctime = current_time(old_dir);
3057         return 0;
3058 }
3059
3060 static int shmem_symlink(struct inode *dir, struct dentry *dentry, const char *symname)
3061 {
3062         int error;
3063         int len;
3064         struct inode *inode;
3065         struct page *page;
3066
3067         len = strlen(symname) + 1;
3068         if (len > PAGE_SIZE)
3069                 return -ENAMETOOLONG;
3070
3071         inode = shmem_get_inode(dir->i_sb, dir, S_IFLNK | 0777, 0,
3072                                 VM_NORESERVE);
3073         if (!inode)
3074                 return -ENOSPC;
3075
3076         error = security_inode_init_security(inode, dir, &dentry->d_name,
3077                                              shmem_initxattrs, NULL);
3078         if (error) {
3079                 if (error != -EOPNOTSUPP) {
3080                         iput(inode);
3081                         return error;
3082                 }
3083                 error = 0;
3084         }
3085
3086         inode->i_size = len-1;
3087         if (len <= SHORT_SYMLINK_LEN) {
3088                 inode->i_link = kmemdup(symname, len, GFP_KERNEL);
3089                 if (!inode->i_link) {
3090                         iput(inode);
3091                         return -ENOMEM;
3092                 }
3093                 inode->i_op = &shmem_short_symlink_operations;
3094         } else {
3095                 inode_nohighmem(inode);
3096                 error = shmem_getpage(inode, 0, &page, SGP_WRITE);
3097                 if (error) {
3098                         iput(inode);
3099                         return error;
3100                 }
3101                 inode->i_mapping->a_ops = &shmem_aops;
3102                 inode->i_op = &shmem_symlink_inode_operations;
3103                 memcpy(page_address(page), symname, len);
3104                 SetPageUptodate(page);
3105                 set_page_dirty(page);
3106                 unlock_page(page);
3107                 put_page(page);
3108         }
3109         dir->i_size += BOGO_DIRENT_SIZE;
3110         dir->i_ctime = dir->i_mtime = current_time(dir);
3111         d_instantiate(dentry, inode);
3112         dget(dentry);
3113         return 0;
3114 }
3115
3116 static void shmem_put_link(void *arg)
3117 {
3118         mark_page_accessed(arg);
3119         put_page(arg);
3120 }
3121
3122 static const char *shmem_get_link(struct dentry *dentry,
3123                                   struct inode *inode,
3124                                   struct delayed_call *done)
3125 {
3126         struct page *page = NULL;
3127         int error;
3128         if (!dentry) {
3129                 page = find_get_page(inode->i_mapping, 0);
3130                 if (!page)
3131                         return ERR_PTR(-ECHILD);
3132                 if (!PageUptodate(page)) {
3133                         put_page(page);
3134                         return ERR_PTR(-ECHILD);
3135                 }
3136         } else {
3137                 error = shmem_getpage(inode, 0, &page, SGP_READ);
3138                 if (error)
3139                         return ERR_PTR(error);
3140                 unlock_page(page);
3141         }
3142         set_delayed_call(done, shmem_put_link, page);
3143         return page_address(page);
3144 }
3145
3146 #ifdef CONFIG_TMPFS_XATTR
3147 /*
3148  * Superblocks without xattr inode operations may get some security.* xattr
3149  * support from the LSM "for free". As soon as we have any other xattrs
3150  * like ACLs, we also need to implement the security.* handlers at
3151  * filesystem level, though.
3152  */
3153
3154 /*
3155  * Callback for security_inode_init_security() for acquiring xattrs.
3156  */
3157 static int shmem_initxattrs(struct inode *inode,
3158                             const struct xattr *xattr_array,
3159                             void *fs_info)
3160 {
3161         struct shmem_inode_info *info = SHMEM_I(inode);
3162         const struct xattr *xattr;
3163         struct simple_xattr *new_xattr;
3164         size_t len;
3165
3166         for (xattr = xattr_array; xattr->name != NULL; xattr++) {
3167                 new_xattr = simple_xattr_alloc(xattr->value, xattr->value_len);
3168                 if (!new_xattr)
3169                         return -ENOMEM;
3170
3171                 len = strlen(xattr->name) + 1;
3172                 new_xattr->name = kmalloc(XATTR_SECURITY_PREFIX_LEN + len,
3173                                           GFP_KERNEL);
3174                 if (!new_xattr->name) {
3175                         kfree(new_xattr);
3176                         return -ENOMEM;
3177                 }
3178
3179                 memcpy(new_xattr->name, XATTR_SECURITY_PREFIX,
3180                        XATTR_SECURITY_PREFIX_LEN);
3181                 memcpy(new_xattr->name + XATTR_SECURITY_PREFIX_LEN,
3182                        xattr->name, len);
3183
3184                 simple_xattr_list_add(&info->xattrs, new_xattr);
3185         }
3186
3187         return 0;
3188 }
3189
3190 static int shmem_xattr_handler_get(const struct xattr_handler *handler,
3191                                    struct dentry *unused, struct inode *inode,
3192                                    const char *name, void *buffer, size_t size)
3193 {
3194         struct shmem_inode_info *info = SHMEM_I(inode);
3195
3196         name = xattr_full_name(handler, name);
3197         return simple_xattr_get(&info->xattrs, name, buffer, size);
3198 }
3199
3200 static int shmem_xattr_handler_set(const struct xattr_handler *handler,
3201                                    struct dentry *unused, struct inode *inode,
3202                                    const char *name, const void *value,
3203                                    size_t size, int flags)
3204 {
3205         struct shmem_inode_info *info = SHMEM_I(inode);
3206
3207         name = xattr_full_name(handler, name);
3208         return simple_xattr_set(&info->xattrs, name, value, size, flags);
3209 }
3210
3211 static const struct xattr_handler shmem_security_xattr_handler = {
3212         .prefix = XATTR_SECURITY_PREFIX,
3213         .get = shmem_xattr_handler_get,
3214         .set = shmem_xattr_handler_set,
3215 };
3216
3217 static const struct xattr_handler shmem_trusted_xattr_handler = {
3218         .prefix = XATTR_TRUSTED_PREFIX,
3219         .get = shmem_xattr_handler_get,
3220         .set = shmem_xattr_handler_set,
3221 };
3222
3223 static const struct xattr_handler *shmem_xattr_handlers[] = {
3224 #ifdef CONFIG_TMPFS_POSIX_ACL
3225         &posix_acl_access_xattr_handler,
3226         &posix_acl_default_xattr_handler,
3227 #endif
3228         &shmem_security_xattr_handler,
3229         &shmem_trusted_xattr_handler,
3230         NULL
3231 };
3232
3233 static ssize_t shmem_listxattr(struct dentry *dentry, char *buffer, size_t size)
3234 {
3235         struct shmem_inode_info *info = SHMEM_I(d_inode(dentry));
3236         return simple_xattr_list(d_inode(dentry), &info->xattrs, buffer, size);
3237 }
3238 #endif /* CONFIG_TMPFS_XATTR */
3239
3240 static const struct inode_operations shmem_short_symlink_operations = {
3241         .get_link       = simple_get_link,
3242 #ifdef CONFIG_TMPFS_XATTR
3243         .listxattr      = shmem_listxattr,
3244 #endif
3245 };
3246
3247 static const struct inode_operations shmem_symlink_inode_operations = {
3248         .get_link       = shmem_get_link,
3249 #ifdef CONFIG_TMPFS_XATTR
3250         .listxattr      = shmem_listxattr,
3251 #endif
3252 };
3253
3254 static struct dentry *shmem_get_parent(struct dentry *child)
3255 {
3256         return ERR_PTR(-ESTALE);
3257 }
3258
3259 static int shmem_match(struct inode *ino, void *vfh)
3260 {
3261         __u32 *fh = vfh;
3262         __u64 inum = fh[2];
3263         inum = (inum << 32) | fh[1];
3264         return ino->i_ino == inum && fh[0] == ino->i_generation;
3265 }
3266
3267 /* Find any alias of inode, but prefer a hashed alias */
3268 static struct dentry *shmem_find_alias(struct inode *inode)
3269 {
3270         struct dentry *alias = d_find_alias(inode);
3271
3272         return alias ?: d_find_any_alias(inode);
3273 }
3274
3275
3276 static struct dentry *shmem_fh_to_dentry(struct super_block *sb,
3277                 struct fid *fid, int fh_len, int fh_type)
3278 {
3279         struct inode *inode;
3280         struct dentry *dentry = NULL;
3281         u64 inum;
3282
3283         if (fh_len < 3)
3284                 return NULL;
3285
3286         inum = fid->raw[2];
3287         inum = (inum << 32) | fid->raw[1];
3288
3289         inode = ilookup5(sb, (unsigned long)(inum + fid->raw[0]),
3290                         shmem_match, fid->raw);
3291         if (inode) {
3292                 dentry = shmem_find_alias(inode);
3293                 iput(inode);
3294         }
3295
3296         return dentry;
3297 }
3298
3299 static int shmem_encode_fh(struct inode *inode, __u32 *fh, int *len,