mm: rid swapoff of quadratic complexity
[sfrench/cifs-2.6.git] / mm / swapfile.c
1 /*
2  *  linux/mm/swapfile.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *  Swap reorganised 29.12.95, Stephen Tweedie
6  */
7
8 #include <linux/mm.h>
9 #include <linux/sched/mm.h>
10 #include <linux/sched/task.h>
11 #include <linux/hugetlb.h>
12 #include <linux/mman.h>
13 #include <linux/slab.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/swap.h>
16 #include <linux/vmalloc.h>
17 #include <linux/pagemap.h>
18 #include <linux/namei.h>
19 #include <linux/shmem_fs.h>
20 #include <linux/blkdev.h>
21 #include <linux/random.h>
22 #include <linux/writeback.h>
23 #include <linux/proc_fs.h>
24 #include <linux/seq_file.h>
25 #include <linux/init.h>
26 #include <linux/ksm.h>
27 #include <linux/rmap.h>
28 #include <linux/security.h>
29 #include <linux/backing-dev.h>
30 #include <linux/mutex.h>
31 #include <linux/capability.h>
32 #include <linux/syscalls.h>
33 #include <linux/memcontrol.h>
34 #include <linux/poll.h>
35 #include <linux/oom.h>
36 #include <linux/frontswap.h>
37 #include <linux/swapfile.h>
38 #include <linux/export.h>
39 #include <linux/swap_slots.h>
40 #include <linux/sort.h>
41
42 #include <asm/pgtable.h>
43 #include <asm/tlbflush.h>
44 #include <linux/swapops.h>
45 #include <linux/swap_cgroup.h>
46
47 static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
48                                  unsigned char);
49 static void free_swap_count_continuations(struct swap_info_struct *);
50 static sector_t map_swap_entry(swp_entry_t, struct block_device**);
51
52 DEFINE_SPINLOCK(swap_lock);
53 static unsigned int nr_swapfiles;
54 atomic_long_t nr_swap_pages;
55 /*
56  * Some modules use swappable objects and may try to swap them out under
57  * memory pressure (via the shrinker). Before doing so, they may wish to
58  * check to see if any swap space is available.
59  */
60 EXPORT_SYMBOL_GPL(nr_swap_pages);
61 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
62 long total_swap_pages;
63 static int least_priority = -1;
64
65 static const char Bad_file[] = "Bad swap file entry ";
66 static const char Unused_file[] = "Unused swap file entry ";
67 static const char Bad_offset[] = "Bad swap offset entry ";
68 static const char Unused_offset[] = "Unused swap offset entry ";
69
70 /*
71  * all active swap_info_structs
72  * protected with swap_lock, and ordered by priority.
73  */
74 PLIST_HEAD(swap_active_head);
75
76 /*
77  * all available (active, not full) swap_info_structs
78  * protected with swap_avail_lock, ordered by priority.
79  * This is used by get_swap_page() instead of swap_active_head
80  * because swap_active_head includes all swap_info_structs,
81  * but get_swap_page() doesn't need to look at full ones.
82  * This uses its own lock instead of swap_lock because when a
83  * swap_info_struct changes between not-full/full, it needs to
84  * add/remove itself to/from this list, but the swap_info_struct->lock
85  * is held and the locking order requires swap_lock to be taken
86  * before any swap_info_struct->lock.
87  */
88 static struct plist_head *swap_avail_heads;
89 static DEFINE_SPINLOCK(swap_avail_lock);
90
91 struct swap_info_struct *swap_info[MAX_SWAPFILES];
92
93 static DEFINE_MUTEX(swapon_mutex);
94
95 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
96 /* Activity counter to indicate that a swapon or swapoff has occurred */
97 static atomic_t proc_poll_event = ATOMIC_INIT(0);
98
99 atomic_t nr_rotate_swap = ATOMIC_INIT(0);
100
101 static inline unsigned char swap_count(unsigned char ent)
102 {
103         return ent & ~SWAP_HAS_CACHE;   /* may include COUNT_CONTINUED flag */
104 }
105
106 /* Reclaim the swap entry anyway if possible */
107 #define TTRS_ANYWAY             0x1
108 /*
109  * Reclaim the swap entry if there are no more mappings of the
110  * corresponding page
111  */
112 #define TTRS_UNMAPPED           0x2
113 /* Reclaim the swap entry if swap is getting full*/
114 #define TTRS_FULL               0x4
115
116 /* returns 1 if swap entry is freed */
117 static int __try_to_reclaim_swap(struct swap_info_struct *si,
118                                  unsigned long offset, unsigned long flags)
119 {
120         swp_entry_t entry = swp_entry(si->type, offset);
121         struct page *page;
122         int ret = 0;
123
124         page = find_get_page(swap_address_space(entry), offset);
125         if (!page)
126                 return 0;
127         /*
128          * When this function is called from scan_swap_map_slots() and it's
129          * called by vmscan.c at reclaiming pages. So, we hold a lock on a page,
130          * here. We have to use trylock for avoiding deadlock. This is a special
131          * case and you should use try_to_free_swap() with explicit lock_page()
132          * in usual operations.
133          */
134         if (trylock_page(page)) {
135                 if ((flags & TTRS_ANYWAY) ||
136                     ((flags & TTRS_UNMAPPED) && !page_mapped(page)) ||
137                     ((flags & TTRS_FULL) && mem_cgroup_swap_full(page)))
138                         ret = try_to_free_swap(page);
139                 unlock_page(page);
140         }
141         put_page(page);
142         return ret;
143 }
144
145 /*
146  * swapon tell device that all the old swap contents can be discarded,
147  * to allow the swap device to optimize its wear-levelling.
148  */
149 static int discard_swap(struct swap_info_struct *si)
150 {
151         struct swap_extent *se;
152         sector_t start_block;
153         sector_t nr_blocks;
154         int err = 0;
155
156         /* Do not discard the swap header page! */
157         se = &si->first_swap_extent;
158         start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
159         nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
160         if (nr_blocks) {
161                 err = blkdev_issue_discard(si->bdev, start_block,
162                                 nr_blocks, GFP_KERNEL, 0);
163                 if (err)
164                         return err;
165                 cond_resched();
166         }
167
168         list_for_each_entry(se, &si->first_swap_extent.list, list) {
169                 start_block = se->start_block << (PAGE_SHIFT - 9);
170                 nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
171
172                 err = blkdev_issue_discard(si->bdev, start_block,
173                                 nr_blocks, GFP_KERNEL, 0);
174                 if (err)
175                         break;
176
177                 cond_resched();
178         }
179         return err;             /* That will often be -EOPNOTSUPP */
180 }
181
182 /*
183  * swap allocation tell device that a cluster of swap can now be discarded,
184  * to allow the swap device to optimize its wear-levelling.
185  */
186 static void discard_swap_cluster(struct swap_info_struct *si,
187                                  pgoff_t start_page, pgoff_t nr_pages)
188 {
189         struct swap_extent *se = si->curr_swap_extent;
190         int found_extent = 0;
191
192         while (nr_pages) {
193                 if (se->start_page <= start_page &&
194                     start_page < se->start_page + se->nr_pages) {
195                         pgoff_t offset = start_page - se->start_page;
196                         sector_t start_block = se->start_block + offset;
197                         sector_t nr_blocks = se->nr_pages - offset;
198
199                         if (nr_blocks > nr_pages)
200                                 nr_blocks = nr_pages;
201                         start_page += nr_blocks;
202                         nr_pages -= nr_blocks;
203
204                         if (!found_extent++)
205                                 si->curr_swap_extent = se;
206
207                         start_block <<= PAGE_SHIFT - 9;
208                         nr_blocks <<= PAGE_SHIFT - 9;
209                         if (blkdev_issue_discard(si->bdev, start_block,
210                                     nr_blocks, GFP_NOIO, 0))
211                                 break;
212                 }
213
214                 se = list_next_entry(se, list);
215         }
216 }
217
218 #ifdef CONFIG_THP_SWAP
219 #define SWAPFILE_CLUSTER        HPAGE_PMD_NR
220
221 #define swap_entry_size(size)   (size)
222 #else
223 #define SWAPFILE_CLUSTER        256
224
225 /*
226  * Define swap_entry_size() as constant to let compiler to optimize
227  * out some code if !CONFIG_THP_SWAP
228  */
229 #define swap_entry_size(size)   1
230 #endif
231 #define LATENCY_LIMIT           256
232
233 static inline void cluster_set_flag(struct swap_cluster_info *info,
234         unsigned int flag)
235 {
236         info->flags = flag;
237 }
238
239 static inline unsigned int cluster_count(struct swap_cluster_info *info)
240 {
241         return info->data;
242 }
243
244 static inline void cluster_set_count(struct swap_cluster_info *info,
245                                      unsigned int c)
246 {
247         info->data = c;
248 }
249
250 static inline void cluster_set_count_flag(struct swap_cluster_info *info,
251                                          unsigned int c, unsigned int f)
252 {
253         info->flags = f;
254         info->data = c;
255 }
256
257 static inline unsigned int cluster_next(struct swap_cluster_info *info)
258 {
259         return info->data;
260 }
261
262 static inline void cluster_set_next(struct swap_cluster_info *info,
263                                     unsigned int n)
264 {
265         info->data = n;
266 }
267
268 static inline void cluster_set_next_flag(struct swap_cluster_info *info,
269                                          unsigned int n, unsigned int f)
270 {
271         info->flags = f;
272         info->data = n;
273 }
274
275 static inline bool cluster_is_free(struct swap_cluster_info *info)
276 {
277         return info->flags & CLUSTER_FLAG_FREE;
278 }
279
280 static inline bool cluster_is_null(struct swap_cluster_info *info)
281 {
282         return info->flags & CLUSTER_FLAG_NEXT_NULL;
283 }
284
285 static inline void cluster_set_null(struct swap_cluster_info *info)
286 {
287         info->flags = CLUSTER_FLAG_NEXT_NULL;
288         info->data = 0;
289 }
290
291 static inline bool cluster_is_huge(struct swap_cluster_info *info)
292 {
293         if (IS_ENABLED(CONFIG_THP_SWAP))
294                 return info->flags & CLUSTER_FLAG_HUGE;
295         return false;
296 }
297
298 static inline void cluster_clear_huge(struct swap_cluster_info *info)
299 {
300         info->flags &= ~CLUSTER_FLAG_HUGE;
301 }
302
303 static inline struct swap_cluster_info *lock_cluster(struct swap_info_struct *si,
304                                                      unsigned long offset)
305 {
306         struct swap_cluster_info *ci;
307
308         ci = si->cluster_info;
309         if (ci) {
310                 ci += offset / SWAPFILE_CLUSTER;
311                 spin_lock(&ci->lock);
312         }
313         return ci;
314 }
315
316 static inline void unlock_cluster(struct swap_cluster_info *ci)
317 {
318         if (ci)
319                 spin_unlock(&ci->lock);
320 }
321
322 /*
323  * Determine the locking method in use for this device.  Return
324  * swap_cluster_info if SSD-style cluster-based locking is in place.
325  */
326 static inline struct swap_cluster_info *lock_cluster_or_swap_info(
327                 struct swap_info_struct *si, unsigned long offset)
328 {
329         struct swap_cluster_info *ci;
330
331         /* Try to use fine-grained SSD-style locking if available: */
332         ci = lock_cluster(si, offset);
333         /* Otherwise, fall back to traditional, coarse locking: */
334         if (!ci)
335                 spin_lock(&si->lock);
336
337         return ci;
338 }
339
340 static inline void unlock_cluster_or_swap_info(struct swap_info_struct *si,
341                                                struct swap_cluster_info *ci)
342 {
343         if (ci)
344                 unlock_cluster(ci);
345         else
346                 spin_unlock(&si->lock);
347 }
348
349 static inline bool cluster_list_empty(struct swap_cluster_list *list)
350 {
351         return cluster_is_null(&list->head);
352 }
353
354 static inline unsigned int cluster_list_first(struct swap_cluster_list *list)
355 {
356         return cluster_next(&list->head);
357 }
358
359 static void cluster_list_init(struct swap_cluster_list *list)
360 {
361         cluster_set_null(&list->head);
362         cluster_set_null(&list->tail);
363 }
364
365 static void cluster_list_add_tail(struct swap_cluster_list *list,
366                                   struct swap_cluster_info *ci,
367                                   unsigned int idx)
368 {
369         if (cluster_list_empty(list)) {
370                 cluster_set_next_flag(&list->head, idx, 0);
371                 cluster_set_next_flag(&list->tail, idx, 0);
372         } else {
373                 struct swap_cluster_info *ci_tail;
374                 unsigned int tail = cluster_next(&list->tail);
375
376                 /*
377                  * Nested cluster lock, but both cluster locks are
378                  * only acquired when we held swap_info_struct->lock
379                  */
380                 ci_tail = ci + tail;
381                 spin_lock_nested(&ci_tail->lock, SINGLE_DEPTH_NESTING);
382                 cluster_set_next(ci_tail, idx);
383                 spin_unlock(&ci_tail->lock);
384                 cluster_set_next_flag(&list->tail, idx, 0);
385         }
386 }
387
388 static unsigned int cluster_list_del_first(struct swap_cluster_list *list,
389                                            struct swap_cluster_info *ci)
390 {
391         unsigned int idx;
392
393         idx = cluster_next(&list->head);
394         if (cluster_next(&list->tail) == idx) {
395                 cluster_set_null(&list->head);
396                 cluster_set_null(&list->tail);
397         } else
398                 cluster_set_next_flag(&list->head,
399                                       cluster_next(&ci[idx]), 0);
400
401         return idx;
402 }
403
404 /* Add a cluster to discard list and schedule it to do discard */
405 static void swap_cluster_schedule_discard(struct swap_info_struct *si,
406                 unsigned int idx)
407 {
408         /*
409          * If scan_swap_map() can't find a free cluster, it will check
410          * si->swap_map directly. To make sure the discarding cluster isn't
411          * taken by scan_swap_map(), mark the swap entries bad (occupied). It
412          * will be cleared after discard
413          */
414         memset(si->swap_map + idx * SWAPFILE_CLUSTER,
415                         SWAP_MAP_BAD, SWAPFILE_CLUSTER);
416
417         cluster_list_add_tail(&si->discard_clusters, si->cluster_info, idx);
418
419         schedule_work(&si->discard_work);
420 }
421
422 static void __free_cluster(struct swap_info_struct *si, unsigned long idx)
423 {
424         struct swap_cluster_info *ci = si->cluster_info;
425
426         cluster_set_flag(ci + idx, CLUSTER_FLAG_FREE);
427         cluster_list_add_tail(&si->free_clusters, ci, idx);
428 }
429
430 /*
431  * Doing discard actually. After a cluster discard is finished, the cluster
432  * will be added to free cluster list. caller should hold si->lock.
433 */
434 static void swap_do_scheduled_discard(struct swap_info_struct *si)
435 {
436         struct swap_cluster_info *info, *ci;
437         unsigned int idx;
438
439         info = si->cluster_info;
440
441         while (!cluster_list_empty(&si->discard_clusters)) {
442                 idx = cluster_list_del_first(&si->discard_clusters, info);
443                 spin_unlock(&si->lock);
444
445                 discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
446                                 SWAPFILE_CLUSTER);
447
448                 spin_lock(&si->lock);
449                 ci = lock_cluster(si, idx * SWAPFILE_CLUSTER);
450                 __free_cluster(si, idx);
451                 memset(si->swap_map + idx * SWAPFILE_CLUSTER,
452                                 0, SWAPFILE_CLUSTER);
453                 unlock_cluster(ci);
454         }
455 }
456
457 static void swap_discard_work(struct work_struct *work)
458 {
459         struct swap_info_struct *si;
460
461         si = container_of(work, struct swap_info_struct, discard_work);
462
463         spin_lock(&si->lock);
464         swap_do_scheduled_discard(si);
465         spin_unlock(&si->lock);
466 }
467
468 static void alloc_cluster(struct swap_info_struct *si, unsigned long idx)
469 {
470         struct swap_cluster_info *ci = si->cluster_info;
471
472         VM_BUG_ON(cluster_list_first(&si->free_clusters) != idx);
473         cluster_list_del_first(&si->free_clusters, ci);
474         cluster_set_count_flag(ci + idx, 0, 0);
475 }
476
477 static void free_cluster(struct swap_info_struct *si, unsigned long idx)
478 {
479         struct swap_cluster_info *ci = si->cluster_info + idx;
480
481         VM_BUG_ON(cluster_count(ci) != 0);
482         /*
483          * If the swap is discardable, prepare discard the cluster
484          * instead of free it immediately. The cluster will be freed
485          * after discard.
486          */
487         if ((si->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
488             (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
489                 swap_cluster_schedule_discard(si, idx);
490                 return;
491         }
492
493         __free_cluster(si, idx);
494 }
495
496 /*
497  * The cluster corresponding to page_nr will be used. The cluster will be
498  * removed from free cluster list and its usage counter will be increased.
499  */
500 static void inc_cluster_info_page(struct swap_info_struct *p,
501         struct swap_cluster_info *cluster_info, unsigned long page_nr)
502 {
503         unsigned long idx = page_nr / SWAPFILE_CLUSTER;
504
505         if (!cluster_info)
506                 return;
507         if (cluster_is_free(&cluster_info[idx]))
508                 alloc_cluster(p, idx);
509
510         VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
511         cluster_set_count(&cluster_info[idx],
512                 cluster_count(&cluster_info[idx]) + 1);
513 }
514
515 /*
516  * The cluster corresponding to page_nr decreases one usage. If the usage
517  * counter becomes 0, which means no page in the cluster is in using, we can
518  * optionally discard the cluster and add it to free cluster list.
519  */
520 static void dec_cluster_info_page(struct swap_info_struct *p,
521         struct swap_cluster_info *cluster_info, unsigned long page_nr)
522 {
523         unsigned long idx = page_nr / SWAPFILE_CLUSTER;
524
525         if (!cluster_info)
526                 return;
527
528         VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
529         cluster_set_count(&cluster_info[idx],
530                 cluster_count(&cluster_info[idx]) - 1);
531
532         if (cluster_count(&cluster_info[idx]) == 0)
533                 free_cluster(p, idx);
534 }
535
536 /*
537  * It's possible scan_swap_map() uses a free cluster in the middle of free
538  * cluster list. Avoiding such abuse to avoid list corruption.
539  */
540 static bool
541 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
542         unsigned long offset)
543 {
544         struct percpu_cluster *percpu_cluster;
545         bool conflict;
546
547         offset /= SWAPFILE_CLUSTER;
548         conflict = !cluster_list_empty(&si->free_clusters) &&
549                 offset != cluster_list_first(&si->free_clusters) &&
550                 cluster_is_free(&si->cluster_info[offset]);
551
552         if (!conflict)
553                 return false;
554
555         percpu_cluster = this_cpu_ptr(si->percpu_cluster);
556         cluster_set_null(&percpu_cluster->index);
557         return true;
558 }
559
560 /*
561  * Try to get a swap entry from current cpu's swap entry pool (a cluster). This
562  * might involve allocating a new cluster for current CPU too.
563  */
564 static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
565         unsigned long *offset, unsigned long *scan_base)
566 {
567         struct percpu_cluster *cluster;
568         struct swap_cluster_info *ci;
569         bool found_free;
570         unsigned long tmp, max;
571
572 new_cluster:
573         cluster = this_cpu_ptr(si->percpu_cluster);
574         if (cluster_is_null(&cluster->index)) {
575                 if (!cluster_list_empty(&si->free_clusters)) {
576                         cluster->index = si->free_clusters.head;
577                         cluster->next = cluster_next(&cluster->index) *
578                                         SWAPFILE_CLUSTER;
579                 } else if (!cluster_list_empty(&si->discard_clusters)) {
580                         /*
581                          * we don't have free cluster but have some clusters in
582                          * discarding, do discard now and reclaim them
583                          */
584                         swap_do_scheduled_discard(si);
585                         *scan_base = *offset = si->cluster_next;
586                         goto new_cluster;
587                 } else
588                         return false;
589         }
590
591         found_free = false;
592
593         /*
594          * Other CPUs can use our cluster if they can't find a free cluster,
595          * check if there is still free entry in the cluster
596          */
597         tmp = cluster->next;
598         max = min_t(unsigned long, si->max,
599                     (cluster_next(&cluster->index) + 1) * SWAPFILE_CLUSTER);
600         if (tmp >= max) {
601                 cluster_set_null(&cluster->index);
602                 goto new_cluster;
603         }
604         ci = lock_cluster(si, tmp);
605         while (tmp < max) {
606                 if (!si->swap_map[tmp]) {
607                         found_free = true;
608                         break;
609                 }
610                 tmp++;
611         }
612         unlock_cluster(ci);
613         if (!found_free) {
614                 cluster_set_null(&cluster->index);
615                 goto new_cluster;
616         }
617         cluster->next = tmp + 1;
618         *offset = tmp;
619         *scan_base = tmp;
620         return found_free;
621 }
622
623 static void __del_from_avail_list(struct swap_info_struct *p)
624 {
625         int nid;
626
627         for_each_node(nid)
628                 plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]);
629 }
630
631 static void del_from_avail_list(struct swap_info_struct *p)
632 {
633         spin_lock(&swap_avail_lock);
634         __del_from_avail_list(p);
635         spin_unlock(&swap_avail_lock);
636 }
637
638 static void swap_range_alloc(struct swap_info_struct *si, unsigned long offset,
639                              unsigned int nr_entries)
640 {
641         unsigned int end = offset + nr_entries - 1;
642
643         if (offset == si->lowest_bit)
644                 si->lowest_bit += nr_entries;
645         if (end == si->highest_bit)
646                 si->highest_bit -= nr_entries;
647         si->inuse_pages += nr_entries;
648         if (si->inuse_pages == si->pages) {
649                 si->lowest_bit = si->max;
650                 si->highest_bit = 0;
651                 del_from_avail_list(si);
652         }
653 }
654
655 static void add_to_avail_list(struct swap_info_struct *p)
656 {
657         int nid;
658
659         spin_lock(&swap_avail_lock);
660         for_each_node(nid) {
661                 WARN_ON(!plist_node_empty(&p->avail_lists[nid]));
662                 plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]);
663         }
664         spin_unlock(&swap_avail_lock);
665 }
666
667 static void swap_range_free(struct swap_info_struct *si, unsigned long offset,
668                             unsigned int nr_entries)
669 {
670         unsigned long end = offset + nr_entries - 1;
671         void (*swap_slot_free_notify)(struct block_device *, unsigned long);
672
673         if (offset < si->lowest_bit)
674                 si->lowest_bit = offset;
675         if (end > si->highest_bit) {
676                 bool was_full = !si->highest_bit;
677
678                 si->highest_bit = end;
679                 if (was_full && (si->flags & SWP_WRITEOK))
680                         add_to_avail_list(si);
681         }
682         atomic_long_add(nr_entries, &nr_swap_pages);
683         si->inuse_pages -= nr_entries;
684         if (si->flags & SWP_BLKDEV)
685                 swap_slot_free_notify =
686                         si->bdev->bd_disk->fops->swap_slot_free_notify;
687         else
688                 swap_slot_free_notify = NULL;
689         while (offset <= end) {
690                 frontswap_invalidate_page(si->type, offset);
691                 if (swap_slot_free_notify)
692                         swap_slot_free_notify(si->bdev, offset);
693                 offset++;
694         }
695 }
696
697 static int scan_swap_map_slots(struct swap_info_struct *si,
698                                unsigned char usage, int nr,
699                                swp_entry_t slots[])
700 {
701         struct swap_cluster_info *ci;
702         unsigned long offset;
703         unsigned long scan_base;
704         unsigned long last_in_cluster = 0;
705         int latency_ration = LATENCY_LIMIT;
706         int n_ret = 0;
707
708         if (nr > SWAP_BATCH)
709                 nr = SWAP_BATCH;
710
711         /*
712          * We try to cluster swap pages by allocating them sequentially
713          * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
714          * way, however, we resort to first-free allocation, starting
715          * a new cluster.  This prevents us from scattering swap pages
716          * all over the entire swap partition, so that we reduce
717          * overall disk seek times between swap pages.  -- sct
718          * But we do now try to find an empty cluster.  -Andrea
719          * And we let swap pages go all over an SSD partition.  Hugh
720          */
721
722         si->flags += SWP_SCANNING;
723         scan_base = offset = si->cluster_next;
724
725         /* SSD algorithm */
726         if (si->cluster_info) {
727                 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
728                         goto checks;
729                 else
730                         goto scan;
731         }
732
733         if (unlikely(!si->cluster_nr--)) {
734                 if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
735                         si->cluster_nr = SWAPFILE_CLUSTER - 1;
736                         goto checks;
737                 }
738
739                 spin_unlock(&si->lock);
740
741                 /*
742                  * If seek is expensive, start searching for new cluster from
743                  * start of partition, to minimize the span of allocated swap.
744                  * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info
745                  * case, just handled by scan_swap_map_try_ssd_cluster() above.
746                  */
747                 scan_base = offset = si->lowest_bit;
748                 last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
749
750                 /* Locate the first empty (unaligned) cluster */
751                 for (; last_in_cluster <= si->highest_bit; offset++) {
752                         if (si->swap_map[offset])
753                                 last_in_cluster = offset + SWAPFILE_CLUSTER;
754                         else if (offset == last_in_cluster) {
755                                 spin_lock(&si->lock);
756                                 offset -= SWAPFILE_CLUSTER - 1;
757                                 si->cluster_next = offset;
758                                 si->cluster_nr = SWAPFILE_CLUSTER - 1;
759                                 goto checks;
760                         }
761                         if (unlikely(--latency_ration < 0)) {
762                                 cond_resched();
763                                 latency_ration = LATENCY_LIMIT;
764                         }
765                 }
766
767                 offset = scan_base;
768                 spin_lock(&si->lock);
769                 si->cluster_nr = SWAPFILE_CLUSTER - 1;
770         }
771
772 checks:
773         if (si->cluster_info) {
774                 while (scan_swap_map_ssd_cluster_conflict(si, offset)) {
775                 /* take a break if we already got some slots */
776                         if (n_ret)
777                                 goto done;
778                         if (!scan_swap_map_try_ssd_cluster(si, &offset,
779                                                         &scan_base))
780                                 goto scan;
781                 }
782         }
783         if (!(si->flags & SWP_WRITEOK))
784                 goto no_page;
785         if (!si->highest_bit)
786                 goto no_page;
787         if (offset > si->highest_bit)
788                 scan_base = offset = si->lowest_bit;
789
790         ci = lock_cluster(si, offset);
791         /* reuse swap entry of cache-only swap if not busy. */
792         if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
793                 int swap_was_freed;
794                 unlock_cluster(ci);
795                 spin_unlock(&si->lock);
796                 swap_was_freed = __try_to_reclaim_swap(si, offset, TTRS_ANYWAY);
797                 spin_lock(&si->lock);
798                 /* entry was freed successfully, try to use this again */
799                 if (swap_was_freed)
800                         goto checks;
801                 goto scan; /* check next one */
802         }
803
804         if (si->swap_map[offset]) {
805                 unlock_cluster(ci);
806                 if (!n_ret)
807                         goto scan;
808                 else
809                         goto done;
810         }
811         si->swap_map[offset] = usage;
812         inc_cluster_info_page(si, si->cluster_info, offset);
813         unlock_cluster(ci);
814
815         swap_range_alloc(si, offset, 1);
816         si->cluster_next = offset + 1;
817         slots[n_ret++] = swp_entry(si->type, offset);
818
819         /* got enough slots or reach max slots? */
820         if ((n_ret == nr) || (offset >= si->highest_bit))
821                 goto done;
822
823         /* search for next available slot */
824
825         /* time to take a break? */
826         if (unlikely(--latency_ration < 0)) {
827                 if (n_ret)
828                         goto done;
829                 spin_unlock(&si->lock);
830                 cond_resched();
831                 spin_lock(&si->lock);
832                 latency_ration = LATENCY_LIMIT;
833         }
834
835         /* try to get more slots in cluster */
836         if (si->cluster_info) {
837                 if (scan_swap_map_try_ssd_cluster(si, &offset, &scan_base))
838                         goto checks;
839                 else
840                         goto done;
841         }
842         /* non-ssd case */
843         ++offset;
844
845         /* non-ssd case, still more slots in cluster? */
846         if (si->cluster_nr && !si->swap_map[offset]) {
847                 --si->cluster_nr;
848                 goto checks;
849         }
850
851 done:
852         si->flags -= SWP_SCANNING;
853         return n_ret;
854
855 scan:
856         spin_unlock(&si->lock);
857         while (++offset <= si->highest_bit) {
858                 if (!si->swap_map[offset]) {
859                         spin_lock(&si->lock);
860                         goto checks;
861                 }
862                 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
863                         spin_lock(&si->lock);
864                         goto checks;
865                 }
866                 if (unlikely(--latency_ration < 0)) {
867                         cond_resched();
868                         latency_ration = LATENCY_LIMIT;
869                 }
870         }
871         offset = si->lowest_bit;
872         while (offset < scan_base) {
873                 if (!si->swap_map[offset]) {
874                         spin_lock(&si->lock);
875                         goto checks;
876                 }
877                 if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
878                         spin_lock(&si->lock);
879                         goto checks;
880                 }
881                 if (unlikely(--latency_ration < 0)) {
882                         cond_resched();
883                         latency_ration = LATENCY_LIMIT;
884                 }
885                 offset++;
886         }
887         spin_lock(&si->lock);
888
889 no_page:
890         si->flags -= SWP_SCANNING;
891         return n_ret;
892 }
893
894 static int swap_alloc_cluster(struct swap_info_struct *si, swp_entry_t *slot)
895 {
896         unsigned long idx;
897         struct swap_cluster_info *ci;
898         unsigned long offset, i;
899         unsigned char *map;
900
901         /*
902          * Should not even be attempting cluster allocations when huge
903          * page swap is disabled.  Warn and fail the allocation.
904          */
905         if (!IS_ENABLED(CONFIG_THP_SWAP)) {
906                 VM_WARN_ON_ONCE(1);
907                 return 0;
908         }
909
910         if (cluster_list_empty(&si->free_clusters))
911                 return 0;
912
913         idx = cluster_list_first(&si->free_clusters);
914         offset = idx * SWAPFILE_CLUSTER;
915         ci = lock_cluster(si, offset);
916         alloc_cluster(si, idx);
917         cluster_set_count_flag(ci, SWAPFILE_CLUSTER, CLUSTER_FLAG_HUGE);
918
919         map = si->swap_map + offset;
920         for (i = 0; i < SWAPFILE_CLUSTER; i++)
921                 map[i] = SWAP_HAS_CACHE;
922         unlock_cluster(ci);
923         swap_range_alloc(si, offset, SWAPFILE_CLUSTER);
924         *slot = swp_entry(si->type, offset);
925
926         return 1;
927 }
928
929 static void swap_free_cluster(struct swap_info_struct *si, unsigned long idx)
930 {
931         unsigned long offset = idx * SWAPFILE_CLUSTER;
932         struct swap_cluster_info *ci;
933
934         ci = lock_cluster(si, offset);
935         memset(si->swap_map + offset, 0, SWAPFILE_CLUSTER);
936         cluster_set_count_flag(ci, 0, 0);
937         free_cluster(si, idx);
938         unlock_cluster(ci);
939         swap_range_free(si, offset, SWAPFILE_CLUSTER);
940 }
941
942 static unsigned long scan_swap_map(struct swap_info_struct *si,
943                                    unsigned char usage)
944 {
945         swp_entry_t entry;
946         int n_ret;
947
948         n_ret = scan_swap_map_slots(si, usage, 1, &entry);
949
950         if (n_ret)
951                 return swp_offset(entry);
952         else
953                 return 0;
954
955 }
956
957 int get_swap_pages(int n_goal, swp_entry_t swp_entries[], int entry_size)
958 {
959         unsigned long size = swap_entry_size(entry_size);
960         struct swap_info_struct *si, *next;
961         long avail_pgs;
962         int n_ret = 0;
963         int node;
964
965         /* Only single cluster request supported */
966         WARN_ON_ONCE(n_goal > 1 && size == SWAPFILE_CLUSTER);
967
968         avail_pgs = atomic_long_read(&nr_swap_pages) / size;
969         if (avail_pgs <= 0)
970                 goto noswap;
971
972         if (n_goal > SWAP_BATCH)
973                 n_goal = SWAP_BATCH;
974
975         if (n_goal > avail_pgs)
976                 n_goal = avail_pgs;
977
978         atomic_long_sub(n_goal * size, &nr_swap_pages);
979
980         spin_lock(&swap_avail_lock);
981
982 start_over:
983         node = numa_node_id();
984         plist_for_each_entry_safe(si, next, &swap_avail_heads[node], avail_lists[node]) {
985                 /* requeue si to after same-priority siblings */
986                 plist_requeue(&si->avail_lists[node], &swap_avail_heads[node]);
987                 spin_unlock(&swap_avail_lock);
988                 spin_lock(&si->lock);
989                 if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) {
990                         spin_lock(&swap_avail_lock);
991                         if (plist_node_empty(&si->avail_lists[node])) {
992                                 spin_unlock(&si->lock);
993                                 goto nextsi;
994                         }
995                         WARN(!si->highest_bit,
996                              "swap_info %d in list but !highest_bit\n",
997                              si->type);
998                         WARN(!(si->flags & SWP_WRITEOK),
999                              "swap_info %d in list but !SWP_WRITEOK\n",
1000                              si->type);
1001                         __del_from_avail_list(si);
1002                         spin_unlock(&si->lock);
1003                         goto nextsi;
1004                 }
1005                 if (size == SWAPFILE_CLUSTER) {
1006                         if (!(si->flags & SWP_FS))
1007                                 n_ret = swap_alloc_cluster(si, swp_entries);
1008                 } else
1009                         n_ret = scan_swap_map_slots(si, SWAP_HAS_CACHE,
1010                                                     n_goal, swp_entries);
1011                 spin_unlock(&si->lock);
1012                 if (n_ret || size == SWAPFILE_CLUSTER)
1013                         goto check_out;
1014                 pr_debug("scan_swap_map of si %d failed to find offset\n",
1015                         si->type);
1016
1017                 spin_lock(&swap_avail_lock);
1018 nextsi:
1019                 /*
1020                  * if we got here, it's likely that si was almost full before,
1021                  * and since scan_swap_map() can drop the si->lock, multiple
1022                  * callers probably all tried to get a page from the same si
1023                  * and it filled up before we could get one; or, the si filled
1024                  * up between us dropping swap_avail_lock and taking si->lock.
1025                  * Since we dropped the swap_avail_lock, the swap_avail_head
1026                  * list may have been modified; so if next is still in the
1027                  * swap_avail_head list then try it, otherwise start over
1028                  * if we have not gotten any slots.
1029                  */
1030                 if (plist_node_empty(&next->avail_lists[node]))
1031                         goto start_over;
1032         }
1033
1034         spin_unlock(&swap_avail_lock);
1035
1036 check_out:
1037         if (n_ret < n_goal)
1038                 atomic_long_add((long)(n_goal - n_ret) * size,
1039                                 &nr_swap_pages);
1040 noswap:
1041         return n_ret;
1042 }
1043
1044 /* The only caller of this function is now suspend routine */
1045 swp_entry_t get_swap_page_of_type(int type)
1046 {
1047         struct swap_info_struct *si;
1048         pgoff_t offset;
1049
1050         si = swap_info[type];
1051         spin_lock(&si->lock);
1052         if (si && (si->flags & SWP_WRITEOK)) {
1053                 atomic_long_dec(&nr_swap_pages);
1054                 /* This is called for allocating swap entry, not cache */
1055                 offset = scan_swap_map(si, 1);
1056                 if (offset) {
1057                         spin_unlock(&si->lock);
1058                         return swp_entry(type, offset);
1059                 }
1060                 atomic_long_inc(&nr_swap_pages);
1061         }
1062         spin_unlock(&si->lock);
1063         return (swp_entry_t) {0};
1064 }
1065
1066 static struct swap_info_struct *__swap_info_get(swp_entry_t entry)
1067 {
1068         struct swap_info_struct *p;
1069         unsigned long offset, type;
1070
1071         if (!entry.val)
1072                 goto out;
1073         type = swp_type(entry);
1074         if (type >= nr_swapfiles)
1075                 goto bad_nofile;
1076         p = swap_info[type];
1077         if (!(p->flags & SWP_USED))
1078                 goto bad_device;
1079         offset = swp_offset(entry);
1080         if (offset >= p->max)
1081                 goto bad_offset;
1082         return p;
1083
1084 bad_offset:
1085         pr_err("swap_info_get: %s%08lx\n", Bad_offset, entry.val);
1086         goto out;
1087 bad_device:
1088         pr_err("swap_info_get: %s%08lx\n", Unused_file, entry.val);
1089         goto out;
1090 bad_nofile:
1091         pr_err("swap_info_get: %s%08lx\n", Bad_file, entry.val);
1092 out:
1093         return NULL;
1094 }
1095
1096 static struct swap_info_struct *_swap_info_get(swp_entry_t entry)
1097 {
1098         struct swap_info_struct *p;
1099
1100         p = __swap_info_get(entry);
1101         if (!p)
1102                 goto out;
1103         if (!p->swap_map[swp_offset(entry)])
1104                 goto bad_free;
1105         return p;
1106
1107 bad_free:
1108         pr_err("swap_info_get: %s%08lx\n", Unused_offset, entry.val);
1109         goto out;
1110 out:
1111         return NULL;
1112 }
1113
1114 static struct swap_info_struct *swap_info_get(swp_entry_t entry)
1115 {
1116         struct swap_info_struct *p;
1117
1118         p = _swap_info_get(entry);
1119         if (p)
1120                 spin_lock(&p->lock);
1121         return p;
1122 }
1123
1124 static struct swap_info_struct *swap_info_get_cont(swp_entry_t entry,
1125                                         struct swap_info_struct *q)
1126 {
1127         struct swap_info_struct *p;
1128
1129         p = _swap_info_get(entry);
1130
1131         if (p != q) {
1132                 if (q != NULL)
1133                         spin_unlock(&q->lock);
1134                 if (p != NULL)
1135                         spin_lock(&p->lock);
1136         }
1137         return p;
1138 }
1139
1140 static unsigned char __swap_entry_free_locked(struct swap_info_struct *p,
1141                                               unsigned long offset,
1142                                               unsigned char usage)
1143 {
1144         unsigned char count;
1145         unsigned char has_cache;
1146
1147         count = p->swap_map[offset];
1148
1149         has_cache = count & SWAP_HAS_CACHE;
1150         count &= ~SWAP_HAS_CACHE;
1151
1152         if (usage == SWAP_HAS_CACHE) {
1153                 VM_BUG_ON(!has_cache);
1154                 has_cache = 0;
1155         } else if (count == SWAP_MAP_SHMEM) {
1156                 /*
1157                  * Or we could insist on shmem.c using a special
1158                  * swap_shmem_free() and free_shmem_swap_and_cache()...
1159                  */
1160                 count = 0;
1161         } else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
1162                 if (count == COUNT_CONTINUED) {
1163                         if (swap_count_continued(p, offset, count))
1164                                 count = SWAP_MAP_MAX | COUNT_CONTINUED;
1165                         else
1166                                 count = SWAP_MAP_MAX;
1167                 } else
1168                         count--;
1169         }
1170
1171         usage = count | has_cache;
1172         p->swap_map[offset] = usage ? : SWAP_HAS_CACHE;
1173
1174         return usage;
1175 }
1176
1177 static unsigned char __swap_entry_free(struct swap_info_struct *p,
1178                                        swp_entry_t entry, unsigned char usage)
1179 {
1180         struct swap_cluster_info *ci;
1181         unsigned long offset = swp_offset(entry);
1182
1183         ci = lock_cluster_or_swap_info(p, offset);
1184         usage = __swap_entry_free_locked(p, offset, usage);
1185         unlock_cluster_or_swap_info(p, ci);
1186         if (!usage)
1187                 free_swap_slot(entry);
1188
1189         return usage;
1190 }
1191
1192 static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry)
1193 {
1194         struct swap_cluster_info *ci;
1195         unsigned long offset = swp_offset(entry);
1196         unsigned char count;
1197
1198         ci = lock_cluster(p, offset);
1199         count = p->swap_map[offset];
1200         VM_BUG_ON(count != SWAP_HAS_CACHE);
1201         p->swap_map[offset] = 0;
1202         dec_cluster_info_page(p, p->cluster_info, offset);
1203         unlock_cluster(ci);
1204
1205         mem_cgroup_uncharge_swap(entry, 1);
1206         swap_range_free(p, offset, 1);
1207 }
1208
1209 /*
1210  * Caller has made sure that the swap device corresponding to entry
1211  * is still around or has not been recycled.
1212  */
1213 void swap_free(swp_entry_t entry)
1214 {
1215         struct swap_info_struct *p;
1216
1217         p = _swap_info_get(entry);
1218         if (p)
1219                 __swap_entry_free(p, entry, 1);
1220 }
1221
1222 /*
1223  * Called after dropping swapcache to decrease refcnt to swap entries.
1224  */
1225 void put_swap_page(struct page *page, swp_entry_t entry)
1226 {
1227         unsigned long offset = swp_offset(entry);
1228         unsigned long idx = offset / SWAPFILE_CLUSTER;
1229         struct swap_cluster_info *ci;
1230         struct swap_info_struct *si;
1231         unsigned char *map;
1232         unsigned int i, free_entries = 0;
1233         unsigned char val;
1234         int size = swap_entry_size(hpage_nr_pages(page));
1235
1236         si = _swap_info_get(entry);
1237         if (!si)
1238                 return;
1239
1240         ci = lock_cluster_or_swap_info(si, offset);
1241         if (size == SWAPFILE_CLUSTER) {
1242                 VM_BUG_ON(!cluster_is_huge(ci));
1243                 map = si->swap_map + offset;
1244                 for (i = 0; i < SWAPFILE_CLUSTER; i++) {
1245                         val = map[i];
1246                         VM_BUG_ON(!(val & SWAP_HAS_CACHE));
1247                         if (val == SWAP_HAS_CACHE)
1248                                 free_entries++;
1249                 }
1250                 cluster_clear_huge(ci);
1251                 if (free_entries == SWAPFILE_CLUSTER) {
1252                         unlock_cluster_or_swap_info(si, ci);
1253                         spin_lock(&si->lock);
1254                         mem_cgroup_uncharge_swap(entry, SWAPFILE_CLUSTER);
1255                         swap_free_cluster(si, idx);
1256                         spin_unlock(&si->lock);
1257                         return;
1258                 }
1259         }
1260         for (i = 0; i < size; i++, entry.val++) {
1261                 if (!__swap_entry_free_locked(si, offset + i, SWAP_HAS_CACHE)) {
1262                         unlock_cluster_or_swap_info(si, ci);
1263                         free_swap_slot(entry);
1264                         if (i == size - 1)
1265                                 return;
1266                         lock_cluster_or_swap_info(si, offset);
1267                 }
1268         }
1269         unlock_cluster_or_swap_info(si, ci);
1270 }
1271
1272 #ifdef CONFIG_THP_SWAP
1273 int split_swap_cluster(swp_entry_t entry)
1274 {
1275         struct swap_info_struct *si;
1276         struct swap_cluster_info *ci;
1277         unsigned long offset = swp_offset(entry);
1278
1279         si = _swap_info_get(entry);
1280         if (!si)
1281                 return -EBUSY;
1282         ci = lock_cluster(si, offset);
1283         cluster_clear_huge(ci);
1284         unlock_cluster(ci);
1285         return 0;
1286 }
1287 #endif
1288
1289 static int swp_entry_cmp(const void *ent1, const void *ent2)
1290 {
1291         const swp_entry_t *e1 = ent1, *e2 = ent2;
1292
1293         return (int)swp_type(*e1) - (int)swp_type(*e2);
1294 }
1295
1296 void swapcache_free_entries(swp_entry_t *entries, int n)
1297 {
1298         struct swap_info_struct *p, *prev;
1299         int i;
1300
1301         if (n <= 0)
1302                 return;
1303
1304         prev = NULL;
1305         p = NULL;
1306
1307         /*
1308          * Sort swap entries by swap device, so each lock is only taken once.
1309          * nr_swapfiles isn't absolutely correct, but the overhead of sort() is
1310          * so low that it isn't necessary to optimize further.
1311          */
1312         if (nr_swapfiles > 1)
1313                 sort(entries, n, sizeof(entries[0]), swp_entry_cmp, NULL);
1314         for (i = 0; i < n; ++i) {
1315                 p = swap_info_get_cont(entries[i], prev);
1316                 if (p)
1317                         swap_entry_free(p, entries[i]);
1318                 prev = p;
1319         }
1320         if (p)
1321                 spin_unlock(&p->lock);
1322 }
1323
1324 /*
1325  * How many references to page are currently swapped out?
1326  * This does not give an exact answer when swap count is continued,
1327  * but does include the high COUNT_CONTINUED flag to allow for that.
1328  */
1329 int page_swapcount(struct page *page)
1330 {
1331         int count = 0;
1332         struct swap_info_struct *p;
1333         struct swap_cluster_info *ci;
1334         swp_entry_t entry;
1335         unsigned long offset;
1336
1337         entry.val = page_private(page);
1338         p = _swap_info_get(entry);
1339         if (p) {
1340                 offset = swp_offset(entry);
1341                 ci = lock_cluster_or_swap_info(p, offset);
1342                 count = swap_count(p->swap_map[offset]);
1343                 unlock_cluster_or_swap_info(p, ci);
1344         }
1345         return count;
1346 }
1347
1348 int __swap_count(struct swap_info_struct *si, swp_entry_t entry)
1349 {
1350         pgoff_t offset = swp_offset(entry);
1351
1352         return swap_count(si->swap_map[offset]);
1353 }
1354
1355 static int swap_swapcount(struct swap_info_struct *si, swp_entry_t entry)
1356 {
1357         int count = 0;
1358         pgoff_t offset = swp_offset(entry);
1359         struct swap_cluster_info *ci;
1360
1361         ci = lock_cluster_or_swap_info(si, offset);
1362         count = swap_count(si->swap_map[offset]);
1363         unlock_cluster_or_swap_info(si, ci);
1364         return count;
1365 }
1366
1367 /*
1368  * How many references to @entry are currently swapped out?
1369  * This does not give an exact answer when swap count is continued,
1370  * but does include the high COUNT_CONTINUED flag to allow for that.
1371  */
1372 int __swp_swapcount(swp_entry_t entry)
1373 {
1374         int count = 0;
1375         struct swap_info_struct *si;
1376
1377         si = __swap_info_get(entry);
1378         if (si)
1379                 count = swap_swapcount(si, entry);
1380         return count;
1381 }
1382
1383 /*
1384  * How many references to @entry are currently swapped out?
1385  * This considers COUNT_CONTINUED so it returns exact answer.
1386  */
1387 int swp_swapcount(swp_entry_t entry)
1388 {
1389         int count, tmp_count, n;
1390         struct swap_info_struct *p;
1391         struct swap_cluster_info *ci;
1392         struct page *page;
1393         pgoff_t offset;
1394         unsigned char *map;
1395
1396         p = _swap_info_get(entry);
1397         if (!p)
1398                 return 0;
1399
1400         offset = swp_offset(entry);
1401
1402         ci = lock_cluster_or_swap_info(p, offset);
1403
1404         count = swap_count(p->swap_map[offset]);
1405         if (!(count & COUNT_CONTINUED))
1406                 goto out;
1407
1408         count &= ~COUNT_CONTINUED;
1409         n = SWAP_MAP_MAX + 1;
1410
1411         page = vmalloc_to_page(p->swap_map + offset);
1412         offset &= ~PAGE_MASK;
1413         VM_BUG_ON(page_private(page) != SWP_CONTINUED);
1414
1415         do {
1416                 page = list_next_entry(page, lru);
1417                 map = kmap_atomic(page);
1418                 tmp_count = map[offset];
1419                 kunmap_atomic(map);
1420
1421                 count += (tmp_count & ~COUNT_CONTINUED) * n;
1422                 n *= (SWAP_CONT_MAX + 1);
1423         } while (tmp_count & COUNT_CONTINUED);
1424 out:
1425         unlock_cluster_or_swap_info(p, ci);
1426         return count;
1427 }
1428
1429 static bool swap_page_trans_huge_swapped(struct swap_info_struct *si,
1430                                          swp_entry_t entry)
1431 {
1432         struct swap_cluster_info *ci;
1433         unsigned char *map = si->swap_map;
1434         unsigned long roffset = swp_offset(entry);
1435         unsigned long offset = round_down(roffset, SWAPFILE_CLUSTER);
1436         int i;
1437         bool ret = false;
1438
1439         ci = lock_cluster_or_swap_info(si, offset);
1440         if (!ci || !cluster_is_huge(ci)) {
1441                 if (swap_count(map[roffset]))
1442                         ret = true;
1443                 goto unlock_out;
1444         }
1445         for (i = 0; i < SWAPFILE_CLUSTER; i++) {
1446                 if (swap_count(map[offset + i])) {
1447                         ret = true;
1448                         break;
1449                 }
1450         }
1451 unlock_out:
1452         unlock_cluster_or_swap_info(si, ci);
1453         return ret;
1454 }
1455
1456 static bool page_swapped(struct page *page)
1457 {
1458         swp_entry_t entry;
1459         struct swap_info_struct *si;
1460
1461         if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page)))
1462                 return page_swapcount(page) != 0;
1463
1464         page = compound_head(page);
1465         entry.val = page_private(page);
1466         si = _swap_info_get(entry);
1467         if (si)
1468                 return swap_page_trans_huge_swapped(si, entry);
1469         return false;
1470 }
1471
1472 static int page_trans_huge_map_swapcount(struct page *page, int *total_mapcount,
1473                                          int *total_swapcount)
1474 {
1475         int i, map_swapcount, _total_mapcount, _total_swapcount;
1476         unsigned long offset = 0;
1477         struct swap_info_struct *si;
1478         struct swap_cluster_info *ci = NULL;
1479         unsigned char *map = NULL;
1480         int mapcount, swapcount = 0;
1481
1482         /* hugetlbfs shouldn't call it */
1483         VM_BUG_ON_PAGE(PageHuge(page), page);
1484
1485         if (!IS_ENABLED(CONFIG_THP_SWAP) || likely(!PageTransCompound(page))) {
1486                 mapcount = page_trans_huge_mapcount(page, total_mapcount);
1487                 if (PageSwapCache(page))
1488                         swapcount = page_swapcount(page);
1489                 if (total_swapcount)
1490                         *total_swapcount = swapcount;
1491                 return mapcount + swapcount;
1492         }
1493
1494         page = compound_head(page);
1495
1496         _total_mapcount = _total_swapcount = map_swapcount = 0;
1497         if (PageSwapCache(page)) {
1498                 swp_entry_t entry;
1499
1500                 entry.val = page_private(page);
1501                 si = _swap_info_get(entry);
1502                 if (si) {
1503                         map = si->swap_map;
1504                         offset = swp_offset(entry);
1505                 }
1506         }
1507         if (map)
1508                 ci = lock_cluster(si, offset);
1509         for (i = 0; i < HPAGE_PMD_NR; i++) {
1510                 mapcount = atomic_read(&page[i]._mapcount) + 1;
1511                 _total_mapcount += mapcount;
1512                 if (map) {
1513                         swapcount = swap_count(map[offset + i]);
1514                         _total_swapcount += swapcount;
1515                 }
1516                 map_swapcount = max(map_swapcount, mapcount + swapcount);
1517         }
1518         unlock_cluster(ci);
1519         if (PageDoubleMap(page)) {
1520                 map_swapcount -= 1;
1521                 _total_mapcount -= HPAGE_PMD_NR;
1522         }
1523         mapcount = compound_mapcount(page);
1524         map_swapcount += mapcount;
1525         _total_mapcount += mapcount;
1526         if (total_mapcount)
1527                 *total_mapcount = _total_mapcount;
1528         if (total_swapcount)
1529                 *total_swapcount = _total_swapcount;
1530
1531         return map_swapcount;
1532 }
1533
1534 /*
1535  * We can write to an anon page without COW if there are no other references
1536  * to it.  And as a side-effect, free up its swap: because the old content
1537  * on disk will never be read, and seeking back there to write new content
1538  * later would only waste time away from clustering.
1539  *
1540  * NOTE: total_map_swapcount should not be relied upon by the caller if
1541  * reuse_swap_page() returns false, but it may be always overwritten
1542  * (see the other implementation for CONFIG_SWAP=n).
1543  */
1544 bool reuse_swap_page(struct page *page, int *total_map_swapcount)
1545 {
1546         int count, total_mapcount, total_swapcount;
1547
1548         VM_BUG_ON_PAGE(!PageLocked(page), page);
1549         if (unlikely(PageKsm(page)))
1550                 return false;
1551         count = page_trans_huge_map_swapcount(page, &total_mapcount,
1552                                               &total_swapcount);
1553         if (total_map_swapcount)
1554                 *total_map_swapcount = total_mapcount + total_swapcount;
1555         if (count == 1 && PageSwapCache(page) &&
1556             (likely(!PageTransCompound(page)) ||
1557              /* The remaining swap count will be freed soon */
1558              total_swapcount == page_swapcount(page))) {
1559                 if (!PageWriteback(page)) {
1560                         page = compound_head(page);
1561                         delete_from_swap_cache(page);
1562                         SetPageDirty(page);
1563                 } else {
1564                         swp_entry_t entry;
1565                         struct swap_info_struct *p;
1566
1567                         entry.val = page_private(page);
1568                         p = swap_info_get(entry);
1569                         if (p->flags & SWP_STABLE_WRITES) {
1570                                 spin_unlock(&p->lock);
1571                                 return false;
1572                         }
1573                         spin_unlock(&p->lock);
1574                 }
1575         }
1576
1577         return count <= 1;
1578 }
1579
1580 /*
1581  * If swap is getting full, or if there are no more mappings of this page,
1582  * then try_to_free_swap is called to free its swap space.
1583  */
1584 int try_to_free_swap(struct page *page)
1585 {
1586         VM_BUG_ON_PAGE(!PageLocked(page), page);
1587
1588         if (!PageSwapCache(page))
1589                 return 0;
1590         if (PageWriteback(page))
1591                 return 0;
1592         if (page_swapped(page))
1593                 return 0;
1594
1595         /*
1596          * Once hibernation has begun to create its image of memory,
1597          * there's a danger that one of the calls to try_to_free_swap()
1598          * - most probably a call from __try_to_reclaim_swap() while
1599          * hibernation is allocating its own swap pages for the image,
1600          * but conceivably even a call from memory reclaim - will free
1601          * the swap from a page which has already been recorded in the
1602          * image as a clean swapcache page, and then reuse its swap for
1603          * another page of the image.  On waking from hibernation, the
1604          * original page might be freed under memory pressure, then
1605          * later read back in from swap, now with the wrong data.
1606          *
1607          * Hibernation suspends storage while it is writing the image
1608          * to disk so check that here.
1609          */
1610         if (pm_suspended_storage())
1611                 return 0;
1612
1613         page = compound_head(page);
1614         delete_from_swap_cache(page);
1615         SetPageDirty(page);
1616         return 1;
1617 }
1618
1619 /*
1620  * Free the swap entry like above, but also try to
1621  * free the page cache entry if it is the last user.
1622  */
1623 int free_swap_and_cache(swp_entry_t entry)
1624 {
1625         struct swap_info_struct *p;
1626         unsigned char count;
1627
1628         if (non_swap_entry(entry))
1629                 return 1;
1630
1631         p = _swap_info_get(entry);
1632         if (p) {
1633                 count = __swap_entry_free(p, entry, 1);
1634                 if (count == SWAP_HAS_CACHE &&
1635                     !swap_page_trans_huge_swapped(p, entry))
1636                         __try_to_reclaim_swap(p, swp_offset(entry),
1637                                               TTRS_UNMAPPED | TTRS_FULL);
1638         }
1639         return p != NULL;
1640 }
1641
1642 #ifdef CONFIG_HIBERNATION
1643 /*
1644  * Find the swap type that corresponds to given device (if any).
1645  *
1646  * @offset - number of the PAGE_SIZE-sized block of the device, starting
1647  * from 0, in which the swap header is expected to be located.
1648  *
1649  * This is needed for the suspend to disk (aka swsusp).
1650  */
1651 int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
1652 {
1653         struct block_device *bdev = NULL;
1654         int type;
1655
1656         if (device)
1657                 bdev = bdget(device);
1658
1659         spin_lock(&swap_lock);
1660         for (type = 0; type < nr_swapfiles; type++) {
1661                 struct swap_info_struct *sis = swap_info[type];
1662
1663                 if (!(sis->flags & SWP_WRITEOK))
1664                         continue;
1665
1666                 if (!bdev) {
1667                         if (bdev_p)
1668                                 *bdev_p = bdgrab(sis->bdev);
1669
1670                         spin_unlock(&swap_lock);
1671                         return type;
1672                 }
1673                 if (bdev == sis->bdev) {
1674                         struct swap_extent *se = &sis->first_swap_extent;
1675
1676                         if (se->start_block == offset) {
1677                                 if (bdev_p)
1678                                         *bdev_p = bdgrab(sis->bdev);
1679
1680                                 spin_unlock(&swap_lock);
1681                                 bdput(bdev);
1682                                 return type;
1683                         }
1684                 }
1685         }
1686         spin_unlock(&swap_lock);
1687         if (bdev)
1688                 bdput(bdev);
1689
1690         return -ENODEV;
1691 }
1692
1693 /*
1694  * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
1695  * corresponding to given index in swap_info (swap type).
1696  */
1697 sector_t swapdev_block(int type, pgoff_t offset)
1698 {
1699         struct block_device *bdev;
1700
1701         if ((unsigned int)type >= nr_swapfiles)
1702                 return 0;
1703         if (!(swap_info[type]->flags & SWP_WRITEOK))
1704                 return 0;
1705         return map_swap_entry(swp_entry(type, offset), &bdev);
1706 }
1707
1708 /*
1709  * Return either the total number of swap pages of given type, or the number
1710  * of free pages of that type (depending on @free)
1711  *
1712  * This is needed for software suspend
1713  */
1714 unsigned int count_swap_pages(int type, int free)
1715 {
1716         unsigned int n = 0;
1717
1718         spin_lock(&swap_lock);
1719         if ((unsigned int)type < nr_swapfiles) {
1720                 struct swap_info_struct *sis = swap_info[type];
1721
1722                 spin_lock(&sis->lock);
1723                 if (sis->flags & SWP_WRITEOK) {
1724                         n = sis->pages;
1725                         if (free)
1726                                 n -= sis->inuse_pages;
1727                 }
1728                 spin_unlock(&sis->lock);
1729         }
1730         spin_unlock(&swap_lock);
1731         return n;
1732 }
1733 #endif /* CONFIG_HIBERNATION */
1734
1735 static inline int pte_same_as_swp(pte_t pte, pte_t swp_pte)
1736 {
1737         return pte_same(pte_swp_clear_soft_dirty(pte), swp_pte);
1738 }
1739
1740 /*
1741  * No need to decide whether this PTE shares the swap entry with others,
1742  * just let do_wp_page work it out if a write is requested later - to
1743  * force COW, vm_page_prot omits write permission from any private vma.
1744  */
1745 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
1746                 unsigned long addr, swp_entry_t entry, struct page *page)
1747 {
1748         struct page *swapcache;
1749         struct mem_cgroup *memcg;
1750         spinlock_t *ptl;
1751         pte_t *pte;
1752         int ret = 1;
1753
1754         swapcache = page;
1755         page = ksm_might_need_to_copy(page, vma, addr);
1756         if (unlikely(!page))
1757                 return -ENOMEM;
1758
1759         if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL,
1760                                 &memcg, false)) {
1761                 ret = -ENOMEM;
1762                 goto out_nolock;
1763         }
1764
1765         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1766         if (unlikely(!pte_same_as_swp(*pte, swp_entry_to_pte(entry)))) {
1767                 mem_cgroup_cancel_charge(page, memcg, false);
1768                 ret = 0;
1769                 goto out;
1770         }
1771
1772         dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1773         inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
1774         get_page(page);
1775         set_pte_at(vma->vm_mm, addr, pte,
1776                    pte_mkold(mk_pte(page, vma->vm_page_prot)));
1777         if (page == swapcache) {
1778                 page_add_anon_rmap(page, vma, addr, false);
1779                 mem_cgroup_commit_charge(page, memcg, true, false);
1780         } else { /* ksm created a completely new copy */
1781                 page_add_new_anon_rmap(page, vma, addr, false);
1782                 mem_cgroup_commit_charge(page, memcg, false, false);
1783                 lru_cache_add_active_or_unevictable(page, vma);
1784         }
1785         swap_free(entry);
1786         /*
1787          * Move the page to the active list so it is not
1788          * immediately swapped out again after swapon.
1789          */
1790         activate_page(page);
1791 out:
1792         pte_unmap_unlock(pte, ptl);
1793 out_nolock:
1794         if (page != swapcache) {
1795                 unlock_page(page);
1796                 put_page(page);
1797         }
1798         return ret;
1799 }
1800
1801 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1802                         unsigned long addr, unsigned long end,
1803                         unsigned int type, bool frontswap,
1804                         unsigned long *fs_pages_to_unuse)
1805 {
1806         struct page *page;
1807         swp_entry_t entry;
1808         pte_t *pte;
1809         struct swap_info_struct *si;
1810         unsigned long offset;
1811         int ret = 0;
1812         volatile unsigned char *swap_map;
1813
1814         si = swap_info[type];
1815         pte = pte_offset_map(pmd, addr);
1816         do {
1817                 struct vm_fault vmf;
1818
1819                 if (!is_swap_pte(*pte))
1820                         continue;
1821
1822                 entry = pte_to_swp_entry(*pte);
1823                 if (swp_type(entry) != type)
1824                         continue;
1825
1826                 offset = swp_offset(entry);
1827                 if (frontswap && !frontswap_test(si, offset))
1828                         continue;
1829
1830                 pte_unmap(pte);
1831                 swap_map = &si->swap_map[offset];
1832                 vmf.vma = vma;
1833                 vmf.address = addr;
1834                 vmf.pmd = pmd;
1835                 page = swapin_readahead(entry, GFP_HIGHUSER_MOVABLE, &vmf);
1836                 if (!page) {
1837                         if (*swap_map == 0 || *swap_map == SWAP_MAP_BAD)
1838                                 goto try_next;
1839                         return -ENOMEM;
1840                 }
1841
1842                 lock_page(page);
1843                 wait_on_page_writeback(page);
1844                 ret = unuse_pte(vma, pmd, addr, entry, page);
1845                 if (ret < 0) {
1846                         unlock_page(page);
1847                         put_page(page);
1848                         goto out;
1849                 }
1850
1851                 try_to_free_swap(page);
1852                 unlock_page(page);
1853                 put_page(page);
1854
1855                 if (*fs_pages_to_unuse && !--(*fs_pages_to_unuse)) {
1856                         ret = FRONTSWAP_PAGES_UNUSED;
1857                         goto out;
1858                 }
1859 try_next:
1860                 pte = pte_offset_map(pmd, addr);
1861         } while (pte++, addr += PAGE_SIZE, addr != end);
1862         pte_unmap(pte - 1);
1863
1864         ret = 0;
1865 out:
1866         return ret;
1867 }
1868
1869 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
1870                                 unsigned long addr, unsigned long end,
1871                                 unsigned int type, bool frontswap,
1872                                 unsigned long *fs_pages_to_unuse)
1873 {
1874         pmd_t *pmd;
1875         unsigned long next;
1876         int ret;
1877
1878         pmd = pmd_offset(pud, addr);
1879         do {
1880                 cond_resched();
1881                 next = pmd_addr_end(addr, end);
1882                 if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1883                         continue;
1884                 ret = unuse_pte_range(vma, pmd, addr, next, type,
1885                                       frontswap, fs_pages_to_unuse);
1886                 if (ret)
1887                         return ret;
1888         } while (pmd++, addr = next, addr != end);
1889         return 0;
1890 }
1891
1892 static inline int unuse_pud_range(struct vm_area_struct *vma, p4d_t *p4d,
1893                                 unsigned long addr, unsigned long end,
1894                                 unsigned int type, bool frontswap,
1895                                 unsigned long *fs_pages_to_unuse)
1896 {
1897         pud_t *pud;
1898         unsigned long next;
1899         int ret;
1900
1901         pud = pud_offset(p4d, addr);
1902         do {
1903                 next = pud_addr_end(addr, end);
1904                 if (pud_none_or_clear_bad(pud))
1905                         continue;
1906                 ret = unuse_pmd_range(vma, pud, addr, next, type,
1907                                       frontswap, fs_pages_to_unuse);
1908                 if (ret)
1909                         return ret;
1910         } while (pud++, addr = next, addr != end);
1911         return 0;
1912 }
1913
1914 static inline int unuse_p4d_range(struct vm_area_struct *vma, pgd_t *pgd,
1915                                 unsigned long addr, unsigned long end,
1916                                 unsigned int type, bool frontswap,
1917                                 unsigned long *fs_pages_to_unuse)
1918 {
1919         p4d_t *p4d;
1920         unsigned long next;
1921         int ret;
1922
1923         p4d = p4d_offset(pgd, addr);
1924         do {
1925                 next = p4d_addr_end(addr, end);
1926                 if (p4d_none_or_clear_bad(p4d))
1927                         continue;
1928                 ret = unuse_pud_range(vma, p4d, addr, next, type,
1929                                       frontswap, fs_pages_to_unuse);
1930                 if (ret)
1931                         return ret;
1932         } while (p4d++, addr = next, addr != end);
1933         return 0;
1934 }
1935
1936 static int unuse_vma(struct vm_area_struct *vma, unsigned int type,
1937                      bool frontswap, unsigned long *fs_pages_to_unuse)
1938 {
1939         pgd_t *pgd;
1940         unsigned long addr, end, next;
1941         int ret;
1942
1943         addr = vma->vm_start;
1944         end = vma->vm_end;
1945
1946         pgd = pgd_offset(vma->vm_mm, addr);
1947         do {
1948                 next = pgd_addr_end(addr, end);
1949                 if (pgd_none_or_clear_bad(pgd))
1950                         continue;
1951                 ret = unuse_p4d_range(vma, pgd, addr, next, type,
1952                                       frontswap, fs_pages_to_unuse);
1953                 if (ret)
1954                         return ret;
1955         } while (pgd++, addr = next, addr != end);
1956         return 0;
1957 }
1958
1959 static int unuse_mm(struct mm_struct *mm, unsigned int type,
1960                     bool frontswap, unsigned long *fs_pages_to_unuse)
1961 {
1962         struct vm_area_struct *vma;
1963         int ret = 0;
1964
1965         down_read(&mm->mmap_sem);
1966         for (vma = mm->mmap; vma; vma = vma->vm_next) {
1967                 if (vma->anon_vma) {
1968                         ret = unuse_vma(vma, type, frontswap,
1969                                         fs_pages_to_unuse);
1970                         if (ret)
1971                                 break;
1972                 }
1973                 cond_resched();
1974         }
1975         up_read(&mm->mmap_sem);
1976         return ret;
1977 }
1978
1979 /*
1980  * Scan swap_map (or frontswap_map if frontswap parameter is true)
1981  * from current position to next entry still in use. Return 0
1982  * if there are no inuse entries after prev till end of the map.
1983  */
1984 static unsigned int find_next_to_unuse(struct swap_info_struct *si,
1985                                         unsigned int prev, bool frontswap)
1986 {
1987         unsigned int i;
1988         unsigned char count;
1989
1990         /*
1991          * No need for swap_lock here: we're just looking
1992          * for whether an entry is in use, not modifying it; false
1993          * hits are okay, and sys_swapoff() has already prevented new
1994          * allocations from this area (while holding swap_lock).
1995          */
1996         for (i = prev + 1; i < si->max; i++) {
1997                 count = READ_ONCE(si->swap_map[i]);
1998                 if (count && swap_count(count) != SWAP_MAP_BAD)
1999                         if (!frontswap || frontswap_test(si, i))
2000                                 break;
2001                 if ((i % LATENCY_LIMIT) == 0)
2002                         cond_resched();
2003         }
2004
2005         if (i == si->max)
2006                 i = 0;
2007
2008         return i;
2009 }
2010
2011 /*
2012  * If the boolean frontswap is true, only unuse pages_to_unuse pages;
2013  * pages_to_unuse==0 means all pages; ignored if frontswap is false
2014  */
2015 #define SWAP_UNUSE_MAX_TRIES 3
2016 int try_to_unuse(unsigned int type, bool frontswap,
2017                  unsigned long pages_to_unuse)
2018 {
2019         struct mm_struct *prev_mm;
2020         struct mm_struct *mm;
2021         struct list_head *p;
2022         int retval = 0;
2023         struct swap_info_struct *si = swap_info[type];
2024         struct page *page;
2025         swp_entry_t entry;
2026         unsigned int i;
2027         int retries = 0;
2028
2029         if (!si->inuse_pages)
2030                 return 0;
2031
2032         if (!frontswap)
2033                 pages_to_unuse = 0;
2034
2035 retry:
2036         retval = shmem_unuse(type, frontswap, &pages_to_unuse);
2037         if (retval)
2038                 goto out;
2039
2040         prev_mm = &init_mm;
2041         mmget(prev_mm);
2042
2043         spin_lock(&mmlist_lock);
2044         p = &init_mm.mmlist;
2045         while ((p = p->next) != &init_mm.mmlist) {
2046                 if (signal_pending(current)) {
2047                         retval = -EINTR;
2048                         break;
2049                 }
2050
2051                 mm = list_entry(p, struct mm_struct, mmlist);
2052                 if (!mmget_not_zero(mm))
2053                         continue;
2054                 spin_unlock(&mmlist_lock);
2055                 mmput(prev_mm);
2056                 prev_mm = mm;
2057                 retval = unuse_mm(mm, type, frontswap, &pages_to_unuse);
2058
2059                 if (retval) {
2060                         mmput(prev_mm);
2061                         goto out;
2062                 }
2063
2064                 /*
2065                  * Make sure that we aren't completely killing
2066                  * interactive performance.
2067                  */
2068                 cond_resched();
2069                 spin_lock(&mmlist_lock);
2070         }
2071         spin_unlock(&mmlist_lock);
2072
2073         mmput(prev_mm);
2074
2075         i = 0;
2076         while ((i = find_next_to_unuse(si, i, frontswap)) != 0) {
2077
2078                 entry = swp_entry(type, i);
2079                 page = find_get_page(swap_address_space(entry), i);
2080                 if (!page)
2081                         continue;
2082
2083                 /*
2084                  * It is conceivable that a racing task removed this page from
2085                  * swap cache just before we acquired the page lock. The page
2086                  * might even be back in swap cache on another swap area. But
2087                  * that is okay, try_to_free_swap() only removes stale pages.
2088                  */
2089                 lock_page(page);
2090                 wait_on_page_writeback(page);
2091                 try_to_free_swap(page);
2092                 unlock_page(page);
2093                 put_page(page);
2094
2095                 /*
2096                  * For frontswap, we just need to unuse pages_to_unuse, if
2097                  * it was specified. Need not check frontswap again here as
2098                  * we already zeroed out pages_to_unuse if not frontswap.
2099                  */
2100                 if (pages_to_unuse && --pages_to_unuse == 0)
2101                         goto out;
2102         }
2103
2104         /*
2105          * Lets check again to see if there are still swap entries in the map.
2106          * If yes, we would need to do retry the unuse logic again.
2107          * Under global memory pressure, swap entries can be reinserted back
2108          * into process space after the mmlist loop above passes over them.
2109          * Its not worth continuosuly retrying to unuse the swap in this case.
2110          * So we try SWAP_UNUSE_MAX_TRIES times.
2111          */
2112         if (++retries >= SWAP_UNUSE_MAX_TRIES)
2113                 retval = -EBUSY;
2114         else if (si->inuse_pages)
2115                 goto retry;
2116
2117 out:
2118         return (retval == FRONTSWAP_PAGES_UNUSED) ? 0 : retval;
2119 }
2120
2121 /*
2122  * After a successful try_to_unuse, if no swap is now in use, we know
2123  * we can empty the mmlist.  swap_lock must be held on entry and exit.
2124  * Note that mmlist_lock nests inside swap_lock, and an mm must be
2125  * added to the mmlist just after page_duplicate - before would be racy.
2126  */
2127 static void drain_mmlist(void)
2128 {
2129         struct list_head *p, *next;
2130         unsigned int type;
2131
2132         for (type = 0; type < nr_swapfiles; type++)
2133                 if (swap_info[type]->inuse_pages)
2134                         return;
2135         spin_lock(&mmlist_lock);
2136         list_for_each_safe(p, next, &init_mm.mmlist)
2137                 list_del_init(p);
2138         spin_unlock(&mmlist_lock);
2139 }
2140
2141 /*
2142  * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
2143  * corresponds to page offset for the specified swap entry.
2144  * Note that the type of this function is sector_t, but it returns page offset
2145  * into the bdev, not sector offset.
2146  */
2147 static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
2148 {
2149         struct swap_info_struct *sis;
2150         struct swap_extent *start_se;
2151         struct swap_extent *se;
2152         pgoff_t offset;
2153
2154         sis = swap_info[swp_type(entry)];
2155         *bdev = sis->bdev;
2156
2157         offset = swp_offset(entry);
2158         start_se = sis->curr_swap_extent;
2159         se = start_se;
2160
2161         for ( ; ; ) {
2162                 if (se->start_page <= offset &&
2163                                 offset < (se->start_page + se->nr_pages)) {
2164                         return se->start_block + (offset - se->start_page);
2165                 }
2166                 se = list_next_entry(se, list);
2167                 sis->curr_swap_extent = se;
2168                 BUG_ON(se == start_se);         /* It *must* be present */
2169         }
2170 }
2171
2172 /*
2173  * Returns the page offset into bdev for the specified page's swap entry.
2174  */
2175 sector_t map_swap_page(struct page *page, struct block_device **bdev)
2176 {
2177         swp_entry_t entry;
2178         entry.val = page_private(page);
2179         return map_swap_entry(entry, bdev);
2180 }
2181
2182 /*
2183  * Free all of a swapdev's extent information
2184  */
2185 static void destroy_swap_extents(struct swap_info_struct *sis)
2186 {
2187         while (!list_empty(&sis->first_swap_extent.list)) {
2188                 struct swap_extent *se;
2189
2190                 se = list_first_entry(&sis->first_swap_extent.list,
2191                                 struct swap_extent, list);
2192                 list_del(&se->list);
2193                 kfree(se);
2194         }
2195
2196         if (sis->flags & SWP_ACTIVATED) {
2197                 struct file *swap_file = sis->swap_file;
2198                 struct address_space *mapping = swap_file->f_mapping;
2199
2200                 sis->flags &= ~SWP_ACTIVATED;
2201                 if (mapping->a_ops->swap_deactivate)
2202                         mapping->a_ops->swap_deactivate(swap_file);
2203         }
2204 }
2205
2206 /*
2207  * Add a block range (and the corresponding page range) into this swapdev's
2208  * extent list.  The extent list is kept sorted in page order.
2209  *
2210  * This function rather assumes that it is called in ascending page order.
2211  */
2212 int
2213 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
2214                 unsigned long nr_pages, sector_t start_block)
2215 {
2216         struct swap_extent *se;
2217         struct swap_extent *new_se;
2218         struct list_head *lh;
2219
2220         if (start_page == 0) {
2221                 se = &sis->first_swap_extent;
2222                 sis->curr_swap_extent = se;
2223                 se->start_page = 0;
2224                 se->nr_pages = nr_pages;
2225                 se->start_block = start_block;
2226                 return 1;
2227         } else {
2228                 lh = sis->first_swap_extent.list.prev;  /* Highest extent */
2229                 se = list_entry(lh, struct swap_extent, list);
2230                 BUG_ON(se->start_page + se->nr_pages != start_page);
2231                 if (se->start_block + se->nr_pages == start_block) {
2232                         /* Merge it */
2233                         se->nr_pages += nr_pages;
2234                         return 0;
2235                 }
2236         }
2237
2238         /*
2239          * No merge.  Insert a new extent, preserving ordering.
2240          */
2241         new_se = kmalloc(sizeof(*se), GFP_KERNEL);
2242         if (new_se == NULL)
2243                 return -ENOMEM;
2244         new_se->start_page = start_page;
2245         new_se->nr_pages = nr_pages;
2246         new_se->start_block = start_block;
2247
2248         list_add_tail(&new_se->list, &sis->first_swap_extent.list);
2249         return 1;
2250 }
2251 EXPORT_SYMBOL_GPL(add_swap_extent);
2252
2253 /*
2254  * A `swap extent' is a simple thing which maps a contiguous range of pages
2255  * onto a contiguous range of disk blocks.  An ordered list of swap extents
2256  * is built at swapon time and is then used at swap_writepage/swap_readpage
2257  * time for locating where on disk a page belongs.
2258  *
2259  * If the swapfile is an S_ISBLK block device, a single extent is installed.
2260  * This is done so that the main operating code can treat S_ISBLK and S_ISREG
2261  * swap files identically.
2262  *
2263  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
2264  * extent list operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
2265  * swapfiles are handled *identically* after swapon time.
2266  *
2267  * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
2268  * and will parse them into an ordered extent list, in PAGE_SIZE chunks.  If
2269  * some stray blocks are found which do not fall within the PAGE_SIZE alignment
2270  * requirements, they are simply tossed out - we will never use those blocks
2271  * for swapping.
2272  *
2273  * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon.  This
2274  * prevents root from shooting her foot off by ftruncating an in-use swapfile,
2275  * which will scribble on the fs.
2276  *
2277  * The amount of disk space which a single swap extent represents varies.
2278  * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
2279  * extents in the list.  To avoid much list walking, we cache the previous
2280  * search location in `curr_swap_extent', and start new searches from there.
2281  * This is extremely effective.  The average number of iterations in
2282  * map_swap_page() has been measured at about 0.3 per page.  - akpm.
2283  */
2284 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
2285 {
2286         struct file *swap_file = sis->swap_file;
2287         struct address_space *mapping = swap_file->f_mapping;
2288         struct inode *inode = mapping->host;
2289         int ret;
2290
2291         if (S_ISBLK(inode->i_mode)) {
2292                 ret = add_swap_extent(sis, 0, sis->max, 0);
2293                 *span = sis->pages;
2294                 return ret;
2295         }
2296
2297         if (mapping->a_ops->swap_activate) {
2298                 ret = mapping->a_ops->swap_activate(sis, swap_file, span);
2299                 if (ret >= 0)
2300                         sis->flags |= SWP_ACTIVATED;
2301                 if (!ret) {
2302                         sis->flags |= SWP_FS;
2303                         ret = add_swap_extent(sis, 0, sis->max, 0);
2304                         *span = sis->pages;
2305                 }
2306                 return ret;
2307         }
2308
2309         return generic_swapfile_activate(sis, swap_file, span);
2310 }
2311
2312 static int swap_node(struct swap_info_struct *p)
2313 {
2314         struct block_device *bdev;
2315
2316         if (p->bdev)
2317                 bdev = p->bdev;
2318         else
2319                 bdev = p->swap_file->f_inode->i_sb->s_bdev;
2320
2321         return bdev ? bdev->bd_disk->node_id : NUMA_NO_NODE;
2322 }
2323
2324 static void _enable_swap_info(struct swap_info_struct *p, int prio,
2325                                 unsigned char *swap_map,
2326                                 struct swap_cluster_info *cluster_info)
2327 {
2328         int i;
2329
2330         if (prio >= 0)
2331                 p->prio = prio;
2332         else
2333                 p->prio = --least_priority;
2334         /*
2335          * the plist prio is negated because plist ordering is
2336          * low-to-high, while swap ordering is high-to-low
2337          */
2338         p->list.prio = -p->prio;
2339         for_each_node(i) {
2340                 if (p->prio >= 0)
2341                         p->avail_lists[i].prio = -p->prio;
2342                 else {
2343                         if (swap_node(p) == i)
2344                                 p->avail_lists[i].prio = 1;
2345                         else
2346                                 p->avail_lists[i].prio = -p->prio;
2347                 }
2348         }
2349         p->swap_map = swap_map;
2350         p->cluster_info = cluster_info;
2351         p->flags |= SWP_WRITEOK;
2352         atomic_long_add(p->pages, &nr_swap_pages);
2353         total_swap_pages += p->pages;
2354
2355         assert_spin_locked(&swap_lock);
2356         /*
2357          * both lists are plists, and thus priority ordered.
2358          * swap_active_head needs to be priority ordered for swapoff(),
2359          * which on removal of any swap_info_struct with an auto-assigned
2360          * (i.e. negative) priority increments the auto-assigned priority
2361          * of any lower-priority swap_info_structs.
2362          * swap_avail_head needs to be priority ordered for get_swap_page(),
2363          * which allocates swap pages from the highest available priority
2364          * swap_info_struct.
2365          */
2366         plist_add(&p->list, &swap_active_head);
2367         add_to_avail_list(p);
2368 }
2369
2370 static void enable_swap_info(struct swap_info_struct *p, int prio,
2371                                 unsigned char *swap_map,
2372                                 struct swap_cluster_info *cluster_info,
2373                                 unsigned long *frontswap_map)
2374 {
2375         frontswap_init(p->type, frontswap_map);
2376         spin_lock(&swap_lock);
2377         spin_lock(&p->lock);
2378          _enable_swap_info(p, prio, swap_map, cluster_info);
2379         spin_unlock(&p->lock);
2380         spin_unlock(&swap_lock);
2381 }
2382
2383 static void reinsert_swap_info(struct swap_info_struct *p)
2384 {
2385         spin_lock(&swap_lock);
2386         spin_lock(&p->lock);
2387         _enable_swap_info(p, p->prio, p->swap_map, p->cluster_info);
2388         spin_unlock(&p->lock);
2389         spin_unlock(&swap_lock);
2390 }
2391
2392 bool has_usable_swap(void)
2393 {
2394         bool ret = true;
2395
2396         spin_lock(&swap_lock);
2397         if (plist_head_empty(&swap_active_head))
2398                 ret = false;
2399         spin_unlock(&swap_lock);
2400         return ret;
2401 }
2402
2403 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
2404 {
2405         struct swap_info_struct *p = NULL;
2406         unsigned char *swap_map;
2407         struct swap_cluster_info *cluster_info;
2408         unsigned long *frontswap_map;
2409         struct file *swap_file, *victim;
2410         struct address_space *mapping;
2411         struct inode *inode;
2412         struct filename *pathname;
2413         int err, found = 0;
2414         unsigned int old_block_size;
2415
2416         if (!capable(CAP_SYS_ADMIN))
2417                 return -EPERM;
2418
2419         BUG_ON(!current->mm);
2420
2421         pathname = getname(specialfile);
2422         if (IS_ERR(pathname))
2423                 return PTR_ERR(pathname);
2424
2425         victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
2426         err = PTR_ERR(victim);
2427         if (IS_ERR(victim))
2428                 goto out;
2429
2430         mapping = victim->f_mapping;
2431         spin_lock(&swap_lock);
2432         plist_for_each_entry(p, &swap_active_head, list) {
2433                 if (p->flags & SWP_WRITEOK) {
2434                         if (p->swap_file->f_mapping == mapping) {
2435                                 found = 1;
2436                                 break;
2437                         }
2438                 }
2439         }
2440         if (!found) {
2441                 err = -EINVAL;
2442                 spin_unlock(&swap_lock);
2443                 goto out_dput;
2444         }
2445         if (!security_vm_enough_memory_mm(current->mm, p->pages))
2446                 vm_unacct_memory(p->pages);
2447         else {
2448                 err = -ENOMEM;
2449                 spin_unlock(&swap_lock);
2450                 goto out_dput;
2451         }
2452         del_from_avail_list(p);
2453         spin_lock(&p->lock);
2454         if (p->prio < 0) {
2455                 struct swap_info_struct *si = p;
2456                 int nid;
2457
2458                 plist_for_each_entry_continue(si, &swap_active_head, list) {
2459                         si->prio++;
2460                         si->list.prio--;
2461                         for_each_node(nid) {
2462                                 if (si->avail_lists[nid].prio != 1)
2463                                         si->avail_lists[nid].prio--;
2464                         }
2465                 }
2466                 least_priority++;
2467         }
2468         plist_del(&p->list, &swap_active_head);
2469         atomic_long_sub(p->pages, &nr_swap_pages);
2470         total_swap_pages -= p->pages;
2471         p->flags &= ~SWP_WRITEOK;
2472         spin_unlock(&p->lock);
2473         spin_unlock(&swap_lock);
2474
2475         disable_swap_slots_cache_lock();
2476
2477         set_current_oom_origin();
2478         err = try_to_unuse(p->type, false, 0); /* force unuse all pages */
2479         clear_current_oom_origin();
2480
2481         if (err) {
2482                 /* re-insert swap space back into swap_list */
2483                 reinsert_swap_info(p);
2484                 reenable_swap_slots_cache_unlock();
2485                 goto out_dput;
2486         }
2487
2488         reenable_swap_slots_cache_unlock();
2489
2490         flush_work(&p->discard_work);
2491
2492         destroy_swap_extents(p);
2493         if (p->flags & SWP_CONTINUED)
2494                 free_swap_count_continuations(p);
2495
2496         if (!p->bdev || !blk_queue_nonrot(bdev_get_queue(p->bdev)))
2497                 atomic_dec(&nr_rotate_swap);
2498
2499         mutex_lock(&swapon_mutex);
2500         spin_lock(&swap_lock);
2501         spin_lock(&p->lock);
2502         drain_mmlist();
2503
2504         /* wait for anyone still in scan_swap_map */
2505         p->highest_bit = 0;             /* cuts scans short */
2506         while (p->flags >= SWP_SCANNING) {
2507                 spin_unlock(&p->lock);
2508                 spin_unlock(&swap_lock);
2509                 schedule_timeout_uninterruptible(1);
2510                 spin_lock(&swap_lock);
2511                 spin_lock(&p->lock);
2512         }
2513
2514         swap_file = p->swap_file;
2515         old_block_size = p->old_block_size;
2516         p->swap_file = NULL;
2517         p->max = 0;
2518         swap_map = p->swap_map;
2519         p->swap_map = NULL;
2520         cluster_info = p->cluster_info;
2521         p->cluster_info = NULL;
2522         frontswap_map = frontswap_map_get(p);
2523         spin_unlock(&p->lock);
2524         spin_unlock(&swap_lock);
2525         frontswap_invalidate_area(p->type);
2526         frontswap_map_set(p, NULL);
2527         mutex_unlock(&swapon_mutex);
2528         free_percpu(p->percpu_cluster);
2529         p->percpu_cluster = NULL;
2530         vfree(swap_map);
2531         kvfree(cluster_info);
2532         kvfree(frontswap_map);
2533         /* Destroy swap account information */
2534         swap_cgroup_swapoff(p->type);
2535         exit_swap_address_space(p->type);
2536
2537         inode = mapping->host;
2538         if (S_ISBLK(inode->i_mode)) {
2539                 struct block_device *bdev = I_BDEV(inode);
2540                 set_blocksize(bdev, old_block_size);
2541                 blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2542         } else {
2543                 inode_lock(inode);
2544                 inode->i_flags &= ~S_SWAPFILE;
2545                 inode_unlock(inode);
2546         }
2547         filp_close(swap_file, NULL);
2548
2549         /*
2550          * Clear the SWP_USED flag after all resources are freed so that swapon
2551          * can reuse this swap_info in alloc_swap_info() safely.  It is ok to
2552          * not hold p->lock after we cleared its SWP_WRITEOK.
2553          */
2554         spin_lock(&swap_lock);
2555         p->flags = 0;
2556         spin_unlock(&swap_lock);
2557
2558         err = 0;
2559         atomic_inc(&proc_poll_event);
2560         wake_up_interruptible(&proc_poll_wait);
2561
2562 out_dput:
2563         filp_close(victim, NULL);
2564 out:
2565         putname(pathname);
2566         return err;
2567 }
2568
2569 #ifdef CONFIG_PROC_FS
2570 static __poll_t swaps_poll(struct file *file, poll_table *wait)
2571 {
2572         struct seq_file *seq = file->private_data;
2573
2574         poll_wait(file, &proc_poll_wait, wait);
2575
2576         if (seq->poll_event != atomic_read(&proc_poll_event)) {
2577                 seq->poll_event = atomic_read(&proc_poll_event);
2578                 return EPOLLIN | EPOLLRDNORM | EPOLLERR | EPOLLPRI;
2579         }
2580
2581         return EPOLLIN | EPOLLRDNORM;
2582 }
2583
2584 /* iterator */
2585 static void *swap_start(struct seq_file *swap, loff_t *pos)
2586 {
2587         struct swap_info_struct *si;
2588         int type;
2589         loff_t l = *pos;
2590
2591         mutex_lock(&swapon_mutex);
2592
2593         if (!l)
2594                 return SEQ_START_TOKEN;
2595
2596         for (type = 0; type < nr_swapfiles; type++) {
2597                 smp_rmb();      /* read nr_swapfiles before swap_info[type] */
2598                 si = swap_info[type];
2599                 if (!(si->flags & SWP_USED) || !si->swap_map)
2600                         continue;
2601                 if (!--l)
2602                         return si;
2603         }
2604
2605         return NULL;
2606 }
2607
2608 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
2609 {
2610         struct swap_info_struct *si = v;
2611         int type;
2612
2613         if (v == SEQ_START_TOKEN)
2614                 type = 0;
2615         else
2616                 type = si->type + 1;
2617
2618         for (; type < nr_swapfiles; type++) {
2619                 smp_rmb();      /* read nr_swapfiles before swap_info[type] */
2620                 si = swap_info[type];
2621                 if (!(si->flags & SWP_USED) || !si->swap_map)
2622                         continue;
2623                 ++*pos;
2624                 return si;
2625         }
2626
2627         return NULL;
2628 }
2629
2630 static void swap_stop(struct seq_file *swap, void *v)
2631 {
2632         mutex_unlock(&swapon_mutex);
2633 }
2634
2635 static int swap_show(struct seq_file *swap, void *v)
2636 {
2637         struct swap_info_struct *si = v;
2638         struct file *file;
2639         int len;
2640
2641         if (si == SEQ_START_TOKEN) {
2642                 seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
2643                 return 0;
2644         }
2645
2646         file = si->swap_file;
2647         len = seq_file_path(swap, file, " \t\n\\");
2648         seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
2649                         len < 40 ? 40 - len : 1, " ",
2650                         S_ISBLK(file_inode(file)->i_mode) ?
2651                                 "partition" : "file\t",
2652                         si->pages << (PAGE_SHIFT - 10),
2653                         si->inuse_pages << (PAGE_SHIFT - 10),
2654                         si->prio);
2655         return 0;
2656 }
2657
2658 static const struct seq_operations swaps_op = {
2659         .start =        swap_start,
2660         .next =         swap_next,
2661         .stop =         swap_stop,
2662         .show =         swap_show
2663 };
2664
2665 static int swaps_open(struct inode *inode, struct file *file)
2666 {
2667         struct seq_file *seq;
2668         int ret;
2669
2670         ret = seq_open(file, &swaps_op);
2671         if (ret)
2672                 return ret;
2673
2674         seq = file->private_data;
2675         seq->poll_event = atomic_read(&proc_poll_event);
2676         return 0;
2677 }
2678
2679 static const struct file_operations proc_swaps_operations = {
2680         .open           = swaps_open,
2681         .read           = seq_read,
2682         .llseek         = seq_lseek,
2683         .release        = seq_release,
2684         .poll           = swaps_poll,
2685 };
2686
2687 static int __init procswaps_init(void)
2688 {
2689         proc_create("swaps", 0, NULL, &proc_swaps_operations);
2690         return 0;
2691 }
2692 __initcall(procswaps_init);
2693 #endif /* CONFIG_PROC_FS */
2694
2695 #ifdef MAX_SWAPFILES_CHECK
2696 static int __init max_swapfiles_check(void)
2697 {
2698         MAX_SWAPFILES_CHECK();
2699         return 0;
2700 }
2701 late_initcall(max_swapfiles_check);
2702 #endif
2703
2704 static struct swap_info_struct *alloc_swap_info(void)
2705 {
2706         struct swap_info_struct *p;
2707         unsigned int type;
2708         int i;
2709         int size = sizeof(*p) + nr_node_ids * sizeof(struct plist_node);
2710
2711         p = kvzalloc(size, GFP_KERNEL);
2712         if (!p)
2713                 return ERR_PTR(-ENOMEM);
2714
2715         spin_lock(&swap_lock);
2716         for (type = 0; type < nr_swapfiles; type++) {
2717                 if (!(swap_info[type]->flags & SWP_USED))
2718                         break;
2719         }
2720         if (type >= MAX_SWAPFILES) {
2721                 spin_unlock(&swap_lock);
2722                 kvfree(p);
2723                 return ERR_PTR(-EPERM);
2724         }
2725         if (type >= nr_swapfiles) {
2726                 p->type = type;
2727                 swap_info[type] = p;
2728                 /*
2729                  * Write swap_info[type] before nr_swapfiles, in case a
2730                  * racing procfs swap_start() or swap_next() is reading them.
2731                  * (We never shrink nr_swapfiles, we never free this entry.)
2732                  */
2733                 smp_wmb();
2734                 nr_swapfiles++;
2735         } else {
2736                 kvfree(p);
2737                 p = swap_info[type];
2738                 /*
2739                  * Do not memset this entry: a racing procfs swap_next()
2740                  * would be relying on p->type to remain valid.
2741                  */
2742         }
2743         INIT_LIST_HEAD(&p->first_swap_extent.list);
2744         plist_node_init(&p->list, 0);
2745         for_each_node(i)
2746                 plist_node_init(&p->avail_lists[i], 0);
2747         p->flags = SWP_USED;
2748         spin_unlock(&swap_lock);
2749         spin_lock_init(&p->lock);
2750         spin_lock_init(&p->cont_lock);
2751
2752         return p;
2753 }
2754
2755 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
2756 {
2757         int error;
2758
2759         if (S_ISBLK(inode->i_mode)) {
2760                 p->bdev = bdgrab(I_BDEV(inode));
2761                 error = blkdev_get(p->bdev,
2762                                    FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
2763                 if (error < 0) {
2764                         p->bdev = NULL;
2765                         return error;
2766                 }
2767                 p->old_block_size = block_size(p->bdev);
2768                 error = set_blocksize(p->bdev, PAGE_SIZE);
2769                 if (error < 0)
2770                         return error;
2771                 p->flags |= SWP_BLKDEV;
2772         } else if (S_ISREG(inode->i_mode)) {
2773                 p->bdev = inode->i_sb->s_bdev;
2774                 inode_lock(inode);
2775                 if (IS_SWAPFILE(inode))
2776                         return -EBUSY;
2777         } else
2778                 return -EINVAL;
2779
2780         return 0;
2781 }
2782
2783
2784 /*
2785  * Find out how many pages are allowed for a single swap device. There
2786  * are two limiting factors:
2787  * 1) the number of bits for the swap offset in the swp_entry_t type, and
2788  * 2) the number of bits in the swap pte, as defined by the different
2789  * architectures.
2790  *
2791  * In order to find the largest possible bit mask, a swap entry with
2792  * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
2793  * decoded to a swp_entry_t again, and finally the swap offset is
2794  * extracted.
2795  *
2796  * This will mask all the bits from the initial ~0UL mask that can't
2797  * be encoded in either the swp_entry_t or the architecture definition
2798  * of a swap pte.
2799  */
2800 unsigned long generic_max_swapfile_size(void)
2801 {
2802         return swp_offset(pte_to_swp_entry(
2803                         swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2804 }
2805
2806 /* Can be overridden by an architecture for additional checks. */
2807 __weak unsigned long max_swapfile_size(void)
2808 {
2809         return generic_max_swapfile_size();
2810 }
2811
2812 static unsigned long read_swap_header(struct swap_info_struct *p,
2813                                         union swap_header *swap_header,
2814                                         struct inode *inode)
2815 {
2816         int i;
2817         unsigned long maxpages;
2818         unsigned long swapfilepages;
2819         unsigned long last_page;
2820
2821         if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2822                 pr_err("Unable to find swap-space signature\n");
2823                 return 0;
2824         }
2825
2826         /* swap partition endianess hack... */
2827         if (swab32(swap_header->info.version) == 1) {
2828                 swab32s(&swap_header->info.version);
2829                 swab32s(&swap_header->info.last_page);
2830                 swab32s(&swap_header->info.nr_badpages);
2831                 if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2832                         return 0;
2833                 for (i = 0; i < swap_header->info.nr_badpages; i++)
2834                         swab32s(&swap_header->info.badpages[i]);
2835         }
2836         /* Check the swap header's sub-version */
2837         if (swap_header->info.version != 1) {
2838                 pr_warn("Unable to handle swap header version %d\n",
2839                         swap_header->info.version);
2840                 return 0;
2841         }
2842
2843         p->lowest_bit  = 1;
2844         p->cluster_next = 1;
2845         p->cluster_nr = 0;
2846
2847         maxpages = max_swapfile_size();
2848         last_page = swap_header->info.last_page;
2849         if (!last_page) {
2850                 pr_warn("Empty swap-file\n");
2851                 return 0;
2852         }
2853         if (last_page > maxpages) {
2854                 pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2855                         maxpages << (PAGE_SHIFT - 10),
2856                         last_page << (PAGE_SHIFT - 10));
2857         }
2858         if (maxpages > last_page) {
2859                 maxpages = last_page + 1;
2860                 /* p->max is an unsigned int: don't overflow it */
2861                 if ((unsigned int)maxpages == 0)
2862                         maxpages = UINT_MAX;
2863         }
2864         p->highest_bit = maxpages - 1;
2865
2866         if (!maxpages)
2867                 return 0;
2868         swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
2869         if (swapfilepages && maxpages > swapfilepages) {
2870                 pr_warn("Swap area shorter than signature indicates\n");
2871                 return 0;
2872         }
2873         if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
2874                 return 0;
2875         if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2876                 return 0;
2877
2878         return maxpages;
2879 }
2880
2881 #define SWAP_CLUSTER_INFO_COLS                                          \
2882         DIV_ROUND_UP(L1_CACHE_BYTES, sizeof(struct swap_cluster_info))
2883 #define SWAP_CLUSTER_SPACE_COLS                                         \
2884         DIV_ROUND_UP(SWAP_ADDRESS_SPACE_PAGES, SWAPFILE_CLUSTER)
2885 #define SWAP_CLUSTER_COLS                                               \
2886         max_t(unsigned int, SWAP_CLUSTER_INFO_COLS, SWAP_CLUSTER_SPACE_COLS)
2887
2888 static int setup_swap_map_and_extents(struct swap_info_struct *p,
2889                                         union swap_header *swap_header,
2890                                         unsigned char *swap_map,
2891                                         struct swap_cluster_info *cluster_info,
2892                                         unsigned long maxpages,
2893                                         sector_t *span)
2894 {
2895         unsigned int j, k;
2896         unsigned int nr_good_pages;
2897         int nr_extents;
2898         unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2899         unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS;
2900         unsigned long i, idx;
2901
2902         nr_good_pages = maxpages - 1;   /* omit header page */
2903
2904         cluster_list_init(&p->free_clusters);
2905         cluster_list_init(&p->discard_clusters);
2906
2907         for (i = 0; i < swap_header->info.nr_badpages; i++) {
2908                 unsigned int page_nr = swap_header->info.badpages[i];
2909                 if (page_nr == 0 || page_nr > swap_header->info.last_page)
2910                         return -EINVAL;
2911                 if (page_nr < maxpages) {
2912                         swap_map[page_nr] = SWAP_MAP_BAD;
2913                         nr_good_pages--;
2914                         /*
2915                          * Haven't marked the cluster free yet, no list
2916                          * operation involved
2917                          */
2918                         inc_cluster_info_page(p, cluster_info, page_nr);
2919                 }
2920         }
2921
2922         /* Haven't marked the cluster free yet, no list operation involved */
2923         for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
2924                 inc_cluster_info_page(p, cluster_info, i);
2925
2926         if (nr_good_pages) {
2927                 swap_map[0] = SWAP_MAP_BAD;
2928                 /*
2929                  * Not mark the cluster free yet, no list
2930                  * operation involved
2931                  */
2932                 inc_cluster_info_page(p, cluster_info, 0);
2933                 p->max = maxpages;
2934                 p->pages = nr_good_pages;
2935                 nr_extents = setup_swap_extents(p, span);
2936                 if (nr_extents < 0)
2937                         return nr_extents;
2938                 nr_good_pages = p->pages;
2939         }
2940         if (!nr_good_pages) {
2941                 pr_warn("Empty swap-file\n");
2942                 return -EINVAL;
2943         }
2944
2945         if (!cluster_info)
2946                 return nr_extents;
2947
2948
2949         /*
2950          * Reduce false cache line sharing between cluster_info and
2951          * sharing same address space.
2952          */
2953         for (k = 0; k < SWAP_CLUSTER_COLS; k++) {
2954                 j = (k + col) % SWAP_CLUSTER_COLS;
2955                 for (i = 0; i < DIV_ROUND_UP(nr_clusters, SWAP_CLUSTER_COLS); i++) {
2956                         idx = i * SWAP_CLUSTER_COLS + j;
2957                         if (idx >= nr_clusters)
2958                                 continue;
2959                         if (cluster_count(&cluster_info[idx]))
2960                                 continue;
2961                         cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
2962                         cluster_list_add_tail(&p->free_clusters, cluster_info,
2963                                               idx);
2964                 }
2965         }
2966         return nr_extents;
2967 }
2968
2969 /*
2970  * Helper to sys_swapon determining if a given swap
2971  * backing device queue supports DISCARD operations.
2972  */
2973 static bool swap_discardable(struct swap_info_struct *si)
2974 {
2975         struct request_queue *q = bdev_get_queue(si->bdev);
2976
2977         if (!q || !blk_queue_discard(q))
2978                 return false;
2979
2980         return true;
2981 }
2982
2983 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
2984 {
2985         struct swap_info_struct *p;
2986         struct filename *name;
2987         struct file *swap_file = NULL;
2988         struct address_space *mapping;
2989         int prio;
2990         int error;
2991         union swap_header *swap_header;
2992         int nr_extents;
2993         sector_t span;
2994         unsigned long maxpages;
2995         unsigned char *swap_map = NULL;
2996         struct swap_cluster_info *cluster_info = NULL;
2997         unsigned long *frontswap_map = NULL;
2998         struct page *page = NULL;
2999         struct inode *inode = NULL;
3000         bool inced_nr_rotate_swap = false;
3001
3002         if (swap_flags & ~SWAP_FLAGS_VALID)
3003                 return -EINVAL;
3004
3005         if (!capable(CAP_SYS_ADMIN))
3006                 return -EPERM;
3007
3008         if (!swap_avail_heads)
3009                 return -ENOMEM;
3010
3011         p = alloc_swap_info();
3012         if (IS_ERR(p))
3013                 return PTR_ERR(p);
3014
3015         INIT_WORK(&p->discard_work, swap_discard_work);
3016
3017         name = getname(specialfile);
3018         if (IS_ERR(name)) {
3019                 error = PTR_ERR(name);
3020                 name = NULL;
3021                 goto bad_swap;
3022         }
3023         swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
3024         if (IS_ERR(swap_file)) {
3025                 error = PTR_ERR(swap_file);
3026                 swap_file = NULL;
3027                 goto bad_swap;
3028         }
3029
3030         p->swap_file = swap_file;
3031         mapping = swap_file->f_mapping;
3032         inode = mapping->host;
3033
3034         /* If S_ISREG(inode->i_mode) will do inode_lock(inode); */
3035         error = claim_swapfile(p, inode);
3036         if (unlikely(error))
3037                 goto bad_swap;
3038
3039         /*
3040          * Read the swap header.
3041          */
3042         if (!mapping->a_ops->readpage) {
3043                 error = -EINVAL;
3044                 goto bad_swap;
3045         }
3046         page = read_mapping_page(mapping, 0, swap_file);
3047         if (IS_ERR(page)) {
3048                 error = PTR_ERR(page);
3049                 goto bad_swap;
3050         }
3051         swap_header = kmap(page);
3052
3053         maxpages = read_swap_header(p, swap_header, inode);
3054         if (unlikely(!maxpages)) {
3055                 error = -EINVAL;
3056                 goto bad_swap;
3057         }
3058
3059         /* OK, set up the swap map and apply the bad block list */
3060         swap_map = vzalloc(maxpages);
3061         if (!swap_map) {
3062                 error = -ENOMEM;
3063                 goto bad_swap;
3064         }
3065
3066         if (bdi_cap_stable_pages_required(inode_to_bdi(inode)))
3067                 p->flags |= SWP_STABLE_WRITES;
3068
3069         if (bdi_cap_synchronous_io(inode_to_bdi(inode)))
3070                 p->flags |= SWP_SYNCHRONOUS_IO;
3071
3072         if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
3073                 int cpu;
3074                 unsigned long ci, nr_cluster;
3075
3076                 p->flags |= SWP_SOLIDSTATE;
3077                 /*
3078                  * select a random position to start with to help wear leveling
3079                  * SSD
3080                  */
3081                 p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
3082                 nr_cluster = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
3083
3084                 cluster_info = kvcalloc(nr_cluster, sizeof(*cluster_info),
3085                                         GFP_KERNEL);
3086                 if (!cluster_info) {
3087                         error = -ENOMEM;
3088                         goto bad_swap;
3089                 }
3090
3091                 for (ci = 0; ci < nr_cluster; ci++)
3092                         spin_lock_init(&((cluster_info + ci)->lock));
3093
3094                 p->percpu_cluster = alloc_percpu(struct percpu_cluster);
3095                 if (!p->percpu_cluster) {
3096                         error = -ENOMEM;
3097                         goto bad_swap;
3098                 }
3099                 for_each_possible_cpu(cpu) {
3100                         struct percpu_cluster *cluster;
3101                         cluster = per_cpu_ptr(p->percpu_cluster, cpu);
3102                         cluster_set_null(&cluster->index);
3103                 }
3104         } else {
3105                 atomic_inc(&nr_rotate_swap);
3106                 inced_nr_rotate_swap = true;
3107         }
3108
3109         error = swap_cgroup_swapon(p->type, maxpages);
3110         if (error)
3111                 goto bad_swap;
3112
3113         nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
3114                 cluster_info, maxpages, &span);
3115         if (unlikely(nr_extents < 0)) {
3116                 error = nr_extents;
3117                 goto bad_swap;
3118         }
3119         /* frontswap enabled? set up bit-per-page map for frontswap */
3120         if (IS_ENABLED(CONFIG_FRONTSWAP))
3121                 frontswap_map = kvcalloc(BITS_TO_LONGS(maxpages),
3122                                          sizeof(long),
3123                                          GFP_KERNEL);
3124
3125         if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {
3126                 /*
3127                  * When discard is enabled for swap with no particular
3128                  * policy flagged, we set all swap discard flags here in
3129                  * order to sustain backward compatibility with older
3130                  * swapon(8) releases.
3131                  */
3132                 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
3133                              SWP_PAGE_DISCARD);
3134
3135                 /*
3136                  * By flagging sys_swapon, a sysadmin can tell us to
3137                  * either do single-time area discards only, or to just
3138                  * perform discards for released swap page-clusters.
3139                  * Now it's time to adjust the p->flags accordingly.
3140                  */
3141                 if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
3142                         p->flags &= ~SWP_PAGE_DISCARD;
3143                 else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
3144                         p->flags &= ~SWP_AREA_DISCARD;
3145
3146                 /* issue a swapon-time discard if it's still required */
3147                 if (p->flags & SWP_AREA_DISCARD) {
3148                         int err = discard_swap(p);
3149                         if (unlikely(err))
3150                                 pr_err("swapon: discard_swap(%p): %d\n",
3151                                         p, err);
3152                 }
3153         }
3154
3155         error = init_swap_address_space(p->type, maxpages);
3156         if (error)
3157                 goto bad_swap;
3158
3159         mutex_lock(&swapon_mutex);
3160         prio = -1;
3161         if (swap_flags & SWAP_FLAG_PREFER)
3162                 prio =
3163                   (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
3164         enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
3165
3166         pr_info("Adding %uk swap on %s.  Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
3167                 p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
3168                 nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
3169                 (p->flags & SWP_SOLIDSTATE) ? "SS" : "",
3170                 (p->flags & SWP_DISCARDABLE) ? "D" : "",
3171                 (p->flags & SWP_AREA_DISCARD) ? "s" : "",
3172                 (p->flags & SWP_PAGE_DISCARD) ? "c" : "",
3173                 (frontswap_map) ? "FS" : "");
3174
3175         mutex_unlock(&swapon_mutex);
3176         atomic_inc(&proc_poll_event);
3177         wake_up_interruptible(&proc_poll_wait);
3178
3179         if (S_ISREG(inode->i_mode))
3180                 inode->i_flags |= S_SWAPFILE;
3181         error = 0;
3182         goto out;
3183 bad_swap:
3184         free_percpu(p->percpu_cluster);
3185         p->percpu_cluster = NULL;
3186         if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
3187                 set_blocksize(p->bdev, p->old_block_size);
3188                 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
3189         }
3190         destroy_swap_extents(p);
3191         swap_cgroup_swapoff(p->type);
3192         spin_lock(&swap_lock);
3193         p->swap_file = NULL;
3194         p->flags = 0;
3195         spin_unlock(&swap_lock);
3196         vfree(swap_map);
3197         kvfree(cluster_info);
3198         kvfree(frontswap_map);
3199         if (inced_nr_rotate_swap)
3200                 atomic_dec(&nr_rotate_swap);
3201         if (swap_file) {
3202                 if (inode && S_ISREG(inode->i_mode)) {
3203                         inode_unlock(inode);
3204                         inode = NULL;
3205                 }
3206                 filp_close(swap_file, NULL);
3207         }
3208 out:
3209         if (page && !IS_ERR(page)) {
3210                 kunmap(page);
3211                 put_page(page);
3212         }
3213         if (name)
3214                 putname(name);
3215         if (inode && S_ISREG(inode->i_mode))
3216                 inode_unlock(inode);
3217         if (!error)
3218                 enable_swap_slots_cache();
3219         return error;
3220 }
3221
3222 void si_swapinfo(struct sysinfo *val)
3223 {
3224         unsigned int type;
3225         unsigned long nr_to_be_unused = 0;
3226
3227         spin_lock(&swap_lock);
3228         for (type = 0; type < nr_swapfiles; type++) {
3229                 struct swap_info_struct *si = swap_info[type];
3230
3231                 if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
3232                         nr_to_be_unused += si->inuse_pages;
3233         }
3234         val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
3235         val->totalswap = total_swap_pages + nr_to_be_unused;
3236         spin_unlock(&swap_lock);
3237 }
3238
3239 /*
3240  * Verify that a swap entry is valid and increment its swap map count.
3241  *
3242  * Returns error code in following case.
3243  * - success -> 0
3244  * - swp_entry is invalid -> EINVAL
3245  * - swp_entry is migration entry -> EINVAL
3246  * - swap-cache reference is requested but there is already one. -> EEXIST
3247  * - swap-cache reference is requested but the entry is not used. -> ENOENT
3248  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
3249  */
3250 static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
3251 {
3252         struct swap_info_struct *p;
3253         struct swap_cluster_info *ci;
3254         unsigned long offset, type;
3255         unsigned char count;
3256         unsigned char has_cache;
3257         int err = -EINVAL;
3258
3259         if (non_swap_entry(entry))
3260                 goto out;
3261
3262         type = swp_type(entry);
3263         if (type >= nr_swapfiles)
3264                 goto bad_file;
3265         p = swap_info[type];
3266         offset = swp_offset(entry);
3267         if (unlikely(offset >= p->max))
3268                 goto out;
3269
3270         ci = lock_cluster_or_swap_info(p, offset);
3271
3272         count = p->swap_map[offset];
3273
3274         /*
3275          * swapin_readahead() doesn't check if a swap entry is valid, so the
3276          * swap entry could be SWAP_MAP_BAD. Check here with lock held.
3277          */
3278         if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
3279                 err = -ENOENT;
3280                 goto unlock_out;
3281         }
3282
3283         has_cache = count & SWAP_HAS_CACHE;
3284         count &= ~SWAP_HAS_CACHE;
3285         err = 0;
3286
3287         if (usage == SWAP_HAS_CACHE) {
3288
3289                 /* set SWAP_HAS_CACHE if there is no cache and entry is used */
3290                 if (!has_cache && count)
3291                         has_cache = SWAP_HAS_CACHE;
3292                 else if (has_cache)             /* someone else added cache */
3293                         err = -EEXIST;
3294                 else                            /* no users remaining */
3295                         err = -ENOENT;
3296
3297         } else if (count || has_cache) {
3298
3299                 if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
3300                         count += usage;
3301                 else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
3302                         err = -EINVAL;
3303                 else if (swap_count_continued(p, offset, count))
3304                         count = COUNT_CONTINUED;
3305                 else
3306                         err = -ENOMEM;
3307         } else
3308                 err = -ENOENT;                  /* unused swap entry */
3309
3310         p->swap_map[offset] = count | has_cache;
3311
3312 unlock_out:
3313         unlock_cluster_or_swap_info(p, ci);
3314 out:
3315         return err;
3316
3317 bad_file:
3318         pr_err("swap_dup: %s%08lx\n", Bad_file, entry.val);
3319         goto out;
3320 }
3321
3322 /*
3323  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
3324  * (in which case its reference count is never incremented).
3325  */
3326 void swap_shmem_alloc(swp_entry_t entry)
3327 {
3328         __swap_duplicate(entry, SWAP_MAP_SHMEM);
3329 }
3330
3331 /*
3332  * Increase reference count of swap entry by 1.
3333  * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
3334  * but could not be atomically allocated.  Returns 0, just as if it succeeded,
3335  * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
3336  * might occur if a page table entry has got corrupted.
3337  */
3338 int swap_duplicate(swp_entry_t entry)
3339 {
3340         int err = 0;
3341
3342         while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
3343                 err = add_swap_count_continuation(entry, GFP_ATOMIC);
3344         return err;
3345 }
3346
3347 /*
3348  * @entry: swap entry for which we allocate swap cache.
3349  *
3350  * Called when allocating swap cache for existing swap entry,
3351  * This can return error codes. Returns 0 at success.
3352  * -EBUSY means there is a swap cache.
3353  * Note: return code is different from swap_duplicate().
3354  */
3355 int swapcache_prepare(swp_entry_t entry)
3356 {
3357         return __swap_duplicate(entry, SWAP_HAS_CACHE);
3358 }
3359
3360 struct swap_info_struct *swp_swap_info(swp_entry_t entry)
3361 {
3362         return swap_info[swp_type(entry)];
3363 }
3364
3365 struct swap_info_struct *page_swap_info(struct page *page)
3366 {
3367         swp_entry_t entry = { .val = page_private(page) };
3368         return swp_swap_info(entry);
3369 }
3370
3371 /*
3372  * out-of-line __page_file_ methods to avoid include hell.
3373  */
3374 struct address_space *__page_file_mapping(struct page *page)
3375 {
3376         return page_swap_info(page)->swap_file->f_mapping;