xref: /linux/mm/swapfile.c (revision d91517839e5d95adc0cf4b28caa7af62a71de526)
1 /*
2  *  linux/mm/swapfile.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *  Swap reorganised 29.12.95, Stephen Tweedie
6  */
7 
8 #include <linux/mm.h>
9 #include <linux/hugetlb.h>
10 #include <linux/mman.h>
11 #include <linux/slab.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/swap.h>
14 #include <linux/vmalloc.h>
15 #include <linux/pagemap.h>
16 #include <linux/namei.h>
17 #include <linux/shmem_fs.h>
18 #include <linux/blkdev.h>
19 #include <linux/random.h>
20 #include <linux/writeback.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/init.h>
24 #include <linux/ksm.h>
25 #include <linux/rmap.h>
26 #include <linux/security.h>
27 #include <linux/backing-dev.h>
28 #include <linux/mutex.h>
29 #include <linux/capability.h>
30 #include <linux/syscalls.h>
31 #include <linux/memcontrol.h>
32 #include <linux/poll.h>
33 #include <linux/oom.h>
34 #include <linux/frontswap.h>
35 #include <linux/swapfile.h>
36 #include <linux/export.h>
37 
38 #include <asm/pgtable.h>
39 #include <asm/tlbflush.h>
40 #include <linux/swapops.h>
41 #include <linux/page_cgroup.h>
42 
43 static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
44 				 unsigned char);
45 static void free_swap_count_continuations(struct swap_info_struct *);
46 static sector_t map_swap_entry(swp_entry_t, struct block_device**);
47 
48 DEFINE_SPINLOCK(swap_lock);
49 static unsigned int nr_swapfiles;
50 atomic_long_t nr_swap_pages;
51 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
52 long total_swap_pages;
53 static int least_priority;
54 static atomic_t highest_priority_index = ATOMIC_INIT(-1);
55 
56 static const char Bad_file[] = "Bad swap file entry ";
57 static const char Unused_file[] = "Unused swap file entry ";
58 static const char Bad_offset[] = "Bad swap offset entry ";
59 static const char Unused_offset[] = "Unused swap offset entry ";
60 
61 struct swap_list_t swap_list = {-1, -1};
62 
63 struct swap_info_struct *swap_info[MAX_SWAPFILES];
64 
65 static DEFINE_MUTEX(swapon_mutex);
66 
67 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
68 /* Activity counter to indicate that a swapon or swapoff has occurred */
69 static atomic_t proc_poll_event = ATOMIC_INIT(0);
70 
71 static inline unsigned char swap_count(unsigned char ent)
72 {
73 	return ent & ~SWAP_HAS_CACHE;	/* may include SWAP_HAS_CONT flag */
74 }
75 
76 /* returns 1 if swap entry is freed */
77 static int
78 __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset)
79 {
80 	swp_entry_t entry = swp_entry(si->type, offset);
81 	struct page *page;
82 	int ret = 0;
83 
84 	page = find_get_page(swap_address_space(entry), entry.val);
85 	if (!page)
86 		return 0;
87 	/*
88 	 * This function is called from scan_swap_map() and it's called
89 	 * by vmscan.c at reclaiming pages. So, we hold a lock on a page, here.
90 	 * We have to use trylock for avoiding deadlock. This is a special
91 	 * case and you should use try_to_free_swap() with explicit lock_page()
92 	 * in usual operations.
93 	 */
94 	if (trylock_page(page)) {
95 		ret = try_to_free_swap(page);
96 		unlock_page(page);
97 	}
98 	page_cache_release(page);
99 	return ret;
100 }
101 
102 /*
103  * swapon tell device that all the old swap contents can be discarded,
104  * to allow the swap device to optimize its wear-levelling.
105  */
106 static int discard_swap(struct swap_info_struct *si)
107 {
108 	struct swap_extent *se;
109 	sector_t start_block;
110 	sector_t nr_blocks;
111 	int err = 0;
112 
113 	/* Do not discard the swap header page! */
114 	se = &si->first_swap_extent;
115 	start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
116 	nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
117 	if (nr_blocks) {
118 		err = blkdev_issue_discard(si->bdev, start_block,
119 				nr_blocks, GFP_KERNEL, 0);
120 		if (err)
121 			return err;
122 		cond_resched();
123 	}
124 
125 	list_for_each_entry(se, &si->first_swap_extent.list, list) {
126 		start_block = se->start_block << (PAGE_SHIFT - 9);
127 		nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
128 
129 		err = blkdev_issue_discard(si->bdev, start_block,
130 				nr_blocks, GFP_KERNEL, 0);
131 		if (err)
132 			break;
133 
134 		cond_resched();
135 	}
136 	return err;		/* That will often be -EOPNOTSUPP */
137 }
138 
139 /*
140  * swap allocation tell device that a cluster of swap can now be discarded,
141  * to allow the swap device to optimize its wear-levelling.
142  */
143 static void discard_swap_cluster(struct swap_info_struct *si,
144 				 pgoff_t start_page, pgoff_t nr_pages)
145 {
146 	struct swap_extent *se = si->curr_swap_extent;
147 	int found_extent = 0;
148 
149 	while (nr_pages) {
150 		struct list_head *lh;
151 
152 		if (se->start_page <= start_page &&
153 		    start_page < se->start_page + se->nr_pages) {
154 			pgoff_t offset = start_page - se->start_page;
155 			sector_t start_block = se->start_block + offset;
156 			sector_t nr_blocks = se->nr_pages - offset;
157 
158 			if (nr_blocks > nr_pages)
159 				nr_blocks = nr_pages;
160 			start_page += nr_blocks;
161 			nr_pages -= nr_blocks;
162 
163 			if (!found_extent++)
164 				si->curr_swap_extent = se;
165 
166 			start_block <<= PAGE_SHIFT - 9;
167 			nr_blocks <<= PAGE_SHIFT - 9;
168 			if (blkdev_issue_discard(si->bdev, start_block,
169 				    nr_blocks, GFP_NOIO, 0))
170 				break;
171 		}
172 
173 		lh = se->list.next;
174 		se = list_entry(lh, struct swap_extent, list);
175 	}
176 }
177 
178 #define SWAPFILE_CLUSTER	256
179 #define LATENCY_LIMIT		256
180 
181 static inline void cluster_set_flag(struct swap_cluster_info *info,
182 	unsigned int flag)
183 {
184 	info->flags = flag;
185 }
186 
187 static inline unsigned int cluster_count(struct swap_cluster_info *info)
188 {
189 	return info->data;
190 }
191 
192 static inline void cluster_set_count(struct swap_cluster_info *info,
193 				     unsigned int c)
194 {
195 	info->data = c;
196 }
197 
198 static inline void cluster_set_count_flag(struct swap_cluster_info *info,
199 					 unsigned int c, unsigned int f)
200 {
201 	info->flags = f;
202 	info->data = c;
203 }
204 
205 static inline unsigned int cluster_next(struct swap_cluster_info *info)
206 {
207 	return info->data;
208 }
209 
210 static inline void cluster_set_next(struct swap_cluster_info *info,
211 				    unsigned int n)
212 {
213 	info->data = n;
214 }
215 
216 static inline void cluster_set_next_flag(struct swap_cluster_info *info,
217 					 unsigned int n, unsigned int f)
218 {
219 	info->flags = f;
220 	info->data = n;
221 }
222 
223 static inline bool cluster_is_free(struct swap_cluster_info *info)
224 {
225 	return info->flags & CLUSTER_FLAG_FREE;
226 }
227 
228 static inline bool cluster_is_null(struct swap_cluster_info *info)
229 {
230 	return info->flags & CLUSTER_FLAG_NEXT_NULL;
231 }
232 
233 static inline void cluster_set_null(struct swap_cluster_info *info)
234 {
235 	info->flags = CLUSTER_FLAG_NEXT_NULL;
236 	info->data = 0;
237 }
238 
239 /* Add a cluster to discard list and schedule it to do discard */
240 static void swap_cluster_schedule_discard(struct swap_info_struct *si,
241 		unsigned int idx)
242 {
243 	/*
244 	 * If scan_swap_map() can't find a free cluster, it will check
245 	 * si->swap_map directly. To make sure the discarding cluster isn't
246 	 * taken by scan_swap_map(), mark the swap entries bad (occupied). It
247 	 * will be cleared after discard
248 	 */
249 	memset(si->swap_map + idx * SWAPFILE_CLUSTER,
250 			SWAP_MAP_BAD, SWAPFILE_CLUSTER);
251 
252 	if (cluster_is_null(&si->discard_cluster_head)) {
253 		cluster_set_next_flag(&si->discard_cluster_head,
254 						idx, 0);
255 		cluster_set_next_flag(&si->discard_cluster_tail,
256 						idx, 0);
257 	} else {
258 		unsigned int tail = cluster_next(&si->discard_cluster_tail);
259 		cluster_set_next(&si->cluster_info[tail], idx);
260 		cluster_set_next_flag(&si->discard_cluster_tail,
261 						idx, 0);
262 	}
263 
264 	schedule_work(&si->discard_work);
265 }
266 
267 /*
268  * Doing discard actually. After a cluster discard is finished, the cluster
269  * will be added to free cluster list. caller should hold si->lock.
270 */
271 static void swap_do_scheduled_discard(struct swap_info_struct *si)
272 {
273 	struct swap_cluster_info *info;
274 	unsigned int idx;
275 
276 	info = si->cluster_info;
277 
278 	while (!cluster_is_null(&si->discard_cluster_head)) {
279 		idx = cluster_next(&si->discard_cluster_head);
280 
281 		cluster_set_next_flag(&si->discard_cluster_head,
282 						cluster_next(&info[idx]), 0);
283 		if (cluster_next(&si->discard_cluster_tail) == idx) {
284 			cluster_set_null(&si->discard_cluster_head);
285 			cluster_set_null(&si->discard_cluster_tail);
286 		}
287 		spin_unlock(&si->lock);
288 
289 		discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
290 				SWAPFILE_CLUSTER);
291 
292 		spin_lock(&si->lock);
293 		cluster_set_flag(&info[idx], CLUSTER_FLAG_FREE);
294 		if (cluster_is_null(&si->free_cluster_head)) {
295 			cluster_set_next_flag(&si->free_cluster_head,
296 						idx, 0);
297 			cluster_set_next_flag(&si->free_cluster_tail,
298 						idx, 0);
299 		} else {
300 			unsigned int tail;
301 
302 			tail = cluster_next(&si->free_cluster_tail);
303 			cluster_set_next(&info[tail], idx);
304 			cluster_set_next_flag(&si->free_cluster_tail,
305 						idx, 0);
306 		}
307 		memset(si->swap_map + idx * SWAPFILE_CLUSTER,
308 				0, SWAPFILE_CLUSTER);
309 	}
310 }
311 
312 static void swap_discard_work(struct work_struct *work)
313 {
314 	struct swap_info_struct *si;
315 
316 	si = container_of(work, struct swap_info_struct, discard_work);
317 
318 	spin_lock(&si->lock);
319 	swap_do_scheduled_discard(si);
320 	spin_unlock(&si->lock);
321 }
322 
323 /*
324  * The cluster corresponding to page_nr will be used. The cluster will be
325  * removed from free cluster list and its usage counter will be increased.
326  */
327 static void inc_cluster_info_page(struct swap_info_struct *p,
328 	struct swap_cluster_info *cluster_info, unsigned long page_nr)
329 {
330 	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
331 
332 	if (!cluster_info)
333 		return;
334 	if (cluster_is_free(&cluster_info[idx])) {
335 		VM_BUG_ON(cluster_next(&p->free_cluster_head) != idx);
336 		cluster_set_next_flag(&p->free_cluster_head,
337 			cluster_next(&cluster_info[idx]), 0);
338 		if (cluster_next(&p->free_cluster_tail) == idx) {
339 			cluster_set_null(&p->free_cluster_tail);
340 			cluster_set_null(&p->free_cluster_head);
341 		}
342 		cluster_set_count_flag(&cluster_info[idx], 0, 0);
343 	}
344 
345 	VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
346 	cluster_set_count(&cluster_info[idx],
347 		cluster_count(&cluster_info[idx]) + 1);
348 }
349 
350 /*
351  * The cluster corresponding to page_nr decreases one usage. If the usage
352  * counter becomes 0, which means no page in the cluster is in using, we can
353  * optionally discard the cluster and add it to free cluster list.
354  */
355 static void dec_cluster_info_page(struct swap_info_struct *p,
356 	struct swap_cluster_info *cluster_info, unsigned long page_nr)
357 {
358 	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
359 
360 	if (!cluster_info)
361 		return;
362 
363 	VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
364 	cluster_set_count(&cluster_info[idx],
365 		cluster_count(&cluster_info[idx]) - 1);
366 
367 	if (cluster_count(&cluster_info[idx]) == 0) {
368 		/*
369 		 * If the swap is discardable, prepare discard the cluster
370 		 * instead of free it immediately. The cluster will be freed
371 		 * after discard.
372 		 */
373 		if ((p->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
374 				 (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
375 			swap_cluster_schedule_discard(p, idx);
376 			return;
377 		}
378 
379 		cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
380 		if (cluster_is_null(&p->free_cluster_head)) {
381 			cluster_set_next_flag(&p->free_cluster_head, idx, 0);
382 			cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
383 		} else {
384 			unsigned int tail = cluster_next(&p->free_cluster_tail);
385 			cluster_set_next(&cluster_info[tail], idx);
386 			cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
387 		}
388 	}
389 }
390 
391 /*
392  * It's possible scan_swap_map() uses a free cluster in the middle of free
393  * cluster list. Avoiding such abuse to avoid list corruption.
394  */
395 static bool
396 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
397 	unsigned long offset)
398 {
399 	struct percpu_cluster *percpu_cluster;
400 	bool conflict;
401 
402 	offset /= SWAPFILE_CLUSTER;
403 	conflict = !cluster_is_null(&si->free_cluster_head) &&
404 		offset != cluster_next(&si->free_cluster_head) &&
405 		cluster_is_free(&si->cluster_info[offset]);
406 
407 	if (!conflict)
408 		return false;
409 
410 	percpu_cluster = this_cpu_ptr(si->percpu_cluster);
411 	cluster_set_null(&percpu_cluster->index);
412 	return true;
413 }
414 
415 /*
416  * Try to get a swap entry from current cpu's swap entry pool (a cluster). This
417  * might involve allocating a new cluster for current CPU too.
418  */
419 static void scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
420 	unsigned long *offset, unsigned long *scan_base)
421 {
422 	struct percpu_cluster *cluster;
423 	bool found_free;
424 	unsigned long tmp;
425 
426 new_cluster:
427 	cluster = this_cpu_ptr(si->percpu_cluster);
428 	if (cluster_is_null(&cluster->index)) {
429 		if (!cluster_is_null(&si->free_cluster_head)) {
430 			cluster->index = si->free_cluster_head;
431 			cluster->next = cluster_next(&cluster->index) *
432 					SWAPFILE_CLUSTER;
433 		} else if (!cluster_is_null(&si->discard_cluster_head)) {
434 			/*
435 			 * we don't have free cluster but have some clusters in
436 			 * discarding, do discard now and reclaim them
437 			 */
438 			swap_do_scheduled_discard(si);
439 			*scan_base = *offset = si->cluster_next;
440 			goto new_cluster;
441 		} else
442 			return;
443 	}
444 
445 	found_free = false;
446 
447 	/*
448 	 * Other CPUs can use our cluster if they can't find a free cluster,
449 	 * check if there is still free entry in the cluster
450 	 */
451 	tmp = cluster->next;
452 	while (tmp < si->max && tmp < (cluster_next(&cluster->index) + 1) *
453 	       SWAPFILE_CLUSTER) {
454 		if (!si->swap_map[tmp]) {
455 			found_free = true;
456 			break;
457 		}
458 		tmp++;
459 	}
460 	if (!found_free) {
461 		cluster_set_null(&cluster->index);
462 		goto new_cluster;
463 	}
464 	cluster->next = tmp + 1;
465 	*offset = tmp;
466 	*scan_base = tmp;
467 }
468 
469 static unsigned long scan_swap_map(struct swap_info_struct *si,
470 				   unsigned char usage)
471 {
472 	unsigned long offset;
473 	unsigned long scan_base;
474 	unsigned long last_in_cluster = 0;
475 	int latency_ration = LATENCY_LIMIT;
476 
477 	/*
478 	 * We try to cluster swap pages by allocating them sequentially
479 	 * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
480 	 * way, however, we resort to first-free allocation, starting
481 	 * a new cluster.  This prevents us from scattering swap pages
482 	 * all over the entire swap partition, so that we reduce
483 	 * overall disk seek times between swap pages.  -- sct
484 	 * But we do now try to find an empty cluster.  -Andrea
485 	 * And we let swap pages go all over an SSD partition.  Hugh
486 	 */
487 
488 	si->flags += SWP_SCANNING;
489 	scan_base = offset = si->cluster_next;
490 
491 	/* SSD algorithm */
492 	if (si->cluster_info) {
493 		scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
494 		goto checks;
495 	}
496 
497 	if (unlikely(!si->cluster_nr--)) {
498 		if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
499 			si->cluster_nr = SWAPFILE_CLUSTER - 1;
500 			goto checks;
501 		}
502 
503 		spin_unlock(&si->lock);
504 
505 		/*
506 		 * If seek is expensive, start searching for new cluster from
507 		 * start of partition, to minimize the span of allocated swap.
508 		 * But if seek is cheap, search from our current position, so
509 		 * that swap is allocated from all over the partition: if the
510 		 * Flash Translation Layer only remaps within limited zones,
511 		 * we don't want to wear out the first zone too quickly.
512 		 */
513 		if (!(si->flags & SWP_SOLIDSTATE))
514 			scan_base = offset = si->lowest_bit;
515 		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
516 
517 		/* Locate the first empty (unaligned) cluster */
518 		for (; last_in_cluster <= si->highest_bit; offset++) {
519 			if (si->swap_map[offset])
520 				last_in_cluster = offset + SWAPFILE_CLUSTER;
521 			else if (offset == last_in_cluster) {
522 				spin_lock(&si->lock);
523 				offset -= SWAPFILE_CLUSTER - 1;
524 				si->cluster_next = offset;
525 				si->cluster_nr = SWAPFILE_CLUSTER - 1;
526 				goto checks;
527 			}
528 			if (unlikely(--latency_ration < 0)) {
529 				cond_resched();
530 				latency_ration = LATENCY_LIMIT;
531 			}
532 		}
533 
534 		offset = si->lowest_bit;
535 		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
536 
537 		/* Locate the first empty (unaligned) cluster */
538 		for (; last_in_cluster < scan_base; offset++) {
539 			if (si->swap_map[offset])
540 				last_in_cluster = offset + SWAPFILE_CLUSTER;
541 			else if (offset == last_in_cluster) {
542 				spin_lock(&si->lock);
543 				offset -= SWAPFILE_CLUSTER - 1;
544 				si->cluster_next = offset;
545 				si->cluster_nr = SWAPFILE_CLUSTER - 1;
546 				goto checks;
547 			}
548 			if (unlikely(--latency_ration < 0)) {
549 				cond_resched();
550 				latency_ration = LATENCY_LIMIT;
551 			}
552 		}
553 
554 		offset = scan_base;
555 		spin_lock(&si->lock);
556 		si->cluster_nr = SWAPFILE_CLUSTER - 1;
557 	}
558 
559 checks:
560 	if (si->cluster_info) {
561 		while (scan_swap_map_ssd_cluster_conflict(si, offset))
562 			scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
563 	}
564 	if (!(si->flags & SWP_WRITEOK))
565 		goto no_page;
566 	if (!si->highest_bit)
567 		goto no_page;
568 	if (offset > si->highest_bit)
569 		scan_base = offset = si->lowest_bit;
570 
571 	/* reuse swap entry of cache-only swap if not busy. */
572 	if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
573 		int swap_was_freed;
574 		spin_unlock(&si->lock);
575 		swap_was_freed = __try_to_reclaim_swap(si, offset);
576 		spin_lock(&si->lock);
577 		/* entry was freed successfully, try to use this again */
578 		if (swap_was_freed)
579 			goto checks;
580 		goto scan; /* check next one */
581 	}
582 
583 	if (si->swap_map[offset])
584 		goto scan;
585 
586 	if (offset == si->lowest_bit)
587 		si->lowest_bit++;
588 	if (offset == si->highest_bit)
589 		si->highest_bit--;
590 	si->inuse_pages++;
591 	if (si->inuse_pages == si->pages) {
592 		si->lowest_bit = si->max;
593 		si->highest_bit = 0;
594 	}
595 	si->swap_map[offset] = usage;
596 	inc_cluster_info_page(si, si->cluster_info, offset);
597 	si->cluster_next = offset + 1;
598 	si->flags -= SWP_SCANNING;
599 
600 	return offset;
601 
602 scan:
603 	spin_unlock(&si->lock);
604 	while (++offset <= si->highest_bit) {
605 		if (!si->swap_map[offset]) {
606 			spin_lock(&si->lock);
607 			goto checks;
608 		}
609 		if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
610 			spin_lock(&si->lock);
611 			goto checks;
612 		}
613 		if (unlikely(--latency_ration < 0)) {
614 			cond_resched();
615 			latency_ration = LATENCY_LIMIT;
616 		}
617 	}
618 	offset = si->lowest_bit;
619 	while (offset < scan_base) {
620 		if (!si->swap_map[offset]) {
621 			spin_lock(&si->lock);
622 			goto checks;
623 		}
624 		if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
625 			spin_lock(&si->lock);
626 			goto checks;
627 		}
628 		if (unlikely(--latency_ration < 0)) {
629 			cond_resched();
630 			latency_ration = LATENCY_LIMIT;
631 		}
632 		offset++;
633 	}
634 	spin_lock(&si->lock);
635 
636 no_page:
637 	si->flags -= SWP_SCANNING;
638 	return 0;
639 }
640 
641 swp_entry_t get_swap_page(void)
642 {
643 	struct swap_info_struct *si;
644 	pgoff_t offset;
645 	int type, next;
646 	int wrapped = 0;
647 	int hp_index;
648 
649 	spin_lock(&swap_lock);
650 	if (atomic_long_read(&nr_swap_pages) <= 0)
651 		goto noswap;
652 	atomic_long_dec(&nr_swap_pages);
653 
654 	for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) {
655 		hp_index = atomic_xchg(&highest_priority_index, -1);
656 		/*
657 		 * highest_priority_index records current highest priority swap
658 		 * type which just frees swap entries. If its priority is
659 		 * higher than that of swap_list.next swap type, we use it.  It
660 		 * isn't protected by swap_lock, so it can be an invalid value
661 		 * if the corresponding swap type is swapoff. We double check
662 		 * the flags here. It's even possible the swap type is swapoff
663 		 * and swapon again and its priority is changed. In such rare
664 		 * case, low prority swap type might be used, but eventually
665 		 * high priority swap will be used after several rounds of
666 		 * swap.
667 		 */
668 		if (hp_index != -1 && hp_index != type &&
669 		    swap_info[type]->prio < swap_info[hp_index]->prio &&
670 		    (swap_info[hp_index]->flags & SWP_WRITEOK)) {
671 			type = hp_index;
672 			swap_list.next = type;
673 		}
674 
675 		si = swap_info[type];
676 		next = si->next;
677 		if (next < 0 ||
678 		    (!wrapped && si->prio != swap_info[next]->prio)) {
679 			next = swap_list.head;
680 			wrapped++;
681 		}
682 
683 		spin_lock(&si->lock);
684 		if (!si->highest_bit) {
685 			spin_unlock(&si->lock);
686 			continue;
687 		}
688 		if (!(si->flags & SWP_WRITEOK)) {
689 			spin_unlock(&si->lock);
690 			continue;
691 		}
692 
693 		swap_list.next = next;
694 
695 		spin_unlock(&swap_lock);
696 		/* This is called for allocating swap entry for cache */
697 		offset = scan_swap_map(si, SWAP_HAS_CACHE);
698 		spin_unlock(&si->lock);
699 		if (offset)
700 			return swp_entry(type, offset);
701 		spin_lock(&swap_lock);
702 		next = swap_list.next;
703 	}
704 
705 	atomic_long_inc(&nr_swap_pages);
706 noswap:
707 	spin_unlock(&swap_lock);
708 	return (swp_entry_t) {0};
709 }
710 
711 /* The only caller of this function is now suspend routine */
712 swp_entry_t get_swap_page_of_type(int type)
713 {
714 	struct swap_info_struct *si;
715 	pgoff_t offset;
716 
717 	si = swap_info[type];
718 	spin_lock(&si->lock);
719 	if (si && (si->flags & SWP_WRITEOK)) {
720 		atomic_long_dec(&nr_swap_pages);
721 		/* This is called for allocating swap entry, not cache */
722 		offset = scan_swap_map(si, 1);
723 		if (offset) {
724 			spin_unlock(&si->lock);
725 			return swp_entry(type, offset);
726 		}
727 		atomic_long_inc(&nr_swap_pages);
728 	}
729 	spin_unlock(&si->lock);
730 	return (swp_entry_t) {0};
731 }
732 
733 static struct swap_info_struct *swap_info_get(swp_entry_t entry)
734 {
735 	struct swap_info_struct *p;
736 	unsigned long offset, type;
737 
738 	if (!entry.val)
739 		goto out;
740 	type = swp_type(entry);
741 	if (type >= nr_swapfiles)
742 		goto bad_nofile;
743 	p = swap_info[type];
744 	if (!(p->flags & SWP_USED))
745 		goto bad_device;
746 	offset = swp_offset(entry);
747 	if (offset >= p->max)
748 		goto bad_offset;
749 	if (!p->swap_map[offset])
750 		goto bad_free;
751 	spin_lock(&p->lock);
752 	return p;
753 
754 bad_free:
755 	pr_err("swap_free: %s%08lx\n", Unused_offset, entry.val);
756 	goto out;
757 bad_offset:
758 	pr_err("swap_free: %s%08lx\n", Bad_offset, entry.val);
759 	goto out;
760 bad_device:
761 	pr_err("swap_free: %s%08lx\n", Unused_file, entry.val);
762 	goto out;
763 bad_nofile:
764 	pr_err("swap_free: %s%08lx\n", Bad_file, entry.val);
765 out:
766 	return NULL;
767 }
768 
769 /*
770  * This swap type frees swap entry, check if it is the highest priority swap
771  * type which just frees swap entry. get_swap_page() uses
772  * highest_priority_index to search highest priority swap type. The
773  * swap_info_struct.lock can't protect us if there are multiple swap types
774  * active, so we use atomic_cmpxchg.
775  */
776 static void set_highest_priority_index(int type)
777 {
778 	int old_hp_index, new_hp_index;
779 
780 	do {
781 		old_hp_index = atomic_read(&highest_priority_index);
782 		if (old_hp_index != -1 &&
783 			swap_info[old_hp_index]->prio >= swap_info[type]->prio)
784 			break;
785 		new_hp_index = type;
786 	} while (atomic_cmpxchg(&highest_priority_index,
787 		old_hp_index, new_hp_index) != old_hp_index);
788 }
789 
790 static unsigned char swap_entry_free(struct swap_info_struct *p,
791 				     swp_entry_t entry, unsigned char usage)
792 {
793 	unsigned long offset = swp_offset(entry);
794 	unsigned char count;
795 	unsigned char has_cache;
796 
797 	count = p->swap_map[offset];
798 	has_cache = count & SWAP_HAS_CACHE;
799 	count &= ~SWAP_HAS_CACHE;
800 
801 	if (usage == SWAP_HAS_CACHE) {
802 		VM_BUG_ON(!has_cache);
803 		has_cache = 0;
804 	} else if (count == SWAP_MAP_SHMEM) {
805 		/*
806 		 * Or we could insist on shmem.c using a special
807 		 * swap_shmem_free() and free_shmem_swap_and_cache()...
808 		 */
809 		count = 0;
810 	} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
811 		if (count == COUNT_CONTINUED) {
812 			if (swap_count_continued(p, offset, count))
813 				count = SWAP_MAP_MAX | COUNT_CONTINUED;
814 			else
815 				count = SWAP_MAP_MAX;
816 		} else
817 			count--;
818 	}
819 
820 	if (!count)
821 		mem_cgroup_uncharge_swap(entry);
822 
823 	usage = count | has_cache;
824 	p->swap_map[offset] = usage;
825 
826 	/* free if no reference */
827 	if (!usage) {
828 		dec_cluster_info_page(p, p->cluster_info, offset);
829 		if (offset < p->lowest_bit)
830 			p->lowest_bit = offset;
831 		if (offset > p->highest_bit)
832 			p->highest_bit = offset;
833 		set_highest_priority_index(p->type);
834 		atomic_long_inc(&nr_swap_pages);
835 		p->inuse_pages--;
836 		frontswap_invalidate_page(p->type, offset);
837 		if (p->flags & SWP_BLKDEV) {
838 			struct gendisk *disk = p->bdev->bd_disk;
839 			if (disk->fops->swap_slot_free_notify)
840 				disk->fops->swap_slot_free_notify(p->bdev,
841 								  offset);
842 		}
843 	}
844 
845 	return usage;
846 }
847 
848 /*
849  * Caller has made sure that the swap device corresponding to entry
850  * is still around or has not been recycled.
851  */
852 void swap_free(swp_entry_t entry)
853 {
854 	struct swap_info_struct *p;
855 
856 	p = swap_info_get(entry);
857 	if (p) {
858 		swap_entry_free(p, entry, 1);
859 		spin_unlock(&p->lock);
860 	}
861 }
862 
863 /*
864  * Called after dropping swapcache to decrease refcnt to swap entries.
865  */
866 void swapcache_free(swp_entry_t entry, struct page *page)
867 {
868 	struct swap_info_struct *p;
869 	unsigned char count;
870 
871 	p = swap_info_get(entry);
872 	if (p) {
873 		count = swap_entry_free(p, entry, SWAP_HAS_CACHE);
874 		if (page)
875 			mem_cgroup_uncharge_swapcache(page, entry, count != 0);
876 		spin_unlock(&p->lock);
877 	}
878 }
879 
880 /*
881  * How many references to page are currently swapped out?
882  * This does not give an exact answer when swap count is continued,
883  * but does include the high COUNT_CONTINUED flag to allow for that.
884  */
885 int page_swapcount(struct page *page)
886 {
887 	int count = 0;
888 	struct swap_info_struct *p;
889 	swp_entry_t entry;
890 
891 	entry.val = page_private(page);
892 	p = swap_info_get(entry);
893 	if (p) {
894 		count = swap_count(p->swap_map[swp_offset(entry)]);
895 		spin_unlock(&p->lock);
896 	}
897 	return count;
898 }
899 
900 /*
901  * We can write to an anon page without COW if there are no other references
902  * to it.  And as a side-effect, free up its swap: because the old content
903  * on disk will never be read, and seeking back there to write new content
904  * later would only waste time away from clustering.
905  */
906 int reuse_swap_page(struct page *page)
907 {
908 	int count;
909 
910 	VM_BUG_ON_PAGE(!PageLocked(page), page);
911 	if (unlikely(PageKsm(page)))
912 		return 0;
913 	count = page_mapcount(page);
914 	if (count <= 1 && PageSwapCache(page)) {
915 		count += page_swapcount(page);
916 		if (count == 1 && !PageWriteback(page)) {
917 			delete_from_swap_cache(page);
918 			SetPageDirty(page);
919 		}
920 	}
921 	return count <= 1;
922 }
923 
924 /*
925  * If swap is getting full, or if there are no more mappings of this page,
926  * then try_to_free_swap is called to free its swap space.
927  */
928 int try_to_free_swap(struct page *page)
929 {
930 	VM_BUG_ON_PAGE(!PageLocked(page), page);
931 
932 	if (!PageSwapCache(page))
933 		return 0;
934 	if (PageWriteback(page))
935 		return 0;
936 	if (page_swapcount(page))
937 		return 0;
938 
939 	/*
940 	 * Once hibernation has begun to create its image of memory,
941 	 * there's a danger that one of the calls to try_to_free_swap()
942 	 * - most probably a call from __try_to_reclaim_swap() while
943 	 * hibernation is allocating its own swap pages for the image,
944 	 * but conceivably even a call from memory reclaim - will free
945 	 * the swap from a page which has already been recorded in the
946 	 * image as a clean swapcache page, and then reuse its swap for
947 	 * another page of the image.  On waking from hibernation, the
948 	 * original page might be freed under memory pressure, then
949 	 * later read back in from swap, now with the wrong data.
950 	 *
951 	 * Hibernation suspends storage while it is writing the image
952 	 * to disk so check that here.
953 	 */
954 	if (pm_suspended_storage())
955 		return 0;
956 
957 	delete_from_swap_cache(page);
958 	SetPageDirty(page);
959 	return 1;
960 }
961 
962 /*
963  * Free the swap entry like above, but also try to
964  * free the page cache entry if it is the last user.
965  */
966 int free_swap_and_cache(swp_entry_t entry)
967 {
968 	struct swap_info_struct *p;
969 	struct page *page = NULL;
970 
971 	if (non_swap_entry(entry))
972 		return 1;
973 
974 	p = swap_info_get(entry);
975 	if (p) {
976 		if (swap_entry_free(p, entry, 1) == SWAP_HAS_CACHE) {
977 			page = find_get_page(swap_address_space(entry),
978 						entry.val);
979 			if (page && !trylock_page(page)) {
980 				page_cache_release(page);
981 				page = NULL;
982 			}
983 		}
984 		spin_unlock(&p->lock);
985 	}
986 	if (page) {
987 		/*
988 		 * Not mapped elsewhere, or swap space full? Free it!
989 		 * Also recheck PageSwapCache now page is locked (above).
990 		 */
991 		if (PageSwapCache(page) && !PageWriteback(page) &&
992 				(!page_mapped(page) || vm_swap_full())) {
993 			delete_from_swap_cache(page);
994 			SetPageDirty(page);
995 		}
996 		unlock_page(page);
997 		page_cache_release(page);
998 	}
999 	return p != NULL;
1000 }
1001 
1002 #ifdef CONFIG_HIBERNATION
1003 /*
1004  * Find the swap type that corresponds to given device (if any).
1005  *
1006  * @offset - number of the PAGE_SIZE-sized block of the device, starting
1007  * from 0, in which the swap header is expected to be located.
1008  *
1009  * This is needed for the suspend to disk (aka swsusp).
1010  */
1011 int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
1012 {
1013 	struct block_device *bdev = NULL;
1014 	int type;
1015 
1016 	if (device)
1017 		bdev = bdget(device);
1018 
1019 	spin_lock(&swap_lock);
1020 	for (type = 0; type < nr_swapfiles; type++) {
1021 		struct swap_info_struct *sis = swap_info[type];
1022 
1023 		if (!(sis->flags & SWP_WRITEOK))
1024 			continue;
1025 
1026 		if (!bdev) {
1027 			if (bdev_p)
1028 				*bdev_p = bdgrab(sis->bdev);
1029 
1030 			spin_unlock(&swap_lock);
1031 			return type;
1032 		}
1033 		if (bdev == sis->bdev) {
1034 			struct swap_extent *se = &sis->first_swap_extent;
1035 
1036 			if (se->start_block == offset) {
1037 				if (bdev_p)
1038 					*bdev_p = bdgrab(sis->bdev);
1039 
1040 				spin_unlock(&swap_lock);
1041 				bdput(bdev);
1042 				return type;
1043 			}
1044 		}
1045 	}
1046 	spin_unlock(&swap_lock);
1047 	if (bdev)
1048 		bdput(bdev);
1049 
1050 	return -ENODEV;
1051 }
1052 
1053 /*
1054  * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
1055  * corresponding to given index in swap_info (swap type).
1056  */
1057 sector_t swapdev_block(int type, pgoff_t offset)
1058 {
1059 	struct block_device *bdev;
1060 
1061 	if ((unsigned int)type >= nr_swapfiles)
1062 		return 0;
1063 	if (!(swap_info[type]->flags & SWP_WRITEOK))
1064 		return 0;
1065 	return map_swap_entry(swp_entry(type, offset), &bdev);
1066 }
1067 
1068 /*
1069  * Return either the total number of swap pages of given type, or the number
1070  * of free pages of that type (depending on @free)
1071  *
1072  * This is needed for software suspend
1073  */
1074 unsigned int count_swap_pages(int type, int free)
1075 {
1076 	unsigned int n = 0;
1077 
1078 	spin_lock(&swap_lock);
1079 	if ((unsigned int)type < nr_swapfiles) {
1080 		struct swap_info_struct *sis = swap_info[type];
1081 
1082 		spin_lock(&sis->lock);
1083 		if (sis->flags & SWP_WRITEOK) {
1084 			n = sis->pages;
1085 			if (free)
1086 				n -= sis->inuse_pages;
1087 		}
1088 		spin_unlock(&sis->lock);
1089 	}
1090 	spin_unlock(&swap_lock);
1091 	return n;
1092 }
1093 #endif /* CONFIG_HIBERNATION */
1094 
1095 static inline int maybe_same_pte(pte_t pte, pte_t swp_pte)
1096 {
1097 #ifdef CONFIG_MEM_SOFT_DIRTY
1098 	/*
1099 	 * When pte keeps soft dirty bit the pte generated
1100 	 * from swap entry does not has it, still it's same
1101 	 * pte from logical point of view.
1102 	 */
1103 	pte_t swp_pte_dirty = pte_swp_mksoft_dirty(swp_pte);
1104 	return pte_same(pte, swp_pte) || pte_same(pte, swp_pte_dirty);
1105 #else
1106 	return pte_same(pte, swp_pte);
1107 #endif
1108 }
1109 
1110 /*
1111  * No need to decide whether this PTE shares the swap entry with others,
1112  * just let do_wp_page work it out if a write is requested later - to
1113  * force COW, vm_page_prot omits write permission from any private vma.
1114  */
1115 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
1116 		unsigned long addr, swp_entry_t entry, struct page *page)
1117 {
1118 	struct page *swapcache;
1119 	struct mem_cgroup *memcg;
1120 	spinlock_t *ptl;
1121 	pte_t *pte;
1122 	int ret = 1;
1123 
1124 	swapcache = page;
1125 	page = ksm_might_need_to_copy(page, vma, addr);
1126 	if (unlikely(!page))
1127 		return -ENOMEM;
1128 
1129 	if (mem_cgroup_try_charge_swapin(vma->vm_mm, page,
1130 					 GFP_KERNEL, &memcg)) {
1131 		ret = -ENOMEM;
1132 		goto out_nolock;
1133 	}
1134 
1135 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1136 	if (unlikely(!maybe_same_pte(*pte, swp_entry_to_pte(entry)))) {
1137 		mem_cgroup_cancel_charge_swapin(memcg);
1138 		ret = 0;
1139 		goto out;
1140 	}
1141 
1142 	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1143 	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
1144 	get_page(page);
1145 	set_pte_at(vma->vm_mm, addr, pte,
1146 		   pte_mkold(mk_pte(page, vma->vm_page_prot)));
1147 	if (page == swapcache)
1148 		page_add_anon_rmap(page, vma, addr);
1149 	else /* ksm created a completely new copy */
1150 		page_add_new_anon_rmap(page, vma, addr);
1151 	mem_cgroup_commit_charge_swapin(page, memcg);
1152 	swap_free(entry);
1153 	/*
1154 	 * Move the page to the active list so it is not
1155 	 * immediately swapped out again after swapon.
1156 	 */
1157 	activate_page(page);
1158 out:
1159 	pte_unmap_unlock(pte, ptl);
1160 out_nolock:
1161 	if (page != swapcache) {
1162 		unlock_page(page);
1163 		put_page(page);
1164 	}
1165 	return ret;
1166 }
1167 
1168 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1169 				unsigned long addr, unsigned long end,
1170 				swp_entry_t entry, struct page *page)
1171 {
1172 	pte_t swp_pte = swp_entry_to_pte(entry);
1173 	pte_t *pte;
1174 	int ret = 0;
1175 
1176 	/*
1177 	 * We don't actually need pte lock while scanning for swp_pte: since
1178 	 * we hold page lock and mmap_sem, swp_pte cannot be inserted into the
1179 	 * page table while we're scanning; though it could get zapped, and on
1180 	 * some architectures (e.g. x86_32 with PAE) we might catch a glimpse
1181 	 * of unmatched parts which look like swp_pte, so unuse_pte must
1182 	 * recheck under pte lock.  Scanning without pte lock lets it be
1183 	 * preemptable whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE.
1184 	 */
1185 	pte = pte_offset_map(pmd, addr);
1186 	do {
1187 		/*
1188 		 * swapoff spends a _lot_ of time in this loop!
1189 		 * Test inline before going to call unuse_pte.
1190 		 */
1191 		if (unlikely(maybe_same_pte(*pte, swp_pte))) {
1192 			pte_unmap(pte);
1193 			ret = unuse_pte(vma, pmd, addr, entry, page);
1194 			if (ret)
1195 				goto out;
1196 			pte = pte_offset_map(pmd, addr);
1197 		}
1198 	} while (pte++, addr += PAGE_SIZE, addr != end);
1199 	pte_unmap(pte - 1);
1200 out:
1201 	return ret;
1202 }
1203 
1204 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
1205 				unsigned long addr, unsigned long end,
1206 				swp_entry_t entry, struct page *page)
1207 {
1208 	pmd_t *pmd;
1209 	unsigned long next;
1210 	int ret;
1211 
1212 	pmd = pmd_offset(pud, addr);
1213 	do {
1214 		next = pmd_addr_end(addr, end);
1215 		if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1216 			continue;
1217 		ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
1218 		if (ret)
1219 			return ret;
1220 	} while (pmd++, addr = next, addr != end);
1221 	return 0;
1222 }
1223 
1224 static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
1225 				unsigned long addr, unsigned long end,
1226 				swp_entry_t entry, struct page *page)
1227 {
1228 	pud_t *pud;
1229 	unsigned long next;
1230 	int ret;
1231 
1232 	pud = pud_offset(pgd, addr);
1233 	do {
1234 		next = pud_addr_end(addr, end);
1235 		if (pud_none_or_clear_bad(pud))
1236 			continue;
1237 		ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
1238 		if (ret)
1239 			return ret;
1240 	} while (pud++, addr = next, addr != end);
1241 	return 0;
1242 }
1243 
1244 static int unuse_vma(struct vm_area_struct *vma,
1245 				swp_entry_t entry, struct page *page)
1246 {
1247 	pgd_t *pgd;
1248 	unsigned long addr, end, next;
1249 	int ret;
1250 
1251 	if (page_anon_vma(page)) {
1252 		addr = page_address_in_vma(page, vma);
1253 		if (addr == -EFAULT)
1254 			return 0;
1255 		else
1256 			end = addr + PAGE_SIZE;
1257 	} else {
1258 		addr = vma->vm_start;
1259 		end = vma->vm_end;
1260 	}
1261 
1262 	pgd = pgd_offset(vma->vm_mm, addr);
1263 	do {
1264 		next = pgd_addr_end(addr, end);
1265 		if (pgd_none_or_clear_bad(pgd))
1266 			continue;
1267 		ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
1268 		if (ret)
1269 			return ret;
1270 	} while (pgd++, addr = next, addr != end);
1271 	return 0;
1272 }
1273 
1274 static int unuse_mm(struct mm_struct *mm,
1275 				swp_entry_t entry, struct page *page)
1276 {
1277 	struct vm_area_struct *vma;
1278 	int ret = 0;
1279 
1280 	if (!down_read_trylock(&mm->mmap_sem)) {
1281 		/*
1282 		 * Activate page so shrink_inactive_list is unlikely to unmap
1283 		 * its ptes while lock is dropped, so swapoff can make progress.
1284 		 */
1285 		activate_page(page);
1286 		unlock_page(page);
1287 		down_read(&mm->mmap_sem);
1288 		lock_page(page);
1289 	}
1290 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
1291 		if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
1292 			break;
1293 	}
1294 	up_read(&mm->mmap_sem);
1295 	return (ret < 0)? ret: 0;
1296 }
1297 
1298 /*
1299  * Scan swap_map (or frontswap_map if frontswap parameter is true)
1300  * from current position to next entry still in use.
1301  * Recycle to start on reaching the end, returning 0 when empty.
1302  */
1303 static unsigned int find_next_to_unuse(struct swap_info_struct *si,
1304 					unsigned int prev, bool frontswap)
1305 {
1306 	unsigned int max = si->max;
1307 	unsigned int i = prev;
1308 	unsigned char count;
1309 
1310 	/*
1311 	 * No need for swap_lock here: we're just looking
1312 	 * for whether an entry is in use, not modifying it; false
1313 	 * hits are okay, and sys_swapoff() has already prevented new
1314 	 * allocations from this area (while holding swap_lock).
1315 	 */
1316 	for (;;) {
1317 		if (++i >= max) {
1318 			if (!prev) {
1319 				i = 0;
1320 				break;
1321 			}
1322 			/*
1323 			 * No entries in use at top of swap_map,
1324 			 * loop back to start and recheck there.
1325 			 */
1326 			max = prev + 1;
1327 			prev = 0;
1328 			i = 1;
1329 		}
1330 		if (frontswap) {
1331 			if (frontswap_test(si, i))
1332 				break;
1333 			else
1334 				continue;
1335 		}
1336 		count = ACCESS_ONCE(si->swap_map[i]);
1337 		if (count && swap_count(count) != SWAP_MAP_BAD)
1338 			break;
1339 	}
1340 	return i;
1341 }
1342 
1343 /*
1344  * We completely avoid races by reading each swap page in advance,
1345  * and then search for the process using it.  All the necessary
1346  * page table adjustments can then be made atomically.
1347  *
1348  * if the boolean frontswap is true, only unuse pages_to_unuse pages;
1349  * pages_to_unuse==0 means all pages; ignored if frontswap is false
1350  */
1351 int try_to_unuse(unsigned int type, bool frontswap,
1352 		 unsigned long pages_to_unuse)
1353 {
1354 	struct swap_info_struct *si = swap_info[type];
1355 	struct mm_struct *start_mm;
1356 	volatile unsigned char *swap_map; /* swap_map is accessed without
1357 					   * locking. Mark it as volatile
1358 					   * to prevent compiler doing
1359 					   * something odd.
1360 					   */
1361 	unsigned char swcount;
1362 	struct page *page;
1363 	swp_entry_t entry;
1364 	unsigned int i = 0;
1365 	int retval = 0;
1366 
1367 	/*
1368 	 * When searching mms for an entry, a good strategy is to
1369 	 * start at the first mm we freed the previous entry from
1370 	 * (though actually we don't notice whether we or coincidence
1371 	 * freed the entry).  Initialize this start_mm with a hold.
1372 	 *
1373 	 * A simpler strategy would be to start at the last mm we
1374 	 * freed the previous entry from; but that would take less
1375 	 * advantage of mmlist ordering, which clusters forked mms
1376 	 * together, child after parent.  If we race with dup_mmap(), we
1377 	 * prefer to resolve parent before child, lest we miss entries
1378 	 * duplicated after we scanned child: using last mm would invert
1379 	 * that.
1380 	 */
1381 	start_mm = &init_mm;
1382 	atomic_inc(&init_mm.mm_users);
1383 
1384 	/*
1385 	 * Keep on scanning until all entries have gone.  Usually,
1386 	 * one pass through swap_map is enough, but not necessarily:
1387 	 * there are races when an instance of an entry might be missed.
1388 	 */
1389 	while ((i = find_next_to_unuse(si, i, frontswap)) != 0) {
1390 		if (signal_pending(current)) {
1391 			retval = -EINTR;
1392 			break;
1393 		}
1394 
1395 		/*
1396 		 * Get a page for the entry, using the existing swap
1397 		 * cache page if there is one.  Otherwise, get a clean
1398 		 * page and read the swap into it.
1399 		 */
1400 		swap_map = &si->swap_map[i];
1401 		entry = swp_entry(type, i);
1402 		page = read_swap_cache_async(entry,
1403 					GFP_HIGHUSER_MOVABLE, NULL, 0);
1404 		if (!page) {
1405 			/*
1406 			 * Either swap_duplicate() failed because entry
1407 			 * has been freed independently, and will not be
1408 			 * reused since sys_swapoff() already disabled
1409 			 * allocation from here, or alloc_page() failed.
1410 			 */
1411 			swcount = *swap_map;
1412 			/*
1413 			 * We don't hold lock here, so the swap entry could be
1414 			 * SWAP_MAP_BAD (when the cluster is discarding).
1415 			 * Instead of fail out, We can just skip the swap
1416 			 * entry because swapoff will wait for discarding
1417 			 * finish anyway.
1418 			 */
1419 			if (!swcount || swcount == SWAP_MAP_BAD)
1420 				continue;
1421 			retval = -ENOMEM;
1422 			break;
1423 		}
1424 
1425 		/*
1426 		 * Don't hold on to start_mm if it looks like exiting.
1427 		 */
1428 		if (atomic_read(&start_mm->mm_users) == 1) {
1429 			mmput(start_mm);
1430 			start_mm = &init_mm;
1431 			atomic_inc(&init_mm.mm_users);
1432 		}
1433 
1434 		/*
1435 		 * Wait for and lock page.  When do_swap_page races with
1436 		 * try_to_unuse, do_swap_page can handle the fault much
1437 		 * faster than try_to_unuse can locate the entry.  This
1438 		 * apparently redundant "wait_on_page_locked" lets try_to_unuse
1439 		 * defer to do_swap_page in such a case - in some tests,
1440 		 * do_swap_page and try_to_unuse repeatedly compete.
1441 		 */
1442 		wait_on_page_locked(page);
1443 		wait_on_page_writeback(page);
1444 		lock_page(page);
1445 		wait_on_page_writeback(page);
1446 
1447 		/*
1448 		 * Remove all references to entry.
1449 		 */
1450 		swcount = *swap_map;
1451 		if (swap_count(swcount) == SWAP_MAP_SHMEM) {
1452 			retval = shmem_unuse(entry, page);
1453 			/* page has already been unlocked and released */
1454 			if (retval < 0)
1455 				break;
1456 			continue;
1457 		}
1458 		if (swap_count(swcount) && start_mm != &init_mm)
1459 			retval = unuse_mm(start_mm, entry, page);
1460 
1461 		if (swap_count(*swap_map)) {
1462 			int set_start_mm = (*swap_map >= swcount);
1463 			struct list_head *p = &start_mm->mmlist;
1464 			struct mm_struct *new_start_mm = start_mm;
1465 			struct mm_struct *prev_mm = start_mm;
1466 			struct mm_struct *mm;
1467 
1468 			atomic_inc(&new_start_mm->mm_users);
1469 			atomic_inc(&prev_mm->mm_users);
1470 			spin_lock(&mmlist_lock);
1471 			while (swap_count(*swap_map) && !retval &&
1472 					(p = p->next) != &start_mm->mmlist) {
1473 				mm = list_entry(p, struct mm_struct, mmlist);
1474 				if (!atomic_inc_not_zero(&mm->mm_users))
1475 					continue;
1476 				spin_unlock(&mmlist_lock);
1477 				mmput(prev_mm);
1478 				prev_mm = mm;
1479 
1480 				cond_resched();
1481 
1482 				swcount = *swap_map;
1483 				if (!swap_count(swcount)) /* any usage ? */
1484 					;
1485 				else if (mm == &init_mm)
1486 					set_start_mm = 1;
1487 				else
1488 					retval = unuse_mm(mm, entry, page);
1489 
1490 				if (set_start_mm && *swap_map < swcount) {
1491 					mmput(new_start_mm);
1492 					atomic_inc(&mm->mm_users);
1493 					new_start_mm = mm;
1494 					set_start_mm = 0;
1495 				}
1496 				spin_lock(&mmlist_lock);
1497 			}
1498 			spin_unlock(&mmlist_lock);
1499 			mmput(prev_mm);
1500 			mmput(start_mm);
1501 			start_mm = new_start_mm;
1502 		}
1503 		if (retval) {
1504 			unlock_page(page);
1505 			page_cache_release(page);
1506 			break;
1507 		}
1508 
1509 		/*
1510 		 * If a reference remains (rare), we would like to leave
1511 		 * the page in the swap cache; but try_to_unmap could
1512 		 * then re-duplicate the entry once we drop page lock,
1513 		 * so we might loop indefinitely; also, that page could
1514 		 * not be swapped out to other storage meanwhile.  So:
1515 		 * delete from cache even if there's another reference,
1516 		 * after ensuring that the data has been saved to disk -
1517 		 * since if the reference remains (rarer), it will be
1518 		 * read from disk into another page.  Splitting into two
1519 		 * pages would be incorrect if swap supported "shared
1520 		 * private" pages, but they are handled by tmpfs files.
1521 		 *
1522 		 * Given how unuse_vma() targets one particular offset
1523 		 * in an anon_vma, once the anon_vma has been determined,
1524 		 * this splitting happens to be just what is needed to
1525 		 * handle where KSM pages have been swapped out: re-reading
1526 		 * is unnecessarily slow, but we can fix that later on.
1527 		 */
1528 		if (swap_count(*swap_map) &&
1529 		     PageDirty(page) && PageSwapCache(page)) {
1530 			struct writeback_control wbc = {
1531 				.sync_mode = WB_SYNC_NONE,
1532 			};
1533 
1534 			swap_writepage(page, &wbc);
1535 			lock_page(page);
1536 			wait_on_page_writeback(page);
1537 		}
1538 
1539 		/*
1540 		 * It is conceivable that a racing task removed this page from
1541 		 * swap cache just before we acquired the page lock at the top,
1542 		 * or while we dropped it in unuse_mm().  The page might even
1543 		 * be back in swap cache on another swap area: that we must not
1544 		 * delete, since it may not have been written out to swap yet.
1545 		 */
1546 		if (PageSwapCache(page) &&
1547 		    likely(page_private(page) == entry.val))
1548 			delete_from_swap_cache(page);
1549 
1550 		/*
1551 		 * So we could skip searching mms once swap count went
1552 		 * to 1, we did not mark any present ptes as dirty: must
1553 		 * mark page dirty so shrink_page_list will preserve it.
1554 		 */
1555 		SetPageDirty(page);
1556 		unlock_page(page);
1557 		page_cache_release(page);
1558 
1559 		/*
1560 		 * Make sure that we aren't completely killing
1561 		 * interactive performance.
1562 		 */
1563 		cond_resched();
1564 		if (frontswap && pages_to_unuse > 0) {
1565 			if (!--pages_to_unuse)
1566 				break;
1567 		}
1568 	}
1569 
1570 	mmput(start_mm);
1571 	return retval;
1572 }
1573 
1574 /*
1575  * After a successful try_to_unuse, if no swap is now in use, we know
1576  * we can empty the mmlist.  swap_lock must be held on entry and exit.
1577  * Note that mmlist_lock nests inside swap_lock, and an mm must be
1578  * added to the mmlist just after page_duplicate - before would be racy.
1579  */
1580 static void drain_mmlist(void)
1581 {
1582 	struct list_head *p, *next;
1583 	unsigned int type;
1584 
1585 	for (type = 0; type < nr_swapfiles; type++)
1586 		if (swap_info[type]->inuse_pages)
1587 			return;
1588 	spin_lock(&mmlist_lock);
1589 	list_for_each_safe(p, next, &init_mm.mmlist)
1590 		list_del_init(p);
1591 	spin_unlock(&mmlist_lock);
1592 }
1593 
1594 /*
1595  * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
1596  * corresponds to page offset for the specified swap entry.
1597  * Note that the type of this function is sector_t, but it returns page offset
1598  * into the bdev, not sector offset.
1599  */
1600 static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
1601 {
1602 	struct swap_info_struct *sis;
1603 	struct swap_extent *start_se;
1604 	struct swap_extent *se;
1605 	pgoff_t offset;
1606 
1607 	sis = swap_info[swp_type(entry)];
1608 	*bdev = sis->bdev;
1609 
1610 	offset = swp_offset(entry);
1611 	start_se = sis->curr_swap_extent;
1612 	se = start_se;
1613 
1614 	for ( ; ; ) {
1615 		struct list_head *lh;
1616 
1617 		if (se->start_page <= offset &&
1618 				offset < (se->start_page + se->nr_pages)) {
1619 			return se->start_block + (offset - se->start_page);
1620 		}
1621 		lh = se->list.next;
1622 		se = list_entry(lh, struct swap_extent, list);
1623 		sis->curr_swap_extent = se;
1624 		BUG_ON(se == start_se);		/* It *must* be present */
1625 	}
1626 }
1627 
1628 /*
1629  * Returns the page offset into bdev for the specified page's swap entry.
1630  */
1631 sector_t map_swap_page(struct page *page, struct block_device **bdev)
1632 {
1633 	swp_entry_t entry;
1634 	entry.val = page_private(page);
1635 	return map_swap_entry(entry, bdev);
1636 }
1637 
1638 /*
1639  * Free all of a swapdev's extent information
1640  */
1641 static void destroy_swap_extents(struct swap_info_struct *sis)
1642 {
1643 	while (!list_empty(&sis->first_swap_extent.list)) {
1644 		struct swap_extent *se;
1645 
1646 		se = list_entry(sis->first_swap_extent.list.next,
1647 				struct swap_extent, list);
1648 		list_del(&se->list);
1649 		kfree(se);
1650 	}
1651 
1652 	if (sis->flags & SWP_FILE) {
1653 		struct file *swap_file = sis->swap_file;
1654 		struct address_space *mapping = swap_file->f_mapping;
1655 
1656 		sis->flags &= ~SWP_FILE;
1657 		mapping->a_ops->swap_deactivate(swap_file);
1658 	}
1659 }
1660 
1661 /*
1662  * Add a block range (and the corresponding page range) into this swapdev's
1663  * extent list.  The extent list is kept sorted in page order.
1664  *
1665  * This function rather assumes that it is called in ascending page order.
1666  */
1667 int
1668 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
1669 		unsigned long nr_pages, sector_t start_block)
1670 {
1671 	struct swap_extent *se;
1672 	struct swap_extent *new_se;
1673 	struct list_head *lh;
1674 
1675 	if (start_page == 0) {
1676 		se = &sis->first_swap_extent;
1677 		sis->curr_swap_extent = se;
1678 		se->start_page = 0;
1679 		se->nr_pages = nr_pages;
1680 		se->start_block = start_block;
1681 		return 1;
1682 	} else {
1683 		lh = sis->first_swap_extent.list.prev;	/* Highest extent */
1684 		se = list_entry(lh, struct swap_extent, list);
1685 		BUG_ON(se->start_page + se->nr_pages != start_page);
1686 		if (se->start_block + se->nr_pages == start_block) {
1687 			/* Merge it */
1688 			se->nr_pages += nr_pages;
1689 			return 0;
1690 		}
1691 	}
1692 
1693 	/*
1694 	 * No merge.  Insert a new extent, preserving ordering.
1695 	 */
1696 	new_se = kmalloc(sizeof(*se), GFP_KERNEL);
1697 	if (new_se == NULL)
1698 		return -ENOMEM;
1699 	new_se->start_page = start_page;
1700 	new_se->nr_pages = nr_pages;
1701 	new_se->start_block = start_block;
1702 
1703 	list_add_tail(&new_se->list, &sis->first_swap_extent.list);
1704 	return 1;
1705 }
1706 
1707 /*
1708  * A `swap extent' is a simple thing which maps a contiguous range of pages
1709  * onto a contiguous range of disk blocks.  An ordered list of swap extents
1710  * is built at swapon time and is then used at swap_writepage/swap_readpage
1711  * time for locating where on disk a page belongs.
1712  *
1713  * If the swapfile is an S_ISBLK block device, a single extent is installed.
1714  * This is done so that the main operating code can treat S_ISBLK and S_ISREG
1715  * swap files identically.
1716  *
1717  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
1718  * extent list operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
1719  * swapfiles are handled *identically* after swapon time.
1720  *
1721  * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
1722  * and will parse them into an ordered extent list, in PAGE_SIZE chunks.  If
1723  * some stray blocks are found which do not fall within the PAGE_SIZE alignment
1724  * requirements, they are simply tossed out - we will never use those blocks
1725  * for swapping.
1726  *
1727  * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon.  This
1728  * prevents root from shooting her foot off by ftruncating an in-use swapfile,
1729  * which will scribble on the fs.
1730  *
1731  * The amount of disk space which a single swap extent represents varies.
1732  * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
1733  * extents in the list.  To avoid much list walking, we cache the previous
1734  * search location in `curr_swap_extent', and start new searches from there.
1735  * This is extremely effective.  The average number of iterations in
1736  * map_swap_page() has been measured at about 0.3 per page.  - akpm.
1737  */
1738 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
1739 {
1740 	struct file *swap_file = sis->swap_file;
1741 	struct address_space *mapping = swap_file->f_mapping;
1742 	struct inode *inode = mapping->host;
1743 	int ret;
1744 
1745 	if (S_ISBLK(inode->i_mode)) {
1746 		ret = add_swap_extent(sis, 0, sis->max, 0);
1747 		*span = sis->pages;
1748 		return ret;
1749 	}
1750 
1751 	if (mapping->a_ops->swap_activate) {
1752 		ret = mapping->a_ops->swap_activate(sis, swap_file, span);
1753 		if (!ret) {
1754 			sis->flags |= SWP_FILE;
1755 			ret = add_swap_extent(sis, 0, sis->max, 0);
1756 			*span = sis->pages;
1757 		}
1758 		return ret;
1759 	}
1760 
1761 	return generic_swapfile_activate(sis, swap_file, span);
1762 }
1763 
1764 static void _enable_swap_info(struct swap_info_struct *p, int prio,
1765 				unsigned char *swap_map,
1766 				struct swap_cluster_info *cluster_info)
1767 {
1768 	int i, prev;
1769 
1770 	if (prio >= 0)
1771 		p->prio = prio;
1772 	else
1773 		p->prio = --least_priority;
1774 	p->swap_map = swap_map;
1775 	p->cluster_info = cluster_info;
1776 	p->flags |= SWP_WRITEOK;
1777 	atomic_long_add(p->pages, &nr_swap_pages);
1778 	total_swap_pages += p->pages;
1779 
1780 	/* insert swap space into swap_list: */
1781 	prev = -1;
1782 	for (i = swap_list.head; i >= 0; i = swap_info[i]->next) {
1783 		if (p->prio >= swap_info[i]->prio)
1784 			break;
1785 		prev = i;
1786 	}
1787 	p->next = i;
1788 	if (prev < 0)
1789 		swap_list.head = swap_list.next = p->type;
1790 	else
1791 		swap_info[prev]->next = p->type;
1792 }
1793 
1794 static void enable_swap_info(struct swap_info_struct *p, int prio,
1795 				unsigned char *swap_map,
1796 				struct swap_cluster_info *cluster_info,
1797 				unsigned long *frontswap_map)
1798 {
1799 	frontswap_init(p->type, frontswap_map);
1800 	spin_lock(&swap_lock);
1801 	spin_lock(&p->lock);
1802 	 _enable_swap_info(p, prio, swap_map, cluster_info);
1803 	spin_unlock(&p->lock);
1804 	spin_unlock(&swap_lock);
1805 }
1806 
1807 static void reinsert_swap_info(struct swap_info_struct *p)
1808 {
1809 	spin_lock(&swap_lock);
1810 	spin_lock(&p->lock);
1811 	_enable_swap_info(p, p->prio, p->swap_map, p->cluster_info);
1812 	spin_unlock(&p->lock);
1813 	spin_unlock(&swap_lock);
1814 }
1815 
1816 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
1817 {
1818 	struct swap_info_struct *p = NULL;
1819 	unsigned char *swap_map;
1820 	struct swap_cluster_info *cluster_info;
1821 	unsigned long *frontswap_map;
1822 	struct file *swap_file, *victim;
1823 	struct address_space *mapping;
1824 	struct inode *inode;
1825 	struct filename *pathname;
1826 	int i, type, prev;
1827 	int err;
1828 	unsigned int old_block_size;
1829 
1830 	if (!capable(CAP_SYS_ADMIN))
1831 		return -EPERM;
1832 
1833 	BUG_ON(!current->mm);
1834 
1835 	pathname = getname(specialfile);
1836 	if (IS_ERR(pathname))
1837 		return PTR_ERR(pathname);
1838 
1839 	victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
1840 	err = PTR_ERR(victim);
1841 	if (IS_ERR(victim))
1842 		goto out;
1843 
1844 	mapping = victim->f_mapping;
1845 	prev = -1;
1846 	spin_lock(&swap_lock);
1847 	for (type = swap_list.head; type >= 0; type = swap_info[type]->next) {
1848 		p = swap_info[type];
1849 		if (p->flags & SWP_WRITEOK) {
1850 			if (p->swap_file->f_mapping == mapping)
1851 				break;
1852 		}
1853 		prev = type;
1854 	}
1855 	if (type < 0) {
1856 		err = -EINVAL;
1857 		spin_unlock(&swap_lock);
1858 		goto out_dput;
1859 	}
1860 	if (!security_vm_enough_memory_mm(current->mm, p->pages))
1861 		vm_unacct_memory(p->pages);
1862 	else {
1863 		err = -ENOMEM;
1864 		spin_unlock(&swap_lock);
1865 		goto out_dput;
1866 	}
1867 	if (prev < 0)
1868 		swap_list.head = p->next;
1869 	else
1870 		swap_info[prev]->next = p->next;
1871 	if (type == swap_list.next) {
1872 		/* just pick something that's safe... */
1873 		swap_list.next = swap_list.head;
1874 	}
1875 	spin_lock(&p->lock);
1876 	if (p->prio < 0) {
1877 		for (i = p->next; i >= 0; i = swap_info[i]->next)
1878 			swap_info[i]->prio = p->prio--;
1879 		least_priority++;
1880 	}
1881 	atomic_long_sub(p->pages, &nr_swap_pages);
1882 	total_swap_pages -= p->pages;
1883 	p->flags &= ~SWP_WRITEOK;
1884 	spin_unlock(&p->lock);
1885 	spin_unlock(&swap_lock);
1886 
1887 	set_current_oom_origin();
1888 	err = try_to_unuse(type, false, 0); /* force all pages to be unused */
1889 	clear_current_oom_origin();
1890 
1891 	if (err) {
1892 		/* re-insert swap space back into swap_list */
1893 		reinsert_swap_info(p);
1894 		goto out_dput;
1895 	}
1896 
1897 	flush_work(&p->discard_work);
1898 
1899 	destroy_swap_extents(p);
1900 	if (p->flags & SWP_CONTINUED)
1901 		free_swap_count_continuations(p);
1902 
1903 	mutex_lock(&swapon_mutex);
1904 	spin_lock(&swap_lock);
1905 	spin_lock(&p->lock);
1906 	drain_mmlist();
1907 
1908 	/* wait for anyone still in scan_swap_map */
1909 	p->highest_bit = 0;		/* cuts scans short */
1910 	while (p->flags >= SWP_SCANNING) {
1911 		spin_unlock(&p->lock);
1912 		spin_unlock(&swap_lock);
1913 		schedule_timeout_uninterruptible(1);
1914 		spin_lock(&swap_lock);
1915 		spin_lock(&p->lock);
1916 	}
1917 
1918 	swap_file = p->swap_file;
1919 	old_block_size = p->old_block_size;
1920 	p->swap_file = NULL;
1921 	p->max = 0;
1922 	swap_map = p->swap_map;
1923 	p->swap_map = NULL;
1924 	cluster_info = p->cluster_info;
1925 	p->cluster_info = NULL;
1926 	p->flags = 0;
1927 	frontswap_map = frontswap_map_get(p);
1928 	spin_unlock(&p->lock);
1929 	spin_unlock(&swap_lock);
1930 	frontswap_invalidate_area(type);
1931 	frontswap_map_set(p, NULL);
1932 	mutex_unlock(&swapon_mutex);
1933 	free_percpu(p->percpu_cluster);
1934 	p->percpu_cluster = NULL;
1935 	vfree(swap_map);
1936 	vfree(cluster_info);
1937 	vfree(frontswap_map);
1938 	/* Destroy swap account information */
1939 	swap_cgroup_swapoff(type);
1940 
1941 	inode = mapping->host;
1942 	if (S_ISBLK(inode->i_mode)) {
1943 		struct block_device *bdev = I_BDEV(inode);
1944 		set_blocksize(bdev, old_block_size);
1945 		blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
1946 	} else {
1947 		mutex_lock(&inode->i_mutex);
1948 		inode->i_flags &= ~S_SWAPFILE;
1949 		mutex_unlock(&inode->i_mutex);
1950 	}
1951 	filp_close(swap_file, NULL);
1952 	err = 0;
1953 	atomic_inc(&proc_poll_event);
1954 	wake_up_interruptible(&proc_poll_wait);
1955 
1956 out_dput:
1957 	filp_close(victim, NULL);
1958 out:
1959 	putname(pathname);
1960 	return err;
1961 }
1962 
1963 #ifdef CONFIG_PROC_FS
1964 static unsigned swaps_poll(struct file *file, poll_table *wait)
1965 {
1966 	struct seq_file *seq = file->private_data;
1967 
1968 	poll_wait(file, &proc_poll_wait, wait);
1969 
1970 	if (seq->poll_event != atomic_read(&proc_poll_event)) {
1971 		seq->poll_event = atomic_read(&proc_poll_event);
1972 		return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
1973 	}
1974 
1975 	return POLLIN | POLLRDNORM;
1976 }
1977 
1978 /* iterator */
1979 static void *swap_start(struct seq_file *swap, loff_t *pos)
1980 {
1981 	struct swap_info_struct *si;
1982 	int type;
1983 	loff_t l = *pos;
1984 
1985 	mutex_lock(&swapon_mutex);
1986 
1987 	if (!l)
1988 		return SEQ_START_TOKEN;
1989 
1990 	for (type = 0; type < nr_swapfiles; type++) {
1991 		smp_rmb();	/* read nr_swapfiles before swap_info[type] */
1992 		si = swap_info[type];
1993 		if (!(si->flags & SWP_USED) || !si->swap_map)
1994 			continue;
1995 		if (!--l)
1996 			return si;
1997 	}
1998 
1999 	return NULL;
2000 }
2001 
2002 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
2003 {
2004 	struct swap_info_struct *si = v;
2005 	int type;
2006 
2007 	if (v == SEQ_START_TOKEN)
2008 		type = 0;
2009 	else
2010 		type = si->type + 1;
2011 
2012 	for (; type < nr_swapfiles; type++) {
2013 		smp_rmb();	/* read nr_swapfiles before swap_info[type] */
2014 		si = swap_info[type];
2015 		if (!(si->flags & SWP_USED) || !si->swap_map)
2016 			continue;
2017 		++*pos;
2018 		return si;
2019 	}
2020 
2021 	return NULL;
2022 }
2023 
2024 static void swap_stop(struct seq_file *swap, void *v)
2025 {
2026 	mutex_unlock(&swapon_mutex);
2027 }
2028 
2029 static int swap_show(struct seq_file *swap, void *v)
2030 {
2031 	struct swap_info_struct *si = v;
2032 	struct file *file;
2033 	int len;
2034 
2035 	if (si == SEQ_START_TOKEN) {
2036 		seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
2037 		return 0;
2038 	}
2039 
2040 	file = si->swap_file;
2041 	len = seq_path(swap, &file->f_path, " \t\n\\");
2042 	seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
2043 			len < 40 ? 40 - len : 1, " ",
2044 			S_ISBLK(file_inode(file)->i_mode) ?
2045 				"partition" : "file\t",
2046 			si->pages << (PAGE_SHIFT - 10),
2047 			si->inuse_pages << (PAGE_SHIFT - 10),
2048 			si->prio);
2049 	return 0;
2050 }
2051 
2052 static const struct seq_operations swaps_op = {
2053 	.start =	swap_start,
2054 	.next =		swap_next,
2055 	.stop =		swap_stop,
2056 	.show =		swap_show
2057 };
2058 
2059 static int swaps_open(struct inode *inode, struct file *file)
2060 {
2061 	struct seq_file *seq;
2062 	int ret;
2063 
2064 	ret = seq_open(file, &swaps_op);
2065 	if (ret)
2066 		return ret;
2067 
2068 	seq = file->private_data;
2069 	seq->poll_event = atomic_read(&proc_poll_event);
2070 	return 0;
2071 }
2072 
2073 static const struct file_operations proc_swaps_operations = {
2074 	.open		= swaps_open,
2075 	.read		= seq_read,
2076 	.llseek		= seq_lseek,
2077 	.release	= seq_release,
2078 	.poll		= swaps_poll,
2079 };
2080 
2081 static int __init procswaps_init(void)
2082 {
2083 	proc_create("swaps", 0, NULL, &proc_swaps_operations);
2084 	return 0;
2085 }
2086 __initcall(procswaps_init);
2087 #endif /* CONFIG_PROC_FS */
2088 
2089 #ifdef MAX_SWAPFILES_CHECK
2090 static int __init max_swapfiles_check(void)
2091 {
2092 	MAX_SWAPFILES_CHECK();
2093 	return 0;
2094 }
2095 late_initcall(max_swapfiles_check);
2096 #endif
2097 
2098 static struct swap_info_struct *alloc_swap_info(void)
2099 {
2100 	struct swap_info_struct *p;
2101 	unsigned int type;
2102 
2103 	p = kzalloc(sizeof(*p), GFP_KERNEL);
2104 	if (!p)
2105 		return ERR_PTR(-ENOMEM);
2106 
2107 	spin_lock(&swap_lock);
2108 	for (type = 0; type < nr_swapfiles; type++) {
2109 		if (!(swap_info[type]->flags & SWP_USED))
2110 			break;
2111 	}
2112 	if (type >= MAX_SWAPFILES) {
2113 		spin_unlock(&swap_lock);
2114 		kfree(p);
2115 		return ERR_PTR(-EPERM);
2116 	}
2117 	if (type >= nr_swapfiles) {
2118 		p->type = type;
2119 		swap_info[type] = p;
2120 		/*
2121 		 * Write swap_info[type] before nr_swapfiles, in case a
2122 		 * racing procfs swap_start() or swap_next() is reading them.
2123 		 * (We never shrink nr_swapfiles, we never free this entry.)
2124 		 */
2125 		smp_wmb();
2126 		nr_swapfiles++;
2127 	} else {
2128 		kfree(p);
2129 		p = swap_info[type];
2130 		/*
2131 		 * Do not memset this entry: a racing procfs swap_next()
2132 		 * would be relying on p->type to remain valid.
2133 		 */
2134 	}
2135 	INIT_LIST_HEAD(&p->first_swap_extent.list);
2136 	p->flags = SWP_USED;
2137 	p->next = -1;
2138 	spin_unlock(&swap_lock);
2139 	spin_lock_init(&p->lock);
2140 
2141 	return p;
2142 }
2143 
2144 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
2145 {
2146 	int error;
2147 
2148 	if (S_ISBLK(inode->i_mode)) {
2149 		p->bdev = bdgrab(I_BDEV(inode));
2150 		error = blkdev_get(p->bdev,
2151 				   FMODE_READ | FMODE_WRITE | FMODE_EXCL,
2152 				   sys_swapon);
2153 		if (error < 0) {
2154 			p->bdev = NULL;
2155 			return -EINVAL;
2156 		}
2157 		p->old_block_size = block_size(p->bdev);
2158 		error = set_blocksize(p->bdev, PAGE_SIZE);
2159 		if (error < 0)
2160 			return error;
2161 		p->flags |= SWP_BLKDEV;
2162 	} else if (S_ISREG(inode->i_mode)) {
2163 		p->bdev = inode->i_sb->s_bdev;
2164 		mutex_lock(&inode->i_mutex);
2165 		if (IS_SWAPFILE(inode))
2166 			return -EBUSY;
2167 	} else
2168 		return -EINVAL;
2169 
2170 	return 0;
2171 }
2172 
2173 static unsigned long read_swap_header(struct swap_info_struct *p,
2174 					union swap_header *swap_header,
2175 					struct inode *inode)
2176 {
2177 	int i;
2178 	unsigned long maxpages;
2179 	unsigned long swapfilepages;
2180 	unsigned long last_page;
2181 
2182 	if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2183 		pr_err("Unable to find swap-space signature\n");
2184 		return 0;
2185 	}
2186 
2187 	/* swap partition endianess hack... */
2188 	if (swab32(swap_header->info.version) == 1) {
2189 		swab32s(&swap_header->info.version);
2190 		swab32s(&swap_header->info.last_page);
2191 		swab32s(&swap_header->info.nr_badpages);
2192 		for (i = 0; i < swap_header->info.nr_badpages; i++)
2193 			swab32s(&swap_header->info.badpages[i]);
2194 	}
2195 	/* Check the swap header's sub-version */
2196 	if (swap_header->info.version != 1) {
2197 		pr_warn("Unable to handle swap header version %d\n",
2198 			swap_header->info.version);
2199 		return 0;
2200 	}
2201 
2202 	p->lowest_bit  = 1;
2203 	p->cluster_next = 1;
2204 	p->cluster_nr = 0;
2205 
2206 	/*
2207 	 * Find out how many pages are allowed for a single swap
2208 	 * device. There are two limiting factors: 1) the number
2209 	 * of bits for the swap offset in the swp_entry_t type, and
2210 	 * 2) the number of bits in the swap pte as defined by the
2211 	 * different architectures. In order to find the
2212 	 * largest possible bit mask, a swap entry with swap type 0
2213 	 * and swap offset ~0UL is created, encoded to a swap pte,
2214 	 * decoded to a swp_entry_t again, and finally the swap
2215 	 * offset is extracted. This will mask all the bits from
2216 	 * the initial ~0UL mask that can't be encoded in either
2217 	 * the swp_entry_t or the architecture definition of a
2218 	 * swap pte.
2219 	 */
2220 	maxpages = swp_offset(pte_to_swp_entry(
2221 			swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2222 	last_page = swap_header->info.last_page;
2223 	if (last_page > maxpages) {
2224 		pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2225 			maxpages << (PAGE_SHIFT - 10),
2226 			last_page << (PAGE_SHIFT - 10));
2227 	}
2228 	if (maxpages > last_page) {
2229 		maxpages = last_page + 1;
2230 		/* p->max is an unsigned int: don't overflow it */
2231 		if ((unsigned int)maxpages == 0)
2232 			maxpages = UINT_MAX;
2233 	}
2234 	p->highest_bit = maxpages - 1;
2235 
2236 	if (!maxpages)
2237 		return 0;
2238 	swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
2239 	if (swapfilepages && maxpages > swapfilepages) {
2240 		pr_warn("Swap area shorter than signature indicates\n");
2241 		return 0;
2242 	}
2243 	if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
2244 		return 0;
2245 	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2246 		return 0;
2247 
2248 	return maxpages;
2249 }
2250 
2251 static int setup_swap_map_and_extents(struct swap_info_struct *p,
2252 					union swap_header *swap_header,
2253 					unsigned char *swap_map,
2254 					struct swap_cluster_info *cluster_info,
2255 					unsigned long maxpages,
2256 					sector_t *span)
2257 {
2258 	int i;
2259 	unsigned int nr_good_pages;
2260 	int nr_extents;
2261 	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2262 	unsigned long idx = p->cluster_next / SWAPFILE_CLUSTER;
2263 
2264 	nr_good_pages = maxpages - 1;	/* omit header page */
2265 
2266 	cluster_set_null(&p->free_cluster_head);
2267 	cluster_set_null(&p->free_cluster_tail);
2268 	cluster_set_null(&p->discard_cluster_head);
2269 	cluster_set_null(&p->discard_cluster_tail);
2270 
2271 	for (i = 0; i < swap_header->info.nr_badpages; i++) {
2272 		unsigned int page_nr = swap_header->info.badpages[i];
2273 		if (page_nr == 0 || page_nr > swap_header->info.last_page)
2274 			return -EINVAL;
2275 		if (page_nr < maxpages) {
2276 			swap_map[page_nr] = SWAP_MAP_BAD;
2277 			nr_good_pages--;
2278 			/*
2279 			 * Haven't marked the cluster free yet, no list
2280 			 * operation involved
2281 			 */
2282 			inc_cluster_info_page(p, cluster_info, page_nr);
2283 		}
2284 	}
2285 
2286 	/* Haven't marked the cluster free yet, no list operation involved */
2287 	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
2288 		inc_cluster_info_page(p, cluster_info, i);
2289 
2290 	if (nr_good_pages) {
2291 		swap_map[0] = SWAP_MAP_BAD;
2292 		/*
2293 		 * Not mark the cluster free yet, no list
2294 		 * operation involved
2295 		 */
2296 		inc_cluster_info_page(p, cluster_info, 0);
2297 		p->max = maxpages;
2298 		p->pages = nr_good_pages;
2299 		nr_extents = setup_swap_extents(p, span);
2300 		if (nr_extents < 0)
2301 			return nr_extents;
2302 		nr_good_pages = p->pages;
2303 	}
2304 	if (!nr_good_pages) {
2305 		pr_warn("Empty swap-file\n");
2306 		return -EINVAL;
2307 	}
2308 
2309 	if (!cluster_info)
2310 		return nr_extents;
2311 
2312 	for (i = 0; i < nr_clusters; i++) {
2313 		if (!cluster_count(&cluster_info[idx])) {
2314 			cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
2315 			if (cluster_is_null(&p->free_cluster_head)) {
2316 				cluster_set_next_flag(&p->free_cluster_head,
2317 								idx, 0);
2318 				cluster_set_next_flag(&p->free_cluster_tail,
2319 								idx, 0);
2320 			} else {
2321 				unsigned int tail;
2322 
2323 				tail = cluster_next(&p->free_cluster_tail);
2324 				cluster_set_next(&cluster_info[tail], idx);
2325 				cluster_set_next_flag(&p->free_cluster_tail,
2326 								idx, 0);
2327 			}
2328 		}
2329 		idx++;
2330 		if (idx == nr_clusters)
2331 			idx = 0;
2332 	}
2333 	return nr_extents;
2334 }
2335 
2336 /*
2337  * Helper to sys_swapon determining if a given swap
2338  * backing device queue supports DISCARD operations.
2339  */
2340 static bool swap_discardable(struct swap_info_struct *si)
2341 {
2342 	struct request_queue *q = bdev_get_queue(si->bdev);
2343 
2344 	if (!q || !blk_queue_discard(q))
2345 		return false;
2346 
2347 	return true;
2348 }
2349 
2350 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
2351 {
2352 	struct swap_info_struct *p;
2353 	struct filename *name;
2354 	struct file *swap_file = NULL;
2355 	struct address_space *mapping;
2356 	int i;
2357 	int prio;
2358 	int error;
2359 	union swap_header *swap_header;
2360 	int nr_extents;
2361 	sector_t span;
2362 	unsigned long maxpages;
2363 	unsigned char *swap_map = NULL;
2364 	struct swap_cluster_info *cluster_info = NULL;
2365 	unsigned long *frontswap_map = NULL;
2366 	struct page *page = NULL;
2367 	struct inode *inode = NULL;
2368 
2369 	if (swap_flags & ~SWAP_FLAGS_VALID)
2370 		return -EINVAL;
2371 
2372 	if (!capable(CAP_SYS_ADMIN))
2373 		return -EPERM;
2374 
2375 	p = alloc_swap_info();
2376 	if (IS_ERR(p))
2377 		return PTR_ERR(p);
2378 
2379 	INIT_WORK(&p->discard_work, swap_discard_work);
2380 
2381 	name = getname(specialfile);
2382 	if (IS_ERR(name)) {
2383 		error = PTR_ERR(name);
2384 		name = NULL;
2385 		goto bad_swap;
2386 	}
2387 	swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
2388 	if (IS_ERR(swap_file)) {
2389 		error = PTR_ERR(swap_file);
2390 		swap_file = NULL;
2391 		goto bad_swap;
2392 	}
2393 
2394 	p->swap_file = swap_file;
2395 	mapping = swap_file->f_mapping;
2396 
2397 	for (i = 0; i < nr_swapfiles; i++) {
2398 		struct swap_info_struct *q = swap_info[i];
2399 
2400 		if (q == p || !q->swap_file)
2401 			continue;
2402 		if (mapping == q->swap_file->f_mapping) {
2403 			error = -EBUSY;
2404 			goto bad_swap;
2405 		}
2406 	}
2407 
2408 	inode = mapping->host;
2409 	/* If S_ISREG(inode->i_mode) will do mutex_lock(&inode->i_mutex); */
2410 	error = claim_swapfile(p, inode);
2411 	if (unlikely(error))
2412 		goto bad_swap;
2413 
2414 	/*
2415 	 * Read the swap header.
2416 	 */
2417 	if (!mapping->a_ops->readpage) {
2418 		error = -EINVAL;
2419 		goto bad_swap;
2420 	}
2421 	page = read_mapping_page(mapping, 0, swap_file);
2422 	if (IS_ERR(page)) {
2423 		error = PTR_ERR(page);
2424 		goto bad_swap;
2425 	}
2426 	swap_header = kmap(page);
2427 
2428 	maxpages = read_swap_header(p, swap_header, inode);
2429 	if (unlikely(!maxpages)) {
2430 		error = -EINVAL;
2431 		goto bad_swap;
2432 	}
2433 
2434 	/* OK, set up the swap map and apply the bad block list */
2435 	swap_map = vzalloc(maxpages);
2436 	if (!swap_map) {
2437 		error = -ENOMEM;
2438 		goto bad_swap;
2439 	}
2440 	if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
2441 		p->flags |= SWP_SOLIDSTATE;
2442 		/*
2443 		 * select a random position to start with to help wear leveling
2444 		 * SSD
2445 		 */
2446 		p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
2447 
2448 		cluster_info = vzalloc(DIV_ROUND_UP(maxpages,
2449 			SWAPFILE_CLUSTER) * sizeof(*cluster_info));
2450 		if (!cluster_info) {
2451 			error = -ENOMEM;
2452 			goto bad_swap;
2453 		}
2454 		p->percpu_cluster = alloc_percpu(struct percpu_cluster);
2455 		if (!p->percpu_cluster) {
2456 			error = -ENOMEM;
2457 			goto bad_swap;
2458 		}
2459 		for_each_possible_cpu(i) {
2460 			struct percpu_cluster *cluster;
2461 			cluster = per_cpu_ptr(p->percpu_cluster, i);
2462 			cluster_set_null(&cluster->index);
2463 		}
2464 	}
2465 
2466 	error = swap_cgroup_swapon(p->type, maxpages);
2467 	if (error)
2468 		goto bad_swap;
2469 
2470 	nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
2471 		cluster_info, maxpages, &span);
2472 	if (unlikely(nr_extents < 0)) {
2473 		error = nr_extents;
2474 		goto bad_swap;
2475 	}
2476 	/* frontswap enabled? set up bit-per-page map for frontswap */
2477 	if (frontswap_enabled)
2478 		frontswap_map = vzalloc(BITS_TO_LONGS(maxpages) * sizeof(long));
2479 
2480 	if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {
2481 		/*
2482 		 * When discard is enabled for swap with no particular
2483 		 * policy flagged, we set all swap discard flags here in
2484 		 * order to sustain backward compatibility with older
2485 		 * swapon(8) releases.
2486 		 */
2487 		p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
2488 			     SWP_PAGE_DISCARD);
2489 
2490 		/*
2491 		 * By flagging sys_swapon, a sysadmin can tell us to
2492 		 * either do single-time area discards only, or to just
2493 		 * perform discards for released swap page-clusters.
2494 		 * Now it's time to adjust the p->flags accordingly.
2495 		 */
2496 		if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
2497 			p->flags &= ~SWP_PAGE_DISCARD;
2498 		else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
2499 			p->flags &= ~SWP_AREA_DISCARD;
2500 
2501 		/* issue a swapon-time discard if it's still required */
2502 		if (p->flags & SWP_AREA_DISCARD) {
2503 			int err = discard_swap(p);
2504 			if (unlikely(err))
2505 				pr_err("swapon: discard_swap(%p): %d\n",
2506 					p, err);
2507 		}
2508 	}
2509 
2510 	mutex_lock(&swapon_mutex);
2511 	prio = -1;
2512 	if (swap_flags & SWAP_FLAG_PREFER)
2513 		prio =
2514 		  (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
2515 	enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
2516 
2517 	pr_info("Adding %uk swap on %s.  "
2518 			"Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2519 		p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
2520 		nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
2521 		(p->flags & SWP_SOLIDSTATE) ? "SS" : "",
2522 		(p->flags & SWP_DISCARDABLE) ? "D" : "",
2523 		(p->flags & SWP_AREA_DISCARD) ? "s" : "",
2524 		(p->flags & SWP_PAGE_DISCARD) ? "c" : "",
2525 		(frontswap_map) ? "FS" : "");
2526 
2527 	mutex_unlock(&swapon_mutex);
2528 	atomic_inc(&proc_poll_event);
2529 	wake_up_interruptible(&proc_poll_wait);
2530 
2531 	if (S_ISREG(inode->i_mode))
2532 		inode->i_flags |= S_SWAPFILE;
2533 	error = 0;
2534 	goto out;
2535 bad_swap:
2536 	free_percpu(p->percpu_cluster);
2537 	p->percpu_cluster = NULL;
2538 	if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
2539 		set_blocksize(p->bdev, p->old_block_size);
2540 		blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2541 	}
2542 	destroy_swap_extents(p);
2543 	swap_cgroup_swapoff(p->type);
2544 	spin_lock(&swap_lock);
2545 	p->swap_file = NULL;
2546 	p->flags = 0;
2547 	spin_unlock(&swap_lock);
2548 	vfree(swap_map);
2549 	vfree(cluster_info);
2550 	if (swap_file) {
2551 		if (inode && S_ISREG(inode->i_mode)) {
2552 			mutex_unlock(&inode->i_mutex);
2553 			inode = NULL;
2554 		}
2555 		filp_close(swap_file, NULL);
2556 	}
2557 out:
2558 	if (page && !IS_ERR(page)) {
2559 		kunmap(page);
2560 		page_cache_release(page);
2561 	}
2562 	if (name)
2563 		putname(name);
2564 	if (inode && S_ISREG(inode->i_mode))
2565 		mutex_unlock(&inode->i_mutex);
2566 	return error;
2567 }
2568 
2569 void si_swapinfo(struct sysinfo *val)
2570 {
2571 	unsigned int type;
2572 	unsigned long nr_to_be_unused = 0;
2573 
2574 	spin_lock(&swap_lock);
2575 	for (type = 0; type < nr_swapfiles; type++) {
2576 		struct swap_info_struct *si = swap_info[type];
2577 
2578 		if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
2579 			nr_to_be_unused += si->inuse_pages;
2580 	}
2581 	val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
2582 	val->totalswap = total_swap_pages + nr_to_be_unused;
2583 	spin_unlock(&swap_lock);
2584 }
2585 
2586 /*
2587  * Verify that a swap entry is valid and increment its swap map count.
2588  *
2589  * Returns error code in following case.
2590  * - success -> 0
2591  * - swp_entry is invalid -> EINVAL
2592  * - swp_entry is migration entry -> EINVAL
2593  * - swap-cache reference is requested but there is already one. -> EEXIST
2594  * - swap-cache reference is requested but the entry is not used. -> ENOENT
2595  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
2596  */
2597 static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
2598 {
2599 	struct swap_info_struct *p;
2600 	unsigned long offset, type;
2601 	unsigned char count;
2602 	unsigned char has_cache;
2603 	int err = -EINVAL;
2604 
2605 	if (non_swap_entry(entry))
2606 		goto out;
2607 
2608 	type = swp_type(entry);
2609 	if (type >= nr_swapfiles)
2610 		goto bad_file;
2611 	p = swap_info[type];
2612 	offset = swp_offset(entry);
2613 
2614 	spin_lock(&p->lock);
2615 	if (unlikely(offset >= p->max))
2616 		goto unlock_out;
2617 
2618 	count = p->swap_map[offset];
2619 
2620 	/*
2621 	 * swapin_readahead() doesn't check if a swap entry is valid, so the
2622 	 * swap entry could be SWAP_MAP_BAD. Check here with lock held.
2623 	 */
2624 	if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
2625 		err = -ENOENT;
2626 		goto unlock_out;
2627 	}
2628 
2629 	has_cache = count & SWAP_HAS_CACHE;
2630 	count &= ~SWAP_HAS_CACHE;
2631 	err = 0;
2632 
2633 	if (usage == SWAP_HAS_CACHE) {
2634 
2635 		/* set SWAP_HAS_CACHE if there is no cache and entry is used */
2636 		if (!has_cache && count)
2637 			has_cache = SWAP_HAS_CACHE;
2638 		else if (has_cache)		/* someone else added cache */
2639 			err = -EEXIST;
2640 		else				/* no users remaining */
2641 			err = -ENOENT;
2642 
2643 	} else if (count || has_cache) {
2644 
2645 		if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
2646 			count += usage;
2647 		else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
2648 			err = -EINVAL;
2649 		else if (swap_count_continued(p, offset, count))
2650 			count = COUNT_CONTINUED;
2651 		else
2652 			err = -ENOMEM;
2653 	} else
2654 		err = -ENOENT;			/* unused swap entry */
2655 
2656 	p->swap_map[offset] = count | has_cache;
2657 
2658 unlock_out:
2659 	spin_unlock(&p->lock);
2660 out:
2661 	return err;
2662 
2663 bad_file:
2664 	pr_err("swap_dup: %s%08lx\n", Bad_file, entry.val);
2665 	goto out;
2666 }
2667 
2668 /*
2669  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
2670  * (in which case its reference count is never incremented).
2671  */
2672 void swap_shmem_alloc(swp_entry_t entry)
2673 {
2674 	__swap_duplicate(entry, SWAP_MAP_SHMEM);
2675 }
2676 
2677 /*
2678  * Increase reference count of swap entry by 1.
2679  * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
2680  * but could not be atomically allocated.  Returns 0, just as if it succeeded,
2681  * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
2682  * might occur if a page table entry has got corrupted.
2683  */
2684 int swap_duplicate(swp_entry_t entry)
2685 {
2686 	int err = 0;
2687 
2688 	while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
2689 		err = add_swap_count_continuation(entry, GFP_ATOMIC);
2690 	return err;
2691 }
2692 
2693 /*
2694  * @entry: swap entry for which we allocate swap cache.
2695  *
2696  * Called when allocating swap cache for existing swap entry,
2697  * This can return error codes. Returns 0 at success.
2698  * -EBUSY means there is a swap cache.
2699  * Note: return code is different from swap_duplicate().
2700  */
2701 int swapcache_prepare(swp_entry_t entry)
2702 {
2703 	return __swap_duplicate(entry, SWAP_HAS_CACHE);
2704 }
2705 
2706 struct swap_info_struct *page_swap_info(struct page *page)
2707 {
2708 	swp_entry_t swap = { .val = page_private(page) };
2709 	BUG_ON(!PageSwapCache(page));
2710 	return swap_info[swp_type(swap)];
2711 }
2712 
2713 /*
2714  * out-of-line __page_file_ methods to avoid include hell.
2715  */
2716 struct address_space *__page_file_mapping(struct page *page)
2717 {
2718 	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2719 	return page_swap_info(page)->swap_file->f_mapping;
2720 }
2721 EXPORT_SYMBOL_GPL(__page_file_mapping);
2722 
2723 pgoff_t __page_file_index(struct page *page)
2724 {
2725 	swp_entry_t swap = { .val = page_private(page) };
2726 	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2727 	return swp_offset(swap);
2728 }
2729 EXPORT_SYMBOL_GPL(__page_file_index);
2730 
2731 /*
2732  * add_swap_count_continuation - called when a swap count is duplicated
2733  * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
2734  * page of the original vmalloc'ed swap_map, to hold the continuation count
2735  * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
2736  * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
2737  *
2738  * These continuation pages are seldom referenced: the common paths all work
2739  * on the original swap_map, only referring to a continuation page when the
2740  * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
2741  *
2742  * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
2743  * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
2744  * can be called after dropping locks.
2745  */
2746 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
2747 {
2748 	struct swap_info_struct *si;
2749 	struct page *head;
2750 	struct page *page;
2751 	struct page *list_page;
2752 	pgoff_t offset;
2753 	unsigned char count;
2754 
2755 	/*
2756 	 * When debugging, it's easier to use __GFP_ZERO here; but it's better
2757 	 * for latency not to zero a page while GFP_ATOMIC and holding locks.
2758 	 */
2759 	page = alloc_page(gfp_mask | __GFP_HIGHMEM);
2760 
2761 	si = swap_info_get(entry);
2762 	if (!si) {
2763 		/*
2764 		 * An acceptable race has occurred since the failing
2765 		 * __swap_duplicate(): the swap entry has been freed,
2766 		 * perhaps even the whole swap_map cleared for swapoff.
2767 		 */
2768 		goto outer;
2769 	}
2770 
2771 	offset = swp_offset(entry);
2772 	count = si->swap_map[offset] & ~SWAP_HAS_CACHE;
2773 
2774 	if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
2775 		/*
2776 		 * The higher the swap count, the more likely it is that tasks
2777 		 * will race to add swap count continuation: we need to avoid
2778 		 * over-provisioning.
2779 		 */
2780 		goto out;
2781 	}
2782 
2783 	if (!page) {
2784 		spin_unlock(&si->lock);
2785 		return -ENOMEM;
2786 	}
2787 
2788 	/*
2789 	 * We are fortunate that although vmalloc_to_page uses pte_offset_map,
2790 	 * no architecture is using highmem pages for kernel page tables: so it
2791 	 * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps.
2792 	 */
2793 	head = vmalloc_to_page(si->swap_map + offset);
2794 	offset &= ~PAGE_MASK;
2795 
2796 	/*
2797 	 * Page allocation does not initialize the page's lru field,
2798 	 * but it does always reset its private field.
2799 	 */
2800 	if (!page_private(head)) {
2801 		BUG_ON(count & COUNT_CONTINUED);
2802 		INIT_LIST_HEAD(&head->lru);
2803 		set_page_private(head, SWP_CONTINUED);
2804 		si->flags |= SWP_CONTINUED;
2805 	}
2806 
2807 	list_for_each_entry(list_page, &head->lru, lru) {
2808 		unsigned char *map;
2809 
2810 		/*
2811 		 * If the previous map said no continuation, but we've found
2812 		 * a continuation page, free our allocation and use this one.
2813 		 */
2814 		if (!(count & COUNT_CONTINUED))
2815 			goto out;
2816 
2817 		map = kmap_atomic(list_page) + offset;
2818 		count = *map;
2819 		kunmap_atomic(map);
2820 
2821 		/*
2822 		 * If this continuation count now has some space in it,
2823 		 * free our allocation and use this one.
2824 		 */
2825 		if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
2826 			goto out;
2827 	}
2828 
2829 	list_add_tail(&page->lru, &head->lru);
2830 	page = NULL;			/* now it's attached, don't free it */
2831 out:
2832 	spin_unlock(&si->lock);
2833 outer:
2834 	if (page)
2835 		__free_page(page);
2836 	return 0;
2837 }
2838 
2839 /*
2840  * swap_count_continued - when the original swap_map count is incremented
2841  * from SWAP_MAP_MAX, check if there is already a continuation page to carry
2842  * into, carry if so, or else fail until a new continuation page is allocated;
2843  * when the original swap_map count is decremented from 0 with continuation,
2844  * borrow from the continuation and report whether it still holds more.
2845  * Called while __swap_duplicate() or swap_entry_free() holds swap_lock.
2846  */
2847 static bool swap_count_continued(struct swap_info_struct *si,
2848 				 pgoff_t offset, unsigned char count)
2849 {
2850 	struct page *head;
2851 	struct page *page;
2852 	unsigned char *map;
2853 
2854 	head = vmalloc_to_page(si->swap_map + offset);
2855 	if (page_private(head) != SWP_CONTINUED) {
2856 		BUG_ON(count & COUNT_CONTINUED);
2857 		return false;		/* need to add count continuation */
2858 	}
2859 
2860 	offset &= ~PAGE_MASK;
2861 	page = list_entry(head->lru.next, struct page, lru);
2862 	map = kmap_atomic(page) + offset;
2863 
2864 	if (count == SWAP_MAP_MAX)	/* initial increment from swap_map */
2865 		goto init_map;		/* jump over SWAP_CONT_MAX checks */
2866 
2867 	if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
2868 		/*
2869 		 * Think of how you add 1 to 999
2870 		 */
2871 		while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
2872 			kunmap_atomic(map);
2873 			page = list_entry(page->lru.next, struct page, lru);
2874 			BUG_ON(page == head);
2875 			map = kmap_atomic(page) + offset;
2876 		}
2877 		if (*map == SWAP_CONT_MAX) {
2878 			kunmap_atomic(map);
2879 			page = list_entry(page->lru.next, struct page, lru);
2880 			if (page == head)
2881 				return false;	/* add count continuation */
2882 			map = kmap_atomic(page) + offset;
2883 init_map:		*map = 0;		/* we didn't zero the page */
2884 		}
2885 		*map += 1;
2886 		kunmap_atomic(map);
2887 		page = list_entry(page->lru.prev, struct page, lru);
2888 		while (page != head) {
2889 			map = kmap_atomic(page) + offset;
2890 			*map = COUNT_CONTINUED;
2891 			kunmap_atomic(map);
2892 			page = list_entry(page->lru.prev, struct page, lru);
2893 		}
2894 		return true;			/* incremented */
2895 
2896 	} else {				/* decrementing */
2897 		/*
2898 		 * Think of how you subtract 1 from 1000
2899 		 */
2900 		BUG_ON(count != COUNT_CONTINUED);
2901 		while (*map == COUNT_CONTINUED) {
2902 			kunmap_atomic(map);
2903 			page = list_entry(page->lru.next, struct page, lru);
2904 			BUG_ON(page == head);
2905 			map = kmap_atomic(page) + offset;
2906 		}
2907 		BUG_ON(*map == 0);
2908 		*map -= 1;
2909 		if (*map == 0)
2910 			count = 0;
2911 		kunmap_atomic(map);
2912 		page = list_entry(page->lru.prev, struct page, lru);
2913 		while (page != head) {
2914 			map = kmap_atomic(page) + offset;
2915 			*map = SWAP_CONT_MAX | count;
2916 			count = COUNT_CONTINUED;
2917 			kunmap_atomic(map);
2918 			page = list_entry(page->lru.prev, struct page, lru);
2919 		}
2920 		return count == COUNT_CONTINUED;
2921 	}
2922 }
2923 
2924 /*
2925  * free_swap_count_continuations - swapoff free all the continuation pages
2926  * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
2927  */
2928 static void free_swap_count_continuations(struct swap_info_struct *si)
2929 {
2930 	pgoff_t offset;
2931 
2932 	for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
2933 		struct page *head;
2934 		head = vmalloc_to_page(si->swap_map + offset);
2935 		if (page_private(head)) {
2936 			struct list_head *this, *next;
2937 			list_for_each_safe(this, next, &head->lru) {
2938 				struct page *page;
2939 				page = list_entry(this, struct page, lru);
2940 				list_del(this);
2941 				__free_page(page);
2942 			}
2943 		}
2944 	}
2945 }
2946