xref: /linux/kernel/power/swap.c (revision b7df4cc3a088a8ce6973c96731bc792dbf54ce28)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * linux/kernel/power/swap.c
4  *
5  * This file provides functions for reading the suspend image from
6  * and writing it to a swap partition.
7  *
8  * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
9  * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
10  * Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com>
11  */
12 
13 #define pr_fmt(fmt) "PM: " fmt
14 
15 #include <linux/module.h>
16 #include <linux/file.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/device.h>
20 #include <linux/bio.h>
21 #include <linux/blkdev.h>
22 #include <linux/swap.h>
23 #include <linux/swapops.h>
24 #include <linux/pm.h>
25 #include <linux/slab.h>
26 #include <linux/vmalloc.h>
27 #include <linux/cpumask.h>
28 #include <linux/atomic.h>
29 #include <linux/kthread.h>
30 #include <linux/crc32.h>
31 #include <linux/ktime.h>
32 
33 #include "power.h"
34 
35 #define HIBERNATE_SIG	"S1SUSPEND"
36 
37 u32 swsusp_hardware_signature;
38 
39 /*
40  * When reading an {un,}compressed image, we may restore pages in place,
41  * in which case some architectures need these pages cleaning before they
42  * can be executed. We don't know which pages these may be, so clean the lot.
43  */
44 static bool clean_pages_on_read;
45 static bool clean_pages_on_decompress;
46 
47 /*
48  *	The swap map is a data structure used for keeping track of each page
49  *	written to a swap partition.  It consists of many swap_map_page
50  *	structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
51  *	These structures are stored on the swap and linked together with the
52  *	help of the .next_swap member.
53  *
54  *	The swap map is created during suspend.  The swap map pages are
55  *	allocated and populated one at a time, so we only need one memory
56  *	page to set up the entire structure.
57  *
58  *	During resume we pick up all swap_map_page structures into a list.
59  */
60 
61 #define MAP_PAGE_ENTRIES	(PAGE_SIZE / sizeof(sector_t) - 1)
62 
63 /*
64  * Number of free pages that are not high.
65  */
66 static inline unsigned long low_free_pages(void)
67 {
68 	return nr_free_pages() - nr_free_highpages();
69 }
70 
71 /*
72  * Number of pages required to be kept free while writing the image. Always
73  * half of all available low pages before the writing starts.
74  */
75 static inline unsigned long reqd_free_pages(void)
76 {
77 	return low_free_pages() / 2;
78 }
79 
80 struct swap_map_page {
81 	sector_t entries[MAP_PAGE_ENTRIES];
82 	sector_t next_swap;
83 };
84 
85 struct swap_map_page_list {
86 	struct swap_map_page *map;
87 	struct swap_map_page_list *next;
88 };
89 
90 /*
91  *	The swap_map_handle structure is used for handling swap in
92  *	a file-alike way
93  */
94 
95 struct swap_map_handle {
96 	struct swap_map_page *cur;
97 	struct swap_map_page_list *maps;
98 	sector_t cur_swap;
99 	sector_t first_sector;
100 	unsigned int k;
101 	unsigned long reqd_free_pages;
102 	u32 crc32;
103 };
104 
105 struct swsusp_header {
106 	char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) -
107 	              sizeof(u32) - sizeof(u32)];
108 	u32	hw_sig;
109 	u32	crc32;
110 	sector_t image;
111 	unsigned int flags;	/* Flags to pass to the "boot" kernel */
112 	char	orig_sig[10];
113 	char	sig[10];
114 } __packed;
115 
116 static struct swsusp_header *swsusp_header;
117 
118 /*
119  *	The following functions are used for tracing the allocated
120  *	swap pages, so that they can be freed in case of an error.
121  */
122 
123 struct swsusp_extent {
124 	struct rb_node node;
125 	unsigned long start;
126 	unsigned long end;
127 };
128 
129 static struct rb_root swsusp_extents = RB_ROOT;
130 
131 static int swsusp_extents_insert(unsigned long swap_offset)
132 {
133 	struct rb_node **new = &(swsusp_extents.rb_node);
134 	struct rb_node *parent = NULL;
135 	struct swsusp_extent *ext;
136 
137 	/* Figure out where to put the new node */
138 	while (*new) {
139 		ext = rb_entry(*new, struct swsusp_extent, node);
140 		parent = *new;
141 		if (swap_offset < ext->start) {
142 			/* Try to merge */
143 			if (swap_offset == ext->start - 1) {
144 				ext->start--;
145 				return 0;
146 			}
147 			new = &((*new)->rb_left);
148 		} else if (swap_offset > ext->end) {
149 			/* Try to merge */
150 			if (swap_offset == ext->end + 1) {
151 				ext->end++;
152 				return 0;
153 			}
154 			new = &((*new)->rb_right);
155 		} else {
156 			/* It already is in the tree */
157 			return -EINVAL;
158 		}
159 	}
160 	/* Add the new node and rebalance the tree. */
161 	ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL);
162 	if (!ext)
163 		return -ENOMEM;
164 
165 	ext->start = swap_offset;
166 	ext->end = swap_offset;
167 	rb_link_node(&ext->node, parent, new);
168 	rb_insert_color(&ext->node, &swsusp_extents);
169 	return 0;
170 }
171 
172 /*
173  *	alloc_swapdev_block - allocate a swap page and register that it has
174  *	been allocated, so that it can be freed in case of an error.
175  */
176 
177 sector_t alloc_swapdev_block(int swap)
178 {
179 	unsigned long offset;
180 
181 	offset = swp_offset(get_swap_page_of_type(swap));
182 	if (offset) {
183 		if (swsusp_extents_insert(offset))
184 			swap_free(swp_entry(swap, offset));
185 		else
186 			return swapdev_block(swap, offset);
187 	}
188 	return 0;
189 }
190 
191 /*
192  *	free_all_swap_pages - free swap pages allocated for saving image data.
193  *	It also frees the extents used to register which swap entries had been
194  *	allocated.
195  */
196 
197 void free_all_swap_pages(int swap)
198 {
199 	struct rb_node *node;
200 
201 	while ((node = swsusp_extents.rb_node)) {
202 		struct swsusp_extent *ext;
203 		unsigned long offset;
204 
205 		ext = rb_entry(node, struct swsusp_extent, node);
206 		rb_erase(node, &swsusp_extents);
207 		for (offset = ext->start; offset <= ext->end; offset++)
208 			swap_free(swp_entry(swap, offset));
209 
210 		kfree(ext);
211 	}
212 }
213 
214 int swsusp_swap_in_use(void)
215 {
216 	return (swsusp_extents.rb_node != NULL);
217 }
218 
219 /*
220  * General things
221  */
222 
223 static unsigned short root_swap = 0xffff;
224 static struct file *hib_resume_bdev_file;
225 
226 struct hib_bio_batch {
227 	atomic_t		count;
228 	wait_queue_head_t	wait;
229 	blk_status_t		error;
230 	struct blk_plug		plug;
231 };
232 
233 static void hib_init_batch(struct hib_bio_batch *hb)
234 {
235 	atomic_set(&hb->count, 0);
236 	init_waitqueue_head(&hb->wait);
237 	hb->error = BLK_STS_OK;
238 	blk_start_plug(&hb->plug);
239 }
240 
241 static void hib_finish_batch(struct hib_bio_batch *hb)
242 {
243 	blk_finish_plug(&hb->plug);
244 }
245 
246 static void hib_end_io(struct bio *bio)
247 {
248 	struct hib_bio_batch *hb = bio->bi_private;
249 	struct page *page = bio_first_page_all(bio);
250 
251 	if (bio->bi_status) {
252 		pr_alert("Read-error on swap-device (%u:%u:%Lu)\n",
253 			 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
254 			 (unsigned long long)bio->bi_iter.bi_sector);
255 	}
256 
257 	if (bio_data_dir(bio) == WRITE)
258 		put_page(page);
259 	else if (clean_pages_on_read)
260 		flush_icache_range((unsigned long)page_address(page),
261 				   (unsigned long)page_address(page) + PAGE_SIZE);
262 
263 	if (bio->bi_status && !hb->error)
264 		hb->error = bio->bi_status;
265 	if (atomic_dec_and_test(&hb->count))
266 		wake_up(&hb->wait);
267 
268 	bio_put(bio);
269 }
270 
271 static int hib_submit_io(blk_opf_t opf, pgoff_t page_off, void *addr,
272 			 struct hib_bio_batch *hb)
273 {
274 	struct page *page = virt_to_page(addr);
275 	struct bio *bio;
276 	int error = 0;
277 
278 	bio = bio_alloc(file_bdev(hib_resume_bdev_file), 1, opf,
279 			GFP_NOIO | __GFP_HIGH);
280 	bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9);
281 
282 	if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
283 		pr_err("Adding page to bio failed at %llu\n",
284 		       (unsigned long long)bio->bi_iter.bi_sector);
285 		bio_put(bio);
286 		return -EFAULT;
287 	}
288 
289 	if (hb) {
290 		bio->bi_end_io = hib_end_io;
291 		bio->bi_private = hb;
292 		atomic_inc(&hb->count);
293 		submit_bio(bio);
294 	} else {
295 		error = submit_bio_wait(bio);
296 		bio_put(bio);
297 	}
298 
299 	return error;
300 }
301 
302 static int hib_wait_io(struct hib_bio_batch *hb)
303 {
304 	/*
305 	 * We are relying on the behavior of blk_plug that a thread with
306 	 * a plug will flush the plug list before sleeping.
307 	 */
308 	wait_event(hb->wait, atomic_read(&hb->count) == 0);
309 	return blk_status_to_errno(hb->error);
310 }
311 
312 /*
313  * Saving part
314  */
315 static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
316 {
317 	int error;
318 
319 	hib_submit_io(REQ_OP_READ, swsusp_resume_block, swsusp_header, NULL);
320 	if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
321 	    !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
322 		memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
323 		memcpy(swsusp_header->sig, HIBERNATE_SIG, 10);
324 		swsusp_header->image = handle->first_sector;
325 		if (swsusp_hardware_signature) {
326 			swsusp_header->hw_sig = swsusp_hardware_signature;
327 			flags |= SF_HW_SIG;
328 		}
329 		swsusp_header->flags = flags;
330 		if (flags & SF_CRC32_MODE)
331 			swsusp_header->crc32 = handle->crc32;
332 		error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC,
333 				      swsusp_resume_block, swsusp_header, NULL);
334 	} else {
335 		pr_err("Swap header not found!\n");
336 		error = -ENODEV;
337 	}
338 	return error;
339 }
340 
341 /*
342  * Hold the swsusp_header flag. This is used in software_resume() in
343  * 'kernel/power/hibernate' to check if the image is compressed and query
344  * for the compression algorithm support(if so).
345  */
346 unsigned int swsusp_header_flags;
347 
348 /**
349  *	swsusp_swap_check - check if the resume device is a swap device
350  *	and get its index (if so)
351  *
352  *	This is called before saving image
353  */
354 static int swsusp_swap_check(void)
355 {
356 	int res;
357 
358 	if (swsusp_resume_device)
359 		res = swap_type_of(swsusp_resume_device, swsusp_resume_block);
360 	else
361 		res = find_first_swap(&swsusp_resume_device);
362 	if (res < 0)
363 		return res;
364 	root_swap = res;
365 
366 	hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device,
367 			BLK_OPEN_WRITE, NULL, NULL);
368 	if (IS_ERR(hib_resume_bdev_file))
369 		return PTR_ERR(hib_resume_bdev_file);
370 
371 	res = set_blocksize(file_bdev(hib_resume_bdev_file), PAGE_SIZE);
372 	if (res < 0)
373 		fput(hib_resume_bdev_file);
374 
375 	return res;
376 }
377 
378 /**
379  *	write_page - Write one page to given swap location.
380  *	@buf:		Address we're writing.
381  *	@offset:	Offset of the swap page we're writing to.
382  *	@hb:		bio completion batch
383  */
384 
385 static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb)
386 {
387 	void *src;
388 	int ret;
389 
390 	if (!offset)
391 		return -ENOSPC;
392 
393 	if (hb) {
394 		src = (void *)__get_free_page(GFP_NOIO | __GFP_NOWARN |
395 		                              __GFP_NORETRY);
396 		if (src) {
397 			copy_page(src, buf);
398 		} else {
399 			ret = hib_wait_io(hb); /* Free pages */
400 			if (ret)
401 				return ret;
402 			src = (void *)__get_free_page(GFP_NOIO |
403 			                              __GFP_NOWARN |
404 			                              __GFP_NORETRY);
405 			if (src) {
406 				copy_page(src, buf);
407 			} else {
408 				WARN_ON_ONCE(1);
409 				hb = NULL;	/* Go synchronous */
410 				src = buf;
411 			}
412 		}
413 	} else {
414 		src = buf;
415 	}
416 	return hib_submit_io(REQ_OP_WRITE | REQ_SYNC, offset, src, hb);
417 }
418 
419 static void release_swap_writer(struct swap_map_handle *handle)
420 {
421 	if (handle->cur)
422 		free_page((unsigned long)handle->cur);
423 	handle->cur = NULL;
424 }
425 
426 static int get_swap_writer(struct swap_map_handle *handle)
427 {
428 	int ret;
429 
430 	ret = swsusp_swap_check();
431 	if (ret) {
432 		if (ret != -ENOSPC)
433 			pr_err("Cannot find swap device, try swapon -a\n");
434 		return ret;
435 	}
436 	handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
437 	if (!handle->cur) {
438 		ret = -ENOMEM;
439 		goto err_close;
440 	}
441 	handle->cur_swap = alloc_swapdev_block(root_swap);
442 	if (!handle->cur_swap) {
443 		ret = -ENOSPC;
444 		goto err_rel;
445 	}
446 	handle->k = 0;
447 	handle->reqd_free_pages = reqd_free_pages();
448 	handle->first_sector = handle->cur_swap;
449 	return 0;
450 err_rel:
451 	release_swap_writer(handle);
452 err_close:
453 	swsusp_close();
454 	return ret;
455 }
456 
457 static int swap_write_page(struct swap_map_handle *handle, void *buf,
458 		struct hib_bio_batch *hb)
459 {
460 	int error;
461 	sector_t offset;
462 
463 	if (!handle->cur)
464 		return -EINVAL;
465 	offset = alloc_swapdev_block(root_swap);
466 	error = write_page(buf, offset, hb);
467 	if (error)
468 		return error;
469 	handle->cur->entries[handle->k++] = offset;
470 	if (handle->k >= MAP_PAGE_ENTRIES) {
471 		offset = alloc_swapdev_block(root_swap);
472 		if (!offset)
473 			return -ENOSPC;
474 		handle->cur->next_swap = offset;
475 		error = write_page(handle->cur, handle->cur_swap, hb);
476 		if (error)
477 			goto out;
478 		clear_page(handle->cur);
479 		handle->cur_swap = offset;
480 		handle->k = 0;
481 
482 		if (hb && low_free_pages() <= handle->reqd_free_pages) {
483 			error = hib_wait_io(hb);
484 			if (error)
485 				goto out;
486 			/*
487 			 * Recalculate the number of required free pages, to
488 			 * make sure we never take more than half.
489 			 */
490 			handle->reqd_free_pages = reqd_free_pages();
491 		}
492 	}
493  out:
494 	return error;
495 }
496 
497 static int flush_swap_writer(struct swap_map_handle *handle)
498 {
499 	if (handle->cur && handle->cur_swap)
500 		return write_page(handle->cur, handle->cur_swap, NULL);
501 	else
502 		return -EINVAL;
503 }
504 
505 static int swap_writer_finish(struct swap_map_handle *handle,
506 		unsigned int flags, int error)
507 {
508 	if (!error) {
509 		pr_info("S");
510 		error = mark_swapfiles(handle, flags);
511 		pr_cont("|\n");
512 		flush_swap_writer(handle);
513 	}
514 
515 	if (error)
516 		free_all_swap_pages(root_swap);
517 	release_swap_writer(handle);
518 	swsusp_close();
519 
520 	return error;
521 }
522 
523 /*
524  * Bytes we need for compressed data in worst case. We assume(limitation)
525  * this is the worst of all the compression algorithms.
526  */
527 #define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2)
528 
529 /* We need to remember how much compressed data we need to read. */
530 #define CMP_HEADER	sizeof(size_t)
531 
532 /* Number of pages/bytes we'll compress at one time. */
533 #define UNC_PAGES	32
534 #define UNC_SIZE	(UNC_PAGES * PAGE_SIZE)
535 
536 /* Number of pages we need for compressed data (worst case). */
537 #define CMP_PAGES	DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \
538 				CMP_HEADER, PAGE_SIZE)
539 #define CMP_SIZE	(CMP_PAGES * PAGE_SIZE)
540 
541 /* Maximum number of threads for compression/decompression. */
542 #define CMP_THREADS	3
543 
544 /* Minimum/maximum number of pages for read buffering. */
545 #define CMP_MIN_RD_PAGES	1024
546 #define CMP_MAX_RD_PAGES	8192
547 
548 /**
549  *	save_image - save the suspend image data
550  */
551 
552 static int save_image(struct swap_map_handle *handle,
553                       struct snapshot_handle *snapshot,
554                       unsigned int nr_to_write)
555 {
556 	unsigned int m;
557 	int ret;
558 	int nr_pages;
559 	int err2;
560 	struct hib_bio_batch hb;
561 	ktime_t start;
562 	ktime_t stop;
563 
564 	hib_init_batch(&hb);
565 
566 	pr_info("Saving image data pages (%u pages)...\n",
567 		nr_to_write);
568 	m = nr_to_write / 10;
569 	if (!m)
570 		m = 1;
571 	nr_pages = 0;
572 	start = ktime_get();
573 	while (1) {
574 		ret = snapshot_read_next(snapshot);
575 		if (ret <= 0)
576 			break;
577 		ret = swap_write_page(handle, data_of(*snapshot), &hb);
578 		if (ret)
579 			break;
580 		if (!(nr_pages % m))
581 			pr_info("Image saving progress: %3d%%\n",
582 				nr_pages / m * 10);
583 		nr_pages++;
584 	}
585 	err2 = hib_wait_io(&hb);
586 	hib_finish_batch(&hb);
587 	stop = ktime_get();
588 	if (!ret)
589 		ret = err2;
590 	if (!ret)
591 		pr_info("Image saving done\n");
592 	swsusp_show_speed(start, stop, nr_to_write, "Wrote");
593 	return ret;
594 }
595 
596 /*
597  * Structure used for CRC32.
598  */
599 struct crc_data {
600 	struct task_struct *thr;                  /* thread */
601 	atomic_t ready;                           /* ready to start flag */
602 	atomic_t stop;                            /* ready to stop flag */
603 	unsigned run_threads;                     /* nr current threads */
604 	wait_queue_head_t go;                     /* start crc update */
605 	wait_queue_head_t done;                   /* crc update done */
606 	u32 *crc32;                               /* points to handle's crc32 */
607 	size_t *unc_len[CMP_THREADS];             /* uncompressed lengths */
608 	unsigned char *unc[CMP_THREADS];          /* uncompressed data */
609 };
610 
611 /*
612  * CRC32 update function that runs in its own thread.
613  */
614 static int crc32_threadfn(void *data)
615 {
616 	struct crc_data *d = data;
617 	unsigned i;
618 
619 	while (1) {
620 		wait_event(d->go, atomic_read_acquire(&d->ready) ||
621 		                  kthread_should_stop());
622 		if (kthread_should_stop()) {
623 			d->thr = NULL;
624 			atomic_set_release(&d->stop, 1);
625 			wake_up(&d->done);
626 			break;
627 		}
628 		atomic_set(&d->ready, 0);
629 
630 		for (i = 0; i < d->run_threads; i++)
631 			*d->crc32 = crc32_le(*d->crc32,
632 			                     d->unc[i], *d->unc_len[i]);
633 		atomic_set_release(&d->stop, 1);
634 		wake_up(&d->done);
635 	}
636 	return 0;
637 }
638 /*
639  * Structure used for data compression.
640  */
641 struct cmp_data {
642 	struct task_struct *thr;                  /* thread */
643 	struct crypto_comp *cc;                   /* crypto compressor stream */
644 	atomic_t ready;                           /* ready to start flag */
645 	atomic_t stop;                            /* ready to stop flag */
646 	int ret;                                  /* return code */
647 	wait_queue_head_t go;                     /* start compression */
648 	wait_queue_head_t done;                   /* compression done */
649 	size_t unc_len;                           /* uncompressed length */
650 	size_t cmp_len;                           /* compressed length */
651 	unsigned char unc[UNC_SIZE];              /* uncompressed buffer */
652 	unsigned char cmp[CMP_SIZE];              /* compressed buffer */
653 };
654 
655 /* Indicates the image size after compression */
656 static atomic_t compressed_size = ATOMIC_INIT(0);
657 
658 /*
659  * Compression function that runs in its own thread.
660  */
661 static int compress_threadfn(void *data)
662 {
663 	struct cmp_data *d = data;
664 	unsigned int cmp_len = 0;
665 
666 	while (1) {
667 		wait_event(d->go, atomic_read_acquire(&d->ready) ||
668 		                  kthread_should_stop());
669 		if (kthread_should_stop()) {
670 			d->thr = NULL;
671 			d->ret = -1;
672 			atomic_set_release(&d->stop, 1);
673 			wake_up(&d->done);
674 			break;
675 		}
676 		atomic_set(&d->ready, 0);
677 
678 		cmp_len = CMP_SIZE - CMP_HEADER;
679 		d->ret = crypto_comp_compress(d->cc, d->unc, d->unc_len,
680 					      d->cmp + CMP_HEADER,
681 					      &cmp_len);
682 		d->cmp_len = cmp_len;
683 
684 		atomic_set(&compressed_size, atomic_read(&compressed_size) + d->cmp_len);
685 		atomic_set_release(&d->stop, 1);
686 		wake_up(&d->done);
687 	}
688 	return 0;
689 }
690 
691 /**
692  * save_compressed_image - Save the suspend image data after compression.
693  * @handle: Swap map handle to use for saving the image.
694  * @snapshot: Image to read data from.
695  * @nr_to_write: Number of pages to save.
696  */
697 static int save_compressed_image(struct swap_map_handle *handle,
698 				 struct snapshot_handle *snapshot,
699 				 unsigned int nr_to_write)
700 {
701 	unsigned int m;
702 	int ret = 0;
703 	int nr_pages;
704 	int err2;
705 	struct hib_bio_batch hb;
706 	ktime_t start;
707 	ktime_t stop;
708 	size_t off;
709 	unsigned thr, run_threads, nr_threads;
710 	unsigned char *page = NULL;
711 	struct cmp_data *data = NULL;
712 	struct crc_data *crc = NULL;
713 
714 	hib_init_batch(&hb);
715 
716 	atomic_set(&compressed_size, 0);
717 
718 	/*
719 	 * We'll limit the number of threads for compression to limit memory
720 	 * footprint.
721 	 */
722 	nr_threads = num_online_cpus() - 1;
723 	nr_threads = clamp_val(nr_threads, 1, CMP_THREADS);
724 
725 	page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH);
726 	if (!page) {
727 		pr_err("Failed to allocate %s page\n", hib_comp_algo);
728 		ret = -ENOMEM;
729 		goto out_clean;
730 	}
731 
732 	data = vzalloc(array_size(nr_threads, sizeof(*data)));
733 	if (!data) {
734 		pr_err("Failed to allocate %s data\n", hib_comp_algo);
735 		ret = -ENOMEM;
736 		goto out_clean;
737 	}
738 
739 	crc = kzalloc(sizeof(*crc), GFP_KERNEL);
740 	if (!crc) {
741 		pr_err("Failed to allocate crc\n");
742 		ret = -ENOMEM;
743 		goto out_clean;
744 	}
745 
746 	/*
747 	 * Start the compression threads.
748 	 */
749 	for (thr = 0; thr < nr_threads; thr++) {
750 		init_waitqueue_head(&data[thr].go);
751 		init_waitqueue_head(&data[thr].done);
752 
753 		data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0);
754 		if (IS_ERR_OR_NULL(data[thr].cc)) {
755 			pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc));
756 			ret = -EFAULT;
757 			goto out_clean;
758 		}
759 
760 		data[thr].thr = kthread_run(compress_threadfn,
761 		                            &data[thr],
762 		                            "image_compress/%u", thr);
763 		if (IS_ERR(data[thr].thr)) {
764 			data[thr].thr = NULL;
765 			pr_err("Cannot start compression threads\n");
766 			ret = -ENOMEM;
767 			goto out_clean;
768 		}
769 	}
770 
771 	/*
772 	 * Start the CRC32 thread.
773 	 */
774 	init_waitqueue_head(&crc->go);
775 	init_waitqueue_head(&crc->done);
776 
777 	handle->crc32 = 0;
778 	crc->crc32 = &handle->crc32;
779 	for (thr = 0; thr < nr_threads; thr++) {
780 		crc->unc[thr] = data[thr].unc;
781 		crc->unc_len[thr] = &data[thr].unc_len;
782 	}
783 
784 	crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
785 	if (IS_ERR(crc->thr)) {
786 		crc->thr = NULL;
787 		pr_err("Cannot start CRC32 thread\n");
788 		ret = -ENOMEM;
789 		goto out_clean;
790 	}
791 
792 	/*
793 	 * Adjust the number of required free pages after all allocations have
794 	 * been done. We don't want to run out of pages when writing.
795 	 */
796 	handle->reqd_free_pages = reqd_free_pages();
797 
798 	pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo);
799 	pr_info("Compressing and saving image data (%u pages)...\n",
800 		nr_to_write);
801 	m = nr_to_write / 10;
802 	if (!m)
803 		m = 1;
804 	nr_pages = 0;
805 	start = ktime_get();
806 	for (;;) {
807 		for (thr = 0; thr < nr_threads; thr++) {
808 			for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) {
809 				ret = snapshot_read_next(snapshot);
810 				if (ret < 0)
811 					goto out_finish;
812 
813 				if (!ret)
814 					break;
815 
816 				memcpy(data[thr].unc + off,
817 				       data_of(*snapshot), PAGE_SIZE);
818 
819 				if (!(nr_pages % m))
820 					pr_info("Image saving progress: %3d%%\n",
821 						nr_pages / m * 10);
822 				nr_pages++;
823 			}
824 			if (!off)
825 				break;
826 
827 			data[thr].unc_len = off;
828 
829 			atomic_set_release(&data[thr].ready, 1);
830 			wake_up(&data[thr].go);
831 		}
832 
833 		if (!thr)
834 			break;
835 
836 		crc->run_threads = thr;
837 		atomic_set_release(&crc->ready, 1);
838 		wake_up(&crc->go);
839 
840 		for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
841 			wait_event(data[thr].done,
842 				atomic_read_acquire(&data[thr].stop));
843 			atomic_set(&data[thr].stop, 0);
844 
845 			ret = data[thr].ret;
846 
847 			if (ret < 0) {
848 				pr_err("%s compression failed\n", hib_comp_algo);
849 				goto out_finish;
850 			}
851 
852 			if (unlikely(!data[thr].cmp_len ||
853 			             data[thr].cmp_len >
854 				     bytes_worst_compress(data[thr].unc_len))) {
855 				pr_err("Invalid %s compressed length\n", hib_comp_algo);
856 				ret = -1;
857 				goto out_finish;
858 			}
859 
860 			*(size_t *)data[thr].cmp = data[thr].cmp_len;
861 
862 			/*
863 			 * Given we are writing one page at a time to disk, we
864 			 * copy that much from the buffer, although the last
865 			 * bit will likely be smaller than full page. This is
866 			 * OK - we saved the length of the compressed data, so
867 			 * any garbage at the end will be discarded when we
868 			 * read it.
869 			 */
870 			for (off = 0;
871 			     off < CMP_HEADER + data[thr].cmp_len;
872 			     off += PAGE_SIZE) {
873 				memcpy(page, data[thr].cmp + off, PAGE_SIZE);
874 
875 				ret = swap_write_page(handle, page, &hb);
876 				if (ret)
877 					goto out_finish;
878 			}
879 		}
880 
881 		wait_event(crc->done, atomic_read_acquire(&crc->stop));
882 		atomic_set(&crc->stop, 0);
883 	}
884 
885 out_finish:
886 	err2 = hib_wait_io(&hb);
887 	stop = ktime_get();
888 	if (!ret)
889 		ret = err2;
890 	if (!ret)
891 		pr_info("Image saving done\n");
892 	swsusp_show_speed(start, stop, nr_to_write, "Wrote");
893 	pr_info("Image size after compression: %d kbytes\n",
894 		(atomic_read(&compressed_size) / 1024));
895 
896 out_clean:
897 	hib_finish_batch(&hb);
898 	if (crc) {
899 		if (crc->thr)
900 			kthread_stop(crc->thr);
901 		kfree(crc);
902 	}
903 	if (data) {
904 		for (thr = 0; thr < nr_threads; thr++) {
905 			if (data[thr].thr)
906 				kthread_stop(data[thr].thr);
907 			if (data[thr].cc)
908 				crypto_free_comp(data[thr].cc);
909 		}
910 		vfree(data);
911 	}
912 	if (page) free_page((unsigned long)page);
913 
914 	return ret;
915 }
916 
917 /**
918  *	enough_swap - Make sure we have enough swap to save the image.
919  *
920  *	Returns TRUE or FALSE after checking the total amount of swap
921  *	space available from the resume partition.
922  */
923 
924 static int enough_swap(unsigned int nr_pages)
925 {
926 	unsigned int free_swap = count_swap_pages(root_swap, 1);
927 	unsigned int required;
928 
929 	pr_debug("Free swap pages: %u\n", free_swap);
930 
931 	required = PAGES_FOR_IO + nr_pages;
932 	return free_swap > required;
933 }
934 
935 /**
936  *	swsusp_write - Write entire image and metadata.
937  *	@flags: flags to pass to the "boot" kernel in the image header
938  *
939  *	It is important _NOT_ to umount filesystems at this point. We want
940  *	them synced (in case something goes wrong) but we DO not want to mark
941  *	filesystem clean: it is not. (And it does not matter, if we resume
942  *	correctly, we'll mark system clean, anyway.)
943  */
944 
945 int swsusp_write(unsigned int flags)
946 {
947 	struct swap_map_handle handle;
948 	struct snapshot_handle snapshot;
949 	struct swsusp_info *header;
950 	unsigned long pages;
951 	int error;
952 
953 	pages = snapshot_get_image_size();
954 	error = get_swap_writer(&handle);
955 	if (error) {
956 		pr_err("Cannot get swap writer\n");
957 		return error;
958 	}
959 	if (flags & SF_NOCOMPRESS_MODE) {
960 		if (!enough_swap(pages)) {
961 			pr_err("Not enough free swap\n");
962 			error = -ENOSPC;
963 			goto out_finish;
964 		}
965 	}
966 	memset(&snapshot, 0, sizeof(struct snapshot_handle));
967 	error = snapshot_read_next(&snapshot);
968 	if (error < (int)PAGE_SIZE) {
969 		if (error >= 0)
970 			error = -EFAULT;
971 
972 		goto out_finish;
973 	}
974 	header = (struct swsusp_info *)data_of(snapshot);
975 	error = swap_write_page(&handle, header, NULL);
976 	if (!error) {
977 		error = (flags & SF_NOCOMPRESS_MODE) ?
978 			save_image(&handle, &snapshot, pages - 1) :
979 			save_compressed_image(&handle, &snapshot, pages - 1);
980 	}
981 out_finish:
982 	error = swap_writer_finish(&handle, flags, error);
983 	return error;
984 }
985 
986 /*
987  *	The following functions allow us to read data using a swap map
988  *	in a file-like way.
989  */
990 
991 static void release_swap_reader(struct swap_map_handle *handle)
992 {
993 	struct swap_map_page_list *tmp;
994 
995 	while (handle->maps) {
996 		if (handle->maps->map)
997 			free_page((unsigned long)handle->maps->map);
998 		tmp = handle->maps;
999 		handle->maps = handle->maps->next;
1000 		kfree(tmp);
1001 	}
1002 	handle->cur = NULL;
1003 }
1004 
1005 static int get_swap_reader(struct swap_map_handle *handle,
1006 		unsigned int *flags_p)
1007 {
1008 	int error;
1009 	struct swap_map_page_list *tmp, *last;
1010 	sector_t offset;
1011 
1012 	*flags_p = swsusp_header->flags;
1013 
1014 	if (!swsusp_header->image) /* how can this happen? */
1015 		return -EINVAL;
1016 
1017 	handle->cur = NULL;
1018 	last = handle->maps = NULL;
1019 	offset = swsusp_header->image;
1020 	while (offset) {
1021 		tmp = kzalloc(sizeof(*handle->maps), GFP_KERNEL);
1022 		if (!tmp) {
1023 			release_swap_reader(handle);
1024 			return -ENOMEM;
1025 		}
1026 		if (!handle->maps)
1027 			handle->maps = tmp;
1028 		if (last)
1029 			last->next = tmp;
1030 		last = tmp;
1031 
1032 		tmp->map = (struct swap_map_page *)
1033 			   __get_free_page(GFP_NOIO | __GFP_HIGH);
1034 		if (!tmp->map) {
1035 			release_swap_reader(handle);
1036 			return -ENOMEM;
1037 		}
1038 
1039 		error = hib_submit_io(REQ_OP_READ, offset, tmp->map, NULL);
1040 		if (error) {
1041 			release_swap_reader(handle);
1042 			return error;
1043 		}
1044 		offset = tmp->map->next_swap;
1045 	}
1046 	handle->k = 0;
1047 	handle->cur = handle->maps->map;
1048 	return 0;
1049 }
1050 
1051 static int swap_read_page(struct swap_map_handle *handle, void *buf,
1052 		struct hib_bio_batch *hb)
1053 {
1054 	sector_t offset;
1055 	int error;
1056 	struct swap_map_page_list *tmp;
1057 
1058 	if (!handle->cur)
1059 		return -EINVAL;
1060 	offset = handle->cur->entries[handle->k];
1061 	if (!offset)
1062 		return -EFAULT;
1063 	error = hib_submit_io(REQ_OP_READ, offset, buf, hb);
1064 	if (error)
1065 		return error;
1066 	if (++handle->k >= MAP_PAGE_ENTRIES) {
1067 		handle->k = 0;
1068 		free_page((unsigned long)handle->maps->map);
1069 		tmp = handle->maps;
1070 		handle->maps = handle->maps->next;
1071 		kfree(tmp);
1072 		if (!handle->maps)
1073 			release_swap_reader(handle);
1074 		else
1075 			handle->cur = handle->maps->map;
1076 	}
1077 	return error;
1078 }
1079 
1080 static int swap_reader_finish(struct swap_map_handle *handle)
1081 {
1082 	release_swap_reader(handle);
1083 
1084 	return 0;
1085 }
1086 
1087 /**
1088  *	load_image - load the image using the swap map handle
1089  *	@handle and the snapshot handle @snapshot
1090  *	(assume there are @nr_pages pages to load)
1091  */
1092 
1093 static int load_image(struct swap_map_handle *handle,
1094                       struct snapshot_handle *snapshot,
1095                       unsigned int nr_to_read)
1096 {
1097 	unsigned int m;
1098 	int ret = 0;
1099 	ktime_t start;
1100 	ktime_t stop;
1101 	struct hib_bio_batch hb;
1102 	int err2;
1103 	unsigned nr_pages;
1104 
1105 	hib_init_batch(&hb);
1106 
1107 	clean_pages_on_read = true;
1108 	pr_info("Loading image data pages (%u pages)...\n", nr_to_read);
1109 	m = nr_to_read / 10;
1110 	if (!m)
1111 		m = 1;
1112 	nr_pages = 0;
1113 	start = ktime_get();
1114 	for ( ; ; ) {
1115 		ret = snapshot_write_next(snapshot);
1116 		if (ret <= 0)
1117 			break;
1118 		ret = swap_read_page(handle, data_of(*snapshot), &hb);
1119 		if (ret)
1120 			break;
1121 		if (snapshot->sync_read)
1122 			ret = hib_wait_io(&hb);
1123 		if (ret)
1124 			break;
1125 		if (!(nr_pages % m))
1126 			pr_info("Image loading progress: %3d%%\n",
1127 				nr_pages / m * 10);
1128 		nr_pages++;
1129 	}
1130 	err2 = hib_wait_io(&hb);
1131 	hib_finish_batch(&hb);
1132 	stop = ktime_get();
1133 	if (!ret)
1134 		ret = err2;
1135 	if (!ret) {
1136 		pr_info("Image loading done\n");
1137 		ret = snapshot_write_finalize(snapshot);
1138 		if (!ret && !snapshot_image_loaded(snapshot))
1139 			ret = -ENODATA;
1140 	}
1141 	swsusp_show_speed(start, stop, nr_to_read, "Read");
1142 	return ret;
1143 }
1144 
1145 /*
1146  * Structure used for data decompression.
1147  */
1148 struct dec_data {
1149 	struct task_struct *thr;                  /* thread */
1150 	struct crypto_comp *cc;                   /* crypto compressor stream */
1151 	atomic_t ready;                           /* ready to start flag */
1152 	atomic_t stop;                            /* ready to stop flag */
1153 	int ret;                                  /* return code */
1154 	wait_queue_head_t go;                     /* start decompression */
1155 	wait_queue_head_t done;                   /* decompression done */
1156 	size_t unc_len;                           /* uncompressed length */
1157 	size_t cmp_len;                           /* compressed length */
1158 	unsigned char unc[UNC_SIZE];              /* uncompressed buffer */
1159 	unsigned char cmp[CMP_SIZE];              /* compressed buffer */
1160 };
1161 
1162 /*
1163  * Decompression function that runs in its own thread.
1164  */
1165 static int decompress_threadfn(void *data)
1166 {
1167 	struct dec_data *d = data;
1168 	unsigned int unc_len = 0;
1169 
1170 	while (1) {
1171 		wait_event(d->go, atomic_read_acquire(&d->ready) ||
1172 		                  kthread_should_stop());
1173 		if (kthread_should_stop()) {
1174 			d->thr = NULL;
1175 			d->ret = -1;
1176 			atomic_set_release(&d->stop, 1);
1177 			wake_up(&d->done);
1178 			break;
1179 		}
1180 		atomic_set(&d->ready, 0);
1181 
1182 		unc_len = UNC_SIZE;
1183 		d->ret = crypto_comp_decompress(d->cc, d->cmp + CMP_HEADER, d->cmp_len,
1184 						d->unc, &unc_len);
1185 		d->unc_len = unc_len;
1186 
1187 		if (clean_pages_on_decompress)
1188 			flush_icache_range((unsigned long)d->unc,
1189 					   (unsigned long)d->unc + d->unc_len);
1190 
1191 		atomic_set_release(&d->stop, 1);
1192 		wake_up(&d->done);
1193 	}
1194 	return 0;
1195 }
1196 
1197 /**
1198  * load_compressed_image - Load compressed image data and decompress it.
1199  * @handle: Swap map handle to use for loading data.
1200  * @snapshot: Image to copy uncompressed data into.
1201  * @nr_to_read: Number of pages to load.
1202  */
1203 static int load_compressed_image(struct swap_map_handle *handle,
1204 				 struct snapshot_handle *snapshot,
1205 				 unsigned int nr_to_read)
1206 {
1207 	unsigned int m;
1208 	int ret = 0;
1209 	int eof = 0;
1210 	struct hib_bio_batch hb;
1211 	ktime_t start;
1212 	ktime_t stop;
1213 	unsigned nr_pages;
1214 	size_t off;
1215 	unsigned i, thr, run_threads, nr_threads;
1216 	unsigned ring = 0, pg = 0, ring_size = 0,
1217 	         have = 0, want, need, asked = 0;
1218 	unsigned long read_pages = 0;
1219 	unsigned char **page = NULL;
1220 	struct dec_data *data = NULL;
1221 	struct crc_data *crc = NULL;
1222 
1223 	hib_init_batch(&hb);
1224 
1225 	/*
1226 	 * We'll limit the number of threads for decompression to limit memory
1227 	 * footprint.
1228 	 */
1229 	nr_threads = num_online_cpus() - 1;
1230 	nr_threads = clamp_val(nr_threads, 1, CMP_THREADS);
1231 
1232 	page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page)));
1233 	if (!page) {
1234 		pr_err("Failed to allocate %s page\n", hib_comp_algo);
1235 		ret = -ENOMEM;
1236 		goto out_clean;
1237 	}
1238 
1239 	data = vzalloc(array_size(nr_threads, sizeof(*data)));
1240 	if (!data) {
1241 		pr_err("Failed to allocate %s data\n", hib_comp_algo);
1242 		ret = -ENOMEM;
1243 		goto out_clean;
1244 	}
1245 
1246 	crc = kzalloc(sizeof(*crc), GFP_KERNEL);
1247 	if (!crc) {
1248 		pr_err("Failed to allocate crc\n");
1249 		ret = -ENOMEM;
1250 		goto out_clean;
1251 	}
1252 
1253 	clean_pages_on_decompress = true;
1254 
1255 	/*
1256 	 * Start the decompression threads.
1257 	 */
1258 	for (thr = 0; thr < nr_threads; thr++) {
1259 		init_waitqueue_head(&data[thr].go);
1260 		init_waitqueue_head(&data[thr].done);
1261 
1262 		data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0);
1263 		if (IS_ERR_OR_NULL(data[thr].cc)) {
1264 			pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc));
1265 			ret = -EFAULT;
1266 			goto out_clean;
1267 		}
1268 
1269 		data[thr].thr = kthread_run(decompress_threadfn,
1270 		                            &data[thr],
1271 		                            "image_decompress/%u", thr);
1272 		if (IS_ERR(data[thr].thr)) {
1273 			data[thr].thr = NULL;
1274 			pr_err("Cannot start decompression threads\n");
1275 			ret = -ENOMEM;
1276 			goto out_clean;
1277 		}
1278 	}
1279 
1280 	/*
1281 	 * Start the CRC32 thread.
1282 	 */
1283 	init_waitqueue_head(&crc->go);
1284 	init_waitqueue_head(&crc->done);
1285 
1286 	handle->crc32 = 0;
1287 	crc->crc32 = &handle->crc32;
1288 	for (thr = 0; thr < nr_threads; thr++) {
1289 		crc->unc[thr] = data[thr].unc;
1290 		crc->unc_len[thr] = &data[thr].unc_len;
1291 	}
1292 
1293 	crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
1294 	if (IS_ERR(crc->thr)) {
1295 		crc->thr = NULL;
1296 		pr_err("Cannot start CRC32 thread\n");
1297 		ret = -ENOMEM;
1298 		goto out_clean;
1299 	}
1300 
1301 	/*
1302 	 * Set the number of pages for read buffering.
1303 	 * This is complete guesswork, because we'll only know the real
1304 	 * picture once prepare_image() is called, which is much later on
1305 	 * during the image load phase. We'll assume the worst case and
1306 	 * say that none of the image pages are from high memory.
1307 	 */
1308 	if (low_free_pages() > snapshot_get_image_size())
1309 		read_pages = (low_free_pages() - snapshot_get_image_size()) / 2;
1310 	read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES);
1311 
1312 	for (i = 0; i < read_pages; i++) {
1313 		page[i] = (void *)__get_free_page(i < CMP_PAGES ?
1314 						  GFP_NOIO | __GFP_HIGH :
1315 						  GFP_NOIO | __GFP_NOWARN |
1316 						  __GFP_NORETRY);
1317 
1318 		if (!page[i]) {
1319 			if (i < CMP_PAGES) {
1320 				ring_size = i;
1321 				pr_err("Failed to allocate %s pages\n", hib_comp_algo);
1322 				ret = -ENOMEM;
1323 				goto out_clean;
1324 			} else {
1325 				break;
1326 			}
1327 		}
1328 	}
1329 	want = ring_size = i;
1330 
1331 	pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo);
1332 	pr_info("Loading and decompressing image data (%u pages)...\n",
1333 		nr_to_read);
1334 	m = nr_to_read / 10;
1335 	if (!m)
1336 		m = 1;
1337 	nr_pages = 0;
1338 	start = ktime_get();
1339 
1340 	ret = snapshot_write_next(snapshot);
1341 	if (ret <= 0)
1342 		goto out_finish;
1343 
1344 	for(;;) {
1345 		for (i = 0; !eof && i < want; i++) {
1346 			ret = swap_read_page(handle, page[ring], &hb);
1347 			if (ret) {
1348 				/*
1349 				 * On real read error, finish. On end of data,
1350 				 * set EOF flag and just exit the read loop.
1351 				 */
1352 				if (handle->cur &&
1353 				    handle->cur->entries[handle->k]) {
1354 					goto out_finish;
1355 				} else {
1356 					eof = 1;
1357 					break;
1358 				}
1359 			}
1360 			if (++ring >= ring_size)
1361 				ring = 0;
1362 		}
1363 		asked += i;
1364 		want -= i;
1365 
1366 		/*
1367 		 * We are out of data, wait for some more.
1368 		 */
1369 		if (!have) {
1370 			if (!asked)
1371 				break;
1372 
1373 			ret = hib_wait_io(&hb);
1374 			if (ret)
1375 				goto out_finish;
1376 			have += asked;
1377 			asked = 0;
1378 			if (eof)
1379 				eof = 2;
1380 		}
1381 
1382 		if (crc->run_threads) {
1383 			wait_event(crc->done, atomic_read_acquire(&crc->stop));
1384 			atomic_set(&crc->stop, 0);
1385 			crc->run_threads = 0;
1386 		}
1387 
1388 		for (thr = 0; have && thr < nr_threads; thr++) {
1389 			data[thr].cmp_len = *(size_t *)page[pg];
1390 			if (unlikely(!data[thr].cmp_len ||
1391 			             data[thr].cmp_len >
1392 					bytes_worst_compress(UNC_SIZE))) {
1393 				pr_err("Invalid %s compressed length\n", hib_comp_algo);
1394 				ret = -1;
1395 				goto out_finish;
1396 			}
1397 
1398 			need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER,
1399 			                    PAGE_SIZE);
1400 			if (need > have) {
1401 				if (eof > 1) {
1402 					ret = -1;
1403 					goto out_finish;
1404 				}
1405 				break;
1406 			}
1407 
1408 			for (off = 0;
1409 			     off < CMP_HEADER + data[thr].cmp_len;
1410 			     off += PAGE_SIZE) {
1411 				memcpy(data[thr].cmp + off,
1412 				       page[pg], PAGE_SIZE);
1413 				have--;
1414 				want++;
1415 				if (++pg >= ring_size)
1416 					pg = 0;
1417 			}
1418 
1419 			atomic_set_release(&data[thr].ready, 1);
1420 			wake_up(&data[thr].go);
1421 		}
1422 
1423 		/*
1424 		 * Wait for more data while we are decompressing.
1425 		 */
1426 		if (have < CMP_PAGES && asked) {
1427 			ret = hib_wait_io(&hb);
1428 			if (ret)
1429 				goto out_finish;
1430 			have += asked;
1431 			asked = 0;
1432 			if (eof)
1433 				eof = 2;
1434 		}
1435 
1436 		for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
1437 			wait_event(data[thr].done,
1438 				atomic_read_acquire(&data[thr].stop));
1439 			atomic_set(&data[thr].stop, 0);
1440 
1441 			ret = data[thr].ret;
1442 
1443 			if (ret < 0) {
1444 				pr_err("%s decompression failed\n", hib_comp_algo);
1445 				goto out_finish;
1446 			}
1447 
1448 			if (unlikely(!data[thr].unc_len ||
1449 				data[thr].unc_len > UNC_SIZE ||
1450 				data[thr].unc_len & (PAGE_SIZE - 1))) {
1451 				pr_err("Invalid %s uncompressed length\n", hib_comp_algo);
1452 				ret = -1;
1453 				goto out_finish;
1454 			}
1455 
1456 			for (off = 0;
1457 			     off < data[thr].unc_len; off += PAGE_SIZE) {
1458 				memcpy(data_of(*snapshot),
1459 				       data[thr].unc + off, PAGE_SIZE);
1460 
1461 				if (!(nr_pages % m))
1462 					pr_info("Image loading progress: %3d%%\n",
1463 						nr_pages / m * 10);
1464 				nr_pages++;
1465 
1466 				ret = snapshot_write_next(snapshot);
1467 				if (ret <= 0) {
1468 					crc->run_threads = thr + 1;
1469 					atomic_set_release(&crc->ready, 1);
1470 					wake_up(&crc->go);
1471 					goto out_finish;
1472 				}
1473 			}
1474 		}
1475 
1476 		crc->run_threads = thr;
1477 		atomic_set_release(&crc->ready, 1);
1478 		wake_up(&crc->go);
1479 	}
1480 
1481 out_finish:
1482 	if (crc->run_threads) {
1483 		wait_event(crc->done, atomic_read_acquire(&crc->stop));
1484 		atomic_set(&crc->stop, 0);
1485 	}
1486 	stop = ktime_get();
1487 	if (!ret) {
1488 		pr_info("Image loading done\n");
1489 		ret = snapshot_write_finalize(snapshot);
1490 		if (!ret && !snapshot_image_loaded(snapshot))
1491 			ret = -ENODATA;
1492 		if (!ret) {
1493 			if (swsusp_header->flags & SF_CRC32_MODE) {
1494 				if(handle->crc32 != swsusp_header->crc32) {
1495 					pr_err("Invalid image CRC32!\n");
1496 					ret = -ENODATA;
1497 				}
1498 			}
1499 		}
1500 	}
1501 	swsusp_show_speed(start, stop, nr_to_read, "Read");
1502 out_clean:
1503 	hib_finish_batch(&hb);
1504 	for (i = 0; i < ring_size; i++)
1505 		free_page((unsigned long)page[i]);
1506 	if (crc) {
1507 		if (crc->thr)
1508 			kthread_stop(crc->thr);
1509 		kfree(crc);
1510 	}
1511 	if (data) {
1512 		for (thr = 0; thr < nr_threads; thr++) {
1513 			if (data[thr].thr)
1514 				kthread_stop(data[thr].thr);
1515 			if (data[thr].cc)
1516 				crypto_free_comp(data[thr].cc);
1517 		}
1518 		vfree(data);
1519 	}
1520 	vfree(page);
1521 
1522 	return ret;
1523 }
1524 
1525 /**
1526  *	swsusp_read - read the hibernation image.
1527  *	@flags_p: flags passed by the "frozen" kernel in the image header should
1528  *		  be written into this memory location
1529  */
1530 
1531 int swsusp_read(unsigned int *flags_p)
1532 {
1533 	int error;
1534 	struct swap_map_handle handle;
1535 	struct snapshot_handle snapshot;
1536 	struct swsusp_info *header;
1537 
1538 	memset(&snapshot, 0, sizeof(struct snapshot_handle));
1539 	error = snapshot_write_next(&snapshot);
1540 	if (error < (int)PAGE_SIZE)
1541 		return error < 0 ? error : -EFAULT;
1542 	header = (struct swsusp_info *)data_of(snapshot);
1543 	error = get_swap_reader(&handle, flags_p);
1544 	if (error)
1545 		goto end;
1546 	if (!error)
1547 		error = swap_read_page(&handle, header, NULL);
1548 	if (!error) {
1549 		error = (*flags_p & SF_NOCOMPRESS_MODE) ?
1550 			load_image(&handle, &snapshot, header->pages - 1) :
1551 			load_compressed_image(&handle, &snapshot, header->pages - 1);
1552 	}
1553 	swap_reader_finish(&handle);
1554 end:
1555 	if (!error)
1556 		pr_debug("Image successfully loaded\n");
1557 	else
1558 		pr_debug("Error %d resuming\n", error);
1559 	return error;
1560 }
1561 
1562 static void *swsusp_holder;
1563 
1564 /**
1565  * swsusp_check - Open the resume device and check for the swsusp signature.
1566  * @exclusive: Open the resume device exclusively.
1567  */
1568 
1569 int swsusp_check(bool exclusive)
1570 {
1571 	void *holder = exclusive ? &swsusp_holder : NULL;
1572 	int error;
1573 
1574 	hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device,
1575 				BLK_OPEN_READ, holder, NULL);
1576 	if (!IS_ERR(hib_resume_bdev_file)) {
1577 		set_blocksize(file_bdev(hib_resume_bdev_file), PAGE_SIZE);
1578 		clear_page(swsusp_header);
1579 		error = hib_submit_io(REQ_OP_READ, swsusp_resume_block,
1580 					swsusp_header, NULL);
1581 		if (error)
1582 			goto put;
1583 
1584 		if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
1585 			memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
1586 			swsusp_header_flags = swsusp_header->flags;
1587 			/* Reset swap signature now */
1588 			error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC,
1589 						swsusp_resume_block,
1590 						swsusp_header, NULL);
1591 		} else {
1592 			error = -EINVAL;
1593 		}
1594 		if (!error && swsusp_header->flags & SF_HW_SIG &&
1595 		    swsusp_header->hw_sig != swsusp_hardware_signature) {
1596 			pr_info("Suspend image hardware signature mismatch (%08x now %08x); aborting resume.\n",
1597 				swsusp_header->hw_sig, swsusp_hardware_signature);
1598 			error = -EINVAL;
1599 		}
1600 
1601 put:
1602 		if (error)
1603 			fput(hib_resume_bdev_file);
1604 		else
1605 			pr_debug("Image signature found, resuming\n");
1606 	} else {
1607 		error = PTR_ERR(hib_resume_bdev_file);
1608 	}
1609 
1610 	if (error)
1611 		pr_debug("Image not found (code %d)\n", error);
1612 
1613 	return error;
1614 }
1615 
1616 /**
1617  * swsusp_close - close resume device.
1618  */
1619 
1620 void swsusp_close(void)
1621 {
1622 	if (IS_ERR(hib_resume_bdev_file)) {
1623 		pr_debug("Image device not initialised\n");
1624 		return;
1625 	}
1626 
1627 	fput(hib_resume_bdev_file);
1628 }
1629 
1630 /**
1631  *      swsusp_unmark - Unmark swsusp signature in the resume device
1632  */
1633 
1634 #ifdef CONFIG_SUSPEND
1635 int swsusp_unmark(void)
1636 {
1637 	int error;
1638 
1639 	hib_submit_io(REQ_OP_READ, swsusp_resume_block,
1640 			swsusp_header, NULL);
1641 	if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) {
1642 		memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10);
1643 		error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC,
1644 					swsusp_resume_block,
1645 					swsusp_header, NULL);
1646 	} else {
1647 		pr_err("Cannot find swsusp signature!\n");
1648 		error = -ENODEV;
1649 	}
1650 
1651 	/*
1652 	 * We just returned from suspend, we don't need the image any more.
1653 	 */
1654 	free_all_swap_pages(root_swap);
1655 
1656 	return error;
1657 }
1658 #endif
1659 
1660 static int __init swsusp_header_init(void)
1661 {
1662 	swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL);
1663 	if (!swsusp_header)
1664 		panic("Could not allocate memory for swsusp_header\n");
1665 	return 0;
1666 }
1667 
1668 core_initcall(swsusp_header_init);
1669