1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/kernel/power/swap.c 4 * 5 * This file provides functions for reading the suspend image from 6 * and writing it to a swap partition. 7 * 8 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz> 9 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 10 * Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com> 11 */ 12 13 #define pr_fmt(fmt) "PM: " fmt 14 15 #include <crypto/acompress.h> 16 #include <linux/module.h> 17 #include <linux/file.h> 18 #include <linux/delay.h> 19 #include <linux/bitops.h> 20 #include <linux/device.h> 21 #include <linux/bio.h> 22 #include <linux/blkdev.h> 23 #include <linux/swap.h> 24 #include <linux/swapops.h> 25 #include <linux/pm.h> 26 #include <linux/slab.h> 27 #include <linux/vmalloc.h> 28 #include <linux/cpumask.h> 29 #include <linux/atomic.h> 30 #include <linux/kthread.h> 31 #include <linux/crc32.h> 32 #include <linux/ktime.h> 33 34 #include "power.h" 35 36 #define HIBERNATE_SIG "S1SUSPEND" 37 38 u32 swsusp_hardware_signature; 39 40 /* 41 * When reading an {un,}compressed image, we may restore pages in place, 42 * in which case some architectures need these pages cleaning before they 43 * can be executed. We don't know which pages these may be, so clean the lot. 44 */ 45 static bool clean_pages_on_read; 46 static bool clean_pages_on_decompress; 47 48 /* 49 * The swap map is a data structure used for keeping track of each page 50 * written to a swap partition. It consists of many swap_map_page structures 51 * that contain each an array of MAP_PAGE_ENTRIES swap entries. These 52 * structures are stored on the swap and linked together with the help of the 53 * .next_swap member. 54 * 55 * The swap map is created during suspend. The swap map pages are allocated and 56 * populated one at a time, so we only need one memory page to set up the entire 57 * structure. 58 * 59 * During resume we pick up all swap_map_page structures into a list. 60 */ 61 #define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1) 62 63 /* 64 * Number of free pages that are not high. 65 */ 66 static inline unsigned long low_free_pages(void) 67 { 68 return nr_free_pages() - nr_free_highpages(); 69 } 70 71 /* 72 * Number of pages required to be kept free while writing the image. Always 73 * half of all available low pages before the writing starts. 74 */ 75 static inline unsigned long reqd_free_pages(void) 76 { 77 return low_free_pages() / 2; 78 } 79 80 struct swap_map_page { 81 sector_t entries[MAP_PAGE_ENTRIES]; 82 sector_t next_swap; 83 }; 84 85 struct swap_map_page_list { 86 struct swap_map_page *map; 87 struct swap_map_page_list *next; 88 }; 89 90 /* 91 * The swap_map_handle structure is used for handling swap in a file-alike way. 92 */ 93 struct swap_map_handle { 94 struct swap_map_page *cur; 95 struct swap_map_page_list *maps; 96 sector_t cur_swap; 97 sector_t first_sector; 98 unsigned int k; 99 unsigned long reqd_free_pages; 100 u32 crc32; 101 }; 102 103 struct swsusp_header { 104 char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) - 105 sizeof(u32) - sizeof(u32)]; 106 u32 hw_sig; 107 u32 crc32; 108 sector_t image; 109 unsigned int flags; /* Flags to pass to the "boot" kernel */ 110 char orig_sig[10]; 111 char sig[10]; 112 } __packed; 113 114 static struct swsusp_header *swsusp_header; 115 116 /* 117 * The following functions are used for tracing the allocated swap pages, so 118 * that they can be freed in case of an error. 119 */ 120 struct swsusp_extent { 121 struct rb_node node; 122 unsigned long start; 123 unsigned long end; 124 }; 125 126 static struct rb_root swsusp_extents = RB_ROOT; 127 128 static int swsusp_extents_insert(unsigned long swap_offset) 129 { 130 struct rb_node **new = &(swsusp_extents.rb_node); 131 struct rb_node *parent = NULL; 132 struct swsusp_extent *ext; 133 134 /* Figure out where to put the new node */ 135 while (*new) { 136 ext = rb_entry(*new, struct swsusp_extent, node); 137 parent = *new; 138 if (swap_offset < ext->start) { 139 /* Try to merge */ 140 if (swap_offset == ext->start - 1) { 141 ext->start--; 142 return 0; 143 } 144 new = &((*new)->rb_left); 145 } else if (swap_offset > ext->end) { 146 /* Try to merge */ 147 if (swap_offset == ext->end + 1) { 148 ext->end++; 149 return 0; 150 } 151 new = &((*new)->rb_right); 152 } else { 153 /* It already is in the tree */ 154 return -EINVAL; 155 } 156 } 157 /* Add the new node and rebalance the tree. */ 158 ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL); 159 if (!ext) 160 return -ENOMEM; 161 162 ext->start = swap_offset; 163 ext->end = swap_offset; 164 rb_link_node(&ext->node, parent, new); 165 rb_insert_color(&ext->node, &swsusp_extents); 166 return 0; 167 } 168 169 sector_t alloc_swapdev_block(int swap) 170 { 171 unsigned long offset; 172 173 /* 174 * Allocate a swap page and register that it has been allocated, so that 175 * it can be freed in case of an error. 176 */ 177 offset = swp_offset(get_swap_page_of_type(swap)); 178 if (offset) { 179 if (swsusp_extents_insert(offset)) 180 swap_free(swp_entry(swap, offset)); 181 else 182 return swapdev_block(swap, offset); 183 } 184 return 0; 185 } 186 187 void free_all_swap_pages(int swap) 188 { 189 struct rb_node *node; 190 191 /* 192 * Free swap pages allocated for saving image data. It also frees the 193 * extents used to register which swap entries had been allocated. 194 */ 195 while ((node = swsusp_extents.rb_node)) { 196 struct swsusp_extent *ext; 197 198 ext = rb_entry(node, struct swsusp_extent, node); 199 rb_erase(node, &swsusp_extents); 200 swap_free_nr(swp_entry(swap, ext->start), 201 ext->end - ext->start + 1); 202 203 kfree(ext); 204 } 205 } 206 207 int swsusp_swap_in_use(void) 208 { 209 return (swsusp_extents.rb_node != NULL); 210 } 211 212 /* 213 * General things 214 */ 215 216 static unsigned short root_swap = 0xffff; 217 static struct file *hib_resume_bdev_file; 218 219 struct hib_bio_batch { 220 atomic_t count; 221 wait_queue_head_t wait; 222 blk_status_t error; 223 struct blk_plug plug; 224 }; 225 226 static void hib_init_batch(struct hib_bio_batch *hb) 227 { 228 atomic_set(&hb->count, 0); 229 init_waitqueue_head(&hb->wait); 230 hb->error = BLK_STS_OK; 231 blk_start_plug(&hb->plug); 232 } 233 234 static void hib_finish_batch(struct hib_bio_batch *hb) 235 { 236 blk_finish_plug(&hb->plug); 237 } 238 239 static void hib_end_io(struct bio *bio) 240 { 241 struct hib_bio_batch *hb = bio->bi_private; 242 struct page *page = bio_first_page_all(bio); 243 244 if (bio->bi_status) { 245 pr_alert("Read-error on swap-device (%u:%u:%Lu)\n", 246 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), 247 (unsigned long long)bio->bi_iter.bi_sector); 248 } 249 250 if (bio_data_dir(bio) == WRITE) 251 put_page(page); 252 else if (clean_pages_on_read) 253 flush_icache_range((unsigned long)page_address(page), 254 (unsigned long)page_address(page) + PAGE_SIZE); 255 256 if (bio->bi_status && !hb->error) 257 hb->error = bio->bi_status; 258 if (atomic_dec_and_test(&hb->count)) 259 wake_up(&hb->wait); 260 261 bio_put(bio); 262 } 263 264 static int hib_submit_io_sync(blk_opf_t opf, pgoff_t page_off, void *addr) 265 { 266 return bdev_rw_virt(file_bdev(hib_resume_bdev_file), 267 page_off * (PAGE_SIZE >> 9), addr, PAGE_SIZE, opf); 268 } 269 270 static int hib_submit_io_async(blk_opf_t opf, pgoff_t page_off, void *addr, 271 struct hib_bio_batch *hb) 272 { 273 struct bio *bio; 274 275 bio = bio_alloc(file_bdev(hib_resume_bdev_file), 1, opf, 276 GFP_NOIO | __GFP_HIGH); 277 bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9); 278 bio_add_virt_nofail(bio, addr, PAGE_SIZE); 279 bio->bi_end_io = hib_end_io; 280 bio->bi_private = hb; 281 atomic_inc(&hb->count); 282 submit_bio(bio); 283 return 0; 284 } 285 286 static int hib_wait_io(struct hib_bio_batch *hb) 287 { 288 /* 289 * We are relying on the behavior of blk_plug that a thread with 290 * a plug will flush the plug list before sleeping. 291 */ 292 wait_event(hb->wait, atomic_read(&hb->count) == 0); 293 return blk_status_to_errno(hb->error); 294 } 295 296 /* 297 * Saving part 298 */ 299 300 static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) 301 { 302 int error; 303 304 hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header); 305 if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) || 306 !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) { 307 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10); 308 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10); 309 swsusp_header->image = handle->first_sector; 310 if (swsusp_hardware_signature) { 311 swsusp_header->hw_sig = swsusp_hardware_signature; 312 flags |= SF_HW_SIG; 313 } 314 swsusp_header->flags = flags; 315 if (flags & SF_CRC32_MODE) 316 swsusp_header->crc32 = handle->crc32; 317 error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, 318 swsusp_resume_block, swsusp_header); 319 } else { 320 pr_err("Swap header not found!\n"); 321 error = -ENODEV; 322 } 323 return error; 324 } 325 326 /* 327 * Hold the swsusp_header flag. This is used in software_resume() in 328 * 'kernel/power/hibernate' to check if the image is compressed and query 329 * for the compression algorithm support(if so). 330 */ 331 unsigned int swsusp_header_flags; 332 333 static int swsusp_swap_check(void) 334 { 335 int res; 336 337 /* 338 * Check if the resume device is a swap device and get its index (if so). 339 * This is called before saving the image. 340 */ 341 if (swsusp_resume_device) 342 res = swap_type_of(swsusp_resume_device, swsusp_resume_block); 343 else 344 res = find_first_swap(&swsusp_resume_device); 345 if (res < 0) 346 return res; 347 root_swap = res; 348 349 hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device, 350 BLK_OPEN_WRITE, NULL, NULL); 351 if (IS_ERR(hib_resume_bdev_file)) 352 return PTR_ERR(hib_resume_bdev_file); 353 354 return 0; 355 } 356 357 static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb) 358 { 359 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; 360 void *src; 361 int ret; 362 363 if (!offset) 364 return -ENOSPC; 365 366 if (!hb) 367 goto sync_io; 368 369 src = (void *)__get_free_page(gfp); 370 if (!src) { 371 ret = hib_wait_io(hb); /* Free pages */ 372 if (ret) 373 return ret; 374 src = (void *)__get_free_page(gfp); 375 if (WARN_ON_ONCE(!src)) 376 goto sync_io; 377 } 378 379 copy_page(src, buf); 380 return hib_submit_io_async(REQ_OP_WRITE | REQ_SYNC, offset, src, hb); 381 sync_io: 382 return hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, offset, buf); 383 } 384 385 static void release_swap_writer(struct swap_map_handle *handle) 386 { 387 if (handle->cur) 388 free_page((unsigned long)handle->cur); 389 handle->cur = NULL; 390 } 391 392 static int get_swap_writer(struct swap_map_handle *handle) 393 { 394 int ret; 395 396 ret = swsusp_swap_check(); 397 if (ret) { 398 if (ret != -ENOSPC) 399 pr_err("Cannot find swap device, try swapon -a\n"); 400 return ret; 401 } 402 handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL); 403 if (!handle->cur) { 404 ret = -ENOMEM; 405 goto err_close; 406 } 407 handle->cur_swap = alloc_swapdev_block(root_swap); 408 if (!handle->cur_swap) { 409 ret = -ENOSPC; 410 goto err_rel; 411 } 412 handle->k = 0; 413 handle->reqd_free_pages = reqd_free_pages(); 414 handle->first_sector = handle->cur_swap; 415 return 0; 416 err_rel: 417 release_swap_writer(handle); 418 err_close: 419 swsusp_close(); 420 return ret; 421 } 422 423 static int swap_write_page(struct swap_map_handle *handle, void *buf, 424 struct hib_bio_batch *hb) 425 { 426 int error; 427 sector_t offset; 428 429 if (!handle->cur) 430 return -EINVAL; 431 offset = alloc_swapdev_block(root_swap); 432 error = write_page(buf, offset, hb); 433 if (error) 434 return error; 435 handle->cur->entries[handle->k++] = offset; 436 if (handle->k >= MAP_PAGE_ENTRIES) { 437 offset = alloc_swapdev_block(root_swap); 438 if (!offset) 439 return -ENOSPC; 440 handle->cur->next_swap = offset; 441 error = write_page(handle->cur, handle->cur_swap, hb); 442 if (error) 443 goto out; 444 clear_page(handle->cur); 445 handle->cur_swap = offset; 446 handle->k = 0; 447 448 if (hb && low_free_pages() <= handle->reqd_free_pages) { 449 error = hib_wait_io(hb); 450 if (error) 451 goto out; 452 /* 453 * Recalculate the number of required free pages, to 454 * make sure we never take more than half. 455 */ 456 handle->reqd_free_pages = reqd_free_pages(); 457 } 458 } 459 out: 460 return error; 461 } 462 463 static int flush_swap_writer(struct swap_map_handle *handle) 464 { 465 if (handle->cur && handle->cur_swap) 466 return write_page(handle->cur, handle->cur_swap, NULL); 467 else 468 return -EINVAL; 469 } 470 471 static int swap_writer_finish(struct swap_map_handle *handle, 472 unsigned int flags, int error) 473 { 474 if (!error) { 475 pr_info("S"); 476 error = mark_swapfiles(handle, flags); 477 pr_cont("|\n"); 478 flush_swap_writer(handle); 479 } 480 481 if (error) 482 free_all_swap_pages(root_swap); 483 release_swap_writer(handle); 484 swsusp_close(); 485 486 return error; 487 } 488 489 /* 490 * Bytes we need for compressed data in worst case. We assume(limitation) 491 * this is the worst of all the compression algorithms. 492 */ 493 #define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2) 494 495 /* We need to remember how much compressed data we need to read. */ 496 #define CMP_HEADER sizeof(size_t) 497 498 /* Number of pages/bytes we'll compress at one time. */ 499 #define UNC_PAGES 32 500 #define UNC_SIZE (UNC_PAGES * PAGE_SIZE) 501 502 /* Number of pages we need for compressed data (worst case). */ 503 #define CMP_PAGES DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \ 504 CMP_HEADER, PAGE_SIZE) 505 #define CMP_SIZE (CMP_PAGES * PAGE_SIZE) 506 507 /* Default number of threads for compression/decompression. */ 508 #define CMP_THREADS 3 509 static unsigned int hibernate_compression_threads = CMP_THREADS; 510 511 /* Minimum/maximum number of pages for read buffering. */ 512 #define CMP_MIN_RD_PAGES 1024 513 #define CMP_MAX_RD_PAGES 8192 514 515 static int save_image(struct swap_map_handle *handle, 516 struct snapshot_handle *snapshot, 517 unsigned int nr_to_write) 518 { 519 unsigned int m; 520 int ret; 521 int nr_pages; 522 int err2; 523 struct hib_bio_batch hb; 524 ktime_t start; 525 ktime_t stop; 526 527 hib_init_batch(&hb); 528 529 pr_info("Saving image data pages (%u pages)...\n", 530 nr_to_write); 531 m = nr_to_write / 10; 532 if (!m) 533 m = 1; 534 nr_pages = 0; 535 start = ktime_get(); 536 while (1) { 537 ret = snapshot_read_next(snapshot); 538 if (ret <= 0) 539 break; 540 ret = swap_write_page(handle, data_of(*snapshot), &hb); 541 if (ret) 542 break; 543 if (!(nr_pages % m)) 544 pr_info("Image saving progress: %3d%%\n", 545 nr_pages / m * 10); 546 nr_pages++; 547 } 548 err2 = hib_wait_io(&hb); 549 hib_finish_batch(&hb); 550 stop = ktime_get(); 551 if (!ret) 552 ret = err2; 553 if (!ret) 554 pr_info("Image saving done\n"); 555 swsusp_show_speed(start, stop, nr_to_write, "Wrote"); 556 return ret; 557 } 558 559 /* 560 * Structure used for CRC32. 561 */ 562 struct crc_data { 563 struct task_struct *thr; /* thread */ 564 atomic_t ready; /* ready to start flag */ 565 atomic_t stop; /* ready to stop flag */ 566 unsigned run_threads; /* nr current threads */ 567 wait_queue_head_t go; /* start crc update */ 568 wait_queue_head_t done; /* crc update done */ 569 u32 *crc32; /* points to handle's crc32 */ 570 size_t **unc_len; /* uncompressed lengths */ 571 unsigned char **unc; /* uncompressed data */ 572 }; 573 574 static struct crc_data *alloc_crc_data(int nr_threads) 575 { 576 struct crc_data *crc; 577 578 crc = kzalloc(sizeof(*crc), GFP_KERNEL); 579 if (!crc) 580 return NULL; 581 582 crc->unc = kcalloc(nr_threads, sizeof(*crc->unc), GFP_KERNEL); 583 if (!crc->unc) 584 goto err_free_crc; 585 586 crc->unc_len = kcalloc(nr_threads, sizeof(*crc->unc_len), GFP_KERNEL); 587 if (!crc->unc_len) 588 goto err_free_unc; 589 590 return crc; 591 592 err_free_unc: 593 kfree(crc->unc); 594 err_free_crc: 595 kfree(crc); 596 return NULL; 597 } 598 599 static void free_crc_data(struct crc_data *crc) 600 { 601 if (!crc) 602 return; 603 604 if (crc->thr) 605 kthread_stop(crc->thr); 606 607 kfree(crc->unc_len); 608 kfree(crc->unc); 609 kfree(crc); 610 } 611 612 static int crc32_threadfn(void *data) 613 { 614 struct crc_data *d = data; 615 unsigned i; 616 617 while (1) { 618 wait_event(d->go, atomic_read_acquire(&d->ready) || 619 kthread_should_stop()); 620 if (kthread_should_stop()) { 621 d->thr = NULL; 622 atomic_set_release(&d->stop, 1); 623 wake_up(&d->done); 624 break; 625 } 626 atomic_set(&d->ready, 0); 627 628 for (i = 0; i < d->run_threads; i++) 629 *d->crc32 = crc32_le(*d->crc32, 630 d->unc[i], *d->unc_len[i]); 631 atomic_set_release(&d->stop, 1); 632 wake_up(&d->done); 633 } 634 return 0; 635 } 636 637 /* 638 * Structure used for data compression. 639 */ 640 struct cmp_data { 641 struct task_struct *thr; /* thread */ 642 struct crypto_acomp *cc; /* crypto compressor */ 643 struct acomp_req *cr; /* crypto request */ 644 atomic_t ready; /* ready to start flag */ 645 atomic_t stop; /* ready to stop flag */ 646 int ret; /* return code */ 647 wait_queue_head_t go; /* start compression */ 648 wait_queue_head_t done; /* compression done */ 649 size_t unc_len; /* uncompressed length */ 650 size_t cmp_len; /* compressed length */ 651 unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ 652 unsigned char cmp[CMP_SIZE]; /* compressed buffer */ 653 }; 654 655 /* Indicates the image size after compression */ 656 static atomic64_t compressed_size = ATOMIC_INIT(0); 657 658 static int compress_threadfn(void *data) 659 { 660 struct cmp_data *d = data; 661 662 while (1) { 663 wait_event(d->go, atomic_read_acquire(&d->ready) || 664 kthread_should_stop()); 665 if (kthread_should_stop()) { 666 d->thr = NULL; 667 d->ret = -1; 668 atomic_set_release(&d->stop, 1); 669 wake_up(&d->done); 670 break; 671 } 672 atomic_set(&d->ready, 0); 673 674 acomp_request_set_callback(d->cr, CRYPTO_TFM_REQ_MAY_SLEEP, 675 NULL, NULL); 676 acomp_request_set_src_nondma(d->cr, d->unc, d->unc_len); 677 acomp_request_set_dst_nondma(d->cr, d->cmp + CMP_HEADER, 678 CMP_SIZE - CMP_HEADER); 679 d->ret = crypto_acomp_compress(d->cr); 680 d->cmp_len = d->cr->dlen; 681 682 atomic64_add(d->cmp_len, &compressed_size); 683 atomic_set_release(&d->stop, 1); 684 wake_up(&d->done); 685 } 686 return 0; 687 } 688 689 static int save_compressed_image(struct swap_map_handle *handle, 690 struct snapshot_handle *snapshot, 691 unsigned int nr_to_write) 692 { 693 unsigned int m; 694 int ret = 0; 695 int nr_pages; 696 int err2; 697 struct hib_bio_batch hb; 698 ktime_t start; 699 ktime_t stop; 700 size_t off; 701 unsigned int thr, run_threads, nr_threads; 702 unsigned char *page = NULL; 703 struct cmp_data *data = NULL; 704 struct crc_data *crc = NULL; 705 706 hib_init_batch(&hb); 707 708 atomic64_set(&compressed_size, 0); 709 710 /* 711 * We'll limit the number of threads for compression to limit memory 712 * footprint. 713 */ 714 nr_threads = num_online_cpus() - 1; 715 nr_threads = clamp_val(nr_threads, 1, hibernate_compression_threads); 716 717 page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); 718 if (!page) { 719 pr_err("Failed to allocate %s page\n", hib_comp_algo); 720 ret = -ENOMEM; 721 goto out_clean; 722 } 723 724 data = vcalloc(nr_threads, sizeof(*data)); 725 if (!data) { 726 pr_err("Failed to allocate %s data\n", hib_comp_algo); 727 ret = -ENOMEM; 728 goto out_clean; 729 } 730 731 crc = alloc_crc_data(nr_threads); 732 if (!crc) { 733 pr_err("Failed to allocate crc\n"); 734 ret = -ENOMEM; 735 goto out_clean; 736 } 737 738 /* 739 * Start the compression threads. 740 */ 741 for (thr = 0; thr < nr_threads; thr++) { 742 init_waitqueue_head(&data[thr].go); 743 init_waitqueue_head(&data[thr].done); 744 745 data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC); 746 if (IS_ERR_OR_NULL(data[thr].cc)) { 747 pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); 748 ret = -EFAULT; 749 goto out_clean; 750 } 751 752 data[thr].cr = acomp_request_alloc(data[thr].cc); 753 if (!data[thr].cr) { 754 pr_err("Could not allocate comp request\n"); 755 ret = -ENOMEM; 756 goto out_clean; 757 } 758 759 data[thr].thr = kthread_run(compress_threadfn, 760 &data[thr], 761 "image_compress/%u", thr); 762 if (IS_ERR(data[thr].thr)) { 763 data[thr].thr = NULL; 764 pr_err("Cannot start compression threads\n"); 765 ret = -ENOMEM; 766 goto out_clean; 767 } 768 } 769 770 /* 771 * Start the CRC32 thread. 772 */ 773 init_waitqueue_head(&crc->go); 774 init_waitqueue_head(&crc->done); 775 776 handle->crc32 = 0; 777 crc->crc32 = &handle->crc32; 778 for (thr = 0; thr < nr_threads; thr++) { 779 crc->unc[thr] = data[thr].unc; 780 crc->unc_len[thr] = &data[thr].unc_len; 781 } 782 783 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32"); 784 if (IS_ERR(crc->thr)) { 785 crc->thr = NULL; 786 pr_err("Cannot start CRC32 thread\n"); 787 ret = -ENOMEM; 788 goto out_clean; 789 } 790 791 /* 792 * Adjust the number of required free pages after all allocations have 793 * been done. We don't want to run out of pages when writing. 794 */ 795 handle->reqd_free_pages = reqd_free_pages(); 796 797 pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo); 798 pr_info("Compressing and saving image data (%u pages)...\n", 799 nr_to_write); 800 m = nr_to_write / 10; 801 if (!m) 802 m = 1; 803 nr_pages = 0; 804 start = ktime_get(); 805 for (;;) { 806 for (thr = 0; thr < nr_threads; thr++) { 807 for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) { 808 ret = snapshot_read_next(snapshot); 809 if (ret < 0) 810 goto out_finish; 811 812 if (!ret) 813 break; 814 815 memcpy(data[thr].unc + off, 816 data_of(*snapshot), PAGE_SIZE); 817 818 if (!(nr_pages % m)) 819 pr_info("Image saving progress: %3d%%\n", 820 nr_pages / m * 10); 821 nr_pages++; 822 } 823 if (!off) 824 break; 825 826 data[thr].unc_len = off; 827 828 atomic_set_release(&data[thr].ready, 1); 829 wake_up(&data[thr].go); 830 } 831 832 if (!thr) 833 break; 834 835 crc->run_threads = thr; 836 atomic_set_release(&crc->ready, 1); 837 wake_up(&crc->go); 838 839 for (run_threads = thr, thr = 0; thr < run_threads; thr++) { 840 wait_event(data[thr].done, 841 atomic_read_acquire(&data[thr].stop)); 842 atomic_set(&data[thr].stop, 0); 843 844 ret = data[thr].ret; 845 846 if (ret < 0) { 847 pr_err("%s compression failed\n", hib_comp_algo); 848 goto out_finish; 849 } 850 851 if (unlikely(!data[thr].cmp_len || 852 data[thr].cmp_len > 853 bytes_worst_compress(data[thr].unc_len))) { 854 pr_err("Invalid %s compressed length\n", hib_comp_algo); 855 ret = -1; 856 goto out_finish; 857 } 858 859 *(size_t *)data[thr].cmp = data[thr].cmp_len; 860 861 /* 862 * Given we are writing one page at a time to disk, we 863 * copy that much from the buffer, although the last 864 * bit will likely be smaller than full page. This is 865 * OK - we saved the length of the compressed data, so 866 * any garbage at the end will be discarded when we 867 * read it. 868 */ 869 for (off = 0; 870 off < CMP_HEADER + data[thr].cmp_len; 871 off += PAGE_SIZE) { 872 memcpy(page, data[thr].cmp + off, PAGE_SIZE); 873 874 ret = swap_write_page(handle, page, &hb); 875 if (ret) 876 goto out_finish; 877 } 878 } 879 880 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 881 atomic_set(&crc->stop, 0); 882 } 883 884 out_finish: 885 err2 = hib_wait_io(&hb); 886 stop = ktime_get(); 887 if (!ret) 888 ret = err2; 889 if (!ret) { 890 swsusp_show_speed(start, stop, nr_to_write, "Wrote"); 891 pr_info("Image size after compression: %lld kbytes\n", 892 (atomic64_read(&compressed_size) / 1024)); 893 pr_info("Image saving done\n"); 894 } else { 895 pr_err("Image saving failed: %d\n", ret); 896 } 897 898 out_clean: 899 hib_finish_batch(&hb); 900 free_crc_data(crc); 901 if (data) { 902 for (thr = 0; thr < nr_threads; thr++) { 903 if (data[thr].thr) 904 kthread_stop(data[thr].thr); 905 if (data[thr].cr) 906 acomp_request_free(data[thr].cr); 907 908 if (!IS_ERR_OR_NULL(data[thr].cc)) 909 crypto_free_acomp(data[thr].cc); 910 } 911 vfree(data); 912 } 913 if (page) 914 free_page((unsigned long)page); 915 916 return ret; 917 } 918 919 static int enough_swap(unsigned int nr_pages) 920 { 921 unsigned int free_swap = count_swap_pages(root_swap, 1); 922 unsigned int required; 923 924 pr_debug("Free swap pages: %u\n", free_swap); 925 926 required = PAGES_FOR_IO + nr_pages; 927 return free_swap > required; 928 } 929 930 /** 931 * swsusp_write - Write entire image and metadata. 932 * @flags: flags to pass to the "boot" kernel in the image header 933 * 934 * It is important _NOT_ to umount filesystems at this point. We want them 935 * synced (in case something goes wrong) but we DO not want to mark filesystem 936 * clean: it is not. (And it does not matter, if we resume correctly, we'll mark 937 * system clean, anyway.) 938 * 939 * Return: 0 on success, negative error code on failure. 940 */ 941 int swsusp_write(unsigned int flags) 942 { 943 struct swap_map_handle handle; 944 struct snapshot_handle snapshot; 945 struct swsusp_info *header; 946 unsigned long pages; 947 int error; 948 949 pages = snapshot_get_image_size(); 950 error = get_swap_writer(&handle); 951 if (error) { 952 pr_err("Cannot get swap writer\n"); 953 return error; 954 } 955 if (flags & SF_NOCOMPRESS_MODE) { 956 if (!enough_swap(pages)) { 957 pr_err("Not enough free swap\n"); 958 error = -ENOSPC; 959 goto out_finish; 960 } 961 } 962 memset(&snapshot, 0, sizeof(struct snapshot_handle)); 963 error = snapshot_read_next(&snapshot); 964 if (error < (int)PAGE_SIZE) { 965 if (error >= 0) 966 error = -EFAULT; 967 968 goto out_finish; 969 } 970 header = (struct swsusp_info *)data_of(snapshot); 971 error = swap_write_page(&handle, header, NULL); 972 if (!error) { 973 error = (flags & SF_NOCOMPRESS_MODE) ? 974 save_image(&handle, &snapshot, pages - 1) : 975 save_compressed_image(&handle, &snapshot, pages - 1); 976 } 977 out_finish: 978 error = swap_writer_finish(&handle, flags, error); 979 return error; 980 } 981 982 /* 983 * The following functions allow us to read data using a swap map in a file-like 984 * way. 985 */ 986 987 static void release_swap_reader(struct swap_map_handle *handle) 988 { 989 struct swap_map_page_list *tmp; 990 991 while (handle->maps) { 992 if (handle->maps->map) 993 free_page((unsigned long)handle->maps->map); 994 tmp = handle->maps; 995 handle->maps = handle->maps->next; 996 kfree(tmp); 997 } 998 handle->cur = NULL; 999 } 1000 1001 static int get_swap_reader(struct swap_map_handle *handle, 1002 unsigned int *flags_p) 1003 { 1004 int error; 1005 struct swap_map_page_list *tmp, *last; 1006 sector_t offset; 1007 1008 *flags_p = swsusp_header->flags; 1009 1010 if (!swsusp_header->image) /* how can this happen? */ 1011 return -EINVAL; 1012 1013 handle->cur = NULL; 1014 last = handle->maps = NULL; 1015 offset = swsusp_header->image; 1016 while (offset) { 1017 tmp = kzalloc(sizeof(*handle->maps), GFP_KERNEL); 1018 if (!tmp) { 1019 release_swap_reader(handle); 1020 return -ENOMEM; 1021 } 1022 if (!handle->maps) 1023 handle->maps = tmp; 1024 if (last) 1025 last->next = tmp; 1026 last = tmp; 1027 1028 tmp->map = (struct swap_map_page *) 1029 __get_free_page(GFP_NOIO | __GFP_HIGH); 1030 if (!tmp->map) { 1031 release_swap_reader(handle); 1032 return -ENOMEM; 1033 } 1034 1035 error = hib_submit_io_sync(REQ_OP_READ, offset, tmp->map); 1036 if (error) { 1037 release_swap_reader(handle); 1038 return error; 1039 } 1040 offset = tmp->map->next_swap; 1041 } 1042 handle->k = 0; 1043 handle->cur = handle->maps->map; 1044 return 0; 1045 } 1046 1047 static int swap_read_page(struct swap_map_handle *handle, void *buf, 1048 struct hib_bio_batch *hb) 1049 { 1050 sector_t offset; 1051 int error; 1052 struct swap_map_page_list *tmp; 1053 1054 if (!handle->cur) 1055 return -EINVAL; 1056 offset = handle->cur->entries[handle->k]; 1057 if (!offset) 1058 return -EFAULT; 1059 if (hb) 1060 error = hib_submit_io_async(REQ_OP_READ, offset, buf, hb); 1061 else 1062 error = hib_submit_io_sync(REQ_OP_READ, offset, buf); 1063 if (error) 1064 return error; 1065 if (++handle->k >= MAP_PAGE_ENTRIES) { 1066 handle->k = 0; 1067 free_page((unsigned long)handle->maps->map); 1068 tmp = handle->maps; 1069 handle->maps = handle->maps->next; 1070 kfree(tmp); 1071 if (!handle->maps) 1072 release_swap_reader(handle); 1073 else 1074 handle->cur = handle->maps->map; 1075 } 1076 return error; 1077 } 1078 1079 static int swap_reader_finish(struct swap_map_handle *handle) 1080 { 1081 release_swap_reader(handle); 1082 1083 return 0; 1084 } 1085 1086 static int load_image(struct swap_map_handle *handle, 1087 struct snapshot_handle *snapshot, 1088 unsigned int nr_to_read) 1089 { 1090 unsigned int m; 1091 int ret = 0; 1092 ktime_t start; 1093 ktime_t stop; 1094 struct hib_bio_batch hb; 1095 int err2; 1096 unsigned nr_pages; 1097 1098 hib_init_batch(&hb); 1099 1100 clean_pages_on_read = true; 1101 pr_info("Loading image data pages (%u pages)...\n", nr_to_read); 1102 m = nr_to_read / 10; 1103 if (!m) 1104 m = 1; 1105 nr_pages = 0; 1106 start = ktime_get(); 1107 for ( ; ; ) { 1108 ret = snapshot_write_next(snapshot); 1109 if (ret <= 0) 1110 break; 1111 ret = swap_read_page(handle, data_of(*snapshot), &hb); 1112 if (ret) 1113 break; 1114 if (snapshot->sync_read) 1115 ret = hib_wait_io(&hb); 1116 if (ret) 1117 break; 1118 if (!(nr_pages % m)) 1119 pr_info("Image loading progress: %3d%%\n", 1120 nr_pages / m * 10); 1121 nr_pages++; 1122 } 1123 err2 = hib_wait_io(&hb); 1124 hib_finish_batch(&hb); 1125 stop = ktime_get(); 1126 if (!ret) 1127 ret = err2; 1128 if (!ret) { 1129 pr_info("Image loading done\n"); 1130 ret = snapshot_write_finalize(snapshot); 1131 if (!ret && !snapshot_image_loaded(snapshot)) 1132 ret = -ENODATA; 1133 } 1134 swsusp_show_speed(start, stop, nr_to_read, "Read"); 1135 return ret; 1136 } 1137 1138 /* 1139 * Structure used for data decompression. 1140 */ 1141 struct dec_data { 1142 struct task_struct *thr; /* thread */ 1143 struct crypto_acomp *cc; /* crypto compressor */ 1144 struct acomp_req *cr; /* crypto request */ 1145 atomic_t ready; /* ready to start flag */ 1146 atomic_t stop; /* ready to stop flag */ 1147 int ret; /* return code */ 1148 wait_queue_head_t go; /* start decompression */ 1149 wait_queue_head_t done; /* decompression done */ 1150 size_t unc_len; /* uncompressed length */ 1151 size_t cmp_len; /* compressed length */ 1152 unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ 1153 unsigned char cmp[CMP_SIZE]; /* compressed buffer */ 1154 }; 1155 1156 static int decompress_threadfn(void *data) 1157 { 1158 struct dec_data *d = data; 1159 1160 while (1) { 1161 wait_event(d->go, atomic_read_acquire(&d->ready) || 1162 kthread_should_stop()); 1163 if (kthread_should_stop()) { 1164 d->thr = NULL; 1165 d->ret = -1; 1166 atomic_set_release(&d->stop, 1); 1167 wake_up(&d->done); 1168 break; 1169 } 1170 atomic_set(&d->ready, 0); 1171 1172 acomp_request_set_callback(d->cr, CRYPTO_TFM_REQ_MAY_SLEEP, 1173 NULL, NULL); 1174 acomp_request_set_src_nondma(d->cr, d->cmp + CMP_HEADER, 1175 d->cmp_len); 1176 acomp_request_set_dst_nondma(d->cr, d->unc, UNC_SIZE); 1177 d->ret = crypto_acomp_decompress(d->cr); 1178 d->unc_len = d->cr->dlen; 1179 1180 if (clean_pages_on_decompress) 1181 flush_icache_range((unsigned long)d->unc, 1182 (unsigned long)d->unc + d->unc_len); 1183 1184 atomic_set_release(&d->stop, 1); 1185 wake_up(&d->done); 1186 } 1187 return 0; 1188 } 1189 1190 static int load_compressed_image(struct swap_map_handle *handle, 1191 struct snapshot_handle *snapshot, 1192 unsigned int nr_to_read) 1193 { 1194 unsigned int m; 1195 int ret = 0; 1196 int eof = 0; 1197 struct hib_bio_batch hb; 1198 ktime_t start; 1199 ktime_t stop; 1200 unsigned nr_pages; 1201 size_t off; 1202 unsigned i, thr, run_threads, nr_threads; 1203 unsigned ring = 0, pg = 0, ring_size = 0, 1204 have = 0, want, need, asked = 0; 1205 unsigned long read_pages = 0; 1206 unsigned char **page = NULL; 1207 struct dec_data *data = NULL; 1208 struct crc_data *crc = NULL; 1209 1210 hib_init_batch(&hb); 1211 1212 /* 1213 * We'll limit the number of threads for decompression to limit memory 1214 * footprint. 1215 */ 1216 nr_threads = num_online_cpus() - 1; 1217 nr_threads = clamp_val(nr_threads, 1, hibernate_compression_threads); 1218 1219 page = vmalloc_array(CMP_MAX_RD_PAGES, sizeof(*page)); 1220 if (!page) { 1221 pr_err("Failed to allocate %s page\n", hib_comp_algo); 1222 ret = -ENOMEM; 1223 goto out_clean; 1224 } 1225 1226 data = vcalloc(nr_threads, sizeof(*data)); 1227 if (!data) { 1228 pr_err("Failed to allocate %s data\n", hib_comp_algo); 1229 ret = -ENOMEM; 1230 goto out_clean; 1231 } 1232 1233 crc = alloc_crc_data(nr_threads); 1234 if (!crc) { 1235 pr_err("Failed to allocate crc\n"); 1236 ret = -ENOMEM; 1237 goto out_clean; 1238 } 1239 1240 clean_pages_on_decompress = true; 1241 1242 /* 1243 * Start the decompression threads. 1244 */ 1245 for (thr = 0; thr < nr_threads; thr++) { 1246 init_waitqueue_head(&data[thr].go); 1247 init_waitqueue_head(&data[thr].done); 1248 1249 data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC); 1250 if (IS_ERR_OR_NULL(data[thr].cc)) { 1251 pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); 1252 ret = -EFAULT; 1253 goto out_clean; 1254 } 1255 1256 data[thr].cr = acomp_request_alloc(data[thr].cc); 1257 if (!data[thr].cr) { 1258 pr_err("Could not allocate comp request\n"); 1259 ret = -ENOMEM; 1260 goto out_clean; 1261 } 1262 1263 data[thr].thr = kthread_run(decompress_threadfn, 1264 &data[thr], 1265 "image_decompress/%u", thr); 1266 if (IS_ERR(data[thr].thr)) { 1267 data[thr].thr = NULL; 1268 pr_err("Cannot start decompression threads\n"); 1269 ret = -ENOMEM; 1270 goto out_clean; 1271 } 1272 } 1273 1274 /* 1275 * Start the CRC32 thread. 1276 */ 1277 init_waitqueue_head(&crc->go); 1278 init_waitqueue_head(&crc->done); 1279 1280 handle->crc32 = 0; 1281 crc->crc32 = &handle->crc32; 1282 for (thr = 0; thr < nr_threads; thr++) { 1283 crc->unc[thr] = data[thr].unc; 1284 crc->unc_len[thr] = &data[thr].unc_len; 1285 } 1286 1287 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32"); 1288 if (IS_ERR(crc->thr)) { 1289 crc->thr = NULL; 1290 pr_err("Cannot start CRC32 thread\n"); 1291 ret = -ENOMEM; 1292 goto out_clean; 1293 } 1294 1295 /* 1296 * Set the number of pages for read buffering. 1297 * This is complete guesswork, because we'll only know the real 1298 * picture once prepare_image() is called, which is much later on 1299 * during the image load phase. We'll assume the worst case and 1300 * say that none of the image pages are from high memory. 1301 */ 1302 if (low_free_pages() > snapshot_get_image_size()) 1303 read_pages = (low_free_pages() - snapshot_get_image_size()) / 2; 1304 read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES); 1305 1306 for (i = 0; i < read_pages; i++) { 1307 page[i] = (void *)__get_free_page(i < CMP_PAGES ? 1308 GFP_NOIO | __GFP_HIGH : 1309 GFP_NOIO | __GFP_NOWARN | 1310 __GFP_NORETRY); 1311 1312 if (!page[i]) { 1313 if (i < CMP_PAGES) { 1314 ring_size = i; 1315 pr_err("Failed to allocate %s pages\n", hib_comp_algo); 1316 ret = -ENOMEM; 1317 goto out_clean; 1318 } else { 1319 break; 1320 } 1321 } 1322 } 1323 want = ring_size = i; 1324 1325 pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo); 1326 pr_info("Loading and decompressing image data (%u pages)...\n", 1327 nr_to_read); 1328 m = nr_to_read / 10; 1329 if (!m) 1330 m = 1; 1331 nr_pages = 0; 1332 start = ktime_get(); 1333 1334 ret = snapshot_write_next(snapshot); 1335 if (ret <= 0) 1336 goto out_finish; 1337 1338 for(;;) { 1339 for (i = 0; !eof && i < want; i++) { 1340 ret = swap_read_page(handle, page[ring], &hb); 1341 if (ret) { 1342 /* 1343 * On real read error, finish. On end of data, 1344 * set EOF flag and just exit the read loop. 1345 */ 1346 if (handle->cur && 1347 handle->cur->entries[handle->k]) { 1348 goto out_finish; 1349 } else { 1350 eof = 1; 1351 break; 1352 } 1353 } 1354 if (++ring >= ring_size) 1355 ring = 0; 1356 } 1357 asked += i; 1358 want -= i; 1359 1360 /* 1361 * We are out of data, wait for some more. 1362 */ 1363 if (!have) { 1364 if (!asked) 1365 break; 1366 1367 ret = hib_wait_io(&hb); 1368 if (ret) 1369 goto out_finish; 1370 have += asked; 1371 asked = 0; 1372 if (eof) 1373 eof = 2; 1374 } 1375 1376 if (crc->run_threads) { 1377 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 1378 atomic_set(&crc->stop, 0); 1379 crc->run_threads = 0; 1380 } 1381 1382 for (thr = 0; have && thr < nr_threads; thr++) { 1383 data[thr].cmp_len = *(size_t *)page[pg]; 1384 if (unlikely(!data[thr].cmp_len || 1385 data[thr].cmp_len > 1386 bytes_worst_compress(UNC_SIZE))) { 1387 pr_err("Invalid %s compressed length\n", hib_comp_algo); 1388 ret = -1; 1389 goto out_finish; 1390 } 1391 1392 need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER, 1393 PAGE_SIZE); 1394 if (need > have) { 1395 if (eof > 1) { 1396 ret = -1; 1397 goto out_finish; 1398 } 1399 break; 1400 } 1401 1402 for (off = 0; 1403 off < CMP_HEADER + data[thr].cmp_len; 1404 off += PAGE_SIZE) { 1405 memcpy(data[thr].cmp + off, 1406 page[pg], PAGE_SIZE); 1407 have--; 1408 want++; 1409 if (++pg >= ring_size) 1410 pg = 0; 1411 } 1412 1413 atomic_set_release(&data[thr].ready, 1); 1414 wake_up(&data[thr].go); 1415 } 1416 1417 /* 1418 * Wait for more data while we are decompressing. 1419 */ 1420 if (have < CMP_PAGES && asked) { 1421 ret = hib_wait_io(&hb); 1422 if (ret) 1423 goto out_finish; 1424 have += asked; 1425 asked = 0; 1426 if (eof) 1427 eof = 2; 1428 } 1429 1430 for (run_threads = thr, thr = 0; thr < run_threads; thr++) { 1431 wait_event(data[thr].done, 1432 atomic_read_acquire(&data[thr].stop)); 1433 atomic_set(&data[thr].stop, 0); 1434 1435 ret = data[thr].ret; 1436 1437 if (ret < 0) { 1438 pr_err("%s decompression failed\n", hib_comp_algo); 1439 goto out_finish; 1440 } 1441 1442 if (unlikely(!data[thr].unc_len || 1443 data[thr].unc_len > UNC_SIZE || 1444 data[thr].unc_len & (PAGE_SIZE - 1))) { 1445 pr_err("Invalid %s uncompressed length\n", hib_comp_algo); 1446 ret = -1; 1447 goto out_finish; 1448 } 1449 1450 for (off = 0; 1451 off < data[thr].unc_len; off += PAGE_SIZE) { 1452 memcpy(data_of(*snapshot), 1453 data[thr].unc + off, PAGE_SIZE); 1454 1455 if (!(nr_pages % m)) 1456 pr_info("Image loading progress: %3d%%\n", 1457 nr_pages / m * 10); 1458 nr_pages++; 1459 1460 ret = snapshot_write_next(snapshot); 1461 if (ret <= 0) { 1462 crc->run_threads = thr + 1; 1463 atomic_set_release(&crc->ready, 1); 1464 wake_up(&crc->go); 1465 goto out_finish; 1466 } 1467 } 1468 } 1469 1470 crc->run_threads = thr; 1471 atomic_set_release(&crc->ready, 1); 1472 wake_up(&crc->go); 1473 } 1474 1475 out_finish: 1476 if (crc->run_threads) { 1477 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 1478 atomic_set(&crc->stop, 0); 1479 } 1480 stop = ktime_get(); 1481 if (!ret) { 1482 pr_info("Image loading done\n"); 1483 ret = snapshot_write_finalize(snapshot); 1484 if (!ret && !snapshot_image_loaded(snapshot)) 1485 ret = -ENODATA; 1486 if (!ret) { 1487 if (swsusp_header->flags & SF_CRC32_MODE) { 1488 if(handle->crc32 != swsusp_header->crc32) { 1489 pr_err("Invalid image CRC32!\n"); 1490 ret = -ENODATA; 1491 } 1492 } 1493 } 1494 } 1495 swsusp_show_speed(start, stop, nr_to_read, "Read"); 1496 out_clean: 1497 hib_finish_batch(&hb); 1498 for (i = 0; i < ring_size; i++) 1499 free_page((unsigned long)page[i]); 1500 free_crc_data(crc); 1501 if (data) { 1502 for (thr = 0; thr < nr_threads; thr++) { 1503 if (data[thr].thr) 1504 kthread_stop(data[thr].thr); 1505 if (data[thr].cr) 1506 acomp_request_free(data[thr].cr); 1507 1508 if (!IS_ERR_OR_NULL(data[thr].cc)) 1509 crypto_free_acomp(data[thr].cc); 1510 } 1511 vfree(data); 1512 } 1513 vfree(page); 1514 1515 return ret; 1516 } 1517 1518 /** 1519 * swsusp_read - read the hibernation image. 1520 * @flags_p: flags passed by the "frozen" kernel in the image header should 1521 * be written into this memory location 1522 * 1523 * Return: 0 on success, negative error code on failure. 1524 */ 1525 int swsusp_read(unsigned int *flags_p) 1526 { 1527 int error; 1528 struct swap_map_handle handle; 1529 struct snapshot_handle snapshot; 1530 struct swsusp_info *header; 1531 1532 memset(&snapshot, 0, sizeof(struct snapshot_handle)); 1533 error = snapshot_write_next(&snapshot); 1534 if (error < (int)PAGE_SIZE) 1535 return error < 0 ? error : -EFAULT; 1536 header = (struct swsusp_info *)data_of(snapshot); 1537 error = get_swap_reader(&handle, flags_p); 1538 if (error) 1539 goto end; 1540 if (!error) 1541 error = swap_read_page(&handle, header, NULL); 1542 if (!error) { 1543 error = (*flags_p & SF_NOCOMPRESS_MODE) ? 1544 load_image(&handle, &snapshot, header->pages - 1) : 1545 load_compressed_image(&handle, &snapshot, header->pages - 1); 1546 } 1547 swap_reader_finish(&handle); 1548 end: 1549 if (!error) 1550 pr_debug("Image successfully loaded\n"); 1551 else 1552 pr_debug("Error %d resuming\n", error); 1553 return error; 1554 } 1555 1556 static void *swsusp_holder; 1557 1558 /** 1559 * swsusp_check - Open the resume device and check for the swsusp signature. 1560 * @exclusive: Open the resume device exclusively. 1561 * 1562 * Return: 0 if a valid image is found, negative error code otherwise. 1563 */ 1564 int swsusp_check(bool exclusive) 1565 { 1566 void *holder = exclusive ? &swsusp_holder : NULL; 1567 int error; 1568 1569 hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device, 1570 BLK_OPEN_READ, holder, NULL); 1571 if (!IS_ERR(hib_resume_bdev_file)) { 1572 clear_page(swsusp_header); 1573 error = hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, 1574 swsusp_header); 1575 if (error) 1576 goto put; 1577 1578 if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { 1579 memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); 1580 swsusp_header_flags = swsusp_header->flags; 1581 /* Reset swap signature now */ 1582 error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, 1583 swsusp_resume_block, 1584 swsusp_header); 1585 } else { 1586 error = -EINVAL; 1587 } 1588 if (!error && swsusp_header->flags & SF_HW_SIG && 1589 swsusp_header->hw_sig != swsusp_hardware_signature) { 1590 pr_info("Suspend image hardware signature mismatch (%08x now %08x); aborting resume.\n", 1591 swsusp_header->hw_sig, swsusp_hardware_signature); 1592 error = -EINVAL; 1593 } 1594 1595 put: 1596 if (error) 1597 bdev_fput(hib_resume_bdev_file); 1598 else 1599 pr_debug("Image signature found, resuming\n"); 1600 } else { 1601 error = PTR_ERR(hib_resume_bdev_file); 1602 } 1603 1604 if (error) 1605 pr_debug("Image not found (code %d)\n", error); 1606 1607 return error; 1608 } 1609 1610 /** 1611 * swsusp_close - close resume device. 1612 */ 1613 void swsusp_close(void) 1614 { 1615 if (IS_ERR(hib_resume_bdev_file)) { 1616 pr_debug("Image device not initialised\n"); 1617 return; 1618 } 1619 1620 fput(hib_resume_bdev_file); 1621 } 1622 1623 /** 1624 * swsusp_unmark - Unmark swsusp signature in the resume device 1625 * 1626 * Return: 0 on success, negative error code on failure. 1627 */ 1628 #ifdef CONFIG_SUSPEND 1629 int swsusp_unmark(void) 1630 { 1631 int error; 1632 1633 hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header); 1634 if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) { 1635 memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10); 1636 error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, 1637 swsusp_resume_block, 1638 swsusp_header); 1639 } else { 1640 pr_err("Cannot find swsusp signature!\n"); 1641 error = -ENODEV; 1642 } 1643 1644 /* 1645 * We just returned from suspend, we don't need the image any more. 1646 */ 1647 free_all_swap_pages(root_swap); 1648 1649 return error; 1650 } 1651 #endif 1652 1653 static ssize_t hibernate_compression_threads_show(struct kobject *kobj, 1654 struct kobj_attribute *attr, char *buf) 1655 { 1656 return sysfs_emit(buf, "%d\n", hibernate_compression_threads); 1657 } 1658 1659 static ssize_t hibernate_compression_threads_store(struct kobject *kobj, 1660 struct kobj_attribute *attr, 1661 const char *buf, size_t n) 1662 { 1663 unsigned long val; 1664 1665 if (kstrtoul(buf, 0, &val)) 1666 return -EINVAL; 1667 1668 if (val < 1) 1669 return -EINVAL; 1670 1671 hibernate_compression_threads = val; 1672 return n; 1673 } 1674 power_attr(hibernate_compression_threads); 1675 1676 static struct attribute *g[] = { 1677 &hibernate_compression_threads_attr.attr, 1678 NULL, 1679 }; 1680 1681 static const struct attribute_group attr_group = { 1682 .attrs = g, 1683 }; 1684 1685 static int __init swsusp_header_init(void) 1686 { 1687 int error; 1688 1689 error = sysfs_create_group(power_kobj, &attr_group); 1690 if (error) 1691 return -ENOMEM; 1692 1693 swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL); 1694 if (!swsusp_header) 1695 panic("Could not allocate memory for swsusp_header\n"); 1696 return 0; 1697 } 1698 1699 core_initcall(swsusp_header_init); 1700 1701 static int __init hibernate_compression_threads_setup(char *str) 1702 { 1703 int rc = kstrtouint(str, 0, &hibernate_compression_threads); 1704 1705 if (rc) 1706 return rc; 1707 1708 if (hibernate_compression_threads < 1) 1709 hibernate_compression_threads = CMP_THREADS; 1710 1711 return 1; 1712 1713 } 1714 1715 __setup("hibernate_compression_threads=", hibernate_compression_threads_setup); 1716