1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/kernel/power/swap.c 4 * 5 * This file provides functions for reading the suspend image from 6 * and writing it to a swap partition. 7 * 8 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz> 9 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 10 * Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com> 11 */ 12 13 #define pr_fmt(fmt) "PM: " fmt 14 15 #include <crypto/acompress.h> 16 #include <linux/module.h> 17 #include <linux/file.h> 18 #include <linux/delay.h> 19 #include <linux/bitops.h> 20 #include <linux/device.h> 21 #include <linux/bio.h> 22 #include <linux/blkdev.h> 23 #include <linux/swap.h> 24 #include <linux/swapops.h> 25 #include <linux/pm.h> 26 #include <linux/slab.h> 27 #include <linux/vmalloc.h> 28 #include <linux/cpumask.h> 29 #include <linux/atomic.h> 30 #include <linux/kthread.h> 31 #include <linux/crc32.h> 32 #include <linux/ktime.h> 33 34 #include "power.h" 35 36 #define HIBERNATE_SIG "S1SUSPEND" 37 38 u32 swsusp_hardware_signature; 39 40 /* 41 * When reading an {un,}compressed image, we may restore pages in place, 42 * in which case some architectures need these pages cleaning before they 43 * can be executed. We don't know which pages these may be, so clean the lot. 44 */ 45 static bool clean_pages_on_read; 46 static bool clean_pages_on_decompress; 47 48 /* 49 * The swap map is a data structure used for keeping track of each page 50 * written to a swap partition. It consists of many swap_map_page structures 51 * that contain each an array of MAP_PAGE_ENTRIES swap entries. These 52 * structures are stored on the swap and linked together with the help of the 53 * .next_swap member. 54 * 55 * The swap map is created during suspend. The swap map pages are allocated and 56 * populated one at a time, so we only need one memory page to set up the entire 57 * structure. 58 * 59 * During resume we pick up all swap_map_page structures into a list. 60 */ 61 #define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1) 62 63 /* 64 * Number of free pages that are not high. 65 */ 66 static inline unsigned long low_free_pages(void) 67 { 68 return nr_free_pages() - nr_free_highpages(); 69 } 70 71 /* 72 * Number of pages required to be kept free while writing the image. Always 73 * half of all available low pages before the writing starts. 74 */ 75 static inline unsigned long reqd_free_pages(void) 76 { 77 return low_free_pages() / 2; 78 } 79 80 struct swap_map_page { 81 sector_t entries[MAP_PAGE_ENTRIES]; 82 sector_t next_swap; 83 }; 84 85 struct swap_map_page_list { 86 struct swap_map_page *map; 87 struct swap_map_page_list *next; 88 }; 89 90 /* 91 * The swap_map_handle structure is used for handling swap in a file-alike way. 92 */ 93 struct swap_map_handle { 94 struct swap_map_page *cur; 95 struct swap_map_page_list *maps; 96 sector_t cur_swap; 97 sector_t first_sector; 98 unsigned int k; 99 unsigned long reqd_free_pages; 100 u32 crc32; 101 }; 102 103 struct swsusp_header { 104 char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) - 105 sizeof(u32) - sizeof(u32)]; 106 u32 hw_sig; 107 u32 crc32; 108 sector_t image; 109 unsigned int flags; /* Flags to pass to the "boot" kernel */ 110 char orig_sig[10]; 111 char sig[10]; 112 } __packed; 113 114 static struct swsusp_header *swsusp_header; 115 116 /* 117 * The following functions are used for tracing the allocated swap pages, so 118 * that they can be freed in case of an error. 119 */ 120 struct swsusp_extent { 121 struct rb_node node; 122 unsigned long start; 123 unsigned long end; 124 }; 125 126 static struct rb_root swsusp_extents = RB_ROOT; 127 128 static int swsusp_extents_insert(unsigned long swap_offset) 129 { 130 struct rb_node **new = &(swsusp_extents.rb_node); 131 struct rb_node *parent = NULL; 132 struct swsusp_extent *ext; 133 134 /* Figure out where to put the new node */ 135 while (*new) { 136 ext = rb_entry(*new, struct swsusp_extent, node); 137 parent = *new; 138 if (swap_offset < ext->start) { 139 /* Try to merge */ 140 if (swap_offset == ext->start - 1) { 141 ext->start--; 142 return 0; 143 } 144 new = &((*new)->rb_left); 145 } else if (swap_offset > ext->end) { 146 /* Try to merge */ 147 if (swap_offset == ext->end + 1) { 148 ext->end++; 149 return 0; 150 } 151 new = &((*new)->rb_right); 152 } else { 153 /* It already is in the tree */ 154 return -EINVAL; 155 } 156 } 157 /* Add the new node and rebalance the tree. */ 158 ext = kzalloc_obj(struct swsusp_extent); 159 if (!ext) 160 return -ENOMEM; 161 162 ext->start = swap_offset; 163 ext->end = swap_offset; 164 rb_link_node(&ext->node, parent, new); 165 rb_insert_color(&ext->node, &swsusp_extents); 166 return 0; 167 } 168 169 sector_t alloc_swapdev_block(int swap) 170 { 171 unsigned long offset; 172 173 /* 174 * Allocate a swap page and register that it has been allocated, so that 175 * it can be freed in case of an error. 176 */ 177 offset = swp_offset(swap_alloc_hibernation_slot(swap)); 178 if (offset) { 179 if (swsusp_extents_insert(offset)) 180 swap_free_hibernation_slot(swp_entry(swap, offset)); 181 else 182 return swapdev_block(swap, offset); 183 } 184 return 0; 185 } 186 187 void free_all_swap_pages(int swap) 188 { 189 unsigned long offset; 190 struct rb_node *node; 191 192 /* 193 * Free swap pages allocated for saving image data. It also frees the 194 * extents used to register which swap entries had been allocated. 195 */ 196 while ((node = swsusp_extents.rb_node)) { 197 struct swsusp_extent *ext; 198 199 ext = rb_entry(node, struct swsusp_extent, node); 200 rb_erase(node, &swsusp_extents); 201 202 for (offset = ext->start; offset <= ext->end; offset++) 203 swap_free_hibernation_slot(swp_entry(swap, offset)); 204 205 kfree(ext); 206 } 207 } 208 209 int swsusp_swap_in_use(void) 210 { 211 return (swsusp_extents.rb_node != NULL); 212 } 213 214 /* 215 * General things 216 */ 217 218 static unsigned short root_swap = 0xffff; 219 static struct file *hib_resume_bdev_file; 220 221 struct hib_bio_batch { 222 atomic_t count; 223 wait_queue_head_t wait; 224 blk_status_t error; 225 struct blk_plug plug; 226 }; 227 228 static void hib_init_batch(struct hib_bio_batch *hb) 229 { 230 atomic_set(&hb->count, 0); 231 init_waitqueue_head(&hb->wait); 232 hb->error = BLK_STS_OK; 233 blk_start_plug(&hb->plug); 234 } 235 236 static void hib_finish_batch(struct hib_bio_batch *hb) 237 { 238 blk_finish_plug(&hb->plug); 239 } 240 241 static void hib_end_io(struct bio *bio) 242 { 243 struct hib_bio_batch *hb = bio->bi_private; 244 struct page *page = bio_first_page_all(bio); 245 246 if (bio->bi_status) { 247 pr_alert("Read-error on swap-device (%u:%u:%Lu)\n", 248 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), 249 (unsigned long long)bio->bi_iter.bi_sector); 250 } 251 252 if (bio_data_dir(bio) == WRITE) 253 put_page(page); 254 else if (clean_pages_on_read) 255 flush_icache_range((unsigned long)page_address(page), 256 (unsigned long)page_address(page) + PAGE_SIZE); 257 258 if (bio->bi_status && !hb->error) 259 hb->error = bio->bi_status; 260 if (atomic_dec_and_test(&hb->count)) 261 wake_up(&hb->wait); 262 263 bio_put(bio); 264 } 265 266 static int hib_submit_io_sync(blk_opf_t opf, pgoff_t page_off, void *addr) 267 { 268 return bdev_rw_virt(file_bdev(hib_resume_bdev_file), 269 page_off * (PAGE_SIZE >> 9), addr, PAGE_SIZE, opf); 270 } 271 272 static int hib_submit_io_async(blk_opf_t opf, pgoff_t page_off, void *addr, 273 struct hib_bio_batch *hb) 274 { 275 struct bio *bio; 276 277 bio = bio_alloc(file_bdev(hib_resume_bdev_file), 1, opf, 278 GFP_NOIO | __GFP_HIGH); 279 bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9); 280 bio_add_virt_nofail(bio, addr, PAGE_SIZE); 281 bio->bi_end_io = hib_end_io; 282 bio->bi_private = hb; 283 atomic_inc(&hb->count); 284 submit_bio(bio); 285 return 0; 286 } 287 288 static int hib_wait_io(struct hib_bio_batch *hb) 289 { 290 /* 291 * We are relying on the behavior of blk_plug that a thread with 292 * a plug will flush the plug list before sleeping. 293 */ 294 wait_event(hb->wait, atomic_read(&hb->count) == 0); 295 return blk_status_to_errno(hb->error); 296 } 297 298 /* 299 * Saving part 300 */ 301 302 static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) 303 { 304 int error; 305 306 hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header); 307 if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) || 308 !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) { 309 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10); 310 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10); 311 swsusp_header->image = handle->first_sector; 312 if (swsusp_hardware_signature) { 313 swsusp_header->hw_sig = swsusp_hardware_signature; 314 flags |= SF_HW_SIG; 315 } 316 swsusp_header->flags = flags; 317 if (flags & SF_CRC32_MODE) 318 swsusp_header->crc32 = handle->crc32; 319 error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, 320 swsusp_resume_block, swsusp_header); 321 } else { 322 pr_err("Swap header not found!\n"); 323 error = -ENODEV; 324 } 325 return error; 326 } 327 328 /* 329 * Hold the swsusp_header flag. This is used in software_resume() in 330 * 'kernel/power/hibernate' to check if the image is compressed and query 331 * for the compression algorithm support(if so). 332 */ 333 unsigned int swsusp_header_flags; 334 335 static int swsusp_swap_check(void) 336 { 337 int res; 338 339 /* 340 * Check if the resume device is a swap device and get its index (if so). 341 * This is called before saving the image. 342 */ 343 if (swsusp_resume_device) 344 res = find_hibernation_swap_type(swsusp_resume_device, swsusp_resume_block); 345 else 346 res = find_first_swap(&swsusp_resume_device); 347 if (res < 0) 348 return res; 349 root_swap = res; 350 351 hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device, 352 BLK_OPEN_WRITE, NULL, NULL); 353 if (IS_ERR(hib_resume_bdev_file)) 354 return PTR_ERR(hib_resume_bdev_file); 355 356 return 0; 357 } 358 359 static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb) 360 { 361 gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY; 362 void *src; 363 int ret; 364 365 if (!offset) 366 return -ENOSPC; 367 368 if (!hb) 369 goto sync_io; 370 371 src = (void *)__get_free_page(gfp); 372 if (!src) { 373 ret = hib_wait_io(hb); /* Free pages */ 374 if (ret) 375 return ret; 376 src = (void *)__get_free_page(gfp); 377 if (WARN_ON_ONCE(!src)) 378 goto sync_io; 379 } 380 381 copy_page(src, buf); 382 return hib_submit_io_async(REQ_OP_WRITE | REQ_SYNC, offset, src, hb); 383 sync_io: 384 return hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, offset, buf); 385 } 386 387 static void release_swap_writer(struct swap_map_handle *handle) 388 { 389 if (handle->cur) 390 free_page((unsigned long)handle->cur); 391 handle->cur = NULL; 392 } 393 394 static int get_swap_writer(struct swap_map_handle *handle) 395 { 396 int ret; 397 398 ret = swsusp_swap_check(); 399 if (ret) { 400 if (ret != -ENOSPC) 401 pr_err("Cannot find swap device, try swapon -a\n"); 402 return ret; 403 } 404 handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL); 405 if (!handle->cur) { 406 ret = -ENOMEM; 407 goto err_close; 408 } 409 handle->cur_swap = alloc_swapdev_block(root_swap); 410 if (!handle->cur_swap) { 411 ret = -ENOSPC; 412 goto err_rel; 413 } 414 handle->k = 0; 415 handle->reqd_free_pages = reqd_free_pages(); 416 handle->first_sector = handle->cur_swap; 417 return 0; 418 err_rel: 419 release_swap_writer(handle); 420 err_close: 421 swsusp_close(); 422 return ret; 423 } 424 425 static int swap_write_page(struct swap_map_handle *handle, void *buf, 426 struct hib_bio_batch *hb) 427 { 428 int error; 429 sector_t offset; 430 431 if (!handle->cur) 432 return -EINVAL; 433 434 /* 435 * If the current map page is full, allocate and link next one first. 436 * Delaying this until here avoids writing an empty swap map page when 437 * the image size is an exact MAP_PAGE_ENTRIES multiple. 438 */ 439 if (handle->k >= MAP_PAGE_ENTRIES) { 440 offset = alloc_swapdev_block(root_swap); 441 if (!offset) 442 return -ENOSPC; 443 444 handle->cur->next_swap = offset; 445 error = write_page(handle->cur, handle->cur_swap, hb); 446 if (error) 447 return error; 448 449 clear_page(handle->cur); 450 handle->cur_swap = offset; 451 handle->k = 0; 452 453 if (hb && low_free_pages() <= handle->reqd_free_pages) { 454 error = hib_wait_io(hb); 455 if (error) 456 return error; 457 /* 458 * Recalculate the number of required free pages, to 459 * make sure we never take more than half. 460 */ 461 handle->reqd_free_pages = reqd_free_pages(); 462 } 463 } 464 465 offset = alloc_swapdev_block(root_swap); 466 error = write_page(buf, offset, hb); 467 if (error) 468 return error; 469 handle->cur->entries[handle->k++] = offset; 470 return 0; 471 } 472 473 static int flush_swap_writer(struct swap_map_handle *handle) 474 { 475 if (handle->cur && handle->cur_swap && handle->k) 476 return write_page(handle->cur, handle->cur_swap, NULL); 477 else if (handle->cur && handle->cur_swap) 478 return 0; 479 else 480 return -EINVAL; 481 } 482 483 static int swap_writer_finish(struct swap_map_handle *handle, 484 unsigned int flags, int error) 485 { 486 if (!error) { 487 pr_info("S"); 488 error = mark_swapfiles(handle, flags); 489 pr_cont("|\n"); 490 flush_swap_writer(handle); 491 } 492 493 if (error) 494 free_all_swap_pages(root_swap); 495 release_swap_writer(handle); 496 swsusp_close(); 497 498 return error; 499 } 500 501 /* 502 * Bytes we need for compressed data in worst case. We assume(limitation) 503 * this is the worst of all the compression algorithms. 504 */ 505 #define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2) 506 507 /* We need to remember how much compressed data we need to read. */ 508 #define CMP_HEADER sizeof(size_t) 509 510 /* Number of pages/bytes we'll compress at one time. */ 511 #define UNC_PAGES 32 512 #define UNC_SIZE (UNC_PAGES * PAGE_SIZE) 513 514 /* Number of pages we need for compressed data (worst case). */ 515 #define CMP_PAGES DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \ 516 CMP_HEADER, PAGE_SIZE) 517 #define CMP_SIZE (CMP_PAGES * PAGE_SIZE) 518 519 /* Default number of threads for compression/decompression. */ 520 #define CMP_THREADS 3 521 static unsigned int hibernate_compression_threads = CMP_THREADS; 522 523 /* Minimum/maximum number of pages for read buffering. */ 524 #define CMP_MIN_RD_PAGES 1024 525 #define CMP_MAX_RD_PAGES 8192 526 527 static int save_image(struct swap_map_handle *handle, 528 struct snapshot_handle *snapshot, 529 unsigned int nr_to_write) 530 { 531 unsigned int m; 532 int ret; 533 int nr_pages; 534 int err2; 535 struct hib_bio_batch hb; 536 ktime_t start; 537 ktime_t stop; 538 539 hib_init_batch(&hb); 540 541 pr_info("Saving image data pages (%u pages)...\n", 542 nr_to_write); 543 m = nr_to_write / 10; 544 if (!m) 545 m = 1; 546 nr_pages = 0; 547 start = ktime_get(); 548 while (1) { 549 ret = snapshot_read_next(snapshot); 550 if (ret <= 0) 551 break; 552 ret = swap_write_page(handle, data_of(*snapshot), &hb); 553 if (ret) 554 break; 555 if (!(nr_pages % m)) 556 pr_info("Image saving progress: %3d%%\n", 557 nr_pages / m * 10); 558 nr_pages++; 559 } 560 err2 = hib_wait_io(&hb); 561 hib_finish_batch(&hb); 562 stop = ktime_get(); 563 if (!ret) 564 ret = err2; 565 if (!ret) 566 pr_info("Image saving done\n"); 567 swsusp_show_speed(start, stop, nr_to_write, "Wrote"); 568 return ret; 569 } 570 571 /* 572 * Structure used for CRC32. 573 */ 574 struct crc_data { 575 struct task_struct *thr; /* thread */ 576 atomic_t ready; /* ready to start flag */ 577 atomic_t stop; /* ready to stop flag */ 578 unsigned run_threads; /* nr current threads */ 579 wait_queue_head_t go; /* start crc update */ 580 wait_queue_head_t done; /* crc update done */ 581 u32 *crc32; /* points to handle's crc32 */ 582 size_t **unc_len; /* uncompressed lengths */ 583 unsigned char *unc[]; /* uncompressed data */ 584 }; 585 586 static struct crc_data *alloc_crc_data(int nr_threads) 587 { 588 struct crc_data *crc; 589 590 crc = kzalloc_flex(*crc, unc, nr_threads); 591 if (!crc) 592 return NULL; 593 594 crc->unc_len = kzalloc_objs(*crc->unc_len, nr_threads); 595 if (!crc->unc_len) 596 goto err_free_crc; 597 598 return crc; 599 600 err_free_crc: 601 kfree(crc); 602 return NULL; 603 } 604 605 static void free_crc_data(struct crc_data *crc) 606 { 607 if (!crc) 608 return; 609 610 if (crc->thr) 611 kthread_stop(crc->thr); 612 613 kfree(crc->unc_len); 614 kfree(crc); 615 } 616 617 static int crc32_threadfn(void *data) 618 { 619 struct crc_data *d = data; 620 unsigned i; 621 622 while (1) { 623 wait_event(d->go, atomic_read_acquire(&d->ready) || 624 kthread_should_stop()); 625 if (kthread_should_stop()) { 626 d->thr = NULL; 627 atomic_set_release(&d->stop, 1); 628 wake_up(&d->done); 629 break; 630 } 631 atomic_set(&d->ready, 0); 632 633 for (i = 0; i < d->run_threads; i++) 634 *d->crc32 = crc32_le(*d->crc32, 635 d->unc[i], *d->unc_len[i]); 636 atomic_set_release(&d->stop, 1); 637 wake_up(&d->done); 638 } 639 return 0; 640 } 641 642 /* 643 * Structure used for data compression. 644 */ 645 struct cmp_data { 646 struct task_struct *thr; /* thread */ 647 struct crypto_acomp *cc; /* crypto compressor */ 648 struct acomp_req *cr; /* crypto request */ 649 atomic_t ready; /* ready to start flag */ 650 atomic_t stop; /* ready to stop flag */ 651 int ret; /* return code */ 652 wait_queue_head_t go; /* start compression */ 653 wait_queue_head_t done; /* compression done */ 654 size_t unc_len; /* uncompressed length */ 655 size_t cmp_len; /* compressed length */ 656 unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ 657 unsigned char cmp[CMP_SIZE]; /* compressed buffer */ 658 }; 659 660 /* Indicates the image size after compression */ 661 static atomic64_t compressed_size = ATOMIC_INIT(0); 662 663 static int compress_threadfn(void *data) 664 { 665 struct cmp_data *d = data; 666 667 while (1) { 668 wait_event(d->go, atomic_read_acquire(&d->ready) || 669 kthread_should_stop()); 670 if (kthread_should_stop()) { 671 d->thr = NULL; 672 d->ret = -1; 673 atomic_set_release(&d->stop, 1); 674 wake_up(&d->done); 675 break; 676 } 677 atomic_set(&d->ready, 0); 678 679 acomp_request_set_callback(d->cr, CRYPTO_TFM_REQ_MAY_SLEEP, 680 NULL, NULL); 681 acomp_request_set_src_nondma(d->cr, d->unc, d->unc_len); 682 acomp_request_set_dst_nondma(d->cr, d->cmp + CMP_HEADER, 683 CMP_SIZE - CMP_HEADER); 684 d->ret = crypto_acomp_compress(d->cr); 685 d->cmp_len = d->cr->dlen; 686 687 atomic64_add(d->cmp_len, &compressed_size); 688 atomic_set_release(&d->stop, 1); 689 wake_up(&d->done); 690 } 691 return 0; 692 } 693 694 static int save_compressed_image(struct swap_map_handle *handle, 695 struct snapshot_handle *snapshot, 696 unsigned int nr_to_write) 697 { 698 unsigned int m; 699 int ret = 0; 700 int nr_pages; 701 int err2; 702 struct hib_bio_batch hb; 703 ktime_t start; 704 ktime_t stop; 705 size_t off; 706 unsigned int thr, run_threads, nr_threads; 707 unsigned char *page = NULL; 708 struct cmp_data *data = NULL; 709 struct crc_data *crc = NULL; 710 711 hib_init_batch(&hb); 712 713 atomic64_set(&compressed_size, 0); 714 715 /* 716 * We'll limit the number of threads for compression to limit memory 717 * footprint. 718 */ 719 nr_threads = num_online_cpus() - 1; 720 nr_threads = clamp_val(nr_threads, 1, hibernate_compression_threads); 721 722 page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); 723 if (!page) { 724 pr_err("Failed to allocate %s page\n", hib_comp_algo); 725 ret = -ENOMEM; 726 goto out_clean; 727 } 728 729 data = vcalloc(nr_threads, sizeof(*data)); 730 if (!data) { 731 pr_err("Failed to allocate %s data\n", hib_comp_algo); 732 ret = -ENOMEM; 733 goto out_clean; 734 } 735 736 crc = alloc_crc_data(nr_threads); 737 if (!crc) { 738 pr_err("Failed to allocate crc\n"); 739 ret = -ENOMEM; 740 goto out_clean; 741 } 742 743 /* 744 * Start the compression threads. 745 */ 746 for (thr = 0; thr < nr_threads; thr++) { 747 init_waitqueue_head(&data[thr].go); 748 init_waitqueue_head(&data[thr].done); 749 750 data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC); 751 if (IS_ERR_OR_NULL(data[thr].cc)) { 752 pr_err("Could not allocate comp stream %pe\n", data[thr].cc); 753 ret = -EFAULT; 754 goto out_clean; 755 } 756 757 data[thr].cr = acomp_request_alloc(data[thr].cc); 758 if (!data[thr].cr) { 759 pr_err("Could not allocate comp request\n"); 760 ret = -ENOMEM; 761 goto out_clean; 762 } 763 764 data[thr].thr = kthread_run(compress_threadfn, 765 &data[thr], 766 "image_compress/%u", thr); 767 if (IS_ERR(data[thr].thr)) { 768 data[thr].thr = NULL; 769 pr_err("Cannot start compression threads\n"); 770 ret = -ENOMEM; 771 goto out_clean; 772 } 773 } 774 775 /* 776 * Start the CRC32 thread. 777 */ 778 init_waitqueue_head(&crc->go); 779 init_waitqueue_head(&crc->done); 780 781 handle->crc32 = 0; 782 crc->crc32 = &handle->crc32; 783 for (thr = 0; thr < nr_threads; thr++) { 784 crc->unc[thr] = data[thr].unc; 785 crc->unc_len[thr] = &data[thr].unc_len; 786 } 787 788 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32"); 789 if (IS_ERR(crc->thr)) { 790 crc->thr = NULL; 791 pr_err("Cannot start CRC32 thread\n"); 792 ret = -ENOMEM; 793 goto out_clean; 794 } 795 796 /* 797 * Adjust the number of required free pages after all allocations have 798 * been done. We don't want to run out of pages when writing. 799 */ 800 handle->reqd_free_pages = reqd_free_pages(); 801 802 pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo); 803 pr_info("Compressing and saving image data (%u pages)...\n", 804 nr_to_write); 805 m = nr_to_write / 10; 806 if (!m) 807 m = 1; 808 nr_pages = 0; 809 start = ktime_get(); 810 for (;;) { 811 for (thr = 0; thr < nr_threads; thr++) { 812 for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) { 813 ret = snapshot_read_next(snapshot); 814 if (ret < 0) 815 goto out_finish; 816 817 if (!ret) 818 break; 819 820 memcpy(data[thr].unc + off, 821 data_of(*snapshot), PAGE_SIZE); 822 823 if (!(nr_pages % m)) 824 pr_info("Image saving progress: %3d%%\n", 825 nr_pages / m * 10); 826 nr_pages++; 827 } 828 if (!off) 829 break; 830 831 data[thr].unc_len = off; 832 833 atomic_set_release(&data[thr].ready, 1); 834 wake_up(&data[thr].go); 835 } 836 837 if (!thr) 838 break; 839 840 crc->run_threads = thr; 841 atomic_set_release(&crc->ready, 1); 842 wake_up(&crc->go); 843 844 for (run_threads = thr, thr = 0; thr < run_threads; thr++) { 845 wait_event(data[thr].done, 846 atomic_read_acquire(&data[thr].stop)); 847 atomic_set(&data[thr].stop, 0); 848 849 ret = data[thr].ret; 850 851 if (ret < 0) { 852 pr_err("%s compression failed\n", hib_comp_algo); 853 goto out_finish; 854 } 855 856 if (unlikely(!data[thr].cmp_len || 857 data[thr].cmp_len > 858 bytes_worst_compress(data[thr].unc_len))) { 859 pr_err("Invalid %s compressed length\n", hib_comp_algo); 860 ret = -1; 861 goto out_finish; 862 } 863 864 *(size_t *)data[thr].cmp = data[thr].cmp_len; 865 866 /* 867 * Given we are writing one page at a time to disk, we 868 * copy that much from the buffer, although the last 869 * bit will likely be smaller than full page. This is 870 * OK - we saved the length of the compressed data, so 871 * any garbage at the end will be discarded when we 872 * read it. 873 */ 874 for (off = 0; 875 off < CMP_HEADER + data[thr].cmp_len; 876 off += PAGE_SIZE) { 877 memcpy(page, data[thr].cmp + off, PAGE_SIZE); 878 879 ret = swap_write_page(handle, page, &hb); 880 if (ret) 881 goto out_finish; 882 } 883 } 884 885 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 886 atomic_set(&crc->stop, 0); 887 } 888 889 out_finish: 890 err2 = hib_wait_io(&hb); 891 stop = ktime_get(); 892 if (!ret) 893 ret = err2; 894 if (!ret) { 895 swsusp_show_speed(start, stop, nr_to_write, "Wrote"); 896 pr_info("Image size after compression: %lld kbytes\n", 897 (atomic64_read(&compressed_size) / 1024)); 898 pr_info("Image saving done\n"); 899 } else { 900 pr_err("Image saving failed: %d\n", ret); 901 } 902 903 out_clean: 904 hib_finish_batch(&hb); 905 free_crc_data(crc); 906 if (data) { 907 for (thr = 0; thr < nr_threads; thr++) { 908 if (data[thr].thr) 909 kthread_stop(data[thr].thr); 910 911 acomp_request_free(data[thr].cr); 912 913 if (!IS_ERR_OR_NULL(data[thr].cc)) 914 crypto_free_acomp(data[thr].cc); 915 } 916 vfree(data); 917 } 918 if (page) 919 free_page((unsigned long)page); 920 921 return ret; 922 } 923 924 static int enough_swap(unsigned int nr_pages) 925 { 926 unsigned int free_swap = count_swap_pages(root_swap, 1); 927 unsigned int required; 928 929 pr_debug("Free swap pages: %u\n", free_swap); 930 931 required = PAGES_FOR_IO + nr_pages; 932 return free_swap > required; 933 } 934 935 /** 936 * swsusp_write - Write entire image and metadata. 937 * @flags: flags to pass to the "boot" kernel in the image header 938 * 939 * It is important _NOT_ to umount filesystems at this point. We want them 940 * synced (in case something goes wrong) but we DO not want to mark filesystem 941 * clean: it is not. (And it does not matter, if we resume correctly, we'll mark 942 * system clean, anyway.) 943 * 944 * Return: 0 on success, negative error code on failure. 945 */ 946 int swsusp_write(unsigned int flags) 947 { 948 struct swap_map_handle handle; 949 struct snapshot_handle snapshot; 950 struct swsusp_info *header; 951 unsigned long pages; 952 int error; 953 954 pages = snapshot_get_image_size(); 955 error = get_swap_writer(&handle); 956 if (error) { 957 pr_err("Cannot get swap writer\n"); 958 return error; 959 } 960 if (flags & SF_NOCOMPRESS_MODE) { 961 if (!enough_swap(pages)) { 962 pr_err("Not enough free swap\n"); 963 error = -ENOSPC; 964 goto out_finish; 965 } 966 } 967 memset(&snapshot, 0, sizeof(struct snapshot_handle)); 968 error = snapshot_read_next(&snapshot); 969 if (error < (int)PAGE_SIZE) { 970 if (error >= 0) 971 error = -EFAULT; 972 973 goto out_finish; 974 } 975 header = (struct swsusp_info *)data_of(snapshot); 976 error = swap_write_page(&handle, header, NULL); 977 if (!error) { 978 error = (flags & SF_NOCOMPRESS_MODE) ? 979 save_image(&handle, &snapshot, pages - 1) : 980 save_compressed_image(&handle, &snapshot, pages - 1); 981 } 982 out_finish: 983 error = swap_writer_finish(&handle, flags, error); 984 return error; 985 } 986 987 /* 988 * The following functions allow us to read data using a swap map in a file-like 989 * way. 990 */ 991 992 static void release_swap_reader(struct swap_map_handle *handle) 993 { 994 struct swap_map_page_list *tmp; 995 996 while (handle->maps) { 997 if (handle->maps->map) 998 free_page((unsigned long)handle->maps->map); 999 tmp = handle->maps; 1000 handle->maps = handle->maps->next; 1001 kfree(tmp); 1002 } 1003 handle->cur = NULL; 1004 } 1005 1006 static int get_swap_reader(struct swap_map_handle *handle, 1007 unsigned int *flags_p) 1008 { 1009 int error; 1010 struct swap_map_page_list *tmp, *last; 1011 sector_t offset; 1012 1013 *flags_p = swsusp_header->flags; 1014 1015 if (!swsusp_header->image) /* how can this happen? */ 1016 return -EINVAL; 1017 1018 handle->cur = NULL; 1019 last = handle->maps = NULL; 1020 offset = swsusp_header->image; 1021 while (offset) { 1022 tmp = kzalloc_obj(*handle->maps); 1023 if (!tmp) { 1024 release_swap_reader(handle); 1025 return -ENOMEM; 1026 } 1027 if (!handle->maps) 1028 handle->maps = tmp; 1029 if (last) 1030 last->next = tmp; 1031 last = tmp; 1032 1033 tmp->map = (struct swap_map_page *) 1034 __get_free_page(GFP_NOIO | __GFP_HIGH); 1035 if (!tmp->map) { 1036 release_swap_reader(handle); 1037 return -ENOMEM; 1038 } 1039 1040 error = hib_submit_io_sync(REQ_OP_READ, offset, tmp->map); 1041 if (error) { 1042 release_swap_reader(handle); 1043 return error; 1044 } 1045 offset = tmp->map->next_swap; 1046 } 1047 handle->k = 0; 1048 handle->cur = handle->maps->map; 1049 return 0; 1050 } 1051 1052 static int swap_read_page(struct swap_map_handle *handle, void *buf, 1053 struct hib_bio_batch *hb) 1054 { 1055 sector_t offset; 1056 int error; 1057 struct swap_map_page_list *tmp; 1058 1059 if (!handle->cur) 1060 return -EINVAL; 1061 offset = handle->cur->entries[handle->k]; 1062 if (!offset) 1063 return -EFAULT; 1064 if (hb) 1065 error = hib_submit_io_async(REQ_OP_READ, offset, buf, hb); 1066 else 1067 error = hib_submit_io_sync(REQ_OP_READ, offset, buf); 1068 if (error) 1069 return error; 1070 if (++handle->k >= MAP_PAGE_ENTRIES) { 1071 handle->k = 0; 1072 free_page((unsigned long)handle->maps->map); 1073 tmp = handle->maps; 1074 handle->maps = handle->maps->next; 1075 kfree(tmp); 1076 if (!handle->maps) 1077 release_swap_reader(handle); 1078 else 1079 handle->cur = handle->maps->map; 1080 } 1081 return error; 1082 } 1083 1084 static int swap_reader_finish(struct swap_map_handle *handle) 1085 { 1086 release_swap_reader(handle); 1087 1088 return 0; 1089 } 1090 1091 static int load_image(struct swap_map_handle *handle, 1092 struct snapshot_handle *snapshot, 1093 unsigned int nr_to_read) 1094 { 1095 unsigned int m; 1096 int ret = 0; 1097 ktime_t start; 1098 ktime_t stop; 1099 struct hib_bio_batch hb; 1100 int err2; 1101 unsigned nr_pages; 1102 1103 hib_init_batch(&hb); 1104 1105 clean_pages_on_read = true; 1106 pr_info("Loading image data pages (%u pages)...\n", nr_to_read); 1107 m = nr_to_read / 10; 1108 if (!m) 1109 m = 1; 1110 nr_pages = 0; 1111 start = ktime_get(); 1112 for ( ; ; ) { 1113 ret = snapshot_write_next(snapshot); 1114 if (ret <= 0) 1115 break; 1116 ret = swap_read_page(handle, data_of(*snapshot), &hb); 1117 if (ret) 1118 break; 1119 if (snapshot->sync_read) 1120 ret = hib_wait_io(&hb); 1121 if (ret) 1122 break; 1123 if (!(nr_pages % m)) 1124 pr_info("Image loading progress: %3d%%\n", 1125 nr_pages / m * 10); 1126 nr_pages++; 1127 } 1128 err2 = hib_wait_io(&hb); 1129 hib_finish_batch(&hb); 1130 stop = ktime_get(); 1131 if (!ret) 1132 ret = err2; 1133 if (!ret) { 1134 pr_info("Image loading done\n"); 1135 ret = snapshot_write_finalize(snapshot); 1136 if (!ret && !snapshot_image_loaded(snapshot)) 1137 ret = -ENODATA; 1138 } 1139 swsusp_show_speed(start, stop, nr_to_read, "Read"); 1140 return ret; 1141 } 1142 1143 /* 1144 * Structure used for data decompression. 1145 */ 1146 struct dec_data { 1147 struct task_struct *thr; /* thread */ 1148 struct crypto_acomp *cc; /* crypto compressor */ 1149 struct acomp_req *cr; /* crypto request */ 1150 atomic_t ready; /* ready to start flag */ 1151 atomic_t stop; /* ready to stop flag */ 1152 int ret; /* return code */ 1153 wait_queue_head_t go; /* start decompression */ 1154 wait_queue_head_t done; /* decompression done */ 1155 size_t unc_len; /* uncompressed length */ 1156 size_t cmp_len; /* compressed length */ 1157 unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ 1158 unsigned char cmp[CMP_SIZE]; /* compressed buffer */ 1159 }; 1160 1161 static int decompress_threadfn(void *data) 1162 { 1163 struct dec_data *d = data; 1164 1165 while (1) { 1166 wait_event(d->go, atomic_read_acquire(&d->ready) || 1167 kthread_should_stop()); 1168 if (kthread_should_stop()) { 1169 d->thr = NULL; 1170 d->ret = -1; 1171 atomic_set_release(&d->stop, 1); 1172 wake_up(&d->done); 1173 break; 1174 } 1175 atomic_set(&d->ready, 0); 1176 1177 acomp_request_set_callback(d->cr, CRYPTO_TFM_REQ_MAY_SLEEP, 1178 NULL, NULL); 1179 acomp_request_set_src_nondma(d->cr, d->cmp + CMP_HEADER, 1180 d->cmp_len); 1181 acomp_request_set_dst_nondma(d->cr, d->unc, UNC_SIZE); 1182 d->ret = crypto_acomp_decompress(d->cr); 1183 d->unc_len = d->cr->dlen; 1184 1185 if (clean_pages_on_decompress) 1186 flush_icache_range((unsigned long)d->unc, 1187 (unsigned long)d->unc + d->unc_len); 1188 1189 atomic_set_release(&d->stop, 1); 1190 wake_up(&d->done); 1191 } 1192 return 0; 1193 } 1194 1195 static int load_compressed_image(struct swap_map_handle *handle, 1196 struct snapshot_handle *snapshot, 1197 unsigned int nr_to_read) 1198 { 1199 unsigned int m; 1200 int ret = 0; 1201 int eof = 0; 1202 struct hib_bio_batch hb; 1203 ktime_t start; 1204 ktime_t stop; 1205 unsigned nr_pages; 1206 size_t off; 1207 unsigned i, thr, run_threads, nr_threads; 1208 unsigned ring = 0, pg = 0, ring_size = 0, 1209 have = 0, want, need, asked = 0; 1210 unsigned long read_pages = 0; 1211 unsigned char **page = NULL; 1212 struct dec_data *data = NULL; 1213 struct crc_data *crc = NULL; 1214 1215 hib_init_batch(&hb); 1216 1217 /* 1218 * We'll limit the number of threads for decompression to limit memory 1219 * footprint. 1220 */ 1221 nr_threads = num_online_cpus() - 1; 1222 nr_threads = clamp_val(nr_threads, 1, hibernate_compression_threads); 1223 1224 page = vmalloc_array(CMP_MAX_RD_PAGES, sizeof(*page)); 1225 if (!page) { 1226 pr_err("Failed to allocate %s page\n", hib_comp_algo); 1227 ret = -ENOMEM; 1228 goto out_clean; 1229 } 1230 1231 data = vcalloc(nr_threads, sizeof(*data)); 1232 if (!data) { 1233 pr_err("Failed to allocate %s data\n", hib_comp_algo); 1234 ret = -ENOMEM; 1235 goto out_clean; 1236 } 1237 1238 crc = alloc_crc_data(nr_threads); 1239 if (!crc) { 1240 pr_err("Failed to allocate crc\n"); 1241 ret = -ENOMEM; 1242 goto out_clean; 1243 } 1244 1245 clean_pages_on_decompress = true; 1246 1247 /* 1248 * Start the decompression threads. 1249 */ 1250 for (thr = 0; thr < nr_threads; thr++) { 1251 init_waitqueue_head(&data[thr].go); 1252 init_waitqueue_head(&data[thr].done); 1253 1254 data[thr].cc = crypto_alloc_acomp(hib_comp_algo, 0, CRYPTO_ALG_ASYNC); 1255 if (IS_ERR_OR_NULL(data[thr].cc)) { 1256 pr_err("Could not allocate comp stream %pe\n", data[thr].cc); 1257 ret = -EFAULT; 1258 goto out_clean; 1259 } 1260 1261 data[thr].cr = acomp_request_alloc(data[thr].cc); 1262 if (!data[thr].cr) { 1263 pr_err("Could not allocate comp request\n"); 1264 ret = -ENOMEM; 1265 goto out_clean; 1266 } 1267 1268 data[thr].thr = kthread_run(decompress_threadfn, 1269 &data[thr], 1270 "image_decompress/%u", thr); 1271 if (IS_ERR(data[thr].thr)) { 1272 data[thr].thr = NULL; 1273 pr_err("Cannot start decompression threads\n"); 1274 ret = -ENOMEM; 1275 goto out_clean; 1276 } 1277 } 1278 1279 /* 1280 * Start the CRC32 thread. 1281 */ 1282 init_waitqueue_head(&crc->go); 1283 init_waitqueue_head(&crc->done); 1284 1285 handle->crc32 = 0; 1286 crc->crc32 = &handle->crc32; 1287 for (thr = 0; thr < nr_threads; thr++) { 1288 crc->unc[thr] = data[thr].unc; 1289 crc->unc_len[thr] = &data[thr].unc_len; 1290 } 1291 1292 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32"); 1293 if (IS_ERR(crc->thr)) { 1294 crc->thr = NULL; 1295 pr_err("Cannot start CRC32 thread\n"); 1296 ret = -ENOMEM; 1297 goto out_clean; 1298 } 1299 1300 /* 1301 * Set the number of pages for read buffering. 1302 * This is complete guesswork, because we'll only know the real 1303 * picture once prepare_image() is called, which is much later on 1304 * during the image load phase. We'll assume the worst case and 1305 * say that none of the image pages are from high memory. 1306 */ 1307 if (low_free_pages() > snapshot_get_image_size()) 1308 read_pages = (low_free_pages() - snapshot_get_image_size()) / 2; 1309 read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES); 1310 1311 for (i = 0; i < read_pages; i++) { 1312 page[i] = (void *)__get_free_page(i < CMP_PAGES ? 1313 GFP_NOIO | __GFP_HIGH : 1314 GFP_NOIO | __GFP_NOWARN | 1315 __GFP_NORETRY); 1316 1317 if (!page[i]) { 1318 if (i < CMP_PAGES) { 1319 ring_size = i; 1320 pr_err("Failed to allocate %s pages\n", hib_comp_algo); 1321 ret = -ENOMEM; 1322 goto out_clean; 1323 } else { 1324 break; 1325 } 1326 } 1327 } 1328 want = ring_size = i; 1329 1330 pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo); 1331 pr_info("Loading and decompressing image data (%u pages)...\n", 1332 nr_to_read); 1333 m = nr_to_read / 10; 1334 if (!m) 1335 m = 1; 1336 nr_pages = 0; 1337 start = ktime_get(); 1338 1339 ret = snapshot_write_next(snapshot); 1340 if (ret <= 0) 1341 goto out_finish; 1342 1343 for(;;) { 1344 for (i = 0; !eof && i < want; i++) { 1345 ret = swap_read_page(handle, page[ring], &hb); 1346 if (ret) { 1347 /* 1348 * On real read error, finish. On end of data, 1349 * set EOF flag and just exit the read loop. 1350 */ 1351 if (handle->cur && 1352 handle->cur->entries[handle->k]) { 1353 goto out_finish; 1354 } else { 1355 eof = 1; 1356 break; 1357 } 1358 } 1359 if (++ring >= ring_size) 1360 ring = 0; 1361 } 1362 asked += i; 1363 want -= i; 1364 1365 /* 1366 * We are out of data, wait for some more. 1367 */ 1368 if (!have) { 1369 if (!asked) 1370 break; 1371 1372 ret = hib_wait_io(&hb); 1373 if (ret) 1374 goto out_finish; 1375 have += asked; 1376 asked = 0; 1377 if (eof) 1378 eof = 2; 1379 } 1380 1381 if (crc->run_threads) { 1382 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 1383 atomic_set(&crc->stop, 0); 1384 crc->run_threads = 0; 1385 } 1386 1387 for (thr = 0; have && thr < nr_threads; thr++) { 1388 data[thr].cmp_len = *(size_t *)page[pg]; 1389 if (unlikely(!data[thr].cmp_len || 1390 data[thr].cmp_len > 1391 bytes_worst_compress(UNC_SIZE))) { 1392 pr_err("Invalid %s compressed length\n", hib_comp_algo); 1393 ret = -1; 1394 goto out_finish; 1395 } 1396 1397 need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER, 1398 PAGE_SIZE); 1399 if (need > have) { 1400 if (eof > 1) { 1401 ret = -1; 1402 goto out_finish; 1403 } 1404 break; 1405 } 1406 1407 for (off = 0; 1408 off < CMP_HEADER + data[thr].cmp_len; 1409 off += PAGE_SIZE) { 1410 memcpy(data[thr].cmp + off, 1411 page[pg], PAGE_SIZE); 1412 have--; 1413 want++; 1414 if (++pg >= ring_size) 1415 pg = 0; 1416 } 1417 1418 atomic_set_release(&data[thr].ready, 1); 1419 wake_up(&data[thr].go); 1420 } 1421 1422 /* 1423 * Wait for more data while we are decompressing. 1424 */ 1425 if (have < CMP_PAGES && asked) { 1426 ret = hib_wait_io(&hb); 1427 if (ret) 1428 goto out_finish; 1429 have += asked; 1430 asked = 0; 1431 if (eof) 1432 eof = 2; 1433 } 1434 1435 for (run_threads = thr, thr = 0; thr < run_threads; thr++) { 1436 wait_event(data[thr].done, 1437 atomic_read_acquire(&data[thr].stop)); 1438 atomic_set(&data[thr].stop, 0); 1439 1440 ret = data[thr].ret; 1441 1442 if (ret < 0) { 1443 pr_err("%s decompression failed\n", hib_comp_algo); 1444 goto out_finish; 1445 } 1446 1447 if (unlikely(!data[thr].unc_len || 1448 data[thr].unc_len > UNC_SIZE || 1449 data[thr].unc_len & (PAGE_SIZE - 1))) { 1450 pr_err("Invalid %s uncompressed length\n", hib_comp_algo); 1451 ret = -1; 1452 goto out_finish; 1453 } 1454 1455 for (off = 0; 1456 off < data[thr].unc_len; off += PAGE_SIZE) { 1457 memcpy(data_of(*snapshot), 1458 data[thr].unc + off, PAGE_SIZE); 1459 1460 if (!(nr_pages % m)) 1461 pr_info("Image loading progress: %3d%%\n", 1462 nr_pages / m * 10); 1463 nr_pages++; 1464 1465 ret = snapshot_write_next(snapshot); 1466 if (ret <= 0) { 1467 crc->run_threads = thr + 1; 1468 atomic_set_release(&crc->ready, 1); 1469 wake_up(&crc->go); 1470 goto out_finish; 1471 } 1472 } 1473 } 1474 1475 crc->run_threads = thr; 1476 atomic_set_release(&crc->ready, 1); 1477 wake_up(&crc->go); 1478 } 1479 1480 out_finish: 1481 if (crc->run_threads) { 1482 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 1483 atomic_set(&crc->stop, 0); 1484 } 1485 stop = ktime_get(); 1486 if (!ret) { 1487 pr_info("Image loading done\n"); 1488 ret = snapshot_write_finalize(snapshot); 1489 if (!ret && !snapshot_image_loaded(snapshot)) 1490 ret = -ENODATA; 1491 if (!ret) { 1492 if (swsusp_header->flags & SF_CRC32_MODE) { 1493 if(handle->crc32 != swsusp_header->crc32) { 1494 pr_err("Invalid image CRC32!\n"); 1495 ret = -ENODATA; 1496 } 1497 } 1498 } 1499 } 1500 swsusp_show_speed(start, stop, nr_to_read, "Read"); 1501 out_clean: 1502 hib_finish_batch(&hb); 1503 for (i = 0; i < ring_size; i++) 1504 free_page((unsigned long)page[i]); 1505 free_crc_data(crc); 1506 if (data) { 1507 for (thr = 0; thr < nr_threads; thr++) { 1508 if (data[thr].thr) 1509 kthread_stop(data[thr].thr); 1510 1511 acomp_request_free(data[thr].cr); 1512 1513 if (!IS_ERR_OR_NULL(data[thr].cc)) 1514 crypto_free_acomp(data[thr].cc); 1515 } 1516 vfree(data); 1517 } 1518 vfree(page); 1519 1520 return ret; 1521 } 1522 1523 /** 1524 * swsusp_read - read the hibernation image. 1525 * @flags_p: flags passed by the "frozen" kernel in the image header should 1526 * be written into this memory location 1527 * 1528 * Return: 0 on success, negative error code on failure. 1529 */ 1530 int swsusp_read(unsigned int *flags_p) 1531 { 1532 int error; 1533 struct swap_map_handle handle; 1534 struct snapshot_handle snapshot; 1535 struct swsusp_info *header; 1536 1537 memset(&snapshot, 0, sizeof(struct snapshot_handle)); 1538 error = snapshot_write_next(&snapshot); 1539 if (error < (int)PAGE_SIZE) 1540 return error < 0 ? error : -EFAULT; 1541 header = (struct swsusp_info *)data_of(snapshot); 1542 error = get_swap_reader(&handle, flags_p); 1543 if (error) 1544 goto end; 1545 if (!error) 1546 error = swap_read_page(&handle, header, NULL); 1547 if (!error) { 1548 error = (*flags_p & SF_NOCOMPRESS_MODE) ? 1549 load_image(&handle, &snapshot, header->pages - 1) : 1550 load_compressed_image(&handle, &snapshot, header->pages - 1); 1551 } 1552 swap_reader_finish(&handle); 1553 end: 1554 if (!error) 1555 pr_debug("Image successfully loaded\n"); 1556 else 1557 pr_debug("Error %d resuming\n", error); 1558 return error; 1559 } 1560 1561 static void *swsusp_holder; 1562 1563 /** 1564 * swsusp_check - Open the resume device and check for the swsusp signature. 1565 * @exclusive: Open the resume device exclusively. 1566 * 1567 * Return: 0 if a valid image is found, negative error code otherwise. 1568 */ 1569 int swsusp_check(bool exclusive) 1570 { 1571 void *holder = exclusive ? &swsusp_holder : NULL; 1572 int error; 1573 1574 hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device, 1575 BLK_OPEN_READ, holder, NULL); 1576 if (!IS_ERR(hib_resume_bdev_file)) { 1577 clear_page(swsusp_header); 1578 error = hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, 1579 swsusp_header); 1580 if (error) 1581 goto put; 1582 1583 if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { 1584 memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); 1585 swsusp_header_flags = swsusp_header->flags; 1586 /* Reset swap signature now */ 1587 error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, 1588 swsusp_resume_block, 1589 swsusp_header); 1590 } else { 1591 error = -EINVAL; 1592 } 1593 if (!error && swsusp_header->flags & SF_HW_SIG && 1594 swsusp_header->hw_sig != swsusp_hardware_signature) { 1595 pr_info("Suspend image hardware signature mismatch (%08x now %08x); aborting resume.\n", 1596 swsusp_header->hw_sig, swsusp_hardware_signature); 1597 error = -EINVAL; 1598 } 1599 1600 put: 1601 if (error) 1602 bdev_fput(hib_resume_bdev_file); 1603 else 1604 pr_debug("Image signature found, resuming\n"); 1605 } else { 1606 error = PTR_ERR(hib_resume_bdev_file); 1607 } 1608 1609 if (error) 1610 pr_debug("Image not found (code %d)\n", error); 1611 1612 return error; 1613 } 1614 1615 /** 1616 * swsusp_close - close resume device. 1617 */ 1618 void swsusp_close(void) 1619 { 1620 if (IS_ERR(hib_resume_bdev_file)) { 1621 pr_debug("Image device not initialised\n"); 1622 return; 1623 } 1624 1625 fput(hib_resume_bdev_file); 1626 } 1627 1628 /** 1629 * swsusp_unmark - Unmark swsusp signature in the resume device 1630 * 1631 * Return: 0 on success, negative error code on failure. 1632 */ 1633 #ifdef CONFIG_SUSPEND 1634 int swsusp_unmark(void) 1635 { 1636 int error; 1637 1638 hib_submit_io_sync(REQ_OP_READ, swsusp_resume_block, swsusp_header); 1639 if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) { 1640 memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10); 1641 error = hib_submit_io_sync(REQ_OP_WRITE | REQ_SYNC, 1642 swsusp_resume_block, 1643 swsusp_header); 1644 } else { 1645 pr_err("Cannot find swsusp signature!\n"); 1646 error = -ENODEV; 1647 } 1648 1649 /* 1650 * We just returned from suspend, we don't need the image any more. 1651 */ 1652 free_all_swap_pages(root_swap); 1653 1654 return error; 1655 } 1656 #endif 1657 1658 static ssize_t hibernate_compression_threads_show(struct kobject *kobj, 1659 struct kobj_attribute *attr, char *buf) 1660 { 1661 return sysfs_emit(buf, "%d\n", hibernate_compression_threads); 1662 } 1663 1664 static ssize_t hibernate_compression_threads_store(struct kobject *kobj, 1665 struct kobj_attribute *attr, 1666 const char *buf, size_t n) 1667 { 1668 unsigned long val; 1669 1670 if (kstrtoul(buf, 0, &val)) 1671 return -EINVAL; 1672 1673 if (val < 1) 1674 return -EINVAL; 1675 1676 hibernate_compression_threads = val; 1677 return n; 1678 } 1679 power_attr(hibernate_compression_threads); 1680 1681 static struct attribute *g[] = { 1682 &hibernate_compression_threads_attr.attr, 1683 NULL, 1684 }; 1685 1686 static const struct attribute_group attr_group = { 1687 .attrs = g, 1688 }; 1689 1690 static int __init swsusp_header_init(void) 1691 { 1692 int error; 1693 1694 error = sysfs_create_group(power_kobj, &attr_group); 1695 if (error) 1696 return -ENOMEM; 1697 1698 swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL); 1699 if (!swsusp_header) 1700 panic("Could not allocate memory for swsusp_header\n"); 1701 return 0; 1702 } 1703 1704 core_initcall(swsusp_header_init); 1705 1706 static int __init hibernate_compression_threads_setup(char *str) 1707 { 1708 int rc = kstrtouint(str, 0, &hibernate_compression_threads); 1709 1710 if (rc) 1711 return rc; 1712 1713 if (hibernate_compression_threads < 1) 1714 hibernate_compression_threads = CMP_THREADS; 1715 1716 return 1; 1717 1718 } 1719 1720 __setup("hibernate_compression_threads=", hibernate_compression_threads_setup); 1721