1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/kernel/power/swap.c 4 * 5 * This file provides functions for reading the suspend image from 6 * and writing it to a swap partition. 7 * 8 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz> 9 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl> 10 * Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com> 11 */ 12 13 #define pr_fmt(fmt) "PM: " fmt 14 15 #include <linux/module.h> 16 #include <linux/file.h> 17 #include <linux/delay.h> 18 #include <linux/bitops.h> 19 #include <linux/device.h> 20 #include <linux/bio.h> 21 #include <linux/blkdev.h> 22 #include <linux/swap.h> 23 #include <linux/swapops.h> 24 #include <linux/pm.h> 25 #include <linux/slab.h> 26 #include <linux/vmalloc.h> 27 #include <linux/cpumask.h> 28 #include <linux/atomic.h> 29 #include <linux/kthread.h> 30 #include <linux/crc32.h> 31 #include <linux/ktime.h> 32 33 #include "power.h" 34 35 #define HIBERNATE_SIG "S1SUSPEND" 36 37 u32 swsusp_hardware_signature; 38 39 /* 40 * When reading an {un,}compressed image, we may restore pages in place, 41 * in which case some architectures need these pages cleaning before they 42 * can be executed. We don't know which pages these may be, so clean the lot. 43 */ 44 static bool clean_pages_on_read; 45 static bool clean_pages_on_decompress; 46 47 /* 48 * The swap map is a data structure used for keeping track of each page 49 * written to a swap partition. It consists of many swap_map_page 50 * structures that contain each an array of MAP_PAGE_ENTRIES swap entries. 51 * These structures are stored on the swap and linked together with the 52 * help of the .next_swap member. 53 * 54 * The swap map is created during suspend. The swap map pages are 55 * allocated and populated one at a time, so we only need one memory 56 * page to set up the entire structure. 57 * 58 * During resume we pick up all swap_map_page structures into a list. 59 */ 60 61 #define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1) 62 63 /* 64 * Number of free pages that are not high. 65 */ 66 static inline unsigned long low_free_pages(void) 67 { 68 return nr_free_pages() - nr_free_highpages(); 69 } 70 71 /* 72 * Number of pages required to be kept free while writing the image. Always 73 * half of all available low pages before the writing starts. 74 */ 75 static inline unsigned long reqd_free_pages(void) 76 { 77 return low_free_pages() / 2; 78 } 79 80 struct swap_map_page { 81 sector_t entries[MAP_PAGE_ENTRIES]; 82 sector_t next_swap; 83 }; 84 85 struct swap_map_page_list { 86 struct swap_map_page *map; 87 struct swap_map_page_list *next; 88 }; 89 90 /* 91 * The swap_map_handle structure is used for handling swap in 92 * a file-alike way 93 */ 94 95 struct swap_map_handle { 96 struct swap_map_page *cur; 97 struct swap_map_page_list *maps; 98 sector_t cur_swap; 99 sector_t first_sector; 100 unsigned int k; 101 unsigned long reqd_free_pages; 102 u32 crc32; 103 }; 104 105 struct swsusp_header { 106 char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) - 107 sizeof(u32) - sizeof(u32)]; 108 u32 hw_sig; 109 u32 crc32; 110 sector_t image; 111 unsigned int flags; /* Flags to pass to the "boot" kernel */ 112 char orig_sig[10]; 113 char sig[10]; 114 } __packed; 115 116 static struct swsusp_header *swsusp_header; 117 118 /* 119 * The following functions are used for tracing the allocated 120 * swap pages, so that they can be freed in case of an error. 121 */ 122 123 struct swsusp_extent { 124 struct rb_node node; 125 unsigned long start; 126 unsigned long end; 127 }; 128 129 static struct rb_root swsusp_extents = RB_ROOT; 130 131 static int swsusp_extents_insert(unsigned long swap_offset) 132 { 133 struct rb_node **new = &(swsusp_extents.rb_node); 134 struct rb_node *parent = NULL; 135 struct swsusp_extent *ext; 136 137 /* Figure out where to put the new node */ 138 while (*new) { 139 ext = rb_entry(*new, struct swsusp_extent, node); 140 parent = *new; 141 if (swap_offset < ext->start) { 142 /* Try to merge */ 143 if (swap_offset == ext->start - 1) { 144 ext->start--; 145 return 0; 146 } 147 new = &((*new)->rb_left); 148 } else if (swap_offset > ext->end) { 149 /* Try to merge */ 150 if (swap_offset == ext->end + 1) { 151 ext->end++; 152 return 0; 153 } 154 new = &((*new)->rb_right); 155 } else { 156 /* It already is in the tree */ 157 return -EINVAL; 158 } 159 } 160 /* Add the new node and rebalance the tree. */ 161 ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL); 162 if (!ext) 163 return -ENOMEM; 164 165 ext->start = swap_offset; 166 ext->end = swap_offset; 167 rb_link_node(&ext->node, parent, new); 168 rb_insert_color(&ext->node, &swsusp_extents); 169 return 0; 170 } 171 172 /* 173 * alloc_swapdev_block - allocate a swap page and register that it has 174 * been allocated, so that it can be freed in case of an error. 175 */ 176 177 sector_t alloc_swapdev_block(int swap) 178 { 179 unsigned long offset; 180 181 offset = swp_offset(get_swap_page_of_type(swap)); 182 if (offset) { 183 if (swsusp_extents_insert(offset)) 184 swap_free(swp_entry(swap, offset)); 185 else 186 return swapdev_block(swap, offset); 187 } 188 return 0; 189 } 190 191 /* 192 * free_all_swap_pages - free swap pages allocated for saving image data. 193 * It also frees the extents used to register which swap entries had been 194 * allocated. 195 */ 196 197 void free_all_swap_pages(int swap) 198 { 199 struct rb_node *node; 200 201 while ((node = swsusp_extents.rb_node)) { 202 struct swsusp_extent *ext; 203 unsigned long offset; 204 205 ext = rb_entry(node, struct swsusp_extent, node); 206 rb_erase(node, &swsusp_extents); 207 for (offset = ext->start; offset <= ext->end; offset++) 208 swap_free(swp_entry(swap, offset)); 209 210 kfree(ext); 211 } 212 } 213 214 int swsusp_swap_in_use(void) 215 { 216 return (swsusp_extents.rb_node != NULL); 217 } 218 219 /* 220 * General things 221 */ 222 223 static unsigned short root_swap = 0xffff; 224 static struct file *hib_resume_bdev_file; 225 226 struct hib_bio_batch { 227 atomic_t count; 228 wait_queue_head_t wait; 229 blk_status_t error; 230 struct blk_plug plug; 231 }; 232 233 static void hib_init_batch(struct hib_bio_batch *hb) 234 { 235 atomic_set(&hb->count, 0); 236 init_waitqueue_head(&hb->wait); 237 hb->error = BLK_STS_OK; 238 blk_start_plug(&hb->plug); 239 } 240 241 static void hib_finish_batch(struct hib_bio_batch *hb) 242 { 243 blk_finish_plug(&hb->plug); 244 } 245 246 static void hib_end_io(struct bio *bio) 247 { 248 struct hib_bio_batch *hb = bio->bi_private; 249 struct page *page = bio_first_page_all(bio); 250 251 if (bio->bi_status) { 252 pr_alert("Read-error on swap-device (%u:%u:%Lu)\n", 253 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)), 254 (unsigned long long)bio->bi_iter.bi_sector); 255 } 256 257 if (bio_data_dir(bio) == WRITE) 258 put_page(page); 259 else if (clean_pages_on_read) 260 flush_icache_range((unsigned long)page_address(page), 261 (unsigned long)page_address(page) + PAGE_SIZE); 262 263 if (bio->bi_status && !hb->error) 264 hb->error = bio->bi_status; 265 if (atomic_dec_and_test(&hb->count)) 266 wake_up(&hb->wait); 267 268 bio_put(bio); 269 } 270 271 static int hib_submit_io(blk_opf_t opf, pgoff_t page_off, void *addr, 272 struct hib_bio_batch *hb) 273 { 274 struct page *page = virt_to_page(addr); 275 struct bio *bio; 276 int error = 0; 277 278 bio = bio_alloc(file_bdev(hib_resume_bdev_file), 1, opf, 279 GFP_NOIO | __GFP_HIGH); 280 bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9); 281 282 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) { 283 pr_err("Adding page to bio failed at %llu\n", 284 (unsigned long long)bio->bi_iter.bi_sector); 285 bio_put(bio); 286 return -EFAULT; 287 } 288 289 if (hb) { 290 bio->bi_end_io = hib_end_io; 291 bio->bi_private = hb; 292 atomic_inc(&hb->count); 293 submit_bio(bio); 294 } else { 295 error = submit_bio_wait(bio); 296 bio_put(bio); 297 } 298 299 return error; 300 } 301 302 static int hib_wait_io(struct hib_bio_batch *hb) 303 { 304 /* 305 * We are relying on the behavior of blk_plug that a thread with 306 * a plug will flush the plug list before sleeping. 307 */ 308 wait_event(hb->wait, atomic_read(&hb->count) == 0); 309 return blk_status_to_errno(hb->error); 310 } 311 312 /* 313 * Saving part 314 */ 315 static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags) 316 { 317 int error; 318 319 hib_submit_io(REQ_OP_READ, swsusp_resume_block, swsusp_header, NULL); 320 if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) || 321 !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) { 322 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10); 323 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10); 324 swsusp_header->image = handle->first_sector; 325 if (swsusp_hardware_signature) { 326 swsusp_header->hw_sig = swsusp_hardware_signature; 327 flags |= SF_HW_SIG; 328 } 329 swsusp_header->flags = flags; 330 if (flags & SF_CRC32_MODE) 331 swsusp_header->crc32 = handle->crc32; 332 error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC, 333 swsusp_resume_block, swsusp_header, NULL); 334 } else { 335 pr_err("Swap header not found!\n"); 336 error = -ENODEV; 337 } 338 return error; 339 } 340 341 /* 342 * Hold the swsusp_header flag. This is used in software_resume() in 343 * 'kernel/power/hibernate' to check if the image is compressed and query 344 * for the compression algorithm support(if so). 345 */ 346 unsigned int swsusp_header_flags; 347 348 /** 349 * swsusp_swap_check - check if the resume device is a swap device 350 * and get its index (if so) 351 * 352 * This is called before saving image 353 */ 354 static int swsusp_swap_check(void) 355 { 356 int res; 357 358 if (swsusp_resume_device) 359 res = swap_type_of(swsusp_resume_device, swsusp_resume_block); 360 else 361 res = find_first_swap(&swsusp_resume_device); 362 if (res < 0) 363 return res; 364 root_swap = res; 365 366 hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device, 367 BLK_OPEN_WRITE, NULL, NULL); 368 if (IS_ERR(hib_resume_bdev_file)) 369 return PTR_ERR(hib_resume_bdev_file); 370 371 return 0; 372 } 373 374 /** 375 * write_page - Write one page to given swap location. 376 * @buf: Address we're writing. 377 * @offset: Offset of the swap page we're writing to. 378 * @hb: bio completion batch 379 */ 380 381 static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb) 382 { 383 void *src; 384 int ret; 385 386 if (!offset) 387 return -ENOSPC; 388 389 if (hb) { 390 src = (void *)__get_free_page(GFP_NOIO | __GFP_NOWARN | 391 __GFP_NORETRY); 392 if (src) { 393 copy_page(src, buf); 394 } else { 395 ret = hib_wait_io(hb); /* Free pages */ 396 if (ret) 397 return ret; 398 src = (void *)__get_free_page(GFP_NOIO | 399 __GFP_NOWARN | 400 __GFP_NORETRY); 401 if (src) { 402 copy_page(src, buf); 403 } else { 404 WARN_ON_ONCE(1); 405 hb = NULL; /* Go synchronous */ 406 src = buf; 407 } 408 } 409 } else { 410 src = buf; 411 } 412 return hib_submit_io(REQ_OP_WRITE | REQ_SYNC, offset, src, hb); 413 } 414 415 static void release_swap_writer(struct swap_map_handle *handle) 416 { 417 if (handle->cur) 418 free_page((unsigned long)handle->cur); 419 handle->cur = NULL; 420 } 421 422 static int get_swap_writer(struct swap_map_handle *handle) 423 { 424 int ret; 425 426 ret = swsusp_swap_check(); 427 if (ret) { 428 if (ret != -ENOSPC) 429 pr_err("Cannot find swap device, try swapon -a\n"); 430 return ret; 431 } 432 handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL); 433 if (!handle->cur) { 434 ret = -ENOMEM; 435 goto err_close; 436 } 437 handle->cur_swap = alloc_swapdev_block(root_swap); 438 if (!handle->cur_swap) { 439 ret = -ENOSPC; 440 goto err_rel; 441 } 442 handle->k = 0; 443 handle->reqd_free_pages = reqd_free_pages(); 444 handle->first_sector = handle->cur_swap; 445 return 0; 446 err_rel: 447 release_swap_writer(handle); 448 err_close: 449 swsusp_close(); 450 return ret; 451 } 452 453 static int swap_write_page(struct swap_map_handle *handle, void *buf, 454 struct hib_bio_batch *hb) 455 { 456 int error; 457 sector_t offset; 458 459 if (!handle->cur) 460 return -EINVAL; 461 offset = alloc_swapdev_block(root_swap); 462 error = write_page(buf, offset, hb); 463 if (error) 464 return error; 465 handle->cur->entries[handle->k++] = offset; 466 if (handle->k >= MAP_PAGE_ENTRIES) { 467 offset = alloc_swapdev_block(root_swap); 468 if (!offset) 469 return -ENOSPC; 470 handle->cur->next_swap = offset; 471 error = write_page(handle->cur, handle->cur_swap, hb); 472 if (error) 473 goto out; 474 clear_page(handle->cur); 475 handle->cur_swap = offset; 476 handle->k = 0; 477 478 if (hb && low_free_pages() <= handle->reqd_free_pages) { 479 error = hib_wait_io(hb); 480 if (error) 481 goto out; 482 /* 483 * Recalculate the number of required free pages, to 484 * make sure we never take more than half. 485 */ 486 handle->reqd_free_pages = reqd_free_pages(); 487 } 488 } 489 out: 490 return error; 491 } 492 493 static int flush_swap_writer(struct swap_map_handle *handle) 494 { 495 if (handle->cur && handle->cur_swap) 496 return write_page(handle->cur, handle->cur_swap, NULL); 497 else 498 return -EINVAL; 499 } 500 501 static int swap_writer_finish(struct swap_map_handle *handle, 502 unsigned int flags, int error) 503 { 504 if (!error) { 505 pr_info("S"); 506 error = mark_swapfiles(handle, flags); 507 pr_cont("|\n"); 508 flush_swap_writer(handle); 509 } 510 511 if (error) 512 free_all_swap_pages(root_swap); 513 release_swap_writer(handle); 514 swsusp_close(); 515 516 return error; 517 } 518 519 /* 520 * Bytes we need for compressed data in worst case. We assume(limitation) 521 * this is the worst of all the compression algorithms. 522 */ 523 #define bytes_worst_compress(x) ((x) + ((x) / 16) + 64 + 3 + 2) 524 525 /* We need to remember how much compressed data we need to read. */ 526 #define CMP_HEADER sizeof(size_t) 527 528 /* Number of pages/bytes we'll compress at one time. */ 529 #define UNC_PAGES 32 530 #define UNC_SIZE (UNC_PAGES * PAGE_SIZE) 531 532 /* Number of pages we need for compressed data (worst case). */ 533 #define CMP_PAGES DIV_ROUND_UP(bytes_worst_compress(UNC_SIZE) + \ 534 CMP_HEADER, PAGE_SIZE) 535 #define CMP_SIZE (CMP_PAGES * PAGE_SIZE) 536 537 /* Maximum number of threads for compression/decompression. */ 538 #define CMP_THREADS 3 539 540 /* Minimum/maximum number of pages for read buffering. */ 541 #define CMP_MIN_RD_PAGES 1024 542 #define CMP_MAX_RD_PAGES 8192 543 544 /** 545 * save_image - save the suspend image data 546 */ 547 548 static int save_image(struct swap_map_handle *handle, 549 struct snapshot_handle *snapshot, 550 unsigned int nr_to_write) 551 { 552 unsigned int m; 553 int ret; 554 int nr_pages; 555 int err2; 556 struct hib_bio_batch hb; 557 ktime_t start; 558 ktime_t stop; 559 560 hib_init_batch(&hb); 561 562 pr_info("Saving image data pages (%u pages)...\n", 563 nr_to_write); 564 m = nr_to_write / 10; 565 if (!m) 566 m = 1; 567 nr_pages = 0; 568 start = ktime_get(); 569 while (1) { 570 ret = snapshot_read_next(snapshot); 571 if (ret <= 0) 572 break; 573 ret = swap_write_page(handle, data_of(*snapshot), &hb); 574 if (ret) 575 break; 576 if (!(nr_pages % m)) 577 pr_info("Image saving progress: %3d%%\n", 578 nr_pages / m * 10); 579 nr_pages++; 580 } 581 err2 = hib_wait_io(&hb); 582 hib_finish_batch(&hb); 583 stop = ktime_get(); 584 if (!ret) 585 ret = err2; 586 if (!ret) 587 pr_info("Image saving done\n"); 588 swsusp_show_speed(start, stop, nr_to_write, "Wrote"); 589 return ret; 590 } 591 592 /* 593 * Structure used for CRC32. 594 */ 595 struct crc_data { 596 struct task_struct *thr; /* thread */ 597 atomic_t ready; /* ready to start flag */ 598 atomic_t stop; /* ready to stop flag */ 599 unsigned run_threads; /* nr current threads */ 600 wait_queue_head_t go; /* start crc update */ 601 wait_queue_head_t done; /* crc update done */ 602 u32 *crc32; /* points to handle's crc32 */ 603 size_t *unc_len[CMP_THREADS]; /* uncompressed lengths */ 604 unsigned char *unc[CMP_THREADS]; /* uncompressed data */ 605 }; 606 607 /* 608 * CRC32 update function that runs in its own thread. 609 */ 610 static int crc32_threadfn(void *data) 611 { 612 struct crc_data *d = data; 613 unsigned i; 614 615 while (1) { 616 wait_event(d->go, atomic_read_acquire(&d->ready) || 617 kthread_should_stop()); 618 if (kthread_should_stop()) { 619 d->thr = NULL; 620 atomic_set_release(&d->stop, 1); 621 wake_up(&d->done); 622 break; 623 } 624 atomic_set(&d->ready, 0); 625 626 for (i = 0; i < d->run_threads; i++) 627 *d->crc32 = crc32_le(*d->crc32, 628 d->unc[i], *d->unc_len[i]); 629 atomic_set_release(&d->stop, 1); 630 wake_up(&d->done); 631 } 632 return 0; 633 } 634 /* 635 * Structure used for data compression. 636 */ 637 struct cmp_data { 638 struct task_struct *thr; /* thread */ 639 struct crypto_comp *cc; /* crypto compressor stream */ 640 atomic_t ready; /* ready to start flag */ 641 atomic_t stop; /* ready to stop flag */ 642 int ret; /* return code */ 643 wait_queue_head_t go; /* start compression */ 644 wait_queue_head_t done; /* compression done */ 645 size_t unc_len; /* uncompressed length */ 646 size_t cmp_len; /* compressed length */ 647 unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ 648 unsigned char cmp[CMP_SIZE]; /* compressed buffer */ 649 }; 650 651 /* Indicates the image size after compression */ 652 static atomic_t compressed_size = ATOMIC_INIT(0); 653 654 /* 655 * Compression function that runs in its own thread. 656 */ 657 static int compress_threadfn(void *data) 658 { 659 struct cmp_data *d = data; 660 unsigned int cmp_len = 0; 661 662 while (1) { 663 wait_event(d->go, atomic_read_acquire(&d->ready) || 664 kthread_should_stop()); 665 if (kthread_should_stop()) { 666 d->thr = NULL; 667 d->ret = -1; 668 atomic_set_release(&d->stop, 1); 669 wake_up(&d->done); 670 break; 671 } 672 atomic_set(&d->ready, 0); 673 674 cmp_len = CMP_SIZE - CMP_HEADER; 675 d->ret = crypto_comp_compress(d->cc, d->unc, d->unc_len, 676 d->cmp + CMP_HEADER, 677 &cmp_len); 678 d->cmp_len = cmp_len; 679 680 atomic_set(&compressed_size, atomic_read(&compressed_size) + d->cmp_len); 681 atomic_set_release(&d->stop, 1); 682 wake_up(&d->done); 683 } 684 return 0; 685 } 686 687 /** 688 * save_compressed_image - Save the suspend image data after compression. 689 * @handle: Swap map handle to use for saving the image. 690 * @snapshot: Image to read data from. 691 * @nr_to_write: Number of pages to save. 692 */ 693 static int save_compressed_image(struct swap_map_handle *handle, 694 struct snapshot_handle *snapshot, 695 unsigned int nr_to_write) 696 { 697 unsigned int m; 698 int ret = 0; 699 int nr_pages; 700 int err2; 701 struct hib_bio_batch hb; 702 ktime_t start; 703 ktime_t stop; 704 size_t off; 705 unsigned thr, run_threads, nr_threads; 706 unsigned char *page = NULL; 707 struct cmp_data *data = NULL; 708 struct crc_data *crc = NULL; 709 710 hib_init_batch(&hb); 711 712 atomic_set(&compressed_size, 0); 713 714 /* 715 * We'll limit the number of threads for compression to limit memory 716 * footprint. 717 */ 718 nr_threads = num_online_cpus() - 1; 719 nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); 720 721 page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH); 722 if (!page) { 723 pr_err("Failed to allocate %s page\n", hib_comp_algo); 724 ret = -ENOMEM; 725 goto out_clean; 726 } 727 728 data = vzalloc(array_size(nr_threads, sizeof(*data))); 729 if (!data) { 730 pr_err("Failed to allocate %s data\n", hib_comp_algo); 731 ret = -ENOMEM; 732 goto out_clean; 733 } 734 735 crc = kzalloc(sizeof(*crc), GFP_KERNEL); 736 if (!crc) { 737 pr_err("Failed to allocate crc\n"); 738 ret = -ENOMEM; 739 goto out_clean; 740 } 741 742 /* 743 * Start the compression threads. 744 */ 745 for (thr = 0; thr < nr_threads; thr++) { 746 init_waitqueue_head(&data[thr].go); 747 init_waitqueue_head(&data[thr].done); 748 749 data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); 750 if (IS_ERR_OR_NULL(data[thr].cc)) { 751 pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); 752 ret = -EFAULT; 753 goto out_clean; 754 } 755 756 data[thr].thr = kthread_run(compress_threadfn, 757 &data[thr], 758 "image_compress/%u", thr); 759 if (IS_ERR(data[thr].thr)) { 760 data[thr].thr = NULL; 761 pr_err("Cannot start compression threads\n"); 762 ret = -ENOMEM; 763 goto out_clean; 764 } 765 } 766 767 /* 768 * Start the CRC32 thread. 769 */ 770 init_waitqueue_head(&crc->go); 771 init_waitqueue_head(&crc->done); 772 773 handle->crc32 = 0; 774 crc->crc32 = &handle->crc32; 775 for (thr = 0; thr < nr_threads; thr++) { 776 crc->unc[thr] = data[thr].unc; 777 crc->unc_len[thr] = &data[thr].unc_len; 778 } 779 780 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32"); 781 if (IS_ERR(crc->thr)) { 782 crc->thr = NULL; 783 pr_err("Cannot start CRC32 thread\n"); 784 ret = -ENOMEM; 785 goto out_clean; 786 } 787 788 /* 789 * Adjust the number of required free pages after all allocations have 790 * been done. We don't want to run out of pages when writing. 791 */ 792 handle->reqd_free_pages = reqd_free_pages(); 793 794 pr_info("Using %u thread(s) for %s compression\n", nr_threads, hib_comp_algo); 795 pr_info("Compressing and saving image data (%u pages)...\n", 796 nr_to_write); 797 m = nr_to_write / 10; 798 if (!m) 799 m = 1; 800 nr_pages = 0; 801 start = ktime_get(); 802 for (;;) { 803 for (thr = 0; thr < nr_threads; thr++) { 804 for (off = 0; off < UNC_SIZE; off += PAGE_SIZE) { 805 ret = snapshot_read_next(snapshot); 806 if (ret < 0) 807 goto out_finish; 808 809 if (!ret) 810 break; 811 812 memcpy(data[thr].unc + off, 813 data_of(*snapshot), PAGE_SIZE); 814 815 if (!(nr_pages % m)) 816 pr_info("Image saving progress: %3d%%\n", 817 nr_pages / m * 10); 818 nr_pages++; 819 } 820 if (!off) 821 break; 822 823 data[thr].unc_len = off; 824 825 atomic_set_release(&data[thr].ready, 1); 826 wake_up(&data[thr].go); 827 } 828 829 if (!thr) 830 break; 831 832 crc->run_threads = thr; 833 atomic_set_release(&crc->ready, 1); 834 wake_up(&crc->go); 835 836 for (run_threads = thr, thr = 0; thr < run_threads; thr++) { 837 wait_event(data[thr].done, 838 atomic_read_acquire(&data[thr].stop)); 839 atomic_set(&data[thr].stop, 0); 840 841 ret = data[thr].ret; 842 843 if (ret < 0) { 844 pr_err("%s compression failed\n", hib_comp_algo); 845 goto out_finish; 846 } 847 848 if (unlikely(!data[thr].cmp_len || 849 data[thr].cmp_len > 850 bytes_worst_compress(data[thr].unc_len))) { 851 pr_err("Invalid %s compressed length\n", hib_comp_algo); 852 ret = -1; 853 goto out_finish; 854 } 855 856 *(size_t *)data[thr].cmp = data[thr].cmp_len; 857 858 /* 859 * Given we are writing one page at a time to disk, we 860 * copy that much from the buffer, although the last 861 * bit will likely be smaller than full page. This is 862 * OK - we saved the length of the compressed data, so 863 * any garbage at the end will be discarded when we 864 * read it. 865 */ 866 for (off = 0; 867 off < CMP_HEADER + data[thr].cmp_len; 868 off += PAGE_SIZE) { 869 memcpy(page, data[thr].cmp + off, PAGE_SIZE); 870 871 ret = swap_write_page(handle, page, &hb); 872 if (ret) 873 goto out_finish; 874 } 875 } 876 877 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 878 atomic_set(&crc->stop, 0); 879 } 880 881 out_finish: 882 err2 = hib_wait_io(&hb); 883 stop = ktime_get(); 884 if (!ret) 885 ret = err2; 886 if (!ret) 887 pr_info("Image saving done\n"); 888 swsusp_show_speed(start, stop, nr_to_write, "Wrote"); 889 pr_info("Image size after compression: %d kbytes\n", 890 (atomic_read(&compressed_size) / 1024)); 891 892 out_clean: 893 hib_finish_batch(&hb); 894 if (crc) { 895 if (crc->thr) 896 kthread_stop(crc->thr); 897 kfree(crc); 898 } 899 if (data) { 900 for (thr = 0; thr < nr_threads; thr++) { 901 if (data[thr].thr) 902 kthread_stop(data[thr].thr); 903 if (data[thr].cc) 904 crypto_free_comp(data[thr].cc); 905 } 906 vfree(data); 907 } 908 if (page) free_page((unsigned long)page); 909 910 return ret; 911 } 912 913 /** 914 * enough_swap - Make sure we have enough swap to save the image. 915 * 916 * Returns TRUE or FALSE after checking the total amount of swap 917 * space available from the resume partition. 918 */ 919 920 static int enough_swap(unsigned int nr_pages) 921 { 922 unsigned int free_swap = count_swap_pages(root_swap, 1); 923 unsigned int required; 924 925 pr_debug("Free swap pages: %u\n", free_swap); 926 927 required = PAGES_FOR_IO + nr_pages; 928 return free_swap > required; 929 } 930 931 /** 932 * swsusp_write - Write entire image and metadata. 933 * @flags: flags to pass to the "boot" kernel in the image header 934 * 935 * It is important _NOT_ to umount filesystems at this point. We want 936 * them synced (in case something goes wrong) but we DO not want to mark 937 * filesystem clean: it is not. (And it does not matter, if we resume 938 * correctly, we'll mark system clean, anyway.) 939 */ 940 941 int swsusp_write(unsigned int flags) 942 { 943 struct swap_map_handle handle; 944 struct snapshot_handle snapshot; 945 struct swsusp_info *header; 946 unsigned long pages; 947 int error; 948 949 pages = snapshot_get_image_size(); 950 error = get_swap_writer(&handle); 951 if (error) { 952 pr_err("Cannot get swap writer\n"); 953 return error; 954 } 955 if (flags & SF_NOCOMPRESS_MODE) { 956 if (!enough_swap(pages)) { 957 pr_err("Not enough free swap\n"); 958 error = -ENOSPC; 959 goto out_finish; 960 } 961 } 962 memset(&snapshot, 0, sizeof(struct snapshot_handle)); 963 error = snapshot_read_next(&snapshot); 964 if (error < (int)PAGE_SIZE) { 965 if (error >= 0) 966 error = -EFAULT; 967 968 goto out_finish; 969 } 970 header = (struct swsusp_info *)data_of(snapshot); 971 error = swap_write_page(&handle, header, NULL); 972 if (!error) { 973 error = (flags & SF_NOCOMPRESS_MODE) ? 974 save_image(&handle, &snapshot, pages - 1) : 975 save_compressed_image(&handle, &snapshot, pages - 1); 976 } 977 out_finish: 978 error = swap_writer_finish(&handle, flags, error); 979 return error; 980 } 981 982 /* 983 * The following functions allow us to read data using a swap map 984 * in a file-like way. 985 */ 986 987 static void release_swap_reader(struct swap_map_handle *handle) 988 { 989 struct swap_map_page_list *tmp; 990 991 while (handle->maps) { 992 if (handle->maps->map) 993 free_page((unsigned long)handle->maps->map); 994 tmp = handle->maps; 995 handle->maps = handle->maps->next; 996 kfree(tmp); 997 } 998 handle->cur = NULL; 999 } 1000 1001 static int get_swap_reader(struct swap_map_handle *handle, 1002 unsigned int *flags_p) 1003 { 1004 int error; 1005 struct swap_map_page_list *tmp, *last; 1006 sector_t offset; 1007 1008 *flags_p = swsusp_header->flags; 1009 1010 if (!swsusp_header->image) /* how can this happen? */ 1011 return -EINVAL; 1012 1013 handle->cur = NULL; 1014 last = handle->maps = NULL; 1015 offset = swsusp_header->image; 1016 while (offset) { 1017 tmp = kzalloc(sizeof(*handle->maps), GFP_KERNEL); 1018 if (!tmp) { 1019 release_swap_reader(handle); 1020 return -ENOMEM; 1021 } 1022 if (!handle->maps) 1023 handle->maps = tmp; 1024 if (last) 1025 last->next = tmp; 1026 last = tmp; 1027 1028 tmp->map = (struct swap_map_page *) 1029 __get_free_page(GFP_NOIO | __GFP_HIGH); 1030 if (!tmp->map) { 1031 release_swap_reader(handle); 1032 return -ENOMEM; 1033 } 1034 1035 error = hib_submit_io(REQ_OP_READ, offset, tmp->map, NULL); 1036 if (error) { 1037 release_swap_reader(handle); 1038 return error; 1039 } 1040 offset = tmp->map->next_swap; 1041 } 1042 handle->k = 0; 1043 handle->cur = handle->maps->map; 1044 return 0; 1045 } 1046 1047 static int swap_read_page(struct swap_map_handle *handle, void *buf, 1048 struct hib_bio_batch *hb) 1049 { 1050 sector_t offset; 1051 int error; 1052 struct swap_map_page_list *tmp; 1053 1054 if (!handle->cur) 1055 return -EINVAL; 1056 offset = handle->cur->entries[handle->k]; 1057 if (!offset) 1058 return -EFAULT; 1059 error = hib_submit_io(REQ_OP_READ, offset, buf, hb); 1060 if (error) 1061 return error; 1062 if (++handle->k >= MAP_PAGE_ENTRIES) { 1063 handle->k = 0; 1064 free_page((unsigned long)handle->maps->map); 1065 tmp = handle->maps; 1066 handle->maps = handle->maps->next; 1067 kfree(tmp); 1068 if (!handle->maps) 1069 release_swap_reader(handle); 1070 else 1071 handle->cur = handle->maps->map; 1072 } 1073 return error; 1074 } 1075 1076 static int swap_reader_finish(struct swap_map_handle *handle) 1077 { 1078 release_swap_reader(handle); 1079 1080 return 0; 1081 } 1082 1083 /** 1084 * load_image - load the image using the swap map handle 1085 * @handle and the snapshot handle @snapshot 1086 * (assume there are @nr_pages pages to load) 1087 */ 1088 1089 static int load_image(struct swap_map_handle *handle, 1090 struct snapshot_handle *snapshot, 1091 unsigned int nr_to_read) 1092 { 1093 unsigned int m; 1094 int ret = 0; 1095 ktime_t start; 1096 ktime_t stop; 1097 struct hib_bio_batch hb; 1098 int err2; 1099 unsigned nr_pages; 1100 1101 hib_init_batch(&hb); 1102 1103 clean_pages_on_read = true; 1104 pr_info("Loading image data pages (%u pages)...\n", nr_to_read); 1105 m = nr_to_read / 10; 1106 if (!m) 1107 m = 1; 1108 nr_pages = 0; 1109 start = ktime_get(); 1110 for ( ; ; ) { 1111 ret = snapshot_write_next(snapshot); 1112 if (ret <= 0) 1113 break; 1114 ret = swap_read_page(handle, data_of(*snapshot), &hb); 1115 if (ret) 1116 break; 1117 if (snapshot->sync_read) 1118 ret = hib_wait_io(&hb); 1119 if (ret) 1120 break; 1121 if (!(nr_pages % m)) 1122 pr_info("Image loading progress: %3d%%\n", 1123 nr_pages / m * 10); 1124 nr_pages++; 1125 } 1126 err2 = hib_wait_io(&hb); 1127 hib_finish_batch(&hb); 1128 stop = ktime_get(); 1129 if (!ret) 1130 ret = err2; 1131 if (!ret) { 1132 pr_info("Image loading done\n"); 1133 ret = snapshot_write_finalize(snapshot); 1134 if (!ret && !snapshot_image_loaded(snapshot)) 1135 ret = -ENODATA; 1136 } 1137 swsusp_show_speed(start, stop, nr_to_read, "Read"); 1138 return ret; 1139 } 1140 1141 /* 1142 * Structure used for data decompression. 1143 */ 1144 struct dec_data { 1145 struct task_struct *thr; /* thread */ 1146 struct crypto_comp *cc; /* crypto compressor stream */ 1147 atomic_t ready; /* ready to start flag */ 1148 atomic_t stop; /* ready to stop flag */ 1149 int ret; /* return code */ 1150 wait_queue_head_t go; /* start decompression */ 1151 wait_queue_head_t done; /* decompression done */ 1152 size_t unc_len; /* uncompressed length */ 1153 size_t cmp_len; /* compressed length */ 1154 unsigned char unc[UNC_SIZE]; /* uncompressed buffer */ 1155 unsigned char cmp[CMP_SIZE]; /* compressed buffer */ 1156 }; 1157 1158 /* 1159 * Decompression function that runs in its own thread. 1160 */ 1161 static int decompress_threadfn(void *data) 1162 { 1163 struct dec_data *d = data; 1164 unsigned int unc_len = 0; 1165 1166 while (1) { 1167 wait_event(d->go, atomic_read_acquire(&d->ready) || 1168 kthread_should_stop()); 1169 if (kthread_should_stop()) { 1170 d->thr = NULL; 1171 d->ret = -1; 1172 atomic_set_release(&d->stop, 1); 1173 wake_up(&d->done); 1174 break; 1175 } 1176 atomic_set(&d->ready, 0); 1177 1178 unc_len = UNC_SIZE; 1179 d->ret = crypto_comp_decompress(d->cc, d->cmp + CMP_HEADER, d->cmp_len, 1180 d->unc, &unc_len); 1181 d->unc_len = unc_len; 1182 1183 if (clean_pages_on_decompress) 1184 flush_icache_range((unsigned long)d->unc, 1185 (unsigned long)d->unc + d->unc_len); 1186 1187 atomic_set_release(&d->stop, 1); 1188 wake_up(&d->done); 1189 } 1190 return 0; 1191 } 1192 1193 /** 1194 * load_compressed_image - Load compressed image data and decompress it. 1195 * @handle: Swap map handle to use for loading data. 1196 * @snapshot: Image to copy uncompressed data into. 1197 * @nr_to_read: Number of pages to load. 1198 */ 1199 static int load_compressed_image(struct swap_map_handle *handle, 1200 struct snapshot_handle *snapshot, 1201 unsigned int nr_to_read) 1202 { 1203 unsigned int m; 1204 int ret = 0; 1205 int eof = 0; 1206 struct hib_bio_batch hb; 1207 ktime_t start; 1208 ktime_t stop; 1209 unsigned nr_pages; 1210 size_t off; 1211 unsigned i, thr, run_threads, nr_threads; 1212 unsigned ring = 0, pg = 0, ring_size = 0, 1213 have = 0, want, need, asked = 0; 1214 unsigned long read_pages = 0; 1215 unsigned char **page = NULL; 1216 struct dec_data *data = NULL; 1217 struct crc_data *crc = NULL; 1218 1219 hib_init_batch(&hb); 1220 1221 /* 1222 * We'll limit the number of threads for decompression to limit memory 1223 * footprint. 1224 */ 1225 nr_threads = num_online_cpus() - 1; 1226 nr_threads = clamp_val(nr_threads, 1, CMP_THREADS); 1227 1228 page = vmalloc(array_size(CMP_MAX_RD_PAGES, sizeof(*page))); 1229 if (!page) { 1230 pr_err("Failed to allocate %s page\n", hib_comp_algo); 1231 ret = -ENOMEM; 1232 goto out_clean; 1233 } 1234 1235 data = vzalloc(array_size(nr_threads, sizeof(*data))); 1236 if (!data) { 1237 pr_err("Failed to allocate %s data\n", hib_comp_algo); 1238 ret = -ENOMEM; 1239 goto out_clean; 1240 } 1241 1242 crc = kzalloc(sizeof(*crc), GFP_KERNEL); 1243 if (!crc) { 1244 pr_err("Failed to allocate crc\n"); 1245 ret = -ENOMEM; 1246 goto out_clean; 1247 } 1248 1249 clean_pages_on_decompress = true; 1250 1251 /* 1252 * Start the decompression threads. 1253 */ 1254 for (thr = 0; thr < nr_threads; thr++) { 1255 init_waitqueue_head(&data[thr].go); 1256 init_waitqueue_head(&data[thr].done); 1257 1258 data[thr].cc = crypto_alloc_comp(hib_comp_algo, 0, 0); 1259 if (IS_ERR_OR_NULL(data[thr].cc)) { 1260 pr_err("Could not allocate comp stream %ld\n", PTR_ERR(data[thr].cc)); 1261 ret = -EFAULT; 1262 goto out_clean; 1263 } 1264 1265 data[thr].thr = kthread_run(decompress_threadfn, 1266 &data[thr], 1267 "image_decompress/%u", thr); 1268 if (IS_ERR(data[thr].thr)) { 1269 data[thr].thr = NULL; 1270 pr_err("Cannot start decompression threads\n"); 1271 ret = -ENOMEM; 1272 goto out_clean; 1273 } 1274 } 1275 1276 /* 1277 * Start the CRC32 thread. 1278 */ 1279 init_waitqueue_head(&crc->go); 1280 init_waitqueue_head(&crc->done); 1281 1282 handle->crc32 = 0; 1283 crc->crc32 = &handle->crc32; 1284 for (thr = 0; thr < nr_threads; thr++) { 1285 crc->unc[thr] = data[thr].unc; 1286 crc->unc_len[thr] = &data[thr].unc_len; 1287 } 1288 1289 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32"); 1290 if (IS_ERR(crc->thr)) { 1291 crc->thr = NULL; 1292 pr_err("Cannot start CRC32 thread\n"); 1293 ret = -ENOMEM; 1294 goto out_clean; 1295 } 1296 1297 /* 1298 * Set the number of pages for read buffering. 1299 * This is complete guesswork, because we'll only know the real 1300 * picture once prepare_image() is called, which is much later on 1301 * during the image load phase. We'll assume the worst case and 1302 * say that none of the image pages are from high memory. 1303 */ 1304 if (low_free_pages() > snapshot_get_image_size()) 1305 read_pages = (low_free_pages() - snapshot_get_image_size()) / 2; 1306 read_pages = clamp_val(read_pages, CMP_MIN_RD_PAGES, CMP_MAX_RD_PAGES); 1307 1308 for (i = 0; i < read_pages; i++) { 1309 page[i] = (void *)__get_free_page(i < CMP_PAGES ? 1310 GFP_NOIO | __GFP_HIGH : 1311 GFP_NOIO | __GFP_NOWARN | 1312 __GFP_NORETRY); 1313 1314 if (!page[i]) { 1315 if (i < CMP_PAGES) { 1316 ring_size = i; 1317 pr_err("Failed to allocate %s pages\n", hib_comp_algo); 1318 ret = -ENOMEM; 1319 goto out_clean; 1320 } else { 1321 break; 1322 } 1323 } 1324 } 1325 want = ring_size = i; 1326 1327 pr_info("Using %u thread(s) for %s decompression\n", nr_threads, hib_comp_algo); 1328 pr_info("Loading and decompressing image data (%u pages)...\n", 1329 nr_to_read); 1330 m = nr_to_read / 10; 1331 if (!m) 1332 m = 1; 1333 nr_pages = 0; 1334 start = ktime_get(); 1335 1336 ret = snapshot_write_next(snapshot); 1337 if (ret <= 0) 1338 goto out_finish; 1339 1340 for(;;) { 1341 for (i = 0; !eof && i < want; i++) { 1342 ret = swap_read_page(handle, page[ring], &hb); 1343 if (ret) { 1344 /* 1345 * On real read error, finish. On end of data, 1346 * set EOF flag and just exit the read loop. 1347 */ 1348 if (handle->cur && 1349 handle->cur->entries[handle->k]) { 1350 goto out_finish; 1351 } else { 1352 eof = 1; 1353 break; 1354 } 1355 } 1356 if (++ring >= ring_size) 1357 ring = 0; 1358 } 1359 asked += i; 1360 want -= i; 1361 1362 /* 1363 * We are out of data, wait for some more. 1364 */ 1365 if (!have) { 1366 if (!asked) 1367 break; 1368 1369 ret = hib_wait_io(&hb); 1370 if (ret) 1371 goto out_finish; 1372 have += asked; 1373 asked = 0; 1374 if (eof) 1375 eof = 2; 1376 } 1377 1378 if (crc->run_threads) { 1379 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 1380 atomic_set(&crc->stop, 0); 1381 crc->run_threads = 0; 1382 } 1383 1384 for (thr = 0; have && thr < nr_threads; thr++) { 1385 data[thr].cmp_len = *(size_t *)page[pg]; 1386 if (unlikely(!data[thr].cmp_len || 1387 data[thr].cmp_len > 1388 bytes_worst_compress(UNC_SIZE))) { 1389 pr_err("Invalid %s compressed length\n", hib_comp_algo); 1390 ret = -1; 1391 goto out_finish; 1392 } 1393 1394 need = DIV_ROUND_UP(data[thr].cmp_len + CMP_HEADER, 1395 PAGE_SIZE); 1396 if (need > have) { 1397 if (eof > 1) { 1398 ret = -1; 1399 goto out_finish; 1400 } 1401 break; 1402 } 1403 1404 for (off = 0; 1405 off < CMP_HEADER + data[thr].cmp_len; 1406 off += PAGE_SIZE) { 1407 memcpy(data[thr].cmp + off, 1408 page[pg], PAGE_SIZE); 1409 have--; 1410 want++; 1411 if (++pg >= ring_size) 1412 pg = 0; 1413 } 1414 1415 atomic_set_release(&data[thr].ready, 1); 1416 wake_up(&data[thr].go); 1417 } 1418 1419 /* 1420 * Wait for more data while we are decompressing. 1421 */ 1422 if (have < CMP_PAGES && asked) { 1423 ret = hib_wait_io(&hb); 1424 if (ret) 1425 goto out_finish; 1426 have += asked; 1427 asked = 0; 1428 if (eof) 1429 eof = 2; 1430 } 1431 1432 for (run_threads = thr, thr = 0; thr < run_threads; thr++) { 1433 wait_event(data[thr].done, 1434 atomic_read_acquire(&data[thr].stop)); 1435 atomic_set(&data[thr].stop, 0); 1436 1437 ret = data[thr].ret; 1438 1439 if (ret < 0) { 1440 pr_err("%s decompression failed\n", hib_comp_algo); 1441 goto out_finish; 1442 } 1443 1444 if (unlikely(!data[thr].unc_len || 1445 data[thr].unc_len > UNC_SIZE || 1446 data[thr].unc_len & (PAGE_SIZE - 1))) { 1447 pr_err("Invalid %s uncompressed length\n", hib_comp_algo); 1448 ret = -1; 1449 goto out_finish; 1450 } 1451 1452 for (off = 0; 1453 off < data[thr].unc_len; off += PAGE_SIZE) { 1454 memcpy(data_of(*snapshot), 1455 data[thr].unc + off, PAGE_SIZE); 1456 1457 if (!(nr_pages % m)) 1458 pr_info("Image loading progress: %3d%%\n", 1459 nr_pages / m * 10); 1460 nr_pages++; 1461 1462 ret = snapshot_write_next(snapshot); 1463 if (ret <= 0) { 1464 crc->run_threads = thr + 1; 1465 atomic_set_release(&crc->ready, 1); 1466 wake_up(&crc->go); 1467 goto out_finish; 1468 } 1469 } 1470 } 1471 1472 crc->run_threads = thr; 1473 atomic_set_release(&crc->ready, 1); 1474 wake_up(&crc->go); 1475 } 1476 1477 out_finish: 1478 if (crc->run_threads) { 1479 wait_event(crc->done, atomic_read_acquire(&crc->stop)); 1480 atomic_set(&crc->stop, 0); 1481 } 1482 stop = ktime_get(); 1483 if (!ret) { 1484 pr_info("Image loading done\n"); 1485 ret = snapshot_write_finalize(snapshot); 1486 if (!ret && !snapshot_image_loaded(snapshot)) 1487 ret = -ENODATA; 1488 if (!ret) { 1489 if (swsusp_header->flags & SF_CRC32_MODE) { 1490 if(handle->crc32 != swsusp_header->crc32) { 1491 pr_err("Invalid image CRC32!\n"); 1492 ret = -ENODATA; 1493 } 1494 } 1495 } 1496 } 1497 swsusp_show_speed(start, stop, nr_to_read, "Read"); 1498 out_clean: 1499 hib_finish_batch(&hb); 1500 for (i = 0; i < ring_size; i++) 1501 free_page((unsigned long)page[i]); 1502 if (crc) { 1503 if (crc->thr) 1504 kthread_stop(crc->thr); 1505 kfree(crc); 1506 } 1507 if (data) { 1508 for (thr = 0; thr < nr_threads; thr++) { 1509 if (data[thr].thr) 1510 kthread_stop(data[thr].thr); 1511 if (data[thr].cc) 1512 crypto_free_comp(data[thr].cc); 1513 } 1514 vfree(data); 1515 } 1516 vfree(page); 1517 1518 return ret; 1519 } 1520 1521 /** 1522 * swsusp_read - read the hibernation image. 1523 * @flags_p: flags passed by the "frozen" kernel in the image header should 1524 * be written into this memory location 1525 */ 1526 1527 int swsusp_read(unsigned int *flags_p) 1528 { 1529 int error; 1530 struct swap_map_handle handle; 1531 struct snapshot_handle snapshot; 1532 struct swsusp_info *header; 1533 1534 memset(&snapshot, 0, sizeof(struct snapshot_handle)); 1535 error = snapshot_write_next(&snapshot); 1536 if (error < (int)PAGE_SIZE) 1537 return error < 0 ? error : -EFAULT; 1538 header = (struct swsusp_info *)data_of(snapshot); 1539 error = get_swap_reader(&handle, flags_p); 1540 if (error) 1541 goto end; 1542 if (!error) 1543 error = swap_read_page(&handle, header, NULL); 1544 if (!error) { 1545 error = (*flags_p & SF_NOCOMPRESS_MODE) ? 1546 load_image(&handle, &snapshot, header->pages - 1) : 1547 load_compressed_image(&handle, &snapshot, header->pages - 1); 1548 } 1549 swap_reader_finish(&handle); 1550 end: 1551 if (!error) 1552 pr_debug("Image successfully loaded\n"); 1553 else 1554 pr_debug("Error %d resuming\n", error); 1555 return error; 1556 } 1557 1558 static void *swsusp_holder; 1559 1560 /** 1561 * swsusp_check - Open the resume device and check for the swsusp signature. 1562 * @exclusive: Open the resume device exclusively. 1563 */ 1564 1565 int swsusp_check(bool exclusive) 1566 { 1567 void *holder = exclusive ? &swsusp_holder : NULL; 1568 int error; 1569 1570 hib_resume_bdev_file = bdev_file_open_by_dev(swsusp_resume_device, 1571 BLK_OPEN_READ, holder, NULL); 1572 if (!IS_ERR(hib_resume_bdev_file)) { 1573 clear_page(swsusp_header); 1574 error = hib_submit_io(REQ_OP_READ, swsusp_resume_block, 1575 swsusp_header, NULL); 1576 if (error) 1577 goto put; 1578 1579 if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) { 1580 memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10); 1581 swsusp_header_flags = swsusp_header->flags; 1582 /* Reset swap signature now */ 1583 error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC, 1584 swsusp_resume_block, 1585 swsusp_header, NULL); 1586 } else { 1587 error = -EINVAL; 1588 } 1589 if (!error && swsusp_header->flags & SF_HW_SIG && 1590 swsusp_header->hw_sig != swsusp_hardware_signature) { 1591 pr_info("Suspend image hardware signature mismatch (%08x now %08x); aborting resume.\n", 1592 swsusp_header->hw_sig, swsusp_hardware_signature); 1593 error = -EINVAL; 1594 } 1595 1596 put: 1597 if (error) 1598 bdev_fput(hib_resume_bdev_file); 1599 else 1600 pr_debug("Image signature found, resuming\n"); 1601 } else { 1602 error = PTR_ERR(hib_resume_bdev_file); 1603 } 1604 1605 if (error) 1606 pr_debug("Image not found (code %d)\n", error); 1607 1608 return error; 1609 } 1610 1611 /** 1612 * swsusp_close - close resume device. 1613 */ 1614 1615 void swsusp_close(void) 1616 { 1617 if (IS_ERR(hib_resume_bdev_file)) { 1618 pr_debug("Image device not initialised\n"); 1619 return; 1620 } 1621 1622 fput(hib_resume_bdev_file); 1623 } 1624 1625 /** 1626 * swsusp_unmark - Unmark swsusp signature in the resume device 1627 */ 1628 1629 #ifdef CONFIG_SUSPEND 1630 int swsusp_unmark(void) 1631 { 1632 int error; 1633 1634 hib_submit_io(REQ_OP_READ, swsusp_resume_block, 1635 swsusp_header, NULL); 1636 if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) { 1637 memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10); 1638 error = hib_submit_io(REQ_OP_WRITE | REQ_SYNC, 1639 swsusp_resume_block, 1640 swsusp_header, NULL); 1641 } else { 1642 pr_err("Cannot find swsusp signature!\n"); 1643 error = -ENODEV; 1644 } 1645 1646 /* 1647 * We just returned from suspend, we don't need the image any more. 1648 */ 1649 free_all_swap_pages(root_swap); 1650 1651 return error; 1652 } 1653 #endif 1654 1655 static int __init swsusp_header_init(void) 1656 { 1657 swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL); 1658 if (!swsusp_header) 1659 panic("Could not allocate memory for swsusp_header\n"); 1660 return 0; 1661 } 1662 1663 core_initcall(swsusp_header_init); 1664