1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * kernel/power/suspend.c - Suspend to RAM and standby functionality. 4 * 5 * Copyright (c) 2003 Patrick Mochel 6 * Copyright (c) 2003 Open Source Development Lab 7 * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc. 8 */ 9 10 #define pr_fmt(fmt) "PM: " fmt 11 12 #include <linux/string.h> 13 #include <linux/delay.h> 14 #include <linux/errno.h> 15 #include <linux/init.h> 16 #include <linux/console.h> 17 #include <linux/cpu.h> 18 #include <linux/cpuidle.h> 19 #include <linux/gfp.h> 20 #include <linux/io.h> 21 #include <linux/kernel.h> 22 #include <linux/list.h> 23 #include <linux/mm.h> 24 #include <linux/slab.h> 25 #include <linux/export.h> 26 #include <linux/suspend.h> 27 #include <linux/syscore_ops.h> 28 #include <linux/swait.h> 29 #include <linux/ftrace.h> 30 #include <trace/events/power.h> 31 #include <linux/compiler.h> 32 #include <linux/moduleparam.h> 33 34 #include "power.h" 35 36 const char * const pm_labels[] = { 37 [PM_SUSPEND_TO_IDLE] = "freeze", 38 [PM_SUSPEND_STANDBY] = "standby", 39 [PM_SUSPEND_MEM] = "mem", 40 }; 41 const char *pm_states[PM_SUSPEND_MAX]; 42 static const char * const mem_sleep_labels[] = { 43 [PM_SUSPEND_TO_IDLE] = "s2idle", 44 [PM_SUSPEND_STANDBY] = "shallow", 45 [PM_SUSPEND_MEM] = "deep", 46 }; 47 const char *mem_sleep_states[PM_SUSPEND_MAX]; 48 49 suspend_state_t mem_sleep_current = PM_SUSPEND_TO_IDLE; 50 suspend_state_t mem_sleep_default = PM_SUSPEND_MAX; 51 suspend_state_t pm_suspend_target_state; 52 EXPORT_SYMBOL_GPL(pm_suspend_target_state); 53 54 unsigned int pm_suspend_global_flags; 55 EXPORT_SYMBOL_GPL(pm_suspend_global_flags); 56 57 static const struct platform_suspend_ops *suspend_ops; 58 static const struct platform_s2idle_ops *s2idle_ops; 59 static DECLARE_SWAIT_QUEUE_HEAD(s2idle_wait_head); 60 61 enum s2idle_states __read_mostly s2idle_state; 62 static DEFINE_RAW_SPINLOCK(s2idle_lock); 63 64 /** 65 * pm_suspend_default_s2idle - Check if suspend-to-idle is the default suspend. 66 * 67 * Return 'true' if suspend-to-idle has been selected as the default system 68 * suspend method. 69 */ 70 bool pm_suspend_default_s2idle(void) 71 { 72 return mem_sleep_current == PM_SUSPEND_TO_IDLE; 73 } 74 EXPORT_SYMBOL_GPL(pm_suspend_default_s2idle); 75 76 void s2idle_set_ops(const struct platform_s2idle_ops *ops) 77 { 78 unsigned int sleep_flags; 79 80 sleep_flags = lock_system_sleep(); 81 s2idle_ops = ops; 82 unlock_system_sleep(sleep_flags); 83 } 84 85 static void s2idle_begin(void) 86 { 87 s2idle_state = S2IDLE_STATE_NONE; 88 } 89 90 static void s2idle_enter(void) 91 { 92 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, true); 93 94 /* 95 * The correctness of the code below depends on the number of online 96 * CPUs being stable, but CPUs cannot be taken offline or put online 97 * while it is running. 98 * 99 * The s2idle_lock must be acquired before the pending wakeup check to 100 * prevent pm_system_wakeup() from running as a whole between that check 101 * and the subsequent s2idle_state update in which case a wakeup event 102 * would get lost. 103 */ 104 raw_spin_lock_irq(&s2idle_lock); 105 if (pm_wakeup_pending()) 106 goto out; 107 108 s2idle_state = S2IDLE_STATE_ENTER; 109 raw_spin_unlock_irq(&s2idle_lock); 110 111 /* Push all the CPUs into the idle loop. */ 112 wake_up_all_idle_cpus(); 113 /* Make the current CPU wait so it can enter the idle loop too. */ 114 swait_event_exclusive(s2idle_wait_head, 115 s2idle_state == S2IDLE_STATE_WAKE); 116 117 /* 118 * Kick all CPUs to ensure that they resume their timers and restore 119 * consistent system state. 120 */ 121 wake_up_all_idle_cpus(); 122 123 raw_spin_lock_irq(&s2idle_lock); 124 125 out: 126 s2idle_state = S2IDLE_STATE_NONE; 127 raw_spin_unlock_irq(&s2idle_lock); 128 129 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, false); 130 } 131 132 static void s2idle_loop(void) 133 { 134 pm_pr_dbg("suspend-to-idle\n"); 135 136 /* 137 * Suspend-to-idle equals: 138 * frozen processes + suspended devices + idle processors. 139 * Thus s2idle_enter() should be called right after all devices have 140 * been suspended. 141 * 142 * Wakeups during the noirq suspend of devices may be spurious, so try 143 * to avoid them upfront. 144 */ 145 for (;;) { 146 if (s2idle_ops && s2idle_ops->wake) { 147 if (s2idle_ops->wake()) 148 break; 149 } else if (pm_wakeup_pending()) { 150 break; 151 } 152 153 if (s2idle_ops && s2idle_ops->check) 154 s2idle_ops->check(); 155 156 s2idle_enter(); 157 } 158 159 pm_pr_dbg("resume from suspend-to-idle\n"); 160 } 161 162 void s2idle_wake(void) 163 { 164 unsigned long flags; 165 166 raw_spin_lock_irqsave(&s2idle_lock, flags); 167 if (s2idle_state > S2IDLE_STATE_NONE) { 168 s2idle_state = S2IDLE_STATE_WAKE; 169 swake_up_one(&s2idle_wait_head); 170 } 171 raw_spin_unlock_irqrestore(&s2idle_lock, flags); 172 } 173 EXPORT_SYMBOL_GPL(s2idle_wake); 174 175 static bool valid_state(suspend_state_t state) 176 { 177 /* 178 * The PM_SUSPEND_STANDBY and PM_SUSPEND_MEM states require low-level 179 * support and need to be valid to the low-level implementation. 180 * 181 * No ->valid() or ->enter() callback implies that none are valid. 182 */ 183 return suspend_ops && suspend_ops->valid && suspend_ops->valid(state) && 184 suspend_ops->enter; 185 } 186 187 void __init pm_states_init(void) 188 { 189 /* "mem" and "freeze" are always present in /sys/power/state. */ 190 pm_states[PM_SUSPEND_MEM] = pm_labels[PM_SUSPEND_MEM]; 191 pm_states[PM_SUSPEND_TO_IDLE] = pm_labels[PM_SUSPEND_TO_IDLE]; 192 /* 193 * Suspend-to-idle should be supported even without any suspend_ops, 194 * initialize mem_sleep_states[] accordingly here. 195 */ 196 mem_sleep_states[PM_SUSPEND_TO_IDLE] = mem_sleep_labels[PM_SUSPEND_TO_IDLE]; 197 } 198 199 static int __init mem_sleep_default_setup(char *str) 200 { 201 suspend_state_t state; 202 203 for (state = PM_SUSPEND_TO_IDLE; state <= PM_SUSPEND_MEM; state++) 204 if (mem_sleep_labels[state] && 205 !strcmp(str, mem_sleep_labels[state])) { 206 mem_sleep_default = state; 207 mem_sleep_current = state; 208 break; 209 } 210 211 return 1; 212 } 213 __setup("mem_sleep_default=", mem_sleep_default_setup); 214 215 /** 216 * suspend_set_ops - Set the global suspend method table. 217 * @ops: Suspend operations to use. 218 */ 219 void suspend_set_ops(const struct platform_suspend_ops *ops) 220 { 221 unsigned int sleep_flags; 222 223 sleep_flags = lock_system_sleep(); 224 225 suspend_ops = ops; 226 227 if (valid_state(PM_SUSPEND_STANDBY)) { 228 mem_sleep_states[PM_SUSPEND_STANDBY] = mem_sleep_labels[PM_SUSPEND_STANDBY]; 229 pm_states[PM_SUSPEND_STANDBY] = pm_labels[PM_SUSPEND_STANDBY]; 230 if (mem_sleep_default == PM_SUSPEND_STANDBY) 231 mem_sleep_current = PM_SUSPEND_STANDBY; 232 } 233 if (valid_state(PM_SUSPEND_MEM)) { 234 mem_sleep_states[PM_SUSPEND_MEM] = mem_sleep_labels[PM_SUSPEND_MEM]; 235 if (mem_sleep_default >= PM_SUSPEND_MEM) 236 mem_sleep_current = PM_SUSPEND_MEM; 237 } 238 239 unlock_system_sleep(sleep_flags); 240 } 241 EXPORT_SYMBOL_GPL(suspend_set_ops); 242 243 /** 244 * suspend_valid_only_mem - Generic memory-only valid callback. 245 * @state: Target system sleep state. 246 * 247 * Platform drivers that implement mem suspend only and only need to check for 248 * that in their .valid() callback can use this instead of rolling their own 249 * .valid() callback. 250 */ 251 int suspend_valid_only_mem(suspend_state_t state) 252 { 253 return state == PM_SUSPEND_MEM; 254 } 255 EXPORT_SYMBOL_GPL(suspend_valid_only_mem); 256 257 static bool sleep_state_supported(suspend_state_t state) 258 { 259 return state == PM_SUSPEND_TO_IDLE || 260 (valid_state(state) && !cxl_mem_active()); 261 } 262 263 static int platform_suspend_prepare(suspend_state_t state) 264 { 265 return state != PM_SUSPEND_TO_IDLE && suspend_ops->prepare ? 266 suspend_ops->prepare() : 0; 267 } 268 269 static int platform_suspend_prepare_late(suspend_state_t state) 270 { 271 return state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->prepare ? 272 s2idle_ops->prepare() : 0; 273 } 274 275 static int platform_suspend_prepare_noirq(suspend_state_t state) 276 { 277 if (state == PM_SUSPEND_TO_IDLE) 278 return s2idle_ops && s2idle_ops->prepare_late ? 279 s2idle_ops->prepare_late() : 0; 280 281 return suspend_ops->prepare_late ? suspend_ops->prepare_late() : 0; 282 } 283 284 static void platform_resume_noirq(suspend_state_t state) 285 { 286 if (state == PM_SUSPEND_TO_IDLE) { 287 if (s2idle_ops && s2idle_ops->restore_early) 288 s2idle_ops->restore_early(); 289 } else if (suspend_ops->wake) { 290 suspend_ops->wake(); 291 } 292 } 293 294 static void platform_resume_early(suspend_state_t state) 295 { 296 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->restore) 297 s2idle_ops->restore(); 298 } 299 300 static void platform_resume_finish(suspend_state_t state) 301 { 302 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->finish) 303 suspend_ops->finish(); 304 } 305 306 static int platform_suspend_begin(suspend_state_t state) 307 { 308 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->begin) 309 return s2idle_ops->begin(); 310 else if (suspend_ops && suspend_ops->begin) 311 return suspend_ops->begin(state); 312 else 313 return 0; 314 } 315 316 static void platform_resume_end(suspend_state_t state) 317 { 318 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->end) 319 s2idle_ops->end(); 320 else if (suspend_ops && suspend_ops->end) 321 suspend_ops->end(); 322 } 323 324 static void platform_recover(suspend_state_t state) 325 { 326 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->recover) 327 suspend_ops->recover(); 328 } 329 330 static bool platform_suspend_again(suspend_state_t state) 331 { 332 return state != PM_SUSPEND_TO_IDLE && suspend_ops->suspend_again ? 333 suspend_ops->suspend_again() : false; 334 } 335 336 #ifdef CONFIG_PM_DEBUG 337 static unsigned int pm_test_delay = 5; 338 module_param(pm_test_delay, uint, 0644); 339 MODULE_PARM_DESC(pm_test_delay, 340 "Number of seconds to wait before resuming from suspend test"); 341 #endif 342 343 static int suspend_test(int level) 344 { 345 #ifdef CONFIG_PM_DEBUG 346 if (pm_test_level == level) { 347 pr_info("suspend debug: Waiting for %d second(s).\n", 348 pm_test_delay); 349 mdelay(pm_test_delay * 1000); 350 return 1; 351 } 352 #endif /* !CONFIG_PM_DEBUG */ 353 return 0; 354 } 355 356 /** 357 * suspend_prepare - Prepare for entering system sleep state. 358 * @state: Target system sleep state. 359 * 360 * Common code run for every system sleep state that can be entered (except for 361 * hibernation). Run suspend notifiers, allocate the "suspend" console and 362 * freeze processes. 363 */ 364 static int suspend_prepare(suspend_state_t state) 365 { 366 int error; 367 368 if (!sleep_state_supported(state)) 369 return -EPERM; 370 371 pm_prepare_console(); 372 373 error = pm_notifier_call_chain_robust(PM_SUSPEND_PREPARE, PM_POST_SUSPEND); 374 if (error) 375 goto Restore; 376 377 trace_suspend_resume(TPS("freeze_processes"), 0, true); 378 error = suspend_freeze_processes(); 379 trace_suspend_resume(TPS("freeze_processes"), 0, false); 380 if (!error) 381 return 0; 382 383 dpm_save_failed_step(SUSPEND_FREEZE); 384 pm_notifier_call_chain(PM_POST_SUSPEND); 385 Restore: 386 pm_restore_console(); 387 return error; 388 } 389 390 /* default implementation */ 391 void __weak arch_suspend_disable_irqs(void) 392 { 393 local_irq_disable(); 394 } 395 396 /* default implementation */ 397 void __weak arch_suspend_enable_irqs(void) 398 { 399 local_irq_enable(); 400 } 401 402 /** 403 * suspend_enter - Make the system enter the given sleep state. 404 * @state: System sleep state to enter. 405 * @wakeup: Returns information that the sleep state should not be re-entered. 406 * 407 * This function should be called after devices have been suspended. 408 */ 409 static int suspend_enter(suspend_state_t state, bool *wakeup) 410 { 411 int error; 412 413 error = platform_suspend_prepare(state); 414 if (error) 415 goto Platform_finish; 416 417 error = dpm_suspend_late(PMSG_SUSPEND); 418 if (error) { 419 pr_err("late suspend of devices failed\n"); 420 goto Platform_finish; 421 } 422 error = platform_suspend_prepare_late(state); 423 if (error) 424 goto Devices_early_resume; 425 426 error = dpm_suspend_noirq(PMSG_SUSPEND); 427 if (error) { 428 pr_err("noirq suspend of devices failed\n"); 429 goto Platform_early_resume; 430 } 431 error = platform_suspend_prepare_noirq(state); 432 if (error) 433 goto Platform_wake; 434 435 if (suspend_test(TEST_PLATFORM)) 436 goto Platform_wake; 437 438 if (state == PM_SUSPEND_TO_IDLE) { 439 s2idle_loop(); 440 goto Platform_wake; 441 } 442 443 error = pm_sleep_disable_secondary_cpus(); 444 if (error || suspend_test(TEST_CPUS)) 445 goto Enable_cpus; 446 447 arch_suspend_disable_irqs(); 448 BUG_ON(!irqs_disabled()); 449 450 system_state = SYSTEM_SUSPEND; 451 452 error = syscore_suspend(); 453 if (!error) { 454 *wakeup = pm_wakeup_pending(); 455 if (!(suspend_test(TEST_CORE) || *wakeup)) { 456 trace_suspend_resume(TPS("machine_suspend"), 457 state, true); 458 error = suspend_ops->enter(state); 459 trace_suspend_resume(TPS("machine_suspend"), 460 state, false); 461 } else if (*wakeup) { 462 error = -EBUSY; 463 } 464 syscore_resume(); 465 } 466 467 system_state = SYSTEM_RUNNING; 468 469 arch_suspend_enable_irqs(); 470 BUG_ON(irqs_disabled()); 471 472 Enable_cpus: 473 pm_sleep_enable_secondary_cpus(); 474 475 Platform_wake: 476 platform_resume_noirq(state); 477 dpm_resume_noirq(PMSG_RESUME); 478 479 Platform_early_resume: 480 platform_resume_early(state); 481 482 Devices_early_resume: 483 dpm_resume_early(PMSG_RESUME); 484 485 Platform_finish: 486 platform_resume_finish(state); 487 return error; 488 } 489 490 /** 491 * suspend_devices_and_enter - Suspend devices and enter system sleep state. 492 * @state: System sleep state to enter. 493 */ 494 int suspend_devices_and_enter(suspend_state_t state) 495 { 496 int error; 497 bool wakeup = false; 498 499 if (!sleep_state_supported(state)) 500 return -ENOSYS; 501 502 pm_suspend_target_state = state; 503 504 if (state == PM_SUSPEND_TO_IDLE) 505 pm_set_suspend_no_platform(); 506 507 error = platform_suspend_begin(state); 508 if (error) 509 goto Close; 510 511 console_suspend_all(); 512 suspend_test_start(); 513 error = dpm_suspend_start(PMSG_SUSPEND); 514 if (error) { 515 pr_err("Some devices failed to suspend, or early wake event detected\n"); 516 goto Recover_platform; 517 } 518 suspend_test_finish("suspend devices"); 519 if (suspend_test(TEST_DEVICES)) 520 goto Recover_platform; 521 522 do { 523 error = suspend_enter(state, &wakeup); 524 } while (!error && !wakeup && platform_suspend_again(state)); 525 526 Resume_devices: 527 suspend_test_start(); 528 dpm_resume_end(PMSG_RESUME); 529 suspend_test_finish("resume devices"); 530 trace_suspend_resume(TPS("console_resume_all"), state, true); 531 console_resume_all(); 532 trace_suspend_resume(TPS("console_resume_all"), state, false); 533 534 Close: 535 platform_resume_end(state); 536 pm_suspend_target_state = PM_SUSPEND_ON; 537 return error; 538 539 Recover_platform: 540 platform_recover(state); 541 goto Resume_devices; 542 } 543 544 /** 545 * suspend_finish - Clean up before finishing the suspend sequence. 546 * 547 * Call platform code to clean up, restart processes, and free the console that 548 * we've allocated. This routine is not called for hibernation. 549 */ 550 static void suspend_finish(void) 551 { 552 suspend_thaw_processes(); 553 pm_notifier_call_chain(PM_POST_SUSPEND); 554 pm_restore_console(); 555 } 556 557 /** 558 * enter_state - Do common work needed to enter system sleep state. 559 * @state: System sleep state to enter. 560 * 561 * Make sure that no one else is trying to put the system into a sleep state. 562 * Fail if that's not the case. Otherwise, prepare for system suspend, make the 563 * system enter the given sleep state and clean up after wakeup. 564 */ 565 static int enter_state(suspend_state_t state) 566 { 567 int error; 568 569 trace_suspend_resume(TPS("suspend_enter"), state, true); 570 if (state == PM_SUSPEND_TO_IDLE) { 571 #ifdef CONFIG_PM_DEBUG 572 if (pm_test_level != TEST_NONE && pm_test_level <= TEST_CPUS) { 573 pr_warn("Unsupported test mode for suspend to idle, please choose none/freezer/devices/platform.\n"); 574 return -EAGAIN; 575 } 576 #endif 577 } else if (!valid_state(state)) { 578 return -EINVAL; 579 } 580 if (!mutex_trylock(&system_transition_mutex)) 581 return -EBUSY; 582 583 if (state == PM_SUSPEND_TO_IDLE) 584 s2idle_begin(); 585 586 if (sync_on_suspend_enabled) { 587 trace_suspend_resume(TPS("sync_filesystems"), 0, true); 588 ksys_sync_helper(); 589 trace_suspend_resume(TPS("sync_filesystems"), 0, false); 590 } 591 592 pm_pr_dbg("Preparing system for sleep (%s)\n", mem_sleep_labels[state]); 593 pm_suspend_clear_flags(); 594 error = suspend_prepare(state); 595 if (error) 596 goto Unlock; 597 598 if (suspend_test(TEST_FREEZER)) 599 goto Finish; 600 601 trace_suspend_resume(TPS("suspend_enter"), state, false); 602 pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]); 603 pm_restrict_gfp_mask(); 604 error = suspend_devices_and_enter(state); 605 pm_restore_gfp_mask(); 606 607 Finish: 608 events_check_enabled = false; 609 pm_pr_dbg("Finishing wakeup.\n"); 610 suspend_finish(); 611 Unlock: 612 mutex_unlock(&system_transition_mutex); 613 return error; 614 } 615 616 /** 617 * pm_suspend - Externally visible function for suspending the system. 618 * @state: System sleep state to enter. 619 * 620 * Check if the value of @state represents one of the supported states, 621 * execute enter_state() and update system suspend statistics. 622 */ 623 int pm_suspend(suspend_state_t state) 624 { 625 int error; 626 627 if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX) 628 return -EINVAL; 629 630 pr_info("suspend entry (%s)\n", mem_sleep_labels[state]); 631 error = enter_state(state); 632 dpm_save_errno(error); 633 pr_info("suspend exit\n"); 634 return error; 635 } 636 EXPORT_SYMBOL(pm_suspend); 637