1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * kernel/power/suspend.c - Suspend to RAM and standby functionality.
4 *
5 * Copyright (c) 2003 Patrick Mochel
6 * Copyright (c) 2003 Open Source Development Lab
7 * Copyright (c) 2009 Rafael J. Wysocki <rjw@sisk.pl>, Novell Inc.
8 */
9
10 #define pr_fmt(fmt) "PM: " fmt
11
12 #include <linux/string.h>
13 #include <linux/delay.h>
14 #include <linux/errno.h>
15 #include <linux/init.h>
16 #include <linux/console.h>
17 #include <linux/cpu.h>
18 #include <linux/cpuidle.h>
19 #include <linux/gfp.h>
20 #include <linux/io.h>
21 #include <linux/kernel.h>
22 #include <linux/list.h>
23 #include <linux/mm.h>
24 #include <linux/slab.h>
25 #include <linux/export.h>
26 #include <linux/suspend.h>
27 #include <linux/syscore_ops.h>
28 #include <linux/swait.h>
29 #include <linux/ftrace.h>
30 #include <trace/events/power.h>
31 #include <linux/compiler.h>
32 #include <linux/moduleparam.h>
33
34 #include "power.h"
35
36 const char * const pm_labels[] = {
37 [PM_SUSPEND_TO_IDLE] = "freeze",
38 [PM_SUSPEND_STANDBY] = "standby",
39 [PM_SUSPEND_MEM] = "mem",
40 };
41 const char *pm_states[PM_SUSPEND_MAX];
42 static const char * const mem_sleep_labels[] = {
43 [PM_SUSPEND_TO_IDLE] = "s2idle",
44 [PM_SUSPEND_STANDBY] = "shallow",
45 [PM_SUSPEND_MEM] = "deep",
46 };
47 const char *mem_sleep_states[PM_SUSPEND_MAX];
48
49 suspend_state_t mem_sleep_current = PM_SUSPEND_TO_IDLE;
50 suspend_state_t mem_sleep_default = PM_SUSPEND_MAX;
51 suspend_state_t pm_suspend_target_state;
52 EXPORT_SYMBOL_GPL(pm_suspend_target_state);
53
54 unsigned int pm_suspend_global_flags;
55 EXPORT_SYMBOL_GPL(pm_suspend_global_flags);
56
57 static const struct platform_suspend_ops *suspend_ops;
58 static const struct platform_s2idle_ops *s2idle_ops;
59 static DECLARE_SWAIT_QUEUE_HEAD(s2idle_wait_head);
60
61 enum s2idle_states __read_mostly s2idle_state;
62 static DEFINE_RAW_SPINLOCK(s2idle_lock);
63
64 /**
65 * pm_suspend_default_s2idle - Check if suspend-to-idle is the default suspend.
66 *
67 * Return 'true' if suspend-to-idle has been selected as the default system
68 * suspend method.
69 */
pm_suspend_default_s2idle(void)70 bool pm_suspend_default_s2idle(void)
71 {
72 return mem_sleep_current == PM_SUSPEND_TO_IDLE;
73 }
74 EXPORT_SYMBOL_GPL(pm_suspend_default_s2idle);
75
s2idle_set_ops(const struct platform_s2idle_ops * ops)76 void s2idle_set_ops(const struct platform_s2idle_ops *ops)
77 {
78 unsigned int sleep_flags;
79
80 sleep_flags = lock_system_sleep();
81 s2idle_ops = ops;
82 unlock_system_sleep(sleep_flags);
83 }
84
s2idle_begin(void)85 static void s2idle_begin(void)
86 {
87 s2idle_state = S2IDLE_STATE_NONE;
88 }
89
s2idle_enter(void)90 static void s2idle_enter(void)
91 {
92 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, true);
93
94 /*
95 * The correctness of the code below depends on the number of online
96 * CPUs being stable, but CPUs cannot be taken offline or put online
97 * while it is running.
98 *
99 * The s2idle_lock must be acquired before the pending wakeup check to
100 * prevent pm_system_wakeup() from running as a whole between that check
101 * and the subsequent s2idle_state update in which case a wakeup event
102 * would get lost.
103 */
104 raw_spin_lock_irq(&s2idle_lock);
105 if (pm_wakeup_pending())
106 goto out;
107
108 s2idle_state = S2IDLE_STATE_ENTER;
109 raw_spin_unlock_irq(&s2idle_lock);
110
111 /* Push all the CPUs into the idle loop. */
112 wake_up_all_idle_cpus();
113 /* Make the current CPU wait so it can enter the idle loop too. */
114 swait_event_exclusive(s2idle_wait_head,
115 s2idle_state == S2IDLE_STATE_WAKE);
116
117 /*
118 * Kick all CPUs to ensure that they resume their timers and restore
119 * consistent system state.
120 */
121 wake_up_all_idle_cpus();
122
123 raw_spin_lock_irq(&s2idle_lock);
124
125 out:
126 s2idle_state = S2IDLE_STATE_NONE;
127 raw_spin_unlock_irq(&s2idle_lock);
128
129 trace_suspend_resume(TPS("machine_suspend"), PM_SUSPEND_TO_IDLE, false);
130 }
131
s2idle_loop(void)132 static void s2idle_loop(void)
133 {
134 pm_pr_dbg("suspend-to-idle\n");
135
136 /*
137 * Suspend-to-idle equals:
138 * frozen processes + suspended devices + idle processors.
139 * Thus s2idle_enter() should be called right after all devices have
140 * been suspended.
141 *
142 * Wakeups during the noirq suspend of devices may be spurious, so try
143 * to avoid them upfront.
144 */
145 for (;;) {
146 if (s2idle_ops && s2idle_ops->wake) {
147 if (s2idle_ops->wake())
148 break;
149 } else if (pm_wakeup_pending()) {
150 break;
151 }
152
153 if (s2idle_ops && s2idle_ops->check)
154 s2idle_ops->check();
155
156 s2idle_enter();
157 }
158
159 pm_pr_dbg("resume from suspend-to-idle\n");
160 }
161
s2idle_wake(void)162 void s2idle_wake(void)
163 {
164 unsigned long flags;
165
166 raw_spin_lock_irqsave(&s2idle_lock, flags);
167 if (s2idle_state > S2IDLE_STATE_NONE) {
168 s2idle_state = S2IDLE_STATE_WAKE;
169 swake_up_one(&s2idle_wait_head);
170 }
171 raw_spin_unlock_irqrestore(&s2idle_lock, flags);
172 }
173 EXPORT_SYMBOL_GPL(s2idle_wake);
174
valid_state(suspend_state_t state)175 static bool valid_state(suspend_state_t state)
176 {
177 /*
178 * The PM_SUSPEND_STANDBY and PM_SUSPEND_MEM states require low-level
179 * support and need to be valid to the low-level implementation.
180 *
181 * No ->valid() or ->enter() callback implies that none are valid.
182 */
183 return suspend_ops && suspend_ops->valid && suspend_ops->valid(state) &&
184 suspend_ops->enter;
185 }
186
pm_states_init(void)187 void __init pm_states_init(void)
188 {
189 /* "mem" and "freeze" are always present in /sys/power/state. */
190 pm_states[PM_SUSPEND_MEM] = pm_labels[PM_SUSPEND_MEM];
191 pm_states[PM_SUSPEND_TO_IDLE] = pm_labels[PM_SUSPEND_TO_IDLE];
192 /*
193 * Suspend-to-idle should be supported even without any suspend_ops,
194 * initialize mem_sleep_states[] accordingly here.
195 */
196 mem_sleep_states[PM_SUSPEND_TO_IDLE] = mem_sleep_labels[PM_SUSPEND_TO_IDLE];
197 }
198
mem_sleep_default_setup(char * str)199 static int __init mem_sleep_default_setup(char *str)
200 {
201 suspend_state_t state;
202
203 for (state = PM_SUSPEND_TO_IDLE; state <= PM_SUSPEND_MEM; state++)
204 if (mem_sleep_labels[state] &&
205 !strcmp(str, mem_sleep_labels[state])) {
206 mem_sleep_default = state;
207 mem_sleep_current = state;
208 break;
209 }
210
211 return 1;
212 }
213 __setup("mem_sleep_default=", mem_sleep_default_setup);
214
215 /**
216 * suspend_set_ops - Set the global suspend method table.
217 * @ops: Suspend operations to use.
218 */
suspend_set_ops(const struct platform_suspend_ops * ops)219 void suspend_set_ops(const struct platform_suspend_ops *ops)
220 {
221 unsigned int sleep_flags;
222
223 sleep_flags = lock_system_sleep();
224
225 suspend_ops = ops;
226
227 if (valid_state(PM_SUSPEND_STANDBY)) {
228 mem_sleep_states[PM_SUSPEND_STANDBY] = mem_sleep_labels[PM_SUSPEND_STANDBY];
229 pm_states[PM_SUSPEND_STANDBY] = pm_labels[PM_SUSPEND_STANDBY];
230 if (mem_sleep_default == PM_SUSPEND_STANDBY)
231 mem_sleep_current = PM_SUSPEND_STANDBY;
232 }
233 if (valid_state(PM_SUSPEND_MEM)) {
234 mem_sleep_states[PM_SUSPEND_MEM] = mem_sleep_labels[PM_SUSPEND_MEM];
235 if (mem_sleep_default >= PM_SUSPEND_MEM)
236 mem_sleep_current = PM_SUSPEND_MEM;
237 }
238
239 unlock_system_sleep(sleep_flags);
240 }
241 EXPORT_SYMBOL_GPL(suspend_set_ops);
242
243 /**
244 * suspend_valid_only_mem - Generic memory-only valid callback.
245 * @state: Target system sleep state.
246 *
247 * Platform drivers that implement mem suspend only and only need to check for
248 * that in their .valid() callback can use this instead of rolling their own
249 * .valid() callback.
250 */
suspend_valid_only_mem(suspend_state_t state)251 int suspend_valid_only_mem(suspend_state_t state)
252 {
253 return state == PM_SUSPEND_MEM;
254 }
255 EXPORT_SYMBOL_GPL(suspend_valid_only_mem);
256
sleep_state_supported(suspend_state_t state)257 static bool sleep_state_supported(suspend_state_t state)
258 {
259 return state == PM_SUSPEND_TO_IDLE ||
260 (valid_state(state) && !cxl_mem_active());
261 }
262
platform_suspend_prepare(suspend_state_t state)263 static int platform_suspend_prepare(suspend_state_t state)
264 {
265 return state != PM_SUSPEND_TO_IDLE && suspend_ops->prepare ?
266 suspend_ops->prepare() : 0;
267 }
268
platform_suspend_prepare_late(suspend_state_t state)269 static int platform_suspend_prepare_late(suspend_state_t state)
270 {
271 return state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->prepare ?
272 s2idle_ops->prepare() : 0;
273 }
274
platform_suspend_prepare_noirq(suspend_state_t state)275 static int platform_suspend_prepare_noirq(suspend_state_t state)
276 {
277 if (state == PM_SUSPEND_TO_IDLE)
278 return s2idle_ops && s2idle_ops->prepare_late ?
279 s2idle_ops->prepare_late() : 0;
280
281 return suspend_ops->prepare_late ? suspend_ops->prepare_late() : 0;
282 }
283
platform_resume_noirq(suspend_state_t state)284 static void platform_resume_noirq(suspend_state_t state)
285 {
286 if (state == PM_SUSPEND_TO_IDLE) {
287 if (s2idle_ops && s2idle_ops->restore_early)
288 s2idle_ops->restore_early();
289 } else if (suspend_ops->wake) {
290 suspend_ops->wake();
291 }
292 }
293
platform_resume_early(suspend_state_t state)294 static void platform_resume_early(suspend_state_t state)
295 {
296 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->restore)
297 s2idle_ops->restore();
298 }
299
platform_resume_finish(suspend_state_t state)300 static void platform_resume_finish(suspend_state_t state)
301 {
302 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->finish)
303 suspend_ops->finish();
304 }
305
platform_suspend_begin(suspend_state_t state)306 static int platform_suspend_begin(suspend_state_t state)
307 {
308 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->begin)
309 return s2idle_ops->begin();
310 else if (suspend_ops && suspend_ops->begin)
311 return suspend_ops->begin(state);
312 else
313 return 0;
314 }
315
platform_resume_end(suspend_state_t state)316 static void platform_resume_end(suspend_state_t state)
317 {
318 if (state == PM_SUSPEND_TO_IDLE && s2idle_ops && s2idle_ops->end)
319 s2idle_ops->end();
320 else if (suspend_ops && suspend_ops->end)
321 suspend_ops->end();
322 }
323
platform_recover(suspend_state_t state)324 static void platform_recover(suspend_state_t state)
325 {
326 if (state != PM_SUSPEND_TO_IDLE && suspend_ops->recover)
327 suspend_ops->recover();
328 }
329
platform_suspend_again(suspend_state_t state)330 static bool platform_suspend_again(suspend_state_t state)
331 {
332 return state != PM_SUSPEND_TO_IDLE && suspend_ops->suspend_again ?
333 suspend_ops->suspend_again() : false;
334 }
335
336 #ifdef CONFIG_PM_DEBUG
337 static unsigned int pm_test_delay = 5;
338 module_param(pm_test_delay, uint, 0644);
339 MODULE_PARM_DESC(pm_test_delay,
340 "Number of seconds to wait before resuming from suspend test");
341 #endif
342
suspend_test(int level)343 static int suspend_test(int level)
344 {
345 #ifdef CONFIG_PM_DEBUG
346 if (pm_test_level == level) {
347 pr_info("suspend debug: Waiting for %d second(s).\n",
348 pm_test_delay);
349 mdelay(pm_test_delay * 1000);
350 return 1;
351 }
352 #endif /* !CONFIG_PM_DEBUG */
353 return 0;
354 }
355
356 /**
357 * suspend_prepare - Prepare for entering system sleep state.
358 * @state: Target system sleep state.
359 *
360 * Common code run for every system sleep state that can be entered (except for
361 * hibernation). Run suspend notifiers, allocate the "suspend" console and
362 * freeze processes.
363 */
suspend_prepare(suspend_state_t state)364 static int suspend_prepare(suspend_state_t state)
365 {
366 int error;
367
368 if (!sleep_state_supported(state))
369 return -EPERM;
370
371 pm_prepare_console();
372
373 error = pm_notifier_call_chain_robust(PM_SUSPEND_PREPARE, PM_POST_SUSPEND);
374 if (error)
375 goto Restore;
376
377 trace_suspend_resume(TPS("freeze_processes"), 0, true);
378 error = suspend_freeze_processes();
379 trace_suspend_resume(TPS("freeze_processes"), 0, false);
380 if (!error)
381 return 0;
382
383 dpm_save_failed_step(SUSPEND_FREEZE);
384 pm_notifier_call_chain(PM_POST_SUSPEND);
385 Restore:
386 pm_restore_console();
387 return error;
388 }
389
390 /* default implementation */
arch_suspend_disable_irqs(void)391 void __weak arch_suspend_disable_irqs(void)
392 {
393 local_irq_disable();
394 }
395
396 /* default implementation */
arch_suspend_enable_irqs(void)397 void __weak arch_suspend_enable_irqs(void)
398 {
399 local_irq_enable();
400 }
401
402 /**
403 * suspend_enter - Make the system enter the given sleep state.
404 * @state: System sleep state to enter.
405 * @wakeup: Returns information that the sleep state should not be re-entered.
406 *
407 * This function should be called after devices have been suspended.
408 */
suspend_enter(suspend_state_t state,bool * wakeup)409 static int suspend_enter(suspend_state_t state, bool *wakeup)
410 {
411 int error;
412
413 error = platform_suspend_prepare(state);
414 if (error)
415 goto Platform_finish;
416
417 error = dpm_suspend_late(PMSG_SUSPEND);
418 if (error) {
419 pr_err("late suspend of devices failed\n");
420 goto Platform_finish;
421 }
422 error = platform_suspend_prepare_late(state);
423 if (error)
424 goto Devices_early_resume;
425
426 error = dpm_suspend_noirq(PMSG_SUSPEND);
427 if (error) {
428 pr_err("noirq suspend of devices failed\n");
429 goto Platform_early_resume;
430 }
431 error = platform_suspend_prepare_noirq(state);
432 if (error)
433 goto Platform_wake;
434
435 if (suspend_test(TEST_PLATFORM))
436 goto Platform_wake;
437
438 if (state == PM_SUSPEND_TO_IDLE) {
439 s2idle_loop();
440 goto Platform_wake;
441 }
442
443 error = pm_sleep_disable_secondary_cpus();
444 if (error || suspend_test(TEST_CPUS))
445 goto Enable_cpus;
446
447 arch_suspend_disable_irqs();
448 BUG_ON(!irqs_disabled());
449
450 system_state = SYSTEM_SUSPEND;
451
452 error = syscore_suspend();
453 if (!error) {
454 *wakeup = pm_wakeup_pending();
455 if (!(suspend_test(TEST_CORE) || *wakeup)) {
456 trace_suspend_resume(TPS("machine_suspend"),
457 state, true);
458 error = suspend_ops->enter(state);
459 trace_suspend_resume(TPS("machine_suspend"),
460 state, false);
461 } else if (*wakeup) {
462 error = -EBUSY;
463 }
464 syscore_resume();
465 }
466
467 system_state = SYSTEM_RUNNING;
468
469 arch_suspend_enable_irqs();
470 BUG_ON(irqs_disabled());
471
472 Enable_cpus:
473 pm_sleep_enable_secondary_cpus();
474
475 Platform_wake:
476 platform_resume_noirq(state);
477 dpm_resume_noirq(PMSG_RESUME);
478
479 Platform_early_resume:
480 platform_resume_early(state);
481
482 Devices_early_resume:
483 dpm_resume_early(PMSG_RESUME);
484
485 Platform_finish:
486 platform_resume_finish(state);
487 return error;
488 }
489
490 /**
491 * suspend_devices_and_enter - Suspend devices and enter system sleep state.
492 * @state: System sleep state to enter.
493 */
suspend_devices_and_enter(suspend_state_t state)494 int suspend_devices_and_enter(suspend_state_t state)
495 {
496 int error;
497 bool wakeup = false;
498
499 if (!sleep_state_supported(state))
500 return -ENOSYS;
501
502 pm_suspend_target_state = state;
503
504 if (state == PM_SUSPEND_TO_IDLE)
505 pm_set_suspend_no_platform();
506
507 error = platform_suspend_begin(state);
508 if (error)
509 goto Close;
510
511 console_suspend_all();
512 suspend_test_start();
513 error = dpm_suspend_start(PMSG_SUSPEND);
514 if (error) {
515 pr_err("Some devices failed to suspend, or early wake event detected\n");
516 goto Recover_platform;
517 }
518 suspend_test_finish("suspend devices");
519 if (suspend_test(TEST_DEVICES))
520 goto Recover_platform;
521
522 do {
523 error = suspend_enter(state, &wakeup);
524 } while (!error && !wakeup && platform_suspend_again(state));
525
526 Resume_devices:
527 suspend_test_start();
528 dpm_resume_end(PMSG_RESUME);
529 suspend_test_finish("resume devices");
530 trace_suspend_resume(TPS("console_resume_all"), state, true);
531 console_resume_all();
532 trace_suspend_resume(TPS("console_resume_all"), state, false);
533
534 Close:
535 platform_resume_end(state);
536 pm_suspend_target_state = PM_SUSPEND_ON;
537 return error;
538
539 Recover_platform:
540 platform_recover(state);
541 goto Resume_devices;
542 }
543
544 /**
545 * suspend_finish - Clean up before finishing the suspend sequence.
546 *
547 * Call platform code to clean up, restart processes, and free the console that
548 * we've allocated. This routine is not called for hibernation.
549 */
suspend_finish(void)550 static void suspend_finish(void)
551 {
552 suspend_thaw_processes();
553 pm_notifier_call_chain(PM_POST_SUSPEND);
554 pm_restore_console();
555 }
556
557 /**
558 * enter_state - Do common work needed to enter system sleep state.
559 * @state: System sleep state to enter.
560 *
561 * Make sure that no one else is trying to put the system into a sleep state.
562 * Fail if that's not the case. Otherwise, prepare for system suspend, make the
563 * system enter the given sleep state and clean up after wakeup.
564 */
enter_state(suspend_state_t state)565 static int enter_state(suspend_state_t state)
566 {
567 int error;
568
569 trace_suspend_resume(TPS("suspend_enter"), state, true);
570 if (state == PM_SUSPEND_TO_IDLE) {
571 #ifdef CONFIG_PM_DEBUG
572 if (pm_test_level != TEST_NONE && pm_test_level <= TEST_CPUS) {
573 pr_warn("Unsupported test mode for suspend to idle, please choose none/freezer/devices/platform.\n");
574 return -EAGAIN;
575 }
576 #endif
577 } else if (!valid_state(state)) {
578 return -EINVAL;
579 }
580 if (!mutex_trylock(&system_transition_mutex))
581 return -EBUSY;
582
583 if (state == PM_SUSPEND_TO_IDLE)
584 s2idle_begin();
585
586 if (sync_on_suspend_enabled) {
587 trace_suspend_resume(TPS("sync_filesystems"), 0, true);
588 ksys_sync_helper();
589 trace_suspend_resume(TPS("sync_filesystems"), 0, false);
590 }
591
592 pm_pr_dbg("Preparing system for sleep (%s)\n", mem_sleep_labels[state]);
593 pm_suspend_clear_flags();
594 error = suspend_prepare(state);
595 if (error)
596 goto Unlock;
597
598 if (suspend_test(TEST_FREEZER))
599 goto Finish;
600
601 trace_suspend_resume(TPS("suspend_enter"), state, false);
602 pm_pr_dbg("Suspending system (%s)\n", mem_sleep_labels[state]);
603 pm_restrict_gfp_mask();
604 error = suspend_devices_and_enter(state);
605 pm_restore_gfp_mask();
606
607 Finish:
608 events_check_enabled = false;
609 pm_pr_dbg("Finishing wakeup.\n");
610 suspend_finish();
611 Unlock:
612 mutex_unlock(&system_transition_mutex);
613 return error;
614 }
615
616 /**
617 * pm_suspend - Externally visible function for suspending the system.
618 * @state: System sleep state to enter.
619 *
620 * Check if the value of @state represents one of the supported states,
621 * execute enter_state() and update system suspend statistics.
622 */
pm_suspend(suspend_state_t state)623 int pm_suspend(suspend_state_t state)
624 {
625 int error;
626
627 if (state <= PM_SUSPEND_ON || state >= PM_SUSPEND_MAX)
628 return -EINVAL;
629
630 pr_info("suspend entry (%s)\n", mem_sleep_labels[state]);
631 error = enter_state(state);
632 dpm_save_errno(error);
633 pr_info("suspend exit\n");
634 return error;
635 }
636 EXPORT_SYMBOL(pm_suspend);
637