Lines Matching +full:step +full:- +full:down
36 #include <linux/percpu-rwsem.h>
48 * struct cpuhp_cpu_state - Per cpu hotplug state storage
57 * @node: Remote CPU node; for multi-instance, do a
59 * @last: For multi-instance rollback, remember how far we got
63 * @done_up: Signal completion to the issuer of the task for cpu-up
64 * @done_down: Signal completion to the issuer of the task for cpu-down
96 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
98 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
118 * struct cpuhp_step - Hotplug state machine step
119 * @name: Name of the step
120 * @startup: Startup function of the step
121 * @teardown: Teardown function of the step
122 * @cant_stop: Bringup/teardown can't be stopped at this step
152 static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step) in cpuhp_step_empty() argument
154 return bringup ? !step->startup.single : !step->teardown.single; in cpuhp_step_empty()
158 * cpuhp_invoke_callback - Invoke the callbacks for a given state
162 * @node: For multi-instance, do a single entry callback for install/remove
163 * @lastp: For multi-instance rollback, remember how far we got
174 struct cpuhp_step *step = cpuhp_get_step(state); in cpuhp_invoke_callback() local
179 if (st->fail == state) { in cpuhp_invoke_callback()
180 st->fail = CPUHP_INVALID; in cpuhp_invoke_callback()
181 return -EAGAIN; in cpuhp_invoke_callback()
184 if (cpuhp_step_empty(bringup, step)) { in cpuhp_invoke_callback()
189 if (!step->multi_instance) { in cpuhp_invoke_callback()
191 cb = bringup ? step->startup.single : step->teardown.single; in cpuhp_invoke_callback()
193 trace_cpuhp_enter(cpu, st->target, state, cb); in cpuhp_invoke_callback()
195 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
198 cbm = bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
203 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
205 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
211 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
215 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
217 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
232 cbm = !bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
236 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
237 if (!cnt--) in cpuhp_invoke_callback()
240 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
242 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
263 struct completion *done = bringup ? &st->done_up : &st->done_down; in wait_for_ap_thread()
269 struct completion *done = bringup ? &st->done_up : &st->done_down; in complete_ap_thread()
293 * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
329 } else if (now - start < NSEC_PER_MSEC) { in cpuhp_wait_for_sync_state()
345 * cpuhp_ap_report_dead - Update synchronization state to DEAD
386 * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
445 ret = -EIO; in cpuhp_bp_sync_alive()
477 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
574 cpu_hotplug_disabled--; in __cpu_hotplug_enable()
598 * Architectures that need SMT-specific errata handling during SMT hotplug
711 enum cpuhp_state prev_state = st->state; in cpuhp_set_state()
712 bool bringup = st->state < target; in cpuhp_set_state()
714 st->rollback = false; in cpuhp_set_state()
715 st->last = NULL; in cpuhp_set_state()
717 st->target = target; in cpuhp_set_state()
718 st->single = false; in cpuhp_set_state()
719 st->bringup = bringup; in cpuhp_set_state()
730 bool bringup = !st->bringup; in cpuhp_reset_state()
732 st->target = prev_state; in cpuhp_reset_state()
738 if (st->rollback) in cpuhp_reset_state()
741 st->rollback = true; in cpuhp_reset_state()
744 * If we have st->last we need to undo partial multi_instance of this in cpuhp_reset_state()
747 if (!st->last) { in cpuhp_reset_state()
748 if (st->bringup) in cpuhp_reset_state()
749 st->state--; in cpuhp_reset_state()
751 st->state++; in cpuhp_reset_state()
754 st->bringup = bringup; in cpuhp_reset_state()
762 if (!st->single && st->state == st->target) in __cpuhp_kick_ap()
765 st->result = 0; in __cpuhp_kick_ap()
771 st->should_run = true; in __cpuhp_kick_ap()
772 wake_up_process(st->thread); in __cpuhp_kick_ap()
773 wait_for_ap_thread(st, st->bringup); in __cpuhp_kick_ap()
784 if ((ret = st->result)) { in cpuhp_kick_ap()
799 return -ECANCELED; in bringup_wait_for_ap_online()
802 kthread_unpark(st->thread); in bringup_wait_for_ap_online()
812 return -ECANCELED; in bringup_wait_for_ap_online()
820 return -EAGAIN; in cpuhp_kick_ap_alive()
847 if (st->target <= CPUHP_AP_ONLINE_IDLE) in cpuhp_bringup_ap()
850 return cpuhp_kick_ap(cpu, st, st->target); in cpuhp_bringup_ap()
864 return -EAGAIN; in bringup_cpu()
891 if (st->target <= CPUHP_AP_ONLINE_IDLE) in bringup_cpu()
894 return cpuhp_kick_ap(cpu, st, st->target); in bringup_cpu()
905 struct mm_struct *mm = idle->active_mm; in finish_cpu()
912 idle->active_mm = NULL; in finish_cpu()
926 * st->state will be modified ahead of time, to match state_to_run, as if it
936 if (st->state >= target) in cpuhp_next_state()
939 *state_to_run = ++st->state; in cpuhp_next_state()
941 if (st->state <= target) in cpuhp_next_state()
944 *state_to_run = st->state--; in cpuhp_next_state()
972 cpu, bringup ? "UP" : "DOWN", in __cpuhp_invoke_callback_range()
973 cpuhp_get_step(st->state)->name, in __cpuhp_invoke_callback_range()
974 st->state, err); in __cpuhp_invoke_callback_range()
975 ret = -1; in __cpuhp_invoke_callback_range()
1006 * When CPU hotplug is disabled, then taking the CPU down is not in can_rollback_cpu()
1012 return st->state <= CPUHP_BRINGUP_CPU; in can_rollback_cpu()
1018 enum cpuhp_state prev_state = st->state; in cpuhp_up_callbacks()
1024 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_up_callbacks()
1025 st->state); in cpuhp_up_callbacks()
1042 return st->should_run; in cpuhp_should_run()
1053 * - single: runs st->cb_state
1054 * - up: runs ++st->state, while st->state < st->target
1055 * - down: runs st->state--, while st->state > st->target
1062 bool bringup = st->bringup; in cpuhp_thread_fun()
1065 if (WARN_ON_ONCE(!st->should_run)) in cpuhp_thread_fun()
1069 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures in cpuhp_thread_fun()
1070 * that if we see ->should_run we also see the rest of the state. in cpuhp_thread_fun()
1082 if (st->single) { in cpuhp_thread_fun()
1083 state = st->cb_state; in cpuhp_thread_fun()
1084 st->should_run = false; in cpuhp_thread_fun()
1086 st->should_run = cpuhp_next_state(bringup, &state, st, st->target); in cpuhp_thread_fun()
1087 if (!st->should_run) in cpuhp_thread_fun()
1095 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
1101 WARN_ON_ONCE(st->result); in cpuhp_thread_fun()
1103 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
1106 if (st->result) { in cpuhp_thread_fun()
1112 WARN_ON_ONCE(st->rollback); in cpuhp_thread_fun()
1113 st->should_run = false; in cpuhp_thread_fun()
1120 if (!st->should_run) in cpuhp_thread_fun()
1145 if (!st->thread) in cpuhp_invoke_ap_callback()
1148 st->rollback = false; in cpuhp_invoke_ap_callback()
1149 st->last = NULL; in cpuhp_invoke_ap_callback()
1151 st->node = node; in cpuhp_invoke_ap_callback()
1152 st->bringup = bringup; in cpuhp_invoke_ap_callback()
1153 st->cb_state = state; in cpuhp_invoke_ap_callback()
1154 st->single = true; in cpuhp_invoke_ap_callback()
1161 if ((ret = st->result) && st->last) { in cpuhp_invoke_ap_callback()
1162 st->rollback = true; in cpuhp_invoke_ap_callback()
1163 st->bringup = !bringup; in cpuhp_invoke_ap_callback()
1172 st->node = st->last = NULL; in cpuhp_invoke_ap_callback()
1179 enum cpuhp_state prev_state = st->state; in cpuhp_kick_ap_work()
1188 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); in cpuhp_kick_ap_work()
1189 ret = cpuhp_kick_ap(cpu, st, st->target); in cpuhp_kick_ap_work()
1190 trace_cpuhp_exit(cpu, st->state, prev_state, ret); in cpuhp_kick_ap_work()
1210 init_completion(&st->done_up); in cpuhp_init_state()
1211 init_completion(&st->done_down); in cpuhp_init_state()
1228 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
1233 * trivial, there are various non-obvious corner cases, which this function
1244 * This function is called after the cpu is taken down and marked in clear_tasks_mm_cpumask()
1246 * their mm mask. -- Peter Zijlstra in clear_tasks_mm_cpumask()
1248 * full-fledged tasklist_lock. in clear_tasks_mm_cpumask()
1262 arch_clear_mm_cpumask_cpu(cpu, t->mm); in clear_tasks_mm_cpumask()
1268 /* Take this CPU down. */
1272 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); in take_cpu_down()
1282 * down, that the current state is CPUHP_TEARDOWN_CPU - 1. in take_cpu_down()
1284 WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1)); in take_cpu_down()
1302 kthread_park(st->thread); in takedown_cpu()
1318 kthread_unpark(st->thread); in takedown_cpu()
1331 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); in takedown_cpu()
1345 * Callbacks must be re-integrated right away to the RCU state machine. in takedown_cpu()
1365 BUG_ON(st->state != CPUHP_AP_OFFLINE); in cpuhp_report_idle_dead()
1368 st->state = CPUHP_AP_IDLE_DEAD; in cpuhp_report_idle_dead()
1380 enum cpuhp_state prev_state = st->state; in cpuhp_down_callbacks()
1385 pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n", in cpuhp_down_callbacks()
1386 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_down_callbacks()
1387 st->state); in cpuhp_down_callbacks()
1391 if (st->state < prev_state) in cpuhp_down_callbacks()
1407 return -EBUSY; in _cpu_down()
1410 return -EINVAL; in _cpu_down()
1421 if (st->state > CPUHP_TEARDOWN_CPU) { in _cpu_down()
1422 st->target = max((int)target, CPUHP_TEARDOWN_CPU); in _cpu_down()
1435 if (st->state > CPUHP_TEARDOWN_CPU) in _cpu_down()
1438 st->target = target; in _cpu_down()
1441 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need in _cpu_down()
1445 if (ret && st->state < prev_state) { in _cpu_down()
1446 if (st->state == CPUHP_TEARDOWN_CPU) { in _cpu_down()
1474 return _cpu_down(work->cpu, 0, work->target); in __cpu_down_maps_locked()
1486 return -EOPNOTSUPP; in cpu_down_maps_locked()
1488 return -EBUSY; in cpu_down_maps_locked()
1492 * CPU to prevent a deadlock against cfs_b->period_timer. in cpu_down_maps_locked()
1500 return -EBUSY; in cpu_down_maps_locked()
1514 * cpu_device_down - Bring down a cpu device
1525 return cpu_down(dev->id, CPUHP_OFFLINE); in cpu_device_down()
1561 pr_err("Failed to offline CPU%d - error=%d", in smp_shutdown_nonboot_cpus()
1587 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1596 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); in notify_cpu_starting()
1628 st->state = CPUHP_AP_ONLINE_IDLE; in cpuhp_online_idle()
1642 ret = -EINVAL; in _cpu_up()
1650 if (st->state >= target) in _cpu_up()
1653 if (st->state == CPUHP_OFFLINE) { in _cpu_up()
1675 if (st->state > CPUHP_BRINGUP_CPU) { in _cpu_up()
1703 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", in cpu_up()
1705 return -EINVAL; in cpu_up()
1715 err = -EBUSY; in cpu_up()
1719 err = -EPERM; in cpu_up()
1730 * cpu_device_up - Bring up a cpu device
1741 return cpu_up(dev->id, CPUHP_ONLINE); in cpu_device_up()
1757 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1774 pr_err("Failed to bring hibernate-CPU up!\n"); in bringup_hibernate_cpu()
1798 if (!--ncpus) in cpuhp_bringup_mask()
1870 ncpus -= num_online_cpus(); in cpuhp_bringup_cpus_parallel()
1878 /* Bring the not-yet started CPUs up */ in cpuhp_bringup_cpus_parallel()
1908 if (primary == -1) { in freeze_secondary_cpus()
1918 * We take down all of the non-boot CPUs in one shot to avoid races in freeze_secondary_cpus()
1923 pr_info("Disabling non-boot CPUs ...\n"); in freeze_secondary_cpus()
1924 for (cpu = nr_cpu_ids - 1; cpu >= 0; cpu--) { in freeze_secondary_cpus()
1930 error = -EBUSY; in freeze_secondary_cpus()
1940 pr_err("Error taking CPU%d down: %d\n", cpu, error); in freeze_secondary_cpus()
1948 pr_err("Non-boot CPUs are not disabled\n"); in freeze_secondary_cpus()
1979 pr_info("Enabling non-boot CPUs ...\n"); in thaw_secondary_cpus()
2004 return -ENOMEM; in alloc_frozen_cpus()
2112 * On the tear-down path, timers_dead_cpu() must be invoked
2125 * the next step will release it.
2144 * All-in-one CPU bringup state which includes the kick alive.
2275 return -EINVAL; in cpuhp_cb_check()
2287 struct cpuhp_step *step; in cpuhp_reserve_state() local
2291 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN; in cpuhp_reserve_state()
2295 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN; in cpuhp_reserve_state()
2299 return -EINVAL; in cpuhp_reserve_state()
2302 for (i = state; i <= end; i++, step++) { in cpuhp_reserve_state()
2303 if (!step->name) in cpuhp_reserve_state()
2307 return -ENOSPC; in cpuhp_reserve_state()
2336 if (name && sp->name) in cpuhp_store_callbacks()
2337 return -EBUSY; in cpuhp_store_callbacks()
2339 sp->startup.single = startup; in cpuhp_store_callbacks()
2340 sp->teardown.single = teardown; in cpuhp_store_callbacks()
2341 sp->name = name; in cpuhp_store_callbacks()
2342 sp->multi_instance = multi_instance; in cpuhp_store_callbacks()
2343 INIT_HLIST_HEAD(&sp->list); in cpuhp_store_callbacks()
2349 return cpuhp_get_step(state)->teardown.single; in cpuhp_get_teardown_cb()
2353 * Call the startup/teardown function for a step either on the AP or
2397 int cpustate = st->state; in cpuhp_rollback_install()
2419 if (sp->multi_instance == false) in __cpuhp_state_add_instance_cpuslocked()
2420 return -EINVAL; in __cpuhp_state_add_instance_cpuslocked()
2424 if (!invoke || !sp->startup.multi) in __cpuhp_state_add_instance_cpuslocked()
2433 int cpustate = st->state; in __cpuhp_state_add_instance_cpuslocked()
2440 if (sp->teardown.multi) in __cpuhp_state_add_instance_cpuslocked()
2447 hlist_add_head(node, &sp->list); in __cpuhp_state_add_instance_cpuslocked()
2466 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2468 * @name: Name of the step
2495 return -EINVAL; in __cpuhp_setup_state_cpuslocked()
2517 int cpustate = st->state; in __cpuhp_setup_state_cpuslocked()
2566 if (!sp->multi_instance) in __cpuhp_state_remove_instance()
2567 return -EINVAL; in __cpuhp_state_remove_instance()
2581 int cpustate = st->state; in __cpuhp_state_remove_instance()
2597 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2616 if (sp->multi_instance) { in __cpuhp_remove_state_cpuslocked()
2617 WARN(!hlist_empty(&sp->list), in __cpuhp_remove_state_cpuslocked()
2633 int cpustate = st->state; in __cpuhp_remove_state_cpuslocked()
2657 dev->offline = true; in cpuhp_offline_cpu_device()
2659 kobject_uevent(&dev->kobj, KOBJ_OFFLINE); in cpuhp_offline_cpu_device()
2666 dev->offline = false; in cpuhp_online_cpu_device()
2668 kobject_uevent(&dev->kobj, KOBJ_ONLINE); in cpuhp_online_cpu_device()
2744 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in state_show()
2746 return sprintf(buf, "%d\n", st->state); in state_show()
2753 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in target_store()
2763 return -EINVAL; in target_store()
2766 return -EINVAL; in target_store()
2775 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; in target_store()
2780 if (st->state < target) in target_store()
2781 ret = cpu_up(dev->id, target); in target_store()
2782 else if (st->state > target) in target_store()
2783 ret = cpu_down(dev->id, target); in target_store()
2784 else if (WARN_ON(st->target != target)) in target_store()
2785 st->target = target; in target_store()
2794 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in target_show()
2796 return sprintf(buf, "%d\n", st->target); in target_show()
2803 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in fail_store()
2812 st->fail = fail; in fail_store()
2817 return -EINVAL; in fail_store()
2823 return -EINVAL; in fail_store()
2831 if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU) in fail_store()
2832 return -EINVAL; in fail_store()
2839 if (!sp->startup.single && !sp->teardown.single) in fail_store()
2840 ret = -EINVAL; in fail_store()
2845 st->fail = fail; in fail_store()
2853 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in fail_show()
2855 return sprintf(buf, "%d\n", st->fail); in fail_show()
2882 if (sp->name) { in states_show()
2883 cur = sprintf(buf, "%3d: %s\n", i, sp->name); in states_show()
2920 return -EPERM; in __store_smt_control()
2923 return -ENODEV; in __store_smt_control()
2940 return -EINVAL; in __store_smt_control()
2942 return -EINVAL; in __store_smt_control()
2968 return -ENODEV; in __store_smt_control()
3027 int ret = -ENODEV; in cpu_smt_sysfs_init()
3031 ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group); in cpu_smt_sysfs_init()
3048 ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group); in cpuhp_sysfs_init()
3059 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); in cpuhp_sysfs_init()
3076 /* cpu_bit_bitmap[0] is empty - so we can back into it */
3137 * IPI/NMI broadcasts when shutting down CPUs. Invocation from in set_cpu_online()