Lines Matching +full:step +full:- +full:down
36 #include <linux/percpu-rwsem.h>
49 * struct cpuhp_cpu_state - Per cpu hotplug state storage
58 * @node: Remote CPU node; for multi-instance, do a
60 * @last: For multi-instance rollback, remember how far we got
64 * @done_up: Signal completion to the issuer of the task for cpu-up
65 * @done_down: Signal completion to the issuer of the task for cpu-down
97 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
99 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
119 * struct cpuhp_step - Hotplug state machine step
120 * @name: Name of the step
121 * @startup: Startup function of the step
122 * @teardown: Teardown function of the step
123 * @cant_stop: Bringup/teardown can't be stopped at this step
153 static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step) in cpuhp_step_empty() argument
155 return bringup ? !step->startup.single : !step->teardown.single; in cpuhp_step_empty()
159 * cpuhp_invoke_callback - Invoke the callbacks for a given state
163 * @node: For multi-instance, do a single entry callback for install/remove
164 * @lastp: For multi-instance rollback, remember how far we got
175 struct cpuhp_step *step = cpuhp_get_step(state); in cpuhp_invoke_callback() local
180 if (st->fail == state) { in cpuhp_invoke_callback()
181 st->fail = CPUHP_INVALID; in cpuhp_invoke_callback()
182 return -EAGAIN; in cpuhp_invoke_callback()
185 if (cpuhp_step_empty(bringup, step)) { in cpuhp_invoke_callback()
190 if (!step->multi_instance) { in cpuhp_invoke_callback()
192 cb = bringup ? step->startup.single : step->teardown.single; in cpuhp_invoke_callback()
194 trace_cpuhp_enter(cpu, st->target, state, cb); in cpuhp_invoke_callback()
196 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
199 cbm = bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
204 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
206 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
212 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
216 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
218 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
233 cbm = !bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
237 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
238 if (!cnt--) in cpuhp_invoke_callback()
241 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
243 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
264 struct completion *done = bringup ? &st->done_up : &st->done_down; in wait_for_ap_thread()
270 struct completion *done = bringup ? &st->done_up : &st->done_down; in complete_ap_thread()
294 * cpuhp_ap_update_sync_state - Update synchronization state during bringup/teardown
330 } else if (now - start < NSEC_PER_MSEC) { in cpuhp_wait_for_sync_state()
346 * cpuhp_ap_report_dead - Update synchronization state to DEAD
387 * cpuhp_ap_sync_alive - Synchronize AP with the control CPU once it is alive
446 ret = -EIO; in cpuhp_bp_sync_alive()
478 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
576 cpu_hotplug_disabled--; in __cpu_hotplug_enable()
600 * Architectures that need SMT-specific errata handling during SMT hotplug
713 enum cpuhp_state prev_state = st->state; in cpuhp_set_state()
714 bool bringup = st->state < target; in cpuhp_set_state()
716 st->rollback = false; in cpuhp_set_state()
717 st->last = NULL; in cpuhp_set_state()
719 st->target = target; in cpuhp_set_state()
720 st->single = false; in cpuhp_set_state()
721 st->bringup = bringup; in cpuhp_set_state()
732 bool bringup = !st->bringup; in cpuhp_reset_state()
734 st->target = prev_state; in cpuhp_reset_state()
740 if (st->rollback) in cpuhp_reset_state()
743 st->rollback = true; in cpuhp_reset_state()
746 * If we have st->last we need to undo partial multi_instance of this in cpuhp_reset_state()
749 if (!st->last) { in cpuhp_reset_state()
750 if (st->bringup) in cpuhp_reset_state()
751 st->state--; in cpuhp_reset_state()
753 st->state++; in cpuhp_reset_state()
756 st->bringup = bringup; in cpuhp_reset_state()
764 if (!st->single && st->state == st->target) in __cpuhp_kick_ap()
767 st->result = 0; in __cpuhp_kick_ap()
773 st->should_run = true; in __cpuhp_kick_ap()
774 wake_up_process(st->thread); in __cpuhp_kick_ap()
775 wait_for_ap_thread(st, st->bringup); in __cpuhp_kick_ap()
786 if ((ret = st->result)) { in cpuhp_kick_ap()
801 return -ECANCELED; in bringup_wait_for_ap_online()
804 kthread_unpark(st->thread); in bringup_wait_for_ap_online()
814 return -ECANCELED; in bringup_wait_for_ap_online()
822 return -EAGAIN; in cpuhp_kick_ap_alive()
849 if (st->target <= CPUHP_AP_ONLINE_IDLE) in cpuhp_bringup_ap()
852 return cpuhp_kick_ap(cpu, st, st->target); in cpuhp_bringup_ap()
866 return -EAGAIN; in bringup_cpu()
893 if (st->target <= CPUHP_AP_ONLINE_IDLE) in bringup_cpu()
896 return cpuhp_kick_ap(cpu, st, st->target); in bringup_cpu()
907 struct mm_struct *mm = idle->active_mm; in finish_cpu()
914 idle->active_mm = NULL; in finish_cpu()
928 * st->state will be modified ahead of time, to match state_to_run, as if it
938 if (st->state >= target) in cpuhp_next_state()
941 *state_to_run = ++st->state; in cpuhp_next_state()
943 if (st->state <= target) in cpuhp_next_state()
946 *state_to_run = st->state--; in cpuhp_next_state()
974 cpu, bringup ? "UP" : "DOWN", in __cpuhp_invoke_callback_range()
975 cpuhp_get_step(st->state)->name, in __cpuhp_invoke_callback_range()
976 st->state, err); in __cpuhp_invoke_callback_range()
977 ret = -1; in __cpuhp_invoke_callback_range()
1008 * When CPU hotplug is disabled, then taking the CPU down is not in can_rollback_cpu()
1014 return st->state <= CPUHP_BRINGUP_CPU; in can_rollback_cpu()
1020 enum cpuhp_state prev_state = st->state; in cpuhp_up_callbacks()
1026 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_up_callbacks()
1027 st->state); in cpuhp_up_callbacks()
1044 return st->should_run; in cpuhp_should_run()
1055 * - single: runs st->cb_state
1056 * - up: runs ++st->state, while st->state < st->target
1057 * - down: runs st->state--, while st->state > st->target
1064 bool bringup = st->bringup; in cpuhp_thread_fun()
1067 if (WARN_ON_ONCE(!st->should_run)) in cpuhp_thread_fun()
1071 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures in cpuhp_thread_fun()
1072 * that if we see ->should_run we also see the rest of the state. in cpuhp_thread_fun()
1084 if (st->single) { in cpuhp_thread_fun()
1085 state = st->cb_state; in cpuhp_thread_fun()
1086 st->should_run = false; in cpuhp_thread_fun()
1088 st->should_run = cpuhp_next_state(bringup, &state, st, st->target); in cpuhp_thread_fun()
1089 if (!st->should_run) in cpuhp_thread_fun()
1097 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
1103 WARN_ON_ONCE(st->result); in cpuhp_thread_fun()
1105 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
1108 if (st->result) { in cpuhp_thread_fun()
1114 WARN_ON_ONCE(st->rollback); in cpuhp_thread_fun()
1115 st->should_run = false; in cpuhp_thread_fun()
1122 if (!st->should_run) in cpuhp_thread_fun()
1147 if (!st->thread) in cpuhp_invoke_ap_callback()
1150 st->rollback = false; in cpuhp_invoke_ap_callback()
1151 st->last = NULL; in cpuhp_invoke_ap_callback()
1153 st->node = node; in cpuhp_invoke_ap_callback()
1154 st->bringup = bringup; in cpuhp_invoke_ap_callback()
1155 st->cb_state = state; in cpuhp_invoke_ap_callback()
1156 st->single = true; in cpuhp_invoke_ap_callback()
1163 if ((ret = st->result) && st->last) { in cpuhp_invoke_ap_callback()
1164 st->rollback = true; in cpuhp_invoke_ap_callback()
1165 st->bringup = !bringup; in cpuhp_invoke_ap_callback()
1174 st->node = st->last = NULL; in cpuhp_invoke_ap_callback()
1181 enum cpuhp_state prev_state = st->state; in cpuhp_kick_ap_work()
1190 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); in cpuhp_kick_ap_work()
1191 ret = cpuhp_kick_ap(cpu, st, st->target); in cpuhp_kick_ap_work()
1192 trace_cpuhp_exit(cpu, st->state, prev_state, ret); in cpuhp_kick_ap_work()
1212 init_completion(&st->done_up); in cpuhp_init_state()
1213 init_completion(&st->done_down); in cpuhp_init_state()
1230 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
1235 * trivial, there are various non-obvious corner cases, which this function
1246 * This function is called after the cpu is taken down and marked in clear_tasks_mm_cpumask()
1248 * their mm mask. -- Peter Zijlstra in clear_tasks_mm_cpumask()
1250 * full-fledged tasklist_lock. in clear_tasks_mm_cpumask()
1264 arch_clear_mm_cpumask_cpu(cpu, t->mm); in clear_tasks_mm_cpumask()
1270 /* Take this CPU down. */
1274 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); in take_cpu_down()
1284 * down, that the current state is CPUHP_TEARDOWN_CPU - 1. in take_cpu_down()
1286 WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1)); in take_cpu_down()
1304 kthread_park(st->thread); in takedown_cpu()
1317 kthread_unpark(st->thread); in takedown_cpu()
1330 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); in takedown_cpu()
1344 * Callbacks must be re-integrated right away to the RCU state machine. in takedown_cpu()
1364 BUG_ON(st->state != CPUHP_AP_OFFLINE); in cpuhp_report_idle_dead()
1367 st->state = CPUHP_AP_IDLE_DEAD; in cpuhp_report_idle_dead()
1379 enum cpuhp_state prev_state = st->state; in cpuhp_down_callbacks()
1384 pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n", in cpuhp_down_callbacks()
1385 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_down_callbacks()
1386 st->state); in cpuhp_down_callbacks()
1390 if (st->state < prev_state) in cpuhp_down_callbacks()
1406 return -EBUSY; in _cpu_down()
1409 return -EINVAL; in _cpu_down()
1420 if (st->state > CPUHP_TEARDOWN_CPU) { in _cpu_down()
1421 st->target = max((int)target, CPUHP_TEARDOWN_CPU); in _cpu_down()
1434 if (st->state > CPUHP_TEARDOWN_CPU) in _cpu_down()
1437 st->target = target; in _cpu_down()
1440 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need in _cpu_down()
1444 if (ret && st->state < prev_state) { in _cpu_down()
1445 if (st->state == CPUHP_TEARDOWN_CPU) { in _cpu_down()
1468 return _cpu_down(work->cpu, 0, work->target); in __cpu_down_maps_locked()
1480 return -EOPNOTSUPP; in cpu_down_maps_locked()
1482 return -EBUSY; in cpu_down_maps_locked()
1486 * CPU to prevent a deadlock against cfs_b->period_timer. in cpu_down_maps_locked()
1494 return -EBUSY; in cpu_down_maps_locked()
1508 * cpu_device_down - Bring down a cpu device
1519 return cpu_down(dev->id, CPUHP_OFFLINE); in cpu_device_down()
1555 pr_err("Failed to offline CPU%d - error=%d", in smp_shutdown_nonboot_cpus()
1581 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1590 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); in notify_cpu_starting()
1622 st->state = CPUHP_AP_ONLINE_IDLE; in cpuhp_online_idle()
1636 ret = -EINVAL; in _cpu_up()
1644 if (st->state >= target) in _cpu_up()
1647 if (st->state == CPUHP_OFFLINE) { in _cpu_up()
1669 if (st->state > CPUHP_BRINGUP_CPU) { in _cpu_up()
1697 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", in cpu_up()
1699 return -EINVAL; in cpu_up()
1709 err = -EBUSY; in cpu_up()
1713 err = -EPERM; in cpu_up()
1724 * cpu_device_up - Bring up a cpu device
1735 return cpu_up(dev->id, CPUHP_ONLINE); in cpu_device_up()
1751 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1768 pr_err("Failed to bring hibernate-CPU up!\n"); in bringup_hibernate_cpu()
1792 if (!--ncpus) in cpuhp_bringup_mask()
1864 ncpus -= num_online_cpus(); in cpuhp_bringup_cpus_parallel()
1872 /* Bring the not-yet started CPUs up */ in cpuhp_bringup_cpus_parallel()
1902 if (primary == -1) { in freeze_secondary_cpus()
1912 * We take down all of the non-boot CPUs in one shot to avoid races in freeze_secondary_cpus()
1917 pr_info("Disabling non-boot CPUs ...\n"); in freeze_secondary_cpus()
1918 for (cpu = nr_cpu_ids - 1; cpu >= 0; cpu--) { in freeze_secondary_cpus()
1924 error = -EBUSY; in freeze_secondary_cpus()
1934 pr_err("Error taking CPU%d down: %d\n", cpu, error); in freeze_secondary_cpus()
1942 pr_err("Non-boot CPUs are not disabled\n"); in freeze_secondary_cpus()
1973 pr_info("Enabling non-boot CPUs ...\n"); in thaw_secondary_cpus()
1998 return -ENOMEM; in alloc_frozen_cpus()
2101 * On the tear-down path, timers_dead_cpu() must be invoked
2114 * the next step will release it.
2133 * All-in-one CPU bringup state which includes the kick alive.
2264 return -EINVAL; in cpuhp_cb_check()
2276 struct cpuhp_step *step; in cpuhp_reserve_state() local
2280 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN; in cpuhp_reserve_state()
2284 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN; in cpuhp_reserve_state()
2288 return -EINVAL; in cpuhp_reserve_state()
2291 for (i = state; i <= end; i++, step++) { in cpuhp_reserve_state()
2292 if (!step->name) in cpuhp_reserve_state()
2296 return -ENOSPC; in cpuhp_reserve_state()
2325 if (name && sp->name) in cpuhp_store_callbacks()
2326 return -EBUSY; in cpuhp_store_callbacks()
2328 sp->startup.single = startup; in cpuhp_store_callbacks()
2329 sp->teardown.single = teardown; in cpuhp_store_callbacks()
2330 sp->name = name; in cpuhp_store_callbacks()
2331 sp->multi_instance = multi_instance; in cpuhp_store_callbacks()
2332 INIT_HLIST_HEAD(&sp->list); in cpuhp_store_callbacks()
2338 return cpuhp_get_step(state)->teardown.single; in cpuhp_get_teardown_cb()
2342 * Call the startup/teardown function for a step either on the AP or
2386 int cpustate = st->state; in cpuhp_rollback_install()
2408 if (sp->multi_instance == false) in __cpuhp_state_add_instance_cpuslocked()
2409 return -EINVAL; in __cpuhp_state_add_instance_cpuslocked()
2413 if (!invoke || !sp->startup.multi) in __cpuhp_state_add_instance_cpuslocked()
2422 int cpustate = st->state; in __cpuhp_state_add_instance_cpuslocked()
2429 if (sp->teardown.multi) in __cpuhp_state_add_instance_cpuslocked()
2436 hlist_add_head(node, &sp->list); in __cpuhp_state_add_instance_cpuslocked()
2455 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2457 * @name: Name of the step
2484 return -EINVAL; in __cpuhp_setup_state_cpuslocked()
2506 int cpustate = st->state; in __cpuhp_setup_state_cpuslocked()
2555 if (!sp->multi_instance) in __cpuhp_state_remove_instance()
2556 return -EINVAL; in __cpuhp_state_remove_instance()
2570 int cpustate = st->state; in __cpuhp_state_remove_instance()
2586 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2605 if (sp->multi_instance) { in __cpuhp_remove_state_cpuslocked()
2606 WARN(!hlist_empty(&sp->list), in __cpuhp_remove_state_cpuslocked()
2622 int cpustate = st->state; in __cpuhp_remove_state_cpuslocked()
2646 dev->offline = true; in cpuhp_offline_cpu_device()
2648 kobject_uevent(&dev->kobj, KOBJ_OFFLINE); in cpuhp_offline_cpu_device()
2655 dev->offline = false; in cpuhp_online_cpu_device()
2657 kobject_uevent(&dev->kobj, KOBJ_ONLINE); in cpuhp_online_cpu_device()
2733 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in state_show()
2735 return sprintf(buf, "%d\n", st->state); in state_show()
2742 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in target_store()
2752 return -EINVAL; in target_store()
2755 return -EINVAL; in target_store()
2764 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; in target_store()
2769 if (st->state < target) in target_store()
2770 ret = cpu_up(dev->id, target); in target_store()
2771 else if (st->state > target) in target_store()
2772 ret = cpu_down(dev->id, target); in target_store()
2773 else if (WARN_ON(st->target != target)) in target_store()
2774 st->target = target; in target_store()
2783 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in target_show()
2785 return sprintf(buf, "%d\n", st->target); in target_show()
2792 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in fail_store()
2801 st->fail = fail; in fail_store()
2806 return -EINVAL; in fail_store()
2812 return -EINVAL; in fail_store()
2820 if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU) in fail_store()
2821 return -EINVAL; in fail_store()
2828 if (!sp->startup.single && !sp->teardown.single) in fail_store()
2829 ret = -EINVAL; in fail_store()
2834 st->fail = fail; in fail_store()
2842 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in fail_show()
2844 return sprintf(buf, "%d\n", st->fail); in fail_show()
2871 if (sp->name) { in states_show()
2872 cur = sprintf(buf, "%3d: %s\n", i, sp->name); in states_show()
2909 return -EPERM; in __store_smt_control()
2912 return -ENODEV; in __store_smt_control()
2929 return -EINVAL; in __store_smt_control()
2931 return -EINVAL; in __store_smt_control()
2957 return -ENODEV; in __store_smt_control()
3016 int ret = -ENODEV; in cpu_smt_sysfs_init()
3020 ret = sysfs_create_group(&dev_root->kobj, &cpuhp_smt_attr_group); in cpu_smt_sysfs_init()
3037 ret = sysfs_create_group(&dev_root->kobj, &cpuhp_cpu_root_attr_group); in cpuhp_sysfs_init()
3048 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); in cpuhp_sysfs_init()
3065 /* cpu_bit_bitmap[0] is empty - so we can back into it */
3126 * IPI/NMI broadcasts when shutting down CPUs. Invocation from in set_cpu_online()
3175 * All except the cross-thread attack vector are mitigated by default.
3176 * Cross-thread mitigation often requires disabling SMT which is expensive
3177 * so cross-thread mitigations are only partially enabled by default.
3179 * Guest-to-Host and Guest-to-Guest vectors are only needed if KVM support is
3200 * be combined with attack-vector disables which follow them.