1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Common SMP CPU bringup/teardown functions 4 */ 5 #include <linux/cpu.h> 6 #include <linux/err.h> 7 #include <linux/smp.h> 8 #include <linux/delay.h> 9 #include <linux/init.h> 10 #include <linux/list.h> 11 #include <linux/slab.h> 12 #include <linux/sched.h> 13 #include <linux/sched/task.h> 14 #include <linux/export.h> 15 #include <linux/percpu.h> 16 #include <linux/kthread.h> 17 #include <linux/smpboot.h> 18 19 #include "smpboot.h" 20 21 #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD 22 /* 23 * For the hotplug case we keep the task structs around and reuse 24 * them. 25 */ 26 static DEFINE_PER_CPU(struct task_struct *, idle_threads); 27 28 struct task_struct *idle_thread_get(unsigned int cpu) 29 { 30 struct task_struct *tsk = per_cpu(idle_threads, cpu); 31 32 if (!tsk) 33 return ERR_PTR(-ENOMEM); 34 return tsk; 35 } 36 37 void __init idle_thread_set_boot_cpu(void) 38 { 39 per_cpu(idle_threads, smp_processor_id()) = current; 40 } 41 42 /** 43 * idle_init - Initialize the idle thread for a cpu 44 * @cpu: The cpu for which the idle thread should be initialized 45 * 46 * Creates the thread if it does not exist. 47 */ 48 static __always_inline void idle_init(unsigned int cpu) 49 { 50 struct task_struct *tsk = per_cpu(idle_threads, cpu); 51 52 if (!tsk) { 53 tsk = fork_idle(cpu); 54 if (IS_ERR(tsk)) 55 pr_err("SMP: fork_idle() failed for CPU %u\n", cpu); 56 else 57 per_cpu(idle_threads, cpu) = tsk; 58 } 59 } 60 61 /** 62 * idle_threads_init - Initialize idle threads for all cpus 63 */ 64 void __init idle_threads_init(void) 65 { 66 unsigned int cpu, boot_cpu; 67 68 boot_cpu = smp_processor_id(); 69 70 for_each_possible_cpu(cpu) { 71 if (cpu != boot_cpu) 72 idle_init(cpu); 73 } 74 } 75 #endif 76 77 static LIST_HEAD(hotplug_threads); 78 static DEFINE_MUTEX(smpboot_threads_lock); 79 80 struct smpboot_thread_data { 81 unsigned int cpu; 82 unsigned int status; 83 struct smp_hotplug_thread *ht; 84 }; 85 86 enum { 87 HP_THREAD_NONE = 0, 88 HP_THREAD_ACTIVE, 89 HP_THREAD_PARKED, 90 }; 91 92 /** 93 * smpboot_thread_fn - percpu hotplug thread loop function 94 * @data: thread data pointer 95 * 96 * Checks for thread stop and park conditions. Calls the necessary 97 * setup, cleanup, park and unpark functions for the registered 98 * thread. 99 * 100 * Returns 1 when the thread should exit, 0 otherwise. 101 */ 102 static int smpboot_thread_fn(void *data) 103 { 104 struct smpboot_thread_data *td = data; 105 struct smp_hotplug_thread *ht = td->ht; 106 107 while (1) { 108 set_current_state(TASK_INTERRUPTIBLE); 109 preempt_disable(); 110 if (kthread_should_stop()) { 111 __set_current_state(TASK_RUNNING); 112 preempt_enable(); 113 /* cleanup must mirror setup */ 114 if (ht->cleanup && td->status != HP_THREAD_NONE) 115 ht->cleanup(td->cpu, cpu_online(td->cpu)); 116 kfree(td); 117 return 0; 118 } 119 120 if (kthread_should_park()) { 121 __set_current_state(TASK_RUNNING); 122 preempt_enable(); 123 if (ht->park && td->status == HP_THREAD_ACTIVE) { 124 BUG_ON(td->cpu != smp_processor_id()); 125 ht->park(td->cpu); 126 td->status = HP_THREAD_PARKED; 127 } 128 kthread_parkme(); 129 /* We might have been woken for stop */ 130 continue; 131 } 132 133 BUG_ON(td->cpu != smp_processor_id()); 134 135 /* Check for state change setup */ 136 switch (td->status) { 137 case HP_THREAD_NONE: 138 __set_current_state(TASK_RUNNING); 139 preempt_enable(); 140 if (ht->setup) 141 ht->setup(td->cpu); 142 td->status = HP_THREAD_ACTIVE; 143 continue; 144 145 case HP_THREAD_PARKED: 146 __set_current_state(TASK_RUNNING); 147 preempt_enable(); 148 if (ht->unpark) 149 ht->unpark(td->cpu); 150 td->status = HP_THREAD_ACTIVE; 151 continue; 152 } 153 154 if (!ht->thread_should_run(td->cpu)) { 155 preempt_enable_no_resched(); 156 schedule(); 157 } else { 158 __set_current_state(TASK_RUNNING); 159 preempt_enable(); 160 ht->thread_fn(td->cpu); 161 } 162 } 163 } 164 165 static int 166 __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 167 { 168 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 169 struct smpboot_thread_data *td; 170 171 if (tsk) 172 return 0; 173 174 td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu)); 175 if (!td) 176 return -ENOMEM; 177 td->cpu = cpu; 178 td->ht = ht; 179 180 tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu, 181 ht->thread_comm); 182 if (IS_ERR(tsk)) { 183 kfree(td); 184 return PTR_ERR(tsk); 185 } 186 kthread_set_per_cpu(tsk, cpu); 187 /* 188 * Park the thread so that it could start right on the CPU 189 * when it is available. 190 */ 191 kthread_park(tsk); 192 get_task_struct(tsk); 193 *per_cpu_ptr(ht->store, cpu) = tsk; 194 if (ht->create) { 195 /* 196 * Make sure that the task has actually scheduled out 197 * into park position, before calling the create 198 * callback. At least the migration thread callback 199 * requires that the task is off the runqueue. 200 */ 201 if (!wait_task_inactive(tsk, TASK_PARKED)) 202 WARN_ON(1); 203 else 204 ht->create(cpu); 205 } 206 return 0; 207 } 208 209 int smpboot_create_threads(unsigned int cpu) 210 { 211 struct smp_hotplug_thread *cur; 212 int ret = 0; 213 214 mutex_lock(&smpboot_threads_lock); 215 list_for_each_entry(cur, &hotplug_threads, list) { 216 ret = __smpboot_create_thread(cur, cpu); 217 if (ret) 218 break; 219 } 220 mutex_unlock(&smpboot_threads_lock); 221 return ret; 222 } 223 224 static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 225 { 226 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 227 228 if (!ht->selfparking) 229 kthread_unpark(tsk); 230 } 231 232 int smpboot_unpark_threads(unsigned int cpu) 233 { 234 struct smp_hotplug_thread *cur; 235 236 mutex_lock(&smpboot_threads_lock); 237 list_for_each_entry(cur, &hotplug_threads, list) 238 smpboot_unpark_thread(cur, cpu); 239 mutex_unlock(&smpboot_threads_lock); 240 return 0; 241 } 242 243 static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 244 { 245 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 246 247 if (tsk && !ht->selfparking) 248 kthread_park(tsk); 249 } 250 251 int smpboot_park_threads(unsigned int cpu) 252 { 253 struct smp_hotplug_thread *cur; 254 255 mutex_lock(&smpboot_threads_lock); 256 list_for_each_entry_reverse(cur, &hotplug_threads, list) 257 smpboot_park_thread(cur, cpu); 258 mutex_unlock(&smpboot_threads_lock); 259 return 0; 260 } 261 262 static void smpboot_destroy_threads(struct smp_hotplug_thread *ht) 263 { 264 unsigned int cpu; 265 266 /* We need to destroy also the parked threads of offline cpus */ 267 for_each_possible_cpu(cpu) { 268 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 269 270 if (tsk) { 271 kthread_stop_put(tsk); 272 *per_cpu_ptr(ht->store, cpu) = NULL; 273 } 274 } 275 } 276 277 /** 278 * smpboot_register_percpu_thread - Register a per_cpu thread related 279 * to hotplug 280 * @plug_thread: Hotplug thread descriptor 281 * 282 * Creates and starts the threads on all online cpus. 283 */ 284 int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread) 285 { 286 unsigned int cpu; 287 int ret = 0; 288 289 cpus_read_lock(); 290 mutex_lock(&smpboot_threads_lock); 291 for_each_online_cpu(cpu) { 292 ret = __smpboot_create_thread(plug_thread, cpu); 293 if (ret) { 294 smpboot_destroy_threads(plug_thread); 295 goto out; 296 } 297 smpboot_unpark_thread(plug_thread, cpu); 298 } 299 list_add(&plug_thread->list, &hotplug_threads); 300 out: 301 mutex_unlock(&smpboot_threads_lock); 302 cpus_read_unlock(); 303 return ret; 304 } 305 EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread); 306 307 /** 308 * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug 309 * @plug_thread: Hotplug thread descriptor 310 * 311 * Stops all threads on all possible cpus. 312 */ 313 void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread) 314 { 315 cpus_read_lock(); 316 mutex_lock(&smpboot_threads_lock); 317 list_del(&plug_thread->list); 318 smpboot_destroy_threads(plug_thread); 319 mutex_unlock(&smpboot_threads_lock); 320 cpus_read_unlock(); 321 } 322 EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread); 323