1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Common SMP CPU bringup/teardown functions 4 */ 5 #include <linux/cpu.h> 6 #include <linux/err.h> 7 #include <linux/smp.h> 8 #include <linux/delay.h> 9 #include <linux/init.h> 10 #include <linux/list.h> 11 #include <linux/slab.h> 12 #include <linux/sched.h> 13 #include <linux/sched/task.h> 14 #include <linux/export.h> 15 #include <linux/percpu.h> 16 #include <linux/kthread.h> 17 #include <linux/smpboot.h> 18 19 #include "smpboot.h" 20 21 #ifdef CONFIG_GENERIC_SMP_IDLE_THREAD 22 /* 23 * For the hotplug case we keep the task structs around and reuse 24 * them. 25 */ 26 static DEFINE_PER_CPU(struct task_struct *, idle_threads); 27 28 struct task_struct *idle_thread_get(unsigned int cpu) 29 { 30 struct task_struct *tsk = per_cpu(idle_threads, cpu); 31 32 if (!tsk) 33 return ERR_PTR(-ENOMEM); 34 return tsk; 35 } 36 37 void __init idle_thread_set_boot_cpu(void) 38 { 39 per_cpu(idle_threads, smp_processor_id()) = current; 40 } 41 42 /** 43 * idle_init - Initialize the idle thread for a cpu 44 * @cpu: The cpu for which the idle thread should be initialized 45 * 46 * Creates the thread if it does not exist. 47 */ 48 static __always_inline void idle_init(unsigned int cpu) 49 { 50 struct task_struct *tsk = per_cpu(idle_threads, cpu); 51 52 if (!tsk) { 53 tsk = fork_idle(cpu); 54 if (IS_ERR(tsk)) 55 pr_err("SMP: fork_idle() failed for CPU %u\n", cpu); 56 else 57 per_cpu(idle_threads, cpu) = tsk; 58 } 59 } 60 61 /** 62 * idle_threads_init - Initialize idle threads for all cpus 63 */ 64 void __init idle_threads_init(void) 65 { 66 unsigned int cpu, boot_cpu; 67 68 boot_cpu = smp_processor_id(); 69 70 for_each_possible_cpu(cpu) { 71 if (cpu != boot_cpu) 72 idle_init(cpu); 73 } 74 } 75 #endif 76 77 static LIST_HEAD(hotplug_threads); 78 static DEFINE_MUTEX(smpboot_threads_lock); 79 80 struct smpboot_thread_data { 81 unsigned int cpu; 82 unsigned int status; 83 struct smp_hotplug_thread *ht; 84 }; 85 86 enum { 87 HP_THREAD_NONE = 0, 88 HP_THREAD_ACTIVE, 89 HP_THREAD_PARKED, 90 }; 91 92 /** 93 * smpboot_thread_fn - percpu hotplug thread loop function 94 * @data: thread data pointer 95 * 96 * Checks for thread stop and park conditions. Calls the necessary 97 * setup, cleanup, park and unpark functions for the registered 98 * thread. 99 * 100 * Returns 1 when the thread should exit, 0 otherwise. 101 */ 102 static int smpboot_thread_fn(void *data) 103 { 104 struct smpboot_thread_data *td = data; 105 struct smp_hotplug_thread *ht = td->ht; 106 bool should_run; 107 108 while (1) { 109 set_current_state(TASK_INTERRUPTIBLE); 110 preempt_disable(); 111 if (kthread_should_stop()) { 112 __set_current_state(TASK_RUNNING); 113 preempt_enable(); 114 /* cleanup must mirror setup */ 115 if (ht->cleanup && td->status != HP_THREAD_NONE) 116 ht->cleanup(td->cpu, cpu_online(td->cpu)); 117 kfree(td); 118 return 0; 119 } 120 121 should_run = td->status == HP_THREAD_ACTIVE && ht->thread_should_run(td->cpu); 122 if (kthread_should_park() && !should_run) { 123 __set_current_state(TASK_RUNNING); 124 preempt_enable(); 125 if (ht->park && td->status == HP_THREAD_ACTIVE) { 126 BUG_ON(td->cpu != smp_processor_id()); 127 ht->park(td->cpu); 128 td->status = HP_THREAD_PARKED; 129 } 130 kthread_parkme(); 131 /* We might have been woken for stop */ 132 continue; 133 } 134 135 BUG_ON(td->cpu != smp_processor_id()); 136 137 /* Check for state change setup */ 138 switch (td->status) { 139 case HP_THREAD_NONE: 140 __set_current_state(TASK_RUNNING); 141 preempt_enable(); 142 if (ht->setup) 143 ht->setup(td->cpu); 144 td->status = HP_THREAD_ACTIVE; 145 continue; 146 147 case HP_THREAD_PARKED: 148 __set_current_state(TASK_RUNNING); 149 preempt_enable(); 150 if (ht->unpark) 151 ht->unpark(td->cpu); 152 td->status = HP_THREAD_ACTIVE; 153 continue; 154 } 155 156 if (!should_run) { 157 preempt_enable_no_resched(); 158 schedule(); 159 } else { 160 __set_current_state(TASK_RUNNING); 161 preempt_enable(); 162 ht->thread_fn(td->cpu); 163 } 164 } 165 } 166 167 static int 168 __smpboot_create_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 169 { 170 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 171 struct smpboot_thread_data *td; 172 173 if (tsk) 174 return 0; 175 176 td = kzalloc_node(sizeof(*td), GFP_KERNEL, cpu_to_node(cpu)); 177 if (!td) 178 return -ENOMEM; 179 td->cpu = cpu; 180 td->ht = ht; 181 182 tsk = kthread_create_on_cpu(smpboot_thread_fn, td, cpu, 183 ht->thread_comm); 184 if (IS_ERR(tsk)) { 185 kfree(td); 186 return PTR_ERR(tsk); 187 } 188 kthread_set_per_cpu(tsk, cpu); 189 /* 190 * Park the thread so that it could start right on the CPU 191 * when it is available. 192 */ 193 kthread_park(tsk); 194 get_task_struct(tsk); 195 *per_cpu_ptr(ht->store, cpu) = tsk; 196 if (ht->create) { 197 /* 198 * Make sure that the task has actually scheduled out 199 * into park position, before calling the create 200 * callback. At least the migration thread callback 201 * requires that the task is off the runqueue. 202 */ 203 if (!wait_task_inactive(tsk, TASK_PARKED)) 204 WARN_ON(1); 205 else 206 ht->create(cpu); 207 } 208 return 0; 209 } 210 211 int smpboot_create_threads(unsigned int cpu) 212 { 213 struct smp_hotplug_thread *cur; 214 int ret = 0; 215 216 mutex_lock(&smpboot_threads_lock); 217 list_for_each_entry(cur, &hotplug_threads, list) { 218 ret = __smpboot_create_thread(cur, cpu); 219 if (ret) 220 break; 221 } 222 mutex_unlock(&smpboot_threads_lock); 223 return ret; 224 } 225 226 static void smpboot_unpark_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 227 { 228 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 229 230 if (!ht->selfparking) 231 kthread_unpark(tsk); 232 } 233 234 int smpboot_unpark_threads(unsigned int cpu) 235 { 236 struct smp_hotplug_thread *cur; 237 238 mutex_lock(&smpboot_threads_lock); 239 list_for_each_entry(cur, &hotplug_threads, list) 240 smpboot_unpark_thread(cur, cpu); 241 mutex_unlock(&smpboot_threads_lock); 242 return 0; 243 } 244 245 static void smpboot_park_thread(struct smp_hotplug_thread *ht, unsigned int cpu) 246 { 247 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 248 249 if (tsk && !ht->selfparking) 250 kthread_park(tsk); 251 } 252 253 int smpboot_park_threads(unsigned int cpu) 254 { 255 struct smp_hotplug_thread *cur; 256 257 mutex_lock(&smpboot_threads_lock); 258 list_for_each_entry_reverse(cur, &hotplug_threads, list) 259 smpboot_park_thread(cur, cpu); 260 mutex_unlock(&smpboot_threads_lock); 261 return 0; 262 } 263 264 static void smpboot_destroy_threads(struct smp_hotplug_thread *ht) 265 { 266 unsigned int cpu; 267 268 /* We need to destroy also the parked threads of offline cpus */ 269 for_each_possible_cpu(cpu) { 270 struct task_struct *tsk = *per_cpu_ptr(ht->store, cpu); 271 272 if (tsk) { 273 kthread_stop_put(tsk); 274 *per_cpu_ptr(ht->store, cpu) = NULL; 275 } 276 } 277 } 278 279 /** 280 * smpboot_register_percpu_thread - Register a per_cpu thread related 281 * to hotplug 282 * @plug_thread: Hotplug thread descriptor 283 * 284 * Creates and starts the threads on all online cpus. 285 */ 286 int smpboot_register_percpu_thread(struct smp_hotplug_thread *plug_thread) 287 { 288 unsigned int cpu; 289 int ret = 0; 290 291 cpus_read_lock(); 292 mutex_lock(&smpboot_threads_lock); 293 for_each_online_cpu(cpu) { 294 ret = __smpboot_create_thread(plug_thread, cpu); 295 if (ret) { 296 smpboot_destroy_threads(plug_thread); 297 goto out; 298 } 299 smpboot_unpark_thread(plug_thread, cpu); 300 } 301 list_add(&plug_thread->list, &hotplug_threads); 302 out: 303 mutex_unlock(&smpboot_threads_lock); 304 cpus_read_unlock(); 305 return ret; 306 } 307 EXPORT_SYMBOL_GPL(smpboot_register_percpu_thread); 308 309 /** 310 * smpboot_unregister_percpu_thread - Unregister a per_cpu thread related to hotplug 311 * @plug_thread: Hotplug thread descriptor 312 * 313 * Stops all threads on all possible cpus. 314 */ 315 void smpboot_unregister_percpu_thread(struct smp_hotplug_thread *plug_thread) 316 { 317 cpus_read_lock(); 318 mutex_lock(&smpboot_threads_lock); 319 list_del(&plug_thread->list); 320 smpboot_destroy_threads(plug_thread); 321 mutex_unlock(&smpboot_threads_lock); 322 cpus_read_unlock(); 323 } 324 EXPORT_SYMBOL_GPL(smpboot_unregister_percpu_thread); 325