1ffdb5976SRusty Russell /* Copyright 2008, 2005 Rusty Russell rusty@rustcorp.com.au IBM Corporation. 2e5582ca2SRusty Russell * GPL v2 and any later version. 3e5582ca2SRusty Russell */ 41da177e4SLinus Torvalds #include <linux/cpu.h> 51da177e4SLinus Torvalds #include <linux/err.h> 6ee527cd3SPrarit Bhargava #include <linux/kthread.h> 7ee527cd3SPrarit Bhargava #include <linux/module.h> 8ee527cd3SPrarit Bhargava #include <linux/sched.h> 9ee527cd3SPrarit Bhargava #include <linux/stop_machine.h> 101da177e4SLinus Torvalds #include <linux/syscalls.h> 11a12bb444SBenjamin Herrenschmidt #include <linux/interrupt.h> 12a12bb444SBenjamin Herrenschmidt 131da177e4SLinus Torvalds #include <asm/atomic.h> 141da177e4SLinus Torvalds #include <asm/uaccess.h> 151da177e4SLinus Torvalds 16ffdb5976SRusty Russell /* This controls the threads on each CPU. */ 171da177e4SLinus Torvalds enum stopmachine_state { 18ffdb5976SRusty Russell /* Dummy starting state for thread. */ 19ffdb5976SRusty Russell STOPMACHINE_NONE, 20ffdb5976SRusty Russell /* Awaiting everyone to be scheduled. */ 211da177e4SLinus Torvalds STOPMACHINE_PREPARE, 22ffdb5976SRusty Russell /* Disable interrupts. */ 231da177e4SLinus Torvalds STOPMACHINE_DISABLE_IRQ, 24ffdb5976SRusty Russell /* Run the function */ 255c2aed62SJason Baron STOPMACHINE_RUN, 26ffdb5976SRusty Russell /* Exit */ 271da177e4SLinus Torvalds STOPMACHINE_EXIT, 281da177e4SLinus Torvalds }; 29ffdb5976SRusty Russell static enum stopmachine_state state; 301da177e4SLinus Torvalds 315c2aed62SJason Baron struct stop_machine_data { 325c2aed62SJason Baron int (*fn)(void *); 335c2aed62SJason Baron void *data; 34ffdb5976SRusty Russell int fnret; 35ffdb5976SRusty Russell }; 365c2aed62SJason Baron 37ffdb5976SRusty Russell /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */ 38ffdb5976SRusty Russell static unsigned int num_threads; 39ffdb5976SRusty Russell static atomic_t thread_ack; 40ffdb5976SRusty Russell static struct completion finished; 41ffdb5976SRusty Russell static DEFINE_MUTEX(lock); 421da177e4SLinus Torvalds 43ffdb5976SRusty Russell static void set_state(enum stopmachine_state newstate) 441da177e4SLinus Torvalds { 45ffdb5976SRusty Russell /* Reset ack counter. */ 46ffdb5976SRusty Russell atomic_set(&thread_ack, num_threads); 47ffdb5976SRusty Russell smp_wmb(); 48ffdb5976SRusty Russell state = newstate; 49ffdb5976SRusty Russell } 501da177e4SLinus Torvalds 51ffdb5976SRusty Russell /* Last one to ack a state moves to the next state. */ 52ffdb5976SRusty Russell static void ack_state(void) 53ffdb5976SRusty Russell { 54ffdb5976SRusty Russell if (atomic_dec_and_test(&thread_ack)) { 55ffdb5976SRusty Russell /* If we're the last one to ack the EXIT, we're finished. */ 56ffdb5976SRusty Russell if (state == STOPMACHINE_EXIT) 57ffdb5976SRusty Russell complete(&finished); 58ffdb5976SRusty Russell else 59ffdb5976SRusty Russell set_state(state + 1); 60ffdb5976SRusty Russell } 61ffdb5976SRusty Russell } 62d8cb7c1dSAndrew Morton 63ffdb5976SRusty Russell /* This is the actual thread which stops the CPU. It exits by itself rather 64ffdb5976SRusty Russell * than waiting for kthread_stop(), because it's easier for hotplug CPU. */ 65ffdb5976SRusty Russell static int stop_cpu(struct stop_machine_data *smdata) 66ffdb5976SRusty Russell { 67ffdb5976SRusty Russell enum stopmachine_state curstate = STOPMACHINE_NONE; 68ffdb5976SRusty Russell int uninitialized_var(ret); 691da177e4SLinus Torvalds 701da177e4SLinus Torvalds /* Simple state machine */ 71ffdb5976SRusty Russell do { 72ffdb5976SRusty Russell /* Chill out and ensure we re-read stopmachine_state. */ 73ffdb5976SRusty Russell cpu_relax(); 74ffdb5976SRusty Russell if (state != curstate) { 75ffdb5976SRusty Russell curstate = state; 76ffdb5976SRusty Russell switch (curstate) { 77ffdb5976SRusty Russell case STOPMACHINE_DISABLE_IRQ: 781da177e4SLinus Torvalds local_irq_disable(); 79a12bb444SBenjamin Herrenschmidt hard_irq_disable(); 80ffdb5976SRusty Russell break; 81ffdb5976SRusty Russell case STOPMACHINE_RUN: 82ffdb5976SRusty Russell /* |= allows error detection if functions on 83ffdb5976SRusty Russell * multiple CPUs. */ 84ffdb5976SRusty Russell smdata->fnret |= smdata->fn(smdata->data); 85ffdb5976SRusty Russell break; 86ffdb5976SRusty Russell default: 87ffdb5976SRusty Russell break; 881da177e4SLinus Torvalds } 89ffdb5976SRusty Russell ack_state(); 901da177e4SLinus Torvalds } 91ffdb5976SRusty Russell } while (curstate != STOPMACHINE_EXIT); 921da177e4SLinus Torvalds 931da177e4SLinus Torvalds local_irq_enable(); 94ffdb5976SRusty Russell do_exit(0); 95ffdb5976SRusty Russell } 961da177e4SLinus Torvalds 97ffdb5976SRusty Russell /* Callback for CPUs which aren't supposed to do anything. */ 98ffdb5976SRusty Russell static int chill(void *unused) 99ffdb5976SRusty Russell { 1001da177e4SLinus Torvalds return 0; 1011da177e4SLinus Torvalds } 1021da177e4SLinus Torvalds 103*eeec4fadSRusty Russell int __stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) 1041da177e4SLinus Torvalds { 105ffdb5976SRusty Russell int i, err; 106ffdb5976SRusty Russell struct stop_machine_data active, idle; 107ffdb5976SRusty Russell struct task_struct **threads; 1081da177e4SLinus Torvalds 109ffdb5976SRusty Russell active.fn = fn; 110ffdb5976SRusty Russell active.data = data; 111ffdb5976SRusty Russell active.fnret = 0; 112ffdb5976SRusty Russell idle.fn = chill; 113ffdb5976SRusty Russell idle.data = NULL; 1141da177e4SLinus Torvalds 115ffdb5976SRusty Russell /* This could be too big for stack on large machines. */ 116ffdb5976SRusty Russell threads = kcalloc(NR_CPUS, sizeof(threads[0]), GFP_KERNEL); 117ffdb5976SRusty Russell if (!threads) 118ffdb5976SRusty Russell return -ENOMEM; 119ffdb5976SRusty Russell 120ffdb5976SRusty Russell /* Set up initial state. */ 121ffdb5976SRusty Russell mutex_lock(&lock); 122ffdb5976SRusty Russell init_completion(&finished); 123ffdb5976SRusty Russell num_threads = num_online_cpus(); 124ffdb5976SRusty Russell set_state(STOPMACHINE_PREPARE); 1251da177e4SLinus Torvalds 1261da177e4SLinus Torvalds for_each_online_cpu(i) { 127*eeec4fadSRusty Russell struct stop_machine_data *smdata = &idle; 12885653af7SSatoru Takeuchi struct sched_param param = { .sched_priority = MAX_RT_PRIO-1 }; 12985653af7SSatoru Takeuchi 130*eeec4fadSRusty Russell if (!cpus) { 131*eeec4fadSRusty Russell if (i == first_cpu(cpu_online_map)) 132ffdb5976SRusty Russell smdata = &active; 133*eeec4fadSRusty Russell } else { 134*eeec4fadSRusty Russell if (cpu_isset(i, *cpus)) 135*eeec4fadSRusty Russell smdata = &active; 136*eeec4fadSRusty Russell } 137ffdb5976SRusty Russell 138ffdb5976SRusty Russell threads[i] = kthread_create((void *)stop_cpu, smdata, "kstop%u", 139ffdb5976SRusty Russell i); 140ffdb5976SRusty Russell if (IS_ERR(threads[i])) { 141ffdb5976SRusty Russell err = PTR_ERR(threads[i]); 142ffdb5976SRusty Russell threads[i] = NULL; 143ffdb5976SRusty Russell goto kill_threads; 1441da177e4SLinus Torvalds } 145ffdb5976SRusty Russell 146ffdb5976SRusty Russell /* Place it onto correct cpu. */ 147ffdb5976SRusty Russell kthread_bind(threads[i], i); 148ffdb5976SRusty Russell 149ffdb5976SRusty Russell /* Make it highest prio. */ 150ffdb5976SRusty Russell if (sched_setscheduler_nocheck(threads[i], SCHED_FIFO, ¶m)) 151ffdb5976SRusty Russell BUG(); 152ffdb5976SRusty Russell } 153ffdb5976SRusty Russell 154ffdb5976SRusty Russell /* We've created all the threads. Wake them all: hold this CPU so one 155ffdb5976SRusty Russell * doesn't hit this CPU until we're ready. */ 156*eeec4fadSRusty Russell get_cpu(); 157ffdb5976SRusty Russell for_each_online_cpu(i) 158ffdb5976SRusty Russell wake_up_process(threads[i]); 159ffdb5976SRusty Russell 160ffdb5976SRusty Russell /* This will release the thread on our CPU. */ 161ffdb5976SRusty Russell put_cpu(); 162ffdb5976SRusty Russell wait_for_completion(&finished); 163ffdb5976SRusty Russell mutex_unlock(&lock); 164ffdb5976SRusty Russell 165ffdb5976SRusty Russell kfree(threads); 166ffdb5976SRusty Russell 167ffdb5976SRusty Russell return active.fnret; 168ffdb5976SRusty Russell 169ffdb5976SRusty Russell kill_threads: 170ffdb5976SRusty Russell for_each_online_cpu(i) 171ffdb5976SRusty Russell if (threads[i]) 172ffdb5976SRusty Russell kthread_stop(threads[i]); 173ffdb5976SRusty Russell mutex_unlock(&lock); 174ffdb5976SRusty Russell 175ffdb5976SRusty Russell kfree(threads); 176ffdb5976SRusty Russell return err; 1771da177e4SLinus Torvalds } 1781da177e4SLinus Torvalds 179*eeec4fadSRusty Russell int stop_machine(int (*fn)(void *), void *data, const cpumask_t *cpus) 1801da177e4SLinus Torvalds { 1811da177e4SLinus Torvalds int ret; 1821da177e4SLinus Torvalds 1831da177e4SLinus Torvalds /* No CPUs can come up or down during this. */ 18486ef5c9aSGautham R Shenoy get_online_cpus(); 185*eeec4fadSRusty Russell ret = __stop_machine(fn, data, cpus); 18686ef5c9aSGautham R Shenoy put_online_cpus(); 1871da177e4SLinus Torvalds 1881da177e4SLinus Torvalds return ret; 1891da177e4SLinus Torvalds } 190*eeec4fadSRusty Russell EXPORT_SYMBOL_GPL(stop_machine); 191