1 /* 2 * Copyright (C) 2010 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com> 3 * 4 * Provides a framework for enqueueing and running callbacks from hardirq 5 * context. The enqueueing is NMI-safe. 6 */ 7 8 #include <linux/bug.h> 9 #include <linux/kernel.h> 10 #include <linux/export.h> 11 #include <linux/irq_work.h> 12 #include <linux/percpu.h> 13 #include <linux/hardirq.h> 14 #include <asm/processor.h> 15 16 /* 17 * An entry can be in one of four states: 18 * 19 * free NULL, 0 -> {claimed} : free to be used 20 * claimed NULL, 3 -> {pending} : claimed to be enqueued 21 * pending next, 3 -> {busy} : queued, pending callback 22 * busy NULL, 2 -> {free, claimed} : callback in progress, can be claimed 23 */ 24 25 #define IRQ_WORK_PENDING 1UL 26 #define IRQ_WORK_BUSY 2UL 27 #define IRQ_WORK_FLAGS 3UL 28 29 static DEFINE_PER_CPU(struct llist_head, irq_work_list); 30 31 /* 32 * Claim the entry so that no one else will poke at it. 33 */ 34 static bool irq_work_claim(struct irq_work *work) 35 { 36 unsigned long flags, nflags; 37 38 for (;;) { 39 flags = work->flags; 40 if (flags & IRQ_WORK_PENDING) 41 return false; 42 nflags = flags | IRQ_WORK_FLAGS; 43 if (cmpxchg(&work->flags, flags, nflags) == flags) 44 break; 45 cpu_relax(); 46 } 47 48 return true; 49 } 50 51 void __weak arch_irq_work_raise(void) 52 { 53 /* 54 * Lame architectures will get the timer tick callback 55 */ 56 } 57 58 /* 59 * Queue the entry and raise the IPI if needed. 60 */ 61 static void __irq_work_queue(struct irq_work *work) 62 { 63 bool empty; 64 65 preempt_disable(); 66 67 empty = llist_add(&work->llnode, &__get_cpu_var(irq_work_list)); 68 /* The list was empty, raise self-interrupt to start processing. */ 69 if (empty) 70 arch_irq_work_raise(); 71 72 preempt_enable(); 73 } 74 75 /* 76 * Enqueue the irq_work @entry, returns true on success, failure when the 77 * @entry was already enqueued by someone else. 78 * 79 * Can be re-enqueued while the callback is still in progress. 80 */ 81 bool irq_work_queue(struct irq_work *work) 82 { 83 if (!irq_work_claim(work)) { 84 /* 85 * Already enqueued, can't do! 86 */ 87 return false; 88 } 89 90 __irq_work_queue(work); 91 return true; 92 } 93 EXPORT_SYMBOL_GPL(irq_work_queue); 94 95 /* 96 * Run the irq_work entries on this cpu. Requires to be ran from hardirq 97 * context with local IRQs disabled. 98 */ 99 void irq_work_run(void) 100 { 101 struct irq_work *work; 102 struct llist_head *this_list; 103 struct llist_node *llnode; 104 105 this_list = &__get_cpu_var(irq_work_list); 106 if (llist_empty(this_list)) 107 return; 108 109 BUG_ON(!in_irq()); 110 BUG_ON(!irqs_disabled()); 111 112 llnode = llist_del_all(this_list); 113 while (llnode != NULL) { 114 work = llist_entry(llnode, struct irq_work, llnode); 115 116 llnode = llist_next(llnode); 117 118 /* 119 * Clear the PENDING bit, after this point the @work 120 * can be re-used. 121 */ 122 work->flags = IRQ_WORK_BUSY; 123 work->func(work); 124 /* 125 * Clear the BUSY bit and return to the free state if 126 * no-one else claimed it meanwhile. 127 */ 128 (void)cmpxchg(&work->flags, IRQ_WORK_BUSY, 0); 129 } 130 } 131 EXPORT_SYMBOL_GPL(irq_work_run); 132 133 /* 134 * Synchronize against the irq_work @entry, ensures the entry is not 135 * currently in use. 136 */ 137 void irq_work_sync(struct irq_work *work) 138 { 139 WARN_ON_ONCE(irqs_disabled()); 140 141 while (work->flags & IRQ_WORK_BUSY) 142 cpu_relax(); 143 } 144 EXPORT_SYMBOL_GPL(irq_work_sync); 145