1511cbce2SChristoph Hellwig /* 2511cbce2SChristoph Hellwig * Functions related to interrupt-poll handling in the block layer. This 3511cbce2SChristoph Hellwig * is similar to NAPI for network devices. 4511cbce2SChristoph Hellwig */ 5511cbce2SChristoph Hellwig #include <linux/kernel.h> 6511cbce2SChristoph Hellwig #include <linux/module.h> 7511cbce2SChristoph Hellwig #include <linux/init.h> 8511cbce2SChristoph Hellwig #include <linux/bio.h> 9511cbce2SChristoph Hellwig #include <linux/interrupt.h> 10511cbce2SChristoph Hellwig #include <linux/cpu.h> 11511cbce2SChristoph Hellwig #include <linux/irq_poll.h> 12511cbce2SChristoph Hellwig #include <linux/delay.h> 13511cbce2SChristoph Hellwig 14511cbce2SChristoph Hellwig static unsigned int irq_poll_budget __read_mostly = 256; 15511cbce2SChristoph Hellwig 16511cbce2SChristoph Hellwig static DEFINE_PER_CPU(struct list_head, blk_cpu_iopoll); 17511cbce2SChristoph Hellwig 18511cbce2SChristoph Hellwig /** 19511cbce2SChristoph Hellwig * irq_poll_sched - Schedule a run of the iopoll handler 20511cbce2SChristoph Hellwig * @iop: The parent iopoll structure 21511cbce2SChristoph Hellwig * 22511cbce2SChristoph Hellwig * Description: 23511cbce2SChristoph Hellwig * Add this irq_poll structure to the pending poll list and trigger the 24ea51190cSChristoph Hellwig * raise of the blk iopoll softirq. 25511cbce2SChristoph Hellwig **/ 26511cbce2SChristoph Hellwig void irq_poll_sched(struct irq_poll *iop) 27511cbce2SChristoph Hellwig { 28511cbce2SChristoph Hellwig unsigned long flags; 29511cbce2SChristoph Hellwig 30ea51190cSChristoph Hellwig if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) 31ea51190cSChristoph Hellwig return; 32*2ee177e9SBart Van Assche if (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) 33ea51190cSChristoph Hellwig return; 34ea51190cSChristoph Hellwig 35511cbce2SChristoph Hellwig local_irq_save(flags); 36511cbce2SChristoph Hellwig list_add_tail(&iop->list, this_cpu_ptr(&blk_cpu_iopoll)); 37511cbce2SChristoph Hellwig __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); 38511cbce2SChristoph Hellwig local_irq_restore(flags); 39511cbce2SChristoph Hellwig } 40511cbce2SChristoph Hellwig EXPORT_SYMBOL(irq_poll_sched); 41511cbce2SChristoph Hellwig 42511cbce2SChristoph Hellwig /** 43511cbce2SChristoph Hellwig * __irq_poll_complete - Mark this @iop as un-polled again 44511cbce2SChristoph Hellwig * @iop: The parent iopoll structure 45511cbce2SChristoph Hellwig * 46511cbce2SChristoph Hellwig * Description: 47511cbce2SChristoph Hellwig * See irq_poll_complete(). This function must be called with interrupts 48511cbce2SChristoph Hellwig * disabled. 49511cbce2SChristoph Hellwig **/ 5083af187dSChristoph Hellwig static void __irq_poll_complete(struct irq_poll *iop) 51511cbce2SChristoph Hellwig { 52511cbce2SChristoph Hellwig list_del(&iop->list); 53511cbce2SChristoph Hellwig smp_mb__before_atomic(); 54511cbce2SChristoph Hellwig clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); 55511cbce2SChristoph Hellwig } 56511cbce2SChristoph Hellwig 57511cbce2SChristoph Hellwig /** 58511cbce2SChristoph Hellwig * irq_poll_complete - Mark this @iop as un-polled again 59511cbce2SChristoph Hellwig * @iop: The parent iopoll structure 60511cbce2SChristoph Hellwig * 61511cbce2SChristoph Hellwig * Description: 62511cbce2SChristoph Hellwig * If a driver consumes less than the assigned budget in its run of the 63511cbce2SChristoph Hellwig * iopoll handler, it'll end the polled mode by calling this function. The 64ea51190cSChristoph Hellwig * iopoll handler will not be invoked again before irq_poll_sched() 65511cbce2SChristoph Hellwig * is called. 66511cbce2SChristoph Hellwig **/ 67511cbce2SChristoph Hellwig void irq_poll_complete(struct irq_poll *iop) 68511cbce2SChristoph Hellwig { 69511cbce2SChristoph Hellwig unsigned long flags; 70511cbce2SChristoph Hellwig 71511cbce2SChristoph Hellwig local_irq_save(flags); 72511cbce2SChristoph Hellwig __irq_poll_complete(iop); 73511cbce2SChristoph Hellwig local_irq_restore(flags); 74511cbce2SChristoph Hellwig } 75511cbce2SChristoph Hellwig EXPORT_SYMBOL(irq_poll_complete); 76511cbce2SChristoph Hellwig 77511cbce2SChristoph Hellwig static void irq_poll_softirq(struct softirq_action *h) 78511cbce2SChristoph Hellwig { 79511cbce2SChristoph Hellwig struct list_head *list = this_cpu_ptr(&blk_cpu_iopoll); 80511cbce2SChristoph Hellwig int rearm = 0, budget = irq_poll_budget; 81511cbce2SChristoph Hellwig unsigned long start_time = jiffies; 82511cbce2SChristoph Hellwig 83511cbce2SChristoph Hellwig local_irq_disable(); 84511cbce2SChristoph Hellwig 85511cbce2SChristoph Hellwig while (!list_empty(list)) { 86511cbce2SChristoph Hellwig struct irq_poll *iop; 87511cbce2SChristoph Hellwig int work, weight; 88511cbce2SChristoph Hellwig 89511cbce2SChristoph Hellwig /* 90511cbce2SChristoph Hellwig * If softirq window is exhausted then punt. 91511cbce2SChristoph Hellwig */ 92511cbce2SChristoph Hellwig if (budget <= 0 || time_after(jiffies, start_time)) { 93511cbce2SChristoph Hellwig rearm = 1; 94511cbce2SChristoph Hellwig break; 95511cbce2SChristoph Hellwig } 96511cbce2SChristoph Hellwig 97511cbce2SChristoph Hellwig local_irq_enable(); 98511cbce2SChristoph Hellwig 99511cbce2SChristoph Hellwig /* Even though interrupts have been re-enabled, this 100511cbce2SChristoph Hellwig * access is safe because interrupts can only add new 101511cbce2SChristoph Hellwig * entries to the tail of this list, and only ->poll() 102511cbce2SChristoph Hellwig * calls can remove this head entry from the list. 103511cbce2SChristoph Hellwig */ 104511cbce2SChristoph Hellwig iop = list_entry(list->next, struct irq_poll, list); 105511cbce2SChristoph Hellwig 106511cbce2SChristoph Hellwig weight = iop->weight; 107511cbce2SChristoph Hellwig work = 0; 108511cbce2SChristoph Hellwig if (test_bit(IRQ_POLL_F_SCHED, &iop->state)) 109511cbce2SChristoph Hellwig work = iop->poll(iop, weight); 110511cbce2SChristoph Hellwig 111511cbce2SChristoph Hellwig budget -= work; 112511cbce2SChristoph Hellwig 113511cbce2SChristoph Hellwig local_irq_disable(); 114511cbce2SChristoph Hellwig 115511cbce2SChristoph Hellwig /* 116511cbce2SChristoph Hellwig * Drivers must not modify the iopoll state, if they 117511cbce2SChristoph Hellwig * consume their assigned weight (or more, some drivers can't 118511cbce2SChristoph Hellwig * easily just stop processing, they have to complete an 119511cbce2SChristoph Hellwig * entire mask of commands).In such cases this code 120511cbce2SChristoph Hellwig * still "owns" the iopoll instance and therefore can 121511cbce2SChristoph Hellwig * move the instance around on the list at-will. 122511cbce2SChristoph Hellwig */ 123511cbce2SChristoph Hellwig if (work >= weight) { 1240bc92aceSChristoph Hellwig if (test_bit(IRQ_POLL_F_DISABLE, &iop->state)) 125511cbce2SChristoph Hellwig __irq_poll_complete(iop); 126511cbce2SChristoph Hellwig else 127511cbce2SChristoph Hellwig list_move_tail(&iop->list, list); 128511cbce2SChristoph Hellwig } 129511cbce2SChristoph Hellwig } 130511cbce2SChristoph Hellwig 131511cbce2SChristoph Hellwig if (rearm) 132511cbce2SChristoph Hellwig __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); 133511cbce2SChristoph Hellwig 134511cbce2SChristoph Hellwig local_irq_enable(); 135511cbce2SChristoph Hellwig } 136511cbce2SChristoph Hellwig 137511cbce2SChristoph Hellwig /** 138511cbce2SChristoph Hellwig * irq_poll_disable - Disable iopoll on this @iop 139511cbce2SChristoph Hellwig * @iop: The parent iopoll structure 140511cbce2SChristoph Hellwig * 141511cbce2SChristoph Hellwig * Description: 142511cbce2SChristoph Hellwig * Disable io polling and wait for any pending callbacks to have completed. 143511cbce2SChristoph Hellwig **/ 144511cbce2SChristoph Hellwig void irq_poll_disable(struct irq_poll *iop) 145511cbce2SChristoph Hellwig { 146511cbce2SChristoph Hellwig set_bit(IRQ_POLL_F_DISABLE, &iop->state); 147511cbce2SChristoph Hellwig while (test_and_set_bit(IRQ_POLL_F_SCHED, &iop->state)) 148511cbce2SChristoph Hellwig msleep(1); 149511cbce2SChristoph Hellwig clear_bit(IRQ_POLL_F_DISABLE, &iop->state); 150511cbce2SChristoph Hellwig } 151511cbce2SChristoph Hellwig EXPORT_SYMBOL(irq_poll_disable); 152511cbce2SChristoph Hellwig 153511cbce2SChristoph Hellwig /** 154511cbce2SChristoph Hellwig * irq_poll_enable - Enable iopoll on this @iop 155511cbce2SChristoph Hellwig * @iop: The parent iopoll structure 156511cbce2SChristoph Hellwig * 157511cbce2SChristoph Hellwig * Description: 158511cbce2SChristoph Hellwig * Enable iopoll on this @iop. Note that the handler run will not be 159511cbce2SChristoph Hellwig * scheduled, it will only mark it as active. 160511cbce2SChristoph Hellwig **/ 161511cbce2SChristoph Hellwig void irq_poll_enable(struct irq_poll *iop) 162511cbce2SChristoph Hellwig { 163511cbce2SChristoph Hellwig BUG_ON(!test_bit(IRQ_POLL_F_SCHED, &iop->state)); 164511cbce2SChristoph Hellwig smp_mb__before_atomic(); 165511cbce2SChristoph Hellwig clear_bit_unlock(IRQ_POLL_F_SCHED, &iop->state); 166511cbce2SChristoph Hellwig } 167511cbce2SChristoph Hellwig EXPORT_SYMBOL(irq_poll_enable); 168511cbce2SChristoph Hellwig 169511cbce2SChristoph Hellwig /** 170511cbce2SChristoph Hellwig * irq_poll_init - Initialize this @iop 171511cbce2SChristoph Hellwig * @iop: The parent iopoll structure 172511cbce2SChristoph Hellwig * @weight: The default weight (or command completion budget) 173511cbce2SChristoph Hellwig * @poll_fn: The handler to invoke 174511cbce2SChristoph Hellwig * 175511cbce2SChristoph Hellwig * Description: 17678d0264eSChristoph Hellwig * Initialize and enable this irq_poll structure. 177511cbce2SChristoph Hellwig **/ 178511cbce2SChristoph Hellwig void irq_poll_init(struct irq_poll *iop, int weight, irq_poll_fn *poll_fn) 179511cbce2SChristoph Hellwig { 180511cbce2SChristoph Hellwig memset(iop, 0, sizeof(*iop)); 181511cbce2SChristoph Hellwig INIT_LIST_HEAD(&iop->list); 182511cbce2SChristoph Hellwig iop->weight = weight; 183511cbce2SChristoph Hellwig iop->poll = poll_fn; 184511cbce2SChristoph Hellwig } 185511cbce2SChristoph Hellwig EXPORT_SYMBOL(irq_poll_init); 186511cbce2SChristoph Hellwig 187511cbce2SChristoph Hellwig static int irq_poll_cpu_notify(struct notifier_block *self, 188511cbce2SChristoph Hellwig unsigned long action, void *hcpu) 189511cbce2SChristoph Hellwig { 190511cbce2SChristoph Hellwig /* 191511cbce2SChristoph Hellwig * If a CPU goes away, splice its entries to the current CPU 192511cbce2SChristoph Hellwig * and trigger a run of the softirq 193511cbce2SChristoph Hellwig */ 194511cbce2SChristoph Hellwig if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) { 195511cbce2SChristoph Hellwig int cpu = (unsigned long) hcpu; 196511cbce2SChristoph Hellwig 197511cbce2SChristoph Hellwig local_irq_disable(); 198511cbce2SChristoph Hellwig list_splice_init(&per_cpu(blk_cpu_iopoll, cpu), 199511cbce2SChristoph Hellwig this_cpu_ptr(&blk_cpu_iopoll)); 200511cbce2SChristoph Hellwig __raise_softirq_irqoff(IRQ_POLL_SOFTIRQ); 201511cbce2SChristoph Hellwig local_irq_enable(); 202511cbce2SChristoph Hellwig } 203511cbce2SChristoph Hellwig 204511cbce2SChristoph Hellwig return NOTIFY_OK; 205511cbce2SChristoph Hellwig } 206511cbce2SChristoph Hellwig 207511cbce2SChristoph Hellwig static struct notifier_block irq_poll_cpu_notifier = { 208511cbce2SChristoph Hellwig .notifier_call = irq_poll_cpu_notify, 209511cbce2SChristoph Hellwig }; 210511cbce2SChristoph Hellwig 211511cbce2SChristoph Hellwig static __init int irq_poll_setup(void) 212511cbce2SChristoph Hellwig { 213511cbce2SChristoph Hellwig int i; 214511cbce2SChristoph Hellwig 215511cbce2SChristoph Hellwig for_each_possible_cpu(i) 216511cbce2SChristoph Hellwig INIT_LIST_HEAD(&per_cpu(blk_cpu_iopoll, i)); 217511cbce2SChristoph Hellwig 218511cbce2SChristoph Hellwig open_softirq(IRQ_POLL_SOFTIRQ, irq_poll_softirq); 219511cbce2SChristoph Hellwig register_hotcpu_notifier(&irq_poll_cpu_notifier); 220511cbce2SChristoph Hellwig return 0; 221511cbce2SChristoph Hellwig } 222511cbce2SChristoph Hellwig subsys_initcall(irq_poll_setup); 223