1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * IRQ offload/bypass manager 4 * 5 * Copyright (C) 2015 Red Hat, Inc. 6 * Copyright (c) 2015 Linaro Ltd. 7 * 8 * Various virtualization hardware acceleration techniques allow bypassing or 9 * offloading interrupts received from devices around the host kernel. Posted 10 * Interrupts on Intel VT-d systems can allow interrupts to be received 11 * directly by a virtual machine. ARM IRQ Forwarding allows forwarded physical 12 * interrupts to be directly deactivated by the guest. This manager allows 13 * interrupt producers and consumers to find each other to enable this sort of 14 * bypass. 15 */ 16 17 #include <linux/irqbypass.h> 18 #include <linux/list.h> 19 #include <linux/module.h> 20 #include <linux/mutex.h> 21 22 MODULE_LICENSE("GPL v2"); 23 MODULE_DESCRIPTION("IRQ bypass manager utility module"); 24 25 static DEFINE_XARRAY(producers); 26 static DEFINE_XARRAY(consumers); 27 static DEFINE_MUTEX(lock); 28 29 /* @lock must be held when calling connect */ 30 static int __connect(struct irq_bypass_producer *prod, 31 struct irq_bypass_consumer *cons) 32 { 33 int ret = 0; 34 35 if (prod->stop) 36 prod->stop(prod); 37 if (cons->stop) 38 cons->stop(cons); 39 40 if (prod->add_consumer) 41 ret = prod->add_consumer(prod, cons); 42 43 if (!ret) { 44 ret = cons->add_producer(cons, prod); 45 if (ret && prod->del_consumer) 46 prod->del_consumer(prod, cons); 47 } 48 49 if (cons->start) 50 cons->start(cons); 51 if (prod->start) 52 prod->start(prod); 53 54 if (!ret) { 55 prod->consumer = cons; 56 cons->producer = prod; 57 } 58 return ret; 59 } 60 61 /* @lock must be held when calling disconnect */ 62 static void __disconnect(struct irq_bypass_producer *prod, 63 struct irq_bypass_consumer *cons) 64 { 65 if (prod->stop) 66 prod->stop(prod); 67 if (cons->stop) 68 cons->stop(cons); 69 70 cons->del_producer(cons, prod); 71 72 if (prod->del_consumer) 73 prod->del_consumer(prod, cons); 74 75 if (cons->start) 76 cons->start(cons); 77 if (prod->start) 78 prod->start(prod); 79 80 prod->consumer = NULL; 81 cons->producer = NULL; 82 } 83 84 /** 85 * irq_bypass_register_producer - register IRQ bypass producer 86 * @producer: pointer to producer structure 87 * @eventfd: pointer to the eventfd context associated with the producer 88 * @irq: Linux IRQ number of the underlying producer device 89 * 90 * Add the provided IRQ producer to the set of producers and connect with the 91 * consumer with a matching eventfd, if one exists. 92 */ 93 int irq_bypass_register_producer(struct irq_bypass_producer *producer, 94 struct eventfd_ctx *eventfd, int irq) 95 { 96 unsigned long index = (unsigned long)eventfd; 97 struct irq_bypass_consumer *consumer; 98 int ret; 99 100 if (WARN_ON_ONCE(producer->eventfd)) 101 return -EINVAL; 102 103 producer->irq = irq; 104 105 guard(mutex)(&lock); 106 107 ret = xa_insert(&producers, index, producer, GFP_KERNEL); 108 if (ret) 109 return ret; 110 111 consumer = xa_load(&consumers, index); 112 if (consumer) { 113 ret = __connect(producer, consumer); 114 if (ret) { 115 WARN_ON_ONCE(xa_erase(&producers, index) != producer); 116 return ret; 117 } 118 } 119 120 producer->eventfd = eventfd; 121 return 0; 122 } 123 EXPORT_SYMBOL_GPL(irq_bypass_register_producer); 124 125 /** 126 * irq_bypass_unregister_producer - unregister IRQ bypass producer 127 * @producer: pointer to producer structure 128 * 129 * Remove a previously registered IRQ producer (note, it's safe to call this 130 * even if registration was unsuccessful). Disconnect from the associated 131 * consumer, if one exists. 132 */ 133 void irq_bypass_unregister_producer(struct irq_bypass_producer *producer) 134 { 135 unsigned long index = (unsigned long)producer->eventfd; 136 137 if (!producer->eventfd) 138 return; 139 140 guard(mutex)(&lock); 141 142 if (producer->consumer) 143 __disconnect(producer, producer->consumer); 144 145 WARN_ON_ONCE(xa_erase(&producers, index) != producer); 146 producer->eventfd = NULL; 147 } 148 EXPORT_SYMBOL_GPL(irq_bypass_unregister_producer); 149 150 /** 151 * irq_bypass_register_consumer - register IRQ bypass consumer 152 * @consumer: pointer to consumer structure 153 * @eventfd: pointer to the eventfd context associated with the consumer 154 * 155 * Add the provided IRQ consumer to the set of consumers and connect with the 156 * producer with a matching eventfd, if one exists. 157 */ 158 int irq_bypass_register_consumer(struct irq_bypass_consumer *consumer, 159 struct eventfd_ctx *eventfd) 160 { 161 unsigned long index = (unsigned long)eventfd; 162 struct irq_bypass_producer *producer; 163 int ret; 164 165 if (WARN_ON_ONCE(consumer->eventfd)) 166 return -EINVAL; 167 168 if (!consumer->add_producer || !consumer->del_producer) 169 return -EINVAL; 170 171 guard(mutex)(&lock); 172 173 ret = xa_insert(&consumers, index, consumer, GFP_KERNEL); 174 if (ret) 175 return ret; 176 177 producer = xa_load(&producers, index); 178 if (producer) { 179 ret = __connect(producer, consumer); 180 if (ret) { 181 WARN_ON_ONCE(xa_erase(&consumers, index) != consumer); 182 return ret; 183 } 184 } 185 186 consumer->eventfd = eventfd; 187 return 0; 188 } 189 EXPORT_SYMBOL_GPL(irq_bypass_register_consumer); 190 191 /** 192 * irq_bypass_unregister_consumer - unregister IRQ bypass consumer 193 * @consumer: pointer to consumer structure 194 * 195 * Remove a previously registered IRQ consumer (note, it's safe to call this 196 * even if registration was unsuccessful). Disconnect from the associated 197 * producer, if one exists. 198 */ 199 void irq_bypass_unregister_consumer(struct irq_bypass_consumer *consumer) 200 { 201 unsigned long index = (unsigned long)consumer->eventfd; 202 203 if (!consumer->eventfd) 204 return; 205 206 guard(mutex)(&lock); 207 208 if (consumer->producer) 209 __disconnect(consumer->producer, consumer); 210 211 WARN_ON_ONCE(xa_erase(&consumers, index) != consumer); 212 consumer->eventfd = NULL; 213 } 214 EXPORT_SYMBOL_GPL(irq_bypass_unregister_consumer); 215