xref: /linux/virt/lib/irqbypass.c (revision 63eb28bb1402891b1ad2be02a530f29a9dd7f1cd)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * IRQ offload/bypass manager
4  *
5  * Copyright (C) 2015 Red Hat, Inc.
6  * Copyright (c) 2015 Linaro Ltd.
7  *
8  * Various virtualization hardware acceleration techniques allow bypassing or
9  * offloading interrupts received from devices around the host kernel.  Posted
10  * Interrupts on Intel VT-d systems can allow interrupts to be received
11  * directly by a virtual machine.  ARM IRQ Forwarding allows forwarded physical
12  * interrupts to be directly deactivated by the guest.  This manager allows
13  * interrupt producers and consumers to find each other to enable this sort of
14  * bypass.
15  */
16 
17 #include <linux/irqbypass.h>
18 #include <linux/list.h>
19 #include <linux/module.h>
20 #include <linux/mutex.h>
21 
22 MODULE_LICENSE("GPL v2");
23 MODULE_DESCRIPTION("IRQ bypass manager utility module");
24 
25 static DEFINE_XARRAY(producers);
26 static DEFINE_XARRAY(consumers);
27 static DEFINE_MUTEX(lock);
28 
29 /* @lock must be held when calling connect */
__connect(struct irq_bypass_producer * prod,struct irq_bypass_consumer * cons)30 static int __connect(struct irq_bypass_producer *prod,
31 		     struct irq_bypass_consumer *cons)
32 {
33 	int ret = 0;
34 
35 	if (prod->stop)
36 		prod->stop(prod);
37 	if (cons->stop)
38 		cons->stop(cons);
39 
40 	if (prod->add_consumer)
41 		ret = prod->add_consumer(prod, cons);
42 
43 	if (!ret) {
44 		ret = cons->add_producer(cons, prod);
45 		if (ret && prod->del_consumer)
46 			prod->del_consumer(prod, cons);
47 	}
48 
49 	if (cons->start)
50 		cons->start(cons);
51 	if (prod->start)
52 		prod->start(prod);
53 
54 	if (!ret) {
55 		prod->consumer = cons;
56 		cons->producer = prod;
57 	}
58 	return ret;
59 }
60 
61 /* @lock must be held when calling disconnect */
__disconnect(struct irq_bypass_producer * prod,struct irq_bypass_consumer * cons)62 static void __disconnect(struct irq_bypass_producer *prod,
63 			 struct irq_bypass_consumer *cons)
64 {
65 	if (prod->stop)
66 		prod->stop(prod);
67 	if (cons->stop)
68 		cons->stop(cons);
69 
70 	cons->del_producer(cons, prod);
71 
72 	if (prod->del_consumer)
73 		prod->del_consumer(prod, cons);
74 
75 	if (cons->start)
76 		cons->start(cons);
77 	if (prod->start)
78 		prod->start(prod);
79 
80 	prod->consumer = NULL;
81 	cons->producer = NULL;
82 }
83 
84 /**
85  * irq_bypass_register_producer - register IRQ bypass producer
86  * @producer: pointer to producer structure
87  * @eventfd: pointer to the eventfd context associated with the producer
88  * @irq: Linux IRQ number of the underlying producer device
89  *
90  * Add the provided IRQ producer to the set of producers and connect with the
91  * consumer with a matching eventfd, if one exists.
92  */
irq_bypass_register_producer(struct irq_bypass_producer * producer,struct eventfd_ctx * eventfd,int irq)93 int irq_bypass_register_producer(struct irq_bypass_producer *producer,
94 				 struct eventfd_ctx *eventfd, int irq)
95 {
96 	unsigned long index = (unsigned long)eventfd;
97 	struct irq_bypass_consumer *consumer;
98 	int ret;
99 
100 	if (WARN_ON_ONCE(producer->eventfd))
101 		return -EINVAL;
102 
103 	producer->irq = irq;
104 
105 	guard(mutex)(&lock);
106 
107 	ret = xa_insert(&producers, index, producer, GFP_KERNEL);
108 	if (ret)
109 		return ret;
110 
111 	consumer = xa_load(&consumers, index);
112 	if (consumer) {
113 		ret = __connect(producer, consumer);
114 		if (ret) {
115 			WARN_ON_ONCE(xa_erase(&producers, index) != producer);
116 			return ret;
117 		}
118 	}
119 
120 	producer->eventfd = eventfd;
121 	return 0;
122 }
123 EXPORT_SYMBOL_GPL(irq_bypass_register_producer);
124 
125 /**
126  * irq_bypass_unregister_producer - unregister IRQ bypass producer
127  * @producer: pointer to producer structure
128  *
129  * Remove a previously registered IRQ producer (note, it's safe to call this
130  * even if registration was unsuccessful).  Disconnect from the associated
131  * consumer, if one exists.
132  */
irq_bypass_unregister_producer(struct irq_bypass_producer * producer)133 void irq_bypass_unregister_producer(struct irq_bypass_producer *producer)
134 {
135 	unsigned long index = (unsigned long)producer->eventfd;
136 
137 	if (!producer->eventfd)
138 		return;
139 
140 	guard(mutex)(&lock);
141 
142 	if (producer->consumer)
143 		__disconnect(producer, producer->consumer);
144 
145 	WARN_ON_ONCE(xa_erase(&producers, index) != producer);
146 	producer->eventfd = NULL;
147 }
148 EXPORT_SYMBOL_GPL(irq_bypass_unregister_producer);
149 
150 /**
151  * irq_bypass_register_consumer - register IRQ bypass consumer
152  * @consumer: pointer to consumer structure
153  * @eventfd: pointer to the eventfd context associated with the consumer
154  *
155  * Add the provided IRQ consumer to the set of consumers and connect with the
156  * producer with a matching eventfd, if one exists.
157  */
irq_bypass_register_consumer(struct irq_bypass_consumer * consumer,struct eventfd_ctx * eventfd)158 int irq_bypass_register_consumer(struct irq_bypass_consumer *consumer,
159 				 struct eventfd_ctx *eventfd)
160 {
161 	unsigned long index = (unsigned long)eventfd;
162 	struct irq_bypass_producer *producer;
163 	int ret;
164 
165 	if (WARN_ON_ONCE(consumer->eventfd))
166 		return -EINVAL;
167 
168 	if (!consumer->add_producer || !consumer->del_producer)
169 		return -EINVAL;
170 
171 	guard(mutex)(&lock);
172 
173 	ret = xa_insert(&consumers, index, consumer, GFP_KERNEL);
174 	if (ret)
175 		return ret;
176 
177 	producer = xa_load(&producers, index);
178 	if (producer) {
179 		ret = __connect(producer, consumer);
180 		if (ret) {
181 			WARN_ON_ONCE(xa_erase(&consumers, index) != consumer);
182 			return ret;
183 		}
184 	}
185 
186 	consumer->eventfd = eventfd;
187 	return 0;
188 }
189 EXPORT_SYMBOL_GPL(irq_bypass_register_consumer);
190 
191 /**
192  * irq_bypass_unregister_consumer - unregister IRQ bypass consumer
193  * @consumer: pointer to consumer structure
194  *
195  * Remove a previously registered IRQ consumer (note, it's safe to call this
196  * even if registration was unsuccessful).  Disconnect from the associated
197  * producer, if one exists.
198  */
irq_bypass_unregister_consumer(struct irq_bypass_consumer * consumer)199 void irq_bypass_unregister_consumer(struct irq_bypass_consumer *consumer)
200 {
201 	unsigned long index = (unsigned long)consumer->eventfd;
202 
203 	if (!consumer->eventfd)
204 		return;
205 
206 	guard(mutex)(&lock);
207 
208 	if (consumer->producer)
209 		__disconnect(consumer->producer, consumer);
210 
211 	WARN_ON_ONCE(xa_erase(&consumers, index) != consumer);
212 	consumer->eventfd = NULL;
213 }
214 EXPORT_SYMBOL_GPL(irq_bypass_unregister_consumer);
215