1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2f73f8173SAlex Williamson /*
3f73f8173SAlex Williamson * IRQ offload/bypass manager
4f73f8173SAlex Williamson *
5f73f8173SAlex Williamson * Copyright (C) 2015 Red Hat, Inc.
6f73f8173SAlex Williamson * Copyright (c) 2015 Linaro Ltd.
7f73f8173SAlex Williamson *
8f73f8173SAlex Williamson * Various virtualization hardware acceleration techniques allow bypassing or
9f73f8173SAlex Williamson * offloading interrupts received from devices around the host kernel. Posted
10f73f8173SAlex Williamson * Interrupts on Intel VT-d systems can allow interrupts to be received
11f73f8173SAlex Williamson * directly by a virtual machine. ARM IRQ Forwarding allows forwarded physical
12f73f8173SAlex Williamson * interrupts to be directly deactivated by the guest. This manager allows
13f73f8173SAlex Williamson * interrupt producers and consumers to find each other to enable this sort of
14f73f8173SAlex Williamson * bypass.
15f73f8173SAlex Williamson */
16f73f8173SAlex Williamson
17f73f8173SAlex Williamson #include <linux/irqbypass.h>
18f73f8173SAlex Williamson #include <linux/list.h>
19f73f8173SAlex Williamson #include <linux/module.h>
20f73f8173SAlex Williamson #include <linux/mutex.h>
21f73f8173SAlex Williamson
22f73f8173SAlex Williamson MODULE_LICENSE("GPL v2");
23f73f8173SAlex Williamson MODULE_DESCRIPTION("IRQ bypass manager utility module");
24f73f8173SAlex Williamson
25f73f8173SAlex Williamson static LIST_HEAD(producers);
26f73f8173SAlex Williamson static LIST_HEAD(consumers);
27f73f8173SAlex Williamson static DEFINE_MUTEX(lock);
28f73f8173SAlex Williamson
29f73f8173SAlex Williamson /* @lock must be held when calling connect */
__connect(struct irq_bypass_producer * prod,struct irq_bypass_consumer * cons)30f73f8173SAlex Williamson static int __connect(struct irq_bypass_producer *prod,
31f73f8173SAlex Williamson struct irq_bypass_consumer *cons)
32f73f8173SAlex Williamson {
33f73f8173SAlex Williamson int ret = 0;
34f73f8173SAlex Williamson
35f73f8173SAlex Williamson if (prod->stop)
36f73f8173SAlex Williamson prod->stop(prod);
37f73f8173SAlex Williamson if (cons->stop)
38f73f8173SAlex Williamson cons->stop(cons);
39f73f8173SAlex Williamson
40f73f8173SAlex Williamson if (prod->add_consumer)
41f73f8173SAlex Williamson ret = prod->add_consumer(prod, cons);
42f73f8173SAlex Williamson
43*e44b49f6SZhu Lingshan if (!ret) {
44f73f8173SAlex Williamson ret = cons->add_producer(cons, prod);
45*e44b49f6SZhu Lingshan if (ret && prod->del_consumer)
46*e44b49f6SZhu Lingshan prod->del_consumer(prod, cons);
47*e44b49f6SZhu Lingshan }
48f73f8173SAlex Williamson
49f73f8173SAlex Williamson if (cons->start)
50f73f8173SAlex Williamson cons->start(cons);
51f73f8173SAlex Williamson if (prod->start)
52f73f8173SAlex Williamson prod->start(prod);
53*e44b49f6SZhu Lingshan
54f73f8173SAlex Williamson return ret;
55f73f8173SAlex Williamson }
56f73f8173SAlex Williamson
57f73f8173SAlex Williamson /* @lock must be held when calling disconnect */
__disconnect(struct irq_bypass_producer * prod,struct irq_bypass_consumer * cons)58f73f8173SAlex Williamson static void __disconnect(struct irq_bypass_producer *prod,
59f73f8173SAlex Williamson struct irq_bypass_consumer *cons)
60f73f8173SAlex Williamson {
61f73f8173SAlex Williamson if (prod->stop)
62f73f8173SAlex Williamson prod->stop(prod);
63f73f8173SAlex Williamson if (cons->stop)
64f73f8173SAlex Williamson cons->stop(cons);
65f73f8173SAlex Williamson
66f73f8173SAlex Williamson cons->del_producer(cons, prod);
67f73f8173SAlex Williamson
68f73f8173SAlex Williamson if (prod->del_consumer)
69f73f8173SAlex Williamson prod->del_consumer(prod, cons);
70f73f8173SAlex Williamson
71f73f8173SAlex Williamson if (cons->start)
72f73f8173SAlex Williamson cons->start(cons);
73f73f8173SAlex Williamson if (prod->start)
74f73f8173SAlex Williamson prod->start(prod);
75f73f8173SAlex Williamson }
76f73f8173SAlex Williamson
77f73f8173SAlex Williamson /**
78f73f8173SAlex Williamson * irq_bypass_register_producer - register IRQ bypass producer
79f73f8173SAlex Williamson * @producer: pointer to producer structure
80f73f8173SAlex Williamson *
81f73f8173SAlex Williamson * Add the provided IRQ producer to the list of producers and connect
82f73f8173SAlex Williamson * with any matching token found on the IRQ consumers list.
83f73f8173SAlex Williamson */
irq_bypass_register_producer(struct irq_bypass_producer * producer)84f73f8173SAlex Williamson int irq_bypass_register_producer(struct irq_bypass_producer *producer)
85f73f8173SAlex Williamson {
86f73f8173SAlex Williamson struct irq_bypass_producer *tmp;
87f73f8173SAlex Williamson struct irq_bypass_consumer *consumer;
88bbfdafa8SMiaohe Lin int ret;
89f73f8173SAlex Williamson
90b52f3ed0SAlex Williamson if (!producer->token)
91b52f3ed0SAlex Williamson return -EINVAL;
92b52f3ed0SAlex Williamson
93f73f8173SAlex Williamson might_sleep();
94f73f8173SAlex Williamson
95f73f8173SAlex Williamson if (!try_module_get(THIS_MODULE))
96f73f8173SAlex Williamson return -ENODEV;
97f73f8173SAlex Williamson
98f73f8173SAlex Williamson mutex_lock(&lock);
99f73f8173SAlex Williamson
100f73f8173SAlex Williamson list_for_each_entry(tmp, &producers, node) {
101f73f8173SAlex Williamson if (tmp->token == producer->token) {
102bbfdafa8SMiaohe Lin ret = -EBUSY;
103bbfdafa8SMiaohe Lin goto out_err;
104f73f8173SAlex Williamson }
105f73f8173SAlex Williamson }
106f73f8173SAlex Williamson
107f73f8173SAlex Williamson list_for_each_entry(consumer, &consumers, node) {
108f73f8173SAlex Williamson if (consumer->token == producer->token) {
109bbfdafa8SMiaohe Lin ret = __connect(producer, consumer);
110bbfdafa8SMiaohe Lin if (ret)
111bbfdafa8SMiaohe Lin goto out_err;
112f73f8173SAlex Williamson break;
113f73f8173SAlex Williamson }
114f73f8173SAlex Williamson }
115f73f8173SAlex Williamson
116f73f8173SAlex Williamson list_add(&producer->node, &producers);
117f73f8173SAlex Williamson
118f73f8173SAlex Williamson mutex_unlock(&lock);
119f73f8173SAlex Williamson
120f73f8173SAlex Williamson return 0;
121bbfdafa8SMiaohe Lin out_err:
122bbfdafa8SMiaohe Lin mutex_unlock(&lock);
123bbfdafa8SMiaohe Lin module_put(THIS_MODULE);
124bbfdafa8SMiaohe Lin return ret;
125f73f8173SAlex Williamson }
126f73f8173SAlex Williamson EXPORT_SYMBOL_GPL(irq_bypass_register_producer);
127f73f8173SAlex Williamson
128f73f8173SAlex Williamson /**
129f73f8173SAlex Williamson * irq_bypass_unregister_producer - unregister IRQ bypass producer
130f73f8173SAlex Williamson * @producer: pointer to producer structure
131f73f8173SAlex Williamson *
132f73f8173SAlex Williamson * Remove a previously registered IRQ producer from the list of producers
133f73f8173SAlex Williamson * and disconnect it from any connected IRQ consumer.
134f73f8173SAlex Williamson */
irq_bypass_unregister_producer(struct irq_bypass_producer * producer)135f73f8173SAlex Williamson void irq_bypass_unregister_producer(struct irq_bypass_producer *producer)
136f73f8173SAlex Williamson {
137f73f8173SAlex Williamson struct irq_bypass_producer *tmp;
138f73f8173SAlex Williamson struct irq_bypass_consumer *consumer;
139f73f8173SAlex Williamson
140b52f3ed0SAlex Williamson if (!producer->token)
141b52f3ed0SAlex Williamson return;
142b52f3ed0SAlex Williamson
143f73f8173SAlex Williamson might_sleep();
144f73f8173SAlex Williamson
145f73f8173SAlex Williamson if (!try_module_get(THIS_MODULE))
146f73f8173SAlex Williamson return; /* nothing in the list anyway */
147f73f8173SAlex Williamson
148f73f8173SAlex Williamson mutex_lock(&lock);
149f73f8173SAlex Williamson
150f73f8173SAlex Williamson list_for_each_entry(tmp, &producers, node) {
151f73f8173SAlex Williamson if (tmp->token != producer->token)
152f73f8173SAlex Williamson continue;
153f73f8173SAlex Williamson
154f73f8173SAlex Williamson list_for_each_entry(consumer, &consumers, node) {
155f73f8173SAlex Williamson if (consumer->token == producer->token) {
156f73f8173SAlex Williamson __disconnect(producer, consumer);
157f73f8173SAlex Williamson break;
158f73f8173SAlex Williamson }
159f73f8173SAlex Williamson }
160f73f8173SAlex Williamson
161f73f8173SAlex Williamson list_del(&producer->node);
162f73f8173SAlex Williamson module_put(THIS_MODULE);
163f73f8173SAlex Williamson break;
164f73f8173SAlex Williamson }
165f73f8173SAlex Williamson
166f73f8173SAlex Williamson mutex_unlock(&lock);
167f73f8173SAlex Williamson
168f73f8173SAlex Williamson module_put(THIS_MODULE);
169f73f8173SAlex Williamson }
170f73f8173SAlex Williamson EXPORT_SYMBOL_GPL(irq_bypass_unregister_producer);
171f73f8173SAlex Williamson
172f73f8173SAlex Williamson /**
173f73f8173SAlex Williamson * irq_bypass_register_consumer - register IRQ bypass consumer
174f73f8173SAlex Williamson * @consumer: pointer to consumer structure
175f73f8173SAlex Williamson *
176f73f8173SAlex Williamson * Add the provided IRQ consumer to the list of consumers and connect
177f73f8173SAlex Williamson * with any matching token found on the IRQ producer list.
178f73f8173SAlex Williamson */
irq_bypass_register_consumer(struct irq_bypass_consumer * consumer)179f73f8173SAlex Williamson int irq_bypass_register_consumer(struct irq_bypass_consumer *consumer)
180f73f8173SAlex Williamson {
181f73f8173SAlex Williamson struct irq_bypass_consumer *tmp;
182f73f8173SAlex Williamson struct irq_bypass_producer *producer;
1838262fe85SMiaohe Lin int ret;
184f73f8173SAlex Williamson
185b52f3ed0SAlex Williamson if (!consumer->token ||
186b52f3ed0SAlex Williamson !consumer->add_producer || !consumer->del_producer)
187f73f8173SAlex Williamson return -EINVAL;
188f73f8173SAlex Williamson
189f73f8173SAlex Williamson might_sleep();
190f73f8173SAlex Williamson
191f73f8173SAlex Williamson if (!try_module_get(THIS_MODULE))
192f73f8173SAlex Williamson return -ENODEV;
193f73f8173SAlex Williamson
194f73f8173SAlex Williamson mutex_lock(&lock);
195f73f8173SAlex Williamson
196f73f8173SAlex Williamson list_for_each_entry(tmp, &consumers, node) {
1974f3dbdf4SWanpeng Li if (tmp->token == consumer->token || tmp == consumer) {
1988262fe85SMiaohe Lin ret = -EBUSY;
1998262fe85SMiaohe Lin goto out_err;
200f73f8173SAlex Williamson }
201f73f8173SAlex Williamson }
202f73f8173SAlex Williamson
203f73f8173SAlex Williamson list_for_each_entry(producer, &producers, node) {
204f73f8173SAlex Williamson if (producer->token == consumer->token) {
2058262fe85SMiaohe Lin ret = __connect(producer, consumer);
2068262fe85SMiaohe Lin if (ret)
2078262fe85SMiaohe Lin goto out_err;
208f73f8173SAlex Williamson break;
209f73f8173SAlex Williamson }
210f73f8173SAlex Williamson }
211f73f8173SAlex Williamson
212f73f8173SAlex Williamson list_add(&consumer->node, &consumers);
213f73f8173SAlex Williamson
214f73f8173SAlex Williamson mutex_unlock(&lock);
215f73f8173SAlex Williamson
216f73f8173SAlex Williamson return 0;
2178262fe85SMiaohe Lin out_err:
2188262fe85SMiaohe Lin mutex_unlock(&lock);
2198262fe85SMiaohe Lin module_put(THIS_MODULE);
2208262fe85SMiaohe Lin return ret;
221f73f8173SAlex Williamson }
222f73f8173SAlex Williamson EXPORT_SYMBOL_GPL(irq_bypass_register_consumer);
223f73f8173SAlex Williamson
224f73f8173SAlex Williamson /**
225f73f8173SAlex Williamson * irq_bypass_unregister_consumer - unregister IRQ bypass consumer
226f73f8173SAlex Williamson * @consumer: pointer to consumer structure
227f73f8173SAlex Williamson *
228f73f8173SAlex Williamson * Remove a previously registered IRQ consumer from the list of consumers
229f73f8173SAlex Williamson * and disconnect it from any connected IRQ producer.
230f73f8173SAlex Williamson */
irq_bypass_unregister_consumer(struct irq_bypass_consumer * consumer)231f73f8173SAlex Williamson void irq_bypass_unregister_consumer(struct irq_bypass_consumer *consumer)
232f73f8173SAlex Williamson {
233f73f8173SAlex Williamson struct irq_bypass_consumer *tmp;
234f73f8173SAlex Williamson struct irq_bypass_producer *producer;
235f73f8173SAlex Williamson
236b52f3ed0SAlex Williamson if (!consumer->token)
237b52f3ed0SAlex Williamson return;
238b52f3ed0SAlex Williamson
239f73f8173SAlex Williamson might_sleep();
240f73f8173SAlex Williamson
241f73f8173SAlex Williamson if (!try_module_get(THIS_MODULE))
242f73f8173SAlex Williamson return; /* nothing in the list anyway */
243f73f8173SAlex Williamson
244f73f8173SAlex Williamson mutex_lock(&lock);
245f73f8173SAlex Williamson
246f73f8173SAlex Williamson list_for_each_entry(tmp, &consumers, node) {
2474f3dbdf4SWanpeng Li if (tmp != consumer)
248f73f8173SAlex Williamson continue;
249f73f8173SAlex Williamson
250f73f8173SAlex Williamson list_for_each_entry(producer, &producers, node) {
251f73f8173SAlex Williamson if (producer->token == consumer->token) {
252f73f8173SAlex Williamson __disconnect(producer, consumer);
253f73f8173SAlex Williamson break;
254f73f8173SAlex Williamson }
255f73f8173SAlex Williamson }
256f73f8173SAlex Williamson
257f73f8173SAlex Williamson list_del(&consumer->node);
258f73f8173SAlex Williamson module_put(THIS_MODULE);
259f73f8173SAlex Williamson break;
260f73f8173SAlex Williamson }
261f73f8173SAlex Williamson
262f73f8173SAlex Williamson mutex_unlock(&lock);
263f73f8173SAlex Williamson
264f73f8173SAlex Williamson module_put(THIS_MODULE);
265f73f8173SAlex Williamson }
266f73f8173SAlex Williamson EXPORT_SYMBOL_GPL(irq_bypass_unregister_consumer);
267