1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Support for adapter interruptions 4 * 5 * Copyright IBM Corp. 1999, 2007 6 * Author(s): Ingo Adlung <adlung@de.ibm.com> 7 * Cornelia Huck <cornelia.huck@de.ibm.com> 8 * Arnd Bergmann <arndb@de.ibm.com> 9 * Peter Oberparleiter <peter.oberparleiter@de.ibm.com> 10 */ 11 12 #include <linux/init.h> 13 #include <linux/irq.h> 14 #include <linux/kernel_stat.h> 15 #include <linux/module.h> 16 #include <linux/mutex.h> 17 #include <linux/rculist.h> 18 #include <linux/slab.h> 19 #include <linux/dmapool.h> 20 21 #include <asm/airq.h> 22 #include <asm/isc.h> 23 #include <asm/cio.h> 24 25 #include "cio.h" 26 #include "cio_debug.h" 27 #include "ioasm.h" 28 29 static DEFINE_SPINLOCK(airq_lists_lock); 30 static struct hlist_head airq_lists[MAX_ISC+1]; 31 32 static struct dma_pool *airq_iv_cache; 33 34 /** 35 * register_adapter_interrupt() - register adapter interrupt handler 36 * @airq: pointer to adapter interrupt descriptor 37 * 38 * Returns 0 on success, or -EINVAL. 39 */ 40 int register_adapter_interrupt(struct airq_struct *airq) 41 { 42 char dbf_txt[32]; 43 44 if (!airq->handler || airq->isc > MAX_ISC) 45 return -EINVAL; 46 if (!airq->lsi_ptr) { 47 airq->lsi_ptr = cio_dma_zalloc(1); 48 if (!airq->lsi_ptr) 49 return -ENOMEM; 50 airq->flags |= AIRQ_PTR_ALLOCATED; 51 } 52 snprintf(dbf_txt, sizeof(dbf_txt), "rairq:%p", airq); 53 CIO_TRACE_EVENT(4, dbf_txt); 54 isc_register(airq->isc); 55 spin_lock(&airq_lists_lock); 56 hlist_add_head_rcu(&airq->list, &airq_lists[airq->isc]); 57 spin_unlock(&airq_lists_lock); 58 return 0; 59 } 60 EXPORT_SYMBOL(register_adapter_interrupt); 61 62 /** 63 * unregister_adapter_interrupt - unregister adapter interrupt handler 64 * @airq: pointer to adapter interrupt descriptor 65 */ 66 void unregister_adapter_interrupt(struct airq_struct *airq) 67 { 68 char dbf_txt[32]; 69 70 if (hlist_unhashed(&airq->list)) 71 return; 72 snprintf(dbf_txt, sizeof(dbf_txt), "urairq:%p", airq); 73 CIO_TRACE_EVENT(4, dbf_txt); 74 spin_lock(&airq_lists_lock); 75 hlist_del_rcu(&airq->list); 76 spin_unlock(&airq_lists_lock); 77 synchronize_rcu(); 78 isc_unregister(airq->isc); 79 if (airq->flags & AIRQ_PTR_ALLOCATED) { 80 cio_dma_free(airq->lsi_ptr, 1); 81 airq->lsi_ptr = NULL; 82 airq->flags &= ~AIRQ_PTR_ALLOCATED; 83 } 84 } 85 EXPORT_SYMBOL(unregister_adapter_interrupt); 86 87 static irqreturn_t do_airq_interrupt(int irq, void *dummy) 88 { 89 struct tpi_info *tpi_info; 90 struct airq_struct *airq; 91 struct hlist_head *head; 92 93 tpi_info = &get_irq_regs()->tpi_info; 94 trace_s390_cio_adapter_int(tpi_info); 95 head = &airq_lists[tpi_info->isc]; 96 rcu_read_lock(); 97 hlist_for_each_entry_rcu(airq, head, list) 98 if (*airq->lsi_ptr != 0) 99 airq->handler(airq, tpi_info); 100 rcu_read_unlock(); 101 102 return IRQ_HANDLED; 103 } 104 105 void __init init_airq_interrupts(void) 106 { 107 irq_set_chip_and_handler(THIN_INTERRUPT, 108 &dummy_irq_chip, handle_percpu_irq); 109 if (request_irq(THIN_INTERRUPT, do_airq_interrupt, 0, "AIO", NULL)) 110 panic("Failed to register AIO interrupt\n"); 111 } 112 113 static inline unsigned long iv_size(unsigned long bits) 114 { 115 return BITS_TO_LONGS(bits) * sizeof(unsigned long); 116 } 117 118 /** 119 * airq_iv_create - create an interrupt vector 120 * @bits: number of bits in the interrupt vector 121 * @flags: allocation flags 122 * @vec: pointer to pinned guest memory if AIRQ_IV_GUESTVEC 123 * 124 * Returns a pointer to an interrupt vector structure 125 */ 126 struct airq_iv *airq_iv_create(unsigned long bits, unsigned long flags, 127 unsigned long *vec) 128 { 129 struct airq_iv *iv; 130 unsigned long size; 131 132 iv = kzalloc(sizeof(*iv), GFP_KERNEL); 133 if (!iv) 134 goto out; 135 iv->bits = bits; 136 iv->flags = flags; 137 size = iv_size(bits); 138 139 if (flags & AIRQ_IV_CACHELINE) { 140 if ((cache_line_size() * BITS_PER_BYTE) < bits 141 || !airq_iv_cache) 142 goto out_free; 143 144 iv->vector = dma_pool_zalloc(airq_iv_cache, GFP_KERNEL, 145 &iv->vector_dma); 146 if (!iv->vector) 147 goto out_free; 148 } else if (flags & AIRQ_IV_GUESTVEC) { 149 iv->vector = vec; 150 } else { 151 iv->vector = cio_dma_zalloc(size); 152 if (!iv->vector) 153 goto out_free; 154 } 155 if (flags & AIRQ_IV_ALLOC) { 156 iv->avail = kmalloc(size, GFP_KERNEL); 157 if (!iv->avail) 158 goto out_free; 159 memset(iv->avail, 0xff, size); 160 iv->end = 0; 161 } else 162 iv->end = bits; 163 if (flags & AIRQ_IV_BITLOCK) { 164 iv->bitlock = kzalloc(size, GFP_KERNEL); 165 if (!iv->bitlock) 166 goto out_free; 167 } 168 if (flags & AIRQ_IV_PTR) { 169 size = bits * sizeof(unsigned long); 170 iv->ptr = kzalloc(size, GFP_KERNEL); 171 if (!iv->ptr) 172 goto out_free; 173 } 174 if (flags & AIRQ_IV_DATA) { 175 size = bits * sizeof(unsigned int); 176 iv->data = kzalloc(size, GFP_KERNEL); 177 if (!iv->data) 178 goto out_free; 179 } 180 spin_lock_init(&iv->lock); 181 return iv; 182 183 out_free: 184 kfree(iv->ptr); 185 kfree(iv->bitlock); 186 kfree(iv->avail); 187 if (iv->flags & AIRQ_IV_CACHELINE && iv->vector) 188 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma); 189 else if (!(iv->flags & AIRQ_IV_GUESTVEC)) 190 cio_dma_free(iv->vector, size); 191 kfree(iv); 192 out: 193 return NULL; 194 } 195 EXPORT_SYMBOL(airq_iv_create); 196 197 /** 198 * airq_iv_release - release an interrupt vector 199 * @iv: pointer to interrupt vector structure 200 */ 201 void airq_iv_release(struct airq_iv *iv) 202 { 203 kfree(iv->data); 204 kfree(iv->ptr); 205 kfree(iv->bitlock); 206 if (iv->flags & AIRQ_IV_CACHELINE) 207 dma_pool_free(airq_iv_cache, iv->vector, iv->vector_dma); 208 else if (!(iv->flags & AIRQ_IV_GUESTVEC)) 209 cio_dma_free(iv->vector, iv_size(iv->bits)); 210 kfree(iv->avail); 211 kfree(iv); 212 } 213 EXPORT_SYMBOL(airq_iv_release); 214 215 /** 216 * airq_iv_alloc - allocate irq bits from an interrupt vector 217 * @iv: pointer to an interrupt vector structure 218 * @num: number of consecutive irq bits to allocate 219 * 220 * Returns the bit number of the first irq in the allocated block of irqs, 221 * or -1UL if no bit is available or the AIRQ_IV_ALLOC flag has not been 222 * specified 223 */ 224 unsigned long airq_iv_alloc(struct airq_iv *iv, unsigned long num) 225 { 226 unsigned long bit, i, flags; 227 228 if (!iv->avail || num == 0) 229 return -1UL; 230 spin_lock_irqsave(&iv->lock, flags); 231 bit = find_first_bit_inv(iv->avail, iv->bits); 232 while (bit + num <= iv->bits) { 233 for (i = 1; i < num; i++) 234 if (!test_bit_inv(bit + i, iv->avail)) 235 break; 236 if (i >= num) { 237 /* Found a suitable block of irqs */ 238 for (i = 0; i < num; i++) 239 clear_bit_inv(bit + i, iv->avail); 240 if (bit + num >= iv->end) 241 iv->end = bit + num + 1; 242 break; 243 } 244 bit = find_next_bit_inv(iv->avail, iv->bits, bit + i + 1); 245 } 246 if (bit + num > iv->bits) 247 bit = -1UL; 248 spin_unlock_irqrestore(&iv->lock, flags); 249 return bit; 250 } 251 EXPORT_SYMBOL(airq_iv_alloc); 252 253 /** 254 * airq_iv_free - free irq bits of an interrupt vector 255 * @iv: pointer to interrupt vector structure 256 * @bit: number of the first irq bit to free 257 * @num: number of consecutive irq bits to free 258 */ 259 void airq_iv_free(struct airq_iv *iv, unsigned long bit, unsigned long num) 260 { 261 unsigned long i, flags; 262 263 if (!iv->avail || num == 0) 264 return; 265 spin_lock_irqsave(&iv->lock, flags); 266 for (i = 0; i < num; i++) { 267 /* Clear (possibly left over) interrupt bit */ 268 clear_bit_inv(bit + i, iv->vector); 269 /* Make the bit positions available again */ 270 set_bit_inv(bit + i, iv->avail); 271 } 272 if (bit + num >= iv->end) { 273 /* Find new end of bit-field */ 274 while (iv->end > 0 && !test_bit_inv(iv->end - 1, iv->avail)) 275 iv->end--; 276 } 277 spin_unlock_irqrestore(&iv->lock, flags); 278 } 279 EXPORT_SYMBOL(airq_iv_free); 280 281 /** 282 * airq_iv_scan - scan interrupt vector for non-zero bits 283 * @iv: pointer to interrupt vector structure 284 * @start: bit number to start the search 285 * @end: bit number to end the search 286 * 287 * Returns the bit number of the next non-zero interrupt bit, or 288 * -1UL if the scan completed without finding any more any non-zero bits. 289 */ 290 unsigned long airq_iv_scan(struct airq_iv *iv, unsigned long start, 291 unsigned long end) 292 { 293 unsigned long bit; 294 295 /* Find non-zero bit starting from 'ivs->next'. */ 296 bit = find_next_bit_inv(iv->vector, end, start); 297 if (bit >= end) 298 return -1UL; 299 clear_bit_inv(bit, iv->vector); 300 return bit; 301 } 302 EXPORT_SYMBOL(airq_iv_scan); 303 304 int __init airq_init(void) 305 { 306 airq_iv_cache = dma_pool_create("airq_iv_cache", cio_get_dma_css_dev(), 307 cache_line_size(), 308 cache_line_size(), PAGE_SIZE); 309 if (!airq_iv_cache) 310 return -ENOMEM; 311 return 0; 312 } 313