1 // SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) 2 /* 3 * Copyright(c) 2018 - 2020 Intel Corporation. 4 */ 5 6 #include "hfi.h" 7 #include "affinity.h" 8 #include "sdma.h" 9 #include "netdev.h" 10 11 /** 12 * msix_initialize() - Calculate, request and configure MSIx IRQs 13 * @dd: valid hfi1 devdata 14 * 15 */ 16 int msix_initialize(struct hfi1_devdata *dd) 17 { 18 u32 total; 19 int ret; 20 struct hfi1_msix_entry *entries; 21 22 /* 23 * MSIx interrupt count: 24 * one for the general, "slow path" interrupt 25 * one per used SDMA engine 26 * one per kernel receive context 27 * ...any new IRQs should be added here. 28 */ 29 total = 1 + dd->num_sdma + dd->n_krcv_queues + dd->num_netdev_contexts; 30 31 if (total >= CCE_NUM_MSIX_VECTORS) 32 return -EINVAL; 33 34 ret = pci_alloc_irq_vectors(dd->pcidev, total, total, PCI_IRQ_MSIX); 35 if (ret < 0) { 36 dd_dev_err(dd, "pci_alloc_irq_vectors() failed: %d\n", ret); 37 return ret; 38 } 39 40 entries = kzalloc_objs(*dd->msix_info.msix_entries, total); 41 if (!entries) { 42 pci_free_irq_vectors(dd->pcidev); 43 return -ENOMEM; 44 } 45 46 dd->msix_info.msix_entries = entries; 47 spin_lock_init(&dd->msix_info.msix_lock); 48 bitmap_zero(dd->msix_info.in_use_msix, total); 49 dd->msix_info.max_requested = total; 50 dd_dev_info(dd, "%u MSI-X interrupts allocated\n", total); 51 52 return 0; 53 } 54 55 /** 56 * msix_request_irq() - Allocate a free MSIx IRQ 57 * @dd: valid devdata 58 * @arg: context information for the IRQ 59 * @handler: IRQ handler 60 * @thread: IRQ thread handler (could be NULL) 61 * @type: affinty IRQ type 62 * @name: IRQ name 63 * 64 * Allocated an MSIx vector if available, and then create the appropriate 65 * meta data needed to keep track of the pci IRQ request. 66 * 67 * Return: 68 * < 0 Error 69 * >= 0 MSIx vector 70 * 71 */ 72 static int msix_request_irq(struct hfi1_devdata *dd, void *arg, 73 irq_handler_t handler, irq_handler_t thread, 74 enum irq_type type, const char *name) 75 { 76 unsigned long nr; 77 int irq; 78 int ret; 79 struct hfi1_msix_entry *me; 80 81 /* Allocate an MSIx vector */ 82 spin_lock(&dd->msix_info.msix_lock); 83 nr = find_first_zero_bit(dd->msix_info.in_use_msix, 84 dd->msix_info.max_requested); 85 if (nr < dd->msix_info.max_requested) 86 __set_bit(nr, dd->msix_info.in_use_msix); 87 spin_unlock(&dd->msix_info.msix_lock); 88 89 if (nr == dd->msix_info.max_requested) 90 return -ENOSPC; 91 92 if (type < IRQ_SDMA || type >= IRQ_OTHER) 93 return -EINVAL; 94 95 irq = pci_irq_vector(dd->pcidev, nr); 96 ret = pci_request_irq(dd->pcidev, nr, handler, thread, arg, name); 97 if (ret) { 98 dd_dev_err(dd, 99 "%s: request for IRQ %d failed, MSIx %lx, err %d\n", 100 name, irq, nr, ret); 101 spin_lock(&dd->msix_info.msix_lock); 102 __clear_bit(nr, dd->msix_info.in_use_msix); 103 spin_unlock(&dd->msix_info.msix_lock); 104 return ret; 105 } 106 107 /* 108 * assign arg after pci_request_irq call, so it will be 109 * cleaned up 110 */ 111 me = &dd->msix_info.msix_entries[nr]; 112 me->irq = irq; 113 me->arg = arg; 114 me->type = type; 115 116 /* This is a request, so a failure is not fatal */ 117 ret = hfi1_get_irq_affinity(dd, me); 118 if (ret) 119 dd_dev_err(dd, "%s: unable to pin IRQ %d\n", name, ret); 120 121 return nr; 122 } 123 124 static int msix_request_rcd_irq_common(struct hfi1_ctxtdata *rcd, 125 irq_handler_t handler, 126 irq_handler_t thread, 127 const char *name) 128 { 129 int nr = msix_request_irq(rcd->dd, rcd, handler, thread, IRQ_RCVCTXT, 130 name); 131 if (nr < 0) 132 return nr; 133 134 /* 135 * Set the interrupt register and mask for this 136 * context's interrupt. 137 */ 138 rcd->ireg = (IS_RCVAVAIL_START + rcd->ctxt) / 64; 139 rcd->imask = ((u64)1) << ((IS_RCVAVAIL_START + rcd->ctxt) % 64); 140 rcd->msix_intr = nr; 141 remap_intr(rcd->dd, IS_RCVAVAIL_START + rcd->ctxt, nr); 142 143 return 0; 144 } 145 146 /** 147 * msix_request_rcd_irq() - Helper function for RCVAVAIL IRQs 148 * @rcd: valid rcd context 149 * 150 */ 151 int msix_request_rcd_irq(struct hfi1_ctxtdata *rcd) 152 { 153 char name[MAX_NAME_SIZE]; 154 155 snprintf(name, sizeof(name), DRIVER_NAME "_%d kctxt%d", 156 rcd->dd->unit, rcd->ctxt); 157 158 return msix_request_rcd_irq_common(rcd, receive_context_interrupt, 159 receive_context_thread, name); 160 } 161 162 /** 163 * msix_netdev_request_rcd_irq - Helper function for RCVAVAIL IRQs 164 * for netdev context 165 * @rcd: valid netdev contexti 166 */ 167 int msix_netdev_request_rcd_irq(struct hfi1_ctxtdata *rcd) 168 { 169 char name[MAX_NAME_SIZE]; 170 171 snprintf(name, sizeof(name), DRIVER_NAME "_%d nd kctxt%d", 172 rcd->dd->unit, rcd->ctxt); 173 return msix_request_rcd_irq_common(rcd, receive_context_interrupt_napi, 174 NULL, name); 175 } 176 177 /** 178 * msix_request_sdma_irq - Helper for getting SDMA IRQ resources 179 * @sde: valid sdma engine 180 * 181 */ 182 int msix_request_sdma_irq(struct sdma_engine *sde) 183 { 184 int nr; 185 char name[MAX_NAME_SIZE]; 186 187 snprintf(name, sizeof(name), DRIVER_NAME "_%d sdma%d", 188 sde->dd->unit, sde->this_idx); 189 nr = msix_request_irq(sde->dd, sde, sdma_interrupt, NULL, 190 IRQ_SDMA, name); 191 if (nr < 0) 192 return nr; 193 sde->msix_intr = nr; 194 remap_sdma_interrupts(sde->dd, sde->this_idx, nr); 195 196 return 0; 197 } 198 199 /** 200 * msix_request_general_irq - Helper for getting general IRQ 201 * resources 202 * @dd: valid device data 203 */ 204 int msix_request_general_irq(struct hfi1_devdata *dd) 205 { 206 int nr; 207 char name[MAX_NAME_SIZE]; 208 209 snprintf(name, sizeof(name), DRIVER_NAME "_%d", dd->unit); 210 nr = msix_request_irq(dd, dd, general_interrupt, NULL, IRQ_GENERAL, 211 name); 212 if (nr < 0) 213 return nr; 214 215 /* general interrupt must be MSIx vector 0 */ 216 if (nr) { 217 msix_free_irq(dd, (u8)nr); 218 dd_dev_err(dd, "Invalid index %d for GENERAL IRQ\n", nr); 219 return -EINVAL; 220 } 221 222 return 0; 223 } 224 225 /** 226 * enable_sdma_srcs - Helper to enable SDMA IRQ srcs 227 * @dd: valid devdata structure 228 * @i: index of SDMA engine 229 */ 230 static void enable_sdma_srcs(struct hfi1_devdata *dd, int i) 231 { 232 set_intr_bits(dd, IS_SDMA_START + i, IS_SDMA_START + i, true); 233 set_intr_bits(dd, IS_SDMA_PROGRESS_START + i, 234 IS_SDMA_PROGRESS_START + i, true); 235 set_intr_bits(dd, IS_SDMA_IDLE_START + i, IS_SDMA_IDLE_START + i, true); 236 set_intr_bits(dd, IS_SDMAENG_ERR_START + i, IS_SDMAENG_ERR_START + i, 237 true); 238 } 239 240 /** 241 * msix_request_irqs() - Allocate all MSIx IRQs 242 * @dd: valid devdata structure 243 * 244 * Helper function to request the used MSIx IRQs. 245 * 246 */ 247 int msix_request_irqs(struct hfi1_devdata *dd) 248 { 249 int i; 250 int ret = msix_request_general_irq(dd); 251 252 if (ret) 253 return ret; 254 255 for (i = 0; i < dd->num_sdma; i++) { 256 struct sdma_engine *sde = &dd->per_sdma[i]; 257 258 ret = msix_request_sdma_irq(sde); 259 if (ret) 260 return ret; 261 enable_sdma_srcs(sde->dd, i); 262 } 263 264 for (i = 0; i < dd->n_krcv_queues; i++) { 265 struct hfi1_ctxtdata *rcd = hfi1_rcd_get_by_index_safe(dd, i); 266 267 if (rcd) 268 ret = msix_request_rcd_irq(rcd); 269 hfi1_rcd_put(rcd); 270 if (ret) 271 return ret; 272 } 273 274 return 0; 275 } 276 277 /** 278 * msix_free_irq() - Free the specified MSIx resources and IRQ 279 * @dd: valid devdata 280 * @msix_intr: MSIx vector to free. 281 * 282 */ 283 void msix_free_irq(struct hfi1_devdata *dd, u8 msix_intr) 284 { 285 struct hfi1_msix_entry *me; 286 287 if (msix_intr >= dd->msix_info.max_requested) 288 return; 289 290 me = &dd->msix_info.msix_entries[msix_intr]; 291 292 if (!me->arg) /* => no irq, no affinity */ 293 return; 294 295 hfi1_put_irq_affinity(dd, me); 296 pci_free_irq(dd->pcidev, msix_intr, me->arg); 297 298 me->arg = NULL; 299 300 spin_lock(&dd->msix_info.msix_lock); 301 __clear_bit(msix_intr, dd->msix_info.in_use_msix); 302 spin_unlock(&dd->msix_info.msix_lock); 303 } 304 305 /** 306 * msix_clean_up_interrupts - Free all MSIx IRQ resources 307 * @dd: valid device data data structure 308 * 309 * Free the MSIx and associated PCI resources, if they have been allocated. 310 */ 311 void msix_clean_up_interrupts(struct hfi1_devdata *dd) 312 { 313 int i; 314 struct hfi1_msix_entry *me = dd->msix_info.msix_entries; 315 316 /* remove irqs - must happen before disabling/turning off */ 317 for (i = 0; i < dd->msix_info.max_requested; i++, me++) 318 msix_free_irq(dd, i); 319 320 /* clean structures */ 321 kfree(dd->msix_info.msix_entries); 322 dd->msix_info.msix_entries = NULL; 323 dd->msix_info.max_requested = 0; 324 325 pci_free_irq_vectors(dd->pcidev); 326 } 327 328 /** 329 * msix_netdev_synchronize_irq - netdev IRQ synchronize 330 * @dd: valid devdata 331 */ 332 void msix_netdev_synchronize_irq(struct hfi1_devdata *dd) 333 { 334 int i; 335 int ctxt_count = hfi1_netdev_ctxt_count(dd); 336 337 for (i = 0; i < ctxt_count; i++) { 338 struct hfi1_ctxtdata *rcd = hfi1_netdev_get_ctxt(dd, i); 339 struct hfi1_msix_entry *me; 340 341 me = &dd->msix_info.msix_entries[rcd->msix_intr]; 342 343 synchronize_irq(me->irq); 344 } 345 } 346