1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/pci.h> 3 #include <linux/printk.h> 4 #include <linux/slab.h> 5 6 #include "nitrox_dev.h" 7 #include "nitrox_csr.h" 8 #include "nitrox_common.h" 9 #include "nitrox_hal.h" 10 11 /** 12 * One vector for each type of ring 13 * - NPS packet ring, AQMQ ring and ZQMQ ring 14 */ 15 #define NR_RING_VECTORS 3 16 #define NR_NON_RING_VECTORS 1 17 /* base entry for packet ring/port */ 18 #define PKT_RING_MSIX_BASE 0 19 #define NON_RING_MSIX_BASE 192 20 21 /** 22 * nps_pkt_slc_isr - IRQ handler for NPS solicit port 23 * @irq: irq number 24 * @data: argument 25 */ 26 static irqreturn_t nps_pkt_slc_isr(int irq, void *data) 27 { 28 struct nitrox_q_vector *qvec = data; 29 union nps_pkt_slc_cnts slc_cnts; 30 struct nitrox_cmdq *cmdq = qvec->cmdq; 31 32 slc_cnts.value = readq(cmdq->compl_cnt_csr_addr); 33 /* New packet on SLC output port */ 34 if (slc_cnts.s.slc_int) 35 tasklet_hi_schedule(&qvec->resp_tasklet); 36 37 return IRQ_HANDLED; 38 } 39 40 static void clear_nps_core_err_intr(struct nitrox_device *ndev) 41 { 42 u64 value; 43 44 /* Write 1 to clear */ 45 value = nitrox_read_csr(ndev, NPS_CORE_INT); 46 nitrox_write_csr(ndev, NPS_CORE_INT, value); 47 48 dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT 0x%016llx\n", value); 49 } 50 51 static void clear_nps_pkt_err_intr(struct nitrox_device *ndev) 52 { 53 union nps_pkt_int pkt_int; 54 unsigned long value, offset; 55 int i; 56 57 pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT); 58 dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT 0x%016llx\n", 59 pkt_int.value); 60 61 if (pkt_int.s.slc_err) { 62 offset = NPS_PKT_SLC_ERR_TYPE; 63 value = nitrox_read_csr(ndev, offset); 64 nitrox_write_csr(ndev, offset, value); 65 dev_err_ratelimited(DEV(ndev), 66 "NPS_PKT_SLC_ERR_TYPE 0x%016lx\n", value); 67 68 offset = NPS_PKT_SLC_RERR_LO; 69 value = nitrox_read_csr(ndev, offset); 70 nitrox_write_csr(ndev, offset, value); 71 /* enable the solicit ports */ 72 for_each_set_bit(i, &value, BITS_PER_LONG) 73 enable_pkt_solicit_port(ndev, i); 74 75 dev_err_ratelimited(DEV(ndev), 76 "NPS_PKT_SLC_RERR_LO 0x%016lx\n", value); 77 78 offset = NPS_PKT_SLC_RERR_HI; 79 value = nitrox_read_csr(ndev, offset); 80 nitrox_write_csr(ndev, offset, value); 81 dev_err_ratelimited(DEV(ndev), 82 "NPS_PKT_SLC_RERR_HI 0x%016lx\n", value); 83 } 84 85 if (pkt_int.s.in_err) { 86 offset = NPS_PKT_IN_ERR_TYPE; 87 value = nitrox_read_csr(ndev, offset); 88 nitrox_write_csr(ndev, offset, value); 89 dev_err_ratelimited(DEV(ndev), 90 "NPS_PKT_IN_ERR_TYPE 0x%016lx\n", value); 91 offset = NPS_PKT_IN_RERR_LO; 92 value = nitrox_read_csr(ndev, offset); 93 nitrox_write_csr(ndev, offset, value); 94 /* enable the input ring */ 95 for_each_set_bit(i, &value, BITS_PER_LONG) 96 enable_pkt_input_ring(ndev, i); 97 98 dev_err_ratelimited(DEV(ndev), 99 "NPS_PKT_IN_RERR_LO 0x%016lx\n", value); 100 101 offset = NPS_PKT_IN_RERR_HI; 102 value = nitrox_read_csr(ndev, offset); 103 nitrox_write_csr(ndev, offset, value); 104 dev_err_ratelimited(DEV(ndev), 105 "NPS_PKT_IN_RERR_HI 0x%016lx\n", value); 106 } 107 } 108 109 static void clear_pom_err_intr(struct nitrox_device *ndev) 110 { 111 u64 value; 112 113 value = nitrox_read_csr(ndev, POM_INT); 114 nitrox_write_csr(ndev, POM_INT, value); 115 dev_err_ratelimited(DEV(ndev), "POM_INT 0x%016llx\n", value); 116 } 117 118 static void clear_pem_err_intr(struct nitrox_device *ndev) 119 { 120 u64 value; 121 122 value = nitrox_read_csr(ndev, PEM0_INT); 123 nitrox_write_csr(ndev, PEM0_INT, value); 124 dev_err_ratelimited(DEV(ndev), "PEM(0)_INT 0x%016llx\n", value); 125 } 126 127 static void clear_lbc_err_intr(struct nitrox_device *ndev) 128 { 129 union lbc_int lbc_int; 130 u64 value, offset; 131 int i; 132 133 lbc_int.value = nitrox_read_csr(ndev, LBC_INT); 134 dev_err_ratelimited(DEV(ndev), "LBC_INT 0x%016llx\n", lbc_int.value); 135 136 if (lbc_int.s.dma_rd_err) { 137 for (i = 0; i < NR_CLUSTERS; i++) { 138 offset = EFL_CORE_VF_ERR_INT0X(i); 139 value = nitrox_read_csr(ndev, offset); 140 nitrox_write_csr(ndev, offset, value); 141 offset = EFL_CORE_VF_ERR_INT1X(i); 142 value = nitrox_read_csr(ndev, offset); 143 nitrox_write_csr(ndev, offset, value); 144 } 145 } 146 147 if (lbc_int.s.cam_soft_err) { 148 dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n"); 149 invalidate_lbc(ndev); 150 } 151 152 if (lbc_int.s.pref_dat_len_mismatch_err) { 153 offset = LBC_PLM_VF1_64_INT; 154 value = nitrox_read_csr(ndev, offset); 155 nitrox_write_csr(ndev, offset, value); 156 offset = LBC_PLM_VF65_128_INT; 157 value = nitrox_read_csr(ndev, offset); 158 nitrox_write_csr(ndev, offset, value); 159 } 160 161 if (lbc_int.s.rd_dat_len_mismatch_err) { 162 offset = LBC_ELM_VF1_64_INT; 163 value = nitrox_read_csr(ndev, offset); 164 nitrox_write_csr(ndev, offset, value); 165 offset = LBC_ELM_VF65_128_INT; 166 value = nitrox_read_csr(ndev, offset); 167 nitrox_write_csr(ndev, offset, value); 168 } 169 nitrox_write_csr(ndev, LBC_INT, lbc_int.value); 170 } 171 172 static void clear_efl_err_intr(struct nitrox_device *ndev) 173 { 174 int i; 175 176 for (i = 0; i < NR_CLUSTERS; i++) { 177 union efl_core_int core_int; 178 u64 value, offset; 179 180 offset = EFL_CORE_INTX(i); 181 core_int.value = nitrox_read_csr(ndev, offset); 182 nitrox_write_csr(ndev, offset, core_int.value); 183 dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT 0x%016llx\n", 184 i, core_int.value); 185 if (core_int.s.se_err) { 186 offset = EFL_CORE_SE_ERR_INTX(i); 187 value = nitrox_read_csr(ndev, offset); 188 nitrox_write_csr(ndev, offset, value); 189 } 190 } 191 } 192 193 static void clear_bmi_err_intr(struct nitrox_device *ndev) 194 { 195 u64 value; 196 197 value = nitrox_read_csr(ndev, BMI_INT); 198 nitrox_write_csr(ndev, BMI_INT, value); 199 dev_err_ratelimited(DEV(ndev), "BMI_INT 0x%016llx\n", value); 200 } 201 202 static void nps_core_int_tasklet(unsigned long data) 203 { 204 struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data); 205 struct nitrox_device *ndev = qvec->ndev; 206 207 /* if pf mode do queue recovery */ 208 if (ndev->mode == __NDEV_MODE_PF) { 209 } else { 210 /** 211 * if VF(s) enabled communicate the error information 212 * to VF(s) 213 */ 214 } 215 } 216 217 /** 218 * nps_core_int_isr - interrupt handler for NITROX errors and 219 * mailbox communication 220 */ 221 static irqreturn_t nps_core_int_isr(int irq, void *data) 222 { 223 struct nitrox_device *ndev = data; 224 union nps_core_int_active core_int; 225 226 core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE); 227 228 if (core_int.s.nps_core) 229 clear_nps_core_err_intr(ndev); 230 231 if (core_int.s.nps_pkt) 232 clear_nps_pkt_err_intr(ndev); 233 234 if (core_int.s.pom) 235 clear_pom_err_intr(ndev); 236 237 if (core_int.s.pem) 238 clear_pem_err_intr(ndev); 239 240 if (core_int.s.lbc) 241 clear_lbc_err_intr(ndev); 242 243 if (core_int.s.efl) 244 clear_efl_err_intr(ndev); 245 246 if (core_int.s.bmi) 247 clear_bmi_err_intr(ndev); 248 249 /* If more work callback the ISR, set resend */ 250 core_int.s.resend = 1; 251 nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value); 252 253 return IRQ_HANDLED; 254 } 255 256 void nitrox_unregister_interrupts(struct nitrox_device *ndev) 257 { 258 struct pci_dev *pdev = ndev->pdev; 259 int i; 260 261 for (i = 0; i < ndev->num_vecs; i++) { 262 struct nitrox_q_vector *qvec; 263 int vec; 264 265 qvec = ndev->qvec + i; 266 if (!qvec->valid) 267 continue; 268 269 /* get the vector number */ 270 vec = pci_irq_vector(pdev, i); 271 irq_set_affinity_hint(vec, NULL); 272 free_irq(vec, qvec); 273 274 tasklet_disable(&qvec->resp_tasklet); 275 tasklet_kill(&qvec->resp_tasklet); 276 qvec->valid = false; 277 } 278 kfree(ndev->qvec); 279 ndev->qvec = NULL; 280 pci_free_irq_vectors(pdev); 281 } 282 283 int nitrox_register_interrupts(struct nitrox_device *ndev) 284 { 285 struct pci_dev *pdev = ndev->pdev; 286 struct nitrox_q_vector *qvec; 287 int nr_vecs, vec, cpu; 288 int ret, i; 289 290 /* 291 * PF MSI-X vectors 292 * 293 * Entry 0: NPS PKT ring 0 294 * Entry 1: AQMQ ring 0 295 * Entry 2: ZQM ring 0 296 * Entry 3: NPS PKT ring 1 297 * Entry 4: AQMQ ring 1 298 * Entry 5: ZQM ring 1 299 * .... 300 * Entry 192: NPS_CORE_INT_ACTIVE 301 */ 302 nr_vecs = pci_msix_vec_count(pdev); 303 304 /* Enable MSI-X */ 305 ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX); 306 if (ret < 0) { 307 dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs); 308 return ret; 309 } 310 ndev->num_vecs = nr_vecs; 311 312 ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL); 313 if (!ndev->qvec) { 314 pci_free_irq_vectors(pdev); 315 return -ENOMEM; 316 } 317 318 /* request irqs for packet rings/ports */ 319 for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) { 320 qvec = &ndev->qvec[i]; 321 322 qvec->ring = i / NR_RING_VECTORS; 323 if (qvec->ring >= ndev->nr_queues) 324 break; 325 326 qvec->cmdq = &ndev->pkt_inq[qvec->ring]; 327 snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring); 328 /* get the vector number */ 329 vec = pci_irq_vector(pdev, i); 330 ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec); 331 if (ret) { 332 dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n", 333 qvec->ring); 334 goto irq_fail; 335 } 336 cpu = qvec->ring % num_online_cpus(); 337 irq_set_affinity_hint(vec, get_cpu_mask(cpu)); 338 339 tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet, 340 (unsigned long)qvec); 341 qvec->valid = true; 342 } 343 344 /* request irqs for non ring vectors */ 345 i = NON_RING_MSIX_BASE; 346 qvec = &ndev->qvec[i]; 347 qvec->ndev = ndev; 348 349 snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i); 350 /* get the vector number */ 351 vec = pci_irq_vector(pdev, i); 352 ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec); 353 if (ret) { 354 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i); 355 goto irq_fail; 356 } 357 cpu = num_online_cpus(); 358 irq_set_affinity_hint(vec, get_cpu_mask(cpu)); 359 360 tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet, 361 (unsigned long)qvec); 362 qvec->valid = true; 363 364 return 0; 365 366 irq_fail: 367 nitrox_unregister_interrupts(ndev); 368 return ret; 369 } 370 371 void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev) 372 { 373 struct pci_dev *pdev = ndev->pdev; 374 int i; 375 376 for (i = 0; i < ndev->num_vecs; i++) { 377 struct nitrox_q_vector *qvec; 378 int vec; 379 380 qvec = ndev->qvec + i; 381 if (!qvec->valid) 382 continue; 383 384 vec = ndev->iov.msix.vector; 385 irq_set_affinity_hint(vec, NULL); 386 free_irq(vec, qvec); 387 388 tasklet_disable(&qvec->resp_tasklet); 389 tasklet_kill(&qvec->resp_tasklet); 390 qvec->valid = false; 391 } 392 kfree(ndev->qvec); 393 ndev->qvec = NULL; 394 pci_disable_msix(pdev); 395 } 396 397 int nitrox_sriov_register_interupts(struct nitrox_device *ndev) 398 { 399 struct pci_dev *pdev = ndev->pdev; 400 struct nitrox_q_vector *qvec; 401 int vec, cpu; 402 int ret; 403 404 /** 405 * only non ring vectors i.e Entry 192 is available 406 * for PF in SR-IOV mode. 407 */ 408 ndev->iov.msix.entry = NON_RING_MSIX_BASE; 409 ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS); 410 if (ret) { 411 dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n", 412 NON_RING_MSIX_BASE); 413 return ret; 414 } 415 416 qvec = kcalloc(NR_NON_RING_VECTORS, sizeof(*qvec), GFP_KERNEL); 417 if (!qvec) { 418 pci_disable_msix(pdev); 419 return -ENOMEM; 420 } 421 qvec->ndev = ndev; 422 423 ndev->qvec = qvec; 424 ndev->num_vecs = NR_NON_RING_VECTORS; 425 snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", 426 NON_RING_MSIX_BASE); 427 428 vec = ndev->iov.msix.vector; 429 ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec); 430 if (ret) { 431 dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", 432 NON_RING_MSIX_BASE); 433 goto iov_irq_fail; 434 } 435 cpu = num_online_cpus(); 436 irq_set_affinity_hint(vec, get_cpu_mask(cpu)); 437 438 tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet, 439 (unsigned long)qvec); 440 qvec->valid = true; 441 442 return 0; 443 444 iov_irq_fail: 445 nitrox_sriov_unregister_interrupts(ndev); 446 return ret; 447 } 448