xref: /linux/drivers/crypto/cavium/nitrox/nitrox_isr.c (revision a1c3be890440a1769ed6f822376a3e3ab0d42994)
1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/pci.h>
3 #include <linux/printk.h>
4 #include <linux/slab.h>
5 
6 #include "nitrox_dev.h"
7 #include "nitrox_csr.h"
8 #include "nitrox_common.h"
9 #include "nitrox_hal.h"
10 #include "nitrox_isr.h"
11 #include "nitrox_mbx.h"
12 
13 /**
14  * One vector for each type of ring
15  *  - NPS packet ring, AQMQ ring and ZQMQ ring
16  */
17 #define NR_RING_VECTORS 3
18 #define NR_NON_RING_VECTORS 1
19 /* base entry for packet ring/port */
20 #define PKT_RING_MSIX_BASE 0
21 #define NON_RING_MSIX_BASE 192
22 
23 /**
24  * nps_pkt_slc_isr - IRQ handler for NPS solicit port
25  * @irq: irq number
26  * @data: argument
27  */
28 static irqreturn_t nps_pkt_slc_isr(int irq, void *data)
29 {
30 	struct nitrox_q_vector *qvec = data;
31 	union nps_pkt_slc_cnts slc_cnts;
32 	struct nitrox_cmdq *cmdq = qvec->cmdq;
33 
34 	slc_cnts.value = readq(cmdq->compl_cnt_csr_addr);
35 	/* New packet on SLC output port */
36 	if (slc_cnts.s.slc_int)
37 		tasklet_hi_schedule(&qvec->resp_tasklet);
38 
39 	return IRQ_HANDLED;
40 }
41 
42 static void clear_nps_core_err_intr(struct nitrox_device *ndev)
43 {
44 	u64 value;
45 
46 	/* Write 1 to clear */
47 	value = nitrox_read_csr(ndev, NPS_CORE_INT);
48 	nitrox_write_csr(ndev, NPS_CORE_INT, value);
49 
50 	dev_err_ratelimited(DEV(ndev), "NSP_CORE_INT  0x%016llx\n", value);
51 }
52 
53 static void clear_nps_pkt_err_intr(struct nitrox_device *ndev)
54 {
55 	union nps_pkt_int pkt_int;
56 	unsigned long value, offset;
57 	int i;
58 
59 	pkt_int.value = nitrox_read_csr(ndev, NPS_PKT_INT);
60 	dev_err_ratelimited(DEV(ndev), "NPS_PKT_INT  0x%016llx\n",
61 			    pkt_int.value);
62 
63 	if (pkt_int.s.slc_err) {
64 		offset = NPS_PKT_SLC_ERR_TYPE;
65 		value = nitrox_read_csr(ndev, offset);
66 		nitrox_write_csr(ndev, offset, value);
67 		dev_err_ratelimited(DEV(ndev),
68 				    "NPS_PKT_SLC_ERR_TYPE  0x%016lx\n", value);
69 
70 		offset = NPS_PKT_SLC_RERR_LO;
71 		value = nitrox_read_csr(ndev, offset);
72 		nitrox_write_csr(ndev, offset, value);
73 		/* enable the solicit ports */
74 		for_each_set_bit(i, &value, BITS_PER_LONG)
75 			enable_pkt_solicit_port(ndev, i);
76 
77 		dev_err_ratelimited(DEV(ndev),
78 				    "NPS_PKT_SLC_RERR_LO  0x%016lx\n", value);
79 
80 		offset = NPS_PKT_SLC_RERR_HI;
81 		value = nitrox_read_csr(ndev, offset);
82 		nitrox_write_csr(ndev, offset, value);
83 		dev_err_ratelimited(DEV(ndev),
84 				    "NPS_PKT_SLC_RERR_HI  0x%016lx\n", value);
85 	}
86 
87 	if (pkt_int.s.in_err) {
88 		offset = NPS_PKT_IN_ERR_TYPE;
89 		value = nitrox_read_csr(ndev, offset);
90 		nitrox_write_csr(ndev, offset, value);
91 		dev_err_ratelimited(DEV(ndev),
92 				    "NPS_PKT_IN_ERR_TYPE  0x%016lx\n", value);
93 		offset = NPS_PKT_IN_RERR_LO;
94 		value = nitrox_read_csr(ndev, offset);
95 		nitrox_write_csr(ndev, offset, value);
96 		/* enable the input ring */
97 		for_each_set_bit(i, &value, BITS_PER_LONG)
98 			enable_pkt_input_ring(ndev, i);
99 
100 		dev_err_ratelimited(DEV(ndev),
101 				    "NPS_PKT_IN_RERR_LO  0x%016lx\n", value);
102 
103 		offset = NPS_PKT_IN_RERR_HI;
104 		value = nitrox_read_csr(ndev, offset);
105 		nitrox_write_csr(ndev, offset, value);
106 		dev_err_ratelimited(DEV(ndev),
107 				    "NPS_PKT_IN_RERR_HI  0x%016lx\n", value);
108 	}
109 }
110 
111 static void clear_pom_err_intr(struct nitrox_device *ndev)
112 {
113 	u64 value;
114 
115 	value = nitrox_read_csr(ndev, POM_INT);
116 	nitrox_write_csr(ndev, POM_INT, value);
117 	dev_err_ratelimited(DEV(ndev), "POM_INT  0x%016llx\n", value);
118 }
119 
120 static void clear_pem_err_intr(struct nitrox_device *ndev)
121 {
122 	u64 value;
123 
124 	value = nitrox_read_csr(ndev, PEM0_INT);
125 	nitrox_write_csr(ndev, PEM0_INT, value);
126 	dev_err_ratelimited(DEV(ndev), "PEM(0)_INT  0x%016llx\n", value);
127 }
128 
129 static void clear_lbc_err_intr(struct nitrox_device *ndev)
130 {
131 	union lbc_int lbc_int;
132 	u64 value, offset;
133 	int i;
134 
135 	lbc_int.value = nitrox_read_csr(ndev, LBC_INT);
136 	dev_err_ratelimited(DEV(ndev), "LBC_INT  0x%016llx\n", lbc_int.value);
137 
138 	if (lbc_int.s.dma_rd_err) {
139 		for (i = 0; i < NR_CLUSTERS; i++) {
140 			offset = EFL_CORE_VF_ERR_INT0X(i);
141 			value = nitrox_read_csr(ndev, offset);
142 			nitrox_write_csr(ndev, offset, value);
143 			offset = EFL_CORE_VF_ERR_INT1X(i);
144 			value = nitrox_read_csr(ndev, offset);
145 			nitrox_write_csr(ndev, offset, value);
146 		}
147 	}
148 
149 	if (lbc_int.s.cam_soft_err) {
150 		dev_err_ratelimited(DEV(ndev), "CAM_SOFT_ERR, invalidating LBC\n");
151 		invalidate_lbc(ndev);
152 	}
153 
154 	if (lbc_int.s.pref_dat_len_mismatch_err) {
155 		offset = LBC_PLM_VF1_64_INT;
156 		value = nitrox_read_csr(ndev, offset);
157 		nitrox_write_csr(ndev, offset, value);
158 		offset = LBC_PLM_VF65_128_INT;
159 		value = nitrox_read_csr(ndev, offset);
160 		nitrox_write_csr(ndev, offset, value);
161 	}
162 
163 	if (lbc_int.s.rd_dat_len_mismatch_err) {
164 		offset = LBC_ELM_VF1_64_INT;
165 		value = nitrox_read_csr(ndev, offset);
166 		nitrox_write_csr(ndev, offset, value);
167 		offset = LBC_ELM_VF65_128_INT;
168 		value = nitrox_read_csr(ndev, offset);
169 		nitrox_write_csr(ndev, offset, value);
170 	}
171 	nitrox_write_csr(ndev, LBC_INT, lbc_int.value);
172 }
173 
174 static void clear_efl_err_intr(struct nitrox_device *ndev)
175 {
176 	int i;
177 
178 	for (i = 0; i < NR_CLUSTERS; i++) {
179 		union efl_core_int core_int;
180 		u64 value, offset;
181 
182 		offset = EFL_CORE_INTX(i);
183 		core_int.value = nitrox_read_csr(ndev, offset);
184 		nitrox_write_csr(ndev, offset, core_int.value);
185 		dev_err_ratelimited(DEV(ndev), "ELF_CORE(%d)_INT  0x%016llx\n",
186 				    i, core_int.value);
187 		if (core_int.s.se_err) {
188 			offset = EFL_CORE_SE_ERR_INTX(i);
189 			value = nitrox_read_csr(ndev, offset);
190 			nitrox_write_csr(ndev, offset, value);
191 		}
192 	}
193 }
194 
195 static void clear_bmi_err_intr(struct nitrox_device *ndev)
196 {
197 	u64 value;
198 
199 	value = nitrox_read_csr(ndev, BMI_INT);
200 	nitrox_write_csr(ndev, BMI_INT, value);
201 	dev_err_ratelimited(DEV(ndev), "BMI_INT  0x%016llx\n", value);
202 }
203 
204 static void nps_core_int_tasklet(unsigned long data)
205 {
206 	struct nitrox_q_vector *qvec = (void *)(uintptr_t)(data);
207 	struct nitrox_device *ndev = qvec->ndev;
208 
209 	/* if pf mode do queue recovery */
210 	if (ndev->mode == __NDEV_MODE_PF) {
211 	} else {
212 		/**
213 		 * if VF(s) enabled communicate the error information
214 		 * to VF(s)
215 		 */
216 	}
217 }
218 
219 /**
220  * nps_core_int_isr - interrupt handler for NITROX errors and
221  *   mailbox communication
222  */
223 static irqreturn_t nps_core_int_isr(int irq, void *data)
224 {
225 	struct nitrox_q_vector *qvec = data;
226 	struct nitrox_device *ndev = qvec->ndev;
227 	union nps_core_int_active core_int;
228 
229 	core_int.value = nitrox_read_csr(ndev, NPS_CORE_INT_ACTIVE);
230 
231 	if (core_int.s.nps_core)
232 		clear_nps_core_err_intr(ndev);
233 
234 	if (core_int.s.nps_pkt)
235 		clear_nps_pkt_err_intr(ndev);
236 
237 	if (core_int.s.pom)
238 		clear_pom_err_intr(ndev);
239 
240 	if (core_int.s.pem)
241 		clear_pem_err_intr(ndev);
242 
243 	if (core_int.s.lbc)
244 		clear_lbc_err_intr(ndev);
245 
246 	if (core_int.s.efl)
247 		clear_efl_err_intr(ndev);
248 
249 	if (core_int.s.bmi)
250 		clear_bmi_err_intr(ndev);
251 
252 	/* Mailbox interrupt */
253 	if (core_int.s.mbox)
254 		nitrox_pf2vf_mbox_handler(ndev);
255 
256 	/* If more work callback the ISR, set resend */
257 	core_int.s.resend = 1;
258 	nitrox_write_csr(ndev, NPS_CORE_INT_ACTIVE, core_int.value);
259 
260 	return IRQ_HANDLED;
261 }
262 
263 void nitrox_unregister_interrupts(struct nitrox_device *ndev)
264 {
265 	struct pci_dev *pdev = ndev->pdev;
266 	int i;
267 
268 	for (i = 0; i < ndev->num_vecs; i++) {
269 		struct nitrox_q_vector *qvec;
270 		int vec;
271 
272 		qvec = ndev->qvec + i;
273 		if (!qvec->valid)
274 			continue;
275 
276 		/* get the vector number */
277 		vec = pci_irq_vector(pdev, i);
278 		irq_set_affinity_hint(vec, NULL);
279 		free_irq(vec, qvec);
280 
281 		tasklet_disable(&qvec->resp_tasklet);
282 		tasklet_kill(&qvec->resp_tasklet);
283 		qvec->valid = false;
284 	}
285 	kfree(ndev->qvec);
286 	ndev->qvec = NULL;
287 	pci_free_irq_vectors(pdev);
288 }
289 
290 int nitrox_register_interrupts(struct nitrox_device *ndev)
291 {
292 	struct pci_dev *pdev = ndev->pdev;
293 	struct nitrox_q_vector *qvec;
294 	int nr_vecs, vec, cpu;
295 	int ret, i;
296 
297 	/*
298 	 * PF MSI-X vectors
299 	 *
300 	 * Entry 0: NPS PKT ring 0
301 	 * Entry 1: AQMQ ring 0
302 	 * Entry 2: ZQM ring 0
303 	 * Entry 3: NPS PKT ring 1
304 	 * Entry 4: AQMQ ring 1
305 	 * Entry 5: ZQM ring 1
306 	 * ....
307 	 * Entry 192: NPS_CORE_INT_ACTIVE
308 	 */
309 	nr_vecs = pci_msix_vec_count(pdev);
310 
311 	/* Enable MSI-X */
312 	ret = pci_alloc_irq_vectors(pdev, nr_vecs, nr_vecs, PCI_IRQ_MSIX);
313 	if (ret < 0) {
314 		dev_err(DEV(ndev), "msix vectors %d alloc failed\n", nr_vecs);
315 		return ret;
316 	}
317 	ndev->num_vecs = nr_vecs;
318 
319 	ndev->qvec = kcalloc(nr_vecs, sizeof(*qvec), GFP_KERNEL);
320 	if (!ndev->qvec) {
321 		pci_free_irq_vectors(pdev);
322 		return -ENOMEM;
323 	}
324 
325 	/* request irqs for packet rings/ports */
326 	for (i = PKT_RING_MSIX_BASE; i < (nr_vecs - 1); i += NR_RING_VECTORS) {
327 		qvec = &ndev->qvec[i];
328 
329 		qvec->ring = i / NR_RING_VECTORS;
330 		if (qvec->ring >= ndev->nr_queues)
331 			break;
332 
333 		qvec->cmdq = &ndev->pkt_inq[qvec->ring];
334 		snprintf(qvec->name, IRQ_NAMESZ, "nitrox-pkt%d", qvec->ring);
335 		/* get the vector number */
336 		vec = pci_irq_vector(pdev, i);
337 		ret = request_irq(vec, nps_pkt_slc_isr, 0, qvec->name, qvec);
338 		if (ret) {
339 			dev_err(DEV(ndev), "irq failed for pkt ring/port%d\n",
340 				qvec->ring);
341 			goto irq_fail;
342 		}
343 		cpu = qvec->ring % num_online_cpus();
344 		irq_set_affinity_hint(vec, get_cpu_mask(cpu));
345 
346 		tasklet_init(&qvec->resp_tasklet, pkt_slc_resp_tasklet,
347 			     (unsigned long)qvec);
348 		qvec->valid = true;
349 	}
350 
351 	/* request irqs for non ring vectors */
352 	i = NON_RING_MSIX_BASE;
353 	qvec = &ndev->qvec[i];
354 	qvec->ndev = ndev;
355 
356 	snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d", i);
357 	/* get the vector number */
358 	vec = pci_irq_vector(pdev, i);
359 	ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
360 	if (ret) {
361 		dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n", i);
362 		goto irq_fail;
363 	}
364 	cpu = num_online_cpus();
365 	irq_set_affinity_hint(vec, get_cpu_mask(cpu));
366 
367 	tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
368 		     (unsigned long)qvec);
369 	qvec->valid = true;
370 
371 	return 0;
372 
373 irq_fail:
374 	nitrox_unregister_interrupts(ndev);
375 	return ret;
376 }
377 
378 void nitrox_sriov_unregister_interrupts(struct nitrox_device *ndev)
379 {
380 	struct pci_dev *pdev = ndev->pdev;
381 	int i;
382 
383 	for (i = 0; i < ndev->num_vecs; i++) {
384 		struct nitrox_q_vector *qvec;
385 		int vec;
386 
387 		qvec = ndev->qvec + i;
388 		if (!qvec->valid)
389 			continue;
390 
391 		vec = ndev->iov.msix.vector;
392 		irq_set_affinity_hint(vec, NULL);
393 		free_irq(vec, qvec);
394 
395 		tasklet_disable(&qvec->resp_tasklet);
396 		tasklet_kill(&qvec->resp_tasklet);
397 		qvec->valid = false;
398 	}
399 	kfree(ndev->qvec);
400 	ndev->qvec = NULL;
401 	pci_disable_msix(pdev);
402 }
403 
404 int nitrox_sriov_register_interupts(struct nitrox_device *ndev)
405 {
406 	struct pci_dev *pdev = ndev->pdev;
407 	struct nitrox_q_vector *qvec;
408 	int vec, cpu;
409 	int ret;
410 
411 	/**
412 	 * only non ring vectors i.e Entry 192 is available
413 	 * for PF in SR-IOV mode.
414 	 */
415 	ndev->iov.msix.entry = NON_RING_MSIX_BASE;
416 	ret = pci_enable_msix_exact(pdev, &ndev->iov.msix, NR_NON_RING_VECTORS);
417 	if (ret) {
418 		dev_err(DEV(ndev), "failed to allocate nps-core-int%d\n",
419 			NON_RING_MSIX_BASE);
420 		return ret;
421 	}
422 
423 	qvec = kcalloc(NR_NON_RING_VECTORS, sizeof(*qvec), GFP_KERNEL);
424 	if (!qvec) {
425 		pci_disable_msix(pdev);
426 		return -ENOMEM;
427 	}
428 	qvec->ndev = ndev;
429 
430 	ndev->qvec = qvec;
431 	ndev->num_vecs = NR_NON_RING_VECTORS;
432 	snprintf(qvec->name, IRQ_NAMESZ, "nitrox-core-int%d",
433 		 NON_RING_MSIX_BASE);
434 
435 	vec = ndev->iov.msix.vector;
436 	ret = request_irq(vec, nps_core_int_isr, 0, qvec->name, qvec);
437 	if (ret) {
438 		dev_err(DEV(ndev), "irq failed for nitrox-core-int%d\n",
439 			NON_RING_MSIX_BASE);
440 		goto iov_irq_fail;
441 	}
442 	cpu = num_online_cpus();
443 	irq_set_affinity_hint(vec, get_cpu_mask(cpu));
444 
445 	tasklet_init(&qvec->resp_tasklet, nps_core_int_tasklet,
446 		     (unsigned long)qvec);
447 	qvec->valid = true;
448 
449 	return 0;
450 
451 iov_irq_fail:
452 	nitrox_sriov_unregister_interrupts(ndev);
453 	return ret;
454 }
455