1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright 2008 Cisco Systems, Inc. All rights reserved.
4 * Copyright 2007 Nuova Systems, Inc. All rights reserved.
5 */
6 #include <linux/string.h>
7 #include <linux/errno.h>
8 #include <linux/pci.h>
9 #include <linux/interrupt.h>
10 #include <scsi/scsi_transport_fc.h>
11 #include <scsi/fc_frame.h>
12 #include "vnic_dev.h"
13 #include "vnic_intr.h"
14 #include "vnic_stats.h"
15 #include "fnic_io.h"
16 #include "fnic.h"
17
fnic_isr_legacy(int irq,void * data)18 static irqreturn_t fnic_isr_legacy(int irq, void *data)
19 {
20 struct fnic *fnic = data;
21 u32 pba;
22 unsigned long work_done = 0;
23
24 pba = vnic_intr_legacy_pba(fnic->legacy_pba);
25 if (!pba)
26 return IRQ_NONE;
27
28 fnic->fnic_stats.misc_stats.last_isr_time = jiffies;
29 atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count);
30
31 if (pba & (1 << FNIC_INTX_NOTIFY)) {
32 vnic_intr_return_all_credits(&fnic->intr[FNIC_INTX_NOTIFY]);
33 fnic_handle_link_event(fnic);
34 }
35
36 if (pba & (1 << FNIC_INTX_ERR)) {
37 vnic_intr_return_all_credits(&fnic->intr[FNIC_INTX_ERR]);
38 fnic_log_q_error(fnic);
39 }
40
41 if (pba & (1 << FNIC_INTX_DUMMY)) {
42 atomic64_inc(&fnic->fnic_stats.misc_stats.intx_dummy);
43 vnic_intr_return_all_credits(&fnic->intr[FNIC_INTX_DUMMY]);
44 }
45
46 if (pba & (1 << FNIC_INTX_WQ_RQ_COPYWQ)) {
47 work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions, FNIC_MQ_CQ_INDEX);
48 work_done += fnic_wq_cmpl_handler(fnic, -1);
49 work_done += fnic_rq_cmpl_handler(fnic, -1);
50
51 vnic_intr_return_credits(&fnic->intr[FNIC_INTX_WQ_RQ_COPYWQ],
52 work_done,
53 1 /* unmask intr */,
54 1 /* reset intr timer */);
55 }
56
57 return IRQ_HANDLED;
58 }
59
fnic_isr_msi(int irq,void * data)60 static irqreturn_t fnic_isr_msi(int irq, void *data)
61 {
62 struct fnic *fnic = data;
63 unsigned long work_done = 0;
64
65 fnic->fnic_stats.misc_stats.last_isr_time = jiffies;
66 atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count);
67
68 work_done += fnic_wq_copy_cmpl_handler(fnic, io_completions, FNIC_MQ_CQ_INDEX);
69 work_done += fnic_wq_cmpl_handler(fnic, -1);
70 work_done += fnic_rq_cmpl_handler(fnic, -1);
71
72 vnic_intr_return_credits(&fnic->intr[0],
73 work_done,
74 1 /* unmask intr */,
75 1 /* reset intr timer */);
76
77 return IRQ_HANDLED;
78 }
79
fnic_isr_msix_rq(int irq,void * data)80 static irqreturn_t fnic_isr_msix_rq(int irq, void *data)
81 {
82 struct fnic *fnic = data;
83 unsigned long rq_work_done = 0;
84
85 fnic->fnic_stats.misc_stats.last_isr_time = jiffies;
86 atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count);
87
88 rq_work_done = fnic_rq_cmpl_handler(fnic, -1);
89 vnic_intr_return_credits(&fnic->intr[FNIC_MSIX_RQ],
90 rq_work_done,
91 1 /* unmask intr */,
92 1 /* reset intr timer */);
93
94 return IRQ_HANDLED;
95 }
96
fnic_isr_msix_wq(int irq,void * data)97 static irqreturn_t fnic_isr_msix_wq(int irq, void *data)
98 {
99 struct fnic *fnic = data;
100 unsigned long wq_work_done = 0;
101
102 fnic->fnic_stats.misc_stats.last_isr_time = jiffies;
103 atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count);
104
105 wq_work_done = fnic_wq_cmpl_handler(fnic, -1);
106 vnic_intr_return_credits(&fnic->intr[FNIC_MSIX_WQ],
107 wq_work_done,
108 1 /* unmask intr */,
109 1 /* reset intr timer */);
110 return IRQ_HANDLED;
111 }
112
fnic_isr_msix_wq_copy(int irq,void * data)113 static irqreturn_t fnic_isr_msix_wq_copy(int irq, void *data)
114 {
115 struct fnic *fnic = data;
116 unsigned long wq_copy_work_done = 0;
117 int i;
118
119 fnic->fnic_stats.misc_stats.last_isr_time = jiffies;
120 atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count);
121
122 i = irq - fnic->msix[0].irq_num;
123 if (i >= fnic->wq_copy_count + fnic->copy_wq_base ||
124 i < 0 || fnic->msix[i].irq_num != irq) {
125 for (i = fnic->copy_wq_base; i < fnic->wq_copy_count + fnic->copy_wq_base ; i++) {
126 if (fnic->msix[i].irq_num == irq)
127 break;
128 }
129 }
130
131 wq_copy_work_done = fnic_wq_copy_cmpl_handler(fnic, io_completions, i);
132 vnic_intr_return_credits(&fnic->intr[i],
133 wq_copy_work_done,
134 1 /* unmask intr */,
135 1 /* reset intr timer */);
136 return IRQ_HANDLED;
137 }
138
fnic_isr_msix_err_notify(int irq,void * data)139 static irqreturn_t fnic_isr_msix_err_notify(int irq, void *data)
140 {
141 struct fnic *fnic = data;
142
143 fnic->fnic_stats.misc_stats.last_isr_time = jiffies;
144 atomic64_inc(&fnic->fnic_stats.misc_stats.isr_count);
145
146 vnic_intr_return_all_credits(&fnic->intr[fnic->err_intr_offset]);
147 fnic_log_q_error(fnic);
148 fnic_handle_link_event(fnic);
149
150 return IRQ_HANDLED;
151 }
152
fnic_free_intr(struct fnic * fnic)153 void fnic_free_intr(struct fnic *fnic)
154 {
155 int i;
156
157 switch (vnic_dev_get_intr_mode(fnic->vdev)) {
158 case VNIC_DEV_INTR_MODE_INTX:
159 case VNIC_DEV_INTR_MODE_MSI:
160 free_irq(pci_irq_vector(fnic->pdev, 0), fnic);
161 break;
162
163 case VNIC_DEV_INTR_MODE_MSIX:
164 for (i = 0; i < ARRAY_SIZE(fnic->msix); i++)
165 if (fnic->msix[i].requested)
166 free_irq(pci_irq_vector(fnic->pdev, i),
167 fnic->msix[i].devid);
168 break;
169
170 default:
171 break;
172 }
173 }
174
fnic_request_intr(struct fnic * fnic)175 int fnic_request_intr(struct fnic *fnic)
176 {
177 int err = 0;
178 int i;
179
180 switch (vnic_dev_get_intr_mode(fnic->vdev)) {
181
182 case VNIC_DEV_INTR_MODE_INTX:
183 err = request_irq(pci_irq_vector(fnic->pdev, 0),
184 &fnic_isr_legacy, IRQF_SHARED, DRV_NAME, fnic);
185 break;
186
187 case VNIC_DEV_INTR_MODE_MSI:
188 err = request_irq(pci_irq_vector(fnic->pdev, 0), &fnic_isr_msi,
189 0, fnic->name, fnic);
190 break;
191
192 case VNIC_DEV_INTR_MODE_MSIX:
193
194 sprintf(fnic->msix[FNIC_MSIX_RQ].devname,
195 "%.11s-fcs-rq", fnic->name);
196 fnic->msix[FNIC_MSIX_RQ].isr = fnic_isr_msix_rq;
197 fnic->msix[FNIC_MSIX_RQ].devid = fnic;
198
199 sprintf(fnic->msix[FNIC_MSIX_WQ].devname,
200 "%.11s-fcs-wq", fnic->name);
201 fnic->msix[FNIC_MSIX_WQ].isr = fnic_isr_msix_wq;
202 fnic->msix[FNIC_MSIX_WQ].devid = fnic;
203
204 for (i = fnic->copy_wq_base; i < fnic->wq_copy_count + fnic->copy_wq_base; i++) {
205 sprintf(fnic->msix[i].devname,
206 "%.11s-scsi-wq-%d", fnic->name, i-FNIC_MSIX_WQ_COPY);
207 fnic->msix[i].isr = fnic_isr_msix_wq_copy;
208 fnic->msix[i].devid = fnic;
209 }
210
211 sprintf(fnic->msix[fnic->err_intr_offset].devname,
212 "%.11s-err-notify", fnic->name);
213 fnic->msix[fnic->err_intr_offset].isr =
214 fnic_isr_msix_err_notify;
215 fnic->msix[fnic->err_intr_offset].devid = fnic;
216
217 for (i = 0; i < fnic->intr_count; i++) {
218 fnic->msix[i].irq_num = pci_irq_vector(fnic->pdev, i);
219
220 err = request_irq(fnic->msix[i].irq_num,
221 fnic->msix[i].isr, 0,
222 fnic->msix[i].devname,
223 fnic->msix[i].devid);
224 if (err) {
225 FNIC_ISR_DBG(KERN_ERR, fnic,
226 "request_irq failed with error: %d\n",
227 err);
228 fnic_free_intr(fnic);
229 break;
230 }
231 fnic->msix[i].requested = 1;
232 }
233 break;
234
235 default:
236 break;
237 }
238
239 return err;
240 }
241
fnic_set_intr_mode_msix(struct fnic * fnic)242 int fnic_set_intr_mode_msix(struct fnic *fnic)
243 {
244 unsigned int n = ARRAY_SIZE(fnic->rq);
245 unsigned int m = ARRAY_SIZE(fnic->wq);
246 unsigned int o = ARRAY_SIZE(fnic->hw_copy_wq);
247 unsigned int min_irqs = n + m + 1 + 1; /*rq, raw wq, wq, err*/
248 /*
249 * Make driver critical vectors unmanaged, or else it can get tied
250 * to an offline CPU. This can happen when hyper-threading is off.
251 */
252 struct irq_affinity affd = {
253 .pre_vectors = n + m + 1, /* rq, raw wq, 1 ioq */
254 .post_vectors = 1, /* err */
255 };
256 /*
257 * We need n RQs, m WQs, o Copy WQs, n+m+o CQs, and n+m+o+1 INTRs
258 * (last INTR is used for WQ/RQ errors and notification area)
259 */
260 FNIC_ISR_DBG(KERN_INFO, fnic,
261 "rq-array size: %d wq-array size: %d copy-wq array size: %d\n",
262 n, m, o);
263 FNIC_ISR_DBG(KERN_INFO, fnic,
264 "rq_count: %d raw_wq_count: %d wq_copy_count: %d cq_count: %d\n",
265 fnic->rq_count, fnic->raw_wq_count,
266 fnic->wq_copy_count, fnic->cq_count);
267
268 if (fnic->rq_count <= n && fnic->raw_wq_count <= m &&
269 fnic->wq_copy_count <= o) {
270 int vec_count = 0;
271 int vecs = fnic->rq_count + fnic->raw_wq_count + fnic->wq_copy_count + 1;
272
273 vec_count = pci_alloc_irq_vectors_affinity(fnic->pdev, min_irqs,
274 vecs, PCI_IRQ_MSIX|PCI_IRQ_AFFINITY, &affd);
275 FNIC_ISR_DBG(KERN_INFO, fnic,
276 "allocated %d MSI-X vectors\n",
277 vec_count);
278
279 if (vec_count > 0) {
280 if (vec_count < vecs) {
281 FNIC_ISR_DBG(KERN_ERR, fnic,
282 "interrupts number mismatch: vec_count: %d vecs: %d\n",
283 vec_count, vecs);
284 if (vec_count < min_irqs) {
285 FNIC_ISR_DBG(KERN_ERR, fnic,
286 "no interrupts for copy wq\n");
287 return 1;
288 }
289 }
290
291 fnic->rq_count = n;
292 fnic->raw_wq_count = m;
293 fnic->copy_wq_base = fnic->rq_count + fnic->raw_wq_count;
294 fnic->wq_copy_count = vec_count - n - m - 1;
295 fnic->wq_count = fnic->raw_wq_count + fnic->wq_copy_count;
296 if (fnic->cq_count != vec_count - 1) {
297 FNIC_ISR_DBG(KERN_ERR, fnic,
298 "CQ count: %d does not match MSI-X vector count: %d\n",
299 fnic->cq_count, vec_count);
300 fnic->cq_count = vec_count - 1;
301 }
302 fnic->intr_count = vec_count;
303 fnic->err_intr_offset = fnic->rq_count + fnic->wq_count;
304
305 FNIC_ISR_DBG(KERN_INFO, fnic,
306 "rq_count: %d raw_wq_count: %d copy_wq_base: %d\n",
307 fnic->rq_count,
308 fnic->raw_wq_count, fnic->copy_wq_base);
309
310 FNIC_ISR_DBG(KERN_INFO, fnic,
311 "wq_copy_count: %d wq_count: %d cq_count: %d\n",
312 fnic->wq_copy_count,
313 fnic->wq_count, fnic->cq_count);
314
315 FNIC_ISR_DBG(KERN_INFO, fnic,
316 "intr_count: %d err_intr_offset: %u\n",
317 fnic->intr_count,
318 fnic->err_intr_offset);
319
320 vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_MSIX);
321 FNIC_ISR_DBG(KERN_INFO, fnic,
322 "fnic using MSI-X\n");
323 return 0;
324 }
325 }
326 return 1;
327 }
328
fnic_set_intr_mode(struct fnic * fnic)329 int fnic_set_intr_mode(struct fnic *fnic)
330 {
331 int ret_status = 0;
332
333 /*
334 * Set interrupt mode (INTx, MSI, MSI-X) depending
335 * system capabilities.
336 *
337 * Try MSI-X first
338 */
339 ret_status = fnic_set_intr_mode_msix(fnic);
340 if (ret_status == 0)
341 return ret_status;
342
343 /*
344 * Next try MSI
345 * We need 1 RQ, 1 WQ, 1 WQ_COPY, 3 CQs, and 1 INTR
346 */
347 if (fnic->rq_count >= 1 &&
348 fnic->raw_wq_count >= 1 &&
349 fnic->wq_copy_count >= 1 &&
350 fnic->cq_count >= 3 &&
351 fnic->intr_count >= 1 &&
352 pci_alloc_irq_vectors(fnic->pdev, 1, 1, PCI_IRQ_MSI) == 1) {
353 fnic->rq_count = 1;
354 fnic->raw_wq_count = 1;
355 fnic->wq_copy_count = 1;
356 fnic->wq_count = 2;
357 fnic->cq_count = 3;
358 fnic->intr_count = 1;
359 fnic->err_intr_offset = 0;
360
361 FNIC_ISR_DBG(KERN_DEBUG, fnic,
362 "Using MSI Interrupts\n");
363 vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_MSI);
364
365 return 0;
366 }
367
368 /*
369 * Next try INTx
370 * We need 1 RQ, 1 WQ, 1 WQ_COPY, 3 CQs, and 3 INTRs
371 * 1 INTR is used for all 3 queues, 1 INTR for queue errors
372 * 1 INTR for notification area
373 */
374
375 if (fnic->rq_count >= 1 &&
376 fnic->raw_wq_count >= 1 &&
377 fnic->wq_copy_count >= 1 &&
378 fnic->cq_count >= 3 &&
379 fnic->intr_count >= 3) {
380
381 fnic->rq_count = 1;
382 fnic->raw_wq_count = 1;
383 fnic->wq_copy_count = 1;
384 fnic->cq_count = 3;
385 fnic->intr_count = 3;
386
387 FNIC_ISR_DBG(KERN_DEBUG, fnic,
388 "Using Legacy Interrupts\n");
389 vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_INTX);
390
391 return 0;
392 }
393
394 vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_UNKNOWN);
395
396 return -EINVAL;
397 }
398
fnic_clear_intr_mode(struct fnic * fnic)399 void fnic_clear_intr_mode(struct fnic *fnic)
400 {
401 pci_free_irq_vectors(fnic->pdev);
402 vnic_dev_set_intr_mode(fnic->vdev, VNIC_DEV_INTR_MODE_INTX);
403 }
404