1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) Meta Platforms, Inc. and affiliates. */
3
4 #include <linux/pci.h>
5 #include <linux/types.h>
6
7 #include "fbnic.h"
8 #include "fbnic_netdev.h"
9 #include "fbnic_txrx.h"
10
fbnic_fw_msix_intr(int __always_unused irq,void * data)11 static irqreturn_t fbnic_fw_msix_intr(int __always_unused irq, void *data)
12 {
13 struct fbnic_dev *fbd = (struct fbnic_dev *)data;
14
15 fbnic_mbx_poll(fbd);
16
17 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY);
18
19 return IRQ_HANDLED;
20 }
21
__fbnic_fw_enable_mbx(struct fbnic_dev * fbd,int vector)22 static int __fbnic_fw_enable_mbx(struct fbnic_dev *fbd, int vector)
23 {
24 int err;
25
26 /* Initialize mailbox and attempt to poll it into ready state */
27 fbnic_mbx_init(fbd);
28 err = fbnic_mbx_poll_tx_ready(fbd);
29 if (err) {
30 dev_warn(fbd->dev, "FW mailbox did not enter ready state\n");
31 return err;
32 }
33
34 /* Enable interrupt and unmask the vector */
35 enable_irq(vector);
36 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY);
37
38 return 0;
39 }
40
41 /**
42 * fbnic_fw_request_mbx - Configure and initialize Firmware Mailbox
43 * @fbd: Pointer to device to initialize
44 *
45 * This function will allocate the IRQ and then reinitialize the mailbox
46 * starting communication between the host and firmware.
47 *
48 * Return: non-zero on failure.
49 **/
fbnic_fw_request_mbx(struct fbnic_dev * fbd)50 int fbnic_fw_request_mbx(struct fbnic_dev *fbd)
51 {
52 struct pci_dev *pdev = to_pci_dev(fbd->dev);
53 int vector, err;
54
55 WARN_ON(fbd->fw_msix_vector);
56
57 vector = pci_irq_vector(pdev, FBNIC_FW_MSIX_ENTRY);
58 if (vector < 0)
59 return vector;
60
61 /* Request the IRQ for FW Mailbox vector. */
62 err = request_threaded_irq(vector, NULL, &fbnic_fw_msix_intr,
63 IRQF_ONESHOT | IRQF_NO_AUTOEN,
64 dev_name(fbd->dev), fbd);
65 if (err)
66 return err;
67
68 /* Initialize mailbox and attempt to poll it into ready state */
69 err = __fbnic_fw_enable_mbx(fbd, vector);
70 if (err)
71 free_irq(vector, fbd);
72
73 fbd->fw_msix_vector = vector;
74
75 return err;
76 }
77
78 /**
79 * fbnic_fw_disable_mbx - Temporarily place mailbox in standby state
80 * @fbd: Pointer to device
81 *
82 * Shutdown the mailbox by notifying the firmware to stop sending us logs, mask
83 * and synchronize the IRQ, and then clean up the rings.
84 **/
fbnic_fw_disable_mbx(struct fbnic_dev * fbd)85 static void fbnic_fw_disable_mbx(struct fbnic_dev *fbd)
86 {
87 /* Disable interrupt and synchronize the IRQ */
88 disable_irq(fbd->fw_msix_vector);
89
90 /* Mask the vector */
91 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(0), 1u << FBNIC_FW_MSIX_ENTRY);
92
93 /* Make sure disabling logs message is sent, must be done here to
94 * avoid risk of completing without a running interrupt.
95 */
96 fbnic_mbx_flush_tx(fbd);
97 fbnic_mbx_clean(fbd);
98 }
99
100 /**
101 * fbnic_fw_free_mbx - Disable mailbox and place it in standby state
102 * @fbd: Pointer to device to disable
103 *
104 * This function will disable the mailbox interrupt, free any messages still
105 * in the mailbox and place it into a disabled state. The firmware is
106 * expected to see the update and assume that the host is in the reset state.
107 **/
fbnic_fw_free_mbx(struct fbnic_dev * fbd)108 void fbnic_fw_free_mbx(struct fbnic_dev *fbd)
109 {
110 /* Vector has already been freed */
111 if (!fbd->fw_msix_vector)
112 return;
113
114 fbnic_fw_disable_mbx(fbd);
115
116 /* Free the vector */
117 free_irq(fbd->fw_msix_vector, fbd);
118 fbd->fw_msix_vector = 0;
119 }
120
fbnic_mac_msix_intr(int __always_unused irq,void * data)121 static irqreturn_t fbnic_mac_msix_intr(int __always_unused irq, void *data)
122 {
123 struct fbnic_dev *fbd = data;
124 struct fbnic_net *fbn;
125
126 if (fbd->mac->get_link_event(fbd) == FBNIC_LINK_EVENT_NONE) {
127 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0),
128 1u << FBNIC_PCS_MSIX_ENTRY);
129 return IRQ_HANDLED;
130 }
131
132 fbn = netdev_priv(fbd->netdev);
133
134 /* Record link down events */
135 if (!fbd->mac->get_link(fbd, fbn->aui, fbn->fec))
136 phylink_pcs_change(fbn->pcs, false);
137
138 return IRQ_HANDLED;
139 }
140
141 /**
142 * fbnic_mac_request_irq - Configure the MAC to enable it to advertise link
143 * @fbd: Pointer to device to initialize
144 *
145 * This function provides basic bringup for the MAC/PHY IRQ. For now the IRQ
146 * will remain disabled until we start the MAC/PCS/PHY logic via phylink.
147 *
148 * Return: non-zero on failure.
149 **/
fbnic_mac_request_irq(struct fbnic_dev * fbd)150 int fbnic_mac_request_irq(struct fbnic_dev *fbd)
151 {
152 struct pci_dev *pdev = to_pci_dev(fbd->dev);
153 int vector, err;
154
155 WARN_ON(fbd->mac_msix_vector);
156
157 vector = pci_irq_vector(pdev, FBNIC_PCS_MSIX_ENTRY);
158 if (vector < 0)
159 return vector;
160
161 /* Request the IRQ for PCS link vector.
162 * Map PCS cause to it, and unmask it
163 */
164 err = request_irq(vector, &fbnic_mac_msix_intr, 0,
165 fbd->netdev->name, fbd);
166 if (err)
167 return err;
168
169 /* Map and enable interrupt, unmask vector after link is configured */
170 fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX),
171 FBNIC_PCS_MSIX_ENTRY | FBNIC_INTR_MSIX_CTRL_ENABLE);
172
173 fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_RXB_IDX), 0);
174
175 fbd->mac_msix_vector = vector;
176
177 return 0;
178 }
179
180 /**
181 * fbnic_mac_free_irq - Teardown the MAC IRQ to prepare for stopping
182 * @fbd: Pointer to device that is stopping
183 *
184 * This function undoes the work done in fbnic_mac_request_irq and prepares
185 * the device to no longer receive traffic on the host interface.
186 **/
fbnic_mac_free_irq(struct fbnic_dev * fbd)187 void fbnic_mac_free_irq(struct fbnic_dev *fbd)
188 {
189 /* Vector has already been freed */
190 if (!fbd->mac_msix_vector)
191 return;
192
193 /* Disable interrupt */
194 fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX),
195 FBNIC_PCS_MSIX_ENTRY);
196 fbnic_wrfl(fbd);
197
198 /* Synchronize IRQ to prevent race that would unmask vector */
199 synchronize_irq(fbd->mac_msix_vector);
200
201 /* Mask the vector */
202 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(0), 1u << FBNIC_PCS_MSIX_ENTRY);
203
204 /* Free the vector */
205 free_irq(fbd->mac_msix_vector, fbd);
206 fbd->mac_msix_vector = 0;
207 }
208
fbnic_synchronize_irq(struct fbnic_dev * fbd,int nr)209 void fbnic_synchronize_irq(struct fbnic_dev *fbd, int nr)
210 {
211 struct pci_dev *pdev = to_pci_dev(fbd->dev);
212 int irq = pci_irq_vector(pdev, nr);
213
214 if (irq < 0)
215 return;
216
217 synchronize_irq(irq);
218 }
219
fbnic_request_irq(struct fbnic_dev * fbd,int nr,irq_handler_t handler,unsigned long flags,const char * name,void * data)220 int fbnic_request_irq(struct fbnic_dev *fbd, int nr, irq_handler_t handler,
221 unsigned long flags, const char *name, void *data)
222 {
223 struct pci_dev *pdev = to_pci_dev(fbd->dev);
224 int irq = pci_irq_vector(pdev, nr);
225
226 if (irq < 0)
227 return irq;
228
229 return request_irq(irq, handler, flags, name, data);
230 }
231
fbnic_free_irq(struct fbnic_dev * fbd,int nr,void * data)232 void fbnic_free_irq(struct fbnic_dev *fbd, int nr, void *data)
233 {
234 struct pci_dev *pdev = to_pci_dev(fbd->dev);
235 int irq = pci_irq_vector(pdev, nr);
236
237 if (irq < 0)
238 return;
239
240 free_irq(irq, data);
241 }
242
243 struct fbnic_msix_test_data {
244 struct fbnic_dev *fbd;
245 unsigned long test_msix_status[BITS_TO_LONGS(FBNIC_MAX_MSIX_VECS)];
246 int irq_vector[FBNIC_MAX_MSIX_VECS];
247 };
248
fbnic_irq_test(int irq,void * data)249 static irqreturn_t fbnic_irq_test(int irq, void *data)
250 {
251 struct fbnic_msix_test_data *test_data = data;
252 struct fbnic_dev *fbd = test_data->fbd;
253 int i;
254
255 for (i = fbd->num_irqs; i--;) {
256 if (test_data->irq_vector[i] == irq) {
257 set_bit(i, test_data->test_msix_status);
258 break;
259 }
260 }
261
262 return IRQ_HANDLED;
263 }
264
265 /**
266 * fbnic_msix_test - Verify behavior of NIC interrupts
267 * @fbd: device to test
268 *
269 * This function is meant to test the global interrupt registers and the
270 * PCIe IP MSI-X functionality. It essentially goes through and tests
271 * various combinations of the set, clear, and mask bits in order to
272 * verify the behavior is as we expect it to be from the driver.
273 *
274 * Return: See enum fbnic_msix_self_test_codes
275 **/
fbnic_msix_test(struct fbnic_dev * fbd)276 enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd)
277 {
278 enum fbnic_msix_self_test_codes result = FBNIC_TEST_MSIX_SUCCESS;
279 struct pci_dev *pdev = to_pci_dev(fbd->dev);
280 struct fbnic_msix_test_data *test_data;
281 u32 mask = 0;
282 int i;
283
284 /* Allocate bitmap and IRQ vector table */
285 test_data = kzalloc_obj(*test_data);
286
287 /* memory allocation failure */
288 if (!test_data)
289 return FBNIC_TEST_MSIX_NOMEM;
290
291 /* Initialize test data */
292 test_data->fbd = fbd;
293
294 for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) {
295 /* Add IRQ to vector table so it can be found */
296 test_data->irq_vector[i] = pci_irq_vector(pdev, i);
297
298 /* Enable the interrupt */
299 if (!fbnic_request_irq(fbd, i, fbnic_irq_test, 0,
300 fbd->netdev->name, test_data))
301 continue;
302
303 while (i-- > FBNIC_NON_NAPI_VECTORS)
304 fbnic_free_irq(fbd, i, test_data);
305 kfree(test_data);
306
307 /* IRQ request failure */
308 return FBNIC_TEST_MSIX_IRQ_REQ_FAIL;
309 }
310
311 /* Test each bit individually */
312 for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) {
313 mask = 1U << (i % 32);
314
315 /* Start with mask set and interrupt cleared */
316 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask);
317 fbnic_wrfl(fbd);
318 fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask);
319 fbnic_wrfl(fbd);
320
321 /* masking failure to prevent interrupt */
322 result = FBNIC_TEST_MSIX_MASK;
323
324 fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask);
325 fbnic_wrfl(fbd);
326 usleep_range(10000, 11000);
327
328 if (test_bit(i, test_data->test_msix_status))
329 break;
330
331 /* unmasking failure w/ sw status set */
332 result = FBNIC_TEST_MSIX_UNMASK;
333
334 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask);
335 fbnic_wrfl(fbd);
336 usleep_range(10000, 11000);
337
338 if (!test_bit(i, test_data->test_msix_status))
339 break;
340
341 /* interrupt when clearing mask */
342 result = FBNIC_TEST_MSIX_IRQ_CLEAR;
343
344 clear_bit(i, test_data->test_msix_status);
345 fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask);
346 fbnic_wrfl(fbd);
347 usleep_range(10000, 11000);
348
349 if (test_bit(i, test_data->test_msix_status))
350 break;
351
352 /* interrupt not triggering when not masked */
353 result = FBNIC_TEST_MSIX_NO_INTERRUPT;
354
355 fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask);
356 fbnic_wrfl(fbd);
357 usleep_range(10000, 11000);
358
359 if (!test_bit(i, test_data->test_msix_status))
360 break;
361
362 /* status not cleared, or mask not set */
363 result = FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK;
364 if (mask & fbnic_rd32(fbd, FBNIC_INTR_STATUS(i / 32)))
365 break;
366 if (!(mask & fbnic_rd32(fbd, FBNIC_INTR_MASK(i / 32))))
367 break;
368
369 /* Result = 0 - Success */
370 result = FBNIC_TEST_MSIX_SUCCESS;
371
372 clear_bit(i, test_data->test_msix_status);
373 }
374
375 if (i < fbd->num_irqs) {
376 fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask);
377 fbnic_wrfl(fbd);
378 fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask);
379 fbnic_wrfl(fbd);
380 clear_bit(i, test_data->test_msix_status);
381 }
382
383 for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) {
384 /* Test for bits set after testing */
385 if (test_bit(i, test_data->test_msix_status))
386 result = FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST;
387
388 /* Free IRQ */
389 fbnic_free_irq(fbd, i, test_data);
390 }
391
392 kfree(test_data);
393
394 return result;
395 }
396
fbnic_napi_name_irqs(struct fbnic_dev * fbd)397 void fbnic_napi_name_irqs(struct fbnic_dev *fbd)
398 {
399 unsigned int i;
400
401 for (i = 0; i < ARRAY_SIZE(fbd->napi_irq); i++)
402 snprintf(fbd->napi_irq[i].name,
403 sizeof(fbd->napi_irq[i].name),
404 "%s-TxRx-%u", fbd->netdev->name, i);
405 }
406
fbnic_napi_request_irq(struct fbnic_dev * fbd,struct fbnic_napi_vector * nv)407 int fbnic_napi_request_irq(struct fbnic_dev *fbd,
408 struct fbnic_napi_vector *nv)
409 {
410 struct fbnic_net *fbn = netdev_priv(fbd->netdev);
411 int i = fbnic_napi_idx(nv);
412 int err;
413
414 if (!fbd->napi_irq[i].users) {
415 err = fbnic_request_irq(fbd, nv->v_idx,
416 fbnic_msix_clean_rings, 0,
417 fbd->napi_irq[i].name,
418 &fbn->napi[i]);
419 if (err)
420 return err;
421 }
422
423 fbd->napi_irq[i].users++;
424 return 0;
425 }
426
fbnic_napi_free_irq(struct fbnic_dev * fbd,struct fbnic_napi_vector * nv)427 void fbnic_napi_free_irq(struct fbnic_dev *fbd,
428 struct fbnic_napi_vector *nv)
429 {
430 struct fbnic_net *fbn = netdev_priv(fbd->netdev);
431 int i = fbnic_napi_idx(nv);
432
433 if (--fbd->napi_irq[i].users)
434 return;
435
436 fbnic_free_irq(fbd, nv->v_idx, &fbn->napi[i]);
437 }
438
fbnic_free_irqs(struct fbnic_dev * fbd)439 void fbnic_free_irqs(struct fbnic_dev *fbd)
440 {
441 struct pci_dev *pdev = to_pci_dev(fbd->dev);
442
443 fbd->num_irqs = 0;
444
445 pci_free_irq_vectors(pdev);
446 }
447
fbnic_alloc_irqs(struct fbnic_dev * fbd)448 int fbnic_alloc_irqs(struct fbnic_dev *fbd)
449 {
450 unsigned int wanted_irqs = FBNIC_NON_NAPI_VECTORS;
451 struct pci_dev *pdev = to_pci_dev(fbd->dev);
452 int num_irqs;
453
454 wanted_irqs += min_t(unsigned int, num_online_cpus(), FBNIC_MAX_RXQS);
455 num_irqs = pci_alloc_irq_vectors(pdev, FBNIC_NON_NAPI_VECTORS + 1,
456 wanted_irqs, PCI_IRQ_MSIX);
457 if (num_irqs < 0) {
458 dev_err(fbd->dev, "Failed to allocate MSI-X entries\n");
459 return num_irqs;
460 }
461
462 if (num_irqs < wanted_irqs)
463 dev_warn(fbd->dev, "Allocated %d IRQs, expected %d\n",
464 num_irqs, wanted_irqs);
465
466 fbd->num_irqs = num_irqs;
467
468 return 0;
469 }
470