1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3
4 #include <linux/ethtool.h>
5 #include <linux/printk.h>
6 #include <linux/dynamic_debug.h>
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/interrupt.h>
12 #include <linux/pci.h>
13 #include <linux/cpumask.h>
14 #include <linux/crash_dump.h>
15 #include <linux/vmalloc.h>
16 #include <net/page_pool/helpers.h>
17
18 #include "ionic.h"
19 #include "ionic_bus.h"
20 #include "ionic_dev.h"
21 #include "ionic_lif.h"
22 #include "ionic_aux.h"
23 #include "ionic_txrx.h"
24 #include "ionic_ethtool.h"
25 #include "ionic_debugfs.h"
26
27 /* queuetype support level */
28 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
29 [IONIC_QTYPE_ADMINQ] = 0, /* 0 = Base version with CQ support */
30 [IONIC_QTYPE_NOTIFYQ] = 0, /* 0 = Base version */
31 [IONIC_QTYPE_RXQ] = 2, /* 0 = Base version with CQ+SG support
32 * 2 = ... with CMB rings
33 */
34 [IONIC_QTYPE_TXQ] = 3, /* 0 = Base version with CQ+SG support
35 * 1 = ... with Tx SG version 1
36 * 3 = ... with CMB rings
37 */
38 };
39
40 static void ionic_link_status_check(struct ionic_lif *lif);
41 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
42 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
43 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
44
45 static void ionic_txrx_deinit(struct ionic_lif *lif);
46 static int ionic_txrx_init(struct ionic_lif *lif);
47 static int ionic_start_queues(struct ionic_lif *lif);
48 static void ionic_stop_queues(struct ionic_lif *lif);
49 static void ionic_lif_queue_identify(struct ionic_lif *lif);
50
51 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif);
52 static void ionic_unregister_rxq_info(struct ionic_queue *q);
53 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id);
54
ionic_dim_work(struct work_struct * work)55 static void ionic_dim_work(struct work_struct *work)
56 {
57 struct dim *dim = container_of(work, struct dim, work);
58 struct dim_cq_moder cur_moder;
59 struct ionic_intr_info *intr;
60 struct ionic_qcq *qcq;
61 struct ionic_lif *lif;
62 struct ionic_queue *q;
63 u32 new_coal;
64
65 qcq = container_of(dim, struct ionic_qcq, dim);
66 q = &qcq->q;
67 if (q->type == IONIC_QTYPE_RXQ)
68 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
69 else
70 cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
71 lif = q->lif;
72 new_coal = ionic_coal_usec_to_hw(lif->ionic, cur_moder.usec);
73 new_coal = new_coal ? new_coal : 1;
74
75 intr = &qcq->intr;
76 if (intr->dim_coal_hw != new_coal) {
77 intr->dim_coal_hw = new_coal;
78
79 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
80 intr->index, intr->dim_coal_hw);
81 }
82
83 dim->state = DIM_START_MEASURE;
84 }
85
ionic_lif_deferred_work(struct work_struct * work)86 static void ionic_lif_deferred_work(struct work_struct *work)
87 {
88 struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
89 struct ionic_deferred *def = &lif->deferred;
90 struct ionic_deferred_work *w = NULL;
91
92 do {
93 spin_lock_bh(&def->lock);
94 if (!list_empty(&def->list)) {
95 w = list_first_entry(&def->list,
96 struct ionic_deferred_work, list);
97 list_del(&w->list);
98 }
99 spin_unlock_bh(&def->lock);
100
101 if (!w)
102 break;
103
104 switch (w->type) {
105 case IONIC_DW_TYPE_RX_MODE:
106 ionic_lif_rx_mode(lif);
107 break;
108 case IONIC_DW_TYPE_LINK_STATUS:
109 ionic_link_status_check(lif);
110 break;
111 case IONIC_DW_TYPE_LIF_RESET:
112 if (w->fw_status) {
113 ionic_lif_handle_fw_up(lif);
114 } else {
115 ionic_lif_handle_fw_down(lif);
116
117 /* Fire off another watchdog to see
118 * if the FW is already back rather than
119 * waiting another whole cycle
120 */
121 mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
122 }
123 break;
124 default:
125 break;
126 }
127 kfree(w);
128 w = NULL;
129 } while (true);
130 }
131
ionic_lif_deferred_enqueue(struct ionic_lif * lif,struct ionic_deferred_work * work)132 void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
133 struct ionic_deferred_work *work)
134 {
135 spin_lock_bh(&lif->deferred.lock);
136 list_add_tail(&work->list, &lif->deferred.list);
137 spin_unlock_bh(&lif->deferred.lock);
138 queue_work(lif->ionic->wq, &lif->deferred.work);
139 }
140
ionic_link_status_check(struct ionic_lif * lif)141 static void ionic_link_status_check(struct ionic_lif *lif)
142 {
143 struct ionic_dev *idev = &lif->ionic->idev;
144 struct net_device *netdev = lif->netdev;
145 u16 link_status;
146 bool link_up;
147
148 if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
149 return;
150
151 /* Don't put carrier back up if we're in a broken state */
152 if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
153 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
154 return;
155 }
156
157 ionic_reset_link_down_count(idev);
158
159 link_status = le16_to_cpu(lif->info->status.link_status);
160 link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
161
162 if (link_up) {
163 int err = 0;
164
165 if (netdev->flags & IFF_UP && netif_running(netdev)) {
166 mutex_lock(&lif->queue_lock);
167 err = ionic_start_queues(lif);
168 if (err && err != -EBUSY) {
169 netdev_err(netdev,
170 "Failed to start queues: %d\n", err);
171 set_bit(IONIC_LIF_F_BROKEN, lif->state);
172 netif_carrier_off(lif->netdev);
173 }
174 mutex_unlock(&lif->queue_lock);
175 }
176
177 if (!err && !netif_carrier_ok(netdev)) {
178 ionic_port_identify(lif->ionic);
179 netdev_info(netdev, "Link up - %d Gbps\n",
180 le32_to_cpu(lif->info->status.link_speed) / 1000);
181 netif_carrier_on(netdev);
182 }
183 } else {
184 if (netif_carrier_ok(netdev)) {
185 netdev_info(netdev, "Link down\n");
186 netif_carrier_off(netdev);
187 }
188
189 if (netdev->flags & IFF_UP && netif_running(netdev)) {
190 mutex_lock(&lif->queue_lock);
191 ionic_stop_queues(lif);
192 mutex_unlock(&lif->queue_lock);
193 }
194 }
195
196 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
197 }
198
ionic_link_status_check_request(struct ionic_lif * lif,bool can_sleep)199 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
200 {
201 struct ionic_deferred_work *work;
202
203 /* we only need one request outstanding at a time */
204 if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
205 return;
206
207 if (!can_sleep) {
208 work = kzalloc_obj(*work, GFP_ATOMIC);
209 if (!work) {
210 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
211 return;
212 }
213
214 work->type = IONIC_DW_TYPE_LINK_STATUS;
215 ionic_lif_deferred_enqueue(lif, work);
216 } else {
217 ionic_link_status_check(lif);
218 }
219 }
220
ionic_isr(int irq,void * data)221 static irqreturn_t ionic_isr(int irq, void *data)
222 {
223 struct napi_struct *napi = data;
224
225 napi_schedule_irqoff(napi);
226
227 return IRQ_HANDLED;
228 }
229
ionic_request_irq(struct ionic_lif * lif,struct ionic_qcq * qcq)230 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
231 {
232 struct ionic_intr_info *intr = &qcq->intr;
233 struct device *dev = lif->ionic->dev;
234 struct ionic_queue *q = &qcq->q;
235 const char *name;
236
237 if (lif->registered)
238 name = netdev_name(lif->netdev);
239 else
240 name = dev_name(dev);
241
242 snprintf(intr->name, sizeof(intr->name),
243 "%.5s-%.16s-%.8s", IONIC_DRV_NAME, name, q->name);
244
245 return devm_request_irq(dev, intr->vector, ionic_isr,
246 0, intr->name, &qcq->napi);
247 }
248
ionic_intr_alloc(struct ionic_lif * lif,struct ionic_intr_info * intr)249 int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
250 {
251 struct ionic *ionic = lif->ionic;
252 int index, err;
253
254 index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
255 if (index == ionic->nintrs)
256 return -ENOSPC;
257
258 set_bit(index, ionic->intrs);
259 ionic_intr_init(&ionic->idev, intr, index);
260
261 err = ionic_bus_get_irq(ionic, intr->index);
262 if (err < 0) {
263 clear_bit(index, ionic->intrs);
264 return err;
265 }
266
267 intr->vector = err;
268
269 return 0;
270 }
271 EXPORT_SYMBOL_NS(ionic_intr_alloc, "NET_IONIC");
272
ionic_intr_free(struct ionic_lif * lif,int index)273 void ionic_intr_free(struct ionic_lif *lif, int index)
274 {
275 if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < lif->ionic->nintrs)
276 clear_bit(index, lif->ionic->intrs);
277 }
278 EXPORT_SYMBOL_NS(ionic_intr_free, "NET_IONIC");
279
ionic_irq_aff_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)280 static void ionic_irq_aff_notify(struct irq_affinity_notify *notify,
281 const cpumask_t *mask)
282 {
283 struct ionic_intr_info *intr = container_of(notify, struct ionic_intr_info, aff_notify);
284
285 cpumask_copy(*intr->affinity_mask, mask);
286 }
287
ionic_irq_aff_release(struct kref __always_unused * ref)288 static void ionic_irq_aff_release(struct kref __always_unused *ref)
289 {
290 }
291
ionic_qcq_enable(struct ionic_qcq * qcq)292 static int ionic_qcq_enable(struct ionic_qcq *qcq)
293 {
294 struct ionic_queue *q = &qcq->q;
295 struct ionic_lif *lif = q->lif;
296 struct ionic_dev *idev;
297 struct device *dev;
298
299 struct ionic_admin_ctx ctx = {
300 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
301 .cmd.q_control = {
302 .opcode = IONIC_CMD_Q_CONTROL,
303 .lif_index = cpu_to_le16(lif->index),
304 .type = q->type,
305 .index = cpu_to_le32(q->index),
306 .oper = IONIC_Q_ENABLE,
307 },
308 };
309 int ret;
310
311 idev = &lif->ionic->idev;
312 dev = lif->ionic->dev;
313
314 dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
315 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
316
317 if (qcq->flags & IONIC_QCQ_F_INTR)
318 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
319
320 ret = ionic_adminq_post_wait(lif, &ctx);
321 if (ret)
322 return ret;
323
324 if (qcq->flags & IONIC_QCQ_F_INTR) {
325 napi_enable(&qcq->napi);
326 irq_set_affinity_notifier(qcq->intr.vector,
327 &qcq->intr.aff_notify);
328 irq_set_affinity_hint(qcq->intr.vector,
329 *qcq->intr.affinity_mask);
330 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
331 IONIC_INTR_MASK_CLEAR);
332 }
333
334 return 0;
335 }
336
ionic_qcq_disable(struct ionic_lif * lif,struct ionic_qcq * qcq,int fw_err)337 static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
338 {
339 struct ionic_queue *q;
340
341 struct ionic_admin_ctx ctx = {
342 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
343 .cmd.q_control = {
344 .opcode = IONIC_CMD_Q_CONTROL,
345 .oper = IONIC_Q_DISABLE,
346 },
347 };
348
349 if (!qcq) {
350 netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
351 return -ENXIO;
352 }
353
354 q = &qcq->q;
355
356 if (qcq->flags & IONIC_QCQ_F_INTR) {
357 struct ionic_dev *idev = &lif->ionic->idev;
358
359 if (lif->doorbell_wa)
360 cancel_work_sync(&qcq->doorbell_napi_work);
361 cancel_work_sync(&qcq->dim.work);
362 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
363 IONIC_INTR_MASK_SET);
364 synchronize_irq(qcq->intr.vector);
365 irq_set_affinity_notifier(qcq->intr.vector, NULL);
366 irq_set_affinity_hint(qcq->intr.vector, NULL);
367 napi_disable(&qcq->napi);
368 }
369
370 /* If there was a previous fw communcation error, don't bother with
371 * sending the adminq command and just return the same error value.
372 */
373 if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
374 return fw_err;
375
376 ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
377 ctx.cmd.q_control.type = q->type;
378 ctx.cmd.q_control.index = cpu_to_le32(q->index);
379 dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
380 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
381
382 return ionic_adminq_post_wait(lif, &ctx);
383 }
384
ionic_lif_qcq_deinit(struct ionic_lif * lif,struct ionic_qcq * qcq)385 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
386 {
387 struct ionic_dev *idev = &lif->ionic->idev;
388
389 if (!qcq)
390 return;
391
392 if (!(qcq->flags & IONIC_QCQ_F_INITED))
393 return;
394
395 ionic_unregister_rxq_info(&qcq->q);
396 if (qcq->flags & IONIC_QCQ_F_INTR) {
397 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
398 IONIC_INTR_MASK_SET);
399 netif_napi_del(&qcq->napi);
400 }
401
402 qcq->flags &= ~IONIC_QCQ_F_INITED;
403 }
404
ionic_qcq_intr_free(struct ionic_lif * lif,struct ionic_qcq * qcq)405 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
406 {
407 if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
408 return;
409
410 irq_set_affinity_hint(qcq->intr.vector, NULL);
411 devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
412 qcq->intr.vector = 0;
413 ionic_intr_free(lif, qcq->intr.index);
414 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
415 }
416
ionic_qcq_free(struct ionic_lif * lif,struct ionic_qcq * qcq)417 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
418 {
419 struct device *dev = lif->ionic->dev;
420
421 if (!qcq)
422 return;
423
424 ionic_debugfs_del_qcq(qcq);
425
426 if (qcq->q_base) {
427 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
428 qcq->q_base = NULL;
429 qcq->q_base_pa = 0;
430 }
431
432 if (qcq->cmb_q_base) {
433 iounmap(qcq->cmb_q_base);
434 ionic_put_cmb(lif, qcq->cmb_pgid, qcq->cmb_order);
435 qcq->cmb_pgid = 0;
436 qcq->cmb_order = 0;
437 qcq->cmb_q_base = NULL;
438 qcq->cmb_q_base_pa = 0;
439 }
440
441 if (qcq->cq_base) {
442 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
443 qcq->cq_base = NULL;
444 qcq->cq_base_pa = 0;
445 }
446
447 if (qcq->sg_base) {
448 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
449 qcq->sg_base = NULL;
450 qcq->sg_base_pa = 0;
451 }
452
453 page_pool_destroy(qcq->q.page_pool);
454 qcq->q.page_pool = NULL;
455
456 ionic_qcq_intr_free(lif, qcq);
457 vfree(qcq->q.info);
458 qcq->q.info = NULL;
459 }
460
ionic_qcqs_free(struct ionic_lif * lif)461 void ionic_qcqs_free(struct ionic_lif *lif)
462 {
463 struct device *dev = lif->ionic->dev;
464 struct ionic_qcq *adminqcq;
465 unsigned long irqflags;
466
467 if (lif->notifyqcq) {
468 ionic_qcq_free(lif, lif->notifyqcq);
469 devm_kfree(dev, lif->notifyqcq);
470 lif->notifyqcq = NULL;
471 }
472
473 if (lif->adminqcq) {
474 spin_lock_irqsave(&lif->adminq_lock, irqflags);
475 adminqcq = READ_ONCE(lif->adminqcq);
476 lif->adminqcq = NULL;
477 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
478 if (adminqcq) {
479 ionic_qcq_free(lif, adminqcq);
480 devm_kfree(dev, adminqcq);
481 }
482 }
483
484 if (lif->rxqcqs) {
485 devm_kfree(dev, lif->rxqstats);
486 lif->rxqstats = NULL;
487 devm_kfree(dev, lif->rxqcqs);
488 lif->rxqcqs = NULL;
489 }
490
491 if (lif->txqcqs) {
492 devm_kfree(dev, lif->txqstats);
493 lif->txqstats = NULL;
494 devm_kfree(dev, lif->txqcqs);
495 lif->txqcqs = NULL;
496 }
497 }
498
ionic_link_qcq_interrupts(struct ionic_qcq * src_qcq,struct ionic_qcq * n_qcq)499 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
500 struct ionic_qcq *n_qcq)
501 {
502 n_qcq->intr.vector = src_qcq->intr.vector;
503 n_qcq->intr.index = src_qcq->intr.index;
504 }
505
ionic_alloc_qcq_interrupt(struct ionic_lif * lif,struct ionic_qcq * qcq)506 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
507 {
508 cpumask_var_t *affinity_mask;
509 int err;
510
511 if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
512 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
513 return 0;
514 }
515
516 err = ionic_intr_alloc(lif, &qcq->intr);
517 if (err) {
518 netdev_warn(lif->netdev, "no intr for %s: %d\n",
519 qcq->q.name, err);
520 goto err_out;
521 }
522
523 ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
524 IONIC_INTR_MASK_SET);
525
526 err = ionic_request_irq(lif, qcq);
527 if (err) {
528 netdev_warn(lif->netdev, "irq request failed %d\n", err);
529 goto err_out_free_intr;
530 }
531
532 /* try to get the irq on the local numa node first */
533 affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];
534 if (cpumask_empty(*affinity_mask)) {
535 unsigned int cpu;
536
537 cpu = cpumask_local_spread(qcq->intr.index,
538 dev_to_node(lif->ionic->dev));
539 if (cpu != -1)
540 cpumask_set_cpu(cpu, *affinity_mask);
541 }
542
543 qcq->intr.affinity_mask = affinity_mask;
544 qcq->intr.aff_notify.notify = ionic_irq_aff_notify;
545 qcq->intr.aff_notify.release = ionic_irq_aff_release;
546
547 netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
548 return 0;
549
550 err_out_free_intr:
551 ionic_intr_free(lif, qcq->intr.index);
552 err_out:
553 return err;
554 }
555
ionic_qcq_alloc(struct ionic_lif * lif,unsigned int type,unsigned int index,const char * name,unsigned int flags,unsigned int num_descs,unsigned int desc_size,unsigned int cq_desc_size,unsigned int sg_desc_size,unsigned int desc_info_size,unsigned int pid,struct bpf_prog * xdp_prog,struct ionic_qcq ** qcq)556 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
557 unsigned int index,
558 const char *name, unsigned int flags,
559 unsigned int num_descs, unsigned int desc_size,
560 unsigned int cq_desc_size,
561 unsigned int sg_desc_size,
562 unsigned int desc_info_size,
563 unsigned int pid, struct bpf_prog *xdp_prog,
564 struct ionic_qcq **qcq)
565 {
566 struct ionic_dev *idev = &lif->ionic->idev;
567 struct device *dev = lif->ionic->dev;
568 struct ionic_qcq *new;
569 int err;
570
571 *qcq = NULL;
572
573 new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
574 if (!new) {
575 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
576 err = -ENOMEM;
577 goto err_out;
578 }
579
580 new->q.dev = dev;
581 new->flags = flags;
582
583 new->q.info = vcalloc(num_descs, desc_info_size);
584 if (!new->q.info) {
585 netdev_err(lif->netdev, "Cannot allocate queue info\n");
586 err = -ENOMEM;
587 goto err_out_free_qcq;
588 }
589
590 if (type == IONIC_QTYPE_RXQ) {
591 struct page_pool_params pp_params = {
592 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
593 .order = 0,
594 .pool_size = num_descs,
595 .nid = NUMA_NO_NODE,
596 .dev = lif->ionic->dev,
597 .napi = &new->napi,
598 .dma_dir = DMA_FROM_DEVICE,
599 .max_len = PAGE_SIZE,
600 .netdev = lif->netdev,
601 };
602
603 if (xdp_prog)
604 pp_params.dma_dir = DMA_BIDIRECTIONAL;
605
606 new->q.page_pool = page_pool_create(&pp_params);
607 if (IS_ERR(new->q.page_pool)) {
608 netdev_err(lif->netdev, "Cannot create page_pool\n");
609 err = PTR_ERR(new->q.page_pool);
610 new->q.page_pool = NULL;
611 goto err_out_free_q_info;
612 }
613 }
614
615 new->q.type = type;
616 new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
617
618 err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
619 desc_size, sg_desc_size, pid);
620 if (err) {
621 netdev_err(lif->netdev, "Cannot initialize queue\n");
622 goto err_out_free_page_pool;
623 }
624
625 err = ionic_alloc_qcq_interrupt(lif, new);
626 if (err)
627 goto err_out_free_page_pool;
628
629 err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
630 if (err) {
631 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
632 goto err_out_free_irq;
633 }
634
635 if (flags & IONIC_QCQ_F_NOTIFYQ) {
636 int q_size;
637
638 /* q & cq need to be contiguous in NotifyQ, so alloc it all in q
639 * and don't alloc qc. We leave new->qc_size and new->qc_base
640 * as 0 to be sure we don't try to free it later.
641 */
642 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
643 new->q_size = PAGE_SIZE + q_size +
644 ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
645 new->q_base = dma_alloc_coherent(dev, new->q_size,
646 &new->q_base_pa, GFP_KERNEL);
647 if (!new->q_base) {
648 netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
649 err = -ENOMEM;
650 goto err_out_free_irq;
651 }
652 new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
653 new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
654
655 /* Base the NotifyQ cq.base off of the ALIGNed q.base */
656 new->cq.base = PTR_ALIGN(new->q.base + q_size, PAGE_SIZE);
657 new->cq.base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
658 new->cq.bound_q = &new->q;
659 } else {
660 /* regular DMA q descriptors */
661 new->q_size = PAGE_SIZE + (num_descs * desc_size);
662 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
663 GFP_KERNEL);
664 if (!new->q_base) {
665 netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
666 err = -ENOMEM;
667 goto err_out_free_irq;
668 }
669 new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
670 new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
671
672 if (flags & IONIC_QCQ_F_CMB_RINGS) {
673 /* on-chip CMB q descriptors */
674 new->cmb_q_size = num_descs * desc_size;
675 new->cmb_order = order_base_2(new->cmb_q_size / PAGE_SIZE);
676
677 err = ionic_get_cmb(lif, &new->cmb_pgid, &new->cmb_q_base_pa,
678 new->cmb_order, 0, NULL);
679 if (err) {
680 netdev_err(lif->netdev,
681 "Cannot allocate queue order %d from cmb: err %d\n",
682 new->cmb_order, err);
683 goto err_out_free_q;
684 }
685
686 new->cmb_q_base = ioremap_wc(new->cmb_q_base_pa, new->cmb_q_size);
687 if (!new->cmb_q_base) {
688 netdev_err(lif->netdev, "Cannot map queue from cmb\n");
689 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
690 err = -ENOMEM;
691 goto err_out_free_q;
692 }
693
694 new->cmb_q_base_pa -= idev->phy_cmb_pages;
695 new->q.cmb_base = new->cmb_q_base;
696 new->q.cmb_base_pa = new->cmb_q_base_pa;
697 }
698
699 /* cq DMA descriptors */
700 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
701 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
702 GFP_KERNEL);
703 if (!new->cq_base) {
704 netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
705 err = -ENOMEM;
706 goto err_out_free_q;
707 }
708 new->cq.base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
709 new->cq.base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
710 new->cq.bound_q = &new->q;
711 }
712
713 if (flags & IONIC_QCQ_F_SG) {
714 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
715 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
716 GFP_KERNEL);
717 if (!new->sg_base) {
718 netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
719 err = -ENOMEM;
720 goto err_out_free_cq;
721 }
722 new->q.sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
723 new->q.sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
724 }
725
726 INIT_WORK(&new->dim.work, ionic_dim_work);
727 new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
728 if (lif->doorbell_wa)
729 INIT_WORK(&new->doorbell_napi_work, ionic_doorbell_napi_work);
730
731 *qcq = new;
732
733 return 0;
734
735 err_out_free_cq:
736 dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
737 err_out_free_q:
738 if (new->cmb_q_base) {
739 iounmap(new->cmb_q_base);
740 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
741 }
742 dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
743 err_out_free_irq:
744 if (flags & IONIC_QCQ_F_INTR) {
745 devm_free_irq(dev, new->intr.vector, &new->napi);
746 ionic_intr_free(lif, new->intr.index);
747 }
748 err_out_free_page_pool:
749 page_pool_destroy(new->q.page_pool);
750 err_out_free_q_info:
751 vfree(new->q.info);
752 err_out_free_qcq:
753 devm_kfree(dev, new);
754 err_out:
755 dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
756 return err;
757 }
758
ionic_qcqs_alloc(struct ionic_lif * lif)759 static int ionic_qcqs_alloc(struct ionic_lif *lif)
760 {
761 struct device *dev = lif->ionic->dev;
762 unsigned int flags;
763 int err;
764
765 flags = IONIC_QCQ_F_INTR;
766 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
767 IONIC_ADMINQ_LENGTH,
768 sizeof(struct ionic_admin_cmd),
769 sizeof(struct ionic_admin_comp),
770 0,
771 sizeof(struct ionic_admin_desc_info),
772 lif->kern_pid, NULL, &lif->adminqcq);
773 if (err)
774 return err;
775 ionic_debugfs_add_qcq(lif, lif->adminqcq);
776
777 if (lif->ionic->nnqs_per_lif) {
778 flags = IONIC_QCQ_F_NOTIFYQ;
779 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
780 flags, IONIC_NOTIFYQ_LENGTH,
781 sizeof(struct ionic_notifyq_cmd),
782 sizeof(union ionic_notifyq_comp),
783 0,
784 sizeof(struct ionic_admin_desc_info),
785 lif->kern_pid, NULL, &lif->notifyqcq);
786 if (err)
787 goto err_out;
788 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
789
790 /* Let the notifyq ride on the adminq interrupt */
791 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
792 }
793
794 err = -ENOMEM;
795 lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
796 sizeof(*lif->txqcqs), GFP_KERNEL);
797 if (!lif->txqcqs)
798 goto err_out;
799 lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
800 sizeof(*lif->rxqcqs), GFP_KERNEL);
801 if (!lif->rxqcqs)
802 goto err_out;
803
804 lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
805 sizeof(*lif->txqstats), GFP_KERNEL);
806 if (!lif->txqstats)
807 goto err_out;
808 lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
809 sizeof(*lif->rxqstats), GFP_KERNEL);
810 if (!lif->rxqstats)
811 goto err_out;
812
813 return 0;
814
815 err_out:
816 ionic_qcqs_free(lif);
817 return err;
818 }
819
ionic_qcq_sanitize(struct ionic_qcq * qcq)820 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
821 {
822 qcq->q.tail_idx = 0;
823 qcq->q.head_idx = 0;
824 qcq->cq.tail_idx = 0;
825 qcq->cq.done_color = 1;
826 memset(qcq->q_base, 0, qcq->q_size);
827 if (qcq->cmb_q_base)
828 memset_io(qcq->cmb_q_base, 0, qcq->cmb_q_size);
829 memset(qcq->cq_base, 0, qcq->cq_size);
830 memset(qcq->sg_base, 0, qcq->sg_size);
831 }
832
ionic_lif_txq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)833 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
834 {
835 struct device *dev = lif->ionic->dev;
836 struct ionic_queue *q = &qcq->q;
837 struct ionic_cq *cq = &qcq->cq;
838 struct ionic_admin_ctx ctx = {
839 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
840 .cmd.q_init = {
841 .opcode = IONIC_CMD_Q_INIT,
842 .lif_index = cpu_to_le16(lif->index),
843 .type = q->type,
844 .ver = lif->qtype_info[q->type].version,
845 .index = cpu_to_le32(q->index),
846 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
847 IONIC_QINIT_F_SG),
848 .intr_index = cpu_to_le16(qcq->intr.index),
849 .pid = cpu_to_le16(q->pid),
850 .ring_size = ilog2(q->num_descs),
851 .ring_base = cpu_to_le64(q->base_pa),
852 .cq_ring_base = cpu_to_le64(cq->base_pa),
853 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
854 .features = cpu_to_le64(q->features),
855 },
856 };
857 int err;
858
859 if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
860 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
861 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
862 }
863
864 dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
865 dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
866 dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
867 dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
868 dev_dbg(dev, "txq_init.cq_ring_base 0x%llx\n", ctx.cmd.q_init.cq_ring_base);
869 dev_dbg(dev, "txq_init.sg_ring_base 0x%llx\n", ctx.cmd.q_init.sg_ring_base);
870 dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
871 dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
872 dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
873
874 ionic_qcq_sanitize(qcq);
875
876 err = ionic_adminq_post_wait(lif, &ctx);
877 if (err)
878 return err;
879
880 q->hw_type = ctx.comp.q_init.hw_type;
881 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
882 q->dbval = IONIC_DBELL_QID(q->hw_index);
883
884 dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
885 dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
886
887 q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
888 q->dbell_jiffies = jiffies;
889
890 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
891 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
892
893 qcq->flags |= IONIC_QCQ_F_INITED;
894
895 return 0;
896 }
897
ionic_lif_rxq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)898 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
899 {
900 struct device *dev = lif->ionic->dev;
901 struct ionic_queue *q = &qcq->q;
902 struct ionic_cq *cq = &qcq->cq;
903 struct ionic_admin_ctx ctx = {
904 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
905 .cmd.q_init = {
906 .opcode = IONIC_CMD_Q_INIT,
907 .lif_index = cpu_to_le16(lif->index),
908 .type = q->type,
909 .ver = lif->qtype_info[q->type].version,
910 .index = cpu_to_le32(q->index),
911 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ),
912 .intr_index = cpu_to_le16(cq->bound_intr->index),
913 .pid = cpu_to_le16(q->pid),
914 .ring_size = ilog2(q->num_descs),
915 .ring_base = cpu_to_le64(q->base_pa),
916 .cq_ring_base = cpu_to_le64(cq->base_pa),
917 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
918 .features = cpu_to_le64(q->features),
919 },
920 };
921 int err;
922
923 q->partner = NULL;
924
925 /* Only normal RX queues have matching TX queue partners. */
926 if (q->index < lif->nxqs) {
927 if (!lif->txqcqs ||
928 q->index >= lif->ionic->ntxqs_per_lif ||
929 !lif->txqcqs[q->index]) {
930 dev_err(dev, "missing TX queue partner for RX queue %u\n",
931 q->index);
932 return -ENXIO;
933 }
934
935 q->partner = &lif->txqcqs[q->index]->q;
936 q->partner->partner = q;
937 }
938
939 if (!lif->xdp_prog ||
940 (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags))
941 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
942
943 if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
944 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
945 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
946 }
947
948 dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
949 dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
950 dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
951 dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
952 dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
953 dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
954 dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
955
956 ionic_qcq_sanitize(qcq);
957
958 err = ionic_adminq_post_wait(lif, &ctx);
959 if (err)
960 return err;
961
962 q->hw_type = ctx.comp.q_init.hw_type;
963 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
964 q->dbval = IONIC_DBELL_QID(q->hw_index);
965
966 dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
967 dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
968
969 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
970 q->dbell_jiffies = jiffies;
971
972 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
973 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
974 else
975 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
976 err = ionic_register_rxq_info(q, qcq->napi.napi_id);
977 if (err) {
978 netif_napi_del(&qcq->napi);
979 return err;
980 }
981
982 qcq->flags |= IONIC_QCQ_F_INITED;
983
984 return 0;
985 }
986
ionic_lif_create_hwstamp_txq(struct ionic_lif * lif)987 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
988 {
989 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
990 unsigned int txq_i, flags;
991 struct ionic_qcq *txq;
992 u64 features;
993 int err;
994
995 if (lif->hwstamp_txq)
996 return 0;
997
998 features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
999
1000 num_desc = IONIC_MIN_TXRX_DESC;
1001 desc_sz = sizeof(struct ionic_txq_desc);
1002 comp_sz = 2 * sizeof(struct ionic_txq_comp);
1003
1004 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
1005 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
1006 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
1007 else
1008 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
1009
1010 txq_i = lif->ionic->ntxqs_per_lif;
1011 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
1012
1013 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
1014 num_desc, desc_sz, comp_sz, sg_desc_sz,
1015 sizeof(struct ionic_tx_desc_info),
1016 lif->kern_pid, NULL, &txq);
1017 if (err)
1018 goto err_qcq_alloc;
1019
1020 txq->q.features = features;
1021
1022 ionic_link_qcq_interrupts(lif->adminqcq, txq);
1023 ionic_debugfs_add_qcq(lif, txq);
1024
1025 lif->hwstamp_txq = txq;
1026
1027 if (netif_running(lif->netdev)) {
1028 err = ionic_lif_txq_init(lif, txq);
1029 if (err)
1030 goto err_qcq_init;
1031
1032 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1033 err = ionic_qcq_enable(txq);
1034 if (err)
1035 goto err_qcq_enable;
1036 }
1037 }
1038
1039 return 0;
1040
1041 err_qcq_enable:
1042 ionic_lif_qcq_deinit(lif, txq);
1043 err_qcq_init:
1044 lif->hwstamp_txq = NULL;
1045 ionic_debugfs_del_qcq(txq);
1046 ionic_qcq_free(lif, txq);
1047 devm_kfree(lif->ionic->dev, txq);
1048 err_qcq_alloc:
1049 return err;
1050 }
1051
ionic_lif_create_hwstamp_rxq(struct ionic_lif * lif)1052 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
1053 {
1054 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
1055 unsigned int rxq_i, flags;
1056 struct ionic_qcq *rxq;
1057 u64 features;
1058 int err;
1059
1060 if (lif->hwstamp_rxq)
1061 return 0;
1062
1063 features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1064
1065 num_desc = IONIC_MIN_TXRX_DESC;
1066 desc_sz = sizeof(struct ionic_rxq_desc);
1067 comp_sz = 2 * sizeof(struct ionic_rxq_comp);
1068 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
1069
1070 rxq_i = lif->ionic->nrxqs_per_lif;
1071 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
1072
1073 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
1074 num_desc, desc_sz, comp_sz, sg_desc_sz,
1075 sizeof(struct ionic_rx_desc_info),
1076 lif->kern_pid, NULL, &rxq);
1077 if (err)
1078 goto err_qcq_alloc;
1079
1080 rxq->q.features = features;
1081
1082 ionic_link_qcq_interrupts(lif->adminqcq, rxq);
1083 ionic_debugfs_add_qcq(lif, rxq);
1084
1085 lif->hwstamp_rxq = rxq;
1086
1087 if (netif_running(lif->netdev)) {
1088 err = ionic_lif_rxq_init(lif, rxq);
1089 if (err)
1090 goto err_qcq_init;
1091
1092 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1093 ionic_rx_fill(&rxq->q, NULL);
1094 err = ionic_qcq_enable(rxq);
1095 if (err)
1096 goto err_qcq_enable;
1097 }
1098 }
1099
1100 return 0;
1101
1102 err_qcq_enable:
1103 ionic_lif_qcq_deinit(lif, rxq);
1104 err_qcq_init:
1105 lif->hwstamp_rxq = NULL;
1106 ionic_debugfs_del_qcq(rxq);
1107 ionic_qcq_free(lif, rxq);
1108 devm_kfree(lif->ionic->dev, rxq);
1109 err_qcq_alloc:
1110 return err;
1111 }
1112
ionic_lif_config_hwstamp_rxq_all(struct ionic_lif * lif,bool rx_all)1113 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
1114 {
1115 struct ionic_queue_params qparam;
1116
1117 ionic_init_queue_params(lif, &qparam);
1118
1119 if (rx_all)
1120 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1121 else
1122 qparam.rxq_features = 0;
1123
1124 /* if we're not running, just set the values and return */
1125 if (!netif_running(lif->netdev)) {
1126 lif->rxq_features = qparam.rxq_features;
1127 return 0;
1128 }
1129
1130 return ionic_reconfigure_queues(lif, &qparam);
1131 }
1132
ionic_lif_set_hwstamp_txmode(struct ionic_lif * lif,u16 txstamp_mode)1133 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1134 {
1135 struct ionic_admin_ctx ctx = {
1136 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1137 .cmd.lif_setattr = {
1138 .opcode = IONIC_CMD_LIF_SETATTR,
1139 .index = cpu_to_le16(lif->index),
1140 .attr = IONIC_LIF_ATTR_TXSTAMP,
1141 .txstamp_mode = cpu_to_le16(txstamp_mode),
1142 },
1143 };
1144
1145 return ionic_adminq_post_wait(lif, &ctx);
1146 }
1147
ionic_lif_del_hwstamp_rxfilt(struct ionic_lif * lif)1148 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1149 {
1150 struct ionic_admin_ctx ctx = {
1151 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1152 .cmd.rx_filter_del = {
1153 .opcode = IONIC_CMD_RX_FILTER_DEL,
1154 .lif_index = cpu_to_le16(lif->index),
1155 },
1156 };
1157 struct ionic_rx_filter *f;
1158 u32 filter_id;
1159 int err;
1160
1161 spin_lock_bh(&lif->rx_filters.lock);
1162
1163 f = ionic_rx_filter_rxsteer(lif);
1164 if (!f) {
1165 spin_unlock_bh(&lif->rx_filters.lock);
1166 return;
1167 }
1168
1169 filter_id = f->filter_id;
1170 ionic_rx_filter_free(lif, f);
1171
1172 spin_unlock_bh(&lif->rx_filters.lock);
1173
1174 netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1175
1176 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1177
1178 err = ionic_adminq_post_wait(lif, &ctx);
1179 if (err && err != -EEXIST)
1180 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1181 }
1182
ionic_lif_add_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1183 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1184 {
1185 struct ionic_admin_ctx ctx = {
1186 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1187 .cmd.rx_filter_add = {
1188 .opcode = IONIC_CMD_RX_FILTER_ADD,
1189 .lif_index = cpu_to_le16(lif->index),
1190 .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1191 .pkt_class = cpu_to_le64(pkt_class),
1192 },
1193 };
1194 u8 qtype;
1195 u32 qid;
1196 int err;
1197
1198 if (!lif->hwstamp_rxq)
1199 return -EINVAL;
1200
1201 qtype = lif->hwstamp_rxq->q.type;
1202 ctx.cmd.rx_filter_add.qtype = qtype;
1203
1204 qid = lif->hwstamp_rxq->q.index;
1205 ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1206
1207 netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1208 err = ionic_adminq_post_wait(lif, &ctx);
1209 if (err && err != -EEXIST)
1210 return err;
1211
1212 spin_lock_bh(&lif->rx_filters.lock);
1213 err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
1214 spin_unlock_bh(&lif->rx_filters.lock);
1215
1216 return err;
1217 }
1218
ionic_lif_set_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1219 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1220 {
1221 ionic_lif_del_hwstamp_rxfilt(lif);
1222
1223 if (!pkt_class)
1224 return 0;
1225
1226 return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1227 }
1228
ionic_adminq_napi(struct napi_struct * napi,int budget)1229 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1230 {
1231 struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1232 struct ionic_lif *lif = napi_to_cq(napi)->lif;
1233 struct ionic_dev *idev = &lif->ionic->idev;
1234 unsigned long irqflags;
1235 unsigned int flags = 0;
1236 int rx_work = 0;
1237 int tx_work = 0;
1238 int n_work = 0;
1239 int a_work = 0;
1240 int work_done;
1241 int credits;
1242
1243 if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1244 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1245 ionic_notifyq_service, NULL, NULL);
1246
1247 spin_lock_irqsave(&lif->adminq_lock, irqflags);
1248 if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1249 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1250 ionic_adminq_service, NULL, NULL);
1251
1252 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1253
1254 if (lif->hwstamp_rxq)
1255 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1256 ionic_rx_service, NULL, NULL);
1257
1258 if (lif->hwstamp_txq)
1259 tx_work = ionic_tx_cq_service(&lif->hwstamp_txq->cq, budget, !!budget);
1260
1261 work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1262 if (work_done < budget && napi_complete_done(napi, work_done)) {
1263 flags |= IONIC_INTR_CRED_UNMASK;
1264 intr->rearm_count++;
1265 }
1266
1267 if (work_done || flags) {
1268 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1269 credits = n_work + a_work + rx_work + tx_work;
1270 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1271 }
1272
1273 if (lif->doorbell_wa) {
1274 if (!a_work)
1275 ionic_adminq_poke_doorbell(&lif->adminqcq->q);
1276 if (lif->hwstamp_rxq && !rx_work)
1277 ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q);
1278 if (lif->hwstamp_txq && !tx_work)
1279 ionic_txq_poke_doorbell(&lif->hwstamp_txq->q);
1280 }
1281
1282 return work_done;
1283 }
1284
ionic_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * ns)1285 void ionic_get_stats64(struct net_device *netdev,
1286 struct rtnl_link_stats64 *ns)
1287 {
1288 struct ionic_lif *lif = netdev_priv(netdev);
1289 struct ionic_lif_stats *ls;
1290
1291 memset(ns, 0, sizeof(*ns));
1292 ls = &lif->info->stats;
1293
1294 ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1295 le64_to_cpu(ls->rx_mcast_packets) +
1296 le64_to_cpu(ls->rx_bcast_packets);
1297
1298 ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1299 le64_to_cpu(ls->tx_mcast_packets) +
1300 le64_to_cpu(ls->tx_bcast_packets);
1301
1302 ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1303 le64_to_cpu(ls->rx_mcast_bytes) +
1304 le64_to_cpu(ls->rx_bcast_bytes);
1305
1306 ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1307 le64_to_cpu(ls->tx_mcast_bytes) +
1308 le64_to_cpu(ls->tx_bcast_bytes);
1309
1310 ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1311 le64_to_cpu(ls->rx_mcast_drop_packets) +
1312 le64_to_cpu(ls->rx_bcast_drop_packets);
1313
1314 ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1315 le64_to_cpu(ls->tx_mcast_drop_packets) +
1316 le64_to_cpu(ls->tx_bcast_drop_packets);
1317
1318 ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1319
1320 ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1321
1322 ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1323 le64_to_cpu(ls->rx_queue_disabled) +
1324 le64_to_cpu(ls->rx_desc_fetch_error) +
1325 le64_to_cpu(ls->rx_desc_data_error);
1326
1327 ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1328 le64_to_cpu(ls->tx_queue_disabled) +
1329 le64_to_cpu(ls->tx_desc_fetch_error) +
1330 le64_to_cpu(ls->tx_desc_data_error);
1331
1332 ns->rx_errors = ns->rx_over_errors +
1333 ns->rx_missed_errors;
1334
1335 ns->tx_errors = ns->tx_aborted_errors;
1336 }
1337
ionic_addr_add(struct net_device * netdev,const u8 * addr)1338 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1339 {
1340 return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
1341 }
1342
ionic_addr_del(struct net_device * netdev,const u8 * addr)1343 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1344 {
1345 /* Don't delete our own address from the uc list */
1346 if (ether_addr_equal(addr, netdev->dev_addr))
1347 return 0;
1348
1349 return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
1350 }
1351
ionic_lif_rx_mode(struct ionic_lif * lif)1352 void ionic_lif_rx_mode(struct ionic_lif *lif)
1353 {
1354 struct net_device *netdev = lif->netdev;
1355 unsigned int nfilters;
1356 unsigned int nd_flags;
1357 char buf[128];
1358 u16 rx_mode;
1359 int i;
1360 #define REMAIN(__x) (sizeof(buf) - (__x))
1361
1362 mutex_lock(&lif->config_lock);
1363
1364 /* grab the flags once for local use */
1365 nd_flags = netdev->flags;
1366
1367 rx_mode = IONIC_RX_MODE_F_UNICAST;
1368 rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1369 rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1370 rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1371 rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1372
1373 /* sync the filters */
1374 ionic_rx_filter_sync(lif);
1375
1376 /* check for overflow state
1377 * if so, we track that we overflowed and enable NIC PROMISC
1378 * else if the overflow is set and not needed
1379 * we remove our overflow flag and check the netdev flags
1380 * to see if we can disable NIC PROMISC
1381 */
1382 nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1383
1384 if (((lif->nucast + lif->nmcast) >= nfilters) ||
1385 (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
1386 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1387 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1388 } else {
1389 if (!(nd_flags & IFF_PROMISC))
1390 rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1391 if (!(nd_flags & IFF_ALLMULTI))
1392 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1393 }
1394
1395 i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1396 lif->rx_mode, rx_mode);
1397 if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1398 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1399 if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1400 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1401 if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1402 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1403 if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1404 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1405 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1406 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1407 if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1408 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1409 netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1410
1411 if (lif->rx_mode != rx_mode) {
1412 struct ionic_admin_ctx ctx = {
1413 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1414 .cmd.rx_mode_set = {
1415 .opcode = IONIC_CMD_RX_MODE_SET,
1416 .lif_index = cpu_to_le16(lif->index),
1417 },
1418 };
1419 int err;
1420
1421 ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1422 err = ionic_adminq_post_wait(lif, &ctx);
1423 if (err)
1424 netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1425 rx_mode, err);
1426 else
1427 lif->rx_mode = rx_mode;
1428 }
1429
1430 mutex_unlock(&lif->config_lock);
1431 }
1432
ionic_ndo_set_rx_mode(struct net_device * netdev)1433 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1434 {
1435 struct ionic_lif *lif = netdev_priv(netdev);
1436 struct ionic_deferred_work *work;
1437
1438 /* Sync the kernel filter list with the driver filter list */
1439 __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1440 __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1441
1442 /* Shove off the rest of the rxmode work to the work task
1443 * which will include syncing the filters to the firmware.
1444 */
1445 work = kzalloc_obj(*work, GFP_ATOMIC);
1446 if (!work) {
1447 netdev_err(lif->netdev, "rxmode change dropped\n");
1448 return;
1449 }
1450 work->type = IONIC_DW_TYPE_RX_MODE;
1451 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1452 ionic_lif_deferred_enqueue(lif, work);
1453 }
1454
ionic_netdev_features_to_nic(netdev_features_t features)1455 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1456 {
1457 u64 wanted = 0;
1458
1459 if (features & NETIF_F_HW_VLAN_CTAG_TX)
1460 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1461 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1462 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1463 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1464 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1465 if (features & NETIF_F_RXHASH)
1466 wanted |= IONIC_ETH_HW_RX_HASH;
1467 if (features & NETIF_F_RXCSUM)
1468 wanted |= IONIC_ETH_HW_RX_CSUM;
1469 if (features & NETIF_F_SG)
1470 wanted |= IONIC_ETH_HW_TX_SG;
1471 if (features & NETIF_F_HW_CSUM)
1472 wanted |= IONIC_ETH_HW_TX_CSUM;
1473 if (features & NETIF_F_TSO)
1474 wanted |= IONIC_ETH_HW_TSO;
1475 if (features & NETIF_F_TSO6)
1476 wanted |= IONIC_ETH_HW_TSO_IPV6;
1477 if (features & NETIF_F_TSO_ECN)
1478 wanted |= IONIC_ETH_HW_TSO_ECN;
1479 if (features & NETIF_F_GSO_GRE)
1480 wanted |= IONIC_ETH_HW_TSO_GRE;
1481 if (features & NETIF_F_GSO_GRE_CSUM)
1482 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1483 if (features & NETIF_F_GSO_IPXIP4)
1484 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1485 if (features & NETIF_F_GSO_IPXIP6)
1486 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1487 if (features & NETIF_F_GSO_UDP_TUNNEL)
1488 wanted |= IONIC_ETH_HW_TSO_UDP;
1489 if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1490 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1491
1492 return cpu_to_le64(wanted);
1493 }
1494
ionic_set_nic_features(struct ionic_lif * lif,netdev_features_t features)1495 static int ionic_set_nic_features(struct ionic_lif *lif,
1496 netdev_features_t features)
1497 {
1498 struct device *dev = lif->ionic->dev;
1499 struct ionic_admin_ctx ctx = {
1500 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1501 .cmd.lif_setattr = {
1502 .opcode = IONIC_CMD_LIF_SETATTR,
1503 .index = cpu_to_le16(lif->index),
1504 .attr = IONIC_LIF_ATTR_FEATURES,
1505 },
1506 };
1507 u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1508 IONIC_ETH_HW_VLAN_RX_STRIP |
1509 IONIC_ETH_HW_VLAN_RX_FILTER;
1510 u64 old_hw_features;
1511 int err;
1512
1513 ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1514
1515 if (lif->phc)
1516 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1517
1518 err = ionic_adminq_post_wait(lif, &ctx);
1519 if (err)
1520 return err;
1521
1522 old_hw_features = lif->hw_features;
1523 lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1524 ctx.comp.lif_setattr.features);
1525
1526 if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1527 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1528
1529 if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&
1530 !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1531 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1532
1533 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1534 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1535 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1536 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1537 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1538 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1539 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1540 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1541 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1542 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1543 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1544 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1545 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1546 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1547 if (lif->hw_features & IONIC_ETH_HW_TSO)
1548 dev_dbg(dev, "feature ETH_HW_TSO\n");
1549 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1550 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1551 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1552 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1553 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1554 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1555 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1556 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1557 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1558 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1559 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1560 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1561 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1562 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1563 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1564 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1565 if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1566 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1567
1568 return 0;
1569 }
1570
ionic_init_nic_features(struct ionic_lif * lif)1571 static int ionic_init_nic_features(struct ionic_lif *lif)
1572 {
1573 struct net_device *netdev = lif->netdev;
1574 netdev_features_t features;
1575 int err;
1576
1577 /* set up what we expect to support by default */
1578 features = NETIF_F_HW_VLAN_CTAG_TX |
1579 NETIF_F_HW_VLAN_CTAG_RX |
1580 NETIF_F_HW_VLAN_CTAG_FILTER |
1581 NETIF_F_SG |
1582 NETIF_F_HW_CSUM |
1583 NETIF_F_RXCSUM |
1584 NETIF_F_TSO |
1585 NETIF_F_TSO6 |
1586 NETIF_F_TSO_ECN |
1587 NETIF_F_GSO_GRE |
1588 NETIF_F_GSO_GRE_CSUM |
1589 NETIF_F_GSO_IPXIP4 |
1590 NETIF_F_GSO_IPXIP6 |
1591 NETIF_F_GSO_UDP_TUNNEL |
1592 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1593
1594 if (lif->nxqs > 1)
1595 features |= NETIF_F_RXHASH;
1596
1597 err = ionic_set_nic_features(lif, features);
1598 if (err)
1599 return err;
1600
1601 /* tell the netdev what we actually can support */
1602 netdev->features |= NETIF_F_HIGHDMA;
1603
1604 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1605 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1606 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1607 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1608 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1609 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1610 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1611 netdev->hw_features |= NETIF_F_RXHASH;
1612 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1613 netdev->hw_features |= NETIF_F_SG;
1614
1615 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1616 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1617 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1618 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1619 if (lif->hw_features & IONIC_ETH_HW_TSO)
1620 netdev->hw_enc_features |= NETIF_F_TSO;
1621 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1622 netdev->hw_enc_features |= NETIF_F_TSO6;
1623 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1624 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1625 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1626 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1627 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1628 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1629 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1630 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1631 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1632 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1633 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1634 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1635 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1636 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1637
1638 netdev->hw_features |= netdev->hw_enc_features;
1639 netdev->features |= netdev->hw_features;
1640 netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1641
1642 netdev->priv_flags |= IFF_UNICAST_FLT |
1643 IFF_LIVE_ADDR_CHANGE;
1644
1645 netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
1646 NETDEV_XDP_ACT_REDIRECT |
1647 NETDEV_XDP_ACT_RX_SG |
1648 NETDEV_XDP_ACT_NDO_XMIT |
1649 NETDEV_XDP_ACT_NDO_XMIT_SG;
1650
1651 return 0;
1652 }
1653
ionic_set_features(struct net_device * netdev,netdev_features_t features)1654 static int ionic_set_features(struct net_device *netdev,
1655 netdev_features_t features)
1656 {
1657 struct ionic_lif *lif = netdev_priv(netdev);
1658 int err;
1659
1660 netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1661 __func__, (u64)lif->netdev->features, (u64)features);
1662
1663 err = ionic_set_nic_features(lif, features);
1664
1665 return err;
1666 }
1667
ionic_set_attr_mac(struct ionic_lif * lif,u8 * mac)1668 static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)
1669 {
1670 struct ionic_admin_ctx ctx = {
1671 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1672 .cmd.lif_setattr = {
1673 .opcode = IONIC_CMD_LIF_SETATTR,
1674 .index = cpu_to_le16(lif->index),
1675 .attr = IONIC_LIF_ATTR_MAC,
1676 },
1677 };
1678
1679 ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);
1680 return ionic_adminq_post_wait(lif, &ctx);
1681 }
1682
ionic_get_attr_mac(struct ionic_lif * lif,u8 * mac_addr)1683 static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)
1684 {
1685 struct ionic_admin_ctx ctx = {
1686 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1687 .cmd.lif_getattr = {
1688 .opcode = IONIC_CMD_LIF_GETATTR,
1689 .index = cpu_to_le16(lif->index),
1690 .attr = IONIC_LIF_ATTR_MAC,
1691 },
1692 };
1693 int err;
1694
1695 err = ionic_adminq_post_wait(lif, &ctx);
1696 if (err)
1697 return err;
1698
1699 ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);
1700 return 0;
1701 }
1702
ionic_program_mac(struct ionic_lif * lif,u8 * mac)1703 static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)
1704 {
1705 u8 get_mac[ETH_ALEN];
1706 int err;
1707
1708 err = ionic_set_attr_mac(lif, mac);
1709 if (err)
1710 return err;
1711
1712 err = ionic_get_attr_mac(lif, get_mac);
1713 if (err)
1714 return err;
1715
1716 /* To deal with older firmware that silently ignores the set attr mac:
1717 * doesn't actually change the mac and doesn't return an error, so we
1718 * do the get attr to verify whether or not the set actually happened
1719 */
1720 if (!ether_addr_equal(get_mac, mac))
1721 return 1;
1722
1723 return 0;
1724 }
1725
ionic_set_mac_address(struct net_device * netdev,void * sa)1726 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1727 {
1728 struct ionic_lif *lif = netdev_priv(netdev);
1729 struct sockaddr *addr = sa;
1730 u8 *mac;
1731 int err;
1732
1733 mac = (u8 *)addr->sa_data;
1734 if (ether_addr_equal(netdev->dev_addr, mac))
1735 return 0;
1736
1737 /* Only program macs for virtual functions to avoid losing the permanent
1738 * Mac across warm reset/reboot.
1739 */
1740 if (lif->ionic->pdev->is_virtfn) {
1741 err = ionic_program_mac(lif, mac);
1742 if (err < 0)
1743 return err;
1744
1745 if (err > 0)
1746 netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
1747 __func__);
1748 }
1749
1750 err = eth_prepare_mac_addr_change(netdev, addr);
1751 if (err)
1752 return err;
1753
1754 if (!is_zero_ether_addr(netdev->dev_addr)) {
1755 netdev_info(netdev, "deleting mac addr %pM\n",
1756 netdev->dev_addr);
1757 ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
1758 }
1759
1760 eth_commit_mac_addr_change(netdev, addr);
1761 netdev_info(netdev, "updating mac addr %pM\n", mac);
1762
1763 return ionic_lif_addr_add(netdev_priv(netdev), mac);
1764 }
1765
ionic_stop_queues_reconfig(struct ionic_lif * lif)1766 void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1767 {
1768 /* Stop and clean the queues before reconfiguration */
1769 netif_device_detach(lif->netdev);
1770 ionic_stop_queues(lif);
1771 ionic_txrx_deinit(lif);
1772 }
1773
ionic_start_queues_reconfig(struct ionic_lif * lif)1774 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1775 {
1776 int err;
1777
1778 /* Re-init the queues after reconfiguration */
1779
1780 /* The only way txrx_init can fail here is if communication
1781 * with FW is suddenly broken. There's not much we can do
1782 * at this point - error messages have already been printed,
1783 * so we can continue on and the user can eventually do a
1784 * DOWN and UP to try to reset and clear the issue.
1785 */
1786 err = ionic_txrx_init(lif);
1787 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1788 netif_device_attach(lif->netdev);
1789
1790 return err;
1791 }
1792
ionic_xdp_is_valid_mtu(struct ionic_lif * lif,u32 mtu,struct bpf_prog * xdp_prog)1793 static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,
1794 struct bpf_prog *xdp_prog)
1795 {
1796 if (!xdp_prog)
1797 return true;
1798
1799 if (mtu <= IONIC_XDP_MAX_LINEAR_MTU)
1800 return true;
1801
1802 if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags)
1803 return true;
1804
1805 return false;
1806 }
1807
ionic_change_mtu(struct net_device * netdev,int new_mtu)1808 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1809 {
1810 struct ionic_lif *lif = netdev_priv(netdev);
1811 struct ionic_admin_ctx ctx = {
1812 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1813 .cmd.lif_setattr = {
1814 .opcode = IONIC_CMD_LIF_SETATTR,
1815 .index = cpu_to_le16(lif->index),
1816 .attr = IONIC_LIF_ATTR_MTU,
1817 .mtu = cpu_to_le32(new_mtu),
1818 },
1819 };
1820 struct bpf_prog *xdp_prog;
1821 int err;
1822
1823 xdp_prog = READ_ONCE(lif->xdp_prog);
1824 if (!ionic_xdp_is_valid_mtu(lif, new_mtu, xdp_prog))
1825 return -EINVAL;
1826
1827 err = ionic_adminq_post_wait(lif, &ctx);
1828 if (err)
1829 return err;
1830
1831 /* if we're not running, nothing more to do */
1832 if (!netif_running(netdev)) {
1833 WRITE_ONCE(netdev->mtu, new_mtu);
1834 return 0;
1835 }
1836
1837 mutex_lock(&lif->queue_lock);
1838 ionic_stop_queues_reconfig(lif);
1839 WRITE_ONCE(netdev->mtu, new_mtu);
1840 err = ionic_start_queues_reconfig(lif);
1841 mutex_unlock(&lif->queue_lock);
1842
1843 return err;
1844 }
1845
ionic_tx_timeout_work(struct work_struct * ws)1846 static void ionic_tx_timeout_work(struct work_struct *ws)
1847 {
1848 struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1849 int err;
1850
1851 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1852 return;
1853
1854 /* if we were stopped before this scheduled job was launched,
1855 * don't bother the queues as they are already stopped.
1856 */
1857 if (!netif_running(lif->netdev))
1858 return;
1859
1860 mutex_lock(&lif->queue_lock);
1861 ionic_stop_queues_reconfig(lif);
1862 err = ionic_start_queues_reconfig(lif);
1863 mutex_unlock(&lif->queue_lock);
1864
1865 if (err)
1866 dev_err(lif->ionic->dev, "%s: Restarting queues failed\n", __func__);
1867 }
1868
ionic_tx_timeout(struct net_device * netdev,unsigned int txqueue)1869 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1870 {
1871 struct ionic_lif *lif = netdev_priv(netdev);
1872
1873 netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1874 schedule_work(&lif->tx_timeout_work);
1875 }
1876
ionic_vlan_rx_add_vid(struct net_device * netdev,__be16 proto,u16 vid)1877 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1878 u16 vid)
1879 {
1880 struct ionic_lif *lif = netdev_priv(netdev);
1881 int err;
1882
1883 err = ionic_lif_vlan_add(lif, vid);
1884 if (err)
1885 return err;
1886
1887 ionic_lif_rx_mode(lif);
1888
1889 return 0;
1890 }
1891
ionic_vlan_rx_kill_vid(struct net_device * netdev,__be16 proto,u16 vid)1892 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1893 u16 vid)
1894 {
1895 struct ionic_lif *lif = netdev_priv(netdev);
1896 int err;
1897
1898 err = ionic_lif_vlan_del(lif, vid);
1899 if (err)
1900 return err;
1901
1902 ionic_lif_rx_mode(lif);
1903
1904 return 0;
1905 }
1906
ionic_lif_rss_config(struct ionic_lif * lif,const u16 types,const u8 * key,const u32 * indir)1907 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1908 const u8 *key, const u32 *indir)
1909 {
1910 struct ionic_admin_ctx ctx = {
1911 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1912 .cmd.lif_setattr = {
1913 .opcode = IONIC_CMD_LIF_SETATTR,
1914 .attr = IONIC_LIF_ATTR_RSS,
1915 .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1916 },
1917 };
1918 unsigned int i, tbl_sz;
1919
1920 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1921 lif->rss_types = types;
1922 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1923 }
1924
1925 if (key)
1926 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1927
1928 if (indir) {
1929 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1930 for (i = 0; i < tbl_sz; i++)
1931 lif->rss_ind_tbl[i] = indir[i];
1932 }
1933
1934 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1935 IONIC_RSS_HASH_KEY_SIZE);
1936
1937 return ionic_adminq_post_wait(lif, &ctx);
1938 }
1939
ionic_lif_rss_init(struct ionic_lif * lif)1940 static int ionic_lif_rss_init(struct ionic_lif *lif)
1941 {
1942 unsigned int tbl_sz;
1943 unsigned int i;
1944
1945 lif->rss_types = IONIC_RSS_TYPE_IPV4 |
1946 IONIC_RSS_TYPE_IPV4_TCP |
1947 IONIC_RSS_TYPE_IPV4_UDP |
1948 IONIC_RSS_TYPE_IPV6 |
1949 IONIC_RSS_TYPE_IPV6_TCP |
1950 IONIC_RSS_TYPE_IPV6_UDP;
1951
1952 /* Fill indirection table with 'default' values */
1953 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1954 for (i = 0; i < tbl_sz; i++)
1955 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1956
1957 return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1958 }
1959
ionic_lif_rss_deinit(struct ionic_lif * lif)1960 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1961 {
1962 int tbl_sz;
1963
1964 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1965 memset(lif->rss_ind_tbl, 0, tbl_sz);
1966 memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1967
1968 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1969 }
1970
ionic_lif_quiesce(struct ionic_lif * lif)1971 static void ionic_lif_quiesce(struct ionic_lif *lif)
1972 {
1973 struct ionic_admin_ctx ctx = {
1974 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1975 .cmd.lif_setattr = {
1976 .opcode = IONIC_CMD_LIF_SETATTR,
1977 .index = cpu_to_le16(lif->index),
1978 .attr = IONIC_LIF_ATTR_STATE,
1979 .state = IONIC_LIF_QUIESCE,
1980 },
1981 };
1982 int err;
1983
1984 err = ionic_adminq_post_wait(lif, &ctx);
1985 if (err)
1986 netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
1987 }
1988
ionic_txrx_disable(struct ionic_lif * lif)1989 static void ionic_txrx_disable(struct ionic_lif *lif)
1990 {
1991 unsigned int i;
1992 int err = 0;
1993
1994 if (lif->txqcqs) {
1995 for (i = 0; i < lif->nxqs; i++)
1996 err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
1997 }
1998
1999 if (lif->hwstamp_txq)
2000 err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
2001
2002 if (lif->rxqcqs) {
2003 for (i = 0; i < lif->nxqs; i++)
2004 err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
2005 }
2006
2007 if (lif->hwstamp_rxq)
2008 err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
2009
2010 ionic_lif_quiesce(lif);
2011 }
2012
ionic_txrx_deinit(struct ionic_lif * lif)2013 static void ionic_txrx_deinit(struct ionic_lif *lif)
2014 {
2015 unsigned int i;
2016
2017 if (lif->txqcqs) {
2018 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
2019 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2020 ionic_tx_flush(&lif->txqcqs[i]->cq);
2021 ionic_tx_empty(&lif->txqcqs[i]->q);
2022 }
2023 }
2024
2025 if (lif->rxqcqs) {
2026 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
2027 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2028 ionic_rx_empty(&lif->rxqcqs[i]->q);
2029 }
2030 }
2031 lif->rx_mode = 0;
2032
2033 if (lif->hwstamp_txq) {
2034 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
2035 ionic_tx_flush(&lif->hwstamp_txq->cq);
2036 ionic_tx_empty(&lif->hwstamp_txq->q);
2037 }
2038
2039 if (lif->hwstamp_rxq) {
2040 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
2041 ionic_rx_empty(&lif->hwstamp_rxq->q);
2042 }
2043 }
2044
ionic_txrx_free(struct ionic_lif * lif)2045 void ionic_txrx_free(struct ionic_lif *lif)
2046 {
2047 unsigned int i;
2048
2049 if (lif->txqcqs) {
2050 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
2051 ionic_qcq_free(lif, lif->txqcqs[i]);
2052 devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
2053 lif->txqcqs[i] = NULL;
2054 }
2055 }
2056
2057 if (lif->rxqcqs) {
2058 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2059 ionic_qcq_free(lif, lif->rxqcqs[i]);
2060 devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
2061 lif->rxqcqs[i] = NULL;
2062 }
2063 }
2064
2065 if (lif->hwstamp_txq) {
2066 ionic_qcq_free(lif, lif->hwstamp_txq);
2067 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2068 lif->hwstamp_txq = NULL;
2069 }
2070
2071 if (lif->hwstamp_rxq) {
2072 ionic_qcq_free(lif, lif->hwstamp_rxq);
2073 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2074 lif->hwstamp_rxq = NULL;
2075 }
2076 }
2077
ionic_txrx_alloc(struct ionic_lif * lif)2078 static int ionic_txrx_alloc(struct ionic_lif *lif)
2079 {
2080 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2081 unsigned int flags, i;
2082 int err = 0;
2083
2084 num_desc = lif->ntxq_descs;
2085 desc_sz = sizeof(struct ionic_txq_desc);
2086 comp_sz = sizeof(struct ionic_txq_comp);
2087
2088 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2089 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2090 sizeof(struct ionic_txq_sg_desc_v1))
2091 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2092 else
2093 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2094
2095 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2096
2097 if (test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state))
2098 flags |= IONIC_QCQ_F_CMB_RINGS;
2099
2100 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2101 flags |= IONIC_QCQ_F_INTR;
2102
2103 for (i = 0; i < lif->nxqs; i++) {
2104 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2105 num_desc, desc_sz, comp_sz, sg_desc_sz,
2106 sizeof(struct ionic_tx_desc_info),
2107 lif->kern_pid, NULL, &lif->txqcqs[i]);
2108 if (err)
2109 goto err_out;
2110
2111 if (flags & IONIC_QCQ_F_INTR) {
2112 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2113 lif->txqcqs[i]->intr.index,
2114 lif->tx_coalesce_hw);
2115 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2116 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2117 }
2118
2119 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2120 }
2121
2122 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2123
2124 if (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state))
2125 flags |= IONIC_QCQ_F_CMB_RINGS;
2126
2127 num_desc = lif->nrxq_descs;
2128 desc_sz = sizeof(struct ionic_rxq_desc);
2129 comp_sz = sizeof(struct ionic_rxq_comp);
2130 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2131
2132 if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2133 comp_sz *= 2;
2134
2135 for (i = 0; i < lif->nxqs; i++) {
2136 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2137 num_desc, desc_sz, comp_sz, sg_desc_sz,
2138 sizeof(struct ionic_rx_desc_info),
2139 lif->kern_pid, lif->xdp_prog,
2140 &lif->rxqcqs[i]);
2141 if (err)
2142 goto err_out;
2143
2144 lif->rxqcqs[i]->q.features = lif->rxq_features;
2145
2146 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2147 lif->rxqcqs[i]->intr.index,
2148 lif->rx_coalesce_hw);
2149 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2150 lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2151
2152 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2153 ionic_link_qcq_interrupts(lif->rxqcqs[i],
2154 lif->txqcqs[i]);
2155
2156 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2157 }
2158
2159 return 0;
2160
2161 err_out:
2162 ionic_txrx_free(lif);
2163
2164 return err;
2165 }
2166
ionic_txrx_init(struct ionic_lif * lif)2167 static int ionic_txrx_init(struct ionic_lif *lif)
2168 {
2169 unsigned int i;
2170 int err;
2171
2172 for (i = 0; i < lif->nxqs; i++) {
2173 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2174 if (err)
2175 goto err_out;
2176
2177 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2178 if (err) {
2179 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2180 goto err_out;
2181 }
2182 }
2183
2184 if (lif->netdev->features & NETIF_F_RXHASH)
2185 ionic_lif_rss_init(lif);
2186
2187 ionic_lif_rx_mode(lif);
2188
2189 return 0;
2190
2191 err_out:
2192 while (i--) {
2193 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2194 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2195 }
2196
2197 return err;
2198 }
2199
ionic_txrx_enable(struct ionic_lif * lif)2200 static int ionic_txrx_enable(struct ionic_lif *lif)
2201 {
2202 int derr = 0;
2203 int i, err;
2204
2205 ionic_xdp_rxqs_prog_update(lif);
2206
2207 for (i = 0; i < lif->nxqs; i++) {
2208 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2209 dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2210 err = -ENXIO;
2211 goto err_out;
2212 }
2213
2214 ionic_rx_fill(&lif->rxqcqs[i]->q,
2215 READ_ONCE(lif->rxqcqs[i]->q.xdp_prog));
2216 err = ionic_qcq_enable(lif->rxqcqs[i]);
2217 if (err)
2218 goto err_out;
2219
2220 err = ionic_qcq_enable(lif->txqcqs[i]);
2221 if (err) {
2222 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
2223 goto err_out;
2224 }
2225 }
2226
2227 if (lif->hwstamp_rxq) {
2228 ionic_rx_fill(&lif->hwstamp_rxq->q, NULL);
2229 err = ionic_qcq_enable(lif->hwstamp_rxq);
2230 if (err)
2231 goto err_out_hwstamp_rx;
2232 }
2233
2234 if (lif->hwstamp_txq) {
2235 err = ionic_qcq_enable(lif->hwstamp_txq);
2236 if (err)
2237 goto err_out_hwstamp_tx;
2238 }
2239
2240 return 0;
2241
2242 err_out_hwstamp_tx:
2243 if (lif->hwstamp_rxq)
2244 derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
2245 err_out_hwstamp_rx:
2246 i = lif->nxqs;
2247 err_out:
2248 while (i--) {
2249 derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
2250 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
2251 }
2252
2253 ionic_xdp_rxqs_prog_update(lif);
2254
2255 return err;
2256 }
2257
ionic_start_queues(struct ionic_lif * lif)2258 static int ionic_start_queues(struct ionic_lif *lif)
2259 {
2260 int err;
2261
2262 if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2263 return -EIO;
2264
2265 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2266 return -EBUSY;
2267
2268 if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2269 return 0;
2270
2271 err = ionic_txrx_enable(lif);
2272 if (err) {
2273 clear_bit(IONIC_LIF_F_UP, lif->state);
2274 return err;
2275 }
2276 netif_tx_wake_all_queues(lif->netdev);
2277
2278 return 0;
2279 }
2280
ionic_open(struct net_device * netdev)2281 static int ionic_open(struct net_device *netdev)
2282 {
2283 struct ionic_lif *lif = netdev_priv(netdev);
2284 int err;
2285
2286 /* If recovering from a broken state, clear the bit and we'll try again */
2287 if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2288 netdev_info(netdev, "clearing broken state\n");
2289
2290 mutex_lock(&lif->queue_lock);
2291
2292 err = ionic_txrx_alloc(lif);
2293 if (err)
2294 goto err_unlock;
2295
2296 err = ionic_txrx_init(lif);
2297 if (err)
2298 goto err_txrx_free;
2299
2300 err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2301 if (err)
2302 goto err_txrx_deinit;
2303
2304 err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2305 if (err)
2306 goto err_txrx_deinit;
2307
2308 /* don't start the queues until we have link */
2309 if (netif_carrier_ok(netdev)) {
2310 err = ionic_start_queues(lif);
2311 if (err)
2312 goto err_txrx_deinit;
2313 }
2314
2315 /* If hardware timestamping is enabled, but the queues were freed by
2316 * ionic_stop, those need to be reallocated and initialized, too.
2317 */
2318 ionic_lif_hwstamp_recreate_queues(lif);
2319
2320 mutex_unlock(&lif->queue_lock);
2321
2322 return 0;
2323
2324 err_txrx_deinit:
2325 ionic_txrx_deinit(lif);
2326 err_txrx_free:
2327 ionic_txrx_free(lif);
2328 err_unlock:
2329 mutex_unlock(&lif->queue_lock);
2330 return err;
2331 }
2332
ionic_stop_queues(struct ionic_lif * lif)2333 static void ionic_stop_queues(struct ionic_lif *lif)
2334 {
2335 if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2336 return;
2337
2338 netif_tx_disable(lif->netdev);
2339 ionic_txrx_disable(lif);
2340 }
2341
ionic_stop(struct net_device * netdev)2342 static int ionic_stop(struct net_device *netdev)
2343 {
2344 struct ionic_lif *lif = netdev_priv(netdev);
2345
2346 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2347 return 0;
2348
2349 mutex_lock(&lif->queue_lock);
2350 ionic_stop_queues(lif);
2351 ionic_txrx_deinit(lif);
2352 ionic_txrx_free(lif);
2353 mutex_unlock(&lif->queue_lock);
2354
2355 return 0;
2356 }
2357
ionic_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivf)2358 static int ionic_get_vf_config(struct net_device *netdev,
2359 int vf, struct ifla_vf_info *ivf)
2360 {
2361 struct ionic_lif *lif = netdev_priv(netdev);
2362 struct ionic *ionic = lif->ionic;
2363 int ret = 0;
2364
2365 if (!netif_device_present(netdev))
2366 return -EBUSY;
2367
2368 down_read(&ionic->vf_op_lock);
2369
2370 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2371 ret = -EINVAL;
2372 } else {
2373 struct ionic_vf *vfdata = &ionic->vfs[vf];
2374
2375 ivf->vf = vf;
2376 ivf->qos = 0;
2377 ivf->vlan = le16_to_cpu(vfdata->vlanid);
2378 ivf->spoofchk = vfdata->spoofchk;
2379 ivf->linkstate = vfdata->linkstate;
2380 ivf->max_tx_rate = le32_to_cpu(vfdata->maxrate);
2381 ivf->trusted = vfdata->trusted;
2382 ether_addr_copy(ivf->mac, vfdata->macaddr);
2383 }
2384
2385 up_read(&ionic->vf_op_lock);
2386 return ret;
2387 }
2388
ionic_get_vf_stats(struct net_device * netdev,int vf,struct ifla_vf_stats * vf_stats)2389 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2390 struct ifla_vf_stats *vf_stats)
2391 {
2392 struct ionic_lif *lif = netdev_priv(netdev);
2393 struct ionic *ionic = lif->ionic;
2394 struct ionic_lif_stats *vs;
2395 int ret = 0;
2396
2397 if (!netif_device_present(netdev))
2398 return -EBUSY;
2399
2400 down_read(&ionic->vf_op_lock);
2401
2402 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2403 ret = -EINVAL;
2404 } else {
2405 memset(vf_stats, 0, sizeof(*vf_stats));
2406 vs = &ionic->vfs[vf].stats;
2407
2408 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2409 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2410 vf_stats->rx_bytes = le64_to_cpu(vs->rx_ucast_bytes);
2411 vf_stats->tx_bytes = le64_to_cpu(vs->tx_ucast_bytes);
2412 vf_stats->broadcast = le64_to_cpu(vs->rx_bcast_packets);
2413 vf_stats->multicast = le64_to_cpu(vs->rx_mcast_packets);
2414 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2415 le64_to_cpu(vs->rx_mcast_drop_packets) +
2416 le64_to_cpu(vs->rx_bcast_drop_packets);
2417 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2418 le64_to_cpu(vs->tx_mcast_drop_packets) +
2419 le64_to_cpu(vs->tx_bcast_drop_packets);
2420 }
2421
2422 up_read(&ionic->vf_op_lock);
2423 return ret;
2424 }
2425
ionic_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2426 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2427 {
2428 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
2429 struct ionic_lif *lif = netdev_priv(netdev);
2430 struct ionic *ionic = lif->ionic;
2431 int ret;
2432
2433 if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2434 return -EINVAL;
2435
2436 if (!netif_device_present(netdev))
2437 return -EBUSY;
2438
2439 down_write(&ionic->vf_op_lock);
2440
2441 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2442 ret = -EINVAL;
2443 } else {
2444 ether_addr_copy(vfc.macaddr, mac);
2445 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
2446 __func__, vf, vfc.macaddr);
2447
2448 ret = ionic_set_vf_config(ionic, vf, &vfc);
2449 if (!ret)
2450 ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2451 }
2452
2453 up_write(&ionic->vf_op_lock);
2454 return ret;
2455 }
2456
ionic_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2457 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2458 u8 qos, __be16 proto)
2459 {
2460 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
2461 struct ionic_lif *lif = netdev_priv(netdev);
2462 struct ionic *ionic = lif->ionic;
2463 int ret;
2464
2465 /* until someday when we support qos */
2466 if (qos)
2467 return -EINVAL;
2468
2469 if (vlan > 4095)
2470 return -EINVAL;
2471
2472 if (proto != htons(ETH_P_8021Q))
2473 return -EPROTONOSUPPORT;
2474
2475 if (!netif_device_present(netdev))
2476 return -EBUSY;
2477
2478 down_write(&ionic->vf_op_lock);
2479
2480 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2481 ret = -EINVAL;
2482 } else {
2483 vfc.vlanid = cpu_to_le16(vlan);
2484 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
2485 __func__, vf, le16_to_cpu(vfc.vlanid));
2486
2487 ret = ionic_set_vf_config(ionic, vf, &vfc);
2488 if (!ret)
2489 ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2490 }
2491
2492 up_write(&ionic->vf_op_lock);
2493 return ret;
2494 }
2495
ionic_set_vf_rate(struct net_device * netdev,int vf,int tx_min,int tx_max)2496 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2497 int tx_min, int tx_max)
2498 {
2499 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
2500 struct ionic_lif *lif = netdev_priv(netdev);
2501 struct ionic *ionic = lif->ionic;
2502 int ret;
2503
2504 /* setting the min just seems silly */
2505 if (tx_min)
2506 return -EINVAL;
2507
2508 if (!netif_device_present(netdev))
2509 return -EBUSY;
2510
2511 down_write(&ionic->vf_op_lock);
2512
2513 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2514 ret = -EINVAL;
2515 } else {
2516 vfc.maxrate = cpu_to_le32(tx_max);
2517 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
2518 __func__, vf, le32_to_cpu(vfc.maxrate));
2519
2520 ret = ionic_set_vf_config(ionic, vf, &vfc);
2521 if (!ret)
2522 ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2523 }
2524
2525 up_write(&ionic->vf_op_lock);
2526 return ret;
2527 }
2528
ionic_set_vf_spoofchk(struct net_device * netdev,int vf,bool set)2529 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2530 {
2531 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
2532 struct ionic_lif *lif = netdev_priv(netdev);
2533 struct ionic *ionic = lif->ionic;
2534 int ret;
2535
2536 if (!netif_device_present(netdev))
2537 return -EBUSY;
2538
2539 down_write(&ionic->vf_op_lock);
2540
2541 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2542 ret = -EINVAL;
2543 } else {
2544 vfc.spoofchk = set;
2545 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
2546 __func__, vf, vfc.spoofchk);
2547
2548 ret = ionic_set_vf_config(ionic, vf, &vfc);
2549 if (!ret)
2550 ionic->vfs[vf].spoofchk = set;
2551 }
2552
2553 up_write(&ionic->vf_op_lock);
2554 return ret;
2555 }
2556
ionic_set_vf_trust(struct net_device * netdev,int vf,bool set)2557 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2558 {
2559 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
2560 struct ionic_lif *lif = netdev_priv(netdev);
2561 struct ionic *ionic = lif->ionic;
2562 int ret;
2563
2564 if (!netif_device_present(netdev))
2565 return -EBUSY;
2566
2567 down_write(&ionic->vf_op_lock);
2568
2569 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2570 ret = -EINVAL;
2571 } else {
2572 vfc.trust = set;
2573 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
2574 __func__, vf, vfc.trust);
2575
2576 ret = ionic_set_vf_config(ionic, vf, &vfc);
2577 if (!ret)
2578 ionic->vfs[vf].trusted = set;
2579 }
2580
2581 up_write(&ionic->vf_op_lock);
2582 return ret;
2583 }
2584
ionic_set_vf_link_state(struct net_device * netdev,int vf,int set)2585 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2586 {
2587 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
2588 struct ionic_lif *lif = netdev_priv(netdev);
2589 struct ionic *ionic = lif->ionic;
2590 u8 vfls;
2591 int ret;
2592
2593 switch (set) {
2594 case IFLA_VF_LINK_STATE_ENABLE:
2595 vfls = IONIC_VF_LINK_STATUS_UP;
2596 break;
2597 case IFLA_VF_LINK_STATE_DISABLE:
2598 vfls = IONIC_VF_LINK_STATUS_DOWN;
2599 break;
2600 case IFLA_VF_LINK_STATE_AUTO:
2601 vfls = IONIC_VF_LINK_STATUS_AUTO;
2602 break;
2603 default:
2604 return -EINVAL;
2605 }
2606
2607 if (!netif_device_present(netdev))
2608 return -EBUSY;
2609
2610 down_write(&ionic->vf_op_lock);
2611
2612 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2613 ret = -EINVAL;
2614 } else {
2615 vfc.linkstate = vfls;
2616 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
2617 __func__, vf, vfc.linkstate);
2618
2619 ret = ionic_set_vf_config(ionic, vf, &vfc);
2620 if (!ret)
2621 ionic->vfs[vf].linkstate = set;
2622 }
2623
2624 up_write(&ionic->vf_op_lock);
2625 return ret;
2626 }
2627
ionic_vf_attr_replay(struct ionic_lif * lif)2628 static void ionic_vf_attr_replay(struct ionic_lif *lif)
2629 {
2630 struct ionic_vf_setattr_cmd vfc = { };
2631 struct ionic *ionic = lif->ionic;
2632 struct ionic_vf *v;
2633 int i;
2634
2635 if (!ionic->vfs)
2636 return;
2637
2638 down_read(&ionic->vf_op_lock);
2639
2640 for (i = 0; i < ionic->num_vfs; i++) {
2641 v = &ionic->vfs[i];
2642
2643 if (v->stats_pa) {
2644 vfc.attr = IONIC_VF_ATTR_STATSADDR;
2645 vfc.stats_pa = cpu_to_le64(v->stats_pa);
2646 ionic_set_vf_config(ionic, i, &vfc);
2647 vfc.stats_pa = 0;
2648 }
2649
2650 if (!is_zero_ether_addr(v->macaddr)) {
2651 vfc.attr = IONIC_VF_ATTR_MAC;
2652 ether_addr_copy(vfc.macaddr, v->macaddr);
2653 ionic_set_vf_config(ionic, i, &vfc);
2654 eth_zero_addr(vfc.macaddr);
2655 }
2656
2657 if (v->vlanid) {
2658 vfc.attr = IONIC_VF_ATTR_VLAN;
2659 vfc.vlanid = v->vlanid;
2660 ionic_set_vf_config(ionic, i, &vfc);
2661 vfc.vlanid = 0;
2662 }
2663
2664 if (v->maxrate) {
2665 vfc.attr = IONIC_VF_ATTR_RATE;
2666 vfc.maxrate = v->maxrate;
2667 ionic_set_vf_config(ionic, i, &vfc);
2668 vfc.maxrate = 0;
2669 }
2670
2671 if (v->spoofchk) {
2672 vfc.attr = IONIC_VF_ATTR_SPOOFCHK;
2673 vfc.spoofchk = v->spoofchk;
2674 ionic_set_vf_config(ionic, i, &vfc);
2675 vfc.spoofchk = 0;
2676 }
2677
2678 if (v->trusted) {
2679 vfc.attr = IONIC_VF_ATTR_TRUST;
2680 vfc.trust = v->trusted;
2681 ionic_set_vf_config(ionic, i, &vfc);
2682 vfc.trust = 0;
2683 }
2684
2685 if (v->linkstate) {
2686 vfc.attr = IONIC_VF_ATTR_LINKSTATE;
2687 vfc.linkstate = v->linkstate;
2688 ionic_set_vf_config(ionic, i, &vfc);
2689 vfc.linkstate = 0;
2690 }
2691 }
2692
2693 up_read(&ionic->vf_op_lock);
2694
2695 ionic_vf_start(ionic);
2696 }
2697
ionic_unregister_rxq_info(struct ionic_queue * q)2698 static void ionic_unregister_rxq_info(struct ionic_queue *q)
2699 {
2700 struct xdp_rxq_info *xi;
2701
2702 if (!q->xdp_rxq_info)
2703 return;
2704
2705 xi = q->xdp_rxq_info;
2706 q->xdp_rxq_info = NULL;
2707
2708 xdp_rxq_info_unreg(xi);
2709 kfree(xi);
2710 }
2711
ionic_register_rxq_info(struct ionic_queue * q,unsigned int napi_id)2712 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)
2713 {
2714 struct xdp_rxq_info *rxq_info;
2715 int err;
2716
2717 rxq_info = kzalloc_obj(*rxq_info);
2718 if (!rxq_info)
2719 return -ENOMEM;
2720
2721 err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id);
2722 if (err) {
2723 netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n",
2724 q->index, err);
2725 goto err_out;
2726 }
2727
2728 err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool);
2729 if (err) {
2730 netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n",
2731 q->index, err);
2732 xdp_rxq_info_unreg(rxq_info);
2733 goto err_out;
2734 }
2735
2736 q->xdp_rxq_info = rxq_info;
2737
2738 return 0;
2739
2740 err_out:
2741 kfree(rxq_info);
2742 return err;
2743 }
2744
ionic_xdp_rxqs_prog_update(struct ionic_lif * lif)2745 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif)
2746 {
2747 struct bpf_prog *xdp_prog;
2748 unsigned int i;
2749
2750 if (!lif->rxqcqs)
2751 return;
2752
2753 xdp_prog = READ_ONCE(lif->xdp_prog);
2754 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2755 struct ionic_queue *q = &lif->rxqcqs[i]->q;
2756
2757 WRITE_ONCE(q->xdp_prog, xdp_prog);
2758 }
2759 }
2760
ionic_xdp_config(struct net_device * netdev,struct netdev_bpf * bpf)2761 static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
2762 {
2763 struct ionic_lif *lif = netdev_priv(netdev);
2764 struct bpf_prog *old_prog;
2765 u32 maxfs;
2766
2767 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
2768 #define XDP_ERR_SPLIT "XDP not available with split Tx/Rx interrupts"
2769 NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_SPLIT);
2770 netdev_info(lif->netdev, XDP_ERR_SPLIT);
2771 return -EOPNOTSUPP;
2772 }
2773
2774 if (!ionic_xdp_is_valid_mtu(lif, netdev->mtu, bpf->prog)) {
2775 #define XDP_ERR_MTU "MTU is too large for XDP without frags support"
2776 NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_MTU);
2777 netdev_info(lif->netdev, XDP_ERR_MTU);
2778 return -EINVAL;
2779 }
2780
2781 maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
2782 if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags))
2783 maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU);
2784 netdev->max_mtu = maxfs;
2785
2786 if (!netif_running(netdev)) {
2787 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2788 } else if (lif->xdp_prog && bpf->prog) {
2789 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2790 ionic_xdp_rxqs_prog_update(lif);
2791 } else {
2792 struct ionic_queue_params qparams;
2793
2794 ionic_init_queue_params(lif, &qparams);
2795 qparams.xdp_prog = bpf->prog;
2796 mutex_lock(&lif->queue_lock);
2797 ionic_reconfigure_queues(lif, &qparams);
2798 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2799 mutex_unlock(&lif->queue_lock);
2800 }
2801
2802 if (old_prog)
2803 bpf_prog_put(old_prog);
2804
2805 return 0;
2806 }
2807
ionic_xdp(struct net_device * netdev,struct netdev_bpf * bpf)2808 static int ionic_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
2809 {
2810 switch (bpf->command) {
2811 case XDP_SETUP_PROG:
2812 return ionic_xdp_config(netdev, bpf);
2813 default:
2814 return -EINVAL;
2815 }
2816 }
2817
2818 static const struct net_device_ops ionic_netdev_ops = {
2819 .ndo_open = ionic_open,
2820 .ndo_stop = ionic_stop,
2821 .ndo_start_xmit = ionic_start_xmit,
2822 .ndo_bpf = ionic_xdp,
2823 .ndo_xdp_xmit = ionic_xdp_xmit,
2824 .ndo_get_stats64 = ionic_get_stats64,
2825 .ndo_set_rx_mode = ionic_ndo_set_rx_mode,
2826 .ndo_set_features = ionic_set_features,
2827 .ndo_set_mac_address = ionic_set_mac_address,
2828 .ndo_validate_addr = eth_validate_addr,
2829 .ndo_tx_timeout = ionic_tx_timeout,
2830 .ndo_change_mtu = ionic_change_mtu,
2831 .ndo_vlan_rx_add_vid = ionic_vlan_rx_add_vid,
2832 .ndo_vlan_rx_kill_vid = ionic_vlan_rx_kill_vid,
2833 .ndo_set_vf_vlan = ionic_set_vf_vlan,
2834 .ndo_set_vf_trust = ionic_set_vf_trust,
2835 .ndo_set_vf_mac = ionic_set_vf_mac,
2836 .ndo_set_vf_rate = ionic_set_vf_rate,
2837 .ndo_set_vf_spoofchk = ionic_set_vf_spoofchk,
2838 .ndo_get_vf_config = ionic_get_vf_config,
2839 .ndo_set_vf_link_state = ionic_set_vf_link_state,
2840 .ndo_get_vf_stats = ionic_get_vf_stats,
2841 .ndo_hwtstamp_get = ionic_hwstamp_get,
2842 .ndo_hwtstamp_set = ionic_hwstamp_set,
2843 };
2844
ionic_cmb_reconfig(struct ionic_lif * lif,struct ionic_queue_params * qparam)2845 static int ionic_cmb_reconfig(struct ionic_lif *lif,
2846 struct ionic_queue_params *qparam)
2847 {
2848 struct ionic_queue_params start_qparams;
2849 int err = 0;
2850
2851 /* When changing CMB queue parameters, we're using limited
2852 * on-device memory and don't have extra memory to use for
2853 * duplicate allocations, so we free it all first then
2854 * re-allocate with the new parameters.
2855 */
2856
2857 /* Checkpoint for possible unwind */
2858 ionic_init_queue_params(lif, &start_qparams);
2859
2860 /* Stop and free the queues */
2861 ionic_stop_queues_reconfig(lif);
2862 ionic_txrx_free(lif);
2863
2864 /* Set up new qparams */
2865 ionic_set_queue_params(lif, qparam);
2866
2867 if (netif_running(lif->netdev)) {
2868 /* Alloc and start the new configuration */
2869 err = ionic_txrx_alloc(lif);
2870 if (err) {
2871 dev_warn(lif->ionic->dev,
2872 "CMB reconfig failed, restoring values: %d\n", err);
2873
2874 /* Back out the changes */
2875 ionic_set_queue_params(lif, &start_qparams);
2876 err = ionic_txrx_alloc(lif);
2877 if (err) {
2878 dev_err(lif->ionic->dev,
2879 "CMB restore failed: %d\n", err);
2880 goto err_out;
2881 }
2882 }
2883
2884 err = ionic_start_queues_reconfig(lif);
2885 if (err) {
2886 dev_err(lif->ionic->dev,
2887 "CMB reconfig failed: %d\n", err);
2888 goto err_out;
2889 }
2890 }
2891
2892 err_out:
2893 /* This was detached in ionic_stop_queues_reconfig() */
2894 netif_device_attach(lif->netdev);
2895
2896 return err;
2897 }
2898
ionic_swap_queues(struct ionic_qcq * a,struct ionic_qcq * b)2899 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2900 {
2901 /* only swapping the queues and napi, not flags or other stuff */
2902 swap(a->napi, b->napi);
2903
2904 if (a->q.type == IONIC_QTYPE_RXQ) {
2905 swap(a->q.page_pool, b->q.page_pool);
2906 a->q.page_pool->p.napi = &a->napi;
2907 if (b->q.page_pool) /* is NULL when increasing queue count */
2908 b->q.page_pool->p.napi = &b->napi;
2909 }
2910
2911 swap(a->q.features, b->q.features);
2912 swap(a->q.num_descs, b->q.num_descs);
2913 swap(a->q.desc_size, b->q.desc_size);
2914 swap(a->q.base, b->q.base);
2915 swap(a->q.base_pa, b->q.base_pa);
2916 swap(a->q.info, b->q.info);
2917 swap(a->q.xdp_prog, b->q.xdp_prog);
2918 swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
2919 swap(a->q.partner, b->q.partner);
2920 swap(a->q_base, b->q_base);
2921 swap(a->q_base_pa, b->q_base_pa);
2922 swap(a->q_size, b->q_size);
2923
2924 swap(a->q.sg_desc_size, b->q.sg_desc_size);
2925 swap(a->q.sg_base, b->q.sg_base);
2926 swap(a->q.sg_base_pa, b->q.sg_base_pa);
2927 swap(a->sg_base, b->sg_base);
2928 swap(a->sg_base_pa, b->sg_base_pa);
2929 swap(a->sg_size, b->sg_size);
2930
2931 swap(a->cq.num_descs, b->cq.num_descs);
2932 swap(a->cq.desc_size, b->cq.desc_size);
2933 swap(a->cq.base, b->cq.base);
2934 swap(a->cq.base_pa, b->cq.base_pa);
2935 swap(a->cq_base, b->cq_base);
2936 swap(a->cq_base_pa, b->cq_base_pa);
2937 swap(a->cq_size, b->cq_size);
2938
2939 ionic_debugfs_del_qcq(a);
2940 ionic_debugfs_add_qcq(a->q.lif, a);
2941 }
2942
ionic_reconfigure_queues(struct ionic_lif * lif,struct ionic_queue_params * qparam)2943 int ionic_reconfigure_queues(struct ionic_lif *lif,
2944 struct ionic_queue_params *qparam)
2945 {
2946 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2947 struct ionic_qcq **tx_qcqs = NULL;
2948 struct ionic_qcq **rx_qcqs = NULL;
2949 unsigned int flags, i;
2950 int err = 0;
2951
2952 /* Are we changing q params while CMB is on */
2953 if ((test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state) && qparam->cmb_tx) ||
2954 (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state) && qparam->cmb_rx))
2955 return ionic_cmb_reconfig(lif, qparam);
2956
2957 /* allocate temporary qcq arrays to hold new queue structs */
2958 if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2959 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2960 sizeof(struct ionic_qcq *), GFP_KERNEL);
2961 if (!tx_qcqs) {
2962 err = -ENOMEM;
2963 goto err_out;
2964 }
2965 }
2966 if (qparam->nxqs != lif->nxqs ||
2967 qparam->nrxq_descs != lif->nrxq_descs ||
2968 qparam->rxq_features != lif->rxq_features ||
2969 qparam->xdp_prog != lif->xdp_prog) {
2970 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2971 sizeof(struct ionic_qcq *), GFP_KERNEL);
2972 if (!rx_qcqs) {
2973 err = -ENOMEM;
2974 goto err_out;
2975 }
2976 }
2977
2978 /* allocate new desc_info and rings, but leave the interrupt setup
2979 * until later so as to not mess with the still-running queues
2980 */
2981 if (tx_qcqs) {
2982 num_desc = qparam->ntxq_descs;
2983 desc_sz = sizeof(struct ionic_txq_desc);
2984 comp_sz = sizeof(struct ionic_txq_comp);
2985
2986 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2987 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2988 sizeof(struct ionic_txq_sg_desc_v1))
2989 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2990 else
2991 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2992
2993 for (i = 0; i < qparam->nxqs; i++) {
2994 /* If missing, short placeholder qcq needed for swap */
2995 if (!lif->txqcqs[i]) {
2996 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2997 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2998 4, desc_sz, comp_sz, sg_desc_sz,
2999 sizeof(struct ionic_tx_desc_info),
3000 lif->kern_pid, NULL, &lif->txqcqs[i]);
3001 if (err)
3002 goto err_out;
3003 }
3004
3005 flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
3006 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
3007 num_desc, desc_sz, comp_sz, sg_desc_sz,
3008 sizeof(struct ionic_tx_desc_info),
3009 lif->kern_pid, NULL, &tx_qcqs[i]);
3010 if (err)
3011 goto err_out;
3012 }
3013 }
3014
3015 if (rx_qcqs) {
3016 num_desc = qparam->nrxq_descs;
3017 desc_sz = sizeof(struct ionic_rxq_desc);
3018 comp_sz = sizeof(struct ionic_rxq_comp);
3019 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
3020
3021 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
3022 comp_sz *= 2;
3023
3024 for (i = 0; i < qparam->nxqs; i++) {
3025 /* If missing, short placeholder qcq needed for swap */
3026 if (!lif->rxqcqs[i]) {
3027 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
3028 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3029 4, desc_sz, comp_sz, sg_desc_sz,
3030 sizeof(struct ionic_rx_desc_info),
3031 lif->kern_pid, NULL, &lif->rxqcqs[i]);
3032 if (err)
3033 goto err_out;
3034 }
3035
3036 flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
3037 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3038 num_desc, desc_sz, comp_sz, sg_desc_sz,
3039 sizeof(struct ionic_rx_desc_info),
3040 lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]);
3041 if (err)
3042 goto err_out;
3043
3044 rx_qcqs[i]->q.features = qparam->rxq_features;
3045 rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog;
3046 }
3047 }
3048
3049 /* stop and clean the queues */
3050 ionic_stop_queues_reconfig(lif);
3051
3052 if (qparam->nxqs != lif->nxqs) {
3053 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
3054 if (err)
3055 goto err_out_reinit_unlock;
3056 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
3057 if (err) {
3058 netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
3059 goto err_out_reinit_unlock;
3060 }
3061 }
3062
3063 /* swap new desc_info and rings, keeping existing interrupt config */
3064 if (tx_qcqs) {
3065 lif->ntxq_descs = qparam->ntxq_descs;
3066 for (i = 0; i < qparam->nxqs; i++)
3067 ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
3068 }
3069
3070 if (rx_qcqs) {
3071 lif->nrxq_descs = qparam->nrxq_descs;
3072 for (i = 0; i < qparam->nxqs; i++)
3073 ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
3074 }
3075
3076 /* if we need to change the interrupt layout, this is the time */
3077 if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
3078 qparam->nxqs != lif->nxqs) {
3079 if (qparam->intr_split) {
3080 set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3081 } else {
3082 clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3083 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3084 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3085 }
3086
3087 /* Clear existing interrupt assignments. We check for NULL here
3088 * because we're checking the whole array for potential qcqs, not
3089 * just those qcqs that have just been set up.
3090 */
3091 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
3092 if (lif->txqcqs[i])
3093 ionic_qcq_intr_free(lif, lif->txqcqs[i]);
3094 if (lif->rxqcqs[i])
3095 ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
3096 }
3097
3098 /* re-assign the interrupts */
3099 for (i = 0; i < qparam->nxqs; i++) {
3100 lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3101 err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
3102 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3103 lif->rxqcqs[i]->intr.index,
3104 lif->rx_coalesce_hw);
3105
3106 if (qparam->intr_split) {
3107 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3108 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
3109 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3110 lif->txqcqs[i]->intr.index,
3111 lif->tx_coalesce_hw);
3112 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
3113 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
3114 } else {
3115 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3116 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
3117 }
3118 }
3119 }
3120
3121 /* now we can rework the debugfs mappings */
3122 if (tx_qcqs) {
3123 for (i = 0; i < qparam->nxqs; i++) {
3124 ionic_debugfs_del_qcq(lif->txqcqs[i]);
3125 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
3126 }
3127 }
3128
3129 if (rx_qcqs) {
3130 for (i = 0; i < qparam->nxqs; i++) {
3131 ionic_debugfs_del_qcq(lif->rxqcqs[i]);
3132 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
3133 }
3134 }
3135
3136 swap(lif->nxqs, qparam->nxqs);
3137 swap(lif->rxq_features, qparam->rxq_features);
3138
3139 err_out_reinit_unlock:
3140 /* re-init the queues, but don't lose an error code */
3141 if (err)
3142 ionic_start_queues_reconfig(lif);
3143 else
3144 err = ionic_start_queues_reconfig(lif);
3145
3146 err_out:
3147 /* free old allocs without cleaning intr */
3148 for (i = 0; i < qparam->nxqs; i++) {
3149 if (tx_qcqs && tx_qcqs[i]) {
3150 tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3151 ionic_qcq_free(lif, tx_qcqs[i]);
3152 devm_kfree(lif->ionic->dev, tx_qcqs[i]);
3153 tx_qcqs[i] = NULL;
3154 }
3155 if (rx_qcqs && rx_qcqs[i]) {
3156 rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3157 ionic_qcq_free(lif, rx_qcqs[i]);
3158 devm_kfree(lif->ionic->dev, rx_qcqs[i]);
3159 rx_qcqs[i] = NULL;
3160 }
3161 }
3162
3163 /* free q array */
3164 if (rx_qcqs) {
3165 devm_kfree(lif->ionic->dev, rx_qcqs);
3166 rx_qcqs = NULL;
3167 }
3168 if (tx_qcqs) {
3169 devm_kfree(lif->ionic->dev, tx_qcqs);
3170 tx_qcqs = NULL;
3171 }
3172
3173 /* clean the unused dma and info allocations when new set is smaller
3174 * than the full array, but leave the qcq shells in place
3175 */
3176 for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
3177 if (lif->txqcqs && lif->txqcqs[i]) {
3178 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3179 ionic_qcq_free(lif, lif->txqcqs[i]);
3180 }
3181
3182 if (lif->rxqcqs && lif->rxqcqs[i]) {
3183 lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3184 ionic_qcq_free(lif, lif->rxqcqs[i]);
3185 }
3186 }
3187
3188 if (err)
3189 netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
3190
3191 return err;
3192 }
3193
ionic_affinity_masks_alloc(struct ionic * ionic)3194 static int ionic_affinity_masks_alloc(struct ionic *ionic)
3195 {
3196 cpumask_var_t *affinity_masks;
3197 int nintrs = ionic->nintrs;
3198 int i;
3199
3200 affinity_masks = kzalloc_objs(cpumask_var_t, nintrs);
3201 if (!affinity_masks)
3202 return -ENOMEM;
3203
3204 for (i = 0; i < nintrs; i++) {
3205 if (!zalloc_cpumask_var_node(&affinity_masks[i], GFP_KERNEL,
3206 dev_to_node(ionic->dev)))
3207 goto err_out;
3208 }
3209
3210 ionic->affinity_masks = affinity_masks;
3211
3212 return 0;
3213
3214 err_out:
3215 for (--i; i >= 0; i--)
3216 free_cpumask_var(affinity_masks[i]);
3217 kfree(affinity_masks);
3218
3219 return -ENOMEM;
3220 }
3221
ionic_affinity_masks_free(struct ionic * ionic)3222 static void ionic_affinity_masks_free(struct ionic *ionic)
3223 {
3224 int i;
3225
3226 for (i = 0; i < ionic->nintrs; i++)
3227 free_cpumask_var(ionic->affinity_masks[i]);
3228 kfree(ionic->affinity_masks);
3229 ionic->affinity_masks = NULL;
3230 }
3231
ionic_lif_alloc(struct ionic * ionic)3232 int ionic_lif_alloc(struct ionic *ionic)
3233 {
3234 struct device *dev = ionic->dev;
3235 union ionic_lif_identity *lid;
3236 struct net_device *netdev;
3237 struct ionic_lif *lif;
3238 int tbl_sz;
3239 int err;
3240
3241 lid = kzalloc_obj(*lid);
3242 if (!lid)
3243 return -ENOMEM;
3244
3245 netdev = alloc_etherdev_mqs(sizeof(*lif),
3246 ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
3247 if (!netdev) {
3248 dev_err(dev, "Cannot allocate netdev, aborting\n");
3249 err = -ENOMEM;
3250 goto err_out_free_lid;
3251 }
3252
3253 SET_NETDEV_DEV(netdev, dev);
3254
3255 lif = netdev_priv(netdev);
3256 lif->netdev = netdev;
3257 ionic->lif = lif;
3258 lif->ionic = ionic;
3259 netdev->netdev_ops = &ionic_netdev_ops;
3260 ionic_ethtool_set_ops(netdev);
3261
3262 netdev->watchdog_timeo = 5 * HZ;
3263 netif_carrier_off(netdev);
3264
3265 lif->identity = lid;
3266 lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
3267 err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
3268 if (err) {
3269 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
3270 lif->lif_type, err);
3271 goto err_out_free_netdev;
3272 }
3273 lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
3274 le32_to_cpu(lif->identity->eth.min_frame_size));
3275 lif->netdev->max_mtu =
3276 le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
3277
3278 lif->neqs = ionic->neqs_per_lif;
3279 lif->nxqs = ionic->ntxqs_per_lif;
3280
3281 lif->index = 0;
3282
3283 if (is_kdump_kernel()) {
3284 lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
3285 lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
3286 } else {
3287 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
3288 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
3289 }
3290
3291 /* Convert the default coalesce value to actual hw resolution */
3292 lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
3293 lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
3294 lif->rx_coalesce_usecs);
3295 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3296 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3297 set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
3298 set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
3299
3300 snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
3301
3302 mutex_init(&lif->queue_lock);
3303 mutex_init(&lif->config_lock);
3304 mutex_init(&lif->adev_lock);
3305
3306 spin_lock_init(&lif->adminq_lock);
3307
3308 spin_lock_init(&lif->deferred.lock);
3309 INIT_LIST_HEAD(&lif->deferred.list);
3310 INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
3311
3312 /* allocate lif info */
3313 lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
3314 lif->info = dma_alloc_coherent(dev, lif->info_sz,
3315 &lif->info_pa, GFP_KERNEL);
3316 if (!lif->info) {
3317 dev_err(dev, "Failed to allocate lif info, aborting\n");
3318 err = -ENOMEM;
3319 goto err_out_free_mutex;
3320 }
3321
3322 ionic_debugfs_add_lif(lif);
3323
3324 err = ionic_affinity_masks_alloc(ionic);
3325 if (err)
3326 goto err_out_free_lif_info;
3327
3328 /* allocate control queues and txrx queue arrays */
3329 ionic_lif_queue_identify(lif);
3330 err = ionic_qcqs_alloc(lif);
3331 if (err)
3332 goto err_out_free_affinity_masks;
3333
3334 /* allocate rss indirection table */
3335 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
3336 lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
3337 lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
3338 &lif->rss_ind_tbl_pa,
3339 GFP_KERNEL);
3340
3341 if (!lif->rss_ind_tbl) {
3342 err = -ENOMEM;
3343 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
3344 goto err_out_free_qcqs;
3345 }
3346 netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
3347
3348 ionic_lif_alloc_phc(lif);
3349
3350 return 0;
3351
3352 err_out_free_qcqs:
3353 ionic_qcqs_free(lif);
3354 err_out_free_affinity_masks:
3355 ionic_affinity_masks_free(lif->ionic);
3356 err_out_free_lif_info:
3357 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3358 lif->info = NULL;
3359 lif->info_pa = 0;
3360 err_out_free_mutex:
3361 mutex_destroy(&lif->adev_lock);
3362 mutex_destroy(&lif->config_lock);
3363 mutex_destroy(&lif->queue_lock);
3364 err_out_free_netdev:
3365 free_netdev(lif->netdev);
3366 lif = NULL;
3367 err_out_free_lid:
3368 kfree(lid);
3369
3370 return err;
3371 }
3372
ionic_lif_reset(struct ionic_lif * lif)3373 static void ionic_lif_reset(struct ionic_lif *lif)
3374 {
3375 struct ionic_dev *idev = &lif->ionic->idev;
3376
3377 if (!ionic_is_fw_running(idev))
3378 return;
3379
3380 mutex_lock(&lif->ionic->dev_cmd_lock);
3381 ionic_dev_cmd_lif_reset(idev, lif->index);
3382 ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3383 mutex_unlock(&lif->ionic->dev_cmd_lock);
3384 }
3385
ionic_lif_handle_fw_down(struct ionic_lif * lif)3386 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
3387 {
3388 struct ionic *ionic = lif->ionic;
3389
3390 if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
3391 return;
3392
3393 dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
3394
3395 netif_device_detach(lif->netdev);
3396
3397 ionic_auxbus_unregister(ionic->lif);
3398 mutex_lock(&lif->queue_lock);
3399 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
3400 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
3401 ionic_stop_queues(lif);
3402 }
3403
3404 if (netif_running(lif->netdev)) {
3405 ionic_txrx_deinit(lif);
3406 ionic_txrx_free(lif);
3407 }
3408 ionic_lif_deinit(lif);
3409 ionic_reset(ionic);
3410 ionic_qcqs_free(lif);
3411
3412 mutex_unlock(&lif->queue_lock);
3413
3414 clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
3415 dev_info(ionic->dev, "FW Down: LIFs stopped\n");
3416 }
3417
ionic_restart_lif(struct ionic_lif * lif)3418 int ionic_restart_lif(struct ionic_lif *lif)
3419 {
3420 struct ionic *ionic = lif->ionic;
3421 int err;
3422
3423 mutex_lock(&lif->queue_lock);
3424
3425 if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
3426 dev_info(ionic->dev, "FW Up: clearing broken state\n");
3427
3428 err = ionic_qcqs_alloc(lif);
3429 if (err)
3430 goto err_unlock;
3431
3432 err = ionic_lif_init(lif);
3433 if (err)
3434 goto err_qcqs_free;
3435
3436 ionic_vf_attr_replay(lif);
3437
3438 if (lif->registered)
3439 ionic_lif_set_netdev_info(lif);
3440
3441 ionic_rx_filter_replay(lif);
3442
3443 if (netif_running(lif->netdev)) {
3444 err = ionic_txrx_alloc(lif);
3445 if (err)
3446 goto err_lifs_deinit;
3447
3448 err = ionic_txrx_init(lif);
3449 if (err)
3450 goto err_txrx_free;
3451 }
3452
3453 mutex_unlock(&lif->queue_lock);
3454
3455 clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3456 ionic_link_status_check_request(lif, CAN_SLEEP);
3457 netif_device_attach(lif->netdev);
3458 ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
3459
3460 ionic_auxbus_register(ionic->lif);
3461
3462 return 0;
3463
3464 err_txrx_free:
3465 ionic_txrx_free(lif);
3466 err_lifs_deinit:
3467 ionic_lif_deinit(lif);
3468 err_qcqs_free:
3469 ionic_qcqs_free(lif);
3470 err_unlock:
3471 mutex_unlock(&lif->queue_lock);
3472
3473 return err;
3474 }
3475
ionic_lif_handle_fw_up(struct ionic_lif * lif)3476 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
3477 {
3478 struct ionic *ionic = lif->ionic;
3479 int err;
3480
3481 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3482 return;
3483
3484 dev_info(ionic->dev, "FW Up: restarting LIFs\n");
3485
3486 /* This is a little different from what happens at
3487 * probe time because the LIF already exists so we
3488 * just need to reanimate it.
3489 */
3490 ionic_init_devinfo(ionic);
3491 ionic_reset(ionic);
3492 err = ionic_identify(ionic);
3493 if (err)
3494 goto err_out;
3495 err = ionic_port_identify(ionic);
3496 if (err)
3497 goto err_out;
3498 err = ionic_port_init(ionic);
3499 if (err)
3500 goto err_out;
3501
3502 err = ionic_restart_lif(lif);
3503 if (err)
3504 goto err_out;
3505
3506 dev_info(ionic->dev, "FW Up: LIFs restarted\n");
3507
3508 /* restore the hardware timestamping queues */
3509 ionic_lif_hwstamp_replay(lif);
3510
3511 return;
3512
3513 err_out:
3514 dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3515 }
3516
ionic_lif_free(struct ionic_lif * lif)3517 void ionic_lif_free(struct ionic_lif *lif)
3518 {
3519 struct device *dev = lif->ionic->dev;
3520
3521 ionic_lif_free_phc(lif);
3522
3523 /* free rss indirection table */
3524 dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3525 lif->rss_ind_tbl_pa);
3526 lif->rss_ind_tbl = NULL;
3527 lif->rss_ind_tbl_pa = 0;
3528
3529 /* free queues */
3530 ionic_qcqs_free(lif);
3531 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3532 ionic_lif_reset(lif);
3533
3534 ionic_affinity_masks_free(lif->ionic);
3535
3536 /* free lif info */
3537 kfree(lif->identity);
3538 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3539 lif->info = NULL;
3540 lif->info_pa = 0;
3541
3542 mutex_destroy(&lif->config_lock);
3543 mutex_destroy(&lif->queue_lock);
3544 mutex_destroy(&lif->adev_lock);
3545
3546 /* free netdev & lif */
3547 ionic_debugfs_del_lif(lif);
3548 free_netdev(lif->netdev);
3549 }
3550
ionic_lif_deinit(struct ionic_lif * lif)3551 void ionic_lif_deinit(struct ionic_lif *lif)
3552 {
3553 if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3554 return;
3555
3556 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3557 cancel_work_sync(&lif->deferred.work);
3558 cancel_work_sync(&lif->tx_timeout_work);
3559 ionic_rx_filters_deinit(lif);
3560 if (lif->netdev->features & NETIF_F_RXHASH)
3561 ionic_lif_rss_deinit(lif);
3562 }
3563
3564 napi_disable(&lif->adminqcq->napi);
3565 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3566 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3567
3568 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3569 lif->kern_dbpage = NULL;
3570
3571 ionic_lif_reset(lif);
3572 }
3573
ionic_lif_adminq_init(struct ionic_lif * lif)3574 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3575 {
3576 struct device *dev = lif->ionic->dev;
3577 struct ionic_q_init_comp comp;
3578 struct ionic_dev *idev;
3579 struct ionic_qcq *qcq;
3580 struct ionic_queue *q;
3581 int err;
3582
3583 idev = &lif->ionic->idev;
3584 qcq = lif->adminqcq;
3585 q = &qcq->q;
3586
3587 mutex_lock(&lif->ionic->dev_cmd_lock);
3588 ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3589 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3590 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3591 mutex_unlock(&lif->ionic->dev_cmd_lock);
3592 if (err) {
3593 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3594 return err;
3595 }
3596
3597 q->hw_type = comp.hw_type;
3598 q->hw_index = le32_to_cpu(comp.hw_index);
3599 q->dbval = IONIC_DBELL_QID(q->hw_index);
3600
3601 dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3602 dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3603
3604 q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;
3605 q->dbell_jiffies = jiffies;
3606
3607 netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
3608
3609 napi_enable(&qcq->napi);
3610
3611 if (qcq->flags & IONIC_QCQ_F_INTR) {
3612 irq_set_affinity_hint(qcq->intr.vector,
3613 *qcq->intr.affinity_mask);
3614 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3615 IONIC_INTR_MASK_CLEAR);
3616 }
3617
3618 qcq->flags |= IONIC_QCQ_F_INITED;
3619
3620 return 0;
3621 }
3622
ionic_lif_notifyq_init(struct ionic_lif * lif)3623 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3624 {
3625 struct ionic_qcq *qcq = lif->notifyqcq;
3626 struct device *dev = lif->ionic->dev;
3627 struct ionic_queue *q = &qcq->q;
3628 int err;
3629
3630 struct ionic_admin_ctx ctx = {
3631 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3632 .cmd.q_init = {
3633 .opcode = IONIC_CMD_Q_INIT,
3634 .lif_index = cpu_to_le16(lif->index),
3635 .type = q->type,
3636 .ver = lif->qtype_info[q->type].version,
3637 .index = cpu_to_le32(q->index),
3638 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3639 IONIC_QINIT_F_ENA),
3640 .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3641 .pid = cpu_to_le16(q->pid),
3642 .ring_size = ilog2(q->num_descs),
3643 .ring_base = cpu_to_le64(q->base_pa),
3644 }
3645 };
3646
3647 dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3648 dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3649 dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3650 dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3651
3652 err = ionic_adminq_post_wait(lif, &ctx);
3653 if (err)
3654 return err;
3655
3656 lif->last_eid = 0;
3657 q->hw_type = ctx.comp.q_init.hw_type;
3658 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3659 q->dbval = IONIC_DBELL_QID(q->hw_index);
3660
3661 dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3662 dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3663
3664 /* preset the callback info */
3665 q->admin_info[0].ctx = lif;
3666
3667 qcq->flags |= IONIC_QCQ_F_INITED;
3668
3669 return 0;
3670 }
3671
ionic_station_set(struct ionic_lif * lif)3672 static int ionic_station_set(struct ionic_lif *lif)
3673 {
3674 struct net_device *netdev = lif->netdev;
3675 struct ionic_admin_ctx ctx = {
3676 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3677 .cmd.lif_getattr = {
3678 .opcode = IONIC_CMD_LIF_GETATTR,
3679 .index = cpu_to_le16(lif->index),
3680 .attr = IONIC_LIF_ATTR_MAC,
3681 },
3682 };
3683 u8 mac_address[ETH_ALEN];
3684 struct sockaddr addr;
3685 int err;
3686
3687 err = ionic_adminq_post_wait(lif, &ctx);
3688 if (err)
3689 return err;
3690 netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3691 ctx.comp.lif_getattr.mac);
3692 ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);
3693
3694 if (is_zero_ether_addr(mac_address)) {
3695 eth_hw_addr_random(netdev);
3696 netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);
3697 ether_addr_copy(mac_address, netdev->dev_addr);
3698
3699 err = ionic_program_mac(lif, mac_address);
3700 if (err < 0)
3701 return err;
3702
3703 if (err > 0) {
3704 netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",
3705 __func__);
3706 return 0;
3707 }
3708 }
3709
3710 if (!is_zero_ether_addr(netdev->dev_addr)) {
3711 /* If the netdev mac is non-zero and doesn't match the default
3712 * device address, it was set by something earlier and we're
3713 * likely here again after a fw-upgrade reset. We need to be
3714 * sure the netdev mac is in our filter list.
3715 */
3716 if (!ether_addr_equal(mac_address, netdev->dev_addr))
3717 ionic_lif_addr_add(lif, netdev->dev_addr);
3718 } else {
3719 /* Update the netdev mac with the device's mac */
3720 ether_addr_copy(addr.sa_data, mac_address);
3721 addr.sa_family = AF_INET;
3722 err = eth_prepare_mac_addr_change(netdev, &addr);
3723 if (err) {
3724 netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3725 addr.sa_data, err);
3726 return 0;
3727 }
3728
3729 eth_commit_mac_addr_change(netdev, &addr);
3730 }
3731
3732 netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3733 netdev->dev_addr);
3734 ionic_lif_addr_add(lif, netdev->dev_addr);
3735
3736 return 0;
3737 }
3738
ionic_lif_init(struct ionic_lif * lif)3739 int ionic_lif_init(struct ionic_lif *lif)
3740 {
3741 struct ionic_dev *idev = &lif->ionic->idev;
3742 struct device *dev = lif->ionic->dev;
3743 struct ionic_lif_init_comp comp;
3744 int dbpage_num;
3745 int err;
3746
3747 mutex_lock(&lif->ionic->dev_cmd_lock);
3748 ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3749 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3750 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3751 mutex_unlock(&lif->ionic->dev_cmd_lock);
3752 if (err)
3753 return err;
3754
3755 lif->hw_index = le16_to_cpu(comp.hw_index);
3756
3757 /* now that we have the hw_index we can figure out our doorbell page */
3758 lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3759 if (!lif->dbid_count) {
3760 dev_err(dev, "No doorbell pages, aborting\n");
3761 return -EINVAL;
3762 }
3763
3764 lif->kern_pid = 0;
3765 dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3766 lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3767 if (!lif->kern_dbpage) {
3768 dev_err(dev, "Cannot map dbpage, aborting\n");
3769 return -ENOMEM;
3770 }
3771
3772 err = ionic_lif_adminq_init(lif);
3773 if (err)
3774 goto err_out_adminq_deinit;
3775
3776 if (lif->ionic->nnqs_per_lif) {
3777 err = ionic_lif_notifyq_init(lif);
3778 if (err)
3779 goto err_out_notifyq_deinit;
3780 }
3781
3782 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3783 err = ionic_set_nic_features(lif, lif->netdev->features);
3784 else
3785 err = ionic_init_nic_features(lif);
3786 if (err)
3787 goto err_out_notifyq_deinit;
3788
3789 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3790 err = ionic_rx_filters_init(lif);
3791 if (err)
3792 goto err_out_notifyq_deinit;
3793 }
3794
3795 err = ionic_station_set(lif);
3796 if (err)
3797 goto err_out_notifyq_deinit;
3798
3799 lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3800 lif->doorbell_wa = ionic_doorbell_wa(lif->ionic);
3801
3802 set_bit(IONIC_LIF_F_INITED, lif->state);
3803
3804 INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3805
3806 return 0;
3807
3808 err_out_notifyq_deinit:
3809 napi_disable(&lif->adminqcq->napi);
3810 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3811 err_out_adminq_deinit:
3812 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3813 ionic_lif_reset(lif);
3814 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3815 lif->kern_dbpage = NULL;
3816
3817 return err;
3818 }
3819
ionic_lif_set_netdev_info(struct ionic_lif * lif)3820 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3821 {
3822 struct ionic_admin_ctx ctx = {
3823 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3824 .cmd.lif_setattr = {
3825 .opcode = IONIC_CMD_LIF_SETATTR,
3826 .index = cpu_to_le16(lif->index),
3827 .attr = IONIC_LIF_ATTR_NAME,
3828 },
3829 };
3830
3831 strscpy(ctx.cmd.lif_setattr.name, netdev_name(lif->netdev),
3832 sizeof(ctx.cmd.lif_setattr.name));
3833
3834 ionic_adminq_post_wait(lif, &ctx);
3835 }
3836
ionic_netdev_lif(struct net_device * netdev)3837 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3838 {
3839 if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3840 return NULL;
3841
3842 return netdev_priv(netdev);
3843 }
3844
ionic_lif_notify(struct notifier_block * nb,unsigned long event,void * info)3845 static int ionic_lif_notify(struct notifier_block *nb,
3846 unsigned long event, void *info)
3847 {
3848 struct net_device *ndev = netdev_notifier_info_to_dev(info);
3849 struct ionic *ionic = container_of(nb, struct ionic, nb);
3850 struct ionic_lif *lif = ionic_netdev_lif(ndev);
3851
3852 if (!lif || lif->ionic != ionic)
3853 return NOTIFY_DONE;
3854
3855 switch (event) {
3856 case NETDEV_CHANGENAME:
3857 ionic_lif_set_netdev_info(lif);
3858 break;
3859 }
3860
3861 return NOTIFY_DONE;
3862 }
3863
ionic_lif_register(struct ionic_lif * lif)3864 int ionic_lif_register(struct ionic_lif *lif)
3865 {
3866 int err;
3867
3868 ionic_lif_register_phc(lif);
3869
3870 lif->ionic->nb.notifier_call = ionic_lif_notify;
3871
3872 err = register_netdevice_notifier(&lif->ionic->nb);
3873 if (err)
3874 lif->ionic->nb.notifier_call = NULL;
3875
3876 /* only register LIF0 for now */
3877 err = register_netdev(lif->netdev);
3878 if (err) {
3879 dev_err(lif->ionic->dev, "Cannot register net device: %d, aborting\n", err);
3880 ionic_lif_unregister(lif);
3881 return err;
3882 }
3883
3884 ionic_link_status_check_request(lif, CAN_SLEEP);
3885 lif->registered = true;
3886 ionic_lif_set_netdev_info(lif);
3887
3888 return 0;
3889 }
3890
ionic_lif_unregister(struct ionic_lif * lif)3891 void ionic_lif_unregister(struct ionic_lif *lif)
3892 {
3893 if (lif->ionic->nb.notifier_call) {
3894 unregister_netdevice_notifier(&lif->ionic->nb);
3895 lif->ionic->nb.notifier_call = NULL;
3896 }
3897
3898 if (lif->netdev->reg_state == NETREG_REGISTERED)
3899 unregister_netdev(lif->netdev);
3900
3901 ionic_lif_unregister_phc(lif);
3902
3903 lif->registered = false;
3904 }
3905
ionic_lif_queue_identify(struct ionic_lif * lif)3906 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3907 {
3908 union ionic_q_identity __iomem *q_ident;
3909 struct ionic *ionic = lif->ionic;
3910 struct ionic_dev *idev;
3911 u16 max_frags;
3912 int qtype;
3913 int err;
3914
3915 idev = &lif->ionic->idev;
3916 q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3917
3918 for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3919 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3920
3921 /* filter out the ones we know about */
3922 switch (qtype) {
3923 case IONIC_QTYPE_ADMINQ:
3924 case IONIC_QTYPE_NOTIFYQ:
3925 case IONIC_QTYPE_RXQ:
3926 case IONIC_QTYPE_TXQ:
3927 break;
3928 default:
3929 continue;
3930 }
3931
3932 memset(qti, 0, sizeof(*qti));
3933
3934 mutex_lock(&ionic->dev_cmd_lock);
3935 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3936 ionic_qtype_versions[qtype]);
3937 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3938 if (!err) {
3939 qti->version = readb(&q_ident->version);
3940 qti->supported = readb(&q_ident->supported);
3941 qti->features = readq(&q_ident->features);
3942 qti->desc_sz = readw(&q_ident->desc_sz);
3943 qti->comp_sz = readw(&q_ident->comp_sz);
3944 qti->sg_desc_sz = readw(&q_ident->sg_desc_sz);
3945 qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3946 qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3947 }
3948 mutex_unlock(&ionic->dev_cmd_lock);
3949
3950 if (err == -EINVAL) {
3951 dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3952 continue;
3953 } else if (err == -EIO) {
3954 dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3955 return;
3956 } else if (err) {
3957 dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3958 qtype, err);
3959 return;
3960 }
3961
3962 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3963 qtype, qti->version);
3964 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3965 qtype, qti->supported);
3966 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3967 qtype, qti->features);
3968 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3969 qtype, qti->desc_sz);
3970 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3971 qtype, qti->comp_sz);
3972 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3973 qtype, qti->sg_desc_sz);
3974 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3975 qtype, qti->max_sg_elems);
3976 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3977 qtype, qti->sg_desc_stride);
3978
3979 if (qtype == IONIC_QTYPE_TXQ)
3980 max_frags = IONIC_TX_MAX_FRAGS;
3981 else if (qtype == IONIC_QTYPE_RXQ)
3982 max_frags = IONIC_RX_MAX_FRAGS;
3983 else
3984 max_frags = 1;
3985
3986 qti->max_sg_elems = min_t(u16, max_frags - 1, MAX_SKB_FRAGS);
3987 dev_dbg(ionic->dev, "qtype %d max_sg_elems %d\n",
3988 qtype, qti->max_sg_elems);
3989 }
3990 }
3991
ionic_lif_identify(struct ionic * ionic,u8 lif_type,union ionic_lif_identity * lid)3992 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3993 union ionic_lif_identity *lid)
3994 {
3995 struct ionic_dev *idev = &ionic->idev;
3996 size_t sz;
3997 int err;
3998
3999 sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
4000
4001 mutex_lock(&ionic->dev_cmd_lock);
4002 ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
4003 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
4004 memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
4005 mutex_unlock(&ionic->dev_cmd_lock);
4006 if (err)
4007 return (err);
4008
4009 dev_dbg(ionic->dev, "capabilities 0x%llx\n",
4010 le64_to_cpu(lid->capabilities));
4011
4012 dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
4013 le32_to_cpu(lid->eth.max_ucast_filters));
4014 dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
4015 le32_to_cpu(lid->eth.max_mcast_filters));
4016 dev_dbg(ionic->dev, "eth.features 0x%llx\n",
4017 le64_to_cpu(lid->eth.config.features));
4018 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
4019 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
4020 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
4021 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
4022 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
4023 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
4024 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
4025 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
4026 dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
4027 dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
4028 dev_dbg(ionic->dev, "eth.config.mtu %d\n",
4029 le32_to_cpu(lid->eth.config.mtu));
4030
4031 return 0;
4032 }
4033
ionic_lif_size(struct ionic * ionic)4034 int ionic_lif_size(struct ionic *ionic)
4035 {
4036 struct ionic_identity *ident = &ionic->ident;
4037 unsigned int nintrs, dev_nintrs;
4038 union ionic_lif_config *lc;
4039 unsigned int ntxqs_per_lif;
4040 unsigned int nrxqs_per_lif;
4041 unsigned int neqs_per_lif;
4042 unsigned int nnqs_per_lif;
4043 unsigned int nxqs, neqs;
4044 unsigned int min_intrs;
4045 int err;
4046
4047 /* retrieve basic values from FW */
4048 lc = &ident->lif.eth.config;
4049 dev_nintrs = le32_to_cpu(ident->dev.nintrs);
4050 neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
4051 nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
4052 ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
4053 nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
4054
4055 /* limit values to play nice with kdump */
4056 if (is_kdump_kernel()) {
4057 dev_nintrs = 2;
4058 neqs_per_lif = 0;
4059 nnqs_per_lif = 0;
4060 ntxqs_per_lif = 1;
4061 nrxqs_per_lif = 1;
4062 }
4063
4064 /* reserve last queue id for hardware timestamping */
4065 if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
4066 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
4067 lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
4068 } else {
4069 ntxqs_per_lif -= 1;
4070 nrxqs_per_lif -= 1;
4071 }
4072 }
4073
4074 nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
4075 nxqs = min(nxqs, num_online_cpus());
4076 neqs = min(neqs_per_lif, num_online_cpus());
4077
4078 try_again:
4079 /* interrupt usage:
4080 * 1 for master lif adminq/notifyq
4081 * 1 for each CPU for master lif TxRx queue pairs
4082 * whatever's left is for RDMA queues
4083 */
4084 nintrs = 1 + nxqs + neqs;
4085 min_intrs = 2; /* adminq + 1 TxRx queue pair */
4086
4087 if (nintrs > dev_nintrs)
4088 goto try_fewer;
4089
4090 err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
4091 if (err < 0 && err != -ENOSPC) {
4092 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
4093 return err;
4094 }
4095 if (err == -ENOSPC)
4096 goto try_fewer;
4097
4098 if (err != nintrs) {
4099 ionic_bus_free_irq_vectors(ionic);
4100 goto try_fewer;
4101 }
4102
4103 ionic->nnqs_per_lif = nnqs_per_lif;
4104 ionic->neqs_per_lif = neqs;
4105 ionic->ntxqs_per_lif = nxqs;
4106 ionic->nrxqs_per_lif = nxqs;
4107 ionic->nintrs = nintrs;
4108
4109 ionic_debugfs_add_sizes(ionic);
4110
4111 return 0;
4112
4113 try_fewer:
4114 if (nnqs_per_lif > 1) {
4115 nnqs_per_lif >>= 1;
4116 goto try_again;
4117 }
4118 if (neqs > 1) {
4119 neqs >>= 1;
4120 goto try_again;
4121 }
4122 if (nxqs > 1) {
4123 nxqs >>= 1;
4124 goto try_again;
4125 }
4126 dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
4127 return -ENOSPC;
4128 }
4129