1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3
4 #include <linux/ethtool.h>
5 #include <linux/printk.h>
6 #include <linux/dynamic_debug.h>
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/interrupt.h>
12 #include <linux/pci.h>
13 #include <linux/cpumask.h>
14 #include <linux/crash_dump.h>
15 #include <linux/vmalloc.h>
16 #include <net/page_pool/helpers.h>
17
18 #include "ionic.h"
19 #include "ionic_bus.h"
20 #include "ionic_dev.h"
21 #include "ionic_lif.h"
22 #include "ionic_txrx.h"
23 #include "ionic_ethtool.h"
24 #include "ionic_debugfs.h"
25
26 /* queuetype support level */
27 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
28 [IONIC_QTYPE_ADMINQ] = 0, /* 0 = Base version with CQ support */
29 [IONIC_QTYPE_NOTIFYQ] = 0, /* 0 = Base version */
30 [IONIC_QTYPE_RXQ] = 2, /* 0 = Base version with CQ+SG support
31 * 2 = ... with CMB rings
32 */
33 [IONIC_QTYPE_TXQ] = 3, /* 0 = Base version with CQ+SG support
34 * 1 = ... with Tx SG version 1
35 * 3 = ... with CMB rings
36 */
37 };
38
39 static void ionic_link_status_check(struct ionic_lif *lif);
40 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
41 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
42 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
43
44 static void ionic_txrx_deinit(struct ionic_lif *lif);
45 static int ionic_txrx_init(struct ionic_lif *lif);
46 static int ionic_start_queues(struct ionic_lif *lif);
47 static void ionic_stop_queues(struct ionic_lif *lif);
48 static void ionic_lif_queue_identify(struct ionic_lif *lif);
49
50 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif);
51 static void ionic_unregister_rxq_info(struct ionic_queue *q);
52 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id);
53
ionic_dim_work(struct work_struct * work)54 static void ionic_dim_work(struct work_struct *work)
55 {
56 struct dim *dim = container_of(work, struct dim, work);
57 struct dim_cq_moder cur_moder;
58 struct ionic_intr_info *intr;
59 struct ionic_qcq *qcq;
60 struct ionic_lif *lif;
61 struct ionic_queue *q;
62 u32 new_coal;
63
64 qcq = container_of(dim, struct ionic_qcq, dim);
65 q = &qcq->q;
66 if (q->type == IONIC_QTYPE_RXQ)
67 cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
68 else
69 cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
70 lif = q->lif;
71 new_coal = ionic_coal_usec_to_hw(lif->ionic, cur_moder.usec);
72 new_coal = new_coal ? new_coal : 1;
73
74 intr = &qcq->intr;
75 if (intr->dim_coal_hw != new_coal) {
76 intr->dim_coal_hw = new_coal;
77
78 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
79 intr->index, intr->dim_coal_hw);
80 }
81
82 dim->state = DIM_START_MEASURE;
83 }
84
ionic_lif_deferred_work(struct work_struct * work)85 static void ionic_lif_deferred_work(struct work_struct *work)
86 {
87 struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
88 struct ionic_deferred *def = &lif->deferred;
89 struct ionic_deferred_work *w = NULL;
90
91 do {
92 spin_lock_bh(&def->lock);
93 if (!list_empty(&def->list)) {
94 w = list_first_entry(&def->list,
95 struct ionic_deferred_work, list);
96 list_del(&w->list);
97 }
98 spin_unlock_bh(&def->lock);
99
100 if (!w)
101 break;
102
103 switch (w->type) {
104 case IONIC_DW_TYPE_RX_MODE:
105 ionic_lif_rx_mode(lif);
106 break;
107 case IONIC_DW_TYPE_LINK_STATUS:
108 ionic_link_status_check(lif);
109 break;
110 case IONIC_DW_TYPE_LIF_RESET:
111 if (w->fw_status) {
112 ionic_lif_handle_fw_up(lif);
113 } else {
114 ionic_lif_handle_fw_down(lif);
115
116 /* Fire off another watchdog to see
117 * if the FW is already back rather than
118 * waiting another whole cycle
119 */
120 mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
121 }
122 break;
123 default:
124 break;
125 }
126 kfree(w);
127 w = NULL;
128 } while (true);
129 }
130
ionic_lif_deferred_enqueue(struct ionic_lif * lif,struct ionic_deferred_work * work)131 void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
132 struct ionic_deferred_work *work)
133 {
134 spin_lock_bh(&lif->deferred.lock);
135 list_add_tail(&work->list, &lif->deferred.list);
136 spin_unlock_bh(&lif->deferred.lock);
137 queue_work(lif->ionic->wq, &lif->deferred.work);
138 }
139
ionic_link_status_check(struct ionic_lif * lif)140 static void ionic_link_status_check(struct ionic_lif *lif)
141 {
142 struct net_device *netdev = lif->netdev;
143 u16 link_status;
144 bool link_up;
145
146 if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
147 return;
148
149 /* Don't put carrier back up if we're in a broken state */
150 if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
151 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
152 return;
153 }
154
155 link_status = le16_to_cpu(lif->info->status.link_status);
156 link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
157
158 if (link_up) {
159 int err = 0;
160
161 if (netdev->flags & IFF_UP && netif_running(netdev)) {
162 mutex_lock(&lif->queue_lock);
163 err = ionic_start_queues(lif);
164 if (err && err != -EBUSY) {
165 netdev_err(netdev,
166 "Failed to start queues: %d\n", err);
167 set_bit(IONIC_LIF_F_BROKEN, lif->state);
168 netif_carrier_off(lif->netdev);
169 }
170 mutex_unlock(&lif->queue_lock);
171 }
172
173 if (!err && !netif_carrier_ok(netdev)) {
174 ionic_port_identify(lif->ionic);
175 netdev_info(netdev, "Link up - %d Gbps\n",
176 le32_to_cpu(lif->info->status.link_speed) / 1000);
177 netif_carrier_on(netdev);
178 }
179 } else {
180 if (netif_carrier_ok(netdev)) {
181 lif->link_down_count++;
182 netdev_info(netdev, "Link down\n");
183 netif_carrier_off(netdev);
184 }
185
186 if (netdev->flags & IFF_UP && netif_running(netdev)) {
187 mutex_lock(&lif->queue_lock);
188 ionic_stop_queues(lif);
189 mutex_unlock(&lif->queue_lock);
190 }
191 }
192
193 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
194 }
195
ionic_link_status_check_request(struct ionic_lif * lif,bool can_sleep)196 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
197 {
198 struct ionic_deferred_work *work;
199
200 /* we only need one request outstanding at a time */
201 if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
202 return;
203
204 if (!can_sleep) {
205 work = kzalloc(sizeof(*work), GFP_ATOMIC);
206 if (!work) {
207 clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
208 return;
209 }
210
211 work->type = IONIC_DW_TYPE_LINK_STATUS;
212 ionic_lif_deferred_enqueue(lif, work);
213 } else {
214 ionic_link_status_check(lif);
215 }
216 }
217
ionic_isr(int irq,void * data)218 static irqreturn_t ionic_isr(int irq, void *data)
219 {
220 struct napi_struct *napi = data;
221
222 napi_schedule_irqoff(napi);
223
224 return IRQ_HANDLED;
225 }
226
ionic_request_irq(struct ionic_lif * lif,struct ionic_qcq * qcq)227 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
228 {
229 struct ionic_intr_info *intr = &qcq->intr;
230 struct device *dev = lif->ionic->dev;
231 struct ionic_queue *q = &qcq->q;
232 const char *name;
233
234 if (lif->registered)
235 name = netdev_name(lif->netdev);
236 else
237 name = dev_name(dev);
238
239 snprintf(intr->name, sizeof(intr->name),
240 "%.5s-%.16s-%.8s", IONIC_DRV_NAME, name, q->name);
241
242 return devm_request_irq(dev, intr->vector, ionic_isr,
243 0, intr->name, &qcq->napi);
244 }
245
ionic_intr_alloc(struct ionic_lif * lif,struct ionic_intr_info * intr)246 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
247 {
248 struct ionic *ionic = lif->ionic;
249 int index;
250
251 index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
252 if (index == ionic->nintrs) {
253 netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
254 __func__, index, ionic->nintrs);
255 return -ENOSPC;
256 }
257
258 set_bit(index, ionic->intrs);
259 ionic_intr_init(&ionic->idev, intr, index);
260
261 return 0;
262 }
263
ionic_intr_free(struct ionic * ionic,int index)264 static void ionic_intr_free(struct ionic *ionic, int index)
265 {
266 if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
267 clear_bit(index, ionic->intrs);
268 }
269
ionic_irq_aff_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)270 static void ionic_irq_aff_notify(struct irq_affinity_notify *notify,
271 const cpumask_t *mask)
272 {
273 struct ionic_intr_info *intr = container_of(notify, struct ionic_intr_info, aff_notify);
274
275 cpumask_copy(*intr->affinity_mask, mask);
276 }
277
ionic_irq_aff_release(struct kref __always_unused * ref)278 static void ionic_irq_aff_release(struct kref __always_unused *ref)
279 {
280 }
281
ionic_qcq_enable(struct ionic_qcq * qcq)282 static int ionic_qcq_enable(struct ionic_qcq *qcq)
283 {
284 struct ionic_queue *q = &qcq->q;
285 struct ionic_lif *lif = q->lif;
286 struct ionic_dev *idev;
287 struct device *dev;
288
289 struct ionic_admin_ctx ctx = {
290 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
291 .cmd.q_control = {
292 .opcode = IONIC_CMD_Q_CONTROL,
293 .lif_index = cpu_to_le16(lif->index),
294 .type = q->type,
295 .index = cpu_to_le32(q->index),
296 .oper = IONIC_Q_ENABLE,
297 },
298 };
299 int ret;
300
301 idev = &lif->ionic->idev;
302 dev = lif->ionic->dev;
303
304 dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
305 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
306
307 if (qcq->flags & IONIC_QCQ_F_INTR)
308 ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
309
310 ret = ionic_adminq_post_wait(lif, &ctx);
311 if (ret)
312 return ret;
313
314 if (qcq->flags & IONIC_QCQ_F_INTR) {
315 napi_enable(&qcq->napi);
316 irq_set_affinity_notifier(qcq->intr.vector,
317 &qcq->intr.aff_notify);
318 irq_set_affinity_hint(qcq->intr.vector,
319 *qcq->intr.affinity_mask);
320 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
321 IONIC_INTR_MASK_CLEAR);
322 }
323
324 return 0;
325 }
326
ionic_qcq_disable(struct ionic_lif * lif,struct ionic_qcq * qcq,int fw_err)327 static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
328 {
329 struct ionic_queue *q;
330
331 struct ionic_admin_ctx ctx = {
332 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
333 .cmd.q_control = {
334 .opcode = IONIC_CMD_Q_CONTROL,
335 .oper = IONIC_Q_DISABLE,
336 },
337 };
338
339 if (!qcq) {
340 netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
341 return -ENXIO;
342 }
343
344 q = &qcq->q;
345
346 if (qcq->flags & IONIC_QCQ_F_INTR) {
347 struct ionic_dev *idev = &lif->ionic->idev;
348
349 if (lif->doorbell_wa)
350 cancel_work_sync(&qcq->doorbell_napi_work);
351 cancel_work_sync(&qcq->dim.work);
352 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
353 IONIC_INTR_MASK_SET);
354 synchronize_irq(qcq->intr.vector);
355 irq_set_affinity_notifier(qcq->intr.vector, NULL);
356 irq_set_affinity_hint(qcq->intr.vector, NULL);
357 napi_disable(&qcq->napi);
358 }
359
360 /* If there was a previous fw communcation error, don't bother with
361 * sending the adminq command and just return the same error value.
362 */
363 if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
364 return fw_err;
365
366 ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
367 ctx.cmd.q_control.type = q->type;
368 ctx.cmd.q_control.index = cpu_to_le32(q->index);
369 dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
370 ctx.cmd.q_control.index, ctx.cmd.q_control.type);
371
372 return ionic_adminq_post_wait(lif, &ctx);
373 }
374
ionic_lif_qcq_deinit(struct ionic_lif * lif,struct ionic_qcq * qcq)375 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
376 {
377 struct ionic_dev *idev = &lif->ionic->idev;
378
379 if (!qcq)
380 return;
381
382 if (!(qcq->flags & IONIC_QCQ_F_INITED))
383 return;
384
385 ionic_unregister_rxq_info(&qcq->q);
386 if (qcq->flags & IONIC_QCQ_F_INTR) {
387 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
388 IONIC_INTR_MASK_SET);
389 netif_napi_del(&qcq->napi);
390 }
391
392 qcq->flags &= ~IONIC_QCQ_F_INITED;
393 }
394
ionic_qcq_intr_free(struct ionic_lif * lif,struct ionic_qcq * qcq)395 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
396 {
397 if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
398 return;
399
400 irq_set_affinity_hint(qcq->intr.vector, NULL);
401 devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
402 qcq->intr.vector = 0;
403 ionic_intr_free(lif->ionic, qcq->intr.index);
404 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
405 }
406
ionic_qcq_free(struct ionic_lif * lif,struct ionic_qcq * qcq)407 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
408 {
409 struct device *dev = lif->ionic->dev;
410
411 if (!qcq)
412 return;
413
414 ionic_debugfs_del_qcq(qcq);
415
416 if (qcq->q_base) {
417 dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
418 qcq->q_base = NULL;
419 qcq->q_base_pa = 0;
420 }
421
422 if (qcq->cmb_q_base) {
423 iounmap(qcq->cmb_q_base);
424 ionic_put_cmb(lif, qcq->cmb_pgid, qcq->cmb_order);
425 qcq->cmb_pgid = 0;
426 qcq->cmb_order = 0;
427 qcq->cmb_q_base = NULL;
428 qcq->cmb_q_base_pa = 0;
429 }
430
431 if (qcq->cq_base) {
432 dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
433 qcq->cq_base = NULL;
434 qcq->cq_base_pa = 0;
435 }
436
437 if (qcq->sg_base) {
438 dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
439 qcq->sg_base = NULL;
440 qcq->sg_base_pa = 0;
441 }
442
443 page_pool_destroy(qcq->q.page_pool);
444 qcq->q.page_pool = NULL;
445
446 ionic_qcq_intr_free(lif, qcq);
447 vfree(qcq->q.info);
448 qcq->q.info = NULL;
449 }
450
ionic_qcqs_free(struct ionic_lif * lif)451 void ionic_qcqs_free(struct ionic_lif *lif)
452 {
453 struct device *dev = lif->ionic->dev;
454 struct ionic_qcq *adminqcq;
455 unsigned long irqflags;
456
457 if (lif->notifyqcq) {
458 ionic_qcq_free(lif, lif->notifyqcq);
459 devm_kfree(dev, lif->notifyqcq);
460 lif->notifyqcq = NULL;
461 }
462
463 if (lif->adminqcq) {
464 spin_lock_irqsave(&lif->adminq_lock, irqflags);
465 adminqcq = READ_ONCE(lif->adminqcq);
466 lif->adminqcq = NULL;
467 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
468 if (adminqcq) {
469 ionic_qcq_free(lif, adminqcq);
470 devm_kfree(dev, adminqcq);
471 }
472 }
473
474 if (lif->rxqcqs) {
475 devm_kfree(dev, lif->rxqstats);
476 lif->rxqstats = NULL;
477 devm_kfree(dev, lif->rxqcqs);
478 lif->rxqcqs = NULL;
479 }
480
481 if (lif->txqcqs) {
482 devm_kfree(dev, lif->txqstats);
483 lif->txqstats = NULL;
484 devm_kfree(dev, lif->txqcqs);
485 lif->txqcqs = NULL;
486 }
487 }
488
ionic_link_qcq_interrupts(struct ionic_qcq * src_qcq,struct ionic_qcq * n_qcq)489 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
490 struct ionic_qcq *n_qcq)
491 {
492 n_qcq->intr.vector = src_qcq->intr.vector;
493 n_qcq->intr.index = src_qcq->intr.index;
494 }
495
ionic_alloc_qcq_interrupt(struct ionic_lif * lif,struct ionic_qcq * qcq)496 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
497 {
498 cpumask_var_t *affinity_mask;
499 int err;
500
501 if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
502 qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
503 return 0;
504 }
505
506 err = ionic_intr_alloc(lif, &qcq->intr);
507 if (err) {
508 netdev_warn(lif->netdev, "no intr for %s: %d\n",
509 qcq->q.name, err);
510 goto err_out;
511 }
512
513 err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
514 if (err < 0) {
515 netdev_warn(lif->netdev, "no vector for %s: %d\n",
516 qcq->q.name, err);
517 goto err_out_free_intr;
518 }
519 qcq->intr.vector = err;
520 ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
521 IONIC_INTR_MASK_SET);
522
523 err = ionic_request_irq(lif, qcq);
524 if (err) {
525 netdev_warn(lif->netdev, "irq request failed %d\n", err);
526 goto err_out_free_intr;
527 }
528
529 /* try to get the irq on the local numa node first */
530 affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];
531 if (cpumask_empty(*affinity_mask)) {
532 unsigned int cpu;
533
534 cpu = cpumask_local_spread(qcq->intr.index,
535 dev_to_node(lif->ionic->dev));
536 if (cpu != -1)
537 cpumask_set_cpu(cpu, *affinity_mask);
538 }
539
540 qcq->intr.affinity_mask = affinity_mask;
541 qcq->intr.aff_notify.notify = ionic_irq_aff_notify;
542 qcq->intr.aff_notify.release = ionic_irq_aff_release;
543
544 netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
545 return 0;
546
547 err_out_free_intr:
548 ionic_intr_free(lif->ionic, qcq->intr.index);
549 err_out:
550 return err;
551 }
552
ionic_qcq_alloc(struct ionic_lif * lif,unsigned int type,unsigned int index,const char * name,unsigned int flags,unsigned int num_descs,unsigned int desc_size,unsigned int cq_desc_size,unsigned int sg_desc_size,unsigned int desc_info_size,unsigned int pid,struct bpf_prog * xdp_prog,struct ionic_qcq ** qcq)553 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
554 unsigned int index,
555 const char *name, unsigned int flags,
556 unsigned int num_descs, unsigned int desc_size,
557 unsigned int cq_desc_size,
558 unsigned int sg_desc_size,
559 unsigned int desc_info_size,
560 unsigned int pid, struct bpf_prog *xdp_prog,
561 struct ionic_qcq **qcq)
562 {
563 struct ionic_dev *idev = &lif->ionic->idev;
564 struct device *dev = lif->ionic->dev;
565 struct ionic_qcq *new;
566 int err;
567
568 *qcq = NULL;
569
570 new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
571 if (!new) {
572 netdev_err(lif->netdev, "Cannot allocate queue structure\n");
573 err = -ENOMEM;
574 goto err_out;
575 }
576
577 new->q.dev = dev;
578 new->flags = flags;
579
580 new->q.info = vcalloc(num_descs, desc_info_size);
581 if (!new->q.info) {
582 netdev_err(lif->netdev, "Cannot allocate queue info\n");
583 err = -ENOMEM;
584 goto err_out_free_qcq;
585 }
586
587 if (type == IONIC_QTYPE_RXQ) {
588 struct page_pool_params pp_params = {
589 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
590 .order = 0,
591 .pool_size = num_descs,
592 .nid = NUMA_NO_NODE,
593 .dev = lif->ionic->dev,
594 .napi = &new->napi,
595 .dma_dir = DMA_FROM_DEVICE,
596 .max_len = PAGE_SIZE,
597 .netdev = lif->netdev,
598 };
599
600 if (xdp_prog)
601 pp_params.dma_dir = DMA_BIDIRECTIONAL;
602
603 new->q.page_pool = page_pool_create(&pp_params);
604 if (IS_ERR(new->q.page_pool)) {
605 netdev_err(lif->netdev, "Cannot create page_pool\n");
606 err = PTR_ERR(new->q.page_pool);
607 new->q.page_pool = NULL;
608 goto err_out_free_q_info;
609 }
610 }
611
612 new->q.type = type;
613 new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
614
615 err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
616 desc_size, sg_desc_size, pid);
617 if (err) {
618 netdev_err(lif->netdev, "Cannot initialize queue\n");
619 goto err_out_free_page_pool;
620 }
621
622 err = ionic_alloc_qcq_interrupt(lif, new);
623 if (err)
624 goto err_out_free_page_pool;
625
626 err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
627 if (err) {
628 netdev_err(lif->netdev, "Cannot initialize completion queue\n");
629 goto err_out_free_irq;
630 }
631
632 if (flags & IONIC_QCQ_F_NOTIFYQ) {
633 int q_size;
634
635 /* q & cq need to be contiguous in NotifyQ, so alloc it all in q
636 * and don't alloc qc. We leave new->qc_size and new->qc_base
637 * as 0 to be sure we don't try to free it later.
638 */
639 q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
640 new->q_size = PAGE_SIZE + q_size +
641 ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
642 new->q_base = dma_alloc_coherent(dev, new->q_size,
643 &new->q_base_pa, GFP_KERNEL);
644 if (!new->q_base) {
645 netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
646 err = -ENOMEM;
647 goto err_out_free_irq;
648 }
649 new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
650 new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
651
652 /* Base the NotifyQ cq.base off of the ALIGNed q.base */
653 new->cq.base = PTR_ALIGN(new->q.base + q_size, PAGE_SIZE);
654 new->cq.base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
655 new->cq.bound_q = &new->q;
656 } else {
657 /* regular DMA q descriptors */
658 new->q_size = PAGE_SIZE + (num_descs * desc_size);
659 new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
660 GFP_KERNEL);
661 if (!new->q_base) {
662 netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
663 err = -ENOMEM;
664 goto err_out_free_irq;
665 }
666 new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
667 new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
668
669 if (flags & IONIC_QCQ_F_CMB_RINGS) {
670 /* on-chip CMB q descriptors */
671 new->cmb_q_size = num_descs * desc_size;
672 new->cmb_order = order_base_2(new->cmb_q_size / PAGE_SIZE);
673
674 err = ionic_get_cmb(lif, &new->cmb_pgid, &new->cmb_q_base_pa,
675 new->cmb_order);
676 if (err) {
677 netdev_err(lif->netdev,
678 "Cannot allocate queue order %d from cmb: err %d\n",
679 new->cmb_order, err);
680 goto err_out_free_q;
681 }
682
683 new->cmb_q_base = ioremap_wc(new->cmb_q_base_pa, new->cmb_q_size);
684 if (!new->cmb_q_base) {
685 netdev_err(lif->netdev, "Cannot map queue from cmb\n");
686 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
687 err = -ENOMEM;
688 goto err_out_free_q;
689 }
690
691 new->cmb_q_base_pa -= idev->phy_cmb_pages;
692 new->q.cmb_base = new->cmb_q_base;
693 new->q.cmb_base_pa = new->cmb_q_base_pa;
694 }
695
696 /* cq DMA descriptors */
697 new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
698 new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
699 GFP_KERNEL);
700 if (!new->cq_base) {
701 netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
702 err = -ENOMEM;
703 goto err_out_free_q;
704 }
705 new->cq.base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
706 new->cq.base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
707 new->cq.bound_q = &new->q;
708 }
709
710 if (flags & IONIC_QCQ_F_SG) {
711 new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
712 new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
713 GFP_KERNEL);
714 if (!new->sg_base) {
715 netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
716 err = -ENOMEM;
717 goto err_out_free_cq;
718 }
719 new->q.sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
720 new->q.sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
721 }
722
723 INIT_WORK(&new->dim.work, ionic_dim_work);
724 new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
725 if (lif->doorbell_wa)
726 INIT_WORK(&new->doorbell_napi_work, ionic_doorbell_napi_work);
727
728 *qcq = new;
729
730 return 0;
731
732 err_out_free_cq:
733 dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
734 err_out_free_q:
735 if (new->cmb_q_base) {
736 iounmap(new->cmb_q_base);
737 ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
738 }
739 dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
740 err_out_free_irq:
741 if (flags & IONIC_QCQ_F_INTR) {
742 devm_free_irq(dev, new->intr.vector, &new->napi);
743 ionic_intr_free(lif->ionic, new->intr.index);
744 }
745 err_out_free_page_pool:
746 page_pool_destroy(new->q.page_pool);
747 err_out_free_q_info:
748 vfree(new->q.info);
749 err_out_free_qcq:
750 devm_kfree(dev, new);
751 err_out:
752 dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
753 return err;
754 }
755
ionic_qcqs_alloc(struct ionic_lif * lif)756 static int ionic_qcqs_alloc(struct ionic_lif *lif)
757 {
758 struct device *dev = lif->ionic->dev;
759 unsigned int flags;
760 int err;
761
762 flags = IONIC_QCQ_F_INTR;
763 err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
764 IONIC_ADMINQ_LENGTH,
765 sizeof(struct ionic_admin_cmd),
766 sizeof(struct ionic_admin_comp),
767 0,
768 sizeof(struct ionic_admin_desc_info),
769 lif->kern_pid, NULL, &lif->adminqcq);
770 if (err)
771 return err;
772 ionic_debugfs_add_qcq(lif, lif->adminqcq);
773
774 if (lif->ionic->nnqs_per_lif) {
775 flags = IONIC_QCQ_F_NOTIFYQ;
776 err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
777 flags, IONIC_NOTIFYQ_LENGTH,
778 sizeof(struct ionic_notifyq_cmd),
779 sizeof(union ionic_notifyq_comp),
780 0,
781 sizeof(struct ionic_admin_desc_info),
782 lif->kern_pid, NULL, &lif->notifyqcq);
783 if (err)
784 goto err_out;
785 ionic_debugfs_add_qcq(lif, lif->notifyqcq);
786
787 /* Let the notifyq ride on the adminq interrupt */
788 ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
789 }
790
791 err = -ENOMEM;
792 lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
793 sizeof(*lif->txqcqs), GFP_KERNEL);
794 if (!lif->txqcqs)
795 goto err_out;
796 lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
797 sizeof(*lif->rxqcqs), GFP_KERNEL);
798 if (!lif->rxqcqs)
799 goto err_out;
800
801 lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
802 sizeof(*lif->txqstats), GFP_KERNEL);
803 if (!lif->txqstats)
804 goto err_out;
805 lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
806 sizeof(*lif->rxqstats), GFP_KERNEL);
807 if (!lif->rxqstats)
808 goto err_out;
809
810 return 0;
811
812 err_out:
813 ionic_qcqs_free(lif);
814 return err;
815 }
816
ionic_qcq_sanitize(struct ionic_qcq * qcq)817 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
818 {
819 qcq->q.tail_idx = 0;
820 qcq->q.head_idx = 0;
821 qcq->cq.tail_idx = 0;
822 qcq->cq.done_color = 1;
823 memset(qcq->q_base, 0, qcq->q_size);
824 if (qcq->cmb_q_base)
825 memset_io(qcq->cmb_q_base, 0, qcq->cmb_q_size);
826 memset(qcq->cq_base, 0, qcq->cq_size);
827 memset(qcq->sg_base, 0, qcq->sg_size);
828 }
829
ionic_lif_txq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)830 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
831 {
832 struct device *dev = lif->ionic->dev;
833 struct ionic_queue *q = &qcq->q;
834 struct ionic_cq *cq = &qcq->cq;
835 struct ionic_admin_ctx ctx = {
836 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
837 .cmd.q_init = {
838 .opcode = IONIC_CMD_Q_INIT,
839 .lif_index = cpu_to_le16(lif->index),
840 .type = q->type,
841 .ver = lif->qtype_info[q->type].version,
842 .index = cpu_to_le32(q->index),
843 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
844 IONIC_QINIT_F_SG),
845 .intr_index = cpu_to_le16(qcq->intr.index),
846 .pid = cpu_to_le16(q->pid),
847 .ring_size = ilog2(q->num_descs),
848 .ring_base = cpu_to_le64(q->base_pa),
849 .cq_ring_base = cpu_to_le64(cq->base_pa),
850 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
851 .features = cpu_to_le64(q->features),
852 },
853 };
854 int err;
855
856 if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
857 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
858 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
859 }
860
861 dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
862 dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
863 dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
864 dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
865 dev_dbg(dev, "txq_init.cq_ring_base 0x%llx\n", ctx.cmd.q_init.cq_ring_base);
866 dev_dbg(dev, "txq_init.sg_ring_base 0x%llx\n", ctx.cmd.q_init.sg_ring_base);
867 dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
868 dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
869 dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
870
871 ionic_qcq_sanitize(qcq);
872
873 err = ionic_adminq_post_wait(lif, &ctx);
874 if (err)
875 return err;
876
877 q->hw_type = ctx.comp.q_init.hw_type;
878 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
879 q->dbval = IONIC_DBELL_QID(q->hw_index);
880
881 dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
882 dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
883
884 q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
885 q->dbell_jiffies = jiffies;
886
887 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
888 netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
889
890 qcq->flags |= IONIC_QCQ_F_INITED;
891
892 return 0;
893 }
894
ionic_lif_rxq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)895 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
896 {
897 struct device *dev = lif->ionic->dev;
898 struct ionic_queue *q = &qcq->q;
899 struct ionic_cq *cq = &qcq->cq;
900 struct ionic_admin_ctx ctx = {
901 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
902 .cmd.q_init = {
903 .opcode = IONIC_CMD_Q_INIT,
904 .lif_index = cpu_to_le16(lif->index),
905 .type = q->type,
906 .ver = lif->qtype_info[q->type].version,
907 .index = cpu_to_le32(q->index),
908 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ),
909 .intr_index = cpu_to_le16(cq->bound_intr->index),
910 .pid = cpu_to_le16(q->pid),
911 .ring_size = ilog2(q->num_descs),
912 .ring_base = cpu_to_le64(q->base_pa),
913 .cq_ring_base = cpu_to_le64(cq->base_pa),
914 .sg_ring_base = cpu_to_le64(q->sg_base_pa),
915 .features = cpu_to_le64(q->features),
916 },
917 };
918 int err;
919
920 q->partner = &lif->txqcqs[q->index]->q;
921 q->partner->partner = q;
922
923 if (!lif->xdp_prog ||
924 (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags))
925 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
926
927 if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
928 ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
929 ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
930 }
931
932 dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
933 dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
934 dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
935 dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
936 dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
937 dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
938 dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
939
940 ionic_qcq_sanitize(qcq);
941
942 err = ionic_adminq_post_wait(lif, &ctx);
943 if (err)
944 return err;
945
946 q->hw_type = ctx.comp.q_init.hw_type;
947 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
948 q->dbval = IONIC_DBELL_QID(q->hw_index);
949
950 dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
951 dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
952
953 q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
954 q->dbell_jiffies = jiffies;
955
956 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
957 netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
958 else
959 netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
960 err = ionic_register_rxq_info(q, qcq->napi.napi_id);
961 if (err) {
962 netif_napi_del(&qcq->napi);
963 return err;
964 }
965
966 qcq->flags |= IONIC_QCQ_F_INITED;
967
968 return 0;
969 }
970
ionic_lif_create_hwstamp_txq(struct ionic_lif * lif)971 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
972 {
973 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
974 unsigned int txq_i, flags;
975 struct ionic_qcq *txq;
976 u64 features;
977 int err;
978
979 if (lif->hwstamp_txq)
980 return 0;
981
982 features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
983
984 num_desc = IONIC_MIN_TXRX_DESC;
985 desc_sz = sizeof(struct ionic_txq_desc);
986 comp_sz = 2 * sizeof(struct ionic_txq_comp);
987
988 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
989 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
990 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
991 else
992 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
993
994 txq_i = lif->ionic->ntxqs_per_lif;
995 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
996
997 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
998 num_desc, desc_sz, comp_sz, sg_desc_sz,
999 sizeof(struct ionic_tx_desc_info),
1000 lif->kern_pid, NULL, &txq);
1001 if (err)
1002 goto err_qcq_alloc;
1003
1004 txq->q.features = features;
1005
1006 ionic_link_qcq_interrupts(lif->adminqcq, txq);
1007 ionic_debugfs_add_qcq(lif, txq);
1008
1009 lif->hwstamp_txq = txq;
1010
1011 if (netif_running(lif->netdev)) {
1012 err = ionic_lif_txq_init(lif, txq);
1013 if (err)
1014 goto err_qcq_init;
1015
1016 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1017 err = ionic_qcq_enable(txq);
1018 if (err)
1019 goto err_qcq_enable;
1020 }
1021 }
1022
1023 return 0;
1024
1025 err_qcq_enable:
1026 ionic_lif_qcq_deinit(lif, txq);
1027 err_qcq_init:
1028 lif->hwstamp_txq = NULL;
1029 ionic_debugfs_del_qcq(txq);
1030 ionic_qcq_free(lif, txq);
1031 devm_kfree(lif->ionic->dev, txq);
1032 err_qcq_alloc:
1033 return err;
1034 }
1035
ionic_lif_create_hwstamp_rxq(struct ionic_lif * lif)1036 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
1037 {
1038 unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
1039 unsigned int rxq_i, flags;
1040 struct ionic_qcq *rxq;
1041 u64 features;
1042 int err;
1043
1044 if (lif->hwstamp_rxq)
1045 return 0;
1046
1047 features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1048
1049 num_desc = IONIC_MIN_TXRX_DESC;
1050 desc_sz = sizeof(struct ionic_rxq_desc);
1051 comp_sz = 2 * sizeof(struct ionic_rxq_comp);
1052 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
1053
1054 rxq_i = lif->ionic->nrxqs_per_lif;
1055 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
1056
1057 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
1058 num_desc, desc_sz, comp_sz, sg_desc_sz,
1059 sizeof(struct ionic_rx_desc_info),
1060 lif->kern_pid, NULL, &rxq);
1061 if (err)
1062 goto err_qcq_alloc;
1063
1064 rxq->q.features = features;
1065
1066 ionic_link_qcq_interrupts(lif->adminqcq, rxq);
1067 ionic_debugfs_add_qcq(lif, rxq);
1068
1069 lif->hwstamp_rxq = rxq;
1070
1071 if (netif_running(lif->netdev)) {
1072 err = ionic_lif_rxq_init(lif, rxq);
1073 if (err)
1074 goto err_qcq_init;
1075
1076 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1077 ionic_rx_fill(&rxq->q, NULL);
1078 err = ionic_qcq_enable(rxq);
1079 if (err)
1080 goto err_qcq_enable;
1081 }
1082 }
1083
1084 return 0;
1085
1086 err_qcq_enable:
1087 ionic_lif_qcq_deinit(lif, rxq);
1088 err_qcq_init:
1089 lif->hwstamp_rxq = NULL;
1090 ionic_debugfs_del_qcq(rxq);
1091 ionic_qcq_free(lif, rxq);
1092 devm_kfree(lif->ionic->dev, rxq);
1093 err_qcq_alloc:
1094 return err;
1095 }
1096
ionic_lif_config_hwstamp_rxq_all(struct ionic_lif * lif,bool rx_all)1097 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
1098 {
1099 struct ionic_queue_params qparam;
1100
1101 ionic_init_queue_params(lif, &qparam);
1102
1103 if (rx_all)
1104 qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1105 else
1106 qparam.rxq_features = 0;
1107
1108 /* if we're not running, just set the values and return */
1109 if (!netif_running(lif->netdev)) {
1110 lif->rxq_features = qparam.rxq_features;
1111 return 0;
1112 }
1113
1114 return ionic_reconfigure_queues(lif, &qparam);
1115 }
1116
ionic_lif_set_hwstamp_txmode(struct ionic_lif * lif,u16 txstamp_mode)1117 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1118 {
1119 struct ionic_admin_ctx ctx = {
1120 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1121 .cmd.lif_setattr = {
1122 .opcode = IONIC_CMD_LIF_SETATTR,
1123 .index = cpu_to_le16(lif->index),
1124 .attr = IONIC_LIF_ATTR_TXSTAMP,
1125 .txstamp_mode = cpu_to_le16(txstamp_mode),
1126 },
1127 };
1128
1129 return ionic_adminq_post_wait(lif, &ctx);
1130 }
1131
ionic_lif_del_hwstamp_rxfilt(struct ionic_lif * lif)1132 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1133 {
1134 struct ionic_admin_ctx ctx = {
1135 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1136 .cmd.rx_filter_del = {
1137 .opcode = IONIC_CMD_RX_FILTER_DEL,
1138 .lif_index = cpu_to_le16(lif->index),
1139 },
1140 };
1141 struct ionic_rx_filter *f;
1142 u32 filter_id;
1143 int err;
1144
1145 spin_lock_bh(&lif->rx_filters.lock);
1146
1147 f = ionic_rx_filter_rxsteer(lif);
1148 if (!f) {
1149 spin_unlock_bh(&lif->rx_filters.lock);
1150 return;
1151 }
1152
1153 filter_id = f->filter_id;
1154 ionic_rx_filter_free(lif, f);
1155
1156 spin_unlock_bh(&lif->rx_filters.lock);
1157
1158 netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1159
1160 ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1161
1162 err = ionic_adminq_post_wait(lif, &ctx);
1163 if (err && err != -EEXIST)
1164 netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1165 }
1166
ionic_lif_add_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1167 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1168 {
1169 struct ionic_admin_ctx ctx = {
1170 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1171 .cmd.rx_filter_add = {
1172 .opcode = IONIC_CMD_RX_FILTER_ADD,
1173 .lif_index = cpu_to_le16(lif->index),
1174 .match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1175 .pkt_class = cpu_to_le64(pkt_class),
1176 },
1177 };
1178 u8 qtype;
1179 u32 qid;
1180 int err;
1181
1182 if (!lif->hwstamp_rxq)
1183 return -EINVAL;
1184
1185 qtype = lif->hwstamp_rxq->q.type;
1186 ctx.cmd.rx_filter_add.qtype = qtype;
1187
1188 qid = lif->hwstamp_rxq->q.index;
1189 ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1190
1191 netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1192 err = ionic_adminq_post_wait(lif, &ctx);
1193 if (err && err != -EEXIST)
1194 return err;
1195
1196 spin_lock_bh(&lif->rx_filters.lock);
1197 err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
1198 spin_unlock_bh(&lif->rx_filters.lock);
1199
1200 return err;
1201 }
1202
ionic_lif_set_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1203 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1204 {
1205 ionic_lif_del_hwstamp_rxfilt(lif);
1206
1207 if (!pkt_class)
1208 return 0;
1209
1210 return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1211 }
1212
ionic_adminq_napi(struct napi_struct * napi,int budget)1213 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1214 {
1215 struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1216 struct ionic_lif *lif = napi_to_cq(napi)->lif;
1217 struct ionic_dev *idev = &lif->ionic->idev;
1218 unsigned long irqflags;
1219 unsigned int flags = 0;
1220 int rx_work = 0;
1221 int tx_work = 0;
1222 int n_work = 0;
1223 int a_work = 0;
1224 int work_done;
1225 int credits;
1226
1227 if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1228 n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1229 ionic_notifyq_service, NULL, NULL);
1230
1231 spin_lock_irqsave(&lif->adminq_lock, irqflags);
1232 if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1233 a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1234 ionic_adminq_service, NULL, NULL);
1235
1236 spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1237
1238 if (lif->hwstamp_rxq)
1239 rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1240 ionic_rx_service, NULL, NULL);
1241
1242 if (lif->hwstamp_txq)
1243 tx_work = ionic_tx_cq_service(&lif->hwstamp_txq->cq, budget, !!budget);
1244
1245 work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1246 if (work_done < budget && napi_complete_done(napi, work_done)) {
1247 flags |= IONIC_INTR_CRED_UNMASK;
1248 intr->rearm_count++;
1249 }
1250
1251 if (work_done || flags) {
1252 flags |= IONIC_INTR_CRED_RESET_COALESCE;
1253 credits = n_work + a_work + rx_work + tx_work;
1254 ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1255 }
1256
1257 if (lif->doorbell_wa) {
1258 if (!a_work)
1259 ionic_adminq_poke_doorbell(&lif->adminqcq->q);
1260 if (lif->hwstamp_rxq && !rx_work)
1261 ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q);
1262 if (lif->hwstamp_txq && !tx_work)
1263 ionic_txq_poke_doorbell(&lif->hwstamp_txq->q);
1264 }
1265
1266 return work_done;
1267 }
1268
ionic_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * ns)1269 void ionic_get_stats64(struct net_device *netdev,
1270 struct rtnl_link_stats64 *ns)
1271 {
1272 struct ionic_lif *lif = netdev_priv(netdev);
1273 struct ionic_lif_stats *ls;
1274
1275 memset(ns, 0, sizeof(*ns));
1276 ls = &lif->info->stats;
1277
1278 ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1279 le64_to_cpu(ls->rx_mcast_packets) +
1280 le64_to_cpu(ls->rx_bcast_packets);
1281
1282 ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1283 le64_to_cpu(ls->tx_mcast_packets) +
1284 le64_to_cpu(ls->tx_bcast_packets);
1285
1286 ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1287 le64_to_cpu(ls->rx_mcast_bytes) +
1288 le64_to_cpu(ls->rx_bcast_bytes);
1289
1290 ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1291 le64_to_cpu(ls->tx_mcast_bytes) +
1292 le64_to_cpu(ls->tx_bcast_bytes);
1293
1294 ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1295 le64_to_cpu(ls->rx_mcast_drop_packets) +
1296 le64_to_cpu(ls->rx_bcast_drop_packets);
1297
1298 ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1299 le64_to_cpu(ls->tx_mcast_drop_packets) +
1300 le64_to_cpu(ls->tx_bcast_drop_packets);
1301
1302 ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1303
1304 ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1305
1306 ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1307 le64_to_cpu(ls->rx_queue_disabled) +
1308 le64_to_cpu(ls->rx_desc_fetch_error) +
1309 le64_to_cpu(ls->rx_desc_data_error);
1310
1311 ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1312 le64_to_cpu(ls->tx_queue_disabled) +
1313 le64_to_cpu(ls->tx_desc_fetch_error) +
1314 le64_to_cpu(ls->tx_desc_data_error);
1315
1316 ns->rx_errors = ns->rx_over_errors +
1317 ns->rx_missed_errors;
1318
1319 ns->tx_errors = ns->tx_aborted_errors;
1320 }
1321
ionic_addr_add(struct net_device * netdev,const u8 * addr)1322 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1323 {
1324 return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
1325 }
1326
ionic_addr_del(struct net_device * netdev,const u8 * addr)1327 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1328 {
1329 /* Don't delete our own address from the uc list */
1330 if (ether_addr_equal(addr, netdev->dev_addr))
1331 return 0;
1332
1333 return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
1334 }
1335
ionic_lif_rx_mode(struct ionic_lif * lif)1336 void ionic_lif_rx_mode(struct ionic_lif *lif)
1337 {
1338 struct net_device *netdev = lif->netdev;
1339 unsigned int nfilters;
1340 unsigned int nd_flags;
1341 char buf[128];
1342 u16 rx_mode;
1343 int i;
1344 #define REMAIN(__x) (sizeof(buf) - (__x))
1345
1346 mutex_lock(&lif->config_lock);
1347
1348 /* grab the flags once for local use */
1349 nd_flags = netdev->flags;
1350
1351 rx_mode = IONIC_RX_MODE_F_UNICAST;
1352 rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1353 rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1354 rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1355 rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1356
1357 /* sync the filters */
1358 ionic_rx_filter_sync(lif);
1359
1360 /* check for overflow state
1361 * if so, we track that we overflowed and enable NIC PROMISC
1362 * else if the overflow is set and not needed
1363 * we remove our overflow flag and check the netdev flags
1364 * to see if we can disable NIC PROMISC
1365 */
1366 nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1367
1368 if (((lif->nucast + lif->nmcast) >= nfilters) ||
1369 (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
1370 rx_mode |= IONIC_RX_MODE_F_PROMISC;
1371 rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1372 } else {
1373 if (!(nd_flags & IFF_PROMISC))
1374 rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1375 if (!(nd_flags & IFF_ALLMULTI))
1376 rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1377 }
1378
1379 i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1380 lif->rx_mode, rx_mode);
1381 if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1382 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1383 if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1384 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1385 if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1386 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1387 if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1388 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1389 if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1390 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1391 if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1392 i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1393 netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1394
1395 if (lif->rx_mode != rx_mode) {
1396 struct ionic_admin_ctx ctx = {
1397 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1398 .cmd.rx_mode_set = {
1399 .opcode = IONIC_CMD_RX_MODE_SET,
1400 .lif_index = cpu_to_le16(lif->index),
1401 },
1402 };
1403 int err;
1404
1405 ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1406 err = ionic_adminq_post_wait(lif, &ctx);
1407 if (err)
1408 netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1409 rx_mode, err);
1410 else
1411 lif->rx_mode = rx_mode;
1412 }
1413
1414 mutex_unlock(&lif->config_lock);
1415 }
1416
ionic_ndo_set_rx_mode(struct net_device * netdev)1417 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1418 {
1419 struct ionic_lif *lif = netdev_priv(netdev);
1420 struct ionic_deferred_work *work;
1421
1422 /* Sync the kernel filter list with the driver filter list */
1423 __dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1424 __dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1425
1426 /* Shove off the rest of the rxmode work to the work task
1427 * which will include syncing the filters to the firmware.
1428 */
1429 work = kzalloc(sizeof(*work), GFP_ATOMIC);
1430 if (!work) {
1431 netdev_err(lif->netdev, "rxmode change dropped\n");
1432 return;
1433 }
1434 work->type = IONIC_DW_TYPE_RX_MODE;
1435 netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1436 ionic_lif_deferred_enqueue(lif, work);
1437 }
1438
ionic_netdev_features_to_nic(netdev_features_t features)1439 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1440 {
1441 u64 wanted = 0;
1442
1443 if (features & NETIF_F_HW_VLAN_CTAG_TX)
1444 wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1445 if (features & NETIF_F_HW_VLAN_CTAG_RX)
1446 wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1447 if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1448 wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1449 if (features & NETIF_F_RXHASH)
1450 wanted |= IONIC_ETH_HW_RX_HASH;
1451 if (features & NETIF_F_RXCSUM)
1452 wanted |= IONIC_ETH_HW_RX_CSUM;
1453 if (features & NETIF_F_SG)
1454 wanted |= IONIC_ETH_HW_TX_SG;
1455 if (features & NETIF_F_HW_CSUM)
1456 wanted |= IONIC_ETH_HW_TX_CSUM;
1457 if (features & NETIF_F_TSO)
1458 wanted |= IONIC_ETH_HW_TSO;
1459 if (features & NETIF_F_TSO6)
1460 wanted |= IONIC_ETH_HW_TSO_IPV6;
1461 if (features & NETIF_F_TSO_ECN)
1462 wanted |= IONIC_ETH_HW_TSO_ECN;
1463 if (features & NETIF_F_GSO_GRE)
1464 wanted |= IONIC_ETH_HW_TSO_GRE;
1465 if (features & NETIF_F_GSO_GRE_CSUM)
1466 wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1467 if (features & NETIF_F_GSO_IPXIP4)
1468 wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1469 if (features & NETIF_F_GSO_IPXIP6)
1470 wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1471 if (features & NETIF_F_GSO_UDP_TUNNEL)
1472 wanted |= IONIC_ETH_HW_TSO_UDP;
1473 if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1474 wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1475
1476 return cpu_to_le64(wanted);
1477 }
1478
ionic_set_nic_features(struct ionic_lif * lif,netdev_features_t features)1479 static int ionic_set_nic_features(struct ionic_lif *lif,
1480 netdev_features_t features)
1481 {
1482 struct device *dev = lif->ionic->dev;
1483 struct ionic_admin_ctx ctx = {
1484 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1485 .cmd.lif_setattr = {
1486 .opcode = IONIC_CMD_LIF_SETATTR,
1487 .index = cpu_to_le16(lif->index),
1488 .attr = IONIC_LIF_ATTR_FEATURES,
1489 },
1490 };
1491 u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1492 IONIC_ETH_HW_VLAN_RX_STRIP |
1493 IONIC_ETH_HW_VLAN_RX_FILTER;
1494 u64 old_hw_features;
1495 int err;
1496
1497 ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1498
1499 if (lif->phc)
1500 ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1501
1502 err = ionic_adminq_post_wait(lif, &ctx);
1503 if (err)
1504 return err;
1505
1506 old_hw_features = lif->hw_features;
1507 lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1508 ctx.comp.lif_setattr.features);
1509
1510 if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1511 ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1512
1513 if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&
1514 !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1515 dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1516
1517 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1518 dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1519 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1520 dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1521 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1522 dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1523 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1524 dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1525 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1526 dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1527 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1528 dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1529 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1530 dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1531 if (lif->hw_features & IONIC_ETH_HW_TSO)
1532 dev_dbg(dev, "feature ETH_HW_TSO\n");
1533 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1534 dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1535 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1536 dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1537 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1538 dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1539 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1540 dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1541 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1542 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1543 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1544 dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1545 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1546 dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1547 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1548 dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1549 if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1550 dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1551
1552 return 0;
1553 }
1554
ionic_init_nic_features(struct ionic_lif * lif)1555 static int ionic_init_nic_features(struct ionic_lif *lif)
1556 {
1557 struct net_device *netdev = lif->netdev;
1558 netdev_features_t features;
1559 int err;
1560
1561 /* set up what we expect to support by default */
1562 features = NETIF_F_HW_VLAN_CTAG_TX |
1563 NETIF_F_HW_VLAN_CTAG_RX |
1564 NETIF_F_HW_VLAN_CTAG_FILTER |
1565 NETIF_F_SG |
1566 NETIF_F_HW_CSUM |
1567 NETIF_F_RXCSUM |
1568 NETIF_F_TSO |
1569 NETIF_F_TSO6 |
1570 NETIF_F_TSO_ECN |
1571 NETIF_F_GSO_GRE |
1572 NETIF_F_GSO_GRE_CSUM |
1573 NETIF_F_GSO_IPXIP4 |
1574 NETIF_F_GSO_IPXIP6 |
1575 NETIF_F_GSO_UDP_TUNNEL |
1576 NETIF_F_GSO_UDP_TUNNEL_CSUM;
1577
1578 if (lif->nxqs > 1)
1579 features |= NETIF_F_RXHASH;
1580
1581 err = ionic_set_nic_features(lif, features);
1582 if (err)
1583 return err;
1584
1585 /* tell the netdev what we actually can support */
1586 netdev->features |= NETIF_F_HIGHDMA;
1587
1588 if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1589 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1590 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1591 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1592 if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1593 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1594 if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1595 netdev->hw_features |= NETIF_F_RXHASH;
1596 if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1597 netdev->hw_features |= NETIF_F_SG;
1598
1599 if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1600 netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1601 if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1602 netdev->hw_enc_features |= NETIF_F_RXCSUM;
1603 if (lif->hw_features & IONIC_ETH_HW_TSO)
1604 netdev->hw_enc_features |= NETIF_F_TSO;
1605 if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1606 netdev->hw_enc_features |= NETIF_F_TSO6;
1607 if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1608 netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1609 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1610 netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1611 if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1612 netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1613 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1614 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1615 if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1616 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1617 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1618 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1619 if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1620 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1621
1622 netdev->hw_features |= netdev->hw_enc_features;
1623 netdev->features |= netdev->hw_features;
1624 netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1625
1626 netdev->priv_flags |= IFF_UNICAST_FLT |
1627 IFF_LIVE_ADDR_CHANGE;
1628
1629 netdev->xdp_features = NETDEV_XDP_ACT_BASIC |
1630 NETDEV_XDP_ACT_REDIRECT |
1631 NETDEV_XDP_ACT_RX_SG |
1632 NETDEV_XDP_ACT_NDO_XMIT |
1633 NETDEV_XDP_ACT_NDO_XMIT_SG;
1634
1635 return 0;
1636 }
1637
ionic_set_features(struct net_device * netdev,netdev_features_t features)1638 static int ionic_set_features(struct net_device *netdev,
1639 netdev_features_t features)
1640 {
1641 struct ionic_lif *lif = netdev_priv(netdev);
1642 int err;
1643
1644 netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1645 __func__, (u64)lif->netdev->features, (u64)features);
1646
1647 err = ionic_set_nic_features(lif, features);
1648
1649 return err;
1650 }
1651
ionic_set_attr_mac(struct ionic_lif * lif,u8 * mac)1652 static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)
1653 {
1654 struct ionic_admin_ctx ctx = {
1655 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1656 .cmd.lif_setattr = {
1657 .opcode = IONIC_CMD_LIF_SETATTR,
1658 .index = cpu_to_le16(lif->index),
1659 .attr = IONIC_LIF_ATTR_MAC,
1660 },
1661 };
1662
1663 ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);
1664 return ionic_adminq_post_wait(lif, &ctx);
1665 }
1666
ionic_get_attr_mac(struct ionic_lif * lif,u8 * mac_addr)1667 static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)
1668 {
1669 struct ionic_admin_ctx ctx = {
1670 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1671 .cmd.lif_getattr = {
1672 .opcode = IONIC_CMD_LIF_GETATTR,
1673 .index = cpu_to_le16(lif->index),
1674 .attr = IONIC_LIF_ATTR_MAC,
1675 },
1676 };
1677 int err;
1678
1679 err = ionic_adminq_post_wait(lif, &ctx);
1680 if (err)
1681 return err;
1682
1683 ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);
1684 return 0;
1685 }
1686
ionic_program_mac(struct ionic_lif * lif,u8 * mac)1687 static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)
1688 {
1689 u8 get_mac[ETH_ALEN];
1690 int err;
1691
1692 err = ionic_set_attr_mac(lif, mac);
1693 if (err)
1694 return err;
1695
1696 err = ionic_get_attr_mac(lif, get_mac);
1697 if (err)
1698 return err;
1699
1700 /* To deal with older firmware that silently ignores the set attr mac:
1701 * doesn't actually change the mac and doesn't return an error, so we
1702 * do the get attr to verify whether or not the set actually happened
1703 */
1704 if (!ether_addr_equal(get_mac, mac))
1705 return 1;
1706
1707 return 0;
1708 }
1709
ionic_set_mac_address(struct net_device * netdev,void * sa)1710 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1711 {
1712 struct ionic_lif *lif = netdev_priv(netdev);
1713 struct sockaddr *addr = sa;
1714 u8 *mac;
1715 int err;
1716
1717 mac = (u8 *)addr->sa_data;
1718 if (ether_addr_equal(netdev->dev_addr, mac))
1719 return 0;
1720
1721 err = ionic_program_mac(lif, mac);
1722 if (err < 0)
1723 return err;
1724
1725 if (err > 0)
1726 netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
1727 __func__);
1728
1729 err = eth_prepare_mac_addr_change(netdev, addr);
1730 if (err)
1731 return err;
1732
1733 if (!is_zero_ether_addr(netdev->dev_addr)) {
1734 netdev_info(netdev, "deleting mac addr %pM\n",
1735 netdev->dev_addr);
1736 ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
1737 }
1738
1739 eth_commit_mac_addr_change(netdev, addr);
1740 netdev_info(netdev, "updating mac addr %pM\n", mac);
1741
1742 return ionic_lif_addr_add(netdev_priv(netdev), mac);
1743 }
1744
ionic_stop_queues_reconfig(struct ionic_lif * lif)1745 void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1746 {
1747 /* Stop and clean the queues before reconfiguration */
1748 netif_device_detach(lif->netdev);
1749 ionic_stop_queues(lif);
1750 ionic_txrx_deinit(lif);
1751 }
1752
ionic_start_queues_reconfig(struct ionic_lif * lif)1753 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1754 {
1755 int err;
1756
1757 /* Re-init the queues after reconfiguration */
1758
1759 /* The only way txrx_init can fail here is if communication
1760 * with FW is suddenly broken. There's not much we can do
1761 * at this point - error messages have already been printed,
1762 * so we can continue on and the user can eventually do a
1763 * DOWN and UP to try to reset and clear the issue.
1764 */
1765 err = ionic_txrx_init(lif);
1766 ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1767 netif_device_attach(lif->netdev);
1768
1769 return err;
1770 }
1771
ionic_xdp_is_valid_mtu(struct ionic_lif * lif,u32 mtu,struct bpf_prog * xdp_prog)1772 static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,
1773 struct bpf_prog *xdp_prog)
1774 {
1775 if (!xdp_prog)
1776 return true;
1777
1778 if (mtu <= IONIC_XDP_MAX_LINEAR_MTU)
1779 return true;
1780
1781 if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags)
1782 return true;
1783
1784 return false;
1785 }
1786
ionic_change_mtu(struct net_device * netdev,int new_mtu)1787 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1788 {
1789 struct ionic_lif *lif = netdev_priv(netdev);
1790 struct ionic_admin_ctx ctx = {
1791 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1792 .cmd.lif_setattr = {
1793 .opcode = IONIC_CMD_LIF_SETATTR,
1794 .index = cpu_to_le16(lif->index),
1795 .attr = IONIC_LIF_ATTR_MTU,
1796 .mtu = cpu_to_le32(new_mtu),
1797 },
1798 };
1799 struct bpf_prog *xdp_prog;
1800 int err;
1801
1802 xdp_prog = READ_ONCE(lif->xdp_prog);
1803 if (!ionic_xdp_is_valid_mtu(lif, new_mtu, xdp_prog))
1804 return -EINVAL;
1805
1806 err = ionic_adminq_post_wait(lif, &ctx);
1807 if (err)
1808 return err;
1809
1810 /* if we're not running, nothing more to do */
1811 if (!netif_running(netdev)) {
1812 WRITE_ONCE(netdev->mtu, new_mtu);
1813 return 0;
1814 }
1815
1816 mutex_lock(&lif->queue_lock);
1817 ionic_stop_queues_reconfig(lif);
1818 WRITE_ONCE(netdev->mtu, new_mtu);
1819 err = ionic_start_queues_reconfig(lif);
1820 mutex_unlock(&lif->queue_lock);
1821
1822 return err;
1823 }
1824
ionic_tx_timeout_work(struct work_struct * ws)1825 static void ionic_tx_timeout_work(struct work_struct *ws)
1826 {
1827 struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1828 int err;
1829
1830 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1831 return;
1832
1833 /* if we were stopped before this scheduled job was launched,
1834 * don't bother the queues as they are already stopped.
1835 */
1836 if (!netif_running(lif->netdev))
1837 return;
1838
1839 mutex_lock(&lif->queue_lock);
1840 ionic_stop_queues_reconfig(lif);
1841 err = ionic_start_queues_reconfig(lif);
1842 mutex_unlock(&lif->queue_lock);
1843
1844 if (err)
1845 dev_err(lif->ionic->dev, "%s: Restarting queues failed\n", __func__);
1846 }
1847
ionic_tx_timeout(struct net_device * netdev,unsigned int txqueue)1848 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1849 {
1850 struct ionic_lif *lif = netdev_priv(netdev);
1851
1852 netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1853 schedule_work(&lif->tx_timeout_work);
1854 }
1855
ionic_vlan_rx_add_vid(struct net_device * netdev,__be16 proto,u16 vid)1856 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1857 u16 vid)
1858 {
1859 struct ionic_lif *lif = netdev_priv(netdev);
1860 int err;
1861
1862 err = ionic_lif_vlan_add(lif, vid);
1863 if (err)
1864 return err;
1865
1866 ionic_lif_rx_mode(lif);
1867
1868 return 0;
1869 }
1870
ionic_vlan_rx_kill_vid(struct net_device * netdev,__be16 proto,u16 vid)1871 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1872 u16 vid)
1873 {
1874 struct ionic_lif *lif = netdev_priv(netdev);
1875 int err;
1876
1877 err = ionic_lif_vlan_del(lif, vid);
1878 if (err)
1879 return err;
1880
1881 ionic_lif_rx_mode(lif);
1882
1883 return 0;
1884 }
1885
ionic_lif_rss_config(struct ionic_lif * lif,const u16 types,const u8 * key,const u32 * indir)1886 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1887 const u8 *key, const u32 *indir)
1888 {
1889 struct ionic_admin_ctx ctx = {
1890 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1891 .cmd.lif_setattr = {
1892 .opcode = IONIC_CMD_LIF_SETATTR,
1893 .attr = IONIC_LIF_ATTR_RSS,
1894 .rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1895 },
1896 };
1897 unsigned int i, tbl_sz;
1898
1899 if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1900 lif->rss_types = types;
1901 ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1902 }
1903
1904 if (key)
1905 memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1906
1907 if (indir) {
1908 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1909 for (i = 0; i < tbl_sz; i++)
1910 lif->rss_ind_tbl[i] = indir[i];
1911 }
1912
1913 memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1914 IONIC_RSS_HASH_KEY_SIZE);
1915
1916 return ionic_adminq_post_wait(lif, &ctx);
1917 }
1918
ionic_lif_rss_init(struct ionic_lif * lif)1919 static int ionic_lif_rss_init(struct ionic_lif *lif)
1920 {
1921 unsigned int tbl_sz;
1922 unsigned int i;
1923
1924 lif->rss_types = IONIC_RSS_TYPE_IPV4 |
1925 IONIC_RSS_TYPE_IPV4_TCP |
1926 IONIC_RSS_TYPE_IPV4_UDP |
1927 IONIC_RSS_TYPE_IPV6 |
1928 IONIC_RSS_TYPE_IPV6_TCP |
1929 IONIC_RSS_TYPE_IPV6_UDP;
1930
1931 /* Fill indirection table with 'default' values */
1932 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1933 for (i = 0; i < tbl_sz; i++)
1934 lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1935
1936 return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1937 }
1938
ionic_lif_rss_deinit(struct ionic_lif * lif)1939 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1940 {
1941 int tbl_sz;
1942
1943 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1944 memset(lif->rss_ind_tbl, 0, tbl_sz);
1945 memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1946
1947 ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1948 }
1949
ionic_lif_quiesce(struct ionic_lif * lif)1950 static void ionic_lif_quiesce(struct ionic_lif *lif)
1951 {
1952 struct ionic_admin_ctx ctx = {
1953 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1954 .cmd.lif_setattr = {
1955 .opcode = IONIC_CMD_LIF_SETATTR,
1956 .index = cpu_to_le16(lif->index),
1957 .attr = IONIC_LIF_ATTR_STATE,
1958 .state = IONIC_LIF_QUIESCE,
1959 },
1960 };
1961 int err;
1962
1963 err = ionic_adminq_post_wait(lif, &ctx);
1964 if (err)
1965 netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
1966 }
1967
ionic_txrx_disable(struct ionic_lif * lif)1968 static void ionic_txrx_disable(struct ionic_lif *lif)
1969 {
1970 unsigned int i;
1971 int err = 0;
1972
1973 if (lif->txqcqs) {
1974 for (i = 0; i < lif->nxqs; i++)
1975 err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
1976 }
1977
1978 if (lif->hwstamp_txq)
1979 err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
1980
1981 if (lif->rxqcqs) {
1982 for (i = 0; i < lif->nxqs; i++)
1983 err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
1984 }
1985
1986 if (lif->hwstamp_rxq)
1987 err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
1988
1989 ionic_lif_quiesce(lif);
1990 }
1991
ionic_txrx_deinit(struct ionic_lif * lif)1992 static void ionic_txrx_deinit(struct ionic_lif *lif)
1993 {
1994 unsigned int i;
1995
1996 if (lif->txqcqs) {
1997 for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1998 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1999 ionic_tx_flush(&lif->txqcqs[i]->cq);
2000 ionic_tx_empty(&lif->txqcqs[i]->q);
2001 }
2002 }
2003
2004 if (lif->rxqcqs) {
2005 for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
2006 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2007 ionic_rx_empty(&lif->rxqcqs[i]->q);
2008 }
2009 }
2010 lif->rx_mode = 0;
2011
2012 if (lif->hwstamp_txq) {
2013 ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
2014 ionic_tx_flush(&lif->hwstamp_txq->cq);
2015 ionic_tx_empty(&lif->hwstamp_txq->q);
2016 }
2017
2018 if (lif->hwstamp_rxq) {
2019 ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
2020 ionic_rx_empty(&lif->hwstamp_rxq->q);
2021 }
2022 }
2023
ionic_txrx_free(struct ionic_lif * lif)2024 void ionic_txrx_free(struct ionic_lif *lif)
2025 {
2026 unsigned int i;
2027
2028 if (lif->txqcqs) {
2029 for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
2030 ionic_qcq_free(lif, lif->txqcqs[i]);
2031 devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
2032 lif->txqcqs[i] = NULL;
2033 }
2034 }
2035
2036 if (lif->rxqcqs) {
2037 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2038 ionic_qcq_free(lif, lif->rxqcqs[i]);
2039 devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
2040 lif->rxqcqs[i] = NULL;
2041 }
2042 }
2043
2044 if (lif->hwstamp_txq) {
2045 ionic_qcq_free(lif, lif->hwstamp_txq);
2046 devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2047 lif->hwstamp_txq = NULL;
2048 }
2049
2050 if (lif->hwstamp_rxq) {
2051 ionic_qcq_free(lif, lif->hwstamp_rxq);
2052 devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2053 lif->hwstamp_rxq = NULL;
2054 }
2055 }
2056
ionic_txrx_alloc(struct ionic_lif * lif)2057 static int ionic_txrx_alloc(struct ionic_lif *lif)
2058 {
2059 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2060 unsigned int flags, i;
2061 int err = 0;
2062
2063 num_desc = lif->ntxq_descs;
2064 desc_sz = sizeof(struct ionic_txq_desc);
2065 comp_sz = sizeof(struct ionic_txq_comp);
2066
2067 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2068 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2069 sizeof(struct ionic_txq_sg_desc_v1))
2070 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2071 else
2072 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2073
2074 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2075
2076 if (test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state))
2077 flags |= IONIC_QCQ_F_CMB_RINGS;
2078
2079 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2080 flags |= IONIC_QCQ_F_INTR;
2081
2082 for (i = 0; i < lif->nxqs; i++) {
2083 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2084 num_desc, desc_sz, comp_sz, sg_desc_sz,
2085 sizeof(struct ionic_tx_desc_info),
2086 lif->kern_pid, NULL, &lif->txqcqs[i]);
2087 if (err)
2088 goto err_out;
2089
2090 if (flags & IONIC_QCQ_F_INTR) {
2091 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2092 lif->txqcqs[i]->intr.index,
2093 lif->tx_coalesce_hw);
2094 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2095 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2096 }
2097
2098 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2099 }
2100
2101 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2102
2103 if (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state))
2104 flags |= IONIC_QCQ_F_CMB_RINGS;
2105
2106 num_desc = lif->nrxq_descs;
2107 desc_sz = sizeof(struct ionic_rxq_desc);
2108 comp_sz = sizeof(struct ionic_rxq_comp);
2109 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2110
2111 if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2112 comp_sz *= 2;
2113
2114 for (i = 0; i < lif->nxqs; i++) {
2115 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2116 num_desc, desc_sz, comp_sz, sg_desc_sz,
2117 sizeof(struct ionic_rx_desc_info),
2118 lif->kern_pid, lif->xdp_prog,
2119 &lif->rxqcqs[i]);
2120 if (err)
2121 goto err_out;
2122
2123 lif->rxqcqs[i]->q.features = lif->rxq_features;
2124
2125 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2126 lif->rxqcqs[i]->intr.index,
2127 lif->rx_coalesce_hw);
2128 if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2129 lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2130
2131 if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2132 ionic_link_qcq_interrupts(lif->rxqcqs[i],
2133 lif->txqcqs[i]);
2134
2135 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2136 }
2137
2138 return 0;
2139
2140 err_out:
2141 ionic_txrx_free(lif);
2142
2143 return err;
2144 }
2145
ionic_txrx_init(struct ionic_lif * lif)2146 static int ionic_txrx_init(struct ionic_lif *lif)
2147 {
2148 unsigned int i;
2149 int err;
2150
2151 for (i = 0; i < lif->nxqs; i++) {
2152 err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2153 if (err)
2154 goto err_out;
2155
2156 err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2157 if (err) {
2158 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2159 goto err_out;
2160 }
2161 }
2162
2163 if (lif->netdev->features & NETIF_F_RXHASH)
2164 ionic_lif_rss_init(lif);
2165
2166 ionic_lif_rx_mode(lif);
2167
2168 return 0;
2169
2170 err_out:
2171 while (i--) {
2172 ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2173 ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2174 }
2175
2176 return err;
2177 }
2178
ionic_txrx_enable(struct ionic_lif * lif)2179 static int ionic_txrx_enable(struct ionic_lif *lif)
2180 {
2181 int derr = 0;
2182 int i, err;
2183
2184 ionic_xdp_rxqs_prog_update(lif);
2185
2186 for (i = 0; i < lif->nxqs; i++) {
2187 if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2188 dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2189 err = -ENXIO;
2190 goto err_out;
2191 }
2192
2193 ionic_rx_fill(&lif->rxqcqs[i]->q,
2194 READ_ONCE(lif->rxqcqs[i]->q.xdp_prog));
2195 err = ionic_qcq_enable(lif->rxqcqs[i]);
2196 if (err)
2197 goto err_out;
2198
2199 err = ionic_qcq_enable(lif->txqcqs[i]);
2200 if (err) {
2201 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
2202 goto err_out;
2203 }
2204 }
2205
2206 if (lif->hwstamp_rxq) {
2207 ionic_rx_fill(&lif->hwstamp_rxq->q, NULL);
2208 err = ionic_qcq_enable(lif->hwstamp_rxq);
2209 if (err)
2210 goto err_out_hwstamp_rx;
2211 }
2212
2213 if (lif->hwstamp_txq) {
2214 err = ionic_qcq_enable(lif->hwstamp_txq);
2215 if (err)
2216 goto err_out_hwstamp_tx;
2217 }
2218
2219 return 0;
2220
2221 err_out_hwstamp_tx:
2222 if (lif->hwstamp_rxq)
2223 derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
2224 err_out_hwstamp_rx:
2225 i = lif->nxqs;
2226 err_out:
2227 while (i--) {
2228 derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
2229 derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
2230 }
2231
2232 ionic_xdp_rxqs_prog_update(lif);
2233
2234 return err;
2235 }
2236
ionic_start_queues(struct ionic_lif * lif)2237 static int ionic_start_queues(struct ionic_lif *lif)
2238 {
2239 int err;
2240
2241 if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2242 return -EIO;
2243
2244 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2245 return -EBUSY;
2246
2247 if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2248 return 0;
2249
2250 err = ionic_txrx_enable(lif);
2251 if (err) {
2252 clear_bit(IONIC_LIF_F_UP, lif->state);
2253 return err;
2254 }
2255 netif_tx_wake_all_queues(lif->netdev);
2256
2257 return 0;
2258 }
2259
ionic_open(struct net_device * netdev)2260 static int ionic_open(struct net_device *netdev)
2261 {
2262 struct ionic_lif *lif = netdev_priv(netdev);
2263 int err;
2264
2265 /* If recovering from a broken state, clear the bit and we'll try again */
2266 if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2267 netdev_info(netdev, "clearing broken state\n");
2268
2269 mutex_lock(&lif->queue_lock);
2270
2271 err = ionic_txrx_alloc(lif);
2272 if (err)
2273 goto err_unlock;
2274
2275 err = ionic_txrx_init(lif);
2276 if (err)
2277 goto err_txrx_free;
2278
2279 err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2280 if (err)
2281 goto err_txrx_deinit;
2282
2283 err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2284 if (err)
2285 goto err_txrx_deinit;
2286
2287 /* don't start the queues until we have link */
2288 if (netif_carrier_ok(netdev)) {
2289 err = ionic_start_queues(lif);
2290 if (err)
2291 goto err_txrx_deinit;
2292 }
2293
2294 /* If hardware timestamping is enabled, but the queues were freed by
2295 * ionic_stop, those need to be reallocated and initialized, too.
2296 */
2297 ionic_lif_hwstamp_recreate_queues(lif);
2298
2299 mutex_unlock(&lif->queue_lock);
2300
2301 return 0;
2302
2303 err_txrx_deinit:
2304 ionic_txrx_deinit(lif);
2305 err_txrx_free:
2306 ionic_txrx_free(lif);
2307 err_unlock:
2308 mutex_unlock(&lif->queue_lock);
2309 return err;
2310 }
2311
ionic_stop_queues(struct ionic_lif * lif)2312 static void ionic_stop_queues(struct ionic_lif *lif)
2313 {
2314 if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2315 return;
2316
2317 netif_tx_disable(lif->netdev);
2318 ionic_txrx_disable(lif);
2319 }
2320
ionic_stop(struct net_device * netdev)2321 static int ionic_stop(struct net_device *netdev)
2322 {
2323 struct ionic_lif *lif = netdev_priv(netdev);
2324
2325 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2326 return 0;
2327
2328 mutex_lock(&lif->queue_lock);
2329 ionic_stop_queues(lif);
2330 ionic_txrx_deinit(lif);
2331 ionic_txrx_free(lif);
2332 mutex_unlock(&lif->queue_lock);
2333
2334 return 0;
2335 }
2336
ionic_eth_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2337 static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2338 {
2339 struct ionic_lif *lif = netdev_priv(netdev);
2340
2341 switch (cmd) {
2342 case SIOCSHWTSTAMP:
2343 return ionic_lif_hwstamp_set(lif, ifr);
2344 case SIOCGHWTSTAMP:
2345 return ionic_lif_hwstamp_get(lif, ifr);
2346 default:
2347 return -EOPNOTSUPP;
2348 }
2349 }
2350
ionic_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivf)2351 static int ionic_get_vf_config(struct net_device *netdev,
2352 int vf, struct ifla_vf_info *ivf)
2353 {
2354 struct ionic_lif *lif = netdev_priv(netdev);
2355 struct ionic *ionic = lif->ionic;
2356 int ret = 0;
2357
2358 if (!netif_device_present(netdev))
2359 return -EBUSY;
2360
2361 down_read(&ionic->vf_op_lock);
2362
2363 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2364 ret = -EINVAL;
2365 } else {
2366 struct ionic_vf *vfdata = &ionic->vfs[vf];
2367
2368 ivf->vf = vf;
2369 ivf->qos = 0;
2370 ivf->vlan = le16_to_cpu(vfdata->vlanid);
2371 ivf->spoofchk = vfdata->spoofchk;
2372 ivf->linkstate = vfdata->linkstate;
2373 ivf->max_tx_rate = le32_to_cpu(vfdata->maxrate);
2374 ivf->trusted = vfdata->trusted;
2375 ether_addr_copy(ivf->mac, vfdata->macaddr);
2376 }
2377
2378 up_read(&ionic->vf_op_lock);
2379 return ret;
2380 }
2381
ionic_get_vf_stats(struct net_device * netdev,int vf,struct ifla_vf_stats * vf_stats)2382 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2383 struct ifla_vf_stats *vf_stats)
2384 {
2385 struct ionic_lif *lif = netdev_priv(netdev);
2386 struct ionic *ionic = lif->ionic;
2387 struct ionic_lif_stats *vs;
2388 int ret = 0;
2389
2390 if (!netif_device_present(netdev))
2391 return -EBUSY;
2392
2393 down_read(&ionic->vf_op_lock);
2394
2395 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2396 ret = -EINVAL;
2397 } else {
2398 memset(vf_stats, 0, sizeof(*vf_stats));
2399 vs = &ionic->vfs[vf].stats;
2400
2401 vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2402 vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2403 vf_stats->rx_bytes = le64_to_cpu(vs->rx_ucast_bytes);
2404 vf_stats->tx_bytes = le64_to_cpu(vs->tx_ucast_bytes);
2405 vf_stats->broadcast = le64_to_cpu(vs->rx_bcast_packets);
2406 vf_stats->multicast = le64_to_cpu(vs->rx_mcast_packets);
2407 vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2408 le64_to_cpu(vs->rx_mcast_drop_packets) +
2409 le64_to_cpu(vs->rx_bcast_drop_packets);
2410 vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2411 le64_to_cpu(vs->tx_mcast_drop_packets) +
2412 le64_to_cpu(vs->tx_bcast_drop_packets);
2413 }
2414
2415 up_read(&ionic->vf_op_lock);
2416 return ret;
2417 }
2418
ionic_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2419 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2420 {
2421 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
2422 struct ionic_lif *lif = netdev_priv(netdev);
2423 struct ionic *ionic = lif->ionic;
2424 int ret;
2425
2426 if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2427 return -EINVAL;
2428
2429 if (!netif_device_present(netdev))
2430 return -EBUSY;
2431
2432 down_write(&ionic->vf_op_lock);
2433
2434 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2435 ret = -EINVAL;
2436 } else {
2437 ether_addr_copy(vfc.macaddr, mac);
2438 dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
2439 __func__, vf, vfc.macaddr);
2440
2441 ret = ionic_set_vf_config(ionic, vf, &vfc);
2442 if (!ret)
2443 ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2444 }
2445
2446 up_write(&ionic->vf_op_lock);
2447 return ret;
2448 }
2449
ionic_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2450 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2451 u8 qos, __be16 proto)
2452 {
2453 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
2454 struct ionic_lif *lif = netdev_priv(netdev);
2455 struct ionic *ionic = lif->ionic;
2456 int ret;
2457
2458 /* until someday when we support qos */
2459 if (qos)
2460 return -EINVAL;
2461
2462 if (vlan > 4095)
2463 return -EINVAL;
2464
2465 if (proto != htons(ETH_P_8021Q))
2466 return -EPROTONOSUPPORT;
2467
2468 if (!netif_device_present(netdev))
2469 return -EBUSY;
2470
2471 down_write(&ionic->vf_op_lock);
2472
2473 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2474 ret = -EINVAL;
2475 } else {
2476 vfc.vlanid = cpu_to_le16(vlan);
2477 dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
2478 __func__, vf, le16_to_cpu(vfc.vlanid));
2479
2480 ret = ionic_set_vf_config(ionic, vf, &vfc);
2481 if (!ret)
2482 ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2483 }
2484
2485 up_write(&ionic->vf_op_lock);
2486 return ret;
2487 }
2488
ionic_set_vf_rate(struct net_device * netdev,int vf,int tx_min,int tx_max)2489 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2490 int tx_min, int tx_max)
2491 {
2492 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
2493 struct ionic_lif *lif = netdev_priv(netdev);
2494 struct ionic *ionic = lif->ionic;
2495 int ret;
2496
2497 /* setting the min just seems silly */
2498 if (tx_min)
2499 return -EINVAL;
2500
2501 if (!netif_device_present(netdev))
2502 return -EBUSY;
2503
2504 down_write(&ionic->vf_op_lock);
2505
2506 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2507 ret = -EINVAL;
2508 } else {
2509 vfc.maxrate = cpu_to_le32(tx_max);
2510 dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
2511 __func__, vf, le32_to_cpu(vfc.maxrate));
2512
2513 ret = ionic_set_vf_config(ionic, vf, &vfc);
2514 if (!ret)
2515 ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2516 }
2517
2518 up_write(&ionic->vf_op_lock);
2519 return ret;
2520 }
2521
ionic_set_vf_spoofchk(struct net_device * netdev,int vf,bool set)2522 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2523 {
2524 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
2525 struct ionic_lif *lif = netdev_priv(netdev);
2526 struct ionic *ionic = lif->ionic;
2527 int ret;
2528
2529 if (!netif_device_present(netdev))
2530 return -EBUSY;
2531
2532 down_write(&ionic->vf_op_lock);
2533
2534 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2535 ret = -EINVAL;
2536 } else {
2537 vfc.spoofchk = set;
2538 dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
2539 __func__, vf, vfc.spoofchk);
2540
2541 ret = ionic_set_vf_config(ionic, vf, &vfc);
2542 if (!ret)
2543 ionic->vfs[vf].spoofchk = set;
2544 }
2545
2546 up_write(&ionic->vf_op_lock);
2547 return ret;
2548 }
2549
ionic_set_vf_trust(struct net_device * netdev,int vf,bool set)2550 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2551 {
2552 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
2553 struct ionic_lif *lif = netdev_priv(netdev);
2554 struct ionic *ionic = lif->ionic;
2555 int ret;
2556
2557 if (!netif_device_present(netdev))
2558 return -EBUSY;
2559
2560 down_write(&ionic->vf_op_lock);
2561
2562 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2563 ret = -EINVAL;
2564 } else {
2565 vfc.trust = set;
2566 dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
2567 __func__, vf, vfc.trust);
2568
2569 ret = ionic_set_vf_config(ionic, vf, &vfc);
2570 if (!ret)
2571 ionic->vfs[vf].trusted = set;
2572 }
2573
2574 up_write(&ionic->vf_op_lock);
2575 return ret;
2576 }
2577
ionic_set_vf_link_state(struct net_device * netdev,int vf,int set)2578 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2579 {
2580 struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
2581 struct ionic_lif *lif = netdev_priv(netdev);
2582 struct ionic *ionic = lif->ionic;
2583 u8 vfls;
2584 int ret;
2585
2586 switch (set) {
2587 case IFLA_VF_LINK_STATE_ENABLE:
2588 vfls = IONIC_VF_LINK_STATUS_UP;
2589 break;
2590 case IFLA_VF_LINK_STATE_DISABLE:
2591 vfls = IONIC_VF_LINK_STATUS_DOWN;
2592 break;
2593 case IFLA_VF_LINK_STATE_AUTO:
2594 vfls = IONIC_VF_LINK_STATUS_AUTO;
2595 break;
2596 default:
2597 return -EINVAL;
2598 }
2599
2600 if (!netif_device_present(netdev))
2601 return -EBUSY;
2602
2603 down_write(&ionic->vf_op_lock);
2604
2605 if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2606 ret = -EINVAL;
2607 } else {
2608 vfc.linkstate = vfls;
2609 dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
2610 __func__, vf, vfc.linkstate);
2611
2612 ret = ionic_set_vf_config(ionic, vf, &vfc);
2613 if (!ret)
2614 ionic->vfs[vf].linkstate = set;
2615 }
2616
2617 up_write(&ionic->vf_op_lock);
2618 return ret;
2619 }
2620
ionic_vf_attr_replay(struct ionic_lif * lif)2621 static void ionic_vf_attr_replay(struct ionic_lif *lif)
2622 {
2623 struct ionic_vf_setattr_cmd vfc = { };
2624 struct ionic *ionic = lif->ionic;
2625 struct ionic_vf *v;
2626 int i;
2627
2628 if (!ionic->vfs)
2629 return;
2630
2631 down_read(&ionic->vf_op_lock);
2632
2633 for (i = 0; i < ionic->num_vfs; i++) {
2634 v = &ionic->vfs[i];
2635
2636 if (v->stats_pa) {
2637 vfc.attr = IONIC_VF_ATTR_STATSADDR;
2638 vfc.stats_pa = cpu_to_le64(v->stats_pa);
2639 ionic_set_vf_config(ionic, i, &vfc);
2640 vfc.stats_pa = 0;
2641 }
2642
2643 if (!is_zero_ether_addr(v->macaddr)) {
2644 vfc.attr = IONIC_VF_ATTR_MAC;
2645 ether_addr_copy(vfc.macaddr, v->macaddr);
2646 ionic_set_vf_config(ionic, i, &vfc);
2647 eth_zero_addr(vfc.macaddr);
2648 }
2649
2650 if (v->vlanid) {
2651 vfc.attr = IONIC_VF_ATTR_VLAN;
2652 vfc.vlanid = v->vlanid;
2653 ionic_set_vf_config(ionic, i, &vfc);
2654 vfc.vlanid = 0;
2655 }
2656
2657 if (v->maxrate) {
2658 vfc.attr = IONIC_VF_ATTR_RATE;
2659 vfc.maxrate = v->maxrate;
2660 ionic_set_vf_config(ionic, i, &vfc);
2661 vfc.maxrate = 0;
2662 }
2663
2664 if (v->spoofchk) {
2665 vfc.attr = IONIC_VF_ATTR_SPOOFCHK;
2666 vfc.spoofchk = v->spoofchk;
2667 ionic_set_vf_config(ionic, i, &vfc);
2668 vfc.spoofchk = 0;
2669 }
2670
2671 if (v->trusted) {
2672 vfc.attr = IONIC_VF_ATTR_TRUST;
2673 vfc.trust = v->trusted;
2674 ionic_set_vf_config(ionic, i, &vfc);
2675 vfc.trust = 0;
2676 }
2677
2678 if (v->linkstate) {
2679 vfc.attr = IONIC_VF_ATTR_LINKSTATE;
2680 vfc.linkstate = v->linkstate;
2681 ionic_set_vf_config(ionic, i, &vfc);
2682 vfc.linkstate = 0;
2683 }
2684 }
2685
2686 up_read(&ionic->vf_op_lock);
2687
2688 ionic_vf_start(ionic);
2689 }
2690
ionic_unregister_rxq_info(struct ionic_queue * q)2691 static void ionic_unregister_rxq_info(struct ionic_queue *q)
2692 {
2693 struct xdp_rxq_info *xi;
2694
2695 if (!q->xdp_rxq_info)
2696 return;
2697
2698 xi = q->xdp_rxq_info;
2699 q->xdp_rxq_info = NULL;
2700
2701 xdp_rxq_info_unreg(xi);
2702 kfree(xi);
2703 }
2704
ionic_register_rxq_info(struct ionic_queue * q,unsigned int napi_id)2705 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)
2706 {
2707 struct xdp_rxq_info *rxq_info;
2708 int err;
2709
2710 rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
2711 if (!rxq_info)
2712 return -ENOMEM;
2713
2714 err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id);
2715 if (err) {
2716 netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n",
2717 q->index, err);
2718 goto err_out;
2719 }
2720
2721 err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool);
2722 if (err) {
2723 netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n",
2724 q->index, err);
2725 xdp_rxq_info_unreg(rxq_info);
2726 goto err_out;
2727 }
2728
2729 q->xdp_rxq_info = rxq_info;
2730
2731 return 0;
2732
2733 err_out:
2734 kfree(rxq_info);
2735 return err;
2736 }
2737
ionic_xdp_rxqs_prog_update(struct ionic_lif * lif)2738 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif)
2739 {
2740 struct bpf_prog *xdp_prog;
2741 unsigned int i;
2742
2743 if (!lif->rxqcqs)
2744 return;
2745
2746 xdp_prog = READ_ONCE(lif->xdp_prog);
2747 for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2748 struct ionic_queue *q = &lif->rxqcqs[i]->q;
2749
2750 WRITE_ONCE(q->xdp_prog, xdp_prog);
2751 }
2752 }
2753
ionic_xdp_config(struct net_device * netdev,struct netdev_bpf * bpf)2754 static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
2755 {
2756 struct ionic_lif *lif = netdev_priv(netdev);
2757 struct bpf_prog *old_prog;
2758 u32 maxfs;
2759
2760 if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
2761 #define XDP_ERR_SPLIT "XDP not available with split Tx/Rx interrupts"
2762 NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_SPLIT);
2763 netdev_info(lif->netdev, XDP_ERR_SPLIT);
2764 return -EOPNOTSUPP;
2765 }
2766
2767 if (!ionic_xdp_is_valid_mtu(lif, netdev->mtu, bpf->prog)) {
2768 #define XDP_ERR_MTU "MTU is too large for XDP without frags support"
2769 NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_MTU);
2770 netdev_info(lif->netdev, XDP_ERR_MTU);
2771 return -EINVAL;
2772 }
2773
2774 maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
2775 if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags))
2776 maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU);
2777 netdev->max_mtu = maxfs;
2778
2779 if (!netif_running(netdev)) {
2780 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2781 } else if (lif->xdp_prog && bpf->prog) {
2782 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2783 ionic_xdp_rxqs_prog_update(lif);
2784 } else {
2785 struct ionic_queue_params qparams;
2786
2787 ionic_init_queue_params(lif, &qparams);
2788 qparams.xdp_prog = bpf->prog;
2789 mutex_lock(&lif->queue_lock);
2790 ionic_reconfigure_queues(lif, &qparams);
2791 old_prog = xchg(&lif->xdp_prog, bpf->prog);
2792 mutex_unlock(&lif->queue_lock);
2793 }
2794
2795 if (old_prog)
2796 bpf_prog_put(old_prog);
2797
2798 return 0;
2799 }
2800
ionic_xdp(struct net_device * netdev,struct netdev_bpf * bpf)2801 static int ionic_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
2802 {
2803 switch (bpf->command) {
2804 case XDP_SETUP_PROG:
2805 return ionic_xdp_config(netdev, bpf);
2806 default:
2807 return -EINVAL;
2808 }
2809 }
2810
2811 static const struct net_device_ops ionic_netdev_ops = {
2812 .ndo_open = ionic_open,
2813 .ndo_stop = ionic_stop,
2814 .ndo_eth_ioctl = ionic_eth_ioctl,
2815 .ndo_start_xmit = ionic_start_xmit,
2816 .ndo_bpf = ionic_xdp,
2817 .ndo_xdp_xmit = ionic_xdp_xmit,
2818 .ndo_get_stats64 = ionic_get_stats64,
2819 .ndo_set_rx_mode = ionic_ndo_set_rx_mode,
2820 .ndo_set_features = ionic_set_features,
2821 .ndo_set_mac_address = ionic_set_mac_address,
2822 .ndo_validate_addr = eth_validate_addr,
2823 .ndo_tx_timeout = ionic_tx_timeout,
2824 .ndo_change_mtu = ionic_change_mtu,
2825 .ndo_vlan_rx_add_vid = ionic_vlan_rx_add_vid,
2826 .ndo_vlan_rx_kill_vid = ionic_vlan_rx_kill_vid,
2827 .ndo_set_vf_vlan = ionic_set_vf_vlan,
2828 .ndo_set_vf_trust = ionic_set_vf_trust,
2829 .ndo_set_vf_mac = ionic_set_vf_mac,
2830 .ndo_set_vf_rate = ionic_set_vf_rate,
2831 .ndo_set_vf_spoofchk = ionic_set_vf_spoofchk,
2832 .ndo_get_vf_config = ionic_get_vf_config,
2833 .ndo_set_vf_link_state = ionic_set_vf_link_state,
2834 .ndo_get_vf_stats = ionic_get_vf_stats,
2835 };
2836
ionic_cmb_reconfig(struct ionic_lif * lif,struct ionic_queue_params * qparam)2837 static int ionic_cmb_reconfig(struct ionic_lif *lif,
2838 struct ionic_queue_params *qparam)
2839 {
2840 struct ionic_queue_params start_qparams;
2841 int err = 0;
2842
2843 /* When changing CMB queue parameters, we're using limited
2844 * on-device memory and don't have extra memory to use for
2845 * duplicate allocations, so we free it all first then
2846 * re-allocate with the new parameters.
2847 */
2848
2849 /* Checkpoint for possible unwind */
2850 ionic_init_queue_params(lif, &start_qparams);
2851
2852 /* Stop and free the queues */
2853 ionic_stop_queues_reconfig(lif);
2854 ionic_txrx_free(lif);
2855
2856 /* Set up new qparams */
2857 ionic_set_queue_params(lif, qparam);
2858
2859 if (netif_running(lif->netdev)) {
2860 /* Alloc and start the new configuration */
2861 err = ionic_txrx_alloc(lif);
2862 if (err) {
2863 dev_warn(lif->ionic->dev,
2864 "CMB reconfig failed, restoring values: %d\n", err);
2865
2866 /* Back out the changes */
2867 ionic_set_queue_params(lif, &start_qparams);
2868 err = ionic_txrx_alloc(lif);
2869 if (err) {
2870 dev_err(lif->ionic->dev,
2871 "CMB restore failed: %d\n", err);
2872 goto err_out;
2873 }
2874 }
2875
2876 err = ionic_start_queues_reconfig(lif);
2877 if (err) {
2878 dev_err(lif->ionic->dev,
2879 "CMB reconfig failed: %d\n", err);
2880 goto err_out;
2881 }
2882 }
2883
2884 err_out:
2885 /* This was detached in ionic_stop_queues_reconfig() */
2886 netif_device_attach(lif->netdev);
2887
2888 return err;
2889 }
2890
ionic_swap_queues(struct ionic_qcq * a,struct ionic_qcq * b)2891 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2892 {
2893 /* only swapping the queues and napi, not flags or other stuff */
2894 swap(a->napi, b->napi);
2895
2896 if (a->q.type == IONIC_QTYPE_RXQ) {
2897 swap(a->q.page_pool, b->q.page_pool);
2898 a->q.page_pool->p.napi = &a->napi;
2899 if (b->q.page_pool) /* is NULL when increasing queue count */
2900 b->q.page_pool->p.napi = &b->napi;
2901 }
2902
2903 swap(a->q.features, b->q.features);
2904 swap(a->q.num_descs, b->q.num_descs);
2905 swap(a->q.desc_size, b->q.desc_size);
2906 swap(a->q.base, b->q.base);
2907 swap(a->q.base_pa, b->q.base_pa);
2908 swap(a->q.info, b->q.info);
2909 swap(a->q.xdp_prog, b->q.xdp_prog);
2910 swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
2911 swap(a->q.partner, b->q.partner);
2912 swap(a->q_base, b->q_base);
2913 swap(a->q_base_pa, b->q_base_pa);
2914 swap(a->q_size, b->q_size);
2915
2916 swap(a->q.sg_desc_size, b->q.sg_desc_size);
2917 swap(a->q.sg_base, b->q.sg_base);
2918 swap(a->q.sg_base_pa, b->q.sg_base_pa);
2919 swap(a->sg_base, b->sg_base);
2920 swap(a->sg_base_pa, b->sg_base_pa);
2921 swap(a->sg_size, b->sg_size);
2922
2923 swap(a->cq.num_descs, b->cq.num_descs);
2924 swap(a->cq.desc_size, b->cq.desc_size);
2925 swap(a->cq.base, b->cq.base);
2926 swap(a->cq.base_pa, b->cq.base_pa);
2927 swap(a->cq_base, b->cq_base);
2928 swap(a->cq_base_pa, b->cq_base_pa);
2929 swap(a->cq_size, b->cq_size);
2930
2931 ionic_debugfs_del_qcq(a);
2932 ionic_debugfs_add_qcq(a->q.lif, a);
2933 }
2934
ionic_reconfigure_queues(struct ionic_lif * lif,struct ionic_queue_params * qparam)2935 int ionic_reconfigure_queues(struct ionic_lif *lif,
2936 struct ionic_queue_params *qparam)
2937 {
2938 unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2939 struct ionic_qcq **tx_qcqs = NULL;
2940 struct ionic_qcq **rx_qcqs = NULL;
2941 unsigned int flags, i;
2942 int err = 0;
2943
2944 /* Are we changing q params while CMB is on */
2945 if ((test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state) && qparam->cmb_tx) ||
2946 (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state) && qparam->cmb_rx))
2947 return ionic_cmb_reconfig(lif, qparam);
2948
2949 /* allocate temporary qcq arrays to hold new queue structs */
2950 if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2951 tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2952 sizeof(struct ionic_qcq *), GFP_KERNEL);
2953 if (!tx_qcqs) {
2954 err = -ENOMEM;
2955 goto err_out;
2956 }
2957 }
2958 if (qparam->nxqs != lif->nxqs ||
2959 qparam->nrxq_descs != lif->nrxq_descs ||
2960 qparam->rxq_features != lif->rxq_features ||
2961 qparam->xdp_prog != lif->xdp_prog) {
2962 rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2963 sizeof(struct ionic_qcq *), GFP_KERNEL);
2964 if (!rx_qcqs) {
2965 err = -ENOMEM;
2966 goto err_out;
2967 }
2968 }
2969
2970 /* allocate new desc_info and rings, but leave the interrupt setup
2971 * until later so as to not mess with the still-running queues
2972 */
2973 if (tx_qcqs) {
2974 num_desc = qparam->ntxq_descs;
2975 desc_sz = sizeof(struct ionic_txq_desc);
2976 comp_sz = sizeof(struct ionic_txq_comp);
2977
2978 if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2979 lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2980 sizeof(struct ionic_txq_sg_desc_v1))
2981 sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2982 else
2983 sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2984
2985 for (i = 0; i < qparam->nxqs; i++) {
2986 /* If missing, short placeholder qcq needed for swap */
2987 if (!lif->txqcqs[i]) {
2988 flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2989 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2990 4, desc_sz, comp_sz, sg_desc_sz,
2991 sizeof(struct ionic_tx_desc_info),
2992 lif->kern_pid, NULL, &lif->txqcqs[i]);
2993 if (err)
2994 goto err_out;
2995 }
2996
2997 flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2998 err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2999 num_desc, desc_sz, comp_sz, sg_desc_sz,
3000 sizeof(struct ionic_tx_desc_info),
3001 lif->kern_pid, NULL, &tx_qcqs[i]);
3002 if (err)
3003 goto err_out;
3004 }
3005 }
3006
3007 if (rx_qcqs) {
3008 num_desc = qparam->nrxq_descs;
3009 desc_sz = sizeof(struct ionic_rxq_desc);
3010 comp_sz = sizeof(struct ionic_rxq_comp);
3011 sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
3012
3013 if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
3014 comp_sz *= 2;
3015
3016 for (i = 0; i < qparam->nxqs; i++) {
3017 /* If missing, short placeholder qcq needed for swap */
3018 if (!lif->rxqcqs[i]) {
3019 flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
3020 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3021 4, desc_sz, comp_sz, sg_desc_sz,
3022 sizeof(struct ionic_rx_desc_info),
3023 lif->kern_pid, NULL, &lif->rxqcqs[i]);
3024 if (err)
3025 goto err_out;
3026 }
3027
3028 flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
3029 err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3030 num_desc, desc_sz, comp_sz, sg_desc_sz,
3031 sizeof(struct ionic_rx_desc_info),
3032 lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]);
3033 if (err)
3034 goto err_out;
3035
3036 rx_qcqs[i]->q.features = qparam->rxq_features;
3037 rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog;
3038 }
3039 }
3040
3041 /* stop and clean the queues */
3042 ionic_stop_queues_reconfig(lif);
3043
3044 if (qparam->nxqs != lif->nxqs) {
3045 err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
3046 if (err)
3047 goto err_out_reinit_unlock;
3048 err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
3049 if (err) {
3050 netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
3051 goto err_out_reinit_unlock;
3052 }
3053 }
3054
3055 /* swap new desc_info and rings, keeping existing interrupt config */
3056 if (tx_qcqs) {
3057 lif->ntxq_descs = qparam->ntxq_descs;
3058 for (i = 0; i < qparam->nxqs; i++)
3059 ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
3060 }
3061
3062 if (rx_qcqs) {
3063 lif->nrxq_descs = qparam->nrxq_descs;
3064 for (i = 0; i < qparam->nxqs; i++)
3065 ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
3066 }
3067
3068 /* if we need to change the interrupt layout, this is the time */
3069 if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
3070 qparam->nxqs != lif->nxqs) {
3071 if (qparam->intr_split) {
3072 set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3073 } else {
3074 clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3075 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3076 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3077 }
3078
3079 /* Clear existing interrupt assignments. We check for NULL here
3080 * because we're checking the whole array for potential qcqs, not
3081 * just those qcqs that have just been set up.
3082 */
3083 for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
3084 if (lif->txqcqs[i])
3085 ionic_qcq_intr_free(lif, lif->txqcqs[i]);
3086 if (lif->rxqcqs[i])
3087 ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
3088 }
3089
3090 /* re-assign the interrupts */
3091 for (i = 0; i < qparam->nxqs; i++) {
3092 lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3093 err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
3094 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3095 lif->rxqcqs[i]->intr.index,
3096 lif->rx_coalesce_hw);
3097
3098 if (qparam->intr_split) {
3099 lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3100 err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
3101 ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3102 lif->txqcqs[i]->intr.index,
3103 lif->tx_coalesce_hw);
3104 if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
3105 lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
3106 } else {
3107 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3108 ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
3109 }
3110 }
3111 }
3112
3113 /* now we can rework the debugfs mappings */
3114 if (tx_qcqs) {
3115 for (i = 0; i < qparam->nxqs; i++) {
3116 ionic_debugfs_del_qcq(lif->txqcqs[i]);
3117 ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
3118 }
3119 }
3120
3121 if (rx_qcqs) {
3122 for (i = 0; i < qparam->nxqs; i++) {
3123 ionic_debugfs_del_qcq(lif->rxqcqs[i]);
3124 ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
3125 }
3126 }
3127
3128 swap(lif->nxqs, qparam->nxqs);
3129 swap(lif->rxq_features, qparam->rxq_features);
3130
3131 err_out_reinit_unlock:
3132 /* re-init the queues, but don't lose an error code */
3133 if (err)
3134 ionic_start_queues_reconfig(lif);
3135 else
3136 err = ionic_start_queues_reconfig(lif);
3137
3138 err_out:
3139 /* free old allocs without cleaning intr */
3140 for (i = 0; i < qparam->nxqs; i++) {
3141 if (tx_qcqs && tx_qcqs[i]) {
3142 tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3143 ionic_qcq_free(lif, tx_qcqs[i]);
3144 devm_kfree(lif->ionic->dev, tx_qcqs[i]);
3145 tx_qcqs[i] = NULL;
3146 }
3147 if (rx_qcqs && rx_qcqs[i]) {
3148 rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3149 ionic_qcq_free(lif, rx_qcqs[i]);
3150 devm_kfree(lif->ionic->dev, rx_qcqs[i]);
3151 rx_qcqs[i] = NULL;
3152 }
3153 }
3154
3155 /* free q array */
3156 if (rx_qcqs) {
3157 devm_kfree(lif->ionic->dev, rx_qcqs);
3158 rx_qcqs = NULL;
3159 }
3160 if (tx_qcqs) {
3161 devm_kfree(lif->ionic->dev, tx_qcqs);
3162 tx_qcqs = NULL;
3163 }
3164
3165 /* clean the unused dma and info allocations when new set is smaller
3166 * than the full array, but leave the qcq shells in place
3167 */
3168 for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
3169 if (lif->txqcqs && lif->txqcqs[i]) {
3170 lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3171 ionic_qcq_free(lif, lif->txqcqs[i]);
3172 }
3173
3174 if (lif->rxqcqs && lif->rxqcqs[i]) {
3175 lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3176 ionic_qcq_free(lif, lif->rxqcqs[i]);
3177 }
3178 }
3179
3180 if (err)
3181 netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
3182
3183 return err;
3184 }
3185
ionic_affinity_masks_alloc(struct ionic * ionic)3186 static int ionic_affinity_masks_alloc(struct ionic *ionic)
3187 {
3188 cpumask_var_t *affinity_masks;
3189 int nintrs = ionic->nintrs;
3190 int i;
3191
3192 affinity_masks = kcalloc(nintrs, sizeof(cpumask_var_t), GFP_KERNEL);
3193 if (!affinity_masks)
3194 return -ENOMEM;
3195
3196 for (i = 0; i < nintrs; i++) {
3197 if (!zalloc_cpumask_var_node(&affinity_masks[i], GFP_KERNEL,
3198 dev_to_node(ionic->dev)))
3199 goto err_out;
3200 }
3201
3202 ionic->affinity_masks = affinity_masks;
3203
3204 return 0;
3205
3206 err_out:
3207 for (--i; i >= 0; i--)
3208 free_cpumask_var(affinity_masks[i]);
3209 kfree(affinity_masks);
3210
3211 return -ENOMEM;
3212 }
3213
ionic_affinity_masks_free(struct ionic * ionic)3214 static void ionic_affinity_masks_free(struct ionic *ionic)
3215 {
3216 int i;
3217
3218 for (i = 0; i < ionic->nintrs; i++)
3219 free_cpumask_var(ionic->affinity_masks[i]);
3220 kfree(ionic->affinity_masks);
3221 ionic->affinity_masks = NULL;
3222 }
3223
ionic_lif_alloc(struct ionic * ionic)3224 int ionic_lif_alloc(struct ionic *ionic)
3225 {
3226 struct device *dev = ionic->dev;
3227 union ionic_lif_identity *lid;
3228 struct net_device *netdev;
3229 struct ionic_lif *lif;
3230 int tbl_sz;
3231 int err;
3232
3233 lid = kzalloc(sizeof(*lid), GFP_KERNEL);
3234 if (!lid)
3235 return -ENOMEM;
3236
3237 netdev = alloc_etherdev_mqs(sizeof(*lif),
3238 ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
3239 if (!netdev) {
3240 dev_err(dev, "Cannot allocate netdev, aborting\n");
3241 err = -ENOMEM;
3242 goto err_out_free_lid;
3243 }
3244
3245 SET_NETDEV_DEV(netdev, dev);
3246
3247 lif = netdev_priv(netdev);
3248 lif->netdev = netdev;
3249 ionic->lif = lif;
3250 lif->ionic = ionic;
3251 netdev->netdev_ops = &ionic_netdev_ops;
3252 ionic_ethtool_set_ops(netdev);
3253
3254 netdev->watchdog_timeo = 5 * HZ;
3255 netif_carrier_off(netdev);
3256
3257 lif->identity = lid;
3258 lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
3259 err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
3260 if (err) {
3261 dev_err(ionic->dev, "Cannot identify type %d: %d\n",
3262 lif->lif_type, err);
3263 goto err_out_free_netdev;
3264 }
3265 lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
3266 le32_to_cpu(lif->identity->eth.min_frame_size));
3267 lif->netdev->max_mtu =
3268 le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
3269
3270 lif->neqs = ionic->neqs_per_lif;
3271 lif->nxqs = ionic->ntxqs_per_lif;
3272
3273 lif->index = 0;
3274
3275 if (is_kdump_kernel()) {
3276 lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
3277 lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
3278 } else {
3279 lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
3280 lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
3281 }
3282
3283 /* Convert the default coalesce value to actual hw resolution */
3284 lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
3285 lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
3286 lif->rx_coalesce_usecs);
3287 lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3288 lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3289 set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
3290 set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
3291
3292 snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
3293
3294 mutex_init(&lif->queue_lock);
3295 mutex_init(&lif->config_lock);
3296
3297 spin_lock_init(&lif->adminq_lock);
3298
3299 spin_lock_init(&lif->deferred.lock);
3300 INIT_LIST_HEAD(&lif->deferred.list);
3301 INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
3302
3303 /* allocate lif info */
3304 lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
3305 lif->info = dma_alloc_coherent(dev, lif->info_sz,
3306 &lif->info_pa, GFP_KERNEL);
3307 if (!lif->info) {
3308 dev_err(dev, "Failed to allocate lif info, aborting\n");
3309 err = -ENOMEM;
3310 goto err_out_free_mutex;
3311 }
3312
3313 ionic_debugfs_add_lif(lif);
3314
3315 err = ionic_affinity_masks_alloc(ionic);
3316 if (err)
3317 goto err_out_free_lif_info;
3318
3319 /* allocate control queues and txrx queue arrays */
3320 ionic_lif_queue_identify(lif);
3321 err = ionic_qcqs_alloc(lif);
3322 if (err)
3323 goto err_out_free_affinity_masks;
3324
3325 /* allocate rss indirection table */
3326 tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
3327 lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
3328 lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
3329 &lif->rss_ind_tbl_pa,
3330 GFP_KERNEL);
3331
3332 if (!lif->rss_ind_tbl) {
3333 err = -ENOMEM;
3334 dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
3335 goto err_out_free_qcqs;
3336 }
3337 netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
3338
3339 ionic_lif_alloc_phc(lif);
3340
3341 return 0;
3342
3343 err_out_free_qcqs:
3344 ionic_qcqs_free(lif);
3345 err_out_free_affinity_masks:
3346 ionic_affinity_masks_free(lif->ionic);
3347 err_out_free_lif_info:
3348 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3349 lif->info = NULL;
3350 lif->info_pa = 0;
3351 err_out_free_mutex:
3352 mutex_destroy(&lif->config_lock);
3353 mutex_destroy(&lif->queue_lock);
3354 err_out_free_netdev:
3355 free_netdev(lif->netdev);
3356 lif = NULL;
3357 err_out_free_lid:
3358 kfree(lid);
3359
3360 return err;
3361 }
3362
ionic_lif_reset(struct ionic_lif * lif)3363 static void ionic_lif_reset(struct ionic_lif *lif)
3364 {
3365 struct ionic_dev *idev = &lif->ionic->idev;
3366
3367 if (!ionic_is_fw_running(idev))
3368 return;
3369
3370 mutex_lock(&lif->ionic->dev_cmd_lock);
3371 ionic_dev_cmd_lif_reset(idev, lif->index);
3372 ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3373 mutex_unlock(&lif->ionic->dev_cmd_lock);
3374 }
3375
ionic_lif_handle_fw_down(struct ionic_lif * lif)3376 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
3377 {
3378 struct ionic *ionic = lif->ionic;
3379
3380 if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
3381 return;
3382
3383 dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
3384
3385 netif_device_detach(lif->netdev);
3386
3387 mutex_lock(&lif->queue_lock);
3388 if (test_bit(IONIC_LIF_F_UP, lif->state)) {
3389 dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
3390 ionic_stop_queues(lif);
3391 }
3392
3393 if (netif_running(lif->netdev)) {
3394 ionic_txrx_deinit(lif);
3395 ionic_txrx_free(lif);
3396 }
3397 ionic_lif_deinit(lif);
3398 ionic_reset(ionic);
3399 ionic_qcqs_free(lif);
3400
3401 mutex_unlock(&lif->queue_lock);
3402
3403 clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
3404 dev_info(ionic->dev, "FW Down: LIFs stopped\n");
3405 }
3406
ionic_restart_lif(struct ionic_lif * lif)3407 int ionic_restart_lif(struct ionic_lif *lif)
3408 {
3409 struct ionic *ionic = lif->ionic;
3410 int err;
3411
3412 mutex_lock(&lif->queue_lock);
3413
3414 if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
3415 dev_info(ionic->dev, "FW Up: clearing broken state\n");
3416
3417 err = ionic_qcqs_alloc(lif);
3418 if (err)
3419 goto err_unlock;
3420
3421 err = ionic_lif_init(lif);
3422 if (err)
3423 goto err_qcqs_free;
3424
3425 ionic_vf_attr_replay(lif);
3426
3427 if (lif->registered)
3428 ionic_lif_set_netdev_info(lif);
3429
3430 ionic_rx_filter_replay(lif);
3431
3432 if (netif_running(lif->netdev)) {
3433 err = ionic_txrx_alloc(lif);
3434 if (err)
3435 goto err_lifs_deinit;
3436
3437 err = ionic_txrx_init(lif);
3438 if (err)
3439 goto err_txrx_free;
3440 }
3441
3442 mutex_unlock(&lif->queue_lock);
3443
3444 clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3445 ionic_link_status_check_request(lif, CAN_SLEEP);
3446 netif_device_attach(lif->netdev);
3447 ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
3448
3449 return 0;
3450
3451 err_txrx_free:
3452 ionic_txrx_free(lif);
3453 err_lifs_deinit:
3454 ionic_lif_deinit(lif);
3455 err_qcqs_free:
3456 ionic_qcqs_free(lif);
3457 err_unlock:
3458 mutex_unlock(&lif->queue_lock);
3459
3460 return err;
3461 }
3462
ionic_lif_handle_fw_up(struct ionic_lif * lif)3463 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
3464 {
3465 struct ionic *ionic = lif->ionic;
3466 int err;
3467
3468 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3469 return;
3470
3471 dev_info(ionic->dev, "FW Up: restarting LIFs\n");
3472
3473 /* This is a little different from what happens at
3474 * probe time because the LIF already exists so we
3475 * just need to reanimate it.
3476 */
3477 ionic_init_devinfo(ionic);
3478 ionic_reset(ionic);
3479 err = ionic_identify(ionic);
3480 if (err)
3481 goto err_out;
3482 err = ionic_port_identify(ionic);
3483 if (err)
3484 goto err_out;
3485 err = ionic_port_init(ionic);
3486 if (err)
3487 goto err_out;
3488
3489 err = ionic_restart_lif(lif);
3490 if (err)
3491 goto err_out;
3492
3493 dev_info(ionic->dev, "FW Up: LIFs restarted\n");
3494
3495 /* restore the hardware timestamping queues */
3496 ionic_lif_hwstamp_replay(lif);
3497
3498 return;
3499
3500 err_out:
3501 dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3502 }
3503
ionic_lif_free(struct ionic_lif * lif)3504 void ionic_lif_free(struct ionic_lif *lif)
3505 {
3506 struct device *dev = lif->ionic->dev;
3507
3508 ionic_lif_free_phc(lif);
3509
3510 /* free rss indirection table */
3511 dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3512 lif->rss_ind_tbl_pa);
3513 lif->rss_ind_tbl = NULL;
3514 lif->rss_ind_tbl_pa = 0;
3515
3516 /* free queues */
3517 ionic_qcqs_free(lif);
3518 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3519 ionic_lif_reset(lif);
3520
3521 ionic_affinity_masks_free(lif->ionic);
3522
3523 /* free lif info */
3524 kfree(lif->identity);
3525 dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3526 lif->info = NULL;
3527 lif->info_pa = 0;
3528
3529 /* unmap doorbell page */
3530 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3531 lif->kern_dbpage = NULL;
3532
3533 mutex_destroy(&lif->config_lock);
3534 mutex_destroy(&lif->queue_lock);
3535
3536 /* free netdev & lif */
3537 ionic_debugfs_del_lif(lif);
3538 free_netdev(lif->netdev);
3539 }
3540
ionic_lif_deinit(struct ionic_lif * lif)3541 void ionic_lif_deinit(struct ionic_lif *lif)
3542 {
3543 if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3544 return;
3545
3546 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3547 cancel_work_sync(&lif->deferred.work);
3548 cancel_work_sync(&lif->tx_timeout_work);
3549 ionic_rx_filters_deinit(lif);
3550 if (lif->netdev->features & NETIF_F_RXHASH)
3551 ionic_lif_rss_deinit(lif);
3552 }
3553
3554 napi_disable(&lif->adminqcq->napi);
3555 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3556 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3557
3558 ionic_lif_reset(lif);
3559 }
3560
ionic_lif_adminq_init(struct ionic_lif * lif)3561 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3562 {
3563 struct device *dev = lif->ionic->dev;
3564 struct ionic_q_init_comp comp;
3565 struct ionic_dev *idev;
3566 struct ionic_qcq *qcq;
3567 struct ionic_queue *q;
3568 int err;
3569
3570 idev = &lif->ionic->idev;
3571 qcq = lif->adminqcq;
3572 q = &qcq->q;
3573
3574 mutex_lock(&lif->ionic->dev_cmd_lock);
3575 ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3576 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3577 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3578 mutex_unlock(&lif->ionic->dev_cmd_lock);
3579 if (err) {
3580 netdev_err(lif->netdev, "adminq init failed %d\n", err);
3581 return err;
3582 }
3583
3584 q->hw_type = comp.hw_type;
3585 q->hw_index = le32_to_cpu(comp.hw_index);
3586 q->dbval = IONIC_DBELL_QID(q->hw_index);
3587
3588 dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3589 dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3590
3591 q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;
3592 q->dbell_jiffies = jiffies;
3593
3594 netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
3595
3596 napi_enable(&qcq->napi);
3597
3598 if (qcq->flags & IONIC_QCQ_F_INTR) {
3599 irq_set_affinity_hint(qcq->intr.vector,
3600 *qcq->intr.affinity_mask);
3601 ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3602 IONIC_INTR_MASK_CLEAR);
3603 }
3604
3605 qcq->flags |= IONIC_QCQ_F_INITED;
3606
3607 return 0;
3608 }
3609
ionic_lif_notifyq_init(struct ionic_lif * lif)3610 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3611 {
3612 struct ionic_qcq *qcq = lif->notifyqcq;
3613 struct device *dev = lif->ionic->dev;
3614 struct ionic_queue *q = &qcq->q;
3615 int err;
3616
3617 struct ionic_admin_ctx ctx = {
3618 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3619 .cmd.q_init = {
3620 .opcode = IONIC_CMD_Q_INIT,
3621 .lif_index = cpu_to_le16(lif->index),
3622 .type = q->type,
3623 .ver = lif->qtype_info[q->type].version,
3624 .index = cpu_to_le32(q->index),
3625 .flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3626 IONIC_QINIT_F_ENA),
3627 .intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3628 .pid = cpu_to_le16(q->pid),
3629 .ring_size = ilog2(q->num_descs),
3630 .ring_base = cpu_to_le64(q->base_pa),
3631 }
3632 };
3633
3634 dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3635 dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3636 dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3637 dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3638
3639 err = ionic_adminq_post_wait(lif, &ctx);
3640 if (err)
3641 return err;
3642
3643 lif->last_eid = 0;
3644 q->hw_type = ctx.comp.q_init.hw_type;
3645 q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3646 q->dbval = IONIC_DBELL_QID(q->hw_index);
3647
3648 dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3649 dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3650
3651 /* preset the callback info */
3652 q->admin_info[0].ctx = lif;
3653
3654 qcq->flags |= IONIC_QCQ_F_INITED;
3655
3656 return 0;
3657 }
3658
ionic_station_set(struct ionic_lif * lif)3659 static int ionic_station_set(struct ionic_lif *lif)
3660 {
3661 struct net_device *netdev = lif->netdev;
3662 struct ionic_admin_ctx ctx = {
3663 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3664 .cmd.lif_getattr = {
3665 .opcode = IONIC_CMD_LIF_GETATTR,
3666 .index = cpu_to_le16(lif->index),
3667 .attr = IONIC_LIF_ATTR_MAC,
3668 },
3669 };
3670 u8 mac_address[ETH_ALEN];
3671 struct sockaddr addr;
3672 int err;
3673
3674 err = ionic_adminq_post_wait(lif, &ctx);
3675 if (err)
3676 return err;
3677 netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3678 ctx.comp.lif_getattr.mac);
3679 ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);
3680
3681 if (is_zero_ether_addr(mac_address)) {
3682 eth_hw_addr_random(netdev);
3683 netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);
3684 ether_addr_copy(mac_address, netdev->dev_addr);
3685
3686 err = ionic_program_mac(lif, mac_address);
3687 if (err < 0)
3688 return err;
3689
3690 if (err > 0) {
3691 netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",
3692 __func__);
3693 return 0;
3694 }
3695 }
3696
3697 if (!is_zero_ether_addr(netdev->dev_addr)) {
3698 /* If the netdev mac is non-zero and doesn't match the default
3699 * device address, it was set by something earlier and we're
3700 * likely here again after a fw-upgrade reset. We need to be
3701 * sure the netdev mac is in our filter list.
3702 */
3703 if (!ether_addr_equal(mac_address, netdev->dev_addr))
3704 ionic_lif_addr_add(lif, netdev->dev_addr);
3705 } else {
3706 /* Update the netdev mac with the device's mac */
3707 ether_addr_copy(addr.sa_data, mac_address);
3708 addr.sa_family = AF_INET;
3709 err = eth_prepare_mac_addr_change(netdev, &addr);
3710 if (err) {
3711 netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3712 addr.sa_data, err);
3713 return 0;
3714 }
3715
3716 eth_commit_mac_addr_change(netdev, &addr);
3717 }
3718
3719 netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3720 netdev->dev_addr);
3721 ionic_lif_addr_add(lif, netdev->dev_addr);
3722
3723 return 0;
3724 }
3725
ionic_lif_init(struct ionic_lif * lif)3726 int ionic_lif_init(struct ionic_lif *lif)
3727 {
3728 struct ionic_dev *idev = &lif->ionic->idev;
3729 struct device *dev = lif->ionic->dev;
3730 struct ionic_lif_init_comp comp;
3731 int dbpage_num;
3732 int err;
3733
3734 mutex_lock(&lif->ionic->dev_cmd_lock);
3735 ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3736 err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3737 ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3738 mutex_unlock(&lif->ionic->dev_cmd_lock);
3739 if (err)
3740 return err;
3741
3742 lif->hw_index = le16_to_cpu(comp.hw_index);
3743
3744 /* now that we have the hw_index we can figure out our doorbell page */
3745 lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3746 if (!lif->dbid_count) {
3747 dev_err(dev, "No doorbell pages, aborting\n");
3748 return -EINVAL;
3749 }
3750
3751 lif->kern_pid = 0;
3752 dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3753 lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3754 if (!lif->kern_dbpage) {
3755 dev_err(dev, "Cannot map dbpage, aborting\n");
3756 return -ENOMEM;
3757 }
3758
3759 err = ionic_lif_adminq_init(lif);
3760 if (err)
3761 goto err_out_adminq_deinit;
3762
3763 if (lif->ionic->nnqs_per_lif) {
3764 err = ionic_lif_notifyq_init(lif);
3765 if (err)
3766 goto err_out_notifyq_deinit;
3767 }
3768
3769 if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3770 err = ionic_set_nic_features(lif, lif->netdev->features);
3771 else
3772 err = ionic_init_nic_features(lif);
3773 if (err)
3774 goto err_out_notifyq_deinit;
3775
3776 if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3777 err = ionic_rx_filters_init(lif);
3778 if (err)
3779 goto err_out_notifyq_deinit;
3780 }
3781
3782 err = ionic_station_set(lif);
3783 if (err)
3784 goto err_out_notifyq_deinit;
3785
3786 lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3787 lif->doorbell_wa = ionic_doorbell_wa(lif->ionic);
3788
3789 set_bit(IONIC_LIF_F_INITED, lif->state);
3790
3791 INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3792
3793 return 0;
3794
3795 err_out_notifyq_deinit:
3796 napi_disable(&lif->adminqcq->napi);
3797 ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3798 err_out_adminq_deinit:
3799 ionic_lif_qcq_deinit(lif, lif->adminqcq);
3800 ionic_lif_reset(lif);
3801 ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3802 lif->kern_dbpage = NULL;
3803
3804 return err;
3805 }
3806
ionic_lif_notify_work(struct work_struct * ws)3807 static void ionic_lif_notify_work(struct work_struct *ws)
3808 {
3809 }
3810
ionic_lif_set_netdev_info(struct ionic_lif * lif)3811 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3812 {
3813 struct ionic_admin_ctx ctx = {
3814 .work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3815 .cmd.lif_setattr = {
3816 .opcode = IONIC_CMD_LIF_SETATTR,
3817 .index = cpu_to_le16(lif->index),
3818 .attr = IONIC_LIF_ATTR_NAME,
3819 },
3820 };
3821
3822 strscpy(ctx.cmd.lif_setattr.name, netdev_name(lif->netdev),
3823 sizeof(ctx.cmd.lif_setattr.name));
3824
3825 ionic_adminq_post_wait(lif, &ctx);
3826 }
3827
ionic_netdev_lif(struct net_device * netdev)3828 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3829 {
3830 if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3831 return NULL;
3832
3833 return netdev_priv(netdev);
3834 }
3835
ionic_lif_notify(struct notifier_block * nb,unsigned long event,void * info)3836 static int ionic_lif_notify(struct notifier_block *nb,
3837 unsigned long event, void *info)
3838 {
3839 struct net_device *ndev = netdev_notifier_info_to_dev(info);
3840 struct ionic *ionic = container_of(nb, struct ionic, nb);
3841 struct ionic_lif *lif = ionic_netdev_lif(ndev);
3842
3843 if (!lif || lif->ionic != ionic)
3844 return NOTIFY_DONE;
3845
3846 switch (event) {
3847 case NETDEV_CHANGENAME:
3848 ionic_lif_set_netdev_info(lif);
3849 break;
3850 }
3851
3852 return NOTIFY_DONE;
3853 }
3854
ionic_lif_register(struct ionic_lif * lif)3855 int ionic_lif_register(struct ionic_lif *lif)
3856 {
3857 int err;
3858
3859 ionic_lif_register_phc(lif);
3860
3861 INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3862
3863 lif->ionic->nb.notifier_call = ionic_lif_notify;
3864
3865 err = register_netdevice_notifier(&lif->ionic->nb);
3866 if (err)
3867 lif->ionic->nb.notifier_call = NULL;
3868
3869 /* only register LIF0 for now */
3870 err = register_netdev(lif->netdev);
3871 if (err) {
3872 dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3873 ionic_lif_unregister_phc(lif);
3874 return err;
3875 }
3876
3877 ionic_link_status_check_request(lif, CAN_SLEEP);
3878 lif->registered = true;
3879 ionic_lif_set_netdev_info(lif);
3880
3881 return 0;
3882 }
3883
ionic_lif_unregister(struct ionic_lif * lif)3884 void ionic_lif_unregister(struct ionic_lif *lif)
3885 {
3886 if (lif->ionic->nb.notifier_call) {
3887 unregister_netdevice_notifier(&lif->ionic->nb);
3888 cancel_work_sync(&lif->ionic->nb_work);
3889 lif->ionic->nb.notifier_call = NULL;
3890 }
3891
3892 if (lif->netdev->reg_state == NETREG_REGISTERED)
3893 unregister_netdev(lif->netdev);
3894
3895 ionic_lif_unregister_phc(lif);
3896
3897 lif->registered = false;
3898 }
3899
ionic_lif_queue_identify(struct ionic_lif * lif)3900 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3901 {
3902 union ionic_q_identity __iomem *q_ident;
3903 struct ionic *ionic = lif->ionic;
3904 struct ionic_dev *idev;
3905 u16 max_frags;
3906 int qtype;
3907 int err;
3908
3909 idev = &lif->ionic->idev;
3910 q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3911
3912 for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3913 struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3914
3915 /* filter out the ones we know about */
3916 switch (qtype) {
3917 case IONIC_QTYPE_ADMINQ:
3918 case IONIC_QTYPE_NOTIFYQ:
3919 case IONIC_QTYPE_RXQ:
3920 case IONIC_QTYPE_TXQ:
3921 break;
3922 default:
3923 continue;
3924 }
3925
3926 memset(qti, 0, sizeof(*qti));
3927
3928 mutex_lock(&ionic->dev_cmd_lock);
3929 ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3930 ionic_qtype_versions[qtype]);
3931 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3932 if (!err) {
3933 qti->version = readb(&q_ident->version);
3934 qti->supported = readb(&q_ident->supported);
3935 qti->features = readq(&q_ident->features);
3936 qti->desc_sz = readw(&q_ident->desc_sz);
3937 qti->comp_sz = readw(&q_ident->comp_sz);
3938 qti->sg_desc_sz = readw(&q_ident->sg_desc_sz);
3939 qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3940 qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3941 }
3942 mutex_unlock(&ionic->dev_cmd_lock);
3943
3944 if (err == -EINVAL) {
3945 dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3946 continue;
3947 } else if (err == -EIO) {
3948 dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3949 return;
3950 } else if (err) {
3951 dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3952 qtype, err);
3953 return;
3954 }
3955
3956 dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3957 qtype, qti->version);
3958 dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3959 qtype, qti->supported);
3960 dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3961 qtype, qti->features);
3962 dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3963 qtype, qti->desc_sz);
3964 dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3965 qtype, qti->comp_sz);
3966 dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3967 qtype, qti->sg_desc_sz);
3968 dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3969 qtype, qti->max_sg_elems);
3970 dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3971 qtype, qti->sg_desc_stride);
3972
3973 if (qtype == IONIC_QTYPE_TXQ)
3974 max_frags = IONIC_TX_MAX_FRAGS;
3975 else if (qtype == IONIC_QTYPE_RXQ)
3976 max_frags = IONIC_RX_MAX_FRAGS;
3977 else
3978 max_frags = 1;
3979
3980 qti->max_sg_elems = min_t(u16, max_frags - 1, MAX_SKB_FRAGS);
3981 dev_dbg(ionic->dev, "qtype %d max_sg_elems %d\n",
3982 qtype, qti->max_sg_elems);
3983 }
3984 }
3985
ionic_lif_identify(struct ionic * ionic,u8 lif_type,union ionic_lif_identity * lid)3986 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3987 union ionic_lif_identity *lid)
3988 {
3989 struct ionic_dev *idev = &ionic->idev;
3990 size_t sz;
3991 int err;
3992
3993 sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3994
3995 mutex_lock(&ionic->dev_cmd_lock);
3996 ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3997 err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3998 memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3999 mutex_unlock(&ionic->dev_cmd_lock);
4000 if (err)
4001 return (err);
4002
4003 dev_dbg(ionic->dev, "capabilities 0x%llx\n",
4004 le64_to_cpu(lid->capabilities));
4005
4006 dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
4007 le32_to_cpu(lid->eth.max_ucast_filters));
4008 dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
4009 le32_to_cpu(lid->eth.max_mcast_filters));
4010 dev_dbg(ionic->dev, "eth.features 0x%llx\n",
4011 le64_to_cpu(lid->eth.config.features));
4012 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
4013 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
4014 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
4015 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
4016 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
4017 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
4018 dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
4019 le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
4020 dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
4021 dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
4022 dev_dbg(ionic->dev, "eth.config.mtu %d\n",
4023 le32_to_cpu(lid->eth.config.mtu));
4024
4025 return 0;
4026 }
4027
ionic_lif_size(struct ionic * ionic)4028 int ionic_lif_size(struct ionic *ionic)
4029 {
4030 struct ionic_identity *ident = &ionic->ident;
4031 unsigned int nintrs, dev_nintrs;
4032 union ionic_lif_config *lc;
4033 unsigned int ntxqs_per_lif;
4034 unsigned int nrxqs_per_lif;
4035 unsigned int neqs_per_lif;
4036 unsigned int nnqs_per_lif;
4037 unsigned int nxqs, neqs;
4038 unsigned int min_intrs;
4039 int err;
4040
4041 /* retrieve basic values from FW */
4042 lc = &ident->lif.eth.config;
4043 dev_nintrs = le32_to_cpu(ident->dev.nintrs);
4044 neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
4045 nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
4046 ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
4047 nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
4048
4049 /* limit values to play nice with kdump */
4050 if (is_kdump_kernel()) {
4051 dev_nintrs = 2;
4052 neqs_per_lif = 0;
4053 nnqs_per_lif = 0;
4054 ntxqs_per_lif = 1;
4055 nrxqs_per_lif = 1;
4056 }
4057
4058 /* reserve last queue id for hardware timestamping */
4059 if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
4060 if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
4061 lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
4062 } else {
4063 ntxqs_per_lif -= 1;
4064 nrxqs_per_lif -= 1;
4065 }
4066 }
4067
4068 nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
4069 nxqs = min(nxqs, num_online_cpus());
4070 neqs = min(neqs_per_lif, num_online_cpus());
4071
4072 try_again:
4073 /* interrupt usage:
4074 * 1 for master lif adminq/notifyq
4075 * 1 for each CPU for master lif TxRx queue pairs
4076 * whatever's left is for RDMA queues
4077 */
4078 nintrs = 1 + nxqs + neqs;
4079 min_intrs = 2; /* adminq + 1 TxRx queue pair */
4080
4081 if (nintrs > dev_nintrs)
4082 goto try_fewer;
4083
4084 err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
4085 if (err < 0 && err != -ENOSPC) {
4086 dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
4087 return err;
4088 }
4089 if (err == -ENOSPC)
4090 goto try_fewer;
4091
4092 if (err != nintrs) {
4093 ionic_bus_free_irq_vectors(ionic);
4094 goto try_fewer;
4095 }
4096
4097 ionic->nnqs_per_lif = nnqs_per_lif;
4098 ionic->neqs_per_lif = neqs;
4099 ionic->ntxqs_per_lif = nxqs;
4100 ionic->nrxqs_per_lif = nxqs;
4101 ionic->nintrs = nintrs;
4102
4103 ionic_debugfs_add_sizes(ionic);
4104
4105 return 0;
4106
4107 try_fewer:
4108 if (nnqs_per_lif > 1) {
4109 nnqs_per_lif >>= 1;
4110 goto try_again;
4111 }
4112 if (neqs > 1) {
4113 neqs >>= 1;
4114 goto try_again;
4115 }
4116 if (nxqs > 1) {
4117 nxqs >>= 1;
4118 goto try_again;
4119 }
4120 dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
4121 return -ENOSPC;
4122 }
4123