xref: /linux/drivers/net/ethernet/pensando/ionic/ionic_lif.c (revision 9410645520e9b820069761f3450ef6661418e279)
1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
3 
4 #include <linux/ethtool.h>
5 #include <linux/printk.h>
6 #include <linux/dynamic_debug.h>
7 #include <linux/netdevice.h>
8 #include <linux/etherdevice.h>
9 #include <linux/if_vlan.h>
10 #include <linux/rtnetlink.h>
11 #include <linux/interrupt.h>
12 #include <linux/pci.h>
13 #include <linux/cpumask.h>
14 #include <linux/crash_dump.h>
15 #include <linux/vmalloc.h>
16 #include <net/page_pool/helpers.h>
17 
18 #include "ionic.h"
19 #include "ionic_bus.h"
20 #include "ionic_dev.h"
21 #include "ionic_lif.h"
22 #include "ionic_txrx.h"
23 #include "ionic_ethtool.h"
24 #include "ionic_debugfs.h"
25 
26 /* queuetype support level */
27 static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
28 	[IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
29 	[IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
30 	[IONIC_QTYPE_RXQ]     = 2,   /* 0 = Base version with CQ+SG support
31 				      * 2 =       ... with CMB rings
32 				      */
33 	[IONIC_QTYPE_TXQ]     = 3,   /* 0 = Base version with CQ+SG support
34 				      * 1 =       ... with Tx SG version 1
35 				      * 3 =       ... with CMB rings
36 				      */
37 };
38 
39 static void ionic_link_status_check(struct ionic_lif *lif);
40 static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
41 static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
42 static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
43 
44 static void ionic_txrx_deinit(struct ionic_lif *lif);
45 static int ionic_txrx_init(struct ionic_lif *lif);
46 static int ionic_start_queues(struct ionic_lif *lif);
47 static void ionic_stop_queues(struct ionic_lif *lif);
48 static void ionic_lif_queue_identify(struct ionic_lif *lif);
49 
50 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif);
51 static void ionic_unregister_rxq_info(struct ionic_queue *q);
52 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id);
53 
ionic_dim_work(struct work_struct * work)54 static void ionic_dim_work(struct work_struct *work)
55 {
56 	struct dim *dim = container_of(work, struct dim, work);
57 	struct dim_cq_moder cur_moder;
58 	struct ionic_intr_info *intr;
59 	struct ionic_qcq *qcq;
60 	struct ionic_lif *lif;
61 	struct ionic_queue *q;
62 	u32 new_coal;
63 
64 	qcq = container_of(dim, struct ionic_qcq, dim);
65 	q = &qcq->q;
66 	if (q->type == IONIC_QTYPE_RXQ)
67 		cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
68 	else
69 		cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix);
70 	lif = q->lif;
71 	new_coal = ionic_coal_usec_to_hw(lif->ionic, cur_moder.usec);
72 	new_coal = new_coal ? new_coal : 1;
73 
74 	intr = &qcq->intr;
75 	if (intr->dim_coal_hw != new_coal) {
76 		intr->dim_coal_hw = new_coal;
77 
78 		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
79 				     intr->index, intr->dim_coal_hw);
80 	}
81 
82 	dim->state = DIM_START_MEASURE;
83 }
84 
ionic_lif_deferred_work(struct work_struct * work)85 static void ionic_lif_deferred_work(struct work_struct *work)
86 {
87 	struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
88 	struct ionic_deferred *def = &lif->deferred;
89 	struct ionic_deferred_work *w = NULL;
90 
91 	do {
92 		spin_lock_bh(&def->lock);
93 		if (!list_empty(&def->list)) {
94 			w = list_first_entry(&def->list,
95 					     struct ionic_deferred_work, list);
96 			list_del(&w->list);
97 		}
98 		spin_unlock_bh(&def->lock);
99 
100 		if (!w)
101 			break;
102 
103 		switch (w->type) {
104 		case IONIC_DW_TYPE_RX_MODE:
105 			ionic_lif_rx_mode(lif);
106 			break;
107 		case IONIC_DW_TYPE_LINK_STATUS:
108 			ionic_link_status_check(lif);
109 			break;
110 		case IONIC_DW_TYPE_LIF_RESET:
111 			if (w->fw_status) {
112 				ionic_lif_handle_fw_up(lif);
113 			} else {
114 				ionic_lif_handle_fw_down(lif);
115 
116 				/* Fire off another watchdog to see
117 				 * if the FW is already back rather than
118 				 * waiting another whole cycle
119 				 */
120 				mod_timer(&lif->ionic->watchdog_timer, jiffies + 1);
121 			}
122 			break;
123 		default:
124 			break;
125 		}
126 		kfree(w);
127 		w = NULL;
128 	} while (true);
129 }
130 
ionic_lif_deferred_enqueue(struct ionic_lif * lif,struct ionic_deferred_work * work)131 void ionic_lif_deferred_enqueue(struct ionic_lif *lif,
132 				struct ionic_deferred_work *work)
133 {
134 	spin_lock_bh(&lif->deferred.lock);
135 	list_add_tail(&work->list, &lif->deferred.list);
136 	spin_unlock_bh(&lif->deferred.lock);
137 	queue_work(lif->ionic->wq, &lif->deferred.work);
138 }
139 
ionic_link_status_check(struct ionic_lif * lif)140 static void ionic_link_status_check(struct ionic_lif *lif)
141 {
142 	struct net_device *netdev = lif->netdev;
143 	u16 link_status;
144 	bool link_up;
145 
146 	if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
147 		return;
148 
149 	/* Don't put carrier back up if we're in a broken state */
150 	if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
151 		clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
152 		return;
153 	}
154 
155 	link_status = le16_to_cpu(lif->info->status.link_status);
156 	link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
157 
158 	if (link_up) {
159 		int err = 0;
160 
161 		if (netdev->flags & IFF_UP && netif_running(netdev)) {
162 			mutex_lock(&lif->queue_lock);
163 			err = ionic_start_queues(lif);
164 			if (err && err != -EBUSY) {
165 				netdev_err(netdev,
166 					   "Failed to start queues: %d\n", err);
167 				set_bit(IONIC_LIF_F_BROKEN, lif->state);
168 				netif_carrier_off(lif->netdev);
169 			}
170 			mutex_unlock(&lif->queue_lock);
171 		}
172 
173 		if (!err && !netif_carrier_ok(netdev)) {
174 			ionic_port_identify(lif->ionic);
175 			netdev_info(netdev, "Link up - %d Gbps\n",
176 				    le32_to_cpu(lif->info->status.link_speed) / 1000);
177 			netif_carrier_on(netdev);
178 		}
179 	} else {
180 		if (netif_carrier_ok(netdev)) {
181 			lif->link_down_count++;
182 			netdev_info(netdev, "Link down\n");
183 			netif_carrier_off(netdev);
184 		}
185 
186 		if (netdev->flags & IFF_UP && netif_running(netdev)) {
187 			mutex_lock(&lif->queue_lock);
188 			ionic_stop_queues(lif);
189 			mutex_unlock(&lif->queue_lock);
190 		}
191 	}
192 
193 	clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
194 }
195 
ionic_link_status_check_request(struct ionic_lif * lif,bool can_sleep)196 void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
197 {
198 	struct ionic_deferred_work *work;
199 
200 	/* we only need one request outstanding at a time */
201 	if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
202 		return;
203 
204 	if (!can_sleep) {
205 		work = kzalloc(sizeof(*work), GFP_ATOMIC);
206 		if (!work) {
207 			clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
208 			return;
209 		}
210 
211 		work->type = IONIC_DW_TYPE_LINK_STATUS;
212 		ionic_lif_deferred_enqueue(lif, work);
213 	} else {
214 		ionic_link_status_check(lif);
215 	}
216 }
217 
ionic_isr(int irq,void * data)218 static irqreturn_t ionic_isr(int irq, void *data)
219 {
220 	struct napi_struct *napi = data;
221 
222 	napi_schedule_irqoff(napi);
223 
224 	return IRQ_HANDLED;
225 }
226 
ionic_request_irq(struct ionic_lif * lif,struct ionic_qcq * qcq)227 static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
228 {
229 	struct ionic_intr_info *intr = &qcq->intr;
230 	struct device *dev = lif->ionic->dev;
231 	struct ionic_queue *q = &qcq->q;
232 	const char *name;
233 
234 	if (lif->registered)
235 		name = netdev_name(lif->netdev);
236 	else
237 		name = dev_name(dev);
238 
239 	snprintf(intr->name, sizeof(intr->name),
240 		 "%.5s-%.16s-%.8s", IONIC_DRV_NAME, name, q->name);
241 
242 	return devm_request_irq(dev, intr->vector, ionic_isr,
243 				0, intr->name, &qcq->napi);
244 }
245 
ionic_intr_alloc(struct ionic_lif * lif,struct ionic_intr_info * intr)246 static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
247 {
248 	struct ionic *ionic = lif->ionic;
249 	int index;
250 
251 	index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
252 	if (index == ionic->nintrs) {
253 		netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
254 			    __func__, index, ionic->nintrs);
255 		return -ENOSPC;
256 	}
257 
258 	set_bit(index, ionic->intrs);
259 	ionic_intr_init(&ionic->idev, intr, index);
260 
261 	return 0;
262 }
263 
ionic_intr_free(struct ionic * ionic,int index)264 static void ionic_intr_free(struct ionic *ionic, int index)
265 {
266 	if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
267 		clear_bit(index, ionic->intrs);
268 }
269 
ionic_irq_aff_notify(struct irq_affinity_notify * notify,const cpumask_t * mask)270 static void ionic_irq_aff_notify(struct irq_affinity_notify *notify,
271 				 const cpumask_t *mask)
272 {
273 	struct ionic_intr_info *intr = container_of(notify, struct ionic_intr_info, aff_notify);
274 
275 	cpumask_copy(*intr->affinity_mask, mask);
276 }
277 
ionic_irq_aff_release(struct kref __always_unused * ref)278 static void ionic_irq_aff_release(struct kref __always_unused *ref)
279 {
280 }
281 
ionic_qcq_enable(struct ionic_qcq * qcq)282 static int ionic_qcq_enable(struct ionic_qcq *qcq)
283 {
284 	struct ionic_queue *q = &qcq->q;
285 	struct ionic_lif *lif = q->lif;
286 	struct ionic_dev *idev;
287 	struct device *dev;
288 
289 	struct ionic_admin_ctx ctx = {
290 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
291 		.cmd.q_control = {
292 			.opcode = IONIC_CMD_Q_CONTROL,
293 			.lif_index = cpu_to_le16(lif->index),
294 			.type = q->type,
295 			.index = cpu_to_le32(q->index),
296 			.oper = IONIC_Q_ENABLE,
297 		},
298 	};
299 	int ret;
300 
301 	idev = &lif->ionic->idev;
302 	dev = lif->ionic->dev;
303 
304 	dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
305 		ctx.cmd.q_control.index, ctx.cmd.q_control.type);
306 
307 	if (qcq->flags & IONIC_QCQ_F_INTR)
308 		ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
309 
310 	ret = ionic_adminq_post_wait(lif, &ctx);
311 	if (ret)
312 		return ret;
313 
314 	if (qcq->flags & IONIC_QCQ_F_INTR) {
315 		napi_enable(&qcq->napi);
316 		irq_set_affinity_notifier(qcq->intr.vector,
317 					  &qcq->intr.aff_notify);
318 		irq_set_affinity_hint(qcq->intr.vector,
319 				      *qcq->intr.affinity_mask);
320 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
321 				IONIC_INTR_MASK_CLEAR);
322 	}
323 
324 	return 0;
325 }
326 
ionic_qcq_disable(struct ionic_lif * lif,struct ionic_qcq * qcq,int fw_err)327 static int ionic_qcq_disable(struct ionic_lif *lif, struct ionic_qcq *qcq, int fw_err)
328 {
329 	struct ionic_queue *q;
330 
331 	struct ionic_admin_ctx ctx = {
332 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
333 		.cmd.q_control = {
334 			.opcode = IONIC_CMD_Q_CONTROL,
335 			.oper = IONIC_Q_DISABLE,
336 		},
337 	};
338 
339 	if (!qcq) {
340 		netdev_err(lif->netdev, "%s: bad qcq\n", __func__);
341 		return -ENXIO;
342 	}
343 
344 	q = &qcq->q;
345 
346 	if (qcq->flags & IONIC_QCQ_F_INTR) {
347 		struct ionic_dev *idev = &lif->ionic->idev;
348 
349 		if (lif->doorbell_wa)
350 			cancel_work_sync(&qcq->doorbell_napi_work);
351 		cancel_work_sync(&qcq->dim.work);
352 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
353 				IONIC_INTR_MASK_SET);
354 		synchronize_irq(qcq->intr.vector);
355 		irq_set_affinity_notifier(qcq->intr.vector, NULL);
356 		irq_set_affinity_hint(qcq->intr.vector, NULL);
357 		napi_disable(&qcq->napi);
358 	}
359 
360 	/* If there was a previous fw communcation error, don't bother with
361 	 * sending the adminq command and just return the same error value.
362 	 */
363 	if (fw_err == -ETIMEDOUT || fw_err == -ENXIO)
364 		return fw_err;
365 
366 	ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
367 	ctx.cmd.q_control.type = q->type;
368 	ctx.cmd.q_control.index = cpu_to_le32(q->index);
369 	dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
370 		ctx.cmd.q_control.index, ctx.cmd.q_control.type);
371 
372 	return ionic_adminq_post_wait(lif, &ctx);
373 }
374 
ionic_lif_qcq_deinit(struct ionic_lif * lif,struct ionic_qcq * qcq)375 static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
376 {
377 	struct ionic_dev *idev = &lif->ionic->idev;
378 
379 	if (!qcq)
380 		return;
381 
382 	if (!(qcq->flags & IONIC_QCQ_F_INITED))
383 		return;
384 
385 	ionic_unregister_rxq_info(&qcq->q);
386 	if (qcq->flags & IONIC_QCQ_F_INTR) {
387 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
388 				IONIC_INTR_MASK_SET);
389 		netif_napi_del(&qcq->napi);
390 	}
391 
392 	qcq->flags &= ~IONIC_QCQ_F_INITED;
393 }
394 
ionic_qcq_intr_free(struct ionic_lif * lif,struct ionic_qcq * qcq)395 static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
396 {
397 	if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
398 		return;
399 
400 	irq_set_affinity_hint(qcq->intr.vector, NULL);
401 	devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
402 	qcq->intr.vector = 0;
403 	ionic_intr_free(lif->ionic, qcq->intr.index);
404 	qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
405 }
406 
ionic_qcq_free(struct ionic_lif * lif,struct ionic_qcq * qcq)407 static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
408 {
409 	struct device *dev = lif->ionic->dev;
410 
411 	if (!qcq)
412 		return;
413 
414 	ionic_debugfs_del_qcq(qcq);
415 
416 	if (qcq->q_base) {
417 		dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
418 		qcq->q_base = NULL;
419 		qcq->q_base_pa = 0;
420 	}
421 
422 	if (qcq->cmb_q_base) {
423 		iounmap(qcq->cmb_q_base);
424 		ionic_put_cmb(lif, qcq->cmb_pgid, qcq->cmb_order);
425 		qcq->cmb_pgid = 0;
426 		qcq->cmb_order = 0;
427 		qcq->cmb_q_base = NULL;
428 		qcq->cmb_q_base_pa = 0;
429 	}
430 
431 	if (qcq->cq_base) {
432 		dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
433 		qcq->cq_base = NULL;
434 		qcq->cq_base_pa = 0;
435 	}
436 
437 	if (qcq->sg_base) {
438 		dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
439 		qcq->sg_base = NULL;
440 		qcq->sg_base_pa = 0;
441 	}
442 
443 	page_pool_destroy(qcq->q.page_pool);
444 	qcq->q.page_pool = NULL;
445 
446 	ionic_qcq_intr_free(lif, qcq);
447 	vfree(qcq->q.info);
448 	qcq->q.info = NULL;
449 }
450 
ionic_qcqs_free(struct ionic_lif * lif)451 void ionic_qcqs_free(struct ionic_lif *lif)
452 {
453 	struct device *dev = lif->ionic->dev;
454 	struct ionic_qcq *adminqcq;
455 	unsigned long irqflags;
456 
457 	if (lif->notifyqcq) {
458 		ionic_qcq_free(lif, lif->notifyqcq);
459 		devm_kfree(dev, lif->notifyqcq);
460 		lif->notifyqcq = NULL;
461 	}
462 
463 	if (lif->adminqcq) {
464 		spin_lock_irqsave(&lif->adminq_lock, irqflags);
465 		adminqcq = READ_ONCE(lif->adminqcq);
466 		lif->adminqcq = NULL;
467 		spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
468 		if (adminqcq) {
469 			ionic_qcq_free(lif, adminqcq);
470 			devm_kfree(dev, adminqcq);
471 		}
472 	}
473 
474 	if (lif->rxqcqs) {
475 		devm_kfree(dev, lif->rxqstats);
476 		lif->rxqstats = NULL;
477 		devm_kfree(dev, lif->rxqcqs);
478 		lif->rxqcqs = NULL;
479 	}
480 
481 	if (lif->txqcqs) {
482 		devm_kfree(dev, lif->txqstats);
483 		lif->txqstats = NULL;
484 		devm_kfree(dev, lif->txqcqs);
485 		lif->txqcqs = NULL;
486 	}
487 }
488 
ionic_link_qcq_interrupts(struct ionic_qcq * src_qcq,struct ionic_qcq * n_qcq)489 static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
490 				      struct ionic_qcq *n_qcq)
491 {
492 	n_qcq->intr.vector = src_qcq->intr.vector;
493 	n_qcq->intr.index = src_qcq->intr.index;
494 }
495 
ionic_alloc_qcq_interrupt(struct ionic_lif * lif,struct ionic_qcq * qcq)496 static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
497 {
498 	cpumask_var_t *affinity_mask;
499 	int err;
500 
501 	if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
502 		qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
503 		return 0;
504 	}
505 
506 	err = ionic_intr_alloc(lif, &qcq->intr);
507 	if (err) {
508 		netdev_warn(lif->netdev, "no intr for %s: %d\n",
509 			    qcq->q.name, err);
510 		goto err_out;
511 	}
512 
513 	err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
514 	if (err < 0) {
515 		netdev_warn(lif->netdev, "no vector for %s: %d\n",
516 			    qcq->q.name, err);
517 		goto err_out_free_intr;
518 	}
519 	qcq->intr.vector = err;
520 	ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
521 			       IONIC_INTR_MASK_SET);
522 
523 	err = ionic_request_irq(lif, qcq);
524 	if (err) {
525 		netdev_warn(lif->netdev, "irq request failed %d\n", err);
526 		goto err_out_free_intr;
527 	}
528 
529 	/* try to get the irq on the local numa node first */
530 	affinity_mask = &lif->ionic->affinity_masks[qcq->intr.index];
531 	if (cpumask_empty(*affinity_mask)) {
532 		unsigned int cpu;
533 
534 		cpu = cpumask_local_spread(qcq->intr.index,
535 					   dev_to_node(lif->ionic->dev));
536 		if (cpu != -1)
537 			cpumask_set_cpu(cpu, *affinity_mask);
538 	}
539 
540 	qcq->intr.affinity_mask = affinity_mask;
541 	qcq->intr.aff_notify.notify = ionic_irq_aff_notify;
542 	qcq->intr.aff_notify.release = ionic_irq_aff_release;
543 
544 	netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
545 	return 0;
546 
547 err_out_free_intr:
548 	ionic_intr_free(lif->ionic, qcq->intr.index);
549 err_out:
550 	return err;
551 }
552 
ionic_qcq_alloc(struct ionic_lif * lif,unsigned int type,unsigned int index,const char * name,unsigned int flags,unsigned int num_descs,unsigned int desc_size,unsigned int cq_desc_size,unsigned int sg_desc_size,unsigned int desc_info_size,unsigned int pid,struct bpf_prog * xdp_prog,struct ionic_qcq ** qcq)553 static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
554 			   unsigned int index,
555 			   const char *name, unsigned int flags,
556 			   unsigned int num_descs, unsigned int desc_size,
557 			   unsigned int cq_desc_size,
558 			   unsigned int sg_desc_size,
559 			   unsigned int desc_info_size,
560 			   unsigned int pid, struct bpf_prog *xdp_prog,
561 			   struct ionic_qcq **qcq)
562 {
563 	struct ionic_dev *idev = &lif->ionic->idev;
564 	struct device *dev = lif->ionic->dev;
565 	struct ionic_qcq *new;
566 	int err;
567 
568 	*qcq = NULL;
569 
570 	new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
571 	if (!new) {
572 		netdev_err(lif->netdev, "Cannot allocate queue structure\n");
573 		err = -ENOMEM;
574 		goto err_out;
575 	}
576 
577 	new->q.dev = dev;
578 	new->flags = flags;
579 
580 	new->q.info = vcalloc(num_descs, desc_info_size);
581 	if (!new->q.info) {
582 		netdev_err(lif->netdev, "Cannot allocate queue info\n");
583 		err = -ENOMEM;
584 		goto err_out_free_qcq;
585 	}
586 
587 	if (type == IONIC_QTYPE_RXQ) {
588 		struct page_pool_params pp_params = {
589 			.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
590 			.order = 0,
591 			.pool_size = num_descs,
592 			.nid = NUMA_NO_NODE,
593 			.dev = lif->ionic->dev,
594 			.napi = &new->napi,
595 			.dma_dir = DMA_FROM_DEVICE,
596 			.max_len = PAGE_SIZE,
597 			.netdev = lif->netdev,
598 		};
599 
600 		if (xdp_prog)
601 			pp_params.dma_dir = DMA_BIDIRECTIONAL;
602 
603 		new->q.page_pool = page_pool_create(&pp_params);
604 		if (IS_ERR(new->q.page_pool)) {
605 			netdev_err(lif->netdev, "Cannot create page_pool\n");
606 			err = PTR_ERR(new->q.page_pool);
607 			new->q.page_pool = NULL;
608 			goto err_out_free_q_info;
609 		}
610 	}
611 
612 	new->q.type = type;
613 	new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
614 
615 	err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
616 			   desc_size, sg_desc_size, pid);
617 	if (err) {
618 		netdev_err(lif->netdev, "Cannot initialize queue\n");
619 		goto err_out_free_page_pool;
620 	}
621 
622 	err = ionic_alloc_qcq_interrupt(lif, new);
623 	if (err)
624 		goto err_out_free_page_pool;
625 
626 	err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
627 	if (err) {
628 		netdev_err(lif->netdev, "Cannot initialize completion queue\n");
629 		goto err_out_free_irq;
630 	}
631 
632 	if (flags & IONIC_QCQ_F_NOTIFYQ) {
633 		int q_size;
634 
635 		/* q & cq need to be contiguous in NotifyQ, so alloc it all in q
636 		 * and don't alloc qc.  We leave new->qc_size and new->qc_base
637 		 * as 0 to be sure we don't try to free it later.
638 		 */
639 		q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
640 		new->q_size = PAGE_SIZE + q_size +
641 			      ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
642 		new->q_base = dma_alloc_coherent(dev, new->q_size,
643 						 &new->q_base_pa, GFP_KERNEL);
644 		if (!new->q_base) {
645 			netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
646 			err = -ENOMEM;
647 			goto err_out_free_irq;
648 		}
649 		new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
650 		new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
651 
652 		/* Base the NotifyQ cq.base off of the ALIGNed q.base */
653 		new->cq.base = PTR_ALIGN(new->q.base + q_size, PAGE_SIZE);
654 		new->cq.base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
655 		new->cq.bound_q = &new->q;
656 	} else {
657 		/* regular DMA q descriptors */
658 		new->q_size = PAGE_SIZE + (num_descs * desc_size);
659 		new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
660 						 GFP_KERNEL);
661 		if (!new->q_base) {
662 			netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
663 			err = -ENOMEM;
664 			goto err_out_free_irq;
665 		}
666 		new->q.base = PTR_ALIGN(new->q_base, PAGE_SIZE);
667 		new->q.base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
668 
669 		if (flags & IONIC_QCQ_F_CMB_RINGS) {
670 			/* on-chip CMB q descriptors */
671 			new->cmb_q_size = num_descs * desc_size;
672 			new->cmb_order = order_base_2(new->cmb_q_size / PAGE_SIZE);
673 
674 			err = ionic_get_cmb(lif, &new->cmb_pgid, &new->cmb_q_base_pa,
675 					    new->cmb_order);
676 			if (err) {
677 				netdev_err(lif->netdev,
678 					   "Cannot allocate queue order %d from cmb: err %d\n",
679 					   new->cmb_order, err);
680 				goto err_out_free_q;
681 			}
682 
683 			new->cmb_q_base = ioremap_wc(new->cmb_q_base_pa, new->cmb_q_size);
684 			if (!new->cmb_q_base) {
685 				netdev_err(lif->netdev, "Cannot map queue from cmb\n");
686 				ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
687 				err = -ENOMEM;
688 				goto err_out_free_q;
689 			}
690 
691 			new->cmb_q_base_pa -= idev->phy_cmb_pages;
692 			new->q.cmb_base = new->cmb_q_base;
693 			new->q.cmb_base_pa = new->cmb_q_base_pa;
694 		}
695 
696 		/* cq DMA descriptors */
697 		new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
698 		new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
699 						  GFP_KERNEL);
700 		if (!new->cq_base) {
701 			netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
702 			err = -ENOMEM;
703 			goto err_out_free_q;
704 		}
705 		new->cq.base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
706 		new->cq.base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
707 		new->cq.bound_q = &new->q;
708 	}
709 
710 	if (flags & IONIC_QCQ_F_SG) {
711 		new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
712 		new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
713 						  GFP_KERNEL);
714 		if (!new->sg_base) {
715 			netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
716 			err = -ENOMEM;
717 			goto err_out_free_cq;
718 		}
719 		new->q.sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
720 		new->q.sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
721 	}
722 
723 	INIT_WORK(&new->dim.work, ionic_dim_work);
724 	new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE;
725 	if (lif->doorbell_wa)
726 		INIT_WORK(&new->doorbell_napi_work, ionic_doorbell_napi_work);
727 
728 	*qcq = new;
729 
730 	return 0;
731 
732 err_out_free_cq:
733 	dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
734 err_out_free_q:
735 	if (new->cmb_q_base) {
736 		iounmap(new->cmb_q_base);
737 		ionic_put_cmb(lif, new->cmb_pgid, new->cmb_order);
738 	}
739 	dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
740 err_out_free_irq:
741 	if (flags & IONIC_QCQ_F_INTR) {
742 		devm_free_irq(dev, new->intr.vector, &new->napi);
743 		ionic_intr_free(lif->ionic, new->intr.index);
744 	}
745 err_out_free_page_pool:
746 	page_pool_destroy(new->q.page_pool);
747 err_out_free_q_info:
748 	vfree(new->q.info);
749 err_out_free_qcq:
750 	devm_kfree(dev, new);
751 err_out:
752 	dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
753 	return err;
754 }
755 
ionic_qcqs_alloc(struct ionic_lif * lif)756 static int ionic_qcqs_alloc(struct ionic_lif *lif)
757 {
758 	struct device *dev = lif->ionic->dev;
759 	unsigned int flags;
760 	int err;
761 
762 	flags = IONIC_QCQ_F_INTR;
763 	err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
764 			      IONIC_ADMINQ_LENGTH,
765 			      sizeof(struct ionic_admin_cmd),
766 			      sizeof(struct ionic_admin_comp),
767 			      0,
768 			      sizeof(struct ionic_admin_desc_info),
769 			      lif->kern_pid, NULL, &lif->adminqcq);
770 	if (err)
771 		return err;
772 	ionic_debugfs_add_qcq(lif, lif->adminqcq);
773 
774 	if (lif->ionic->nnqs_per_lif) {
775 		flags = IONIC_QCQ_F_NOTIFYQ;
776 		err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
777 				      flags, IONIC_NOTIFYQ_LENGTH,
778 				      sizeof(struct ionic_notifyq_cmd),
779 				      sizeof(union ionic_notifyq_comp),
780 				      0,
781 				      sizeof(struct ionic_admin_desc_info),
782 				      lif->kern_pid, NULL, &lif->notifyqcq);
783 		if (err)
784 			goto err_out;
785 		ionic_debugfs_add_qcq(lif, lif->notifyqcq);
786 
787 		/* Let the notifyq ride on the adminq interrupt */
788 		ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
789 	}
790 
791 	err = -ENOMEM;
792 	lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
793 				   sizeof(*lif->txqcqs), GFP_KERNEL);
794 	if (!lif->txqcqs)
795 		goto err_out;
796 	lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
797 				   sizeof(*lif->rxqcqs), GFP_KERNEL);
798 	if (!lif->rxqcqs)
799 		goto err_out;
800 
801 	lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
802 				     sizeof(*lif->txqstats), GFP_KERNEL);
803 	if (!lif->txqstats)
804 		goto err_out;
805 	lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
806 				     sizeof(*lif->rxqstats), GFP_KERNEL);
807 	if (!lif->rxqstats)
808 		goto err_out;
809 
810 	return 0;
811 
812 err_out:
813 	ionic_qcqs_free(lif);
814 	return err;
815 }
816 
ionic_qcq_sanitize(struct ionic_qcq * qcq)817 static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
818 {
819 	qcq->q.tail_idx = 0;
820 	qcq->q.head_idx = 0;
821 	qcq->cq.tail_idx = 0;
822 	qcq->cq.done_color = 1;
823 	memset(qcq->q_base, 0, qcq->q_size);
824 	if (qcq->cmb_q_base)
825 		memset_io(qcq->cmb_q_base, 0, qcq->cmb_q_size);
826 	memset(qcq->cq_base, 0, qcq->cq_size);
827 	memset(qcq->sg_base, 0, qcq->sg_size);
828 }
829 
ionic_lif_txq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)830 static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
831 {
832 	struct device *dev = lif->ionic->dev;
833 	struct ionic_queue *q = &qcq->q;
834 	struct ionic_cq *cq = &qcq->cq;
835 	struct ionic_admin_ctx ctx = {
836 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
837 		.cmd.q_init = {
838 			.opcode = IONIC_CMD_Q_INIT,
839 			.lif_index = cpu_to_le16(lif->index),
840 			.type = q->type,
841 			.ver = lif->qtype_info[q->type].version,
842 			.index = cpu_to_le32(q->index),
843 			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
844 					     IONIC_QINIT_F_SG),
845 			.intr_index = cpu_to_le16(qcq->intr.index),
846 			.pid = cpu_to_le16(q->pid),
847 			.ring_size = ilog2(q->num_descs),
848 			.ring_base = cpu_to_le64(q->base_pa),
849 			.cq_ring_base = cpu_to_le64(cq->base_pa),
850 			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
851 			.features = cpu_to_le64(q->features),
852 		},
853 	};
854 	int err;
855 
856 	if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
857 		ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
858 		ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
859 	}
860 
861 	dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
862 	dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
863 	dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
864 	dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
865 	dev_dbg(dev, "txq_init.cq_ring_base 0x%llx\n", ctx.cmd.q_init.cq_ring_base);
866 	dev_dbg(dev, "txq_init.sg_ring_base 0x%llx\n", ctx.cmd.q_init.sg_ring_base);
867 	dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
868 	dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
869 	dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
870 
871 	ionic_qcq_sanitize(qcq);
872 
873 	err = ionic_adminq_post_wait(lif, &ctx);
874 	if (err)
875 		return err;
876 
877 	q->hw_type = ctx.comp.q_init.hw_type;
878 	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
879 	q->dbval = IONIC_DBELL_QID(q->hw_index);
880 
881 	dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
882 	dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
883 
884 	q->dbell_deadline = IONIC_TX_DOORBELL_DEADLINE;
885 	q->dbell_jiffies = jiffies;
886 
887 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
888 		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi);
889 
890 	qcq->flags |= IONIC_QCQ_F_INITED;
891 
892 	return 0;
893 }
894 
ionic_lif_rxq_init(struct ionic_lif * lif,struct ionic_qcq * qcq)895 static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
896 {
897 	struct device *dev = lif->ionic->dev;
898 	struct ionic_queue *q = &qcq->q;
899 	struct ionic_cq *cq = &qcq->cq;
900 	struct ionic_admin_ctx ctx = {
901 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
902 		.cmd.q_init = {
903 			.opcode = IONIC_CMD_Q_INIT,
904 			.lif_index = cpu_to_le16(lif->index),
905 			.type = q->type,
906 			.ver = lif->qtype_info[q->type].version,
907 			.index = cpu_to_le32(q->index),
908 			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ),
909 			.intr_index = cpu_to_le16(cq->bound_intr->index),
910 			.pid = cpu_to_le16(q->pid),
911 			.ring_size = ilog2(q->num_descs),
912 			.ring_base = cpu_to_le64(q->base_pa),
913 			.cq_ring_base = cpu_to_le64(cq->base_pa),
914 			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
915 			.features = cpu_to_le64(q->features),
916 		},
917 	};
918 	int err;
919 
920 	q->partner = &lif->txqcqs[q->index]->q;
921 	q->partner->partner = q;
922 
923 	if (!lif->xdp_prog ||
924 	    (lif->xdp_prog->aux && lif->xdp_prog->aux->xdp_has_frags))
925 		ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_SG);
926 
927 	if (qcq->flags & IONIC_QCQ_F_CMB_RINGS) {
928 		ctx.cmd.q_init.flags |= cpu_to_le16(IONIC_QINIT_F_CMB);
929 		ctx.cmd.q_init.ring_base = cpu_to_le64(qcq->cmb_q_base_pa);
930 	}
931 
932 	dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
933 	dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
934 	dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
935 	dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
936 	dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
937 	dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
938 	dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
939 
940 	ionic_qcq_sanitize(qcq);
941 
942 	err = ionic_adminq_post_wait(lif, &ctx);
943 	if (err)
944 		return err;
945 
946 	q->hw_type = ctx.comp.q_init.hw_type;
947 	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
948 	q->dbval = IONIC_DBELL_QID(q->hw_index);
949 
950 	dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
951 	dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
952 
953 	q->dbell_deadline = IONIC_RX_MIN_DOORBELL_DEADLINE;
954 	q->dbell_jiffies = jiffies;
955 
956 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
957 		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi);
958 	else
959 		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi);
960 	err = ionic_register_rxq_info(q, qcq->napi.napi_id);
961 	if (err) {
962 		netif_napi_del(&qcq->napi);
963 		return err;
964 	}
965 
966 	qcq->flags |= IONIC_QCQ_F_INITED;
967 
968 	return 0;
969 }
970 
ionic_lif_create_hwstamp_txq(struct ionic_lif * lif)971 int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
972 {
973 	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
974 	unsigned int txq_i, flags;
975 	struct ionic_qcq *txq;
976 	u64 features;
977 	int err;
978 
979 	if (lif->hwstamp_txq)
980 		return 0;
981 
982 	features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
983 
984 	num_desc = IONIC_MIN_TXRX_DESC;
985 	desc_sz = sizeof(struct ionic_txq_desc);
986 	comp_sz = 2 * sizeof(struct ionic_txq_comp);
987 
988 	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
989 	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
990 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
991 	else
992 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
993 
994 	txq_i = lif->ionic->ntxqs_per_lif;
995 	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
996 
997 	err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
998 			      num_desc, desc_sz, comp_sz, sg_desc_sz,
999 			      sizeof(struct ionic_tx_desc_info),
1000 			      lif->kern_pid, NULL, &txq);
1001 	if (err)
1002 		goto err_qcq_alloc;
1003 
1004 	txq->q.features = features;
1005 
1006 	ionic_link_qcq_interrupts(lif->adminqcq, txq);
1007 	ionic_debugfs_add_qcq(lif, txq);
1008 
1009 	lif->hwstamp_txq = txq;
1010 
1011 	if (netif_running(lif->netdev)) {
1012 		err = ionic_lif_txq_init(lif, txq);
1013 		if (err)
1014 			goto err_qcq_init;
1015 
1016 		if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1017 			err = ionic_qcq_enable(txq);
1018 			if (err)
1019 				goto err_qcq_enable;
1020 		}
1021 	}
1022 
1023 	return 0;
1024 
1025 err_qcq_enable:
1026 	ionic_lif_qcq_deinit(lif, txq);
1027 err_qcq_init:
1028 	lif->hwstamp_txq = NULL;
1029 	ionic_debugfs_del_qcq(txq);
1030 	ionic_qcq_free(lif, txq);
1031 	devm_kfree(lif->ionic->dev, txq);
1032 err_qcq_alloc:
1033 	return err;
1034 }
1035 
ionic_lif_create_hwstamp_rxq(struct ionic_lif * lif)1036 int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
1037 {
1038 	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
1039 	unsigned int rxq_i, flags;
1040 	struct ionic_qcq *rxq;
1041 	u64 features;
1042 	int err;
1043 
1044 	if (lif->hwstamp_rxq)
1045 		return 0;
1046 
1047 	features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1048 
1049 	num_desc = IONIC_MIN_TXRX_DESC;
1050 	desc_sz = sizeof(struct ionic_rxq_desc);
1051 	comp_sz = 2 * sizeof(struct ionic_rxq_comp);
1052 	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
1053 
1054 	rxq_i = lif->ionic->nrxqs_per_lif;
1055 	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
1056 
1057 	err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
1058 			      num_desc, desc_sz, comp_sz, sg_desc_sz,
1059 			      sizeof(struct ionic_rx_desc_info),
1060 			      lif->kern_pid, NULL, &rxq);
1061 	if (err)
1062 		goto err_qcq_alloc;
1063 
1064 	rxq->q.features = features;
1065 
1066 	ionic_link_qcq_interrupts(lif->adminqcq, rxq);
1067 	ionic_debugfs_add_qcq(lif, rxq);
1068 
1069 	lif->hwstamp_rxq = rxq;
1070 
1071 	if (netif_running(lif->netdev)) {
1072 		err = ionic_lif_rxq_init(lif, rxq);
1073 		if (err)
1074 			goto err_qcq_init;
1075 
1076 		if (test_bit(IONIC_LIF_F_UP, lif->state)) {
1077 			ionic_rx_fill(&rxq->q, NULL);
1078 			err = ionic_qcq_enable(rxq);
1079 			if (err)
1080 				goto err_qcq_enable;
1081 		}
1082 	}
1083 
1084 	return 0;
1085 
1086 err_qcq_enable:
1087 	ionic_lif_qcq_deinit(lif, rxq);
1088 err_qcq_init:
1089 	lif->hwstamp_rxq = NULL;
1090 	ionic_debugfs_del_qcq(rxq);
1091 	ionic_qcq_free(lif, rxq);
1092 	devm_kfree(lif->ionic->dev, rxq);
1093 err_qcq_alloc:
1094 	return err;
1095 }
1096 
ionic_lif_config_hwstamp_rxq_all(struct ionic_lif * lif,bool rx_all)1097 int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
1098 {
1099 	struct ionic_queue_params qparam;
1100 
1101 	ionic_init_queue_params(lif, &qparam);
1102 
1103 	if (rx_all)
1104 		qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
1105 	else
1106 		qparam.rxq_features = 0;
1107 
1108 	/* if we're not running, just set the values and return */
1109 	if (!netif_running(lif->netdev)) {
1110 		lif->rxq_features = qparam.rxq_features;
1111 		return 0;
1112 	}
1113 
1114 	return ionic_reconfigure_queues(lif, &qparam);
1115 }
1116 
ionic_lif_set_hwstamp_txmode(struct ionic_lif * lif,u16 txstamp_mode)1117 int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1118 {
1119 	struct ionic_admin_ctx ctx = {
1120 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1121 		.cmd.lif_setattr = {
1122 			.opcode = IONIC_CMD_LIF_SETATTR,
1123 			.index = cpu_to_le16(lif->index),
1124 			.attr = IONIC_LIF_ATTR_TXSTAMP,
1125 			.txstamp_mode = cpu_to_le16(txstamp_mode),
1126 		},
1127 	};
1128 
1129 	return ionic_adminq_post_wait(lif, &ctx);
1130 }
1131 
ionic_lif_del_hwstamp_rxfilt(struct ionic_lif * lif)1132 static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1133 {
1134 	struct ionic_admin_ctx ctx = {
1135 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1136 		.cmd.rx_filter_del = {
1137 			.opcode = IONIC_CMD_RX_FILTER_DEL,
1138 			.lif_index = cpu_to_le16(lif->index),
1139 		},
1140 	};
1141 	struct ionic_rx_filter *f;
1142 	u32 filter_id;
1143 	int err;
1144 
1145 	spin_lock_bh(&lif->rx_filters.lock);
1146 
1147 	f = ionic_rx_filter_rxsteer(lif);
1148 	if (!f) {
1149 		spin_unlock_bh(&lif->rx_filters.lock);
1150 		return;
1151 	}
1152 
1153 	filter_id = f->filter_id;
1154 	ionic_rx_filter_free(lif, f);
1155 
1156 	spin_unlock_bh(&lif->rx_filters.lock);
1157 
1158 	netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1159 
1160 	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1161 
1162 	err = ionic_adminq_post_wait(lif, &ctx);
1163 	if (err && err != -EEXIST)
1164 		netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1165 }
1166 
ionic_lif_add_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1167 static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1168 {
1169 	struct ionic_admin_ctx ctx = {
1170 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1171 		.cmd.rx_filter_add = {
1172 			.opcode = IONIC_CMD_RX_FILTER_ADD,
1173 			.lif_index = cpu_to_le16(lif->index),
1174 			.match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1175 			.pkt_class = cpu_to_le64(pkt_class),
1176 		},
1177 	};
1178 	u8 qtype;
1179 	u32 qid;
1180 	int err;
1181 
1182 	if (!lif->hwstamp_rxq)
1183 		return -EINVAL;
1184 
1185 	qtype = lif->hwstamp_rxq->q.type;
1186 	ctx.cmd.rx_filter_add.qtype = qtype;
1187 
1188 	qid = lif->hwstamp_rxq->q.index;
1189 	ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1190 
1191 	netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1192 	err = ionic_adminq_post_wait(lif, &ctx);
1193 	if (err && err != -EEXIST)
1194 		return err;
1195 
1196 	spin_lock_bh(&lif->rx_filters.lock);
1197 	err = ionic_rx_filter_save(lif, 0, qid, 0, &ctx, IONIC_FILTER_STATE_SYNCED);
1198 	spin_unlock_bh(&lif->rx_filters.lock);
1199 
1200 	return err;
1201 }
1202 
ionic_lif_set_hwstamp_rxfilt(struct ionic_lif * lif,u64 pkt_class)1203 int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1204 {
1205 	ionic_lif_del_hwstamp_rxfilt(lif);
1206 
1207 	if (!pkt_class)
1208 		return 0;
1209 
1210 	return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1211 }
1212 
ionic_adminq_napi(struct napi_struct * napi,int budget)1213 static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1214 {
1215 	struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1216 	struct ionic_lif *lif = napi_to_cq(napi)->lif;
1217 	struct ionic_dev *idev = &lif->ionic->idev;
1218 	unsigned long irqflags;
1219 	unsigned int flags = 0;
1220 	int rx_work = 0;
1221 	int tx_work = 0;
1222 	int n_work = 0;
1223 	int a_work = 0;
1224 	int work_done;
1225 	int credits;
1226 
1227 	if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1228 		n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1229 					  ionic_notifyq_service, NULL, NULL);
1230 
1231 	spin_lock_irqsave(&lif->adminq_lock, irqflags);
1232 	if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1233 		a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1234 					  ionic_adminq_service, NULL, NULL);
1235 
1236 	spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1237 
1238 	if (lif->hwstamp_rxq)
1239 		rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1240 					   ionic_rx_service, NULL, NULL);
1241 
1242 	if (lif->hwstamp_txq)
1243 		tx_work = ionic_tx_cq_service(&lif->hwstamp_txq->cq, budget, !!budget);
1244 
1245 	work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1246 	if (work_done < budget && napi_complete_done(napi, work_done)) {
1247 		flags |= IONIC_INTR_CRED_UNMASK;
1248 		intr->rearm_count++;
1249 	}
1250 
1251 	if (work_done || flags) {
1252 		flags |= IONIC_INTR_CRED_RESET_COALESCE;
1253 		credits = n_work + a_work + rx_work + tx_work;
1254 		ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1255 	}
1256 
1257 	if (lif->doorbell_wa) {
1258 		if (!a_work)
1259 			ionic_adminq_poke_doorbell(&lif->adminqcq->q);
1260 		if (lif->hwstamp_rxq && !rx_work)
1261 			ionic_rxq_poke_doorbell(&lif->hwstamp_rxq->q);
1262 		if (lif->hwstamp_txq && !tx_work)
1263 			ionic_txq_poke_doorbell(&lif->hwstamp_txq->q);
1264 	}
1265 
1266 	return work_done;
1267 }
1268 
ionic_get_stats64(struct net_device * netdev,struct rtnl_link_stats64 * ns)1269 void ionic_get_stats64(struct net_device *netdev,
1270 		       struct rtnl_link_stats64 *ns)
1271 {
1272 	struct ionic_lif *lif = netdev_priv(netdev);
1273 	struct ionic_lif_stats *ls;
1274 
1275 	memset(ns, 0, sizeof(*ns));
1276 	ls = &lif->info->stats;
1277 
1278 	ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1279 			 le64_to_cpu(ls->rx_mcast_packets) +
1280 			 le64_to_cpu(ls->rx_bcast_packets);
1281 
1282 	ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1283 			 le64_to_cpu(ls->tx_mcast_packets) +
1284 			 le64_to_cpu(ls->tx_bcast_packets);
1285 
1286 	ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1287 		       le64_to_cpu(ls->rx_mcast_bytes) +
1288 		       le64_to_cpu(ls->rx_bcast_bytes);
1289 
1290 	ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1291 		       le64_to_cpu(ls->tx_mcast_bytes) +
1292 		       le64_to_cpu(ls->tx_bcast_bytes);
1293 
1294 	ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1295 			 le64_to_cpu(ls->rx_mcast_drop_packets) +
1296 			 le64_to_cpu(ls->rx_bcast_drop_packets);
1297 
1298 	ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1299 			 le64_to_cpu(ls->tx_mcast_drop_packets) +
1300 			 le64_to_cpu(ls->tx_bcast_drop_packets);
1301 
1302 	ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1303 
1304 	ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1305 
1306 	ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1307 			       le64_to_cpu(ls->rx_queue_disabled) +
1308 			       le64_to_cpu(ls->rx_desc_fetch_error) +
1309 			       le64_to_cpu(ls->rx_desc_data_error);
1310 
1311 	ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1312 				le64_to_cpu(ls->tx_queue_disabled) +
1313 				le64_to_cpu(ls->tx_desc_fetch_error) +
1314 				le64_to_cpu(ls->tx_desc_data_error);
1315 
1316 	ns->rx_errors = ns->rx_over_errors +
1317 			ns->rx_missed_errors;
1318 
1319 	ns->tx_errors = ns->tx_aborted_errors;
1320 }
1321 
ionic_addr_add(struct net_device * netdev,const u8 * addr)1322 static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1323 {
1324 	return ionic_lif_list_addr(netdev_priv(netdev), addr, ADD_ADDR);
1325 }
1326 
ionic_addr_del(struct net_device * netdev,const u8 * addr)1327 static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1328 {
1329 	/* Don't delete our own address from the uc list */
1330 	if (ether_addr_equal(addr, netdev->dev_addr))
1331 		return 0;
1332 
1333 	return ionic_lif_list_addr(netdev_priv(netdev), addr, DEL_ADDR);
1334 }
1335 
ionic_lif_rx_mode(struct ionic_lif * lif)1336 void ionic_lif_rx_mode(struct ionic_lif *lif)
1337 {
1338 	struct net_device *netdev = lif->netdev;
1339 	unsigned int nfilters;
1340 	unsigned int nd_flags;
1341 	char buf[128];
1342 	u16 rx_mode;
1343 	int i;
1344 #define REMAIN(__x) (sizeof(buf) - (__x))
1345 
1346 	mutex_lock(&lif->config_lock);
1347 
1348 	/* grab the flags once for local use */
1349 	nd_flags = netdev->flags;
1350 
1351 	rx_mode = IONIC_RX_MODE_F_UNICAST;
1352 	rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1353 	rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1354 	rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1355 	rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1356 
1357 	/* sync the filters */
1358 	ionic_rx_filter_sync(lif);
1359 
1360 	/* check for overflow state
1361 	 *    if so, we track that we overflowed and enable NIC PROMISC
1362 	 *    else if the overflow is set and not needed
1363 	 *       we remove our overflow flag and check the netdev flags
1364 	 *       to see if we can disable NIC PROMISC
1365 	 */
1366 	nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1367 
1368 	if (((lif->nucast + lif->nmcast) >= nfilters) ||
1369 	    (lif->max_vlans && lif->nvlans >= lif->max_vlans)) {
1370 		rx_mode |= IONIC_RX_MODE_F_PROMISC;
1371 		rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1372 	} else {
1373 		if (!(nd_flags & IFF_PROMISC))
1374 			rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1375 		if (!(nd_flags & IFF_ALLMULTI))
1376 			rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1377 	}
1378 
1379 	i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1380 		      lif->rx_mode, rx_mode);
1381 	if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1382 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1383 	if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1384 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1385 	if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1386 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1387 	if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1388 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1389 	if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1390 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1391 	if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1392 		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1393 	netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1394 
1395 	if (lif->rx_mode != rx_mode) {
1396 		struct ionic_admin_ctx ctx = {
1397 			.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1398 			.cmd.rx_mode_set = {
1399 				.opcode = IONIC_CMD_RX_MODE_SET,
1400 				.lif_index = cpu_to_le16(lif->index),
1401 			},
1402 		};
1403 		int err;
1404 
1405 		ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1406 		err = ionic_adminq_post_wait(lif, &ctx);
1407 		if (err)
1408 			netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1409 				    rx_mode, err);
1410 		else
1411 			lif->rx_mode = rx_mode;
1412 	}
1413 
1414 	mutex_unlock(&lif->config_lock);
1415 }
1416 
ionic_ndo_set_rx_mode(struct net_device * netdev)1417 static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1418 {
1419 	struct ionic_lif *lif = netdev_priv(netdev);
1420 	struct ionic_deferred_work *work;
1421 
1422 	/* Sync the kernel filter list with the driver filter list */
1423 	__dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1424 	__dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1425 
1426 	/* Shove off the rest of the rxmode work to the work task
1427 	 * which will include syncing the filters to the firmware.
1428 	 */
1429 	work = kzalloc(sizeof(*work), GFP_ATOMIC);
1430 	if (!work) {
1431 		netdev_err(lif->netdev, "rxmode change dropped\n");
1432 		return;
1433 	}
1434 	work->type = IONIC_DW_TYPE_RX_MODE;
1435 	netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1436 	ionic_lif_deferred_enqueue(lif, work);
1437 }
1438 
ionic_netdev_features_to_nic(netdev_features_t features)1439 static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1440 {
1441 	u64 wanted = 0;
1442 
1443 	if (features & NETIF_F_HW_VLAN_CTAG_TX)
1444 		wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1445 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
1446 		wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1447 	if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1448 		wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1449 	if (features & NETIF_F_RXHASH)
1450 		wanted |= IONIC_ETH_HW_RX_HASH;
1451 	if (features & NETIF_F_RXCSUM)
1452 		wanted |= IONIC_ETH_HW_RX_CSUM;
1453 	if (features & NETIF_F_SG)
1454 		wanted |= IONIC_ETH_HW_TX_SG;
1455 	if (features & NETIF_F_HW_CSUM)
1456 		wanted |= IONIC_ETH_HW_TX_CSUM;
1457 	if (features & NETIF_F_TSO)
1458 		wanted |= IONIC_ETH_HW_TSO;
1459 	if (features & NETIF_F_TSO6)
1460 		wanted |= IONIC_ETH_HW_TSO_IPV6;
1461 	if (features & NETIF_F_TSO_ECN)
1462 		wanted |= IONIC_ETH_HW_TSO_ECN;
1463 	if (features & NETIF_F_GSO_GRE)
1464 		wanted |= IONIC_ETH_HW_TSO_GRE;
1465 	if (features & NETIF_F_GSO_GRE_CSUM)
1466 		wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1467 	if (features & NETIF_F_GSO_IPXIP4)
1468 		wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1469 	if (features & NETIF_F_GSO_IPXIP6)
1470 		wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1471 	if (features & NETIF_F_GSO_UDP_TUNNEL)
1472 		wanted |= IONIC_ETH_HW_TSO_UDP;
1473 	if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1474 		wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1475 
1476 	return cpu_to_le64(wanted);
1477 }
1478 
ionic_set_nic_features(struct ionic_lif * lif,netdev_features_t features)1479 static int ionic_set_nic_features(struct ionic_lif *lif,
1480 				  netdev_features_t features)
1481 {
1482 	struct device *dev = lif->ionic->dev;
1483 	struct ionic_admin_ctx ctx = {
1484 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1485 		.cmd.lif_setattr = {
1486 			.opcode = IONIC_CMD_LIF_SETATTR,
1487 			.index = cpu_to_le16(lif->index),
1488 			.attr = IONIC_LIF_ATTR_FEATURES,
1489 		},
1490 	};
1491 	u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1492 			 IONIC_ETH_HW_VLAN_RX_STRIP |
1493 			 IONIC_ETH_HW_VLAN_RX_FILTER;
1494 	u64 old_hw_features;
1495 	int err;
1496 
1497 	ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1498 
1499 	if (lif->phc)
1500 		ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1501 
1502 	err = ionic_adminq_post_wait(lif, &ctx);
1503 	if (err)
1504 		return err;
1505 
1506 	old_hw_features = lif->hw_features;
1507 	lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1508 				       ctx.comp.lif_setattr.features);
1509 
1510 	if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1511 		ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1512 
1513 	if ((vlan_flags & le64_to_cpu(ctx.cmd.lif_setattr.features)) &&
1514 	    !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1515 		dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1516 
1517 	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1518 		dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1519 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1520 		dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1521 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1522 		dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1523 	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1524 		dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1525 	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1526 		dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1527 	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1528 		dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1529 	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1530 		dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1531 	if (lif->hw_features & IONIC_ETH_HW_TSO)
1532 		dev_dbg(dev, "feature ETH_HW_TSO\n");
1533 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1534 		dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1535 	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1536 		dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1537 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1538 		dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1539 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1540 		dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1541 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1542 		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1543 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1544 		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1545 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1546 		dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1547 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1548 		dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1549 	if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1550 		dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1551 
1552 	return 0;
1553 }
1554 
ionic_init_nic_features(struct ionic_lif * lif)1555 static int ionic_init_nic_features(struct ionic_lif *lif)
1556 {
1557 	struct net_device *netdev = lif->netdev;
1558 	netdev_features_t features;
1559 	int err;
1560 
1561 	/* set up what we expect to support by default */
1562 	features = NETIF_F_HW_VLAN_CTAG_TX |
1563 		   NETIF_F_HW_VLAN_CTAG_RX |
1564 		   NETIF_F_HW_VLAN_CTAG_FILTER |
1565 		   NETIF_F_SG |
1566 		   NETIF_F_HW_CSUM |
1567 		   NETIF_F_RXCSUM |
1568 		   NETIF_F_TSO |
1569 		   NETIF_F_TSO6 |
1570 		   NETIF_F_TSO_ECN |
1571 		   NETIF_F_GSO_GRE |
1572 		   NETIF_F_GSO_GRE_CSUM |
1573 		   NETIF_F_GSO_IPXIP4 |
1574 		   NETIF_F_GSO_IPXIP6 |
1575 		   NETIF_F_GSO_UDP_TUNNEL |
1576 		   NETIF_F_GSO_UDP_TUNNEL_CSUM;
1577 
1578 	if (lif->nxqs > 1)
1579 		features |= NETIF_F_RXHASH;
1580 
1581 	err = ionic_set_nic_features(lif, features);
1582 	if (err)
1583 		return err;
1584 
1585 	/* tell the netdev what we actually can support */
1586 	netdev->features |= NETIF_F_HIGHDMA;
1587 
1588 	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1589 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1590 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1591 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1592 	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1593 		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1594 	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1595 		netdev->hw_features |= NETIF_F_RXHASH;
1596 	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1597 		netdev->hw_features |= NETIF_F_SG;
1598 
1599 	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1600 		netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1601 	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1602 		netdev->hw_enc_features |= NETIF_F_RXCSUM;
1603 	if (lif->hw_features & IONIC_ETH_HW_TSO)
1604 		netdev->hw_enc_features |= NETIF_F_TSO;
1605 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1606 		netdev->hw_enc_features |= NETIF_F_TSO6;
1607 	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1608 		netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1609 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1610 		netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1611 	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1612 		netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1613 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1614 		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1615 	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1616 		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1617 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1618 		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1619 	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1620 		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1621 
1622 	netdev->hw_features |= netdev->hw_enc_features;
1623 	netdev->features |= netdev->hw_features;
1624 	netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1625 
1626 	netdev->priv_flags |= IFF_UNICAST_FLT |
1627 			      IFF_LIVE_ADDR_CHANGE;
1628 
1629 	netdev->xdp_features = NETDEV_XDP_ACT_BASIC    |
1630 			       NETDEV_XDP_ACT_REDIRECT |
1631 			       NETDEV_XDP_ACT_RX_SG    |
1632 			       NETDEV_XDP_ACT_NDO_XMIT |
1633 			       NETDEV_XDP_ACT_NDO_XMIT_SG;
1634 
1635 	return 0;
1636 }
1637 
ionic_set_features(struct net_device * netdev,netdev_features_t features)1638 static int ionic_set_features(struct net_device *netdev,
1639 			      netdev_features_t features)
1640 {
1641 	struct ionic_lif *lif = netdev_priv(netdev);
1642 	int err;
1643 
1644 	netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1645 		   __func__, (u64)lif->netdev->features, (u64)features);
1646 
1647 	err = ionic_set_nic_features(lif, features);
1648 
1649 	return err;
1650 }
1651 
ionic_set_attr_mac(struct ionic_lif * lif,u8 * mac)1652 static int ionic_set_attr_mac(struct ionic_lif *lif, u8 *mac)
1653 {
1654 	struct ionic_admin_ctx ctx = {
1655 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1656 		.cmd.lif_setattr = {
1657 			.opcode = IONIC_CMD_LIF_SETATTR,
1658 			.index = cpu_to_le16(lif->index),
1659 			.attr = IONIC_LIF_ATTR_MAC,
1660 		},
1661 	};
1662 
1663 	ether_addr_copy(ctx.cmd.lif_setattr.mac, mac);
1664 	return ionic_adminq_post_wait(lif, &ctx);
1665 }
1666 
ionic_get_attr_mac(struct ionic_lif * lif,u8 * mac_addr)1667 static int ionic_get_attr_mac(struct ionic_lif *lif, u8 *mac_addr)
1668 {
1669 	struct ionic_admin_ctx ctx = {
1670 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1671 		.cmd.lif_getattr = {
1672 			.opcode = IONIC_CMD_LIF_GETATTR,
1673 			.index = cpu_to_le16(lif->index),
1674 			.attr = IONIC_LIF_ATTR_MAC,
1675 		},
1676 	};
1677 	int err;
1678 
1679 	err = ionic_adminq_post_wait(lif, &ctx);
1680 	if (err)
1681 		return err;
1682 
1683 	ether_addr_copy(mac_addr, ctx.comp.lif_getattr.mac);
1684 	return 0;
1685 }
1686 
ionic_program_mac(struct ionic_lif * lif,u8 * mac)1687 static int ionic_program_mac(struct ionic_lif *lif, u8 *mac)
1688 {
1689 	u8  get_mac[ETH_ALEN];
1690 	int err;
1691 
1692 	err = ionic_set_attr_mac(lif, mac);
1693 	if (err)
1694 		return err;
1695 
1696 	err = ionic_get_attr_mac(lif, get_mac);
1697 	if (err)
1698 		return err;
1699 
1700 	/* To deal with older firmware that silently ignores the set attr mac:
1701 	 * doesn't actually change the mac and doesn't return an error, so we
1702 	 * do the get attr to verify whether or not the set actually happened
1703 	 */
1704 	if (!ether_addr_equal(get_mac, mac))
1705 		return 1;
1706 
1707 	return 0;
1708 }
1709 
ionic_set_mac_address(struct net_device * netdev,void * sa)1710 static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1711 {
1712 	struct ionic_lif *lif = netdev_priv(netdev);
1713 	struct sockaddr *addr = sa;
1714 	u8 *mac;
1715 	int err;
1716 
1717 	mac = (u8 *)addr->sa_data;
1718 	if (ether_addr_equal(netdev->dev_addr, mac))
1719 		return 0;
1720 
1721 	err = ionic_program_mac(lif, mac);
1722 	if (err < 0)
1723 		return err;
1724 
1725 	if (err > 0)
1726 		netdev_dbg(netdev, "%s: SET and GET ATTR Mac are not equal-due to old FW running\n",
1727 			   __func__);
1728 
1729 	err = eth_prepare_mac_addr_change(netdev, addr);
1730 	if (err)
1731 		return err;
1732 
1733 	if (!is_zero_ether_addr(netdev->dev_addr)) {
1734 		netdev_info(netdev, "deleting mac addr %pM\n",
1735 			    netdev->dev_addr);
1736 		ionic_lif_addr_del(netdev_priv(netdev), netdev->dev_addr);
1737 	}
1738 
1739 	eth_commit_mac_addr_change(netdev, addr);
1740 	netdev_info(netdev, "updating mac addr %pM\n", mac);
1741 
1742 	return ionic_lif_addr_add(netdev_priv(netdev), mac);
1743 }
1744 
ionic_stop_queues_reconfig(struct ionic_lif * lif)1745 void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1746 {
1747 	/* Stop and clean the queues before reconfiguration */
1748 	netif_device_detach(lif->netdev);
1749 	ionic_stop_queues(lif);
1750 	ionic_txrx_deinit(lif);
1751 }
1752 
ionic_start_queues_reconfig(struct ionic_lif * lif)1753 static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1754 {
1755 	int err;
1756 
1757 	/* Re-init the queues after reconfiguration */
1758 
1759 	/* The only way txrx_init can fail here is if communication
1760 	 * with FW is suddenly broken.  There's not much we can do
1761 	 * at this point - error messages have already been printed,
1762 	 * so we can continue on and the user can eventually do a
1763 	 * DOWN and UP to try to reset and clear the issue.
1764 	 */
1765 	err = ionic_txrx_init(lif);
1766 	ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1767 	netif_device_attach(lif->netdev);
1768 
1769 	return err;
1770 }
1771 
ionic_xdp_is_valid_mtu(struct ionic_lif * lif,u32 mtu,struct bpf_prog * xdp_prog)1772 static bool ionic_xdp_is_valid_mtu(struct ionic_lif *lif, u32 mtu,
1773 				   struct bpf_prog *xdp_prog)
1774 {
1775 	if (!xdp_prog)
1776 		return true;
1777 
1778 	if (mtu <= IONIC_XDP_MAX_LINEAR_MTU)
1779 		return true;
1780 
1781 	if (xdp_prog->aux && xdp_prog->aux->xdp_has_frags)
1782 		return true;
1783 
1784 	return false;
1785 }
1786 
ionic_change_mtu(struct net_device * netdev,int new_mtu)1787 static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1788 {
1789 	struct ionic_lif *lif = netdev_priv(netdev);
1790 	struct ionic_admin_ctx ctx = {
1791 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1792 		.cmd.lif_setattr = {
1793 			.opcode = IONIC_CMD_LIF_SETATTR,
1794 			.index = cpu_to_le16(lif->index),
1795 			.attr = IONIC_LIF_ATTR_MTU,
1796 			.mtu = cpu_to_le32(new_mtu),
1797 		},
1798 	};
1799 	struct bpf_prog *xdp_prog;
1800 	int err;
1801 
1802 	xdp_prog = READ_ONCE(lif->xdp_prog);
1803 	if (!ionic_xdp_is_valid_mtu(lif, new_mtu, xdp_prog))
1804 		return -EINVAL;
1805 
1806 	err = ionic_adminq_post_wait(lif, &ctx);
1807 	if (err)
1808 		return err;
1809 
1810 	/* if we're not running, nothing more to do */
1811 	if (!netif_running(netdev)) {
1812 		WRITE_ONCE(netdev->mtu, new_mtu);
1813 		return 0;
1814 	}
1815 
1816 	mutex_lock(&lif->queue_lock);
1817 	ionic_stop_queues_reconfig(lif);
1818 	WRITE_ONCE(netdev->mtu, new_mtu);
1819 	err = ionic_start_queues_reconfig(lif);
1820 	mutex_unlock(&lif->queue_lock);
1821 
1822 	return err;
1823 }
1824 
ionic_tx_timeout_work(struct work_struct * ws)1825 static void ionic_tx_timeout_work(struct work_struct *ws)
1826 {
1827 	struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1828 	int err;
1829 
1830 	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1831 		return;
1832 
1833 	/* if we were stopped before this scheduled job was launched,
1834 	 * don't bother the queues as they are already stopped.
1835 	 */
1836 	if (!netif_running(lif->netdev))
1837 		return;
1838 
1839 	mutex_lock(&lif->queue_lock);
1840 	ionic_stop_queues_reconfig(lif);
1841 	err = ionic_start_queues_reconfig(lif);
1842 	mutex_unlock(&lif->queue_lock);
1843 
1844 	if (err)
1845 		dev_err(lif->ionic->dev, "%s: Restarting queues failed\n", __func__);
1846 }
1847 
ionic_tx_timeout(struct net_device * netdev,unsigned int txqueue)1848 static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1849 {
1850 	struct ionic_lif *lif = netdev_priv(netdev);
1851 
1852 	netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1853 	schedule_work(&lif->tx_timeout_work);
1854 }
1855 
ionic_vlan_rx_add_vid(struct net_device * netdev,__be16 proto,u16 vid)1856 static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1857 				 u16 vid)
1858 {
1859 	struct ionic_lif *lif = netdev_priv(netdev);
1860 	int err;
1861 
1862 	err = ionic_lif_vlan_add(lif, vid);
1863 	if (err)
1864 		return err;
1865 
1866 	ionic_lif_rx_mode(lif);
1867 
1868 	return 0;
1869 }
1870 
ionic_vlan_rx_kill_vid(struct net_device * netdev,__be16 proto,u16 vid)1871 static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1872 				  u16 vid)
1873 {
1874 	struct ionic_lif *lif = netdev_priv(netdev);
1875 	int err;
1876 
1877 	err = ionic_lif_vlan_del(lif, vid);
1878 	if (err)
1879 		return err;
1880 
1881 	ionic_lif_rx_mode(lif);
1882 
1883 	return 0;
1884 }
1885 
ionic_lif_rss_config(struct ionic_lif * lif,const u16 types,const u8 * key,const u32 * indir)1886 int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1887 			 const u8 *key, const u32 *indir)
1888 {
1889 	struct ionic_admin_ctx ctx = {
1890 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1891 		.cmd.lif_setattr = {
1892 			.opcode = IONIC_CMD_LIF_SETATTR,
1893 			.attr = IONIC_LIF_ATTR_RSS,
1894 			.rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1895 		},
1896 	};
1897 	unsigned int i, tbl_sz;
1898 
1899 	if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1900 		lif->rss_types = types;
1901 		ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1902 	}
1903 
1904 	if (key)
1905 		memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1906 
1907 	if (indir) {
1908 		tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1909 		for (i = 0; i < tbl_sz; i++)
1910 			lif->rss_ind_tbl[i] = indir[i];
1911 	}
1912 
1913 	memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1914 	       IONIC_RSS_HASH_KEY_SIZE);
1915 
1916 	return ionic_adminq_post_wait(lif, &ctx);
1917 }
1918 
ionic_lif_rss_init(struct ionic_lif * lif)1919 static int ionic_lif_rss_init(struct ionic_lif *lif)
1920 {
1921 	unsigned int tbl_sz;
1922 	unsigned int i;
1923 
1924 	lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1925 			 IONIC_RSS_TYPE_IPV4_TCP |
1926 			 IONIC_RSS_TYPE_IPV4_UDP |
1927 			 IONIC_RSS_TYPE_IPV6     |
1928 			 IONIC_RSS_TYPE_IPV6_TCP |
1929 			 IONIC_RSS_TYPE_IPV6_UDP;
1930 
1931 	/* Fill indirection table with 'default' values */
1932 	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1933 	for (i = 0; i < tbl_sz; i++)
1934 		lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1935 
1936 	return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1937 }
1938 
ionic_lif_rss_deinit(struct ionic_lif * lif)1939 static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1940 {
1941 	int tbl_sz;
1942 
1943 	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1944 	memset(lif->rss_ind_tbl, 0, tbl_sz);
1945 	memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1946 
1947 	ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1948 }
1949 
ionic_lif_quiesce(struct ionic_lif * lif)1950 static void ionic_lif_quiesce(struct ionic_lif *lif)
1951 {
1952 	struct ionic_admin_ctx ctx = {
1953 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1954 		.cmd.lif_setattr = {
1955 			.opcode = IONIC_CMD_LIF_SETATTR,
1956 			.index = cpu_to_le16(lif->index),
1957 			.attr = IONIC_LIF_ATTR_STATE,
1958 			.state = IONIC_LIF_QUIESCE,
1959 		},
1960 	};
1961 	int err;
1962 
1963 	err = ionic_adminq_post_wait(lif, &ctx);
1964 	if (err)
1965 		netdev_dbg(lif->netdev, "lif quiesce failed %d\n", err);
1966 }
1967 
ionic_txrx_disable(struct ionic_lif * lif)1968 static void ionic_txrx_disable(struct ionic_lif *lif)
1969 {
1970 	unsigned int i;
1971 	int err = 0;
1972 
1973 	if (lif->txqcqs) {
1974 		for (i = 0; i < lif->nxqs; i++)
1975 			err = ionic_qcq_disable(lif, lif->txqcqs[i], err);
1976 	}
1977 
1978 	if (lif->hwstamp_txq)
1979 		err = ionic_qcq_disable(lif, lif->hwstamp_txq, err);
1980 
1981 	if (lif->rxqcqs) {
1982 		for (i = 0; i < lif->nxqs; i++)
1983 			err = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
1984 	}
1985 
1986 	if (lif->hwstamp_rxq)
1987 		err = ionic_qcq_disable(lif, lif->hwstamp_rxq, err);
1988 
1989 	ionic_lif_quiesce(lif);
1990 }
1991 
ionic_txrx_deinit(struct ionic_lif * lif)1992 static void ionic_txrx_deinit(struct ionic_lif *lif)
1993 {
1994 	unsigned int i;
1995 
1996 	if (lif->txqcqs) {
1997 		for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1998 			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1999 			ionic_tx_flush(&lif->txqcqs[i]->cq);
2000 			ionic_tx_empty(&lif->txqcqs[i]->q);
2001 		}
2002 	}
2003 
2004 	if (lif->rxqcqs) {
2005 		for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
2006 			ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2007 			ionic_rx_empty(&lif->rxqcqs[i]->q);
2008 		}
2009 	}
2010 	lif->rx_mode = 0;
2011 
2012 	if (lif->hwstamp_txq) {
2013 		ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
2014 		ionic_tx_flush(&lif->hwstamp_txq->cq);
2015 		ionic_tx_empty(&lif->hwstamp_txq->q);
2016 	}
2017 
2018 	if (lif->hwstamp_rxq) {
2019 		ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
2020 		ionic_rx_empty(&lif->hwstamp_rxq->q);
2021 	}
2022 }
2023 
ionic_txrx_free(struct ionic_lif * lif)2024 void ionic_txrx_free(struct ionic_lif *lif)
2025 {
2026 	unsigned int i;
2027 
2028 	if (lif->txqcqs) {
2029 		for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
2030 			ionic_qcq_free(lif, lif->txqcqs[i]);
2031 			devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
2032 			lif->txqcqs[i] = NULL;
2033 		}
2034 	}
2035 
2036 	if (lif->rxqcqs) {
2037 		for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2038 			ionic_qcq_free(lif, lif->rxqcqs[i]);
2039 			devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
2040 			lif->rxqcqs[i] = NULL;
2041 		}
2042 	}
2043 
2044 	if (lif->hwstamp_txq) {
2045 		ionic_qcq_free(lif, lif->hwstamp_txq);
2046 		devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2047 		lif->hwstamp_txq = NULL;
2048 	}
2049 
2050 	if (lif->hwstamp_rxq) {
2051 		ionic_qcq_free(lif, lif->hwstamp_rxq);
2052 		devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2053 		lif->hwstamp_rxq = NULL;
2054 	}
2055 }
2056 
ionic_txrx_alloc(struct ionic_lif * lif)2057 static int ionic_txrx_alloc(struct ionic_lif *lif)
2058 {
2059 	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2060 	unsigned int flags, i;
2061 	int err = 0;
2062 
2063 	num_desc = lif->ntxq_descs;
2064 	desc_sz = sizeof(struct ionic_txq_desc);
2065 	comp_sz = sizeof(struct ionic_txq_comp);
2066 
2067 	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2068 	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2069 					  sizeof(struct ionic_txq_sg_desc_v1))
2070 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2071 	else
2072 		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2073 
2074 	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2075 
2076 	if (test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state))
2077 		flags |= IONIC_QCQ_F_CMB_RINGS;
2078 
2079 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2080 		flags |= IONIC_QCQ_F_INTR;
2081 
2082 	for (i = 0; i < lif->nxqs; i++) {
2083 		err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2084 				      num_desc, desc_sz, comp_sz, sg_desc_sz,
2085 				      sizeof(struct ionic_tx_desc_info),
2086 				      lif->kern_pid, NULL, &lif->txqcqs[i]);
2087 		if (err)
2088 			goto err_out;
2089 
2090 		if (flags & IONIC_QCQ_F_INTR) {
2091 			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2092 					     lif->txqcqs[i]->intr.index,
2093 					     lif->tx_coalesce_hw);
2094 			if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2095 				lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2096 		}
2097 
2098 		ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2099 	}
2100 
2101 	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2102 
2103 	if (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state))
2104 		flags |= IONIC_QCQ_F_CMB_RINGS;
2105 
2106 	num_desc = lif->nrxq_descs;
2107 	desc_sz = sizeof(struct ionic_rxq_desc);
2108 	comp_sz = sizeof(struct ionic_rxq_comp);
2109 	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2110 
2111 	if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2112 		comp_sz *= 2;
2113 
2114 	for (i = 0; i < lif->nxqs; i++) {
2115 		err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2116 				      num_desc, desc_sz, comp_sz, sg_desc_sz,
2117 				      sizeof(struct ionic_rx_desc_info),
2118 				      lif->kern_pid, lif->xdp_prog,
2119 				      &lif->rxqcqs[i]);
2120 		if (err)
2121 			goto err_out;
2122 
2123 		lif->rxqcqs[i]->q.features = lif->rxq_features;
2124 
2125 		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2126 				     lif->rxqcqs[i]->intr.index,
2127 				     lif->rx_coalesce_hw);
2128 		if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2129 			lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2130 
2131 		if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2132 			ionic_link_qcq_interrupts(lif->rxqcqs[i],
2133 						  lif->txqcqs[i]);
2134 
2135 		ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2136 	}
2137 
2138 	return 0;
2139 
2140 err_out:
2141 	ionic_txrx_free(lif);
2142 
2143 	return err;
2144 }
2145 
ionic_txrx_init(struct ionic_lif * lif)2146 static int ionic_txrx_init(struct ionic_lif *lif)
2147 {
2148 	unsigned int i;
2149 	int err;
2150 
2151 	for (i = 0; i < lif->nxqs; i++) {
2152 		err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2153 		if (err)
2154 			goto err_out;
2155 
2156 		err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2157 		if (err) {
2158 			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2159 			goto err_out;
2160 		}
2161 	}
2162 
2163 	if (lif->netdev->features & NETIF_F_RXHASH)
2164 		ionic_lif_rss_init(lif);
2165 
2166 	ionic_lif_rx_mode(lif);
2167 
2168 	return 0;
2169 
2170 err_out:
2171 	while (i--) {
2172 		ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2173 		ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2174 	}
2175 
2176 	return err;
2177 }
2178 
ionic_txrx_enable(struct ionic_lif * lif)2179 static int ionic_txrx_enable(struct ionic_lif *lif)
2180 {
2181 	int derr = 0;
2182 	int i, err;
2183 
2184 	ionic_xdp_rxqs_prog_update(lif);
2185 
2186 	for (i = 0; i < lif->nxqs; i++) {
2187 		if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2188 			dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2189 			err = -ENXIO;
2190 			goto err_out;
2191 		}
2192 
2193 		ionic_rx_fill(&lif->rxqcqs[i]->q,
2194 			      READ_ONCE(lif->rxqcqs[i]->q.xdp_prog));
2195 		err = ionic_qcq_enable(lif->rxqcqs[i]);
2196 		if (err)
2197 			goto err_out;
2198 
2199 		err = ionic_qcq_enable(lif->txqcqs[i]);
2200 		if (err) {
2201 			derr = ionic_qcq_disable(lif, lif->rxqcqs[i], err);
2202 			goto err_out;
2203 		}
2204 	}
2205 
2206 	if (lif->hwstamp_rxq) {
2207 		ionic_rx_fill(&lif->hwstamp_rxq->q, NULL);
2208 		err = ionic_qcq_enable(lif->hwstamp_rxq);
2209 		if (err)
2210 			goto err_out_hwstamp_rx;
2211 	}
2212 
2213 	if (lif->hwstamp_txq) {
2214 		err = ionic_qcq_enable(lif->hwstamp_txq);
2215 		if (err)
2216 			goto err_out_hwstamp_tx;
2217 	}
2218 
2219 	return 0;
2220 
2221 err_out_hwstamp_tx:
2222 	if (lif->hwstamp_rxq)
2223 		derr = ionic_qcq_disable(lif, lif->hwstamp_rxq, derr);
2224 err_out_hwstamp_rx:
2225 	i = lif->nxqs;
2226 err_out:
2227 	while (i--) {
2228 		derr = ionic_qcq_disable(lif, lif->txqcqs[i], derr);
2229 		derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr);
2230 	}
2231 
2232 	ionic_xdp_rxqs_prog_update(lif);
2233 
2234 	return err;
2235 }
2236 
ionic_start_queues(struct ionic_lif * lif)2237 static int ionic_start_queues(struct ionic_lif *lif)
2238 {
2239 	int err;
2240 
2241 	if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2242 		return -EIO;
2243 
2244 	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2245 		return -EBUSY;
2246 
2247 	if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2248 		return 0;
2249 
2250 	err = ionic_txrx_enable(lif);
2251 	if (err) {
2252 		clear_bit(IONIC_LIF_F_UP, lif->state);
2253 		return err;
2254 	}
2255 	netif_tx_wake_all_queues(lif->netdev);
2256 
2257 	return 0;
2258 }
2259 
ionic_open(struct net_device * netdev)2260 static int ionic_open(struct net_device *netdev)
2261 {
2262 	struct ionic_lif *lif = netdev_priv(netdev);
2263 	int err;
2264 
2265 	/* If recovering from a broken state, clear the bit and we'll try again */
2266 	if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2267 		netdev_info(netdev, "clearing broken state\n");
2268 
2269 	mutex_lock(&lif->queue_lock);
2270 
2271 	err = ionic_txrx_alloc(lif);
2272 	if (err)
2273 		goto err_unlock;
2274 
2275 	err = ionic_txrx_init(lif);
2276 	if (err)
2277 		goto err_txrx_free;
2278 
2279 	err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2280 	if (err)
2281 		goto err_txrx_deinit;
2282 
2283 	err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2284 	if (err)
2285 		goto err_txrx_deinit;
2286 
2287 	/* don't start the queues until we have link */
2288 	if (netif_carrier_ok(netdev)) {
2289 		err = ionic_start_queues(lif);
2290 		if (err)
2291 			goto err_txrx_deinit;
2292 	}
2293 
2294 	/* If hardware timestamping is enabled, but the queues were freed by
2295 	 * ionic_stop, those need to be reallocated and initialized, too.
2296 	 */
2297 	ionic_lif_hwstamp_recreate_queues(lif);
2298 
2299 	mutex_unlock(&lif->queue_lock);
2300 
2301 	return 0;
2302 
2303 err_txrx_deinit:
2304 	ionic_txrx_deinit(lif);
2305 err_txrx_free:
2306 	ionic_txrx_free(lif);
2307 err_unlock:
2308 	mutex_unlock(&lif->queue_lock);
2309 	return err;
2310 }
2311 
ionic_stop_queues(struct ionic_lif * lif)2312 static void ionic_stop_queues(struct ionic_lif *lif)
2313 {
2314 	if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2315 		return;
2316 
2317 	netif_tx_disable(lif->netdev);
2318 	ionic_txrx_disable(lif);
2319 }
2320 
ionic_stop(struct net_device * netdev)2321 static int ionic_stop(struct net_device *netdev)
2322 {
2323 	struct ionic_lif *lif = netdev_priv(netdev);
2324 
2325 	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2326 		return 0;
2327 
2328 	mutex_lock(&lif->queue_lock);
2329 	ionic_stop_queues(lif);
2330 	ionic_txrx_deinit(lif);
2331 	ionic_txrx_free(lif);
2332 	mutex_unlock(&lif->queue_lock);
2333 
2334 	return 0;
2335 }
2336 
ionic_eth_ioctl(struct net_device * netdev,struct ifreq * ifr,int cmd)2337 static int ionic_eth_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2338 {
2339 	struct ionic_lif *lif = netdev_priv(netdev);
2340 
2341 	switch (cmd) {
2342 	case SIOCSHWTSTAMP:
2343 		return ionic_lif_hwstamp_set(lif, ifr);
2344 	case SIOCGHWTSTAMP:
2345 		return ionic_lif_hwstamp_get(lif, ifr);
2346 	default:
2347 		return -EOPNOTSUPP;
2348 	}
2349 }
2350 
ionic_get_vf_config(struct net_device * netdev,int vf,struct ifla_vf_info * ivf)2351 static int ionic_get_vf_config(struct net_device *netdev,
2352 			       int vf, struct ifla_vf_info *ivf)
2353 {
2354 	struct ionic_lif *lif = netdev_priv(netdev);
2355 	struct ionic *ionic = lif->ionic;
2356 	int ret = 0;
2357 
2358 	if (!netif_device_present(netdev))
2359 		return -EBUSY;
2360 
2361 	down_read(&ionic->vf_op_lock);
2362 
2363 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2364 		ret = -EINVAL;
2365 	} else {
2366 		struct ionic_vf *vfdata = &ionic->vfs[vf];
2367 
2368 		ivf->vf		  = vf;
2369 		ivf->qos	  = 0;
2370 		ivf->vlan         = le16_to_cpu(vfdata->vlanid);
2371 		ivf->spoofchk     = vfdata->spoofchk;
2372 		ivf->linkstate    = vfdata->linkstate;
2373 		ivf->max_tx_rate  = le32_to_cpu(vfdata->maxrate);
2374 		ivf->trusted      = vfdata->trusted;
2375 		ether_addr_copy(ivf->mac, vfdata->macaddr);
2376 	}
2377 
2378 	up_read(&ionic->vf_op_lock);
2379 	return ret;
2380 }
2381 
ionic_get_vf_stats(struct net_device * netdev,int vf,struct ifla_vf_stats * vf_stats)2382 static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2383 			      struct ifla_vf_stats *vf_stats)
2384 {
2385 	struct ionic_lif *lif = netdev_priv(netdev);
2386 	struct ionic *ionic = lif->ionic;
2387 	struct ionic_lif_stats *vs;
2388 	int ret = 0;
2389 
2390 	if (!netif_device_present(netdev))
2391 		return -EBUSY;
2392 
2393 	down_read(&ionic->vf_op_lock);
2394 
2395 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2396 		ret = -EINVAL;
2397 	} else {
2398 		memset(vf_stats, 0, sizeof(*vf_stats));
2399 		vs = &ionic->vfs[vf].stats;
2400 
2401 		vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2402 		vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2403 		vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2404 		vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2405 		vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2406 		vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2407 		vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2408 				       le64_to_cpu(vs->rx_mcast_drop_packets) +
2409 				       le64_to_cpu(vs->rx_bcast_drop_packets);
2410 		vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2411 				       le64_to_cpu(vs->tx_mcast_drop_packets) +
2412 				       le64_to_cpu(vs->tx_bcast_drop_packets);
2413 	}
2414 
2415 	up_read(&ionic->vf_op_lock);
2416 	return ret;
2417 }
2418 
ionic_set_vf_mac(struct net_device * netdev,int vf,u8 * mac)2419 static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2420 {
2421 	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_MAC };
2422 	struct ionic_lif *lif = netdev_priv(netdev);
2423 	struct ionic *ionic = lif->ionic;
2424 	int ret;
2425 
2426 	if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2427 		return -EINVAL;
2428 
2429 	if (!netif_device_present(netdev))
2430 		return -EBUSY;
2431 
2432 	down_write(&ionic->vf_op_lock);
2433 
2434 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2435 		ret = -EINVAL;
2436 	} else {
2437 		ether_addr_copy(vfc.macaddr, mac);
2438 		dev_dbg(ionic->dev, "%s: vf %d macaddr %pM\n",
2439 			__func__, vf, vfc.macaddr);
2440 
2441 		ret = ionic_set_vf_config(ionic, vf, &vfc);
2442 		if (!ret)
2443 			ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2444 	}
2445 
2446 	up_write(&ionic->vf_op_lock);
2447 	return ret;
2448 }
2449 
ionic_set_vf_vlan(struct net_device * netdev,int vf,u16 vlan,u8 qos,__be16 proto)2450 static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2451 			     u8 qos, __be16 proto)
2452 {
2453 	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_VLAN };
2454 	struct ionic_lif *lif = netdev_priv(netdev);
2455 	struct ionic *ionic = lif->ionic;
2456 	int ret;
2457 
2458 	/* until someday when we support qos */
2459 	if (qos)
2460 		return -EINVAL;
2461 
2462 	if (vlan > 4095)
2463 		return -EINVAL;
2464 
2465 	if (proto != htons(ETH_P_8021Q))
2466 		return -EPROTONOSUPPORT;
2467 
2468 	if (!netif_device_present(netdev))
2469 		return -EBUSY;
2470 
2471 	down_write(&ionic->vf_op_lock);
2472 
2473 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2474 		ret = -EINVAL;
2475 	} else {
2476 		vfc.vlanid = cpu_to_le16(vlan);
2477 		dev_dbg(ionic->dev, "%s: vf %d vlan %d\n",
2478 			__func__, vf, le16_to_cpu(vfc.vlanid));
2479 
2480 		ret = ionic_set_vf_config(ionic, vf, &vfc);
2481 		if (!ret)
2482 			ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2483 	}
2484 
2485 	up_write(&ionic->vf_op_lock);
2486 	return ret;
2487 }
2488 
ionic_set_vf_rate(struct net_device * netdev,int vf,int tx_min,int tx_max)2489 static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2490 			     int tx_min, int tx_max)
2491 {
2492 	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_RATE };
2493 	struct ionic_lif *lif = netdev_priv(netdev);
2494 	struct ionic *ionic = lif->ionic;
2495 	int ret;
2496 
2497 	/* setting the min just seems silly */
2498 	if (tx_min)
2499 		return -EINVAL;
2500 
2501 	if (!netif_device_present(netdev))
2502 		return -EBUSY;
2503 
2504 	down_write(&ionic->vf_op_lock);
2505 
2506 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2507 		ret = -EINVAL;
2508 	} else {
2509 		vfc.maxrate = cpu_to_le32(tx_max);
2510 		dev_dbg(ionic->dev, "%s: vf %d maxrate %d\n",
2511 			__func__, vf, le32_to_cpu(vfc.maxrate));
2512 
2513 		ret = ionic_set_vf_config(ionic, vf, &vfc);
2514 		if (!ret)
2515 			ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2516 	}
2517 
2518 	up_write(&ionic->vf_op_lock);
2519 	return ret;
2520 }
2521 
ionic_set_vf_spoofchk(struct net_device * netdev,int vf,bool set)2522 static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2523 {
2524 	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_SPOOFCHK };
2525 	struct ionic_lif *lif = netdev_priv(netdev);
2526 	struct ionic *ionic = lif->ionic;
2527 	int ret;
2528 
2529 	if (!netif_device_present(netdev))
2530 		return -EBUSY;
2531 
2532 	down_write(&ionic->vf_op_lock);
2533 
2534 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2535 		ret = -EINVAL;
2536 	} else {
2537 		vfc.spoofchk = set;
2538 		dev_dbg(ionic->dev, "%s: vf %d spoof %d\n",
2539 			__func__, vf, vfc.spoofchk);
2540 
2541 		ret = ionic_set_vf_config(ionic, vf, &vfc);
2542 		if (!ret)
2543 			ionic->vfs[vf].spoofchk = set;
2544 	}
2545 
2546 	up_write(&ionic->vf_op_lock);
2547 	return ret;
2548 }
2549 
ionic_set_vf_trust(struct net_device * netdev,int vf,bool set)2550 static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2551 {
2552 	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_TRUST };
2553 	struct ionic_lif *lif = netdev_priv(netdev);
2554 	struct ionic *ionic = lif->ionic;
2555 	int ret;
2556 
2557 	if (!netif_device_present(netdev))
2558 		return -EBUSY;
2559 
2560 	down_write(&ionic->vf_op_lock);
2561 
2562 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2563 		ret = -EINVAL;
2564 	} else {
2565 		vfc.trust = set;
2566 		dev_dbg(ionic->dev, "%s: vf %d trust %d\n",
2567 			__func__, vf, vfc.trust);
2568 
2569 		ret = ionic_set_vf_config(ionic, vf, &vfc);
2570 		if (!ret)
2571 			ionic->vfs[vf].trusted = set;
2572 	}
2573 
2574 	up_write(&ionic->vf_op_lock);
2575 	return ret;
2576 }
2577 
ionic_set_vf_link_state(struct net_device * netdev,int vf,int set)2578 static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2579 {
2580 	struct ionic_vf_setattr_cmd vfc = { .attr = IONIC_VF_ATTR_LINKSTATE };
2581 	struct ionic_lif *lif = netdev_priv(netdev);
2582 	struct ionic *ionic = lif->ionic;
2583 	u8 vfls;
2584 	int ret;
2585 
2586 	switch (set) {
2587 	case IFLA_VF_LINK_STATE_ENABLE:
2588 		vfls = IONIC_VF_LINK_STATUS_UP;
2589 		break;
2590 	case IFLA_VF_LINK_STATE_DISABLE:
2591 		vfls = IONIC_VF_LINK_STATUS_DOWN;
2592 		break;
2593 	case IFLA_VF_LINK_STATE_AUTO:
2594 		vfls = IONIC_VF_LINK_STATUS_AUTO;
2595 		break;
2596 	default:
2597 		return -EINVAL;
2598 	}
2599 
2600 	if (!netif_device_present(netdev))
2601 		return -EBUSY;
2602 
2603 	down_write(&ionic->vf_op_lock);
2604 
2605 	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2606 		ret = -EINVAL;
2607 	} else {
2608 		vfc.linkstate = vfls;
2609 		dev_dbg(ionic->dev, "%s: vf %d linkstate %d\n",
2610 			__func__, vf, vfc.linkstate);
2611 
2612 		ret = ionic_set_vf_config(ionic, vf, &vfc);
2613 		if (!ret)
2614 			ionic->vfs[vf].linkstate = set;
2615 	}
2616 
2617 	up_write(&ionic->vf_op_lock);
2618 	return ret;
2619 }
2620 
ionic_vf_attr_replay(struct ionic_lif * lif)2621 static void ionic_vf_attr_replay(struct ionic_lif *lif)
2622 {
2623 	struct ionic_vf_setattr_cmd vfc = { };
2624 	struct ionic *ionic = lif->ionic;
2625 	struct ionic_vf *v;
2626 	int i;
2627 
2628 	if (!ionic->vfs)
2629 		return;
2630 
2631 	down_read(&ionic->vf_op_lock);
2632 
2633 	for (i = 0; i < ionic->num_vfs; i++) {
2634 		v = &ionic->vfs[i];
2635 
2636 		if (v->stats_pa) {
2637 			vfc.attr = IONIC_VF_ATTR_STATSADDR;
2638 			vfc.stats_pa = cpu_to_le64(v->stats_pa);
2639 			ionic_set_vf_config(ionic, i, &vfc);
2640 			vfc.stats_pa = 0;
2641 		}
2642 
2643 		if (!is_zero_ether_addr(v->macaddr)) {
2644 			vfc.attr = IONIC_VF_ATTR_MAC;
2645 			ether_addr_copy(vfc.macaddr, v->macaddr);
2646 			ionic_set_vf_config(ionic, i, &vfc);
2647 			eth_zero_addr(vfc.macaddr);
2648 		}
2649 
2650 		if (v->vlanid) {
2651 			vfc.attr = IONIC_VF_ATTR_VLAN;
2652 			vfc.vlanid = v->vlanid;
2653 			ionic_set_vf_config(ionic, i, &vfc);
2654 			vfc.vlanid = 0;
2655 		}
2656 
2657 		if (v->maxrate) {
2658 			vfc.attr = IONIC_VF_ATTR_RATE;
2659 			vfc.maxrate = v->maxrate;
2660 			ionic_set_vf_config(ionic, i, &vfc);
2661 			vfc.maxrate = 0;
2662 		}
2663 
2664 		if (v->spoofchk) {
2665 			vfc.attr = IONIC_VF_ATTR_SPOOFCHK;
2666 			vfc.spoofchk = v->spoofchk;
2667 			ionic_set_vf_config(ionic, i, &vfc);
2668 			vfc.spoofchk = 0;
2669 		}
2670 
2671 		if (v->trusted) {
2672 			vfc.attr = IONIC_VF_ATTR_TRUST;
2673 			vfc.trust = v->trusted;
2674 			ionic_set_vf_config(ionic, i, &vfc);
2675 			vfc.trust = 0;
2676 		}
2677 
2678 		if (v->linkstate) {
2679 			vfc.attr = IONIC_VF_ATTR_LINKSTATE;
2680 			vfc.linkstate = v->linkstate;
2681 			ionic_set_vf_config(ionic, i, &vfc);
2682 			vfc.linkstate = 0;
2683 		}
2684 	}
2685 
2686 	up_read(&ionic->vf_op_lock);
2687 
2688 	ionic_vf_start(ionic);
2689 }
2690 
ionic_unregister_rxq_info(struct ionic_queue * q)2691 static void ionic_unregister_rxq_info(struct ionic_queue *q)
2692 {
2693 	struct xdp_rxq_info *xi;
2694 
2695 	if (!q->xdp_rxq_info)
2696 		return;
2697 
2698 	xi = q->xdp_rxq_info;
2699 	q->xdp_rxq_info = NULL;
2700 
2701 	xdp_rxq_info_unreg(xi);
2702 	kfree(xi);
2703 }
2704 
ionic_register_rxq_info(struct ionic_queue * q,unsigned int napi_id)2705 static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id)
2706 {
2707 	struct xdp_rxq_info *rxq_info;
2708 	int err;
2709 
2710 	rxq_info = kzalloc(sizeof(*rxq_info), GFP_KERNEL);
2711 	if (!rxq_info)
2712 		return -ENOMEM;
2713 
2714 	err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id);
2715 	if (err) {
2716 		netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n",
2717 			   q->index, err);
2718 		goto err_out;
2719 	}
2720 
2721 	err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool);
2722 	if (err) {
2723 		netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n",
2724 			   q->index, err);
2725 		xdp_rxq_info_unreg(rxq_info);
2726 		goto err_out;
2727 	}
2728 
2729 	q->xdp_rxq_info = rxq_info;
2730 
2731 	return 0;
2732 
2733 err_out:
2734 	kfree(rxq_info);
2735 	return err;
2736 }
2737 
ionic_xdp_rxqs_prog_update(struct ionic_lif * lif)2738 static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif)
2739 {
2740 	struct bpf_prog *xdp_prog;
2741 	unsigned int i;
2742 
2743 	if (!lif->rxqcqs)
2744 		return;
2745 
2746 	xdp_prog = READ_ONCE(lif->xdp_prog);
2747 	for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
2748 		struct ionic_queue *q = &lif->rxqcqs[i]->q;
2749 
2750 		WRITE_ONCE(q->xdp_prog, xdp_prog);
2751 	}
2752 }
2753 
ionic_xdp_config(struct net_device * netdev,struct netdev_bpf * bpf)2754 static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf)
2755 {
2756 	struct ionic_lif *lif = netdev_priv(netdev);
2757 	struct bpf_prog *old_prog;
2758 	u32 maxfs;
2759 
2760 	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state)) {
2761 #define XDP_ERR_SPLIT "XDP not available with split Tx/Rx interrupts"
2762 		NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_SPLIT);
2763 		netdev_info(lif->netdev, XDP_ERR_SPLIT);
2764 		return -EOPNOTSUPP;
2765 	}
2766 
2767 	if (!ionic_xdp_is_valid_mtu(lif, netdev->mtu, bpf->prog)) {
2768 #define XDP_ERR_MTU "MTU is too large for XDP without frags support"
2769 		NL_SET_ERR_MSG_MOD(bpf->extack, XDP_ERR_MTU);
2770 		netdev_info(lif->netdev, XDP_ERR_MTU);
2771 		return -EINVAL;
2772 	}
2773 
2774 	maxfs = __le32_to_cpu(lif->identity->eth.max_frame_size) - VLAN_ETH_HLEN;
2775 	if (bpf->prog && !(bpf->prog->aux && bpf->prog->aux->xdp_has_frags))
2776 		maxfs = min_t(u32, maxfs, IONIC_XDP_MAX_LINEAR_MTU);
2777 	netdev->max_mtu = maxfs;
2778 
2779 	if (!netif_running(netdev)) {
2780 		old_prog = xchg(&lif->xdp_prog, bpf->prog);
2781 	} else if (lif->xdp_prog && bpf->prog) {
2782 		old_prog = xchg(&lif->xdp_prog, bpf->prog);
2783 		ionic_xdp_rxqs_prog_update(lif);
2784 	} else {
2785 		struct ionic_queue_params qparams;
2786 
2787 		ionic_init_queue_params(lif, &qparams);
2788 		qparams.xdp_prog = bpf->prog;
2789 		mutex_lock(&lif->queue_lock);
2790 		ionic_reconfigure_queues(lif, &qparams);
2791 		old_prog = xchg(&lif->xdp_prog, bpf->prog);
2792 		mutex_unlock(&lif->queue_lock);
2793 	}
2794 
2795 	if (old_prog)
2796 		bpf_prog_put(old_prog);
2797 
2798 	return 0;
2799 }
2800 
ionic_xdp(struct net_device * netdev,struct netdev_bpf * bpf)2801 static int ionic_xdp(struct net_device *netdev, struct netdev_bpf *bpf)
2802 {
2803 	switch (bpf->command) {
2804 	case XDP_SETUP_PROG:
2805 		return ionic_xdp_config(netdev, bpf);
2806 	default:
2807 		return -EINVAL;
2808 	}
2809 }
2810 
2811 static const struct net_device_ops ionic_netdev_ops = {
2812 	.ndo_open               = ionic_open,
2813 	.ndo_stop               = ionic_stop,
2814 	.ndo_eth_ioctl		= ionic_eth_ioctl,
2815 	.ndo_start_xmit		= ionic_start_xmit,
2816 	.ndo_bpf		= ionic_xdp,
2817 	.ndo_xdp_xmit		= ionic_xdp_xmit,
2818 	.ndo_get_stats64	= ionic_get_stats64,
2819 	.ndo_set_rx_mode	= ionic_ndo_set_rx_mode,
2820 	.ndo_set_features	= ionic_set_features,
2821 	.ndo_set_mac_address	= ionic_set_mac_address,
2822 	.ndo_validate_addr	= eth_validate_addr,
2823 	.ndo_tx_timeout         = ionic_tx_timeout,
2824 	.ndo_change_mtu         = ionic_change_mtu,
2825 	.ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2826 	.ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2827 	.ndo_set_vf_vlan	= ionic_set_vf_vlan,
2828 	.ndo_set_vf_trust	= ionic_set_vf_trust,
2829 	.ndo_set_vf_mac		= ionic_set_vf_mac,
2830 	.ndo_set_vf_rate	= ionic_set_vf_rate,
2831 	.ndo_set_vf_spoofchk	= ionic_set_vf_spoofchk,
2832 	.ndo_get_vf_config	= ionic_get_vf_config,
2833 	.ndo_set_vf_link_state	= ionic_set_vf_link_state,
2834 	.ndo_get_vf_stats       = ionic_get_vf_stats,
2835 };
2836 
ionic_cmb_reconfig(struct ionic_lif * lif,struct ionic_queue_params * qparam)2837 static int ionic_cmb_reconfig(struct ionic_lif *lif,
2838 			      struct ionic_queue_params *qparam)
2839 {
2840 	struct ionic_queue_params start_qparams;
2841 	int err = 0;
2842 
2843 	/* When changing CMB queue parameters, we're using limited
2844 	 * on-device memory and don't have extra memory to use for
2845 	 * duplicate allocations, so we free it all first then
2846 	 * re-allocate with the new parameters.
2847 	 */
2848 
2849 	/* Checkpoint for possible unwind */
2850 	ionic_init_queue_params(lif, &start_qparams);
2851 
2852 	/* Stop and free the queues */
2853 	ionic_stop_queues_reconfig(lif);
2854 	ionic_txrx_free(lif);
2855 
2856 	/* Set up new qparams */
2857 	ionic_set_queue_params(lif, qparam);
2858 
2859 	if (netif_running(lif->netdev)) {
2860 		/* Alloc and start the new configuration */
2861 		err = ionic_txrx_alloc(lif);
2862 		if (err) {
2863 			dev_warn(lif->ionic->dev,
2864 				 "CMB reconfig failed, restoring values: %d\n", err);
2865 
2866 			/* Back out the changes */
2867 			ionic_set_queue_params(lif, &start_qparams);
2868 			err = ionic_txrx_alloc(lif);
2869 			if (err) {
2870 				dev_err(lif->ionic->dev,
2871 					"CMB restore failed: %d\n", err);
2872 				goto err_out;
2873 			}
2874 		}
2875 
2876 		err = ionic_start_queues_reconfig(lif);
2877 		if (err) {
2878 			dev_err(lif->ionic->dev,
2879 				"CMB reconfig failed: %d\n", err);
2880 			goto err_out;
2881 		}
2882 	}
2883 
2884 err_out:
2885 	/* This was detached in ionic_stop_queues_reconfig() */
2886 	netif_device_attach(lif->netdev);
2887 
2888 	return err;
2889 }
2890 
ionic_swap_queues(struct ionic_qcq * a,struct ionic_qcq * b)2891 static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2892 {
2893 	/* only swapping the queues and napi, not flags or other stuff */
2894 	swap(a->napi,         b->napi);
2895 
2896 	if (a->q.type == IONIC_QTYPE_RXQ) {
2897 		swap(a->q.page_pool, b->q.page_pool);
2898 		a->q.page_pool->p.napi = &a->napi;
2899 		if (b->q.page_pool)  /* is NULL when increasing queue count */
2900 			b->q.page_pool->p.napi = &b->napi;
2901 	}
2902 
2903 	swap(a->q.features,   b->q.features);
2904 	swap(a->q.num_descs,  b->q.num_descs);
2905 	swap(a->q.desc_size,  b->q.desc_size);
2906 	swap(a->q.base,       b->q.base);
2907 	swap(a->q.base_pa,    b->q.base_pa);
2908 	swap(a->q.info,       b->q.info);
2909 	swap(a->q.xdp_prog,   b->q.xdp_prog);
2910 	swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info);
2911 	swap(a->q.partner,    b->q.partner);
2912 	swap(a->q_base,       b->q_base);
2913 	swap(a->q_base_pa,    b->q_base_pa);
2914 	swap(a->q_size,       b->q_size);
2915 
2916 	swap(a->q.sg_desc_size, b->q.sg_desc_size);
2917 	swap(a->q.sg_base,    b->q.sg_base);
2918 	swap(a->q.sg_base_pa, b->q.sg_base_pa);
2919 	swap(a->sg_base,      b->sg_base);
2920 	swap(a->sg_base_pa,   b->sg_base_pa);
2921 	swap(a->sg_size,      b->sg_size);
2922 
2923 	swap(a->cq.num_descs, b->cq.num_descs);
2924 	swap(a->cq.desc_size, b->cq.desc_size);
2925 	swap(a->cq.base,      b->cq.base);
2926 	swap(a->cq.base_pa,   b->cq.base_pa);
2927 	swap(a->cq_base,      b->cq_base);
2928 	swap(a->cq_base_pa,   b->cq_base_pa);
2929 	swap(a->cq_size,      b->cq_size);
2930 
2931 	ionic_debugfs_del_qcq(a);
2932 	ionic_debugfs_add_qcq(a->q.lif, a);
2933 }
2934 
ionic_reconfigure_queues(struct ionic_lif * lif,struct ionic_queue_params * qparam)2935 int ionic_reconfigure_queues(struct ionic_lif *lif,
2936 			     struct ionic_queue_params *qparam)
2937 {
2938 	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2939 	struct ionic_qcq **tx_qcqs = NULL;
2940 	struct ionic_qcq **rx_qcqs = NULL;
2941 	unsigned int flags, i;
2942 	int err = 0;
2943 
2944 	/* Are we changing q params while CMB is on */
2945 	if ((test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state) && qparam->cmb_tx) ||
2946 	    (test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state) && qparam->cmb_rx))
2947 		return ionic_cmb_reconfig(lif, qparam);
2948 
2949 	/* allocate temporary qcq arrays to hold new queue structs */
2950 	if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2951 		tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2952 				       sizeof(struct ionic_qcq *), GFP_KERNEL);
2953 		if (!tx_qcqs) {
2954 			err = -ENOMEM;
2955 			goto err_out;
2956 		}
2957 	}
2958 	if (qparam->nxqs != lif->nxqs ||
2959 	    qparam->nrxq_descs != lif->nrxq_descs ||
2960 	    qparam->rxq_features != lif->rxq_features ||
2961 	    qparam->xdp_prog != lif->xdp_prog) {
2962 		rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2963 				       sizeof(struct ionic_qcq *), GFP_KERNEL);
2964 		if (!rx_qcqs) {
2965 			err = -ENOMEM;
2966 			goto err_out;
2967 		}
2968 	}
2969 
2970 	/* allocate new desc_info and rings, but leave the interrupt setup
2971 	 * until later so as to not mess with the still-running queues
2972 	 */
2973 	if (tx_qcqs) {
2974 		num_desc = qparam->ntxq_descs;
2975 		desc_sz = sizeof(struct ionic_txq_desc);
2976 		comp_sz = sizeof(struct ionic_txq_comp);
2977 
2978 		if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2979 		    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2980 		    sizeof(struct ionic_txq_sg_desc_v1))
2981 			sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2982 		else
2983 			sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2984 
2985 		for (i = 0; i < qparam->nxqs; i++) {
2986 			/* If missing, short placeholder qcq needed for swap */
2987 			if (!lif->txqcqs[i]) {
2988 				flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2989 				err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2990 						      4, desc_sz, comp_sz, sg_desc_sz,
2991 						      sizeof(struct ionic_tx_desc_info),
2992 						      lif->kern_pid, NULL, &lif->txqcqs[i]);
2993 				if (err)
2994 					goto err_out;
2995 			}
2996 
2997 			flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2998 			err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2999 					      num_desc, desc_sz, comp_sz, sg_desc_sz,
3000 					      sizeof(struct ionic_tx_desc_info),
3001 					      lif->kern_pid, NULL, &tx_qcqs[i]);
3002 			if (err)
3003 				goto err_out;
3004 		}
3005 	}
3006 
3007 	if (rx_qcqs) {
3008 		num_desc = qparam->nrxq_descs;
3009 		desc_sz = sizeof(struct ionic_rxq_desc);
3010 		comp_sz = sizeof(struct ionic_rxq_comp);
3011 		sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
3012 
3013 		if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
3014 			comp_sz *= 2;
3015 
3016 		for (i = 0; i < qparam->nxqs; i++) {
3017 			/* If missing, short placeholder qcq needed for swap */
3018 			if (!lif->rxqcqs[i]) {
3019 				flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
3020 				err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3021 						      4, desc_sz, comp_sz, sg_desc_sz,
3022 						      sizeof(struct ionic_rx_desc_info),
3023 						      lif->kern_pid, NULL, &lif->rxqcqs[i]);
3024 				if (err)
3025 					goto err_out;
3026 			}
3027 
3028 			flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
3029 			err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
3030 					      num_desc, desc_sz, comp_sz, sg_desc_sz,
3031 					      sizeof(struct ionic_rx_desc_info),
3032 					      lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]);
3033 			if (err)
3034 				goto err_out;
3035 
3036 			rx_qcqs[i]->q.features = qparam->rxq_features;
3037 			rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog;
3038 		}
3039 	}
3040 
3041 	/* stop and clean the queues */
3042 	ionic_stop_queues_reconfig(lif);
3043 
3044 	if (qparam->nxqs != lif->nxqs) {
3045 		err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
3046 		if (err)
3047 			goto err_out_reinit_unlock;
3048 		err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
3049 		if (err) {
3050 			netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
3051 			goto err_out_reinit_unlock;
3052 		}
3053 	}
3054 
3055 	/* swap new desc_info and rings, keeping existing interrupt config */
3056 	if (tx_qcqs) {
3057 		lif->ntxq_descs = qparam->ntxq_descs;
3058 		for (i = 0; i < qparam->nxqs; i++)
3059 			ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
3060 	}
3061 
3062 	if (rx_qcqs) {
3063 		lif->nrxq_descs = qparam->nrxq_descs;
3064 		for (i = 0; i < qparam->nxqs; i++)
3065 			ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
3066 	}
3067 
3068 	/* if we need to change the interrupt layout, this is the time */
3069 	if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
3070 	    qparam->nxqs != lif->nxqs) {
3071 		if (qparam->intr_split) {
3072 			set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3073 		} else {
3074 			clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
3075 			lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3076 			lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3077 		}
3078 
3079 		/* Clear existing interrupt assignments.  We check for NULL here
3080 		 * because we're checking the whole array for potential qcqs, not
3081 		 * just those qcqs that have just been set up.
3082 		 */
3083 		for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
3084 			if (lif->txqcqs[i])
3085 				ionic_qcq_intr_free(lif, lif->txqcqs[i]);
3086 			if (lif->rxqcqs[i])
3087 				ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
3088 		}
3089 
3090 		/* re-assign the interrupts */
3091 		for (i = 0; i < qparam->nxqs; i++) {
3092 			lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3093 			err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
3094 			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3095 					     lif->rxqcqs[i]->intr.index,
3096 					     lif->rx_coalesce_hw);
3097 
3098 			if (qparam->intr_split) {
3099 				lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
3100 				err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
3101 				ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
3102 						     lif->txqcqs[i]->intr.index,
3103 						     lif->tx_coalesce_hw);
3104 				if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
3105 					lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
3106 			} else {
3107 				lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3108 				ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
3109 			}
3110 		}
3111 	}
3112 
3113 	/* now we can rework the debugfs mappings */
3114 	if (tx_qcqs) {
3115 		for (i = 0; i < qparam->nxqs; i++) {
3116 			ionic_debugfs_del_qcq(lif->txqcqs[i]);
3117 			ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
3118 		}
3119 	}
3120 
3121 	if (rx_qcqs) {
3122 		for (i = 0; i < qparam->nxqs; i++) {
3123 			ionic_debugfs_del_qcq(lif->rxqcqs[i]);
3124 			ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
3125 		}
3126 	}
3127 
3128 	swap(lif->nxqs, qparam->nxqs);
3129 	swap(lif->rxq_features, qparam->rxq_features);
3130 
3131 err_out_reinit_unlock:
3132 	/* re-init the queues, but don't lose an error code */
3133 	if (err)
3134 		ionic_start_queues_reconfig(lif);
3135 	else
3136 		err = ionic_start_queues_reconfig(lif);
3137 
3138 err_out:
3139 	/* free old allocs without cleaning intr */
3140 	for (i = 0; i < qparam->nxqs; i++) {
3141 		if (tx_qcqs && tx_qcqs[i]) {
3142 			tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3143 			ionic_qcq_free(lif, tx_qcqs[i]);
3144 			devm_kfree(lif->ionic->dev, tx_qcqs[i]);
3145 			tx_qcqs[i] = NULL;
3146 		}
3147 		if (rx_qcqs && rx_qcqs[i]) {
3148 			rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3149 			ionic_qcq_free(lif, rx_qcqs[i]);
3150 			devm_kfree(lif->ionic->dev, rx_qcqs[i]);
3151 			rx_qcqs[i] = NULL;
3152 		}
3153 	}
3154 
3155 	/* free q array */
3156 	if (rx_qcqs) {
3157 		devm_kfree(lif->ionic->dev, rx_qcqs);
3158 		rx_qcqs = NULL;
3159 	}
3160 	if (tx_qcqs) {
3161 		devm_kfree(lif->ionic->dev, tx_qcqs);
3162 		tx_qcqs = NULL;
3163 	}
3164 
3165 	/* clean the unused dma and info allocations when new set is smaller
3166 	 * than the full array, but leave the qcq shells in place
3167 	 */
3168 	for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
3169 		if (lif->txqcqs && lif->txqcqs[i]) {
3170 			lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3171 			ionic_qcq_free(lif, lif->txqcqs[i]);
3172 		}
3173 
3174 		if (lif->rxqcqs && lif->rxqcqs[i]) {
3175 			lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
3176 			ionic_qcq_free(lif, lif->rxqcqs[i]);
3177 		}
3178 	}
3179 
3180 	if (err)
3181 		netdev_info(lif->netdev, "%s: failed %d\n", __func__, err);
3182 
3183 	return err;
3184 }
3185 
ionic_affinity_masks_alloc(struct ionic * ionic)3186 static int ionic_affinity_masks_alloc(struct ionic *ionic)
3187 {
3188 	cpumask_var_t *affinity_masks;
3189 	int nintrs = ionic->nintrs;
3190 	int i;
3191 
3192 	affinity_masks = kcalloc(nintrs, sizeof(cpumask_var_t), GFP_KERNEL);
3193 	if (!affinity_masks)
3194 		return -ENOMEM;
3195 
3196 	for (i = 0; i < nintrs; i++) {
3197 		if (!zalloc_cpumask_var_node(&affinity_masks[i], GFP_KERNEL,
3198 					     dev_to_node(ionic->dev)))
3199 			goto err_out;
3200 	}
3201 
3202 	ionic->affinity_masks = affinity_masks;
3203 
3204 	return 0;
3205 
3206 err_out:
3207 	for (--i; i >= 0; i--)
3208 		free_cpumask_var(affinity_masks[i]);
3209 	kfree(affinity_masks);
3210 
3211 	return -ENOMEM;
3212 }
3213 
ionic_affinity_masks_free(struct ionic * ionic)3214 static void ionic_affinity_masks_free(struct ionic *ionic)
3215 {
3216 	int i;
3217 
3218 	for (i = 0; i < ionic->nintrs; i++)
3219 		free_cpumask_var(ionic->affinity_masks[i]);
3220 	kfree(ionic->affinity_masks);
3221 	ionic->affinity_masks = NULL;
3222 }
3223 
ionic_lif_alloc(struct ionic * ionic)3224 int ionic_lif_alloc(struct ionic *ionic)
3225 {
3226 	struct device *dev = ionic->dev;
3227 	union ionic_lif_identity *lid;
3228 	struct net_device *netdev;
3229 	struct ionic_lif *lif;
3230 	int tbl_sz;
3231 	int err;
3232 
3233 	lid = kzalloc(sizeof(*lid), GFP_KERNEL);
3234 	if (!lid)
3235 		return -ENOMEM;
3236 
3237 	netdev = alloc_etherdev_mqs(sizeof(*lif),
3238 				    ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
3239 	if (!netdev) {
3240 		dev_err(dev, "Cannot allocate netdev, aborting\n");
3241 		err = -ENOMEM;
3242 		goto err_out_free_lid;
3243 	}
3244 
3245 	SET_NETDEV_DEV(netdev, dev);
3246 
3247 	lif = netdev_priv(netdev);
3248 	lif->netdev = netdev;
3249 	ionic->lif = lif;
3250 	lif->ionic = ionic;
3251 	netdev->netdev_ops = &ionic_netdev_ops;
3252 	ionic_ethtool_set_ops(netdev);
3253 
3254 	netdev->watchdog_timeo = 5 * HZ;
3255 	netif_carrier_off(netdev);
3256 
3257 	lif->identity = lid;
3258 	lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
3259 	err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
3260 	if (err) {
3261 		dev_err(ionic->dev, "Cannot identify type %d: %d\n",
3262 			lif->lif_type, err);
3263 		goto err_out_free_netdev;
3264 	}
3265 	lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
3266 				     le32_to_cpu(lif->identity->eth.min_frame_size));
3267 	lif->netdev->max_mtu =
3268 		le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
3269 
3270 	lif->neqs = ionic->neqs_per_lif;
3271 	lif->nxqs = ionic->ntxqs_per_lif;
3272 
3273 	lif->index = 0;
3274 
3275 	if (is_kdump_kernel()) {
3276 		lif->ntxq_descs = IONIC_MIN_TXRX_DESC;
3277 		lif->nrxq_descs = IONIC_MIN_TXRX_DESC;
3278 	} else {
3279 		lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
3280 		lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
3281 	}
3282 
3283 	/* Convert the default coalesce value to actual hw resolution */
3284 	lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
3285 	lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
3286 						    lif->rx_coalesce_usecs);
3287 	lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
3288 	lif->tx_coalesce_hw = lif->rx_coalesce_hw;
3289 	set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
3290 	set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
3291 
3292 	snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
3293 
3294 	mutex_init(&lif->queue_lock);
3295 	mutex_init(&lif->config_lock);
3296 
3297 	spin_lock_init(&lif->adminq_lock);
3298 
3299 	spin_lock_init(&lif->deferred.lock);
3300 	INIT_LIST_HEAD(&lif->deferred.list);
3301 	INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
3302 
3303 	/* allocate lif info */
3304 	lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
3305 	lif->info = dma_alloc_coherent(dev, lif->info_sz,
3306 				       &lif->info_pa, GFP_KERNEL);
3307 	if (!lif->info) {
3308 		dev_err(dev, "Failed to allocate lif info, aborting\n");
3309 		err = -ENOMEM;
3310 		goto err_out_free_mutex;
3311 	}
3312 
3313 	ionic_debugfs_add_lif(lif);
3314 
3315 	err = ionic_affinity_masks_alloc(ionic);
3316 	if (err)
3317 		goto err_out_free_lif_info;
3318 
3319 	/* allocate control queues and txrx queue arrays */
3320 	ionic_lif_queue_identify(lif);
3321 	err = ionic_qcqs_alloc(lif);
3322 	if (err)
3323 		goto err_out_free_affinity_masks;
3324 
3325 	/* allocate rss indirection table */
3326 	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
3327 	lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
3328 	lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
3329 					      &lif->rss_ind_tbl_pa,
3330 					      GFP_KERNEL);
3331 
3332 	if (!lif->rss_ind_tbl) {
3333 		err = -ENOMEM;
3334 		dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
3335 		goto err_out_free_qcqs;
3336 	}
3337 	netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
3338 
3339 	ionic_lif_alloc_phc(lif);
3340 
3341 	return 0;
3342 
3343 err_out_free_qcqs:
3344 	ionic_qcqs_free(lif);
3345 err_out_free_affinity_masks:
3346 	ionic_affinity_masks_free(lif->ionic);
3347 err_out_free_lif_info:
3348 	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3349 	lif->info = NULL;
3350 	lif->info_pa = 0;
3351 err_out_free_mutex:
3352 	mutex_destroy(&lif->config_lock);
3353 	mutex_destroy(&lif->queue_lock);
3354 err_out_free_netdev:
3355 	free_netdev(lif->netdev);
3356 	lif = NULL;
3357 err_out_free_lid:
3358 	kfree(lid);
3359 
3360 	return err;
3361 }
3362 
ionic_lif_reset(struct ionic_lif * lif)3363 static void ionic_lif_reset(struct ionic_lif *lif)
3364 {
3365 	struct ionic_dev *idev = &lif->ionic->idev;
3366 
3367 	if (!ionic_is_fw_running(idev))
3368 		return;
3369 
3370 	mutex_lock(&lif->ionic->dev_cmd_lock);
3371 	ionic_dev_cmd_lif_reset(idev, lif->index);
3372 	ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3373 	mutex_unlock(&lif->ionic->dev_cmd_lock);
3374 }
3375 
ionic_lif_handle_fw_down(struct ionic_lif * lif)3376 static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
3377 {
3378 	struct ionic *ionic = lif->ionic;
3379 
3380 	if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
3381 		return;
3382 
3383 	dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
3384 
3385 	netif_device_detach(lif->netdev);
3386 
3387 	mutex_lock(&lif->queue_lock);
3388 	if (test_bit(IONIC_LIF_F_UP, lif->state)) {
3389 		dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
3390 		ionic_stop_queues(lif);
3391 	}
3392 
3393 	if (netif_running(lif->netdev)) {
3394 		ionic_txrx_deinit(lif);
3395 		ionic_txrx_free(lif);
3396 	}
3397 	ionic_lif_deinit(lif);
3398 	ionic_reset(ionic);
3399 	ionic_qcqs_free(lif);
3400 
3401 	mutex_unlock(&lif->queue_lock);
3402 
3403 	clear_bit(IONIC_LIF_F_FW_STOPPING, lif->state);
3404 	dev_info(ionic->dev, "FW Down: LIFs stopped\n");
3405 }
3406 
ionic_restart_lif(struct ionic_lif * lif)3407 int ionic_restart_lif(struct ionic_lif *lif)
3408 {
3409 	struct ionic *ionic = lif->ionic;
3410 	int err;
3411 
3412 	mutex_lock(&lif->queue_lock);
3413 
3414 	if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
3415 		dev_info(ionic->dev, "FW Up: clearing broken state\n");
3416 
3417 	err = ionic_qcqs_alloc(lif);
3418 	if (err)
3419 		goto err_unlock;
3420 
3421 	err = ionic_lif_init(lif);
3422 	if (err)
3423 		goto err_qcqs_free;
3424 
3425 	ionic_vf_attr_replay(lif);
3426 
3427 	if (lif->registered)
3428 		ionic_lif_set_netdev_info(lif);
3429 
3430 	ionic_rx_filter_replay(lif);
3431 
3432 	if (netif_running(lif->netdev)) {
3433 		err = ionic_txrx_alloc(lif);
3434 		if (err)
3435 			goto err_lifs_deinit;
3436 
3437 		err = ionic_txrx_init(lif);
3438 		if (err)
3439 			goto err_txrx_free;
3440 	}
3441 
3442 	mutex_unlock(&lif->queue_lock);
3443 
3444 	clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
3445 	ionic_link_status_check_request(lif, CAN_SLEEP);
3446 	netif_device_attach(lif->netdev);
3447 	ionic_queue_doorbell_check(ionic, IONIC_NAPI_DEADLINE);
3448 
3449 	return 0;
3450 
3451 err_txrx_free:
3452 	ionic_txrx_free(lif);
3453 err_lifs_deinit:
3454 	ionic_lif_deinit(lif);
3455 err_qcqs_free:
3456 	ionic_qcqs_free(lif);
3457 err_unlock:
3458 	mutex_unlock(&lif->queue_lock);
3459 
3460 	return err;
3461 }
3462 
ionic_lif_handle_fw_up(struct ionic_lif * lif)3463 static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
3464 {
3465 	struct ionic *ionic = lif->ionic;
3466 	int err;
3467 
3468 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3469 		return;
3470 
3471 	dev_info(ionic->dev, "FW Up: restarting LIFs\n");
3472 
3473 	/* This is a little different from what happens at
3474 	 * probe time because the LIF already exists so we
3475 	 * just need to reanimate it.
3476 	 */
3477 	ionic_init_devinfo(ionic);
3478 	ionic_reset(ionic);
3479 	err = ionic_identify(ionic);
3480 	if (err)
3481 		goto err_out;
3482 	err = ionic_port_identify(ionic);
3483 	if (err)
3484 		goto err_out;
3485 	err = ionic_port_init(ionic);
3486 	if (err)
3487 		goto err_out;
3488 
3489 	err = ionic_restart_lif(lif);
3490 	if (err)
3491 		goto err_out;
3492 
3493 	dev_info(ionic->dev, "FW Up: LIFs restarted\n");
3494 
3495 	/* restore the hardware timestamping queues */
3496 	ionic_lif_hwstamp_replay(lif);
3497 
3498 	return;
3499 
3500 err_out:
3501 	dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3502 }
3503 
ionic_lif_free(struct ionic_lif * lif)3504 void ionic_lif_free(struct ionic_lif *lif)
3505 {
3506 	struct device *dev = lif->ionic->dev;
3507 
3508 	ionic_lif_free_phc(lif);
3509 
3510 	/* free rss indirection table */
3511 	dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3512 			  lif->rss_ind_tbl_pa);
3513 	lif->rss_ind_tbl = NULL;
3514 	lif->rss_ind_tbl_pa = 0;
3515 
3516 	/* free queues */
3517 	ionic_qcqs_free(lif);
3518 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3519 		ionic_lif_reset(lif);
3520 
3521 	ionic_affinity_masks_free(lif->ionic);
3522 
3523 	/* free lif info */
3524 	kfree(lif->identity);
3525 	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3526 	lif->info = NULL;
3527 	lif->info_pa = 0;
3528 
3529 	/* unmap doorbell page */
3530 	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3531 	lif->kern_dbpage = NULL;
3532 
3533 	mutex_destroy(&lif->config_lock);
3534 	mutex_destroy(&lif->queue_lock);
3535 
3536 	/* free netdev & lif */
3537 	ionic_debugfs_del_lif(lif);
3538 	free_netdev(lif->netdev);
3539 }
3540 
ionic_lif_deinit(struct ionic_lif * lif)3541 void ionic_lif_deinit(struct ionic_lif *lif)
3542 {
3543 	if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3544 		return;
3545 
3546 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3547 		cancel_work_sync(&lif->deferred.work);
3548 		cancel_work_sync(&lif->tx_timeout_work);
3549 		ionic_rx_filters_deinit(lif);
3550 		if (lif->netdev->features & NETIF_F_RXHASH)
3551 			ionic_lif_rss_deinit(lif);
3552 	}
3553 
3554 	napi_disable(&lif->adminqcq->napi);
3555 	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3556 	ionic_lif_qcq_deinit(lif, lif->adminqcq);
3557 
3558 	ionic_lif_reset(lif);
3559 }
3560 
ionic_lif_adminq_init(struct ionic_lif * lif)3561 static int ionic_lif_adminq_init(struct ionic_lif *lif)
3562 {
3563 	struct device *dev = lif->ionic->dev;
3564 	struct ionic_q_init_comp comp;
3565 	struct ionic_dev *idev;
3566 	struct ionic_qcq *qcq;
3567 	struct ionic_queue *q;
3568 	int err;
3569 
3570 	idev = &lif->ionic->idev;
3571 	qcq = lif->adminqcq;
3572 	q = &qcq->q;
3573 
3574 	mutex_lock(&lif->ionic->dev_cmd_lock);
3575 	ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3576 	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3577 	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3578 	mutex_unlock(&lif->ionic->dev_cmd_lock);
3579 	if (err) {
3580 		netdev_err(lif->netdev, "adminq init failed %d\n", err);
3581 		return err;
3582 	}
3583 
3584 	q->hw_type = comp.hw_type;
3585 	q->hw_index = le32_to_cpu(comp.hw_index);
3586 	q->dbval = IONIC_DBELL_QID(q->hw_index);
3587 
3588 	dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3589 	dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3590 
3591 	q->dbell_deadline = IONIC_ADMIN_DOORBELL_DEADLINE;
3592 	q->dbell_jiffies = jiffies;
3593 
3594 	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi);
3595 
3596 	napi_enable(&qcq->napi);
3597 
3598 	if (qcq->flags & IONIC_QCQ_F_INTR) {
3599 		irq_set_affinity_hint(qcq->intr.vector,
3600 				      *qcq->intr.affinity_mask);
3601 		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3602 				IONIC_INTR_MASK_CLEAR);
3603 	}
3604 
3605 	qcq->flags |= IONIC_QCQ_F_INITED;
3606 
3607 	return 0;
3608 }
3609 
ionic_lif_notifyq_init(struct ionic_lif * lif)3610 static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3611 {
3612 	struct ionic_qcq *qcq = lif->notifyqcq;
3613 	struct device *dev = lif->ionic->dev;
3614 	struct ionic_queue *q = &qcq->q;
3615 	int err;
3616 
3617 	struct ionic_admin_ctx ctx = {
3618 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3619 		.cmd.q_init = {
3620 			.opcode = IONIC_CMD_Q_INIT,
3621 			.lif_index = cpu_to_le16(lif->index),
3622 			.type = q->type,
3623 			.ver = lif->qtype_info[q->type].version,
3624 			.index = cpu_to_le32(q->index),
3625 			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3626 					     IONIC_QINIT_F_ENA),
3627 			.intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3628 			.pid = cpu_to_le16(q->pid),
3629 			.ring_size = ilog2(q->num_descs),
3630 			.ring_base = cpu_to_le64(q->base_pa),
3631 		}
3632 	};
3633 
3634 	dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3635 	dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3636 	dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3637 	dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3638 
3639 	err = ionic_adminq_post_wait(lif, &ctx);
3640 	if (err)
3641 		return err;
3642 
3643 	lif->last_eid = 0;
3644 	q->hw_type = ctx.comp.q_init.hw_type;
3645 	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3646 	q->dbval = IONIC_DBELL_QID(q->hw_index);
3647 
3648 	dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3649 	dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3650 
3651 	/* preset the callback info */
3652 	q->admin_info[0].ctx = lif;
3653 
3654 	qcq->flags |= IONIC_QCQ_F_INITED;
3655 
3656 	return 0;
3657 }
3658 
ionic_station_set(struct ionic_lif * lif)3659 static int ionic_station_set(struct ionic_lif *lif)
3660 {
3661 	struct net_device *netdev = lif->netdev;
3662 	struct ionic_admin_ctx ctx = {
3663 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3664 		.cmd.lif_getattr = {
3665 			.opcode = IONIC_CMD_LIF_GETATTR,
3666 			.index = cpu_to_le16(lif->index),
3667 			.attr = IONIC_LIF_ATTR_MAC,
3668 		},
3669 	};
3670 	u8 mac_address[ETH_ALEN];
3671 	struct sockaddr addr;
3672 	int err;
3673 
3674 	err = ionic_adminq_post_wait(lif, &ctx);
3675 	if (err)
3676 		return err;
3677 	netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3678 		   ctx.comp.lif_getattr.mac);
3679 	ether_addr_copy(mac_address, ctx.comp.lif_getattr.mac);
3680 
3681 	if (is_zero_ether_addr(mac_address)) {
3682 		eth_hw_addr_random(netdev);
3683 		netdev_dbg(netdev, "Random Mac generated: %pM\n", netdev->dev_addr);
3684 		ether_addr_copy(mac_address, netdev->dev_addr);
3685 
3686 		err = ionic_program_mac(lif, mac_address);
3687 		if (err < 0)
3688 			return err;
3689 
3690 		if (err > 0) {
3691 			netdev_dbg(netdev, "%s:SET/GET ATTR Mac are not same-due to old FW running\n",
3692 				   __func__);
3693 			return 0;
3694 		}
3695 	}
3696 
3697 	if (!is_zero_ether_addr(netdev->dev_addr)) {
3698 		/* If the netdev mac is non-zero and doesn't match the default
3699 		 * device address, it was set by something earlier and we're
3700 		 * likely here again after a fw-upgrade reset.  We need to be
3701 		 * sure the netdev mac is in our filter list.
3702 		 */
3703 		if (!ether_addr_equal(mac_address, netdev->dev_addr))
3704 			ionic_lif_addr_add(lif, netdev->dev_addr);
3705 	} else {
3706 		/* Update the netdev mac with the device's mac */
3707 		ether_addr_copy(addr.sa_data, mac_address);
3708 		addr.sa_family = AF_INET;
3709 		err = eth_prepare_mac_addr_change(netdev, &addr);
3710 		if (err) {
3711 			netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3712 				    addr.sa_data, err);
3713 			return 0;
3714 		}
3715 
3716 		eth_commit_mac_addr_change(netdev, &addr);
3717 	}
3718 
3719 	netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3720 		   netdev->dev_addr);
3721 	ionic_lif_addr_add(lif, netdev->dev_addr);
3722 
3723 	return 0;
3724 }
3725 
ionic_lif_init(struct ionic_lif * lif)3726 int ionic_lif_init(struct ionic_lif *lif)
3727 {
3728 	struct ionic_dev *idev = &lif->ionic->idev;
3729 	struct device *dev = lif->ionic->dev;
3730 	struct ionic_lif_init_comp comp;
3731 	int dbpage_num;
3732 	int err;
3733 
3734 	mutex_lock(&lif->ionic->dev_cmd_lock);
3735 	ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3736 	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3737 	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3738 	mutex_unlock(&lif->ionic->dev_cmd_lock);
3739 	if (err)
3740 		return err;
3741 
3742 	lif->hw_index = le16_to_cpu(comp.hw_index);
3743 
3744 	/* now that we have the hw_index we can figure out our doorbell page */
3745 	lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3746 	if (!lif->dbid_count) {
3747 		dev_err(dev, "No doorbell pages, aborting\n");
3748 		return -EINVAL;
3749 	}
3750 
3751 	lif->kern_pid = 0;
3752 	dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3753 	lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3754 	if (!lif->kern_dbpage) {
3755 		dev_err(dev, "Cannot map dbpage, aborting\n");
3756 		return -ENOMEM;
3757 	}
3758 
3759 	err = ionic_lif_adminq_init(lif);
3760 	if (err)
3761 		goto err_out_adminq_deinit;
3762 
3763 	if (lif->ionic->nnqs_per_lif) {
3764 		err = ionic_lif_notifyq_init(lif);
3765 		if (err)
3766 			goto err_out_notifyq_deinit;
3767 	}
3768 
3769 	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3770 		err = ionic_set_nic_features(lif, lif->netdev->features);
3771 	else
3772 		err = ionic_init_nic_features(lif);
3773 	if (err)
3774 		goto err_out_notifyq_deinit;
3775 
3776 	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3777 		err = ionic_rx_filters_init(lif);
3778 		if (err)
3779 			goto err_out_notifyq_deinit;
3780 	}
3781 
3782 	err = ionic_station_set(lif);
3783 	if (err)
3784 		goto err_out_notifyq_deinit;
3785 
3786 	lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3787 	lif->doorbell_wa = ionic_doorbell_wa(lif->ionic);
3788 
3789 	set_bit(IONIC_LIF_F_INITED, lif->state);
3790 
3791 	INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3792 
3793 	return 0;
3794 
3795 err_out_notifyq_deinit:
3796 	napi_disable(&lif->adminqcq->napi);
3797 	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3798 err_out_adminq_deinit:
3799 	ionic_lif_qcq_deinit(lif, lif->adminqcq);
3800 	ionic_lif_reset(lif);
3801 	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3802 	lif->kern_dbpage = NULL;
3803 
3804 	return err;
3805 }
3806 
ionic_lif_notify_work(struct work_struct * ws)3807 static void ionic_lif_notify_work(struct work_struct *ws)
3808 {
3809 }
3810 
ionic_lif_set_netdev_info(struct ionic_lif * lif)3811 static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3812 {
3813 	struct ionic_admin_ctx ctx = {
3814 		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3815 		.cmd.lif_setattr = {
3816 			.opcode = IONIC_CMD_LIF_SETATTR,
3817 			.index = cpu_to_le16(lif->index),
3818 			.attr = IONIC_LIF_ATTR_NAME,
3819 		},
3820 	};
3821 
3822 	strscpy(ctx.cmd.lif_setattr.name, netdev_name(lif->netdev),
3823 		sizeof(ctx.cmd.lif_setattr.name));
3824 
3825 	ionic_adminq_post_wait(lif, &ctx);
3826 }
3827 
ionic_netdev_lif(struct net_device * netdev)3828 static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3829 {
3830 	if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3831 		return NULL;
3832 
3833 	return netdev_priv(netdev);
3834 }
3835 
ionic_lif_notify(struct notifier_block * nb,unsigned long event,void * info)3836 static int ionic_lif_notify(struct notifier_block *nb,
3837 			    unsigned long event, void *info)
3838 {
3839 	struct net_device *ndev = netdev_notifier_info_to_dev(info);
3840 	struct ionic *ionic = container_of(nb, struct ionic, nb);
3841 	struct ionic_lif *lif = ionic_netdev_lif(ndev);
3842 
3843 	if (!lif || lif->ionic != ionic)
3844 		return NOTIFY_DONE;
3845 
3846 	switch (event) {
3847 	case NETDEV_CHANGENAME:
3848 		ionic_lif_set_netdev_info(lif);
3849 		break;
3850 	}
3851 
3852 	return NOTIFY_DONE;
3853 }
3854 
ionic_lif_register(struct ionic_lif * lif)3855 int ionic_lif_register(struct ionic_lif *lif)
3856 {
3857 	int err;
3858 
3859 	ionic_lif_register_phc(lif);
3860 
3861 	INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3862 
3863 	lif->ionic->nb.notifier_call = ionic_lif_notify;
3864 
3865 	err = register_netdevice_notifier(&lif->ionic->nb);
3866 	if (err)
3867 		lif->ionic->nb.notifier_call = NULL;
3868 
3869 	/* only register LIF0 for now */
3870 	err = register_netdev(lif->netdev);
3871 	if (err) {
3872 		dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3873 		ionic_lif_unregister_phc(lif);
3874 		return err;
3875 	}
3876 
3877 	ionic_link_status_check_request(lif, CAN_SLEEP);
3878 	lif->registered = true;
3879 	ionic_lif_set_netdev_info(lif);
3880 
3881 	return 0;
3882 }
3883 
ionic_lif_unregister(struct ionic_lif * lif)3884 void ionic_lif_unregister(struct ionic_lif *lif)
3885 {
3886 	if (lif->ionic->nb.notifier_call) {
3887 		unregister_netdevice_notifier(&lif->ionic->nb);
3888 		cancel_work_sync(&lif->ionic->nb_work);
3889 		lif->ionic->nb.notifier_call = NULL;
3890 	}
3891 
3892 	if (lif->netdev->reg_state == NETREG_REGISTERED)
3893 		unregister_netdev(lif->netdev);
3894 
3895 	ionic_lif_unregister_phc(lif);
3896 
3897 	lif->registered = false;
3898 }
3899 
ionic_lif_queue_identify(struct ionic_lif * lif)3900 static void ionic_lif_queue_identify(struct ionic_lif *lif)
3901 {
3902 	union ionic_q_identity __iomem *q_ident;
3903 	struct ionic *ionic = lif->ionic;
3904 	struct ionic_dev *idev;
3905 	u16 max_frags;
3906 	int qtype;
3907 	int err;
3908 
3909 	idev = &lif->ionic->idev;
3910 	q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3911 
3912 	for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3913 		struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3914 
3915 		/* filter out the ones we know about */
3916 		switch (qtype) {
3917 		case IONIC_QTYPE_ADMINQ:
3918 		case IONIC_QTYPE_NOTIFYQ:
3919 		case IONIC_QTYPE_RXQ:
3920 		case IONIC_QTYPE_TXQ:
3921 			break;
3922 		default:
3923 			continue;
3924 		}
3925 
3926 		memset(qti, 0, sizeof(*qti));
3927 
3928 		mutex_lock(&ionic->dev_cmd_lock);
3929 		ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3930 					     ionic_qtype_versions[qtype]);
3931 		err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3932 		if (!err) {
3933 			qti->version   = readb(&q_ident->version);
3934 			qti->supported = readb(&q_ident->supported);
3935 			qti->features  = readq(&q_ident->features);
3936 			qti->desc_sz   = readw(&q_ident->desc_sz);
3937 			qti->comp_sz   = readw(&q_ident->comp_sz);
3938 			qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3939 			qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3940 			qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3941 		}
3942 		mutex_unlock(&ionic->dev_cmd_lock);
3943 
3944 		if (err == -EINVAL) {
3945 			dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3946 			continue;
3947 		} else if (err == -EIO) {
3948 			dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3949 			return;
3950 		} else if (err) {
3951 			dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3952 				qtype, err);
3953 			return;
3954 		}
3955 
3956 		dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3957 			qtype, qti->version);
3958 		dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3959 			qtype, qti->supported);
3960 		dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3961 			qtype, qti->features);
3962 		dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3963 			qtype, qti->desc_sz);
3964 		dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3965 			qtype, qti->comp_sz);
3966 		dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3967 			qtype, qti->sg_desc_sz);
3968 		dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3969 			qtype, qti->max_sg_elems);
3970 		dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3971 			qtype, qti->sg_desc_stride);
3972 
3973 		if (qtype == IONIC_QTYPE_TXQ)
3974 			max_frags = IONIC_TX_MAX_FRAGS;
3975 		else if (qtype == IONIC_QTYPE_RXQ)
3976 			max_frags = IONIC_RX_MAX_FRAGS;
3977 		else
3978 			max_frags = 1;
3979 
3980 		qti->max_sg_elems = min_t(u16, max_frags - 1, MAX_SKB_FRAGS);
3981 		dev_dbg(ionic->dev, "qtype %d max_sg_elems %d\n",
3982 			qtype, qti->max_sg_elems);
3983 	}
3984 }
3985 
ionic_lif_identify(struct ionic * ionic,u8 lif_type,union ionic_lif_identity * lid)3986 int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3987 		       union ionic_lif_identity *lid)
3988 {
3989 	struct ionic_dev *idev = &ionic->idev;
3990 	size_t sz;
3991 	int err;
3992 
3993 	sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3994 
3995 	mutex_lock(&ionic->dev_cmd_lock);
3996 	ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3997 	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3998 	memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3999 	mutex_unlock(&ionic->dev_cmd_lock);
4000 	if (err)
4001 		return (err);
4002 
4003 	dev_dbg(ionic->dev, "capabilities 0x%llx\n",
4004 		le64_to_cpu(lid->capabilities));
4005 
4006 	dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
4007 		le32_to_cpu(lid->eth.max_ucast_filters));
4008 	dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
4009 		le32_to_cpu(lid->eth.max_mcast_filters));
4010 	dev_dbg(ionic->dev, "eth.features 0x%llx\n",
4011 		le64_to_cpu(lid->eth.config.features));
4012 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
4013 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
4014 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
4015 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
4016 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
4017 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
4018 	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
4019 		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
4020 	dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
4021 	dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
4022 	dev_dbg(ionic->dev, "eth.config.mtu %d\n",
4023 		le32_to_cpu(lid->eth.config.mtu));
4024 
4025 	return 0;
4026 }
4027 
ionic_lif_size(struct ionic * ionic)4028 int ionic_lif_size(struct ionic *ionic)
4029 {
4030 	struct ionic_identity *ident = &ionic->ident;
4031 	unsigned int nintrs, dev_nintrs;
4032 	union ionic_lif_config *lc;
4033 	unsigned int ntxqs_per_lif;
4034 	unsigned int nrxqs_per_lif;
4035 	unsigned int neqs_per_lif;
4036 	unsigned int nnqs_per_lif;
4037 	unsigned int nxqs, neqs;
4038 	unsigned int min_intrs;
4039 	int err;
4040 
4041 	/* retrieve basic values from FW */
4042 	lc = &ident->lif.eth.config;
4043 	dev_nintrs = le32_to_cpu(ident->dev.nintrs);
4044 	neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
4045 	nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
4046 	ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
4047 	nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
4048 
4049 	/* limit values to play nice with kdump */
4050 	if (is_kdump_kernel()) {
4051 		dev_nintrs = 2;
4052 		neqs_per_lif = 0;
4053 		nnqs_per_lif = 0;
4054 		ntxqs_per_lif = 1;
4055 		nrxqs_per_lif = 1;
4056 	}
4057 
4058 	/* reserve last queue id for hardware timestamping */
4059 	if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
4060 		if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
4061 			lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
4062 		} else {
4063 			ntxqs_per_lif -= 1;
4064 			nrxqs_per_lif -= 1;
4065 		}
4066 	}
4067 
4068 	nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
4069 	nxqs = min(nxqs, num_online_cpus());
4070 	neqs = min(neqs_per_lif, num_online_cpus());
4071 
4072 try_again:
4073 	/* interrupt usage:
4074 	 *    1 for master lif adminq/notifyq
4075 	 *    1 for each CPU for master lif TxRx queue pairs
4076 	 *    whatever's left is for RDMA queues
4077 	 */
4078 	nintrs = 1 + nxqs + neqs;
4079 	min_intrs = 2;  /* adminq + 1 TxRx queue pair */
4080 
4081 	if (nintrs > dev_nintrs)
4082 		goto try_fewer;
4083 
4084 	err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
4085 	if (err < 0 && err != -ENOSPC) {
4086 		dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
4087 		return err;
4088 	}
4089 	if (err == -ENOSPC)
4090 		goto try_fewer;
4091 
4092 	if (err != nintrs) {
4093 		ionic_bus_free_irq_vectors(ionic);
4094 		goto try_fewer;
4095 	}
4096 
4097 	ionic->nnqs_per_lif = nnqs_per_lif;
4098 	ionic->neqs_per_lif = neqs;
4099 	ionic->ntxqs_per_lif = nxqs;
4100 	ionic->nrxqs_per_lif = nxqs;
4101 	ionic->nintrs = nintrs;
4102 
4103 	ionic_debugfs_add_sizes(ionic);
4104 
4105 	return 0;
4106 
4107 try_fewer:
4108 	if (nnqs_per_lif > 1) {
4109 		nnqs_per_lif >>= 1;
4110 		goto try_again;
4111 	}
4112 	if (neqs > 1) {
4113 		neqs >>= 1;
4114 		goto try_again;
4115 	}
4116 	if (nxqs > 1) {
4117 		nxqs >>= 1;
4118 		goto try_again;
4119 	}
4120 	dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
4121 	return -ENOSPC;
4122 }
4123