xref: /linux/drivers/net/ethernet/chelsio/cxgb4vf/cxgb4vf_main.c (revision bab2c80e5a6c855657482eac9e97f5f3eedb509a)
1 /*
2  * This file is part of the Chelsio T4 PCI-E SR-IOV Virtual Function Ethernet
3  * driver for Linux.
4  *
5  * Copyright (c) 2009-2010 Chelsio Communications, Inc. All rights reserved.
6  *
7  * This software is available to you under a choice of one of two
8  * licenses.  You may choose to be licensed under the terms of the GNU
9  * General Public License (GPL) Version 2, available from the file
10  * COPYING in the main directory of this source tree, or the
11  * OpenIB.org BSD license below:
12  *
13  *     Redistribution and use in source and binary forms, with or
14  *     without modification, are permitted provided that the following
15  *     conditions are met:
16  *
17  *      - Redistributions of source code must retain the above
18  *        copyright notice, this list of conditions and the following
19  *        disclaimer.
20  *
21  *      - Redistributions in binary form must reproduce the above
22  *        copyright notice, this list of conditions and the following
23  *        disclaimer in the documentation and/or other materials
24  *        provided with the distribution.
25  *
26  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
27  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
28  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
29  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
30  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
31  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
32  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
33  * SOFTWARE.
34  */
35 
36 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
37 
38 #include <linux/module.h>
39 #include <linux/moduleparam.h>
40 #include <linux/init.h>
41 #include <linux/pci.h>
42 #include <linux/dma-mapping.h>
43 #include <linux/netdevice.h>
44 #include <linux/etherdevice.h>
45 #include <linux/debugfs.h>
46 #include <linux/ethtool.h>
47 #include <linux/mdio.h>
48 
49 #include "t4vf_common.h"
50 #include "t4vf_defs.h"
51 
52 #include "../cxgb4/t4_regs.h"
53 #include "../cxgb4/t4_msg.h"
54 
55 /*
56  * Generic information about the driver.
57  */
58 #define DRV_VERSION "2.0.0-ko"
59 #define DRV_DESC "Chelsio T4/T5/T6 Virtual Function (VF) Network Driver"
60 
61 /*
62  * Module Parameters.
63  * ==================
64  */
65 
66 /*
67  * Default ethtool "message level" for adapters.
68  */
69 #define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
70 			 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
71 			 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
72 
73 /*
74  * The driver uses the best interrupt scheme available on a platform in the
75  * order MSI-X then MSI.  This parameter determines which of these schemes the
76  * driver may consider as follows:
77  *
78  *     msi = 2: choose from among MSI-X and MSI
79  *     msi = 1: only consider MSI interrupts
80  *
81  * Note that unlike the Physical Function driver, this Virtual Function driver
82  * does _not_ support legacy INTx interrupts (this limitation is mandated by
83  * the PCI-E SR-IOV standard).
84  */
85 #define MSI_MSIX	2
86 #define MSI_MSI		1
87 #define MSI_DEFAULT	MSI_MSIX
88 
89 static int msi = MSI_DEFAULT;
90 
91 module_param(msi, int, 0644);
92 MODULE_PARM_DESC(msi, "whether to use MSI-X or MSI");
93 
94 /*
95  * Fundamental constants.
96  * ======================
97  */
98 
99 enum {
100 	MAX_TXQ_ENTRIES		= 16384,
101 	MAX_RSPQ_ENTRIES	= 16384,
102 	MAX_RX_BUFFERS		= 16384,
103 
104 	MIN_TXQ_ENTRIES		= 32,
105 	MIN_RSPQ_ENTRIES	= 128,
106 	MIN_FL_ENTRIES		= 16,
107 
108 	/*
109 	 * For purposes of manipulating the Free List size we need to
110 	 * recognize that Free Lists are actually Egress Queues (the host
111 	 * produces free buffers which the hardware consumes), Egress Queues
112 	 * indices are all in units of Egress Context Units bytes, and free
113 	 * list entries are 64-bit PCI DMA addresses.  And since the state of
114 	 * the Producer Index == the Consumer Index implies an EMPTY list, we
115 	 * always have at least one Egress Unit's worth of Free List entries
116 	 * unused.  See sge.c for more details ...
117 	 */
118 	EQ_UNIT = SGE_EQ_IDXSIZE,
119 	FL_PER_EQ_UNIT = EQ_UNIT / sizeof(__be64),
120 	MIN_FL_RESID = FL_PER_EQ_UNIT,
121 };
122 
123 /*
124  * Global driver state.
125  * ====================
126  */
127 
128 static struct dentry *cxgb4vf_debugfs_root;
129 
130 /*
131  * OS "Callback" functions.
132  * ========================
133  */
134 
135 /*
136  * The link status has changed on the indicated "port" (Virtual Interface).
137  */
138 void t4vf_os_link_changed(struct adapter *adapter, int pidx, int link_ok)
139 {
140 	struct net_device *dev = adapter->port[pidx];
141 
142 	/*
143 	 * If the port is disabled or the current recorded "link up"
144 	 * status matches the new status, just return.
145 	 */
146 	if (!netif_running(dev) || link_ok == netif_carrier_ok(dev))
147 		return;
148 
149 	/*
150 	 * Tell the OS that the link status has changed and print a short
151 	 * informative message on the console about the event.
152 	 */
153 	if (link_ok) {
154 		const char *s;
155 		const char *fc;
156 		const struct port_info *pi = netdev_priv(dev);
157 
158 		switch (pi->link_cfg.speed) {
159 		case 100:
160 			s = "100Mbps";
161 			break;
162 		case 1000:
163 			s = "1Gbps";
164 			break;
165 		case 10000:
166 			s = "10Gbps";
167 			break;
168 		case 25000:
169 			s = "25Gbps";
170 			break;
171 		case 40000:
172 			s = "40Gbps";
173 			break;
174 		case 100000:
175 			s = "100Gbps";
176 			break;
177 
178 		default:
179 			s = "unknown";
180 			break;
181 		}
182 
183 		switch ((int)pi->link_cfg.fc) {
184 		case PAUSE_RX:
185 			fc = "RX";
186 			break;
187 
188 		case PAUSE_TX:
189 			fc = "TX";
190 			break;
191 
192 		case PAUSE_RX | PAUSE_TX:
193 			fc = "RX/TX";
194 			break;
195 
196 		default:
197 			fc = "no";
198 			break;
199 		}
200 
201 		netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s, fc);
202 	} else {
203 		netdev_info(dev, "link down\n");
204 	}
205 }
206 
207 /*
208  * THe port module type has changed on the indicated "port" (Virtual
209  * Interface).
210  */
211 void t4vf_os_portmod_changed(struct adapter *adapter, int pidx)
212 {
213 	static const char * const mod_str[] = {
214 		NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
215 	};
216 	const struct net_device *dev = adapter->port[pidx];
217 	const struct port_info *pi = netdev_priv(dev);
218 
219 	if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
220 		dev_info(adapter->pdev_dev, "%s: port module unplugged\n",
221 			 dev->name);
222 	else if (pi->mod_type < ARRAY_SIZE(mod_str))
223 		dev_info(adapter->pdev_dev, "%s: %s port module inserted\n",
224 			 dev->name, mod_str[pi->mod_type]);
225 	else if (pi->mod_type == FW_PORT_MOD_TYPE_NOTSUPPORTED)
226 		dev_info(adapter->pdev_dev, "%s: unsupported optical port "
227 			 "module inserted\n", dev->name);
228 	else if (pi->mod_type == FW_PORT_MOD_TYPE_UNKNOWN)
229 		dev_info(adapter->pdev_dev, "%s: unknown port module inserted,"
230 			 "forcing TWINAX\n", dev->name);
231 	else if (pi->mod_type == FW_PORT_MOD_TYPE_ERROR)
232 		dev_info(adapter->pdev_dev, "%s: transceiver module error\n",
233 			 dev->name);
234 	else
235 		dev_info(adapter->pdev_dev, "%s: unknown module type %d "
236 			 "inserted\n", dev->name, pi->mod_type);
237 }
238 
239 /*
240  * Net device operations.
241  * ======================
242  */
243 
244 
245 
246 
247 /*
248  * Perform the MAC and PHY actions needed to enable a "port" (Virtual
249  * Interface).
250  */
251 static int link_start(struct net_device *dev)
252 {
253 	int ret;
254 	struct port_info *pi = netdev_priv(dev);
255 
256 	/*
257 	 * We do not set address filters and promiscuity here, the stack does
258 	 * that step explicitly. Enable vlan accel.
259 	 */
260 	ret = t4vf_set_rxmode(pi->adapter, pi->viid, dev->mtu, -1, -1, -1, 1,
261 			      true);
262 	if (ret == 0) {
263 		ret = t4vf_change_mac(pi->adapter, pi->viid,
264 				      pi->xact_addr_filt, dev->dev_addr, true);
265 		if (ret >= 0) {
266 			pi->xact_addr_filt = ret;
267 			ret = 0;
268 		}
269 	}
270 
271 	/*
272 	 * We don't need to actually "start the link" itself since the
273 	 * firmware will do that for us when the first Virtual Interface
274 	 * is enabled on a port.
275 	 */
276 	if (ret == 0)
277 		ret = t4vf_enable_pi(pi->adapter, pi, true, true);
278 
279 	/* The Virtual Interfaces are connected to an internal switch on the
280 	 * chip which allows VIs attached to the same port to talk to each
281 	 * other even when the port link is down.  As a result, we generally
282 	 * want to always report a VI's link as being "up", provided there are
283 	 * no errors in enabling vi.
284 	 */
285 
286 	if (ret == 0)
287 		netif_carrier_on(dev);
288 
289 	return ret;
290 }
291 
292 /*
293  * Name the MSI-X interrupts.
294  */
295 static void name_msix_vecs(struct adapter *adapter)
296 {
297 	int namelen = sizeof(adapter->msix_info[0].desc) - 1;
298 	int pidx;
299 
300 	/*
301 	 * Firmware events.
302 	 */
303 	snprintf(adapter->msix_info[MSIX_FW].desc, namelen,
304 		 "%s-FWeventq", adapter->name);
305 	adapter->msix_info[MSIX_FW].desc[namelen] = 0;
306 
307 	/*
308 	 * Ethernet queues.
309 	 */
310 	for_each_port(adapter, pidx) {
311 		struct net_device *dev = adapter->port[pidx];
312 		const struct port_info *pi = netdev_priv(dev);
313 		int qs, msi;
314 
315 		for (qs = 0, msi = MSIX_IQFLINT; qs < pi->nqsets; qs++, msi++) {
316 			snprintf(adapter->msix_info[msi].desc, namelen,
317 				 "%s-%d", dev->name, qs);
318 			adapter->msix_info[msi].desc[namelen] = 0;
319 		}
320 	}
321 }
322 
323 /*
324  * Request all of our MSI-X resources.
325  */
326 static int request_msix_queue_irqs(struct adapter *adapter)
327 {
328 	struct sge *s = &adapter->sge;
329 	int rxq, msi, err;
330 
331 	/*
332 	 * Firmware events.
333 	 */
334 	err = request_irq(adapter->msix_info[MSIX_FW].vec, t4vf_sge_intr_msix,
335 			  0, adapter->msix_info[MSIX_FW].desc, &s->fw_evtq);
336 	if (err)
337 		return err;
338 
339 	/*
340 	 * Ethernet queues.
341 	 */
342 	msi = MSIX_IQFLINT;
343 	for_each_ethrxq(s, rxq) {
344 		err = request_irq(adapter->msix_info[msi].vec,
345 				  t4vf_sge_intr_msix, 0,
346 				  adapter->msix_info[msi].desc,
347 				  &s->ethrxq[rxq].rspq);
348 		if (err)
349 			goto err_free_irqs;
350 		msi++;
351 	}
352 	return 0;
353 
354 err_free_irqs:
355 	while (--rxq >= 0)
356 		free_irq(adapter->msix_info[--msi].vec, &s->ethrxq[rxq].rspq);
357 	free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
358 	return err;
359 }
360 
361 /*
362  * Free our MSI-X resources.
363  */
364 static void free_msix_queue_irqs(struct adapter *adapter)
365 {
366 	struct sge *s = &adapter->sge;
367 	int rxq, msi;
368 
369 	free_irq(adapter->msix_info[MSIX_FW].vec, &s->fw_evtq);
370 	msi = MSIX_IQFLINT;
371 	for_each_ethrxq(s, rxq)
372 		free_irq(adapter->msix_info[msi++].vec,
373 			 &s->ethrxq[rxq].rspq);
374 }
375 
376 /*
377  * Turn on NAPI and start up interrupts on a response queue.
378  */
379 static void qenable(struct sge_rspq *rspq)
380 {
381 	napi_enable(&rspq->napi);
382 
383 	/*
384 	 * 0-increment the Going To Sleep register to start the timer and
385 	 * enable interrupts.
386 	 */
387 	t4_write_reg(rspq->adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
388 		     CIDXINC_V(0) |
389 		     SEINTARM_V(rspq->intr_params) |
390 		     INGRESSQID_V(rspq->cntxt_id));
391 }
392 
393 /*
394  * Enable NAPI scheduling and interrupt generation for all Receive Queues.
395  */
396 static void enable_rx(struct adapter *adapter)
397 {
398 	int rxq;
399 	struct sge *s = &adapter->sge;
400 
401 	for_each_ethrxq(s, rxq)
402 		qenable(&s->ethrxq[rxq].rspq);
403 	qenable(&s->fw_evtq);
404 
405 	/*
406 	 * The interrupt queue doesn't use NAPI so we do the 0-increment of
407 	 * its Going To Sleep register here to get it started.
408 	 */
409 	if (adapter->flags & USING_MSI)
410 		t4_write_reg(adapter, T4VF_SGE_BASE_ADDR + SGE_VF_GTS,
411 			     CIDXINC_V(0) |
412 			     SEINTARM_V(s->intrq.intr_params) |
413 			     INGRESSQID_V(s->intrq.cntxt_id));
414 
415 }
416 
417 /*
418  * Wait until all NAPI handlers are descheduled.
419  */
420 static void quiesce_rx(struct adapter *adapter)
421 {
422 	struct sge *s = &adapter->sge;
423 	int rxq;
424 
425 	for_each_ethrxq(s, rxq)
426 		napi_disable(&s->ethrxq[rxq].rspq.napi);
427 	napi_disable(&s->fw_evtq.napi);
428 }
429 
430 /*
431  * Response queue handler for the firmware event queue.
432  */
433 static int fwevtq_handler(struct sge_rspq *rspq, const __be64 *rsp,
434 			  const struct pkt_gl *gl)
435 {
436 	/*
437 	 * Extract response opcode and get pointer to CPL message body.
438 	 */
439 	struct adapter *adapter = rspq->adapter;
440 	u8 opcode = ((const struct rss_header *)rsp)->opcode;
441 	void *cpl = (void *)(rsp + 1);
442 
443 	switch (opcode) {
444 	case CPL_FW6_MSG: {
445 		/*
446 		 * We've received an asynchronous message from the firmware.
447 		 */
448 		const struct cpl_fw6_msg *fw_msg = cpl;
449 		if (fw_msg->type == FW6_TYPE_CMD_RPL)
450 			t4vf_handle_fw_rpl(adapter, fw_msg->data);
451 		break;
452 	}
453 
454 	case CPL_FW4_MSG: {
455 		/* FW can send EGR_UPDATEs encapsulated in a CPL_FW4_MSG.
456 		 */
457 		const struct cpl_sge_egr_update *p = (void *)(rsp + 3);
458 		opcode = CPL_OPCODE_G(ntohl(p->opcode_qid));
459 		if (opcode != CPL_SGE_EGR_UPDATE) {
460 			dev_err(adapter->pdev_dev, "unexpected FW4/CPL %#x on FW event queue\n"
461 				, opcode);
462 			break;
463 		}
464 		cpl = (void *)p;
465 		/*FALLTHROUGH*/
466 	}
467 
468 	case CPL_SGE_EGR_UPDATE: {
469 		/*
470 		 * We've received an Egress Queue Status Update message.  We
471 		 * get these, if the SGE is configured to send these when the
472 		 * firmware passes certain points in processing our TX
473 		 * Ethernet Queue or if we make an explicit request for one.
474 		 * We use these updates to determine when we may need to
475 		 * restart a TX Ethernet Queue which was stopped for lack of
476 		 * free TX Queue Descriptors ...
477 		 */
478 		const struct cpl_sge_egr_update *p = cpl;
479 		unsigned int qid = EGR_QID_G(be32_to_cpu(p->opcode_qid));
480 		struct sge *s = &adapter->sge;
481 		struct sge_txq *tq;
482 		struct sge_eth_txq *txq;
483 		unsigned int eq_idx;
484 
485 		/*
486 		 * Perform sanity checking on the Queue ID to make sure it
487 		 * really refers to one of our TX Ethernet Egress Queues which
488 		 * is active and matches the queue's ID.  None of these error
489 		 * conditions should ever happen so we may want to either make
490 		 * them fatal and/or conditionalized under DEBUG.
491 		 */
492 		eq_idx = EQ_IDX(s, qid);
493 		if (unlikely(eq_idx >= MAX_EGRQ)) {
494 			dev_err(adapter->pdev_dev,
495 				"Egress Update QID %d out of range\n", qid);
496 			break;
497 		}
498 		tq = s->egr_map[eq_idx];
499 		if (unlikely(tq == NULL)) {
500 			dev_err(adapter->pdev_dev,
501 				"Egress Update QID %d TXQ=NULL\n", qid);
502 			break;
503 		}
504 		txq = container_of(tq, struct sge_eth_txq, q);
505 		if (unlikely(tq->abs_id != qid)) {
506 			dev_err(adapter->pdev_dev,
507 				"Egress Update QID %d refers to TXQ %d\n",
508 				qid, tq->abs_id);
509 			break;
510 		}
511 
512 		/*
513 		 * Restart a stopped TX Queue which has less than half of its
514 		 * TX ring in use ...
515 		 */
516 		txq->q.restarts++;
517 		netif_tx_wake_queue(txq->txq);
518 		break;
519 	}
520 
521 	default:
522 		dev_err(adapter->pdev_dev,
523 			"unexpected CPL %#x on FW event queue\n", opcode);
524 	}
525 
526 	return 0;
527 }
528 
529 /*
530  * Allocate SGE TX/RX response queues.  Determine how many sets of SGE queues
531  * to use and initializes them.  We support multiple "Queue Sets" per port if
532  * we have MSI-X, otherwise just one queue set per port.
533  */
534 static int setup_sge_queues(struct adapter *adapter)
535 {
536 	struct sge *s = &adapter->sge;
537 	int err, pidx, msix;
538 
539 	/*
540 	 * Clear "Queue Set" Free List Starving and TX Queue Mapping Error
541 	 * state.
542 	 */
543 	bitmap_zero(s->starving_fl, MAX_EGRQ);
544 
545 	/*
546 	 * If we're using MSI interrupt mode we need to set up a "forwarded
547 	 * interrupt" queue which we'll set up with our MSI vector.  The rest
548 	 * of the ingress queues will be set up to forward their interrupts to
549 	 * this queue ...  This must be first since t4vf_sge_alloc_rxq() uses
550 	 * the intrq's queue ID as the interrupt forwarding queue for the
551 	 * subsequent calls ...
552 	 */
553 	if (adapter->flags & USING_MSI) {
554 		err = t4vf_sge_alloc_rxq(adapter, &s->intrq, false,
555 					 adapter->port[0], 0, NULL, NULL);
556 		if (err)
557 			goto err_free_queues;
558 	}
559 
560 	/*
561 	 * Allocate our ingress queue for asynchronous firmware messages.
562 	 */
563 	err = t4vf_sge_alloc_rxq(adapter, &s->fw_evtq, true, adapter->port[0],
564 				 MSIX_FW, NULL, fwevtq_handler);
565 	if (err)
566 		goto err_free_queues;
567 
568 	/*
569 	 * Allocate each "port"'s initial Queue Sets.  These can be changed
570 	 * later on ... up to the point where any interface on the adapter is
571 	 * brought up at which point lots of things get nailed down
572 	 * permanently ...
573 	 */
574 	msix = MSIX_IQFLINT;
575 	for_each_port(adapter, pidx) {
576 		struct net_device *dev = adapter->port[pidx];
577 		struct port_info *pi = netdev_priv(dev);
578 		struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
579 		struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
580 		int qs;
581 
582 		for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
583 			err = t4vf_sge_alloc_rxq(adapter, &rxq->rspq, false,
584 						 dev, msix++,
585 						 &rxq->fl, t4vf_ethrx_handler);
586 			if (err)
587 				goto err_free_queues;
588 
589 			err = t4vf_sge_alloc_eth_txq(adapter, txq, dev,
590 					     netdev_get_tx_queue(dev, qs),
591 					     s->fw_evtq.cntxt_id);
592 			if (err)
593 				goto err_free_queues;
594 
595 			rxq->rspq.idx = qs;
596 			memset(&rxq->stats, 0, sizeof(rxq->stats));
597 		}
598 	}
599 
600 	/*
601 	 * Create the reverse mappings for the queues.
602 	 */
603 	s->egr_base = s->ethtxq[0].q.abs_id - s->ethtxq[0].q.cntxt_id;
604 	s->ingr_base = s->ethrxq[0].rspq.abs_id - s->ethrxq[0].rspq.cntxt_id;
605 	IQ_MAP(s, s->fw_evtq.abs_id) = &s->fw_evtq;
606 	for_each_port(adapter, pidx) {
607 		struct net_device *dev = adapter->port[pidx];
608 		struct port_info *pi = netdev_priv(dev);
609 		struct sge_eth_rxq *rxq = &s->ethrxq[pi->first_qset];
610 		struct sge_eth_txq *txq = &s->ethtxq[pi->first_qset];
611 		int qs;
612 
613 		for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
614 			IQ_MAP(s, rxq->rspq.abs_id) = &rxq->rspq;
615 			EQ_MAP(s, txq->q.abs_id) = &txq->q;
616 
617 			/*
618 			 * The FW_IQ_CMD doesn't return the Absolute Queue IDs
619 			 * for Free Lists but since all of the Egress Queues
620 			 * (including Free Lists) have Relative Queue IDs
621 			 * which are computed as Absolute - Base Queue ID, we
622 			 * can synthesize the Absolute Queue IDs for the Free
623 			 * Lists.  This is useful for debugging purposes when
624 			 * we want to dump Queue Contexts via the PF Driver.
625 			 */
626 			rxq->fl.abs_id = rxq->fl.cntxt_id + s->egr_base;
627 			EQ_MAP(s, rxq->fl.abs_id) = &rxq->fl;
628 		}
629 	}
630 	return 0;
631 
632 err_free_queues:
633 	t4vf_free_sge_resources(adapter);
634 	return err;
635 }
636 
637 /*
638  * Set up Receive Side Scaling (RSS) to distribute packets to multiple receive
639  * queues.  We configure the RSS CPU lookup table to distribute to the number
640  * of HW receive queues, and the response queue lookup table to narrow that
641  * down to the response queues actually configured for each "port" (Virtual
642  * Interface).  We always configure the RSS mapping for all ports since the
643  * mapping table has plenty of entries.
644  */
645 static int setup_rss(struct adapter *adapter)
646 {
647 	int pidx;
648 
649 	for_each_port(adapter, pidx) {
650 		struct port_info *pi = adap2pinfo(adapter, pidx);
651 		struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
652 		u16 rss[MAX_PORT_QSETS];
653 		int qs, err;
654 
655 		for (qs = 0; qs < pi->nqsets; qs++)
656 			rss[qs] = rxq[qs].rspq.abs_id;
657 
658 		err = t4vf_config_rss_range(adapter, pi->viid,
659 					    0, pi->rss_size, rss, pi->nqsets);
660 		if (err)
661 			return err;
662 
663 		/*
664 		 * Perform Global RSS Mode-specific initialization.
665 		 */
666 		switch (adapter->params.rss.mode) {
667 		case FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL:
668 			/*
669 			 * If Tunnel All Lookup isn't specified in the global
670 			 * RSS Configuration, then we need to specify a
671 			 * default Ingress Queue for any ingress packets which
672 			 * aren't hashed.  We'll use our first ingress queue
673 			 * ...
674 			 */
675 			if (!adapter->params.rss.u.basicvirtual.tnlalllookup) {
676 				union rss_vi_config config;
677 				err = t4vf_read_rss_vi_config(adapter,
678 							      pi->viid,
679 							      &config);
680 				if (err)
681 					return err;
682 				config.basicvirtual.defaultq =
683 					rxq[0].rspq.abs_id;
684 				err = t4vf_write_rss_vi_config(adapter,
685 							       pi->viid,
686 							       &config);
687 				if (err)
688 					return err;
689 			}
690 			break;
691 		}
692 	}
693 
694 	return 0;
695 }
696 
697 /*
698  * Bring the adapter up.  Called whenever we go from no "ports" open to having
699  * one open.  This function performs the actions necessary to make an adapter
700  * operational, such as completing the initialization of HW modules, and
701  * enabling interrupts.  Must be called with the rtnl lock held.  (Note that
702  * this is called "cxgb_up" in the PF Driver.)
703  */
704 static int adapter_up(struct adapter *adapter)
705 {
706 	int err;
707 
708 	/*
709 	 * If this is the first time we've been called, perform basic
710 	 * adapter setup.  Once we've done this, many of our adapter
711 	 * parameters can no longer be changed ...
712 	 */
713 	if ((adapter->flags & FULL_INIT_DONE) == 0) {
714 		err = setup_sge_queues(adapter);
715 		if (err)
716 			return err;
717 		err = setup_rss(adapter);
718 		if (err) {
719 			t4vf_free_sge_resources(adapter);
720 			return err;
721 		}
722 
723 		if (adapter->flags & USING_MSIX)
724 			name_msix_vecs(adapter);
725 		adapter->flags |= FULL_INIT_DONE;
726 	}
727 
728 	/*
729 	 * Acquire our interrupt resources.  We only support MSI-X and MSI.
730 	 */
731 	BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);
732 	if (adapter->flags & USING_MSIX)
733 		err = request_msix_queue_irqs(adapter);
734 	else
735 		err = request_irq(adapter->pdev->irq,
736 				  t4vf_intr_handler(adapter), 0,
737 				  adapter->name, adapter);
738 	if (err) {
739 		dev_err(adapter->pdev_dev, "request_irq failed, err %d\n",
740 			err);
741 		return err;
742 	}
743 
744 	/*
745 	 * Enable NAPI ingress processing and return success.
746 	 */
747 	enable_rx(adapter);
748 	t4vf_sge_start(adapter);
749 
750 	/* Initialize hash mac addr list*/
751 	INIT_LIST_HEAD(&adapter->mac_hlist);
752 	return 0;
753 }
754 
755 /*
756  * Bring the adapter down.  Called whenever the last "port" (Virtual
757  * Interface) closed.  (Note that this routine is called "cxgb_down" in the PF
758  * Driver.)
759  */
760 static void adapter_down(struct adapter *adapter)
761 {
762 	/*
763 	 * Free interrupt resources.
764 	 */
765 	if (adapter->flags & USING_MSIX)
766 		free_msix_queue_irqs(adapter);
767 	else
768 		free_irq(adapter->pdev->irq, adapter);
769 
770 	/*
771 	 * Wait for NAPI handlers to finish.
772 	 */
773 	quiesce_rx(adapter);
774 }
775 
776 /*
777  * Start up a net device.
778  */
779 static int cxgb4vf_open(struct net_device *dev)
780 {
781 	int err;
782 	struct port_info *pi = netdev_priv(dev);
783 	struct adapter *adapter = pi->adapter;
784 
785 	/*
786 	 * If this is the first interface that we're opening on the "adapter",
787 	 * bring the "adapter" up now.
788 	 */
789 	if (adapter->open_device_map == 0) {
790 		err = adapter_up(adapter);
791 		if (err)
792 			return err;
793 	}
794 
795 	/*
796 	 * Note that this interface is up and start everything up ...
797 	 */
798 	err = link_start(dev);
799 	if (err)
800 		goto err_unwind;
801 
802 	pi->vlan_id = t4vf_get_vf_vlan_acl(adapter);
803 
804 	netif_tx_start_all_queues(dev);
805 	set_bit(pi->port_id, &adapter->open_device_map);
806 	return 0;
807 
808 err_unwind:
809 	if (adapter->open_device_map == 0)
810 		adapter_down(adapter);
811 	return err;
812 }
813 
814 /*
815  * Shut down a net device.  This routine is called "cxgb_close" in the PF
816  * Driver ...
817  */
818 static int cxgb4vf_stop(struct net_device *dev)
819 {
820 	struct port_info *pi = netdev_priv(dev);
821 	struct adapter *adapter = pi->adapter;
822 
823 	netif_tx_stop_all_queues(dev);
824 	netif_carrier_off(dev);
825 	t4vf_enable_pi(adapter, pi, false, false);
826 
827 	clear_bit(pi->port_id, &adapter->open_device_map);
828 	if (adapter->open_device_map == 0)
829 		adapter_down(adapter);
830 	return 0;
831 }
832 
833 /*
834  * Translate our basic statistics into the standard "ifconfig" statistics.
835  */
836 static struct net_device_stats *cxgb4vf_get_stats(struct net_device *dev)
837 {
838 	struct t4vf_port_stats stats;
839 	struct port_info *pi = netdev2pinfo(dev);
840 	struct adapter *adapter = pi->adapter;
841 	struct net_device_stats *ns = &dev->stats;
842 	int err;
843 
844 	spin_lock(&adapter->stats_lock);
845 	err = t4vf_get_port_stats(adapter, pi->pidx, &stats);
846 	spin_unlock(&adapter->stats_lock);
847 
848 	memset(ns, 0, sizeof(*ns));
849 	if (err)
850 		return ns;
851 
852 	ns->tx_bytes = (stats.tx_bcast_bytes + stats.tx_mcast_bytes +
853 			stats.tx_ucast_bytes + stats.tx_offload_bytes);
854 	ns->tx_packets = (stats.tx_bcast_frames + stats.tx_mcast_frames +
855 			  stats.tx_ucast_frames + stats.tx_offload_frames);
856 	ns->rx_bytes = (stats.rx_bcast_bytes + stats.rx_mcast_bytes +
857 			stats.rx_ucast_bytes);
858 	ns->rx_packets = (stats.rx_bcast_frames + stats.rx_mcast_frames +
859 			  stats.rx_ucast_frames);
860 	ns->multicast = stats.rx_mcast_frames;
861 	ns->tx_errors = stats.tx_drop_frames;
862 	ns->rx_errors = stats.rx_err_frames;
863 
864 	return ns;
865 }
866 
867 static inline int cxgb4vf_set_addr_hash(struct port_info *pi)
868 {
869 	struct adapter *adapter = pi->adapter;
870 	u64 vec = 0;
871 	bool ucast = false;
872 	struct hash_mac_addr *entry;
873 
874 	/* Calculate the hash vector for the updated list and program it */
875 	list_for_each_entry(entry, &adapter->mac_hlist, list) {
876 		ucast |= is_unicast_ether_addr(entry->addr);
877 		vec |= (1ULL << hash_mac_addr(entry->addr));
878 	}
879 	return t4vf_set_addr_hash(adapter, pi->viid, ucast, vec, false);
880 }
881 
882 static int cxgb4vf_mac_sync(struct net_device *netdev, const u8 *mac_addr)
883 {
884 	struct port_info *pi = netdev_priv(netdev);
885 	struct adapter *adapter = pi->adapter;
886 	int ret;
887 	u64 mhash = 0;
888 	u64 uhash = 0;
889 	bool free = false;
890 	bool ucast = is_unicast_ether_addr(mac_addr);
891 	const u8 *maclist[1] = {mac_addr};
892 	struct hash_mac_addr *new_entry;
893 
894 	ret = t4vf_alloc_mac_filt(adapter, pi->viid, free, 1, maclist,
895 				  NULL, ucast ? &uhash : &mhash, false);
896 	if (ret < 0)
897 		goto out;
898 	/* if hash != 0, then add the addr to hash addr list
899 	 * so on the end we will calculate the hash for the
900 	 * list and program it
901 	 */
902 	if (uhash || mhash) {
903 		new_entry = kzalloc(sizeof(*new_entry), GFP_ATOMIC);
904 		if (!new_entry)
905 			return -ENOMEM;
906 		ether_addr_copy(new_entry->addr, mac_addr);
907 		list_add_tail(&new_entry->list, &adapter->mac_hlist);
908 		ret = cxgb4vf_set_addr_hash(pi);
909 	}
910 out:
911 	return ret < 0 ? ret : 0;
912 }
913 
914 static int cxgb4vf_mac_unsync(struct net_device *netdev, const u8 *mac_addr)
915 {
916 	struct port_info *pi = netdev_priv(netdev);
917 	struct adapter *adapter = pi->adapter;
918 	int ret;
919 	const u8 *maclist[1] = {mac_addr};
920 	struct hash_mac_addr *entry, *tmp;
921 
922 	/* If the MAC address to be removed is in the hash addr
923 	 * list, delete it from the list and update hash vector
924 	 */
925 	list_for_each_entry_safe(entry, tmp, &adapter->mac_hlist, list) {
926 		if (ether_addr_equal(entry->addr, mac_addr)) {
927 			list_del(&entry->list);
928 			kfree(entry);
929 			return cxgb4vf_set_addr_hash(pi);
930 		}
931 	}
932 
933 	ret = t4vf_free_mac_filt(adapter, pi->viid, 1, maclist, false);
934 	return ret < 0 ? -EINVAL : 0;
935 }
936 
937 /*
938  * Set RX properties of a port, such as promiscruity, address filters, and MTU.
939  * If @mtu is -1 it is left unchanged.
940  */
941 static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
942 {
943 	struct port_info *pi = netdev_priv(dev);
944 
945 	__dev_uc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
946 	__dev_mc_sync(dev, cxgb4vf_mac_sync, cxgb4vf_mac_unsync);
947 	return t4vf_set_rxmode(pi->adapter, pi->viid, -1,
948 			       (dev->flags & IFF_PROMISC) != 0,
949 			       (dev->flags & IFF_ALLMULTI) != 0,
950 			       1, -1, sleep_ok);
951 }
952 
953 /*
954  * Set the current receive modes on the device.
955  */
956 static void cxgb4vf_set_rxmode(struct net_device *dev)
957 {
958 	/* unfortunately we can't return errors to the stack */
959 	set_rxmode(dev, -1, false);
960 }
961 
962 /*
963  * Find the entry in the interrupt holdoff timer value array which comes
964  * closest to the specified interrupt holdoff value.
965  */
966 static int closest_timer(const struct sge *s, int us)
967 {
968 	int i, timer_idx = 0, min_delta = INT_MAX;
969 
970 	for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
971 		int delta = us - s->timer_val[i];
972 		if (delta < 0)
973 			delta = -delta;
974 		if (delta < min_delta) {
975 			min_delta = delta;
976 			timer_idx = i;
977 		}
978 	}
979 	return timer_idx;
980 }
981 
982 static int closest_thres(const struct sge *s, int thres)
983 {
984 	int i, delta, pktcnt_idx = 0, min_delta = INT_MAX;
985 
986 	for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
987 		delta = thres - s->counter_val[i];
988 		if (delta < 0)
989 			delta = -delta;
990 		if (delta < min_delta) {
991 			min_delta = delta;
992 			pktcnt_idx = i;
993 		}
994 	}
995 	return pktcnt_idx;
996 }
997 
998 /*
999  * Return a queue's interrupt hold-off time in us.  0 means no timer.
1000  */
1001 static unsigned int qtimer_val(const struct adapter *adapter,
1002 			       const struct sge_rspq *rspq)
1003 {
1004 	unsigned int timer_idx = QINTR_TIMER_IDX_G(rspq->intr_params);
1005 
1006 	return timer_idx < SGE_NTIMERS
1007 		? adapter->sge.timer_val[timer_idx]
1008 		: 0;
1009 }
1010 
1011 /**
1012  *	set_rxq_intr_params - set a queue's interrupt holdoff parameters
1013  *	@adapter: the adapter
1014  *	@rspq: the RX response queue
1015  *	@us: the hold-off time in us, or 0 to disable timer
1016  *	@cnt: the hold-off packet count, or 0 to disable counter
1017  *
1018  *	Sets an RX response queue's interrupt hold-off time and packet count.
1019  *	At least one of the two needs to be enabled for the queue to generate
1020  *	interrupts.
1021  */
1022 static int set_rxq_intr_params(struct adapter *adapter, struct sge_rspq *rspq,
1023 			       unsigned int us, unsigned int cnt)
1024 {
1025 	unsigned int timer_idx;
1026 
1027 	/*
1028 	 * If both the interrupt holdoff timer and count are specified as
1029 	 * zero, default to a holdoff count of 1 ...
1030 	 */
1031 	if ((us | cnt) == 0)
1032 		cnt = 1;
1033 
1034 	/*
1035 	 * If an interrupt holdoff count has been specified, then find the
1036 	 * closest configured holdoff count and use that.  If the response
1037 	 * queue has already been created, then update its queue context
1038 	 * parameters ...
1039 	 */
1040 	if (cnt) {
1041 		int err;
1042 		u32 v, pktcnt_idx;
1043 
1044 		pktcnt_idx = closest_thres(&adapter->sge, cnt);
1045 		if (rspq->desc && rspq->pktcnt_idx != pktcnt_idx) {
1046 			v = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_DMAQ) |
1047 			    FW_PARAMS_PARAM_X_V(
1048 					FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1049 			    FW_PARAMS_PARAM_YZ_V(rspq->cntxt_id);
1050 			err = t4vf_set_params(adapter, 1, &v, &pktcnt_idx);
1051 			if (err)
1052 				return err;
1053 		}
1054 		rspq->pktcnt_idx = pktcnt_idx;
1055 	}
1056 
1057 	/*
1058 	 * Compute the closest holdoff timer index from the supplied holdoff
1059 	 * timer value.
1060 	 */
1061 	timer_idx = (us == 0
1062 		     ? SGE_TIMER_RSTRT_CNTR
1063 		     : closest_timer(&adapter->sge, us));
1064 
1065 	/*
1066 	 * Update the response queue's interrupt coalescing parameters and
1067 	 * return success.
1068 	 */
1069 	rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) |
1070 			     QINTR_CNT_EN_V(cnt > 0));
1071 	return 0;
1072 }
1073 
1074 /*
1075  * Return a version number to identify the type of adapter.  The scheme is:
1076  * - bits 0..9: chip version
1077  * - bits 10..15: chip revision
1078  */
1079 static inline unsigned int mk_adap_vers(const struct adapter *adapter)
1080 {
1081 	/*
1082 	 * Chip version 4, revision 0x3f (cxgb4vf).
1083 	 */
1084 	return CHELSIO_CHIP_VERSION(adapter->params.chip) | (0x3f << 10);
1085 }
1086 
1087 /*
1088  * Execute the specified ioctl command.
1089  */
1090 static int cxgb4vf_do_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
1091 {
1092 	int ret = 0;
1093 
1094 	switch (cmd) {
1095 	    /*
1096 	     * The VF Driver doesn't have access to any of the other
1097 	     * common Ethernet device ioctl()'s (like reading/writing
1098 	     * PHY registers, etc.
1099 	     */
1100 
1101 	default:
1102 		ret = -EOPNOTSUPP;
1103 		break;
1104 	}
1105 	return ret;
1106 }
1107 
1108 /*
1109  * Change the device's MTU.
1110  */
1111 static int cxgb4vf_change_mtu(struct net_device *dev, int new_mtu)
1112 {
1113 	int ret;
1114 	struct port_info *pi = netdev_priv(dev);
1115 
1116 	ret = t4vf_set_rxmode(pi->adapter, pi->viid, new_mtu,
1117 			      -1, -1, -1, -1, true);
1118 	if (!ret)
1119 		dev->mtu = new_mtu;
1120 	return ret;
1121 }
1122 
1123 static netdev_features_t cxgb4vf_fix_features(struct net_device *dev,
1124 	netdev_features_t features)
1125 {
1126 	/*
1127 	 * Since there is no support for separate rx/tx vlan accel
1128 	 * enable/disable make sure tx flag is always in same state as rx.
1129 	 */
1130 	if (features & NETIF_F_HW_VLAN_CTAG_RX)
1131 		features |= NETIF_F_HW_VLAN_CTAG_TX;
1132 	else
1133 		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
1134 
1135 	return features;
1136 }
1137 
1138 static int cxgb4vf_set_features(struct net_device *dev,
1139 	netdev_features_t features)
1140 {
1141 	struct port_info *pi = netdev_priv(dev);
1142 	netdev_features_t changed = dev->features ^ features;
1143 
1144 	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
1145 		t4vf_set_rxmode(pi->adapter, pi->viid, -1, -1, -1, -1,
1146 				features & NETIF_F_HW_VLAN_CTAG_TX, 0);
1147 
1148 	return 0;
1149 }
1150 
1151 /*
1152  * Change the devices MAC address.
1153  */
1154 static int cxgb4vf_set_mac_addr(struct net_device *dev, void *_addr)
1155 {
1156 	int ret;
1157 	struct sockaddr *addr = _addr;
1158 	struct port_info *pi = netdev_priv(dev);
1159 
1160 	if (!is_valid_ether_addr(addr->sa_data))
1161 		return -EADDRNOTAVAIL;
1162 
1163 	ret = t4vf_change_mac(pi->adapter, pi->viid, pi->xact_addr_filt,
1164 			      addr->sa_data, true);
1165 	if (ret < 0)
1166 		return ret;
1167 
1168 	memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
1169 	pi->xact_addr_filt = ret;
1170 	return 0;
1171 }
1172 
1173 #ifdef CONFIG_NET_POLL_CONTROLLER
1174 /*
1175  * Poll all of our receive queues.  This is called outside of normal interrupt
1176  * context.
1177  */
1178 static void cxgb4vf_poll_controller(struct net_device *dev)
1179 {
1180 	struct port_info *pi = netdev_priv(dev);
1181 	struct adapter *adapter = pi->adapter;
1182 
1183 	if (adapter->flags & USING_MSIX) {
1184 		struct sge_eth_rxq *rxq;
1185 		int nqsets;
1186 
1187 		rxq = &adapter->sge.ethrxq[pi->first_qset];
1188 		for (nqsets = pi->nqsets; nqsets; nqsets--) {
1189 			t4vf_sge_intr_msix(0, &rxq->rspq);
1190 			rxq++;
1191 		}
1192 	} else
1193 		t4vf_intr_handler(adapter)(0, adapter);
1194 }
1195 #endif
1196 
1197 /*
1198  * Ethtool operations.
1199  * ===================
1200  *
1201  * Note that we don't support any ethtool operations which change the physical
1202  * state of the port to which we're linked.
1203  */
1204 
1205 /**
1206  *	from_fw_port_mod_type - translate Firmware Port/Module type to Ethtool
1207  *	@port_type: Firmware Port Type
1208  *	@mod_type: Firmware Module Type
1209  *
1210  *	Translate Firmware Port/Module type to Ethtool Port Type.
1211  */
1212 static int from_fw_port_mod_type(enum fw_port_type port_type,
1213 				 enum fw_port_module_type mod_type)
1214 {
1215 	if (port_type == FW_PORT_TYPE_BT_SGMII ||
1216 	    port_type == FW_PORT_TYPE_BT_XFI ||
1217 	    port_type == FW_PORT_TYPE_BT_XAUI) {
1218 		return PORT_TP;
1219 	} else if (port_type == FW_PORT_TYPE_FIBER_XFI ||
1220 		   port_type == FW_PORT_TYPE_FIBER_XAUI) {
1221 		return PORT_FIBRE;
1222 	} else if (port_type == FW_PORT_TYPE_SFP ||
1223 		   port_type == FW_PORT_TYPE_QSFP_10G ||
1224 		   port_type == FW_PORT_TYPE_QSA ||
1225 		   port_type == FW_PORT_TYPE_QSFP ||
1226 		   port_type == FW_PORT_TYPE_CR4_QSFP ||
1227 		   port_type == FW_PORT_TYPE_CR_QSFP ||
1228 		   port_type == FW_PORT_TYPE_CR2_QSFP ||
1229 		   port_type == FW_PORT_TYPE_SFP28) {
1230 		if (mod_type == FW_PORT_MOD_TYPE_LR ||
1231 		    mod_type == FW_PORT_MOD_TYPE_SR ||
1232 		    mod_type == FW_PORT_MOD_TYPE_ER ||
1233 		    mod_type == FW_PORT_MOD_TYPE_LRM)
1234 			return PORT_FIBRE;
1235 		else if (mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1236 			 mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1237 			return PORT_DA;
1238 		else
1239 			return PORT_OTHER;
1240 	} else if (port_type == FW_PORT_TYPE_KR4_100G ||
1241 		   port_type == FW_PORT_TYPE_KR_SFP28 ||
1242 		   port_type == FW_PORT_TYPE_KR_XLAUI) {
1243 		return PORT_NONE;
1244 	}
1245 
1246 	return PORT_OTHER;
1247 }
1248 
1249 /**
1250  *	fw_caps_to_lmm - translate Firmware to ethtool Link Mode Mask
1251  *	@port_type: Firmware Port Type
1252  *	@fw_caps: Firmware Port Capabilities
1253  *	@link_mode_mask: ethtool Link Mode Mask
1254  *
1255  *	Translate a Firmware Port Capabilities specification to an ethtool
1256  *	Link Mode Mask.
1257  */
1258 static void fw_caps_to_lmm(enum fw_port_type port_type,
1259 			   unsigned int fw_caps,
1260 			   unsigned long *link_mode_mask)
1261 {
1262 	#define SET_LMM(__lmm_name) \
1263 		__set_bit(ETHTOOL_LINK_MODE_ ## __lmm_name ## _BIT, \
1264 			  link_mode_mask)
1265 
1266 	#define FW_CAPS_TO_LMM(__fw_name, __lmm_name) \
1267 		do { \
1268 			if (fw_caps & FW_PORT_CAP32_ ## __fw_name) \
1269 				SET_LMM(__lmm_name); \
1270 		} while (0)
1271 
1272 	switch (port_type) {
1273 	case FW_PORT_TYPE_BT_SGMII:
1274 	case FW_PORT_TYPE_BT_XFI:
1275 	case FW_PORT_TYPE_BT_XAUI:
1276 		SET_LMM(TP);
1277 		FW_CAPS_TO_LMM(SPEED_100M, 100baseT_Full);
1278 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1279 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1280 		break;
1281 
1282 	case FW_PORT_TYPE_KX4:
1283 	case FW_PORT_TYPE_KX:
1284 		SET_LMM(Backplane);
1285 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1286 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full);
1287 		break;
1288 
1289 	case FW_PORT_TYPE_KR:
1290 		SET_LMM(Backplane);
1291 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1292 		break;
1293 
1294 	case FW_PORT_TYPE_BP_AP:
1295 		SET_LMM(Backplane);
1296 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1297 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC);
1298 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1299 		break;
1300 
1301 	case FW_PORT_TYPE_BP4_AP:
1302 		SET_LMM(Backplane);
1303 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1304 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseR_FEC);
1305 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1306 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKX4_Full);
1307 		break;
1308 
1309 	case FW_PORT_TYPE_FIBER_XFI:
1310 	case FW_PORT_TYPE_FIBER_XAUI:
1311 	case FW_PORT_TYPE_SFP:
1312 	case FW_PORT_TYPE_QSFP_10G:
1313 	case FW_PORT_TYPE_QSA:
1314 		SET_LMM(FIBRE);
1315 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1316 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1317 		break;
1318 
1319 	case FW_PORT_TYPE_BP40_BA:
1320 	case FW_PORT_TYPE_QSFP:
1321 		SET_LMM(FIBRE);
1322 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1323 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1324 		FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full);
1325 		break;
1326 
1327 	case FW_PORT_TYPE_CR_QSFP:
1328 	case FW_PORT_TYPE_SFP28:
1329 		SET_LMM(FIBRE);
1330 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1331 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseT_Full);
1332 		FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full);
1333 		break;
1334 
1335 	case FW_PORT_TYPE_KR_SFP28:
1336 		SET_LMM(Backplane);
1337 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseT_Full);
1338 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1339 		FW_CAPS_TO_LMM(SPEED_25G, 25000baseKR_Full);
1340 		break;
1341 
1342 	case FW_PORT_TYPE_KR_XLAUI:
1343 		SET_LMM(Backplane);
1344 		FW_CAPS_TO_LMM(SPEED_1G, 1000baseKX_Full);
1345 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseKR_Full);
1346 		FW_CAPS_TO_LMM(SPEED_40G, 40000baseKR4_Full);
1347 		break;
1348 
1349 	case FW_PORT_TYPE_CR2_QSFP:
1350 		SET_LMM(FIBRE);
1351 		FW_CAPS_TO_LMM(SPEED_50G, 50000baseSR2_Full);
1352 		break;
1353 
1354 	case FW_PORT_TYPE_KR4_100G:
1355 	case FW_PORT_TYPE_CR4_QSFP:
1356 		SET_LMM(FIBRE);
1357 		FW_CAPS_TO_LMM(SPEED_1G,  1000baseT_Full);
1358 		FW_CAPS_TO_LMM(SPEED_10G, 10000baseSR_Full);
1359 		FW_CAPS_TO_LMM(SPEED_40G, 40000baseSR4_Full);
1360 		FW_CAPS_TO_LMM(SPEED_25G, 25000baseCR_Full);
1361 		FW_CAPS_TO_LMM(SPEED_50G, 50000baseCR2_Full);
1362 		FW_CAPS_TO_LMM(SPEED_100G, 100000baseCR4_Full);
1363 		break;
1364 
1365 	default:
1366 		break;
1367 	}
1368 
1369 	FW_CAPS_TO_LMM(ANEG, Autoneg);
1370 	FW_CAPS_TO_LMM(802_3_PAUSE, Pause);
1371 	FW_CAPS_TO_LMM(802_3_ASM_DIR, Asym_Pause);
1372 
1373 	#undef FW_CAPS_TO_LMM
1374 	#undef SET_LMM
1375 }
1376 
1377 static int cxgb4vf_get_link_ksettings(struct net_device *dev,
1378 				  struct ethtool_link_ksettings *link_ksettings)
1379 {
1380 	struct port_info *pi = netdev_priv(dev);
1381 	struct ethtool_link_settings *base = &link_ksettings->base;
1382 
1383 	/* For the nonce, the Firmware doesn't send up Port State changes
1384 	 * when the Virtual Interface attached to the Port is down.  So
1385 	 * if it's down, let's grab any changes.
1386 	 */
1387 	if (!netif_running(dev))
1388 		(void)t4vf_update_port_info(pi);
1389 
1390 	ethtool_link_ksettings_zero_link_mode(link_ksettings, supported);
1391 	ethtool_link_ksettings_zero_link_mode(link_ksettings, advertising);
1392 	ethtool_link_ksettings_zero_link_mode(link_ksettings, lp_advertising);
1393 
1394 	base->port = from_fw_port_mod_type(pi->port_type, pi->mod_type);
1395 
1396 	if (pi->mdio_addr >= 0) {
1397 		base->phy_address = pi->mdio_addr;
1398 		base->mdio_support = (pi->port_type == FW_PORT_TYPE_BT_SGMII
1399 				      ? ETH_MDIO_SUPPORTS_C22
1400 				      : ETH_MDIO_SUPPORTS_C45);
1401 	} else {
1402 		base->phy_address = 255;
1403 		base->mdio_support = 0;
1404 	}
1405 
1406 	fw_caps_to_lmm(pi->port_type, pi->link_cfg.pcaps,
1407 		       link_ksettings->link_modes.supported);
1408 	fw_caps_to_lmm(pi->port_type, pi->link_cfg.acaps,
1409 		       link_ksettings->link_modes.advertising);
1410 	fw_caps_to_lmm(pi->port_type, pi->link_cfg.lpacaps,
1411 		       link_ksettings->link_modes.lp_advertising);
1412 
1413 	if (netif_carrier_ok(dev)) {
1414 		base->speed = pi->link_cfg.speed;
1415 		base->duplex = DUPLEX_FULL;
1416 	} else {
1417 		base->speed = SPEED_UNKNOWN;
1418 		base->duplex = DUPLEX_UNKNOWN;
1419 	}
1420 
1421 	if (pi->link_cfg.fc & PAUSE_RX) {
1422 		if (pi->link_cfg.fc & PAUSE_TX) {
1423 			ethtool_link_ksettings_add_link_mode(link_ksettings,
1424 							     advertising,
1425 							     Pause);
1426 		} else {
1427 			ethtool_link_ksettings_add_link_mode(link_ksettings,
1428 							     advertising,
1429 							     Asym_Pause);
1430 		}
1431 	} else if (pi->link_cfg.fc & PAUSE_TX) {
1432 		ethtool_link_ksettings_add_link_mode(link_ksettings,
1433 						     advertising,
1434 						     Asym_Pause);
1435 	}
1436 
1437 	base->autoneg = pi->link_cfg.autoneg;
1438 	if (pi->link_cfg.pcaps & FW_PORT_CAP32_ANEG)
1439 		ethtool_link_ksettings_add_link_mode(link_ksettings,
1440 						     supported, Autoneg);
1441 	if (pi->link_cfg.autoneg)
1442 		ethtool_link_ksettings_add_link_mode(link_ksettings,
1443 						     advertising, Autoneg);
1444 
1445 	return 0;
1446 }
1447 
1448 /* Translate the Firmware FEC value into the ethtool value. */
1449 static inline unsigned int fwcap_to_eth_fec(unsigned int fw_fec)
1450 {
1451 	unsigned int eth_fec = 0;
1452 
1453 	if (fw_fec & FW_PORT_CAP32_FEC_RS)
1454 		eth_fec |= ETHTOOL_FEC_RS;
1455 	if (fw_fec & FW_PORT_CAP32_FEC_BASER_RS)
1456 		eth_fec |= ETHTOOL_FEC_BASER;
1457 
1458 	/* if nothing is set, then FEC is off */
1459 	if (!eth_fec)
1460 		eth_fec = ETHTOOL_FEC_OFF;
1461 
1462 	return eth_fec;
1463 }
1464 
1465 /* Translate Common Code FEC value into ethtool value. */
1466 static inline unsigned int cc_to_eth_fec(unsigned int cc_fec)
1467 {
1468 	unsigned int eth_fec = 0;
1469 
1470 	if (cc_fec & FEC_AUTO)
1471 		eth_fec |= ETHTOOL_FEC_AUTO;
1472 	if (cc_fec & FEC_RS)
1473 		eth_fec |= ETHTOOL_FEC_RS;
1474 	if (cc_fec & FEC_BASER_RS)
1475 		eth_fec |= ETHTOOL_FEC_BASER;
1476 
1477 	/* if nothing is set, then FEC is off */
1478 	if (!eth_fec)
1479 		eth_fec = ETHTOOL_FEC_OFF;
1480 
1481 	return eth_fec;
1482 }
1483 
1484 static int cxgb4vf_get_fecparam(struct net_device *dev,
1485 				struct ethtool_fecparam *fec)
1486 {
1487 	const struct port_info *pi = netdev_priv(dev);
1488 	const struct link_config *lc = &pi->link_cfg;
1489 
1490 	/* Translate the Firmware FEC Support into the ethtool value.  We
1491 	 * always support IEEE 802.3 "automatic" selection of Link FEC type if
1492 	 * any FEC is supported.
1493 	 */
1494 	fec->fec = fwcap_to_eth_fec(lc->pcaps);
1495 	if (fec->fec != ETHTOOL_FEC_OFF)
1496 		fec->fec |= ETHTOOL_FEC_AUTO;
1497 
1498 	/* Translate the current internal FEC parameters into the
1499 	 * ethtool values.
1500 	 */
1501 	fec->active_fec = cc_to_eth_fec(lc->fec);
1502 	return 0;
1503 }
1504 
1505 /*
1506  * Return our driver information.
1507  */
1508 static void cxgb4vf_get_drvinfo(struct net_device *dev,
1509 				struct ethtool_drvinfo *drvinfo)
1510 {
1511 	struct adapter *adapter = netdev2adap(dev);
1512 
1513 	strlcpy(drvinfo->driver, KBUILD_MODNAME, sizeof(drvinfo->driver));
1514 	strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version));
1515 	strlcpy(drvinfo->bus_info, pci_name(to_pci_dev(dev->dev.parent)),
1516 		sizeof(drvinfo->bus_info));
1517 	snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version),
1518 		 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1519 		 FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.fwrev),
1520 		 FW_HDR_FW_VER_MINOR_G(adapter->params.dev.fwrev),
1521 		 FW_HDR_FW_VER_MICRO_G(adapter->params.dev.fwrev),
1522 		 FW_HDR_FW_VER_BUILD_G(adapter->params.dev.fwrev),
1523 		 FW_HDR_FW_VER_MAJOR_G(adapter->params.dev.tprev),
1524 		 FW_HDR_FW_VER_MINOR_G(adapter->params.dev.tprev),
1525 		 FW_HDR_FW_VER_MICRO_G(adapter->params.dev.tprev),
1526 		 FW_HDR_FW_VER_BUILD_G(adapter->params.dev.tprev));
1527 }
1528 
1529 /*
1530  * Return current adapter message level.
1531  */
1532 static u32 cxgb4vf_get_msglevel(struct net_device *dev)
1533 {
1534 	return netdev2adap(dev)->msg_enable;
1535 }
1536 
1537 /*
1538  * Set current adapter message level.
1539  */
1540 static void cxgb4vf_set_msglevel(struct net_device *dev, u32 msglevel)
1541 {
1542 	netdev2adap(dev)->msg_enable = msglevel;
1543 }
1544 
1545 /*
1546  * Return the device's current Queue Set ring size parameters along with the
1547  * allowed maximum values.  Since ethtool doesn't understand the concept of
1548  * multi-queue devices, we just return the current values associated with the
1549  * first Queue Set.
1550  */
1551 static void cxgb4vf_get_ringparam(struct net_device *dev,
1552 				  struct ethtool_ringparam *rp)
1553 {
1554 	const struct port_info *pi = netdev_priv(dev);
1555 	const struct sge *s = &pi->adapter->sge;
1556 
1557 	rp->rx_max_pending = MAX_RX_BUFFERS;
1558 	rp->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1559 	rp->rx_jumbo_max_pending = 0;
1560 	rp->tx_max_pending = MAX_TXQ_ENTRIES;
1561 
1562 	rp->rx_pending = s->ethrxq[pi->first_qset].fl.size - MIN_FL_RESID;
1563 	rp->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1564 	rp->rx_jumbo_pending = 0;
1565 	rp->tx_pending = s->ethtxq[pi->first_qset].q.size;
1566 }
1567 
1568 /*
1569  * Set the Queue Set ring size parameters for the device.  Again, since
1570  * ethtool doesn't allow for the concept of multiple queues per device, we'll
1571  * apply these new values across all of the Queue Sets associated with the
1572  * device -- after vetting them of course!
1573  */
1574 static int cxgb4vf_set_ringparam(struct net_device *dev,
1575 				 struct ethtool_ringparam *rp)
1576 {
1577 	const struct port_info *pi = netdev_priv(dev);
1578 	struct adapter *adapter = pi->adapter;
1579 	struct sge *s = &adapter->sge;
1580 	int qs;
1581 
1582 	if (rp->rx_pending > MAX_RX_BUFFERS ||
1583 	    rp->rx_jumbo_pending ||
1584 	    rp->tx_pending > MAX_TXQ_ENTRIES ||
1585 	    rp->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1586 	    rp->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1587 	    rp->rx_pending < MIN_FL_ENTRIES ||
1588 	    rp->tx_pending < MIN_TXQ_ENTRIES)
1589 		return -EINVAL;
1590 
1591 	if (adapter->flags & FULL_INIT_DONE)
1592 		return -EBUSY;
1593 
1594 	for (qs = pi->first_qset; qs < pi->first_qset + pi->nqsets; qs++) {
1595 		s->ethrxq[qs].fl.size = rp->rx_pending + MIN_FL_RESID;
1596 		s->ethrxq[qs].rspq.size = rp->rx_mini_pending;
1597 		s->ethtxq[qs].q.size = rp->tx_pending;
1598 	}
1599 	return 0;
1600 }
1601 
1602 /*
1603  * Return the interrupt holdoff timer and count for the first Queue Set on the
1604  * device.  Our extension ioctl() (the cxgbtool interface) allows the
1605  * interrupt holdoff timer to be read on all of the device's Queue Sets.
1606  */
1607 static int cxgb4vf_get_coalesce(struct net_device *dev,
1608 				struct ethtool_coalesce *coalesce)
1609 {
1610 	const struct port_info *pi = netdev_priv(dev);
1611 	const struct adapter *adapter = pi->adapter;
1612 	const struct sge_rspq *rspq = &adapter->sge.ethrxq[pi->first_qset].rspq;
1613 
1614 	coalesce->rx_coalesce_usecs = qtimer_val(adapter, rspq);
1615 	coalesce->rx_max_coalesced_frames =
1616 		((rspq->intr_params & QINTR_CNT_EN_F)
1617 		 ? adapter->sge.counter_val[rspq->pktcnt_idx]
1618 		 : 0);
1619 	return 0;
1620 }
1621 
1622 /*
1623  * Set the RX interrupt holdoff timer and count for the first Queue Set on the
1624  * interface.  Our extension ioctl() (the cxgbtool interface) allows us to set
1625  * the interrupt holdoff timer on any of the device's Queue Sets.
1626  */
1627 static int cxgb4vf_set_coalesce(struct net_device *dev,
1628 				struct ethtool_coalesce *coalesce)
1629 {
1630 	const struct port_info *pi = netdev_priv(dev);
1631 	struct adapter *adapter = pi->adapter;
1632 
1633 	return set_rxq_intr_params(adapter,
1634 				   &adapter->sge.ethrxq[pi->first_qset].rspq,
1635 				   coalesce->rx_coalesce_usecs,
1636 				   coalesce->rx_max_coalesced_frames);
1637 }
1638 
1639 /*
1640  * Report current port link pause parameter settings.
1641  */
1642 static void cxgb4vf_get_pauseparam(struct net_device *dev,
1643 				   struct ethtool_pauseparam *pauseparam)
1644 {
1645 	struct port_info *pi = netdev_priv(dev);
1646 
1647 	pauseparam->autoneg = (pi->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1648 	pauseparam->rx_pause = (pi->link_cfg.fc & PAUSE_RX) != 0;
1649 	pauseparam->tx_pause = (pi->link_cfg.fc & PAUSE_TX) != 0;
1650 }
1651 
1652 /*
1653  * Identify the port by blinking the port's LED.
1654  */
1655 static int cxgb4vf_phys_id(struct net_device *dev,
1656 			   enum ethtool_phys_id_state state)
1657 {
1658 	unsigned int val;
1659 	struct port_info *pi = netdev_priv(dev);
1660 
1661 	if (state == ETHTOOL_ID_ACTIVE)
1662 		val = 0xffff;
1663 	else if (state == ETHTOOL_ID_INACTIVE)
1664 		val = 0;
1665 	else
1666 		return -EINVAL;
1667 
1668 	return t4vf_identify_port(pi->adapter, pi->viid, val);
1669 }
1670 
1671 /*
1672  * Port stats maintained per queue of the port.
1673  */
1674 struct queue_port_stats {
1675 	u64 tso;
1676 	u64 tx_csum;
1677 	u64 rx_csum;
1678 	u64 vlan_ex;
1679 	u64 vlan_ins;
1680 	u64 lro_pkts;
1681 	u64 lro_merged;
1682 };
1683 
1684 /*
1685  * Strings for the ETH_SS_STATS statistics set ("ethtool -S").  Note that
1686  * these need to match the order of statistics returned by
1687  * t4vf_get_port_stats().
1688  */
1689 static const char stats_strings[][ETH_GSTRING_LEN] = {
1690 	/*
1691 	 * These must match the layout of the t4vf_port_stats structure.
1692 	 */
1693 	"TxBroadcastBytes  ",
1694 	"TxBroadcastFrames ",
1695 	"TxMulticastBytes  ",
1696 	"TxMulticastFrames ",
1697 	"TxUnicastBytes    ",
1698 	"TxUnicastFrames   ",
1699 	"TxDroppedFrames   ",
1700 	"TxOffloadBytes    ",
1701 	"TxOffloadFrames   ",
1702 	"RxBroadcastBytes  ",
1703 	"RxBroadcastFrames ",
1704 	"RxMulticastBytes  ",
1705 	"RxMulticastFrames ",
1706 	"RxUnicastBytes    ",
1707 	"RxUnicastFrames   ",
1708 	"RxErrorFrames     ",
1709 
1710 	/*
1711 	 * These are accumulated per-queue statistics and must match the
1712 	 * order of the fields in the queue_port_stats structure.
1713 	 */
1714 	"TSO               ",
1715 	"TxCsumOffload     ",
1716 	"RxCsumGood        ",
1717 	"VLANextractions   ",
1718 	"VLANinsertions    ",
1719 	"GROPackets        ",
1720 	"GROMerged         ",
1721 };
1722 
1723 /*
1724  * Return the number of statistics in the specified statistics set.
1725  */
1726 static int cxgb4vf_get_sset_count(struct net_device *dev, int sset)
1727 {
1728 	switch (sset) {
1729 	case ETH_SS_STATS:
1730 		return ARRAY_SIZE(stats_strings);
1731 	default:
1732 		return -EOPNOTSUPP;
1733 	}
1734 	/*NOTREACHED*/
1735 }
1736 
1737 /*
1738  * Return the strings for the specified statistics set.
1739  */
1740 static void cxgb4vf_get_strings(struct net_device *dev,
1741 				u32 sset,
1742 				u8 *data)
1743 {
1744 	switch (sset) {
1745 	case ETH_SS_STATS:
1746 		memcpy(data, stats_strings, sizeof(stats_strings));
1747 		break;
1748 	}
1749 }
1750 
1751 /*
1752  * Small utility routine to accumulate queue statistics across the queues of
1753  * a "port".
1754  */
1755 static void collect_sge_port_stats(const struct adapter *adapter,
1756 				   const struct port_info *pi,
1757 				   struct queue_port_stats *stats)
1758 {
1759 	const struct sge_eth_txq *txq = &adapter->sge.ethtxq[pi->first_qset];
1760 	const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[pi->first_qset];
1761 	int qs;
1762 
1763 	memset(stats, 0, sizeof(*stats));
1764 	for (qs = 0; qs < pi->nqsets; qs++, rxq++, txq++) {
1765 		stats->tso += txq->tso;
1766 		stats->tx_csum += txq->tx_cso;
1767 		stats->rx_csum += rxq->stats.rx_cso;
1768 		stats->vlan_ex += rxq->stats.vlan_ex;
1769 		stats->vlan_ins += txq->vlan_ins;
1770 		stats->lro_pkts += rxq->stats.lro_pkts;
1771 		stats->lro_merged += rxq->stats.lro_merged;
1772 	}
1773 }
1774 
1775 /*
1776  * Return the ETH_SS_STATS statistics set.
1777  */
1778 static void cxgb4vf_get_ethtool_stats(struct net_device *dev,
1779 				      struct ethtool_stats *stats,
1780 				      u64 *data)
1781 {
1782 	struct port_info *pi = netdev2pinfo(dev);
1783 	struct adapter *adapter = pi->adapter;
1784 	int err = t4vf_get_port_stats(adapter, pi->pidx,
1785 				      (struct t4vf_port_stats *)data);
1786 	if (err)
1787 		memset(data, 0, sizeof(struct t4vf_port_stats));
1788 
1789 	data += sizeof(struct t4vf_port_stats) / sizeof(u64);
1790 	collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1791 }
1792 
1793 /*
1794  * Return the size of our register map.
1795  */
1796 static int cxgb4vf_get_regs_len(struct net_device *dev)
1797 {
1798 	return T4VF_REGMAP_SIZE;
1799 }
1800 
1801 /*
1802  * Dump a block of registers, start to end inclusive, into a buffer.
1803  */
1804 static void reg_block_dump(struct adapter *adapter, void *regbuf,
1805 			   unsigned int start, unsigned int end)
1806 {
1807 	u32 *bp = regbuf + start - T4VF_REGMAP_START;
1808 
1809 	for ( ; start <= end; start += sizeof(u32)) {
1810 		/*
1811 		 * Avoid reading the Mailbox Control register since that
1812 		 * can trigger a Mailbox Ownership Arbitration cycle and
1813 		 * interfere with communication with the firmware.
1814 		 */
1815 		if (start == T4VF_CIM_BASE_ADDR + CIM_VF_EXT_MAILBOX_CTRL)
1816 			*bp++ = 0xffff;
1817 		else
1818 			*bp++ = t4_read_reg(adapter, start);
1819 	}
1820 }
1821 
1822 /*
1823  * Copy our entire register map into the provided buffer.
1824  */
1825 static void cxgb4vf_get_regs(struct net_device *dev,
1826 			     struct ethtool_regs *regs,
1827 			     void *regbuf)
1828 {
1829 	struct adapter *adapter = netdev2adap(dev);
1830 
1831 	regs->version = mk_adap_vers(adapter);
1832 
1833 	/*
1834 	 * Fill in register buffer with our register map.
1835 	 */
1836 	memset(regbuf, 0, T4VF_REGMAP_SIZE);
1837 
1838 	reg_block_dump(adapter, regbuf,
1839 		       T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_FIRST,
1840 		       T4VF_SGE_BASE_ADDR + T4VF_MOD_MAP_SGE_LAST);
1841 	reg_block_dump(adapter, regbuf,
1842 		       T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_FIRST,
1843 		       T4VF_MPS_BASE_ADDR + T4VF_MOD_MAP_MPS_LAST);
1844 
1845 	/* T5 adds new registers in the PL Register map.
1846 	 */
1847 	reg_block_dump(adapter, regbuf,
1848 		       T4VF_PL_BASE_ADDR + T4VF_MOD_MAP_PL_FIRST,
1849 		       T4VF_PL_BASE_ADDR + (is_t4(adapter->params.chip)
1850 		       ? PL_VF_WHOAMI_A : PL_VF_REVISION_A));
1851 	reg_block_dump(adapter, regbuf,
1852 		       T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_FIRST,
1853 		       T4VF_CIM_BASE_ADDR + T4VF_MOD_MAP_CIM_LAST);
1854 
1855 	reg_block_dump(adapter, regbuf,
1856 		       T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_FIRST,
1857 		       T4VF_MBDATA_BASE_ADDR + T4VF_MBDATA_LAST);
1858 }
1859 
1860 /*
1861  * Report current Wake On LAN settings.
1862  */
1863 static void cxgb4vf_get_wol(struct net_device *dev,
1864 			    struct ethtool_wolinfo *wol)
1865 {
1866 	wol->supported = 0;
1867 	wol->wolopts = 0;
1868 	memset(&wol->sopass, 0, sizeof(wol->sopass));
1869 }
1870 
1871 /*
1872  * TCP Segmentation Offload flags which we support.
1873  */
1874 #define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
1875 
1876 static const struct ethtool_ops cxgb4vf_ethtool_ops = {
1877 	.get_link_ksettings	= cxgb4vf_get_link_ksettings,
1878 	.get_fecparam		= cxgb4vf_get_fecparam,
1879 	.get_drvinfo		= cxgb4vf_get_drvinfo,
1880 	.get_msglevel		= cxgb4vf_get_msglevel,
1881 	.set_msglevel		= cxgb4vf_set_msglevel,
1882 	.get_ringparam		= cxgb4vf_get_ringparam,
1883 	.set_ringparam		= cxgb4vf_set_ringparam,
1884 	.get_coalesce		= cxgb4vf_get_coalesce,
1885 	.set_coalesce		= cxgb4vf_set_coalesce,
1886 	.get_pauseparam		= cxgb4vf_get_pauseparam,
1887 	.get_link		= ethtool_op_get_link,
1888 	.get_strings		= cxgb4vf_get_strings,
1889 	.set_phys_id		= cxgb4vf_phys_id,
1890 	.get_sset_count		= cxgb4vf_get_sset_count,
1891 	.get_ethtool_stats	= cxgb4vf_get_ethtool_stats,
1892 	.get_regs_len		= cxgb4vf_get_regs_len,
1893 	.get_regs		= cxgb4vf_get_regs,
1894 	.get_wol		= cxgb4vf_get_wol,
1895 };
1896 
1897 /*
1898  * /sys/kernel/debug/cxgb4vf support code and data.
1899  * ================================================
1900  */
1901 
1902 /*
1903  * Show Firmware Mailbox Command/Reply Log
1904  *
1905  * Note that we don't do any locking when dumping the Firmware Mailbox Log so
1906  * it's possible that we can catch things during a log update and therefore
1907  * see partially corrupted log entries.  But i9t's probably Good Enough(tm).
1908  * If we ever decide that we want to make sure that we're dumping a coherent
1909  * log, we'd need to perform locking in the mailbox logging and in
1910  * mboxlog_open() where we'd need to grab the entire mailbox log in one go
1911  * like we do for the Firmware Device Log.  But as stated above, meh ...
1912  */
1913 static int mboxlog_show(struct seq_file *seq, void *v)
1914 {
1915 	struct adapter *adapter = seq->private;
1916 	struct mbox_cmd_log *log = adapter->mbox_log;
1917 	struct mbox_cmd *entry;
1918 	int entry_idx, i;
1919 
1920 	if (v == SEQ_START_TOKEN) {
1921 		seq_printf(seq,
1922 			   "%10s  %15s  %5s  %5s  %s\n",
1923 			   "Seq#", "Tstamp", "Atime", "Etime",
1924 			   "Command/Reply");
1925 		return 0;
1926 	}
1927 
1928 	entry_idx = log->cursor + ((uintptr_t)v - 2);
1929 	if (entry_idx >= log->size)
1930 		entry_idx -= log->size;
1931 	entry = mbox_cmd_log_entry(log, entry_idx);
1932 
1933 	/* skip over unused entries */
1934 	if (entry->timestamp == 0)
1935 		return 0;
1936 
1937 	seq_printf(seq, "%10u  %15llu  %5d  %5d",
1938 		   entry->seqno, entry->timestamp,
1939 		   entry->access, entry->execute);
1940 	for (i = 0; i < MBOX_LEN / 8; i++) {
1941 		u64 flit = entry->cmd[i];
1942 		u32 hi = (u32)(flit >> 32);
1943 		u32 lo = (u32)flit;
1944 
1945 		seq_printf(seq, "  %08x %08x", hi, lo);
1946 	}
1947 	seq_puts(seq, "\n");
1948 	return 0;
1949 }
1950 
1951 static inline void *mboxlog_get_idx(struct seq_file *seq, loff_t pos)
1952 {
1953 	struct adapter *adapter = seq->private;
1954 	struct mbox_cmd_log *log = adapter->mbox_log;
1955 
1956 	return ((pos <= log->size) ? (void *)(uintptr_t)(pos + 1) : NULL);
1957 }
1958 
1959 static void *mboxlog_start(struct seq_file *seq, loff_t *pos)
1960 {
1961 	return *pos ? mboxlog_get_idx(seq, *pos) : SEQ_START_TOKEN;
1962 }
1963 
1964 static void *mboxlog_next(struct seq_file *seq, void *v, loff_t *pos)
1965 {
1966 	++*pos;
1967 	return mboxlog_get_idx(seq, *pos);
1968 }
1969 
1970 static void mboxlog_stop(struct seq_file *seq, void *v)
1971 {
1972 }
1973 
1974 static const struct seq_operations mboxlog_seq_ops = {
1975 	.start = mboxlog_start,
1976 	.next  = mboxlog_next,
1977 	.stop  = mboxlog_stop,
1978 	.show  = mboxlog_show
1979 };
1980 
1981 static int mboxlog_open(struct inode *inode, struct file *file)
1982 {
1983 	int res = seq_open(file, &mboxlog_seq_ops);
1984 
1985 	if (!res) {
1986 		struct seq_file *seq = file->private_data;
1987 
1988 		seq->private = inode->i_private;
1989 	}
1990 	return res;
1991 }
1992 
1993 static const struct file_operations mboxlog_fops = {
1994 	.owner   = THIS_MODULE,
1995 	.open    = mboxlog_open,
1996 	.read    = seq_read,
1997 	.llseek  = seq_lseek,
1998 	.release = seq_release,
1999 };
2000 
2001 /*
2002  * Show SGE Queue Set information.  We display QPL Queues Sets per line.
2003  */
2004 #define QPL	4
2005 
2006 static int sge_qinfo_show(struct seq_file *seq, void *v)
2007 {
2008 	struct adapter *adapter = seq->private;
2009 	int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
2010 	int qs, r = (uintptr_t)v - 1;
2011 
2012 	if (r)
2013 		seq_putc(seq, '\n');
2014 
2015 	#define S3(fmt_spec, s, v) \
2016 		do {\
2017 			seq_printf(seq, "%-12s", s); \
2018 			for (qs = 0; qs < n; ++qs) \
2019 				seq_printf(seq, " %16" fmt_spec, v); \
2020 			seq_putc(seq, '\n'); \
2021 		} while (0)
2022 	#define S(s, v)		S3("s", s, v)
2023 	#define T(s, v)		S3("u", s, txq[qs].v)
2024 	#define R(s, v)		S3("u", s, rxq[qs].v)
2025 
2026 	if (r < eth_entries) {
2027 		const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
2028 		const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
2029 		int n = min(QPL, adapter->sge.ethqsets - QPL * r);
2030 
2031 		S("QType:", "Ethernet");
2032 		S("Interface:",
2033 		  (rxq[qs].rspq.netdev
2034 		   ? rxq[qs].rspq.netdev->name
2035 		   : "N/A"));
2036 		S3("d", "Port:",
2037 		   (rxq[qs].rspq.netdev
2038 		    ? ((struct port_info *)
2039 		       netdev_priv(rxq[qs].rspq.netdev))->port_id
2040 		    : -1));
2041 		T("TxQ ID:", q.abs_id);
2042 		T("TxQ size:", q.size);
2043 		T("TxQ inuse:", q.in_use);
2044 		T("TxQ PIdx:", q.pidx);
2045 		T("TxQ CIdx:", q.cidx);
2046 		R("RspQ ID:", rspq.abs_id);
2047 		R("RspQ size:", rspq.size);
2048 		R("RspQE size:", rspq.iqe_len);
2049 		S3("u", "Intr delay:", qtimer_val(adapter, &rxq[qs].rspq));
2050 		S3("u", "Intr pktcnt:",
2051 		   adapter->sge.counter_val[rxq[qs].rspq.pktcnt_idx]);
2052 		R("RspQ CIdx:", rspq.cidx);
2053 		R("RspQ Gen:", rspq.gen);
2054 		R("FL ID:", fl.abs_id);
2055 		R("FL size:", fl.size - MIN_FL_RESID);
2056 		R("FL avail:", fl.avail);
2057 		R("FL PIdx:", fl.pidx);
2058 		R("FL CIdx:", fl.cidx);
2059 		return 0;
2060 	}
2061 
2062 	r -= eth_entries;
2063 	if (r == 0) {
2064 		const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
2065 
2066 		seq_printf(seq, "%-12s %16s\n", "QType:", "FW event queue");
2067 		seq_printf(seq, "%-12s %16u\n", "RspQ ID:", evtq->abs_id);
2068 		seq_printf(seq, "%-12s %16u\n", "Intr delay:",
2069 			   qtimer_val(adapter, evtq));
2070 		seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
2071 			   adapter->sge.counter_val[evtq->pktcnt_idx]);
2072 		seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", evtq->cidx);
2073 		seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", evtq->gen);
2074 	} else if (r == 1) {
2075 		const struct sge_rspq *intrq = &adapter->sge.intrq;
2076 
2077 		seq_printf(seq, "%-12s %16s\n", "QType:", "Interrupt Queue");
2078 		seq_printf(seq, "%-12s %16u\n", "RspQ ID:", intrq->abs_id);
2079 		seq_printf(seq, "%-12s %16u\n", "Intr delay:",
2080 			   qtimer_val(adapter, intrq));
2081 		seq_printf(seq, "%-12s %16u\n", "Intr pktcnt:",
2082 			   adapter->sge.counter_val[intrq->pktcnt_idx]);
2083 		seq_printf(seq, "%-12s %16u\n", "RspQ Cidx:", intrq->cidx);
2084 		seq_printf(seq, "%-12s %16u\n", "RspQ Gen:", intrq->gen);
2085 	}
2086 
2087 	#undef R
2088 	#undef T
2089 	#undef S
2090 	#undef S3
2091 
2092 	return 0;
2093 }
2094 
2095 /*
2096  * Return the number of "entries" in our "file".  We group the multi-Queue
2097  * sections with QPL Queue Sets per "entry".  The sections of the output are:
2098  *
2099  *     Ethernet RX/TX Queue Sets
2100  *     Firmware Event Queue
2101  *     Forwarded Interrupt Queue (if in MSI mode)
2102  */
2103 static int sge_queue_entries(const struct adapter *adapter)
2104 {
2105 	return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
2106 		((adapter->flags & USING_MSI) != 0);
2107 }
2108 
2109 static void *sge_queue_start(struct seq_file *seq, loff_t *pos)
2110 {
2111 	int entries = sge_queue_entries(seq->private);
2112 
2113 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2114 }
2115 
2116 static void sge_queue_stop(struct seq_file *seq, void *v)
2117 {
2118 }
2119 
2120 static void *sge_queue_next(struct seq_file *seq, void *v, loff_t *pos)
2121 {
2122 	int entries = sge_queue_entries(seq->private);
2123 
2124 	++*pos;
2125 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2126 }
2127 
2128 static const struct seq_operations sge_qinfo_seq_ops = {
2129 	.start = sge_queue_start,
2130 	.next  = sge_queue_next,
2131 	.stop  = sge_queue_stop,
2132 	.show  = sge_qinfo_show
2133 };
2134 
2135 static int sge_qinfo_open(struct inode *inode, struct file *file)
2136 {
2137 	int res = seq_open(file, &sge_qinfo_seq_ops);
2138 
2139 	if (!res) {
2140 		struct seq_file *seq = file->private_data;
2141 		seq->private = inode->i_private;
2142 	}
2143 	return res;
2144 }
2145 
2146 static const struct file_operations sge_qinfo_debugfs_fops = {
2147 	.owner   = THIS_MODULE,
2148 	.open    = sge_qinfo_open,
2149 	.read    = seq_read,
2150 	.llseek  = seq_lseek,
2151 	.release = seq_release,
2152 };
2153 
2154 /*
2155  * Show SGE Queue Set statistics.  We display QPL Queues Sets per line.
2156  */
2157 #define QPL	4
2158 
2159 static int sge_qstats_show(struct seq_file *seq, void *v)
2160 {
2161 	struct adapter *adapter = seq->private;
2162 	int eth_entries = DIV_ROUND_UP(adapter->sge.ethqsets, QPL);
2163 	int qs, r = (uintptr_t)v - 1;
2164 
2165 	if (r)
2166 		seq_putc(seq, '\n');
2167 
2168 	#define S3(fmt, s, v) \
2169 		do { \
2170 			seq_printf(seq, "%-16s", s); \
2171 			for (qs = 0; qs < n; ++qs) \
2172 				seq_printf(seq, " %8" fmt, v); \
2173 			seq_putc(seq, '\n'); \
2174 		} while (0)
2175 	#define S(s, v)		S3("s", s, v)
2176 
2177 	#define T3(fmt, s, v)	S3(fmt, s, txq[qs].v)
2178 	#define T(s, v)		T3("lu", s, v)
2179 
2180 	#define R3(fmt, s, v)	S3(fmt, s, rxq[qs].v)
2181 	#define R(s, v)		R3("lu", s, v)
2182 
2183 	if (r < eth_entries) {
2184 		const struct sge_eth_rxq *rxq = &adapter->sge.ethrxq[r * QPL];
2185 		const struct sge_eth_txq *txq = &adapter->sge.ethtxq[r * QPL];
2186 		int n = min(QPL, adapter->sge.ethqsets - QPL * r);
2187 
2188 		S("QType:", "Ethernet");
2189 		S("Interface:",
2190 		  (rxq[qs].rspq.netdev
2191 		   ? rxq[qs].rspq.netdev->name
2192 		   : "N/A"));
2193 		R3("u", "RspQNullInts:", rspq.unhandled_irqs);
2194 		R("RxPackets:", stats.pkts);
2195 		R("RxCSO:", stats.rx_cso);
2196 		R("VLANxtract:", stats.vlan_ex);
2197 		R("LROmerged:", stats.lro_merged);
2198 		R("LROpackets:", stats.lro_pkts);
2199 		R("RxDrops:", stats.rx_drops);
2200 		T("TSO:", tso);
2201 		T("TxCSO:", tx_cso);
2202 		T("VLANins:", vlan_ins);
2203 		T("TxQFull:", q.stops);
2204 		T("TxQRestarts:", q.restarts);
2205 		T("TxMapErr:", mapping_err);
2206 		R("FLAllocErr:", fl.alloc_failed);
2207 		R("FLLrgAlcErr:", fl.large_alloc_failed);
2208 		R("FLStarving:", fl.starving);
2209 		return 0;
2210 	}
2211 
2212 	r -= eth_entries;
2213 	if (r == 0) {
2214 		const struct sge_rspq *evtq = &adapter->sge.fw_evtq;
2215 
2216 		seq_printf(seq, "%-8s %16s\n", "QType:", "FW event queue");
2217 		seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
2218 			   evtq->unhandled_irqs);
2219 		seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", evtq->cidx);
2220 		seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", evtq->gen);
2221 	} else if (r == 1) {
2222 		const struct sge_rspq *intrq = &adapter->sge.intrq;
2223 
2224 		seq_printf(seq, "%-8s %16s\n", "QType:", "Interrupt Queue");
2225 		seq_printf(seq, "%-16s %8u\n", "RspQNullInts:",
2226 			   intrq->unhandled_irqs);
2227 		seq_printf(seq, "%-16s %8u\n", "RspQ CIdx:", intrq->cidx);
2228 		seq_printf(seq, "%-16s %8u\n", "RspQ Gen:", intrq->gen);
2229 	}
2230 
2231 	#undef R
2232 	#undef T
2233 	#undef S
2234 	#undef R3
2235 	#undef T3
2236 	#undef S3
2237 
2238 	return 0;
2239 }
2240 
2241 /*
2242  * Return the number of "entries" in our "file".  We group the multi-Queue
2243  * sections with QPL Queue Sets per "entry".  The sections of the output are:
2244  *
2245  *     Ethernet RX/TX Queue Sets
2246  *     Firmware Event Queue
2247  *     Forwarded Interrupt Queue (if in MSI mode)
2248  */
2249 static int sge_qstats_entries(const struct adapter *adapter)
2250 {
2251 	return DIV_ROUND_UP(adapter->sge.ethqsets, QPL) + 1 +
2252 		((adapter->flags & USING_MSI) != 0);
2253 }
2254 
2255 static void *sge_qstats_start(struct seq_file *seq, loff_t *pos)
2256 {
2257 	int entries = sge_qstats_entries(seq->private);
2258 
2259 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2260 }
2261 
2262 static void sge_qstats_stop(struct seq_file *seq, void *v)
2263 {
2264 }
2265 
2266 static void *sge_qstats_next(struct seq_file *seq, void *v, loff_t *pos)
2267 {
2268 	int entries = sge_qstats_entries(seq->private);
2269 
2270 	(*pos)++;
2271 	return *pos < entries ? (void *)((uintptr_t)*pos + 1) : NULL;
2272 }
2273 
2274 static const struct seq_operations sge_qstats_seq_ops = {
2275 	.start = sge_qstats_start,
2276 	.next  = sge_qstats_next,
2277 	.stop  = sge_qstats_stop,
2278 	.show  = sge_qstats_show
2279 };
2280 
2281 static int sge_qstats_open(struct inode *inode, struct file *file)
2282 {
2283 	int res = seq_open(file, &sge_qstats_seq_ops);
2284 
2285 	if (res == 0) {
2286 		struct seq_file *seq = file->private_data;
2287 		seq->private = inode->i_private;
2288 	}
2289 	return res;
2290 }
2291 
2292 static const struct file_operations sge_qstats_proc_fops = {
2293 	.owner   = THIS_MODULE,
2294 	.open    = sge_qstats_open,
2295 	.read    = seq_read,
2296 	.llseek  = seq_lseek,
2297 	.release = seq_release,
2298 };
2299 
2300 /*
2301  * Show PCI-E SR-IOV Virtual Function Resource Limits.
2302  */
2303 static int resources_show(struct seq_file *seq, void *v)
2304 {
2305 	struct adapter *adapter = seq->private;
2306 	struct vf_resources *vfres = &adapter->params.vfres;
2307 
2308 	#define S(desc, fmt, var) \
2309 		seq_printf(seq, "%-60s " fmt "\n", \
2310 			   desc " (" #var "):", vfres->var)
2311 
2312 	S("Virtual Interfaces", "%d", nvi);
2313 	S("Egress Queues", "%d", neq);
2314 	S("Ethernet Control", "%d", nethctrl);
2315 	S("Ingress Queues/w Free Lists/Interrupts", "%d", niqflint);
2316 	S("Ingress Queues", "%d", niq);
2317 	S("Traffic Class", "%d", tc);
2318 	S("Port Access Rights Mask", "%#x", pmask);
2319 	S("MAC Address Filters", "%d", nexactf);
2320 	S("Firmware Command Read Capabilities", "%#x", r_caps);
2321 	S("Firmware Command Write/Execute Capabilities", "%#x", wx_caps);
2322 
2323 	#undef S
2324 
2325 	return 0;
2326 }
2327 
2328 static int resources_open(struct inode *inode, struct file *file)
2329 {
2330 	return single_open(file, resources_show, inode->i_private);
2331 }
2332 
2333 static const struct file_operations resources_proc_fops = {
2334 	.owner   = THIS_MODULE,
2335 	.open    = resources_open,
2336 	.read    = seq_read,
2337 	.llseek  = seq_lseek,
2338 	.release = single_release,
2339 };
2340 
2341 /*
2342  * Show Virtual Interfaces.
2343  */
2344 static int interfaces_show(struct seq_file *seq, void *v)
2345 {
2346 	if (v == SEQ_START_TOKEN) {
2347 		seq_puts(seq, "Interface  Port   VIID\n");
2348 	} else {
2349 		struct adapter *adapter = seq->private;
2350 		int pidx = (uintptr_t)v - 2;
2351 		struct net_device *dev = adapter->port[pidx];
2352 		struct port_info *pi = netdev_priv(dev);
2353 
2354 		seq_printf(seq, "%9s  %4d  %#5x\n",
2355 			   dev->name, pi->port_id, pi->viid);
2356 	}
2357 	return 0;
2358 }
2359 
2360 static inline void *interfaces_get_idx(struct adapter *adapter, loff_t pos)
2361 {
2362 	return pos <= adapter->params.nports
2363 		? (void *)(uintptr_t)(pos + 1)
2364 		: NULL;
2365 }
2366 
2367 static void *interfaces_start(struct seq_file *seq, loff_t *pos)
2368 {
2369 	return *pos
2370 		? interfaces_get_idx(seq->private, *pos)
2371 		: SEQ_START_TOKEN;
2372 }
2373 
2374 static void *interfaces_next(struct seq_file *seq, void *v, loff_t *pos)
2375 {
2376 	(*pos)++;
2377 	return interfaces_get_idx(seq->private, *pos);
2378 }
2379 
2380 static void interfaces_stop(struct seq_file *seq, void *v)
2381 {
2382 }
2383 
2384 static const struct seq_operations interfaces_seq_ops = {
2385 	.start = interfaces_start,
2386 	.next  = interfaces_next,
2387 	.stop  = interfaces_stop,
2388 	.show  = interfaces_show
2389 };
2390 
2391 static int interfaces_open(struct inode *inode, struct file *file)
2392 {
2393 	int res = seq_open(file, &interfaces_seq_ops);
2394 
2395 	if (res == 0) {
2396 		struct seq_file *seq = file->private_data;
2397 		seq->private = inode->i_private;
2398 	}
2399 	return res;
2400 }
2401 
2402 static const struct file_operations interfaces_proc_fops = {
2403 	.owner   = THIS_MODULE,
2404 	.open    = interfaces_open,
2405 	.read    = seq_read,
2406 	.llseek  = seq_lseek,
2407 	.release = seq_release,
2408 };
2409 
2410 /*
2411  * /sys/kernel/debugfs/cxgb4vf/ files list.
2412  */
2413 struct cxgb4vf_debugfs_entry {
2414 	const char *name;		/* name of debugfs node */
2415 	umode_t mode;			/* file system mode */
2416 	const struct file_operations *fops;
2417 };
2418 
2419 static struct cxgb4vf_debugfs_entry debugfs_files[] = {
2420 	{ "mboxlog",    0444, &mboxlog_fops },
2421 	{ "sge_qinfo",  0444, &sge_qinfo_debugfs_fops },
2422 	{ "sge_qstats", 0444, &sge_qstats_proc_fops },
2423 	{ "resources",  0444, &resources_proc_fops },
2424 	{ "interfaces", 0444, &interfaces_proc_fops },
2425 };
2426 
2427 /*
2428  * Module and device initialization and cleanup code.
2429  * ==================================================
2430  */
2431 
2432 /*
2433  * Set up out /sys/kernel/debug/cxgb4vf sub-nodes.  We assume that the
2434  * directory (debugfs_root) has already been set up.
2435  */
2436 static int setup_debugfs(struct adapter *adapter)
2437 {
2438 	int i;
2439 
2440 	BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root));
2441 
2442 	/*
2443 	 * Debugfs support is best effort.
2444 	 */
2445 	for (i = 0; i < ARRAY_SIZE(debugfs_files); i++)
2446 		(void)debugfs_create_file(debugfs_files[i].name,
2447 				  debugfs_files[i].mode,
2448 				  adapter->debugfs_root,
2449 				  (void *)adapter,
2450 				  debugfs_files[i].fops);
2451 
2452 	return 0;
2453 }
2454 
2455 /*
2456  * Tear down the /sys/kernel/debug/cxgb4vf sub-nodes created above.  We leave
2457  * it to our caller to tear down the directory (debugfs_root).
2458  */
2459 static void cleanup_debugfs(struct adapter *adapter)
2460 {
2461 	BUG_ON(IS_ERR_OR_NULL(adapter->debugfs_root));
2462 
2463 	/*
2464 	 * Unlike our sister routine cleanup_proc(), we don't need to remove
2465 	 * individual entries because a call will be made to
2466 	 * debugfs_remove_recursive().  We just need to clean up any ancillary
2467 	 * persistent state.
2468 	 */
2469 	/* nothing to do */
2470 }
2471 
2472 /* Figure out how many Ports and Queue Sets we can support.  This depends on
2473  * knowing our Virtual Function Resources and may be called a second time if
2474  * we fall back from MSI-X to MSI Interrupt Mode.
2475  */
2476 static void size_nports_qsets(struct adapter *adapter)
2477 {
2478 	struct vf_resources *vfres = &adapter->params.vfres;
2479 	unsigned int ethqsets, pmask_nports;
2480 
2481 	/* The number of "ports" which we support is equal to the number of
2482 	 * Virtual Interfaces with which we've been provisioned.
2483 	 */
2484 	adapter->params.nports = vfres->nvi;
2485 	if (adapter->params.nports > MAX_NPORTS) {
2486 		dev_warn(adapter->pdev_dev, "only using %d of %d maximum"
2487 			 " allowed virtual interfaces\n", MAX_NPORTS,
2488 			 adapter->params.nports);
2489 		adapter->params.nports = MAX_NPORTS;
2490 	}
2491 
2492 	/* We may have been provisioned with more VIs than the number of
2493 	 * ports we're allowed to access (our Port Access Rights Mask).
2494 	 * This is obviously a configuration conflict but we don't want to
2495 	 * crash the kernel or anything silly just because of that.
2496 	 */
2497 	pmask_nports = hweight32(adapter->params.vfres.pmask);
2498 	if (pmask_nports < adapter->params.nports) {
2499 		dev_warn(adapter->pdev_dev, "only using %d of %d provisioned"
2500 			 " virtual interfaces; limited by Port Access Rights"
2501 			 " mask %#x\n", pmask_nports, adapter->params.nports,
2502 			 adapter->params.vfres.pmask);
2503 		adapter->params.nports = pmask_nports;
2504 	}
2505 
2506 	/* We need to reserve an Ingress Queue for the Asynchronous Firmware
2507 	 * Event Queue.  And if we're using MSI Interrupts, we'll also need to
2508 	 * reserve an Ingress Queue for a Forwarded Interrupts.
2509 	 *
2510 	 * The rest of the FL/Intr-capable ingress queues will be matched up
2511 	 * one-for-one with Ethernet/Control egress queues in order to form
2512 	 * "Queue Sets" which will be aportioned between the "ports".  For
2513 	 * each Queue Set, we'll need the ability to allocate two Egress
2514 	 * Contexts -- one for the Ingress Queue Free List and one for the TX
2515 	 * Ethernet Queue.
2516 	 *
2517 	 * Note that even if we're currently configured to use MSI-X
2518 	 * Interrupts (module variable msi == MSI_MSIX) we may get downgraded
2519 	 * to MSI Interrupts if we can't get enough MSI-X Interrupts.  If that
2520 	 * happens we'll need to adjust things later.
2521 	 */
2522 	ethqsets = vfres->niqflint - 1 - (msi == MSI_MSI);
2523 	if (vfres->nethctrl != ethqsets)
2524 		ethqsets = min(vfres->nethctrl, ethqsets);
2525 	if (vfres->neq < ethqsets*2)
2526 		ethqsets = vfres->neq/2;
2527 	if (ethqsets > MAX_ETH_QSETS)
2528 		ethqsets = MAX_ETH_QSETS;
2529 	adapter->sge.max_ethqsets = ethqsets;
2530 
2531 	if (adapter->sge.max_ethqsets < adapter->params.nports) {
2532 		dev_warn(adapter->pdev_dev, "only using %d of %d available"
2533 			 " virtual interfaces (too few Queue Sets)\n",
2534 			 adapter->sge.max_ethqsets, adapter->params.nports);
2535 		adapter->params.nports = adapter->sge.max_ethqsets;
2536 	}
2537 }
2538 
2539 /*
2540  * Perform early "adapter" initialization.  This is where we discover what
2541  * adapter parameters we're going to be using and initialize basic adapter
2542  * hardware support.
2543  */
2544 static int adap_init0(struct adapter *adapter)
2545 {
2546 	struct sge_params *sge_params = &adapter->params.sge;
2547 	struct sge *s = &adapter->sge;
2548 	int err;
2549 	u32 param, val = 0;
2550 
2551 	/*
2552 	 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux
2553 	 * 2.6.31 and later we can't call pci_reset_function() in order to
2554 	 * issue an FLR because of a self- deadlock on the device semaphore.
2555 	 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the
2556 	 * cases where they're needed -- for instance, some versions of KVM
2557 	 * fail to reset "Assigned Devices" when the VM reboots.  Therefore we
2558 	 * use the firmware based reset in order to reset any per function
2559 	 * state.
2560 	 */
2561 	err = t4vf_fw_reset(adapter);
2562 	if (err < 0) {
2563 		dev_err(adapter->pdev_dev, "FW reset failed: err=%d\n", err);
2564 		return err;
2565 	}
2566 
2567 	/*
2568 	 * Grab basic operational parameters.  These will predominantly have
2569 	 * been set up by the Physical Function Driver or will be hard coded
2570 	 * into the adapter.  We just have to live with them ...  Note that
2571 	 * we _must_ get our VPD parameters before our SGE parameters because
2572 	 * we need to know the adapter's core clock from the VPD in order to
2573 	 * properly decode the SGE Timer Values.
2574 	 */
2575 	err = t4vf_get_dev_params(adapter);
2576 	if (err) {
2577 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2578 			" device parameters: err=%d\n", err);
2579 		return err;
2580 	}
2581 	err = t4vf_get_vpd_params(adapter);
2582 	if (err) {
2583 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2584 			" VPD parameters: err=%d\n", err);
2585 		return err;
2586 	}
2587 	err = t4vf_get_sge_params(adapter);
2588 	if (err) {
2589 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2590 			" SGE parameters: err=%d\n", err);
2591 		return err;
2592 	}
2593 	err = t4vf_get_rss_glb_config(adapter);
2594 	if (err) {
2595 		dev_err(adapter->pdev_dev, "unable to retrieve adapter"
2596 			" RSS parameters: err=%d\n", err);
2597 		return err;
2598 	}
2599 	if (adapter->params.rss.mode !=
2600 	    FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
2601 		dev_err(adapter->pdev_dev, "unable to operate with global RSS"
2602 			" mode %d\n", adapter->params.rss.mode);
2603 		return -EINVAL;
2604 	}
2605 	err = t4vf_sge_init(adapter);
2606 	if (err) {
2607 		dev_err(adapter->pdev_dev, "unable to use adapter parameters:"
2608 			" err=%d\n", err);
2609 		return err;
2610 	}
2611 
2612 	/* If we're running on newer firmware, let it know that we're
2613 	 * prepared to deal with encapsulated CPL messages.  Older
2614 	 * firmware won't understand this and we'll just get
2615 	 * unencapsulated messages ...
2616 	 */
2617 	param = FW_PARAMS_MNEM_V(FW_PARAMS_MNEM_PFVF) |
2618 		FW_PARAMS_PARAM_X_V(FW_PARAMS_PARAM_PFVF_CPLFW4MSG_ENCAP);
2619 	val = 1;
2620 	(void) t4vf_set_params(adapter, 1, &param, &val);
2621 
2622 	/*
2623 	 * Retrieve our RX interrupt holdoff timer values and counter
2624 	 * threshold values from the SGE parameters.
2625 	 */
2626 	s->timer_val[0] = core_ticks_to_us(adapter,
2627 		TIMERVALUE0_G(sge_params->sge_timer_value_0_and_1));
2628 	s->timer_val[1] = core_ticks_to_us(adapter,
2629 		TIMERVALUE1_G(sge_params->sge_timer_value_0_and_1));
2630 	s->timer_val[2] = core_ticks_to_us(adapter,
2631 		TIMERVALUE0_G(sge_params->sge_timer_value_2_and_3));
2632 	s->timer_val[3] = core_ticks_to_us(adapter,
2633 		TIMERVALUE1_G(sge_params->sge_timer_value_2_and_3));
2634 	s->timer_val[4] = core_ticks_to_us(adapter,
2635 		TIMERVALUE0_G(sge_params->sge_timer_value_4_and_5));
2636 	s->timer_val[5] = core_ticks_to_us(adapter,
2637 		TIMERVALUE1_G(sge_params->sge_timer_value_4_and_5));
2638 
2639 	s->counter_val[0] = THRESHOLD_0_G(sge_params->sge_ingress_rx_threshold);
2640 	s->counter_val[1] = THRESHOLD_1_G(sge_params->sge_ingress_rx_threshold);
2641 	s->counter_val[2] = THRESHOLD_2_G(sge_params->sge_ingress_rx_threshold);
2642 	s->counter_val[3] = THRESHOLD_3_G(sge_params->sge_ingress_rx_threshold);
2643 
2644 	/*
2645 	 * Grab our Virtual Interface resource allocation, extract the
2646 	 * features that we're interested in and do a bit of sanity testing on
2647 	 * what we discover.
2648 	 */
2649 	err = t4vf_get_vfres(adapter);
2650 	if (err) {
2651 		dev_err(adapter->pdev_dev, "unable to get virtual interface"
2652 			" resources: err=%d\n", err);
2653 		return err;
2654 	}
2655 
2656 	/* Check for various parameter sanity issues */
2657 	if (adapter->params.vfres.pmask == 0) {
2658 		dev_err(adapter->pdev_dev, "no port access configured\n"
2659 			"usable!\n");
2660 		return -EINVAL;
2661 	}
2662 	if (adapter->params.vfres.nvi == 0) {
2663 		dev_err(adapter->pdev_dev, "no virtual interfaces configured/"
2664 			"usable!\n");
2665 		return -EINVAL;
2666 	}
2667 
2668 	/* Initialize nports and max_ethqsets now that we have our Virtual
2669 	 * Function Resources.
2670 	 */
2671 	size_nports_qsets(adapter);
2672 
2673 	return 0;
2674 }
2675 
2676 static inline void init_rspq(struct sge_rspq *rspq, u8 timer_idx,
2677 			     u8 pkt_cnt_idx, unsigned int size,
2678 			     unsigned int iqe_size)
2679 {
2680 	rspq->intr_params = (QINTR_TIMER_IDX_V(timer_idx) |
2681 			     (pkt_cnt_idx < SGE_NCOUNTERS ?
2682 			      QINTR_CNT_EN_F : 0));
2683 	rspq->pktcnt_idx = (pkt_cnt_idx < SGE_NCOUNTERS
2684 			    ? pkt_cnt_idx
2685 			    : 0);
2686 	rspq->iqe_len = iqe_size;
2687 	rspq->size = size;
2688 }
2689 
2690 /*
2691  * Perform default configuration of DMA queues depending on the number and
2692  * type of ports we found and the number of available CPUs.  Most settings can
2693  * be modified by the admin via ethtool and cxgbtool prior to the adapter
2694  * being brought up for the first time.
2695  */
2696 static void cfg_queues(struct adapter *adapter)
2697 {
2698 	struct sge *s = &adapter->sge;
2699 	int q10g, n10g, qidx, pidx, qs;
2700 	size_t iqe_size;
2701 
2702 	/*
2703 	 * We should not be called till we know how many Queue Sets we can
2704 	 * support.  In particular, this means that we need to know what kind
2705 	 * of interrupts we'll be using ...
2706 	 */
2707 	BUG_ON((adapter->flags & (USING_MSIX|USING_MSI)) == 0);
2708 
2709 	/*
2710 	 * Count the number of 10GbE Virtual Interfaces that we have.
2711 	 */
2712 	n10g = 0;
2713 	for_each_port(adapter, pidx)
2714 		n10g += is_x_10g_port(&adap2pinfo(adapter, pidx)->link_cfg);
2715 
2716 	/*
2717 	 * We default to 1 queue per non-10G port and up to # of cores queues
2718 	 * per 10G port.
2719 	 */
2720 	if (n10g == 0)
2721 		q10g = 0;
2722 	else {
2723 		int n1g = (adapter->params.nports - n10g);
2724 		q10g = (adapter->sge.max_ethqsets - n1g) / n10g;
2725 		if (q10g > num_online_cpus())
2726 			q10g = num_online_cpus();
2727 	}
2728 
2729 	/*
2730 	 * Allocate the "Queue Sets" to the various Virtual Interfaces.
2731 	 * The layout will be established in setup_sge_queues() when the
2732 	 * adapter is brough up for the first time.
2733 	 */
2734 	qidx = 0;
2735 	for_each_port(adapter, pidx) {
2736 		struct port_info *pi = adap2pinfo(adapter, pidx);
2737 
2738 		pi->first_qset = qidx;
2739 		pi->nqsets = is_x_10g_port(&pi->link_cfg) ? q10g : 1;
2740 		qidx += pi->nqsets;
2741 	}
2742 	s->ethqsets = qidx;
2743 
2744 	/*
2745 	 * The Ingress Queue Entry Size for our various Response Queues needs
2746 	 * to be big enough to accommodate the largest message we can receive
2747 	 * from the chip/firmware; which is 64 bytes ...
2748 	 */
2749 	iqe_size = 64;
2750 
2751 	/*
2752 	 * Set up default Queue Set parameters ...  Start off with the
2753 	 * shortest interrupt holdoff timer.
2754 	 */
2755 	for (qs = 0; qs < s->max_ethqsets; qs++) {
2756 		struct sge_eth_rxq *rxq = &s->ethrxq[qs];
2757 		struct sge_eth_txq *txq = &s->ethtxq[qs];
2758 
2759 		init_rspq(&rxq->rspq, 0, 0, 1024, iqe_size);
2760 		rxq->fl.size = 72;
2761 		txq->q.size = 1024;
2762 	}
2763 
2764 	/*
2765 	 * The firmware event queue is used for link state changes and
2766 	 * notifications of TX DMA completions.
2767 	 */
2768 	init_rspq(&s->fw_evtq, SGE_TIMER_RSTRT_CNTR, 0, 512, iqe_size);
2769 
2770 	/*
2771 	 * The forwarded interrupt queue is used when we're in MSI interrupt
2772 	 * mode.  In this mode all interrupts associated with RX queues will
2773 	 * be forwarded to a single queue which we'll associate with our MSI
2774 	 * interrupt vector.  The messages dropped in the forwarded interrupt
2775 	 * queue will indicate which ingress queue needs servicing ...  This
2776 	 * queue needs to be large enough to accommodate all of the ingress
2777 	 * queues which are forwarding their interrupt (+1 to prevent the PIDX
2778 	 * from equalling the CIDX if every ingress queue has an outstanding
2779 	 * interrupt).  The queue doesn't need to be any larger because no
2780 	 * ingress queue will ever have more than one outstanding interrupt at
2781 	 * any time ...
2782 	 */
2783 	init_rspq(&s->intrq, SGE_TIMER_RSTRT_CNTR, 0, MSIX_ENTRIES + 1,
2784 		  iqe_size);
2785 }
2786 
2787 /*
2788  * Reduce the number of Ethernet queues across all ports to at most n.
2789  * n provides at least one queue per port.
2790  */
2791 static void reduce_ethqs(struct adapter *adapter, int n)
2792 {
2793 	int i;
2794 	struct port_info *pi;
2795 
2796 	/*
2797 	 * While we have too many active Ether Queue Sets, interate across the
2798 	 * "ports" and reduce their individual Queue Set allocations.
2799 	 */
2800 	BUG_ON(n < adapter->params.nports);
2801 	while (n < adapter->sge.ethqsets)
2802 		for_each_port(adapter, i) {
2803 			pi = adap2pinfo(adapter, i);
2804 			if (pi->nqsets > 1) {
2805 				pi->nqsets--;
2806 				adapter->sge.ethqsets--;
2807 				if (adapter->sge.ethqsets <= n)
2808 					break;
2809 			}
2810 		}
2811 
2812 	/*
2813 	 * Reassign the starting Queue Sets for each of the "ports" ...
2814 	 */
2815 	n = 0;
2816 	for_each_port(adapter, i) {
2817 		pi = adap2pinfo(adapter, i);
2818 		pi->first_qset = n;
2819 		n += pi->nqsets;
2820 	}
2821 }
2822 
2823 /*
2824  * We need to grab enough MSI-X vectors to cover our interrupt needs.  Ideally
2825  * we get a separate MSI-X vector for every "Queue Set" plus any extras we
2826  * need.  Minimally we need one for every Virtual Interface plus those needed
2827  * for our "extras".  Note that this process may lower the maximum number of
2828  * allowed Queue Sets ...
2829  */
2830 static int enable_msix(struct adapter *adapter)
2831 {
2832 	int i, want, need, nqsets;
2833 	struct msix_entry entries[MSIX_ENTRIES];
2834 	struct sge *s = &adapter->sge;
2835 
2836 	for (i = 0; i < MSIX_ENTRIES; ++i)
2837 		entries[i].entry = i;
2838 
2839 	/*
2840 	 * We _want_ enough MSI-X interrupts to cover all of our "Queue Sets"
2841 	 * plus those needed for our "extras" (for example, the firmware
2842 	 * message queue).  We _need_ at least one "Queue Set" per Virtual
2843 	 * Interface plus those needed for our "extras".  So now we get to see
2844 	 * if the song is right ...
2845 	 */
2846 	want = s->max_ethqsets + MSIX_EXTRAS;
2847 	need = adapter->params.nports + MSIX_EXTRAS;
2848 
2849 	want = pci_enable_msix_range(adapter->pdev, entries, need, want);
2850 	if (want < 0)
2851 		return want;
2852 
2853 	nqsets = want - MSIX_EXTRAS;
2854 	if (nqsets < s->max_ethqsets) {
2855 		dev_warn(adapter->pdev_dev, "only enough MSI-X vectors"
2856 			 " for %d Queue Sets\n", nqsets);
2857 		s->max_ethqsets = nqsets;
2858 		if (nqsets < s->ethqsets)
2859 			reduce_ethqs(adapter, nqsets);
2860 	}
2861 	for (i = 0; i < want; ++i)
2862 		adapter->msix_info[i].vec = entries[i].vector;
2863 
2864 	return 0;
2865 }
2866 
2867 static const struct net_device_ops cxgb4vf_netdev_ops	= {
2868 	.ndo_open		= cxgb4vf_open,
2869 	.ndo_stop		= cxgb4vf_stop,
2870 	.ndo_start_xmit		= t4vf_eth_xmit,
2871 	.ndo_get_stats		= cxgb4vf_get_stats,
2872 	.ndo_set_rx_mode	= cxgb4vf_set_rxmode,
2873 	.ndo_set_mac_address	= cxgb4vf_set_mac_addr,
2874 	.ndo_validate_addr	= eth_validate_addr,
2875 	.ndo_do_ioctl		= cxgb4vf_do_ioctl,
2876 	.ndo_change_mtu		= cxgb4vf_change_mtu,
2877 	.ndo_fix_features	= cxgb4vf_fix_features,
2878 	.ndo_set_features	= cxgb4vf_set_features,
2879 #ifdef CONFIG_NET_POLL_CONTROLLER
2880 	.ndo_poll_controller	= cxgb4vf_poll_controller,
2881 #endif
2882 };
2883 
2884 /*
2885  * "Probe" a device: initialize a device and construct all kernel and driver
2886  * state needed to manage the device.  This routine is called "init_one" in
2887  * the PF Driver ...
2888  */
2889 static int cxgb4vf_pci_probe(struct pci_dev *pdev,
2890 			     const struct pci_device_id *ent)
2891 {
2892 	int pci_using_dac;
2893 	int err, pidx;
2894 	unsigned int pmask;
2895 	struct adapter *adapter;
2896 	struct port_info *pi;
2897 	struct net_device *netdev;
2898 	unsigned int pf;
2899 
2900 	/*
2901 	 * Print our driver banner the first time we're called to initialize a
2902 	 * device.
2903 	 */
2904 	pr_info_once("%s - version %s\n", DRV_DESC, DRV_VERSION);
2905 
2906 	/*
2907 	 * Initialize generic PCI device state.
2908 	 */
2909 	err = pci_enable_device(pdev);
2910 	if (err) {
2911 		dev_err(&pdev->dev, "cannot enable PCI device\n");
2912 		return err;
2913 	}
2914 
2915 	/*
2916 	 * Reserve PCI resources for the device.  If we can't get them some
2917 	 * other driver may have already claimed the device ...
2918 	 */
2919 	err = pci_request_regions(pdev, KBUILD_MODNAME);
2920 	if (err) {
2921 		dev_err(&pdev->dev, "cannot obtain PCI resources\n");
2922 		goto err_disable_device;
2923 	}
2924 
2925 	/*
2926 	 * Set up our DMA mask: try for 64-bit address masking first and
2927 	 * fall back to 32-bit if we can't get 64 bits ...
2928 	 */
2929 	err = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2930 	if (err == 0) {
2931 		err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2932 		if (err) {
2933 			dev_err(&pdev->dev, "unable to obtain 64-bit DMA for"
2934 				" coherent allocations\n");
2935 			goto err_release_regions;
2936 		}
2937 		pci_using_dac = 1;
2938 	} else {
2939 		err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2940 		if (err != 0) {
2941 			dev_err(&pdev->dev, "no usable DMA configuration\n");
2942 			goto err_release_regions;
2943 		}
2944 		pci_using_dac = 0;
2945 	}
2946 
2947 	/*
2948 	 * Enable bus mastering for the device ...
2949 	 */
2950 	pci_set_master(pdev);
2951 
2952 	/*
2953 	 * Allocate our adapter data structure and attach it to the device.
2954 	 */
2955 	adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
2956 	if (!adapter) {
2957 		err = -ENOMEM;
2958 		goto err_release_regions;
2959 	}
2960 	pci_set_drvdata(pdev, adapter);
2961 	adapter->pdev = pdev;
2962 	adapter->pdev_dev = &pdev->dev;
2963 
2964 	adapter->mbox_log = kzalloc(sizeof(*adapter->mbox_log) +
2965 				    (sizeof(struct mbox_cmd) *
2966 				     T4VF_OS_LOG_MBOX_CMDS),
2967 				    GFP_KERNEL);
2968 	if (!adapter->mbox_log) {
2969 		err = -ENOMEM;
2970 		goto err_free_adapter;
2971 	}
2972 	adapter->mbox_log->size = T4VF_OS_LOG_MBOX_CMDS;
2973 
2974 	/*
2975 	 * Initialize SMP data synchronization resources.
2976 	 */
2977 	spin_lock_init(&adapter->stats_lock);
2978 	spin_lock_init(&adapter->mbox_lock);
2979 	INIT_LIST_HEAD(&adapter->mlist.list);
2980 
2981 	/*
2982 	 * Map our I/O registers in BAR0.
2983 	 */
2984 	adapter->regs = pci_ioremap_bar(pdev, 0);
2985 	if (!adapter->regs) {
2986 		dev_err(&pdev->dev, "cannot map device registers\n");
2987 		err = -ENOMEM;
2988 		goto err_free_adapter;
2989 	}
2990 
2991 	/* Wait for the device to become ready before proceeding ...
2992 	 */
2993 	err = t4vf_prep_adapter(adapter);
2994 	if (err) {
2995 		dev_err(adapter->pdev_dev, "device didn't become ready:"
2996 			" err=%d\n", err);
2997 		goto err_unmap_bar0;
2998 	}
2999 
3000 	/* For T5 and later we want to use the new BAR-based User Doorbells,
3001 	 * so we need to map BAR2 here ...
3002 	 */
3003 	if (!is_t4(adapter->params.chip)) {
3004 		adapter->bar2 = ioremap_wc(pci_resource_start(pdev, 2),
3005 					   pci_resource_len(pdev, 2));
3006 		if (!adapter->bar2) {
3007 			dev_err(adapter->pdev_dev, "cannot map BAR2 doorbells\n");
3008 			err = -ENOMEM;
3009 			goto err_unmap_bar0;
3010 		}
3011 	}
3012 	/*
3013 	 * Initialize adapter level features.
3014 	 */
3015 	adapter->name = pci_name(pdev);
3016 	adapter->msg_enable = DFLT_MSG_ENABLE;
3017 
3018 	/* If possible, we use PCIe Relaxed Ordering Attribute to deliver
3019 	 * Ingress Packet Data to Free List Buffers in order to allow for
3020 	 * chipset performance optimizations between the Root Complex and
3021 	 * Memory Controllers.  (Messages to the associated Ingress Queue
3022 	 * notifying new Packet Placement in the Free Lists Buffers will be
3023 	 * send without the Relaxed Ordering Attribute thus guaranteeing that
3024 	 * all preceding PCIe Transaction Layer Packets will be processed
3025 	 * first.)  But some Root Complexes have various issues with Upstream
3026 	 * Transaction Layer Packets with the Relaxed Ordering Attribute set.
3027 	 * The PCIe devices which under the Root Complexes will be cleared the
3028 	 * Relaxed Ordering bit in the configuration space, So we check our
3029 	 * PCIe configuration space to see if it's flagged with advice against
3030 	 * using Relaxed Ordering.
3031 	 */
3032 	if (!pcie_relaxed_ordering_enabled(pdev))
3033 		adapter->flags |= ROOT_NO_RELAXED_ORDERING;
3034 
3035 	err = adap_init0(adapter);
3036 	if (err)
3037 		goto err_unmap_bar;
3038 
3039 	/*
3040 	 * Allocate our "adapter ports" and stitch everything together.
3041 	 */
3042 	pmask = adapter->params.vfres.pmask;
3043 	pf = t4vf_get_pf_from_vf(adapter);
3044 	for_each_port(adapter, pidx) {
3045 		int port_id, viid;
3046 		u8 mac[ETH_ALEN];
3047 		unsigned int naddr = 1;
3048 
3049 		/*
3050 		 * We simplistically allocate our virtual interfaces
3051 		 * sequentially across the port numbers to which we have
3052 		 * access rights.  This should be configurable in some manner
3053 		 * ...
3054 		 */
3055 		if (pmask == 0)
3056 			break;
3057 		port_id = ffs(pmask) - 1;
3058 		pmask &= ~(1 << port_id);
3059 		viid = t4vf_alloc_vi(adapter, port_id);
3060 		if (viid < 0) {
3061 			dev_err(&pdev->dev, "cannot allocate VI for port %d:"
3062 				" err=%d\n", port_id, viid);
3063 			err = viid;
3064 			goto err_free_dev;
3065 		}
3066 
3067 		/*
3068 		 * Allocate our network device and stitch things together.
3069 		 */
3070 		netdev = alloc_etherdev_mq(sizeof(struct port_info),
3071 					   MAX_PORT_QSETS);
3072 		if (netdev == NULL) {
3073 			t4vf_free_vi(adapter, viid);
3074 			err = -ENOMEM;
3075 			goto err_free_dev;
3076 		}
3077 		adapter->port[pidx] = netdev;
3078 		SET_NETDEV_DEV(netdev, &pdev->dev);
3079 		pi = netdev_priv(netdev);
3080 		pi->adapter = adapter;
3081 		pi->pidx = pidx;
3082 		pi->port_id = port_id;
3083 		pi->viid = viid;
3084 
3085 		/*
3086 		 * Initialize the starting state of our "port" and register
3087 		 * it.
3088 		 */
3089 		pi->xact_addr_filt = -1;
3090 		netif_carrier_off(netdev);
3091 		netdev->irq = pdev->irq;
3092 
3093 		netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
3094 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3095 			NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_RXCSUM;
3096 		netdev->vlan_features = NETIF_F_SG | TSO_FLAGS |
3097 			NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3098 			NETIF_F_HIGHDMA;
3099 		netdev->features = netdev->hw_features |
3100 				   NETIF_F_HW_VLAN_CTAG_TX;
3101 		if (pci_using_dac)
3102 			netdev->features |= NETIF_F_HIGHDMA;
3103 
3104 		netdev->priv_flags |= IFF_UNICAST_FLT;
3105 		netdev->min_mtu = 81;
3106 		netdev->max_mtu = ETH_MAX_MTU;
3107 
3108 		netdev->netdev_ops = &cxgb4vf_netdev_ops;
3109 		netdev->ethtool_ops = &cxgb4vf_ethtool_ops;
3110 		netdev->dev_port = pi->port_id;
3111 
3112 		/*
3113 		 * Initialize the hardware/software state for the port.
3114 		 */
3115 		err = t4vf_port_init(adapter, pidx);
3116 		if (err) {
3117 			dev_err(&pdev->dev, "cannot initialize port %d\n",
3118 				pidx);
3119 			goto err_free_dev;
3120 		}
3121 
3122 		err = t4vf_get_vf_mac_acl(adapter, pf, &naddr, mac);
3123 		if (err) {
3124 			dev_err(&pdev->dev,
3125 				"unable to determine MAC ACL address, "
3126 				"continuing anyway.. (status %d)\n", err);
3127 		} else if (naddr && adapter->params.vfres.nvi == 1) {
3128 			struct sockaddr addr;
3129 
3130 			ether_addr_copy(addr.sa_data, mac);
3131 			err = cxgb4vf_set_mac_addr(netdev, &addr);
3132 			if (err) {
3133 				dev_err(&pdev->dev,
3134 					"unable to set MAC address %pM\n",
3135 					mac);
3136 				goto err_free_dev;
3137 			}
3138 			dev_info(&pdev->dev,
3139 				 "Using assigned MAC ACL: %pM\n", mac);
3140 		}
3141 	}
3142 
3143 	/* See what interrupts we'll be using.  If we've been configured to
3144 	 * use MSI-X interrupts, try to enable them but fall back to using
3145 	 * MSI interrupts if we can't enable MSI-X interrupts.  If we can't
3146 	 * get MSI interrupts we bail with the error.
3147 	 */
3148 	if (msi == MSI_MSIX && enable_msix(adapter) == 0)
3149 		adapter->flags |= USING_MSIX;
3150 	else {
3151 		if (msi == MSI_MSIX) {
3152 			dev_info(adapter->pdev_dev,
3153 				 "Unable to use MSI-X Interrupts; falling "
3154 				 "back to MSI Interrupts\n");
3155 
3156 			/* We're going to need a Forwarded Interrupt Queue so
3157 			 * that may cut into how many Queue Sets we can
3158 			 * support.
3159 			 */
3160 			msi = MSI_MSI;
3161 			size_nports_qsets(adapter);
3162 		}
3163 		err = pci_enable_msi(pdev);
3164 		if (err) {
3165 			dev_err(&pdev->dev, "Unable to allocate MSI Interrupts;"
3166 				" err=%d\n", err);
3167 			goto err_free_dev;
3168 		}
3169 		adapter->flags |= USING_MSI;
3170 	}
3171 
3172 	/* Now that we know how many "ports" we have and what interrupt
3173 	 * mechanism we're going to use, we can configure our queue resources.
3174 	 */
3175 	cfg_queues(adapter);
3176 
3177 	/*
3178 	 * The "card" is now ready to go.  If any errors occur during device
3179 	 * registration we do not fail the whole "card" but rather proceed
3180 	 * only with the ports we manage to register successfully.  However we
3181 	 * must register at least one net device.
3182 	 */
3183 	for_each_port(adapter, pidx) {
3184 		struct port_info *pi = netdev_priv(adapter->port[pidx]);
3185 		netdev = adapter->port[pidx];
3186 		if (netdev == NULL)
3187 			continue;
3188 
3189 		netif_set_real_num_tx_queues(netdev, pi->nqsets);
3190 		netif_set_real_num_rx_queues(netdev, pi->nqsets);
3191 
3192 		err = register_netdev(netdev);
3193 		if (err) {
3194 			dev_warn(&pdev->dev, "cannot register net device %s,"
3195 				 " skipping\n", netdev->name);
3196 			continue;
3197 		}
3198 
3199 		set_bit(pidx, &adapter->registered_device_map);
3200 	}
3201 	if (adapter->registered_device_map == 0) {
3202 		dev_err(&pdev->dev, "could not register any net devices\n");
3203 		goto err_disable_interrupts;
3204 	}
3205 
3206 	/*
3207 	 * Set up our debugfs entries.
3208 	 */
3209 	if (!IS_ERR_OR_NULL(cxgb4vf_debugfs_root)) {
3210 		adapter->debugfs_root =
3211 			debugfs_create_dir(pci_name(pdev),
3212 					   cxgb4vf_debugfs_root);
3213 		if (IS_ERR_OR_NULL(adapter->debugfs_root))
3214 			dev_warn(&pdev->dev, "could not create debugfs"
3215 				 " directory");
3216 		else
3217 			setup_debugfs(adapter);
3218 	}
3219 
3220 	/*
3221 	 * Print a short notice on the existence and configuration of the new
3222 	 * VF network device ...
3223 	 */
3224 	for_each_port(adapter, pidx) {
3225 		dev_info(adapter->pdev_dev, "%s: Chelsio VF NIC PCIe %s\n",
3226 			 adapter->port[pidx]->name,
3227 			 (adapter->flags & USING_MSIX) ? "MSI-X" :
3228 			 (adapter->flags & USING_MSI)  ? "MSI" : "");
3229 	}
3230 
3231 	/*
3232 	 * Return success!
3233 	 */
3234 	return 0;
3235 
3236 	/*
3237 	 * Error recovery and exit code.  Unwind state that's been created
3238 	 * so far and return the error.
3239 	 */
3240 err_disable_interrupts:
3241 	if (adapter->flags & USING_MSIX) {
3242 		pci_disable_msix(adapter->pdev);
3243 		adapter->flags &= ~USING_MSIX;
3244 	} else if (adapter->flags & USING_MSI) {
3245 		pci_disable_msi(adapter->pdev);
3246 		adapter->flags &= ~USING_MSI;
3247 	}
3248 
3249 err_free_dev:
3250 	for_each_port(adapter, pidx) {
3251 		netdev = adapter->port[pidx];
3252 		if (netdev == NULL)
3253 			continue;
3254 		pi = netdev_priv(netdev);
3255 		t4vf_free_vi(adapter, pi->viid);
3256 		if (test_bit(pidx, &adapter->registered_device_map))
3257 			unregister_netdev(netdev);
3258 		free_netdev(netdev);
3259 	}
3260 
3261 err_unmap_bar:
3262 	if (!is_t4(adapter->params.chip))
3263 		iounmap(adapter->bar2);
3264 
3265 err_unmap_bar0:
3266 	iounmap(adapter->regs);
3267 
3268 err_free_adapter:
3269 	kfree(adapter->mbox_log);
3270 	kfree(adapter);
3271 
3272 err_release_regions:
3273 	pci_release_regions(pdev);
3274 	pci_clear_master(pdev);
3275 
3276 err_disable_device:
3277 	pci_disable_device(pdev);
3278 
3279 	return err;
3280 }
3281 
3282 /*
3283  * "Remove" a device: tear down all kernel and driver state created in the
3284  * "probe" routine and quiesce the device (disable interrupts, etc.).  (Note
3285  * that this is called "remove_one" in the PF Driver.)
3286  */
3287 static void cxgb4vf_pci_remove(struct pci_dev *pdev)
3288 {
3289 	struct adapter *adapter = pci_get_drvdata(pdev);
3290 
3291 	/*
3292 	 * Tear down driver state associated with device.
3293 	 */
3294 	if (adapter) {
3295 		int pidx;
3296 
3297 		/*
3298 		 * Stop all of our activity.  Unregister network port,
3299 		 * disable interrupts, etc.
3300 		 */
3301 		for_each_port(adapter, pidx)
3302 			if (test_bit(pidx, &adapter->registered_device_map))
3303 				unregister_netdev(adapter->port[pidx]);
3304 		t4vf_sge_stop(adapter);
3305 		if (adapter->flags & USING_MSIX) {
3306 			pci_disable_msix(adapter->pdev);
3307 			adapter->flags &= ~USING_MSIX;
3308 		} else if (adapter->flags & USING_MSI) {
3309 			pci_disable_msi(adapter->pdev);
3310 			adapter->flags &= ~USING_MSI;
3311 		}
3312 
3313 		/*
3314 		 * Tear down our debugfs entries.
3315 		 */
3316 		if (!IS_ERR_OR_NULL(adapter->debugfs_root)) {
3317 			cleanup_debugfs(adapter);
3318 			debugfs_remove_recursive(adapter->debugfs_root);
3319 		}
3320 
3321 		/*
3322 		 * Free all of the various resources which we've acquired ...
3323 		 */
3324 		t4vf_free_sge_resources(adapter);
3325 		for_each_port(adapter, pidx) {
3326 			struct net_device *netdev = adapter->port[pidx];
3327 			struct port_info *pi;
3328 
3329 			if (netdev == NULL)
3330 				continue;
3331 
3332 			pi = netdev_priv(netdev);
3333 			t4vf_free_vi(adapter, pi->viid);
3334 			free_netdev(netdev);
3335 		}
3336 		iounmap(adapter->regs);
3337 		if (!is_t4(adapter->params.chip))
3338 			iounmap(adapter->bar2);
3339 		kfree(adapter->mbox_log);
3340 		kfree(adapter);
3341 	}
3342 
3343 	/*
3344 	 * Disable the device and release its PCI resources.
3345 	 */
3346 	pci_disable_device(pdev);
3347 	pci_clear_master(pdev);
3348 	pci_release_regions(pdev);
3349 }
3350 
3351 /*
3352  * "Shutdown" quiesce the device, stopping Ingress Packet and Interrupt
3353  * delivery.
3354  */
3355 static void cxgb4vf_pci_shutdown(struct pci_dev *pdev)
3356 {
3357 	struct adapter *adapter;
3358 	int pidx;
3359 
3360 	adapter = pci_get_drvdata(pdev);
3361 	if (!adapter)
3362 		return;
3363 
3364 	/* Disable all Virtual Interfaces.  This will shut down the
3365 	 * delivery of all ingress packets into the chip for these
3366 	 * Virtual Interfaces.
3367 	 */
3368 	for_each_port(adapter, pidx)
3369 		if (test_bit(pidx, &adapter->registered_device_map))
3370 			unregister_netdev(adapter->port[pidx]);
3371 
3372 	/* Free up all Queues which will prevent further DMA and
3373 	 * Interrupts allowing various internal pathways to drain.
3374 	 */
3375 	t4vf_sge_stop(adapter);
3376 	if (adapter->flags & USING_MSIX) {
3377 		pci_disable_msix(adapter->pdev);
3378 		adapter->flags &= ~USING_MSIX;
3379 	} else if (adapter->flags & USING_MSI) {
3380 		pci_disable_msi(adapter->pdev);
3381 		adapter->flags &= ~USING_MSI;
3382 	}
3383 
3384 	/*
3385 	 * Free up all Queues which will prevent further DMA and
3386 	 * Interrupts allowing various internal pathways to drain.
3387 	 */
3388 	t4vf_free_sge_resources(adapter);
3389 	pci_set_drvdata(pdev, NULL);
3390 }
3391 
3392 /* Macros needed to support the PCI Device ID Table ...
3393  */
3394 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_BEGIN \
3395 	static const struct pci_device_id cxgb4vf_pci_tbl[] = {
3396 #define CH_PCI_DEVICE_ID_FUNCTION	0x8
3397 
3398 #define CH_PCI_ID_TABLE_ENTRY(devid) \
3399 		{ PCI_VDEVICE(CHELSIO, (devid)), 0 }
3400 
3401 #define CH_PCI_DEVICE_ID_TABLE_DEFINE_END { 0, } }
3402 
3403 #include "../cxgb4/t4_pci_id_tbl.h"
3404 
3405 MODULE_DESCRIPTION(DRV_DESC);
3406 MODULE_AUTHOR("Chelsio Communications");
3407 MODULE_LICENSE("Dual BSD/GPL");
3408 MODULE_VERSION(DRV_VERSION);
3409 MODULE_DEVICE_TABLE(pci, cxgb4vf_pci_tbl);
3410 
3411 static struct pci_driver cxgb4vf_driver = {
3412 	.name		= KBUILD_MODNAME,
3413 	.id_table	= cxgb4vf_pci_tbl,
3414 	.probe		= cxgb4vf_pci_probe,
3415 	.remove		= cxgb4vf_pci_remove,
3416 	.shutdown	= cxgb4vf_pci_shutdown,
3417 };
3418 
3419 /*
3420  * Initialize global driver state.
3421  */
3422 static int __init cxgb4vf_module_init(void)
3423 {
3424 	int ret;
3425 
3426 	/*
3427 	 * Vet our module parameters.
3428 	 */
3429 	if (msi != MSI_MSIX && msi != MSI_MSI) {
3430 		pr_warn("bad module parameter msi=%d; must be %d (MSI-X or MSI) or %d (MSI)\n",
3431 			msi, MSI_MSIX, MSI_MSI);
3432 		return -EINVAL;
3433 	}
3434 
3435 	/* Debugfs support is optional, just warn if this fails */
3436 	cxgb4vf_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3437 	if (IS_ERR_OR_NULL(cxgb4vf_debugfs_root))
3438 		pr_warn("could not create debugfs entry, continuing\n");
3439 
3440 	ret = pci_register_driver(&cxgb4vf_driver);
3441 	if (ret < 0 && !IS_ERR_OR_NULL(cxgb4vf_debugfs_root))
3442 		debugfs_remove(cxgb4vf_debugfs_root);
3443 	return ret;
3444 }
3445 
3446 /*
3447  * Tear down global driver state.
3448  */
3449 static void __exit cxgb4vf_module_exit(void)
3450 {
3451 	pci_unregister_driver(&cxgb4vf_driver);
3452 	debugfs_remove(cxgb4vf_debugfs_root);
3453 }
3454 
3455 module_init(cxgb4vf_module_init);
3456 module_exit(cxgb4vf_module_exit);
3457