xref: /freebsd/sys/dev/cxgbe/t4_vf.c (revision 13ea0450a9c8742119d36f3bf8f47accdce46e54)
1 /*-
2  * Copyright (c) 2016 Chelsio Communications, Inc.
3  * All rights reserved.
4  * Written by: John Baldwin <jhb@FreeBSD.org>
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. Redistributions in binary form must reproduce the above copyright
12  *    notice, this list of conditions and the following disclaimer in the
13  *    documentation and/or other materials provided with the distribution.
14  *
15  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25  * SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __FBSDID("$FreeBSD$");
30 
31 #include "opt_inet.h"
32 #include "opt_inet6.h"
33 
34 #include <sys/param.h>
35 #include <sys/bus.h>
36 #include <sys/conf.h>
37 #include <sys/kernel.h>
38 #include <sys/module.h>
39 #include <sys/priv.h>
40 #include <dev/pci/pcivar.h>
41 #if defined(__i386__) || defined(__amd64__)
42 #include <vm/vm.h>
43 #include <vm/pmap.h>
44 #endif
45 
46 #include "common/common.h"
47 #include "common/t4_regs.h"
48 #include "t4_ioctl.h"
49 #include "t4_mp_ring.h"
50 
51 /*
52  * Some notes:
53  *
54  * The Virtual Interfaces are connected to an internal switch on the chip
55  * which allows VIs attached to the same port to talk to each other even when
56  * the port link is down.  As a result, we might want to always report a
57  * VF's link as being "up".
58  *
59  * XXX: Add a TUNABLE and possible per-device sysctl for this?
60  */
61 
62 struct intrs_and_queues {
63 	uint16_t intr_type;	/* MSI, or MSI-X */
64 	uint16_t nirq;		/* Total # of vectors */
65 	uint16_t ntxq;		/* # of NIC txq's for each port */
66 	uint16_t nrxq;		/* # of NIC rxq's for each port */
67 };
68 
69 struct {
70 	uint16_t device;
71 	char *desc;
72 } t4vf_pciids[] = {
73 	{0x4800, "Chelsio T440-dbg VF"},
74 	{0x4801, "Chelsio T420-CR VF"},
75 	{0x4802, "Chelsio T422-CR VF"},
76 	{0x4803, "Chelsio T440-CR VF"},
77 	{0x4804, "Chelsio T420-BCH VF"},
78 	{0x4805, "Chelsio T440-BCH VF"},
79 	{0x4806, "Chelsio T440-CH VF"},
80 	{0x4807, "Chelsio T420-SO VF"},
81 	{0x4808, "Chelsio T420-CX VF"},
82 	{0x4809, "Chelsio T420-BT VF"},
83 	{0x480a, "Chelsio T404-BT VF"},
84 	{0x480e, "Chelsio T440-LP-CR VF"},
85 }, t5vf_pciids[] = {
86 	{0x5800, "Chelsio T580-dbg VF"},
87 	{0x5801,  "Chelsio T520-CR VF"},	/* 2 x 10G */
88 	{0x5802,  "Chelsio T522-CR VF"},	/* 2 x 10G, 2 X 1G */
89 	{0x5803,  "Chelsio T540-CR VF"},	/* 4 x 10G */
90 	{0x5807,  "Chelsio T520-SO VF"},	/* 2 x 10G, nomem */
91 	{0x5809,  "Chelsio T520-BT VF"},	/* 2 x 10GBaseT */
92 	{0x580a,  "Chelsio T504-BT VF"},	/* 4 x 1G */
93 	{0x580d,  "Chelsio T580-CR VF"},	/* 2 x 40G */
94 	{0x580e,  "Chelsio T540-LP-CR VF"},	/* 4 x 10G */
95 	{0x5810,  "Chelsio T580-LP-CR VF"},	/* 2 x 40G */
96 	{0x5811,  "Chelsio T520-LL-CR VF"},	/* 2 x 10G */
97 	{0x5812,  "Chelsio T560-CR VF"},	/* 1 x 40G, 2 x 10G */
98 	{0x5814,  "Chelsio T580-LP-SO-CR VF"},	/* 2 x 40G, nomem */
99 	{0x5815,  "Chelsio T502-BT VF"},	/* 2 x 1G */
100 #ifdef notyet
101 	{0x5804,  "Chelsio T520-BCH VF"},
102 	{0x5805,  "Chelsio T540-BCH VF"},
103 	{0x5806,  "Chelsio T540-CH VF"},
104 	{0x5808,  "Chelsio T520-CX VF"},
105 	{0x580b,  "Chelsio B520-SR VF"},
106 	{0x580c,  "Chelsio B504-BT VF"},
107 	{0x580f,  "Chelsio Amsterdam VF"},
108 	{0x5813,  "Chelsio T580-CHR VF"},
109 #endif
110 }, t6vf_pciids[] = {
111 	{0x6800, "Chelsio T6-DBG-25 VF"},	/* 2 x 10/25G, debug */
112 	{0x6801, "Chelsio T6225-CR VF"},	/* 2 x 10/25G */
113 	{0x6802, "Chelsio T6225-SO-CR VF"},	/* 2 x 10/25G, nomem */
114 	{0x6803, "Chelsio T6425-CR VF"},	/* 4 x 10/25G */
115 	{0x6804, "Chelsio T6425-SO-CR VF"},	/* 4 x 10/25G, nomem */
116 	{0x6805, "Chelsio T6225-OCP-SO VF"},	/* 2 x 10/25G, nomem */
117 	{0x6806, "Chelsio T62100-OCP-SO VF"},	/* 2 x 40/50/100G, nomem */
118 	{0x6807, "Chelsio T62100-LP-CR VF"},	/* 2 x 40/50/100G */
119 	{0x6808, "Chelsio T62100-SO-CR VF"},	/* 2 x 40/50/100G, nomem */
120 	{0x6809, "Chelsio T6210-BT VF"},	/* 2 x 10GBASE-T */
121 	{0x680d, "Chelsio T62100-CR VF"},	/* 2 x 40/50/100G */
122 	{0x6810, "Chelsio T6-DBG-100 VF"},	/* 2 x 40/50/100G, debug */
123 	{0x6811, "Chelsio T6225-LL-CR VF"},	/* 2 x 10/25G */
124 	{0x6814, "Chelsio T61100-OCP-SO VF"},	/* 1 x 40/50/100G, nomem */
125 	{0x6815, "Chelsio T6201-BT VF"},	/* 2 x 1000BASE-T */
126 
127 	/* Custom */
128 	{0x6880, "Chelsio T6225 80 VF"},
129 	{0x6881, "Chelsio T62100 81 VF"},
130 };
131 
132 static d_ioctl_t t4vf_ioctl;
133 
134 static struct cdevsw t4vf_cdevsw = {
135        .d_version = D_VERSION,
136        .d_ioctl = t4vf_ioctl,
137        .d_name = "t4vf",
138 };
139 
140 static int
141 t4vf_probe(device_t dev)
142 {
143 	uint16_t d;
144 	size_t i;
145 
146 	d = pci_get_device(dev);
147 	for (i = 0; i < nitems(t4vf_pciids); i++) {
148 		if (d == t4vf_pciids[i].device) {
149 			device_set_desc(dev, t4vf_pciids[i].desc);
150 			return (BUS_PROBE_DEFAULT);
151 		}
152 	}
153 	return (ENXIO);
154 }
155 
156 static int
157 t5vf_probe(device_t dev)
158 {
159 	uint16_t d;
160 	size_t i;
161 
162 	d = pci_get_device(dev);
163 	for (i = 0; i < nitems(t5vf_pciids); i++) {
164 		if (d == t5vf_pciids[i].device) {
165 			device_set_desc(dev, t5vf_pciids[i].desc);
166 			return (BUS_PROBE_DEFAULT);
167 		}
168 	}
169 	return (ENXIO);
170 }
171 
172 static int
173 t6vf_probe(device_t dev)
174 {
175 	uint16_t d;
176 	size_t i;
177 
178 	d = pci_get_device(dev);
179 	for (i = 0; i < nitems(t6vf_pciids); i++) {
180 		if (d == t6vf_pciids[i].device) {
181 			device_set_desc(dev, t6vf_pciids[i].desc);
182 			return (BUS_PROBE_DEFAULT);
183 		}
184 	}
185 	return (ENXIO);
186 }
187 
188 #define FW_PARAM_DEV(param) \
189 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
190 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
191 #define FW_PARAM_PFVF(param) \
192 	(V_FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
193 	 V_FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param))
194 
195 static int
196 get_params__pre_init(struct adapter *sc)
197 {
198 	int rc;
199 	uint32_t param[3], val[3];
200 
201 	param[0] = FW_PARAM_DEV(FWREV);
202 	param[1] = FW_PARAM_DEV(TPREV);
203 	param[2] = FW_PARAM_DEV(CCLK);
204 	rc = -t4vf_query_params(sc, nitems(param), param, val);
205 	if (rc != 0) {
206 		device_printf(sc->dev,
207 		    "failed to query parameters (pre_init): %d.\n", rc);
208 		return (rc);
209 	}
210 
211 	sc->params.fw_vers = val[0];
212 	sc->params.tp_vers = val[1];
213 	sc->params.vpd.cclk = val[2];
214 
215 	snprintf(sc->fw_version, sizeof(sc->fw_version), "%u.%u.%u.%u",
216 	    G_FW_HDR_FW_VER_MAJOR(sc->params.fw_vers),
217 	    G_FW_HDR_FW_VER_MINOR(sc->params.fw_vers),
218 	    G_FW_HDR_FW_VER_MICRO(sc->params.fw_vers),
219 	    G_FW_HDR_FW_VER_BUILD(sc->params.fw_vers));
220 
221 	snprintf(sc->tp_version, sizeof(sc->tp_version), "%u.%u.%u.%u",
222 	    G_FW_HDR_FW_VER_MAJOR(sc->params.tp_vers),
223 	    G_FW_HDR_FW_VER_MINOR(sc->params.tp_vers),
224 	    G_FW_HDR_FW_VER_MICRO(sc->params.tp_vers),
225 	    G_FW_HDR_FW_VER_BUILD(sc->params.tp_vers));
226 
227 	return (0);
228 }
229 
230 static int
231 get_params__post_init(struct adapter *sc)
232 {
233 	int rc;
234 
235 	rc = -t4vf_get_sge_params(sc);
236 	if (rc != 0) {
237 		device_printf(sc->dev,
238 		    "unable to retrieve adapter SGE parameters: %d\n", rc);
239 		return (rc);
240 	}
241 
242 	rc = -t4vf_get_rss_glb_config(sc);
243 	if (rc != 0) {
244 		device_printf(sc->dev,
245 		    "unable to retrieve adapter RSS parameters: %d\n", rc);
246 		return (rc);
247 	}
248 	if (sc->params.rss.mode != FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL) {
249 		device_printf(sc->dev,
250 		    "unable to operate with global RSS mode %d\n",
251 		    sc->params.rss.mode);
252 		return (EINVAL);
253 	}
254 
255 	rc = t4_read_chip_settings(sc);
256 	if (rc != 0)
257 		return (rc);
258 
259 	/*
260 	 * Grab our Virtual Interface resource allocation, extract the
261 	 * features that we're interested in and do a bit of sanity testing on
262 	 * what we discover.
263 	 */
264 	rc = -t4vf_get_vfres(sc);
265 	if (rc != 0) {
266 		device_printf(sc->dev,
267 		    "unable to get virtual interface resources: %d\n", rc);
268 		return (rc);
269 	}
270 
271 	/*
272 	 * Check for various parameter sanity issues.
273 	 */
274 	if (sc->params.vfres.pmask == 0) {
275 		device_printf(sc->dev, "no port access configured/usable!\n");
276 		return (EINVAL);
277 	}
278 	if (sc->params.vfres.nvi == 0) {
279 		device_printf(sc->dev,
280 		    "no virtual interfaces configured/usable!\n");
281 		return (EINVAL);
282 	}
283 	sc->params.portvec = sc->params.vfres.pmask;
284 
285 	return (0);
286 }
287 
288 static int
289 set_params__post_init(struct adapter *sc)
290 {
291 	uint32_t param, val;
292 
293 	/* ask for encapsulated CPLs */
294 	param = FW_PARAM_PFVF(CPLFW4MSG_ENCAP);
295 	val = 1;
296 	(void)t4vf_set_params(sc, 1, &param, &val);
297 
298 	/* Enable 32b port caps if the firmware supports it. */
299 	param = FW_PARAM_PFVF(PORT_CAPS32);
300 	val = 1;
301 	if (t4vf_set_params(sc, 1, &param, &val) == 0)
302 		sc->params.port_caps32 = 1;
303 
304 	return (0);
305 }
306 
307 #undef FW_PARAM_PFVF
308 #undef FW_PARAM_DEV
309 
310 static int
311 cfg_itype_and_nqueues(struct adapter *sc, struct intrs_and_queues *iaq)
312 {
313 	struct vf_resources *vfres;
314 	int nrxq, ntxq, nports;
315 	int itype, iq_avail, navail, rc;
316 
317 	/*
318 	 * Figure out the layout of queues across our VIs and ensure
319 	 * we can allocate enough interrupts for our layout.
320 	 */
321 	vfres = &sc->params.vfres;
322 	nports = sc->params.nports;
323 	bzero(iaq, sizeof(*iaq));
324 
325 	for (itype = INTR_MSIX; itype != 0; itype >>= 1) {
326 		if (itype == INTR_INTX)
327 			continue;
328 
329 		if (itype == INTR_MSIX)
330 			navail = pci_msix_count(sc->dev);
331 		else
332 			navail = pci_msi_count(sc->dev);
333 
334 		if (navail == 0)
335 			continue;
336 
337 		iaq->intr_type = itype;
338 
339 		/*
340 		 * XXX: The Linux driver reserves an Ingress Queue for
341 		 * forwarded interrupts when using MSI (but not MSI-X).
342 		 * It seems it just always asks for 2 interrupts and
343 		 * forwards all rxqs to the forwarded interrupt.
344 		 *
345 		 * We must reserve one IRQ for the for the firmware
346 		 * event queue.
347 		 *
348 		 * Every rxq requires an ingress queue with a free
349 		 * list and interrupts and an egress queue.  Every txq
350 		 * requires an ETH egress queue.
351 		 */
352 		iaq->nirq = T4VF_EXTRA_INTR;
353 
354 		/*
355 		 * First, determine how many queues we can allocate.
356 		 * Start by finding the upper bound on rxqs from the
357 		 * limit on ingress queues.
358 		 */
359 		iq_avail = vfres->niqflint - iaq->nirq;
360 		if (iq_avail < nports) {
361 			device_printf(sc->dev,
362 			    "Not enough ingress queues (%d) for %d ports\n",
363 			    vfres->niqflint, nports);
364 			return (ENXIO);
365 		}
366 
367 		/*
368 		 * Try to honor the cap on interrupts.  If there aren't
369 		 * enough interrupts for at least one interrupt per
370 		 * port, then don't bother, we will just forward all
371 		 * interrupts to one interrupt in that case.
372 		 */
373 		if (iaq->nirq + nports <= navail) {
374 			if (iq_avail > navail - iaq->nirq)
375 				iq_avail = navail - iaq->nirq;
376 		}
377 
378 		nrxq = nports * t4_nrxq;
379 		if (nrxq > iq_avail) {
380 			/*
381 			 * Too many ingress queues.  Use what we can.
382 			 */
383 			nrxq = (iq_avail / nports) * nports;
384 		}
385 		KASSERT(nrxq <= iq_avail, ("too many ingress queues"));
386 
387 		/*
388 		 * Next, determine the upper bound on txqs from the limit
389 		 * on ETH queues.
390 		 */
391 		if (vfres->nethctrl < nports) {
392 			device_printf(sc->dev,
393 			    "Not enough ETH queues (%d) for %d ports\n",
394 			    vfres->nethctrl, nports);
395 			return (ENXIO);
396 		}
397 
398 		ntxq = nports * t4_ntxq;
399 		if (ntxq > vfres->nethctrl) {
400 			/*
401 			 * Too many ETH queues.  Use what we can.
402 			 */
403 			ntxq = (vfres->nethctrl / nports) * nports;
404 		}
405 		KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues"));
406 
407 		/*
408 		 * Finally, ensure we have enough egress queues.
409 		 */
410 		if (vfres->neq < nports * 2) {
411 			device_printf(sc->dev,
412 			    "Not enough egress queues (%d) for %d ports\n",
413 			    vfres->neq, nports);
414 			return (ENXIO);
415 		}
416 		if (nrxq + ntxq > vfres->neq) {
417 			/* Just punt and use 1 for everything. */
418 			nrxq = ntxq = nports;
419 		}
420 		KASSERT(nrxq <= iq_avail, ("too many ingress queues"));
421 		KASSERT(ntxq <= vfres->nethctrl, ("too many ETH queues"));
422 		KASSERT(nrxq + ntxq <= vfres->neq, ("too many egress queues"));
423 
424 		/*
425 		 * Do we have enough interrupts?  For MSI the interrupts
426 		 * have to be a power of 2 as well.
427 		 */
428 		iaq->nirq += nrxq;
429 		iaq->ntxq = ntxq;
430 		iaq->nrxq = nrxq;
431 		if (iaq->nirq <= navail &&
432 		    (itype != INTR_MSI || powerof2(iaq->nirq))) {
433 			navail = iaq->nirq;
434 			if (itype == INTR_MSIX)
435 				rc = pci_alloc_msix(sc->dev, &navail);
436 			else
437 				rc = pci_alloc_msi(sc->dev, &navail);
438 			if (rc != 0) {
439 				device_printf(sc->dev,
440 		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
441 				    itype, rc, iaq->nirq, navail);
442 				return (rc);
443 			}
444 			if (navail == iaq->nirq) {
445 				return (0);
446 			}
447 			pci_release_msi(sc->dev);
448 		}
449 
450 		/* Fall back to a single interrupt. */
451 		iaq->nirq = 1;
452 		navail = iaq->nirq;
453 		if (itype == INTR_MSIX)
454 			rc = pci_alloc_msix(sc->dev, &navail);
455 		else
456 			rc = pci_alloc_msi(sc->dev, &navail);
457 		if (rc != 0)
458 			device_printf(sc->dev,
459 		    "failed to allocate vectors:%d, type=%d, req=%d, rcvd=%d\n",
460 			    itype, rc, iaq->nirq, navail);
461 		return (rc);
462 	}
463 
464 	device_printf(sc->dev,
465 	    "failed to find a usable interrupt type.  "
466 	    "allowed=%d, msi-x=%d, msi=%d, intx=1", t4_intr_types,
467 	    pci_msix_count(sc->dev), pci_msi_count(sc->dev));
468 
469 	return (ENXIO);
470 }
471 
472 static int
473 t4vf_attach(device_t dev)
474 {
475 	struct adapter *sc;
476 	int rc = 0, i, j, rqidx, tqidx;
477 	struct make_dev_args mda;
478 	struct intrs_and_queues iaq;
479 	struct sge *s;
480 
481 	sc = device_get_softc(dev);
482 	sc->dev = dev;
483 	pci_enable_busmaster(dev);
484 	pci_set_max_read_req(dev, 4096);
485 	sc->params.pci.mps = pci_get_max_payload(dev);
486 
487 	sc->flags |= IS_VF;
488 	TUNABLE_INT_FETCH("hw.cxgbe.dflags", &sc->debug_flags);
489 
490 	sc->sge_gts_reg = VF_SGE_REG(A_SGE_VF_GTS);
491 	sc->sge_kdoorbell_reg = VF_SGE_REG(A_SGE_VF_KDOORBELL);
492 	snprintf(sc->lockname, sizeof(sc->lockname), "%s",
493 	    device_get_nameunit(dev));
494 	mtx_init(&sc->sc_lock, sc->lockname, 0, MTX_DEF);
495 	t4_add_adapter(sc);
496 
497 	mtx_init(&sc->sfl_lock, "starving freelists", 0, MTX_DEF);
498 	TAILQ_INIT(&sc->sfl);
499 	callout_init_mtx(&sc->sfl_callout, &sc->sfl_lock, 0);
500 
501 	mtx_init(&sc->reg_lock, "indirect register access", 0, MTX_DEF);
502 
503 	rc = t4_map_bars_0_and_4(sc);
504 	if (rc != 0)
505 		goto done; /* error message displayed already */
506 
507 	rc = -t4vf_prep_adapter(sc);
508 	if (rc != 0)
509 		goto done;
510 
511 	t4_init_devnames(sc);
512 	if (sc->names == NULL) {
513 		rc = ENOTSUP;
514 		goto done; /* error message displayed already */
515 	}
516 
517 	/*
518 	 * Leave the 'pf' and 'mbox' values as zero.  This ensures
519 	 * that various firmware messages do not set the fields which
520 	 * is the correct thing to do for a VF.
521 	 */
522 
523 	memset(sc->chan_map, 0xff, sizeof(sc->chan_map));
524 
525 	make_dev_args_init(&mda);
526 	mda.mda_devsw = &t4vf_cdevsw;
527 	mda.mda_uid = UID_ROOT;
528 	mda.mda_gid = GID_WHEEL;
529 	mda.mda_mode = 0600;
530 	mda.mda_si_drv1 = sc;
531 	rc = make_dev_s(&mda, &sc->cdev, "%s", device_get_nameunit(dev));
532 	if (rc != 0)
533 		device_printf(dev, "failed to create nexus char device: %d.\n",
534 		    rc);
535 
536 #if defined(__i386__)
537 	if ((cpu_feature & CPUID_CX8) == 0) {
538 		device_printf(dev, "64 bit atomics not available.\n");
539 		rc = ENOTSUP;
540 		goto done;
541 	}
542 #endif
543 
544 	/*
545 	 * Some environments do not properly handle PCIE FLRs -- e.g. in Linux
546 	 * 2.6.31 and later we can't call pci_reset_function() in order to
547 	 * issue an FLR because of a self- deadlock on the device semaphore.
548 	 * Meanwhile, the OS infrastructure doesn't issue FLRs in all the
549 	 * cases where they're needed -- for instance, some versions of KVM
550 	 * fail to reset "Assigned Devices" when the VM reboots.  Therefore we
551 	 * use the firmware based reset in order to reset any per function
552 	 * state.
553 	 */
554 	rc = -t4vf_fw_reset(sc);
555 	if (rc != 0) {
556 		device_printf(dev, "FW reset failed: %d\n", rc);
557 		goto done;
558 	}
559 	sc->flags |= FW_OK;
560 
561 	/*
562 	 * Grab basic operational parameters.  These will predominantly have
563 	 * been set up by the Physical Function Driver or will be hard coded
564 	 * into the adapter.  We just have to live with them ...  Note that
565 	 * we _must_ get our VPD parameters before our SGE parameters because
566 	 * we need to know the adapter's core clock from the VPD in order to
567 	 * properly decode the SGE Timer Values.
568 	 */
569 	rc = get_params__pre_init(sc);
570 	if (rc != 0)
571 		goto done; /* error message displayed already */
572 	rc = get_params__post_init(sc);
573 	if (rc != 0)
574 		goto done; /* error message displayed already */
575 
576 	rc = set_params__post_init(sc);
577 	if (rc != 0)
578 		goto done; /* error message displayed already */
579 
580 	rc = t4_map_bar_2(sc);
581 	if (rc != 0)
582 		goto done; /* error message displayed already */
583 
584 	rc = t4_create_dma_tag(sc);
585 	if (rc != 0)
586 		goto done; /* error message displayed already */
587 
588 	/*
589 	 * The number of "ports" which we support is equal to the number of
590 	 * Virtual Interfaces with which we've been provisioned.
591 	 */
592 	sc->params.nports = imin(sc->params.vfres.nvi, MAX_NPORTS);
593 
594 	/*
595 	 * We may have been provisioned with more VIs than the number of
596 	 * ports we're allowed to access (our Port Access Rights Mask).
597 	 * Just use a single VI for each port.
598 	 */
599 	sc->params.nports = imin(sc->params.nports,
600 	    bitcount32(sc->params.vfres.pmask));
601 
602 #ifdef notyet
603 	/*
604 	 * XXX: The Linux VF driver will lower nports if it thinks there
605 	 * are too few resources in vfres (niqflint, nethctrl, neq).
606 	 */
607 #endif
608 
609 	/*
610 	 * First pass over all the ports - allocate VIs and initialize some
611 	 * basic parameters like mac address, port type, etc.
612 	 */
613 	for_each_port(sc, i) {
614 		struct port_info *pi;
615 
616 		pi = malloc(sizeof(*pi), M_CXGBE, M_ZERO | M_WAITOK);
617 		sc->port[i] = pi;
618 
619 		/* These must be set before t4_port_init */
620 		pi->adapter = sc;
621 		pi->port_id = i;
622 		pi->nvi = 1;
623 		pi->vi = malloc(sizeof(struct vi_info) * pi->nvi, M_CXGBE,
624 		    M_ZERO | M_WAITOK);
625 
626 		/*
627 		 * Allocate the "main" VI and initialize parameters
628 		 * like mac addr.
629 		 */
630 		rc = -t4_port_init(sc, sc->mbox, sc->pf, 0, i);
631 		if (rc != 0) {
632 			device_printf(dev, "unable to initialize port %d: %d\n",
633 			    i, rc);
634 			free(pi->vi, M_CXGBE);
635 			free(pi, M_CXGBE);
636 			sc->port[i] = NULL;
637 			goto done;
638 		}
639 
640 		/* No t4_link_start. */
641 
642 		snprintf(pi->lockname, sizeof(pi->lockname), "%sp%d",
643 		    device_get_nameunit(dev), i);
644 		mtx_init(&pi->pi_lock, pi->lockname, 0, MTX_DEF);
645 		sc->chan_map[pi->tx_chan] = i;
646 
647 		/* All VIs on this port share this media. */
648 		ifmedia_init(&pi->media, IFM_IMASK, cxgbe_media_change,
649 		    cxgbe_media_status);
650 
651 		pi->dev = device_add_child(dev, sc->names->vf_ifnet_name, -1);
652 		if (pi->dev == NULL) {
653 			device_printf(dev,
654 			    "failed to add device for port %d.\n", i);
655 			rc = ENXIO;
656 			goto done;
657 		}
658 		pi->vi[0].dev = pi->dev;
659 		device_set_softc(pi->dev, pi);
660 	}
661 
662 	/*
663 	 * Interrupt type, # of interrupts, # of rx/tx queues, etc.
664 	 */
665 	rc = cfg_itype_and_nqueues(sc, &iaq);
666 	if (rc != 0)
667 		goto done; /* error message displayed already */
668 
669 	sc->intr_type = iaq.intr_type;
670 	sc->intr_count = iaq.nirq;
671 
672 	s = &sc->sge;
673 	s->nrxq = sc->params.nports * iaq.nrxq;
674 	s->ntxq = sc->params.nports * iaq.ntxq;
675 	s->neq = s->ntxq + s->nrxq;	/* the free list in an rxq is an eq */
676 	s->neq += sc->params.nports;	/* ctrl queues: 1 per port */
677 	s->niq = s->nrxq + 1;		/* 1 extra for firmware event queue */
678 
679 	s->rxq = malloc(s->nrxq * sizeof(struct sge_rxq), M_CXGBE,
680 	    M_ZERO | M_WAITOK);
681 	s->txq = malloc(s->ntxq * sizeof(struct sge_txq), M_CXGBE,
682 	    M_ZERO | M_WAITOK);
683 	s->iqmap = malloc(s->niq * sizeof(struct sge_iq *), M_CXGBE,
684 	    M_ZERO | M_WAITOK);
685 	s->eqmap = malloc(s->neq * sizeof(struct sge_eq *), M_CXGBE,
686 	    M_ZERO | M_WAITOK);
687 
688 	sc->irq = malloc(sc->intr_count * sizeof(struct irq), M_CXGBE,
689 	    M_ZERO | M_WAITOK);
690 
691 	/*
692 	 * Second pass over the ports.  This time we know the number of rx and
693 	 * tx queues that each port should get.
694 	 */
695 	rqidx = tqidx = 0;
696 	for_each_port(sc, i) {
697 		struct port_info *pi = sc->port[i];
698 		struct vi_info *vi;
699 
700 		if (pi == NULL)
701 			continue;
702 
703 		for_each_vi(pi, j, vi) {
704 			vi->pi = pi;
705 			vi->qsize_rxq = t4_qsize_rxq;
706 			vi->qsize_txq = t4_qsize_txq;
707 
708 			vi->first_rxq = rqidx;
709 			vi->first_txq = tqidx;
710 			vi->tmr_idx = t4_tmr_idx;
711 			vi->pktc_idx = t4_pktc_idx;
712 			vi->nrxq = j == 0 ? iaq.nrxq: 1;
713 			vi->ntxq = j == 0 ? iaq.ntxq: 1;
714 
715 			rqidx += vi->nrxq;
716 			tqidx += vi->ntxq;
717 
718 			vi->rsrv_noflowq = 0;
719 		}
720 	}
721 
722 	rc = t4_setup_intr_handlers(sc);
723 	if (rc != 0) {
724 		device_printf(dev,
725 		    "failed to setup interrupt handlers: %d\n", rc);
726 		goto done;
727 	}
728 
729 	rc = bus_generic_attach(dev);
730 	if (rc != 0) {
731 		device_printf(dev,
732 		    "failed to attach all child ports: %d\n", rc);
733 		goto done;
734 	}
735 
736 	device_printf(dev,
737 	    "%d ports, %d %s interrupt%s, %d eq, %d iq\n",
738 	    sc->params.nports, sc->intr_count, sc->intr_type == INTR_MSIX ?
739 	    "MSI-X" : "MSI", sc->intr_count > 1 ? "s" : "", sc->sge.neq,
740 	    sc->sge.niq);
741 
742 done:
743 	if (rc != 0)
744 		t4_detach_common(dev);
745 	else
746 		t4_sysctls(sc);
747 
748 	return (rc);
749 }
750 
751 static void
752 get_regs(struct adapter *sc, struct t4_regdump *regs, uint8_t *buf)
753 {
754 
755 	/* 0x3f is used as the revision for VFs. */
756 	regs->version = chip_id(sc) | (0x3f << 10);
757 	t4_get_regs(sc, buf, regs->len);
758 }
759 
760 static void
761 t4_clr_vi_stats(struct adapter *sc)
762 {
763 	int reg;
764 
765 	for (reg = A_MPS_VF_STAT_TX_VF_BCAST_BYTES_L;
766 	     reg <= A_MPS_VF_STAT_RX_VF_ERR_FRAMES_H; reg += 4)
767 		t4_write_reg(sc, VF_MPS_REG(reg), 0);
768 }
769 
770 static int
771 t4vf_ioctl(struct cdev *dev, unsigned long cmd, caddr_t data, int fflag,
772     struct thread *td)
773 {
774 	int rc;
775 	struct adapter *sc = dev->si_drv1;
776 
777 	rc = priv_check(td, PRIV_DRIVER);
778 	if (rc != 0)
779 		return (rc);
780 
781 	switch (cmd) {
782 	case CHELSIO_T4_GETREG: {
783 		struct t4_reg *edata = (struct t4_reg *)data;
784 
785 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
786 			return (EFAULT);
787 
788 		if (edata->size == 4)
789 			edata->val = t4_read_reg(sc, edata->addr);
790 		else if (edata->size == 8)
791 			edata->val = t4_read_reg64(sc, edata->addr);
792 		else
793 			return (EINVAL);
794 
795 		break;
796 	}
797 	case CHELSIO_T4_SETREG: {
798 		struct t4_reg *edata = (struct t4_reg *)data;
799 
800 		if ((edata->addr & 0x3) != 0 || edata->addr >= sc->mmio_len)
801 			return (EFAULT);
802 
803 		if (edata->size == 4) {
804 			if (edata->val & 0xffffffff00000000)
805 				return (EINVAL);
806 			t4_write_reg(sc, edata->addr, (uint32_t) edata->val);
807 		} else if (edata->size == 8)
808 			t4_write_reg64(sc, edata->addr, edata->val);
809 		else
810 			return (EINVAL);
811 		break;
812 	}
813 	case CHELSIO_T4_REGDUMP: {
814 		struct t4_regdump *regs = (struct t4_regdump *)data;
815 		int reglen = t4_get_regs_len(sc);
816 		uint8_t *buf;
817 
818 		if (regs->len < reglen) {
819 			regs->len = reglen; /* hint to the caller */
820 			return (ENOBUFS);
821 		}
822 
823 		regs->len = reglen;
824 		buf = malloc(reglen, M_CXGBE, M_WAITOK | M_ZERO);
825 		get_regs(sc, regs, buf);
826 		rc = copyout(buf, regs->data, reglen);
827 		free(buf, M_CXGBE);
828 		break;
829 	}
830 	case CHELSIO_T4_CLEAR_STATS: {
831 		int i, v;
832 		u_int port_id = *(uint32_t *)data;
833 		struct port_info *pi;
834 		struct vi_info *vi;
835 
836 		if (port_id >= sc->params.nports)
837 			return (EINVAL);
838 		pi = sc->port[port_id];
839 
840 		/* MAC stats */
841 		pi->tx_parse_error = 0;
842 		t4_clr_vi_stats(sc);
843 
844 		/*
845 		 * Since this command accepts a port, clear stats for
846 		 * all VIs on this port.
847 		 */
848 		for_each_vi(pi, v, vi) {
849 			if (vi->flags & VI_INIT_DONE) {
850 				struct sge_rxq *rxq;
851 				struct sge_txq *txq;
852 
853 				for_each_rxq(vi, i, rxq) {
854 #if defined(INET) || defined(INET6)
855 					rxq->lro.lro_queued = 0;
856 					rxq->lro.lro_flushed = 0;
857 #endif
858 					rxq->rxcsum = 0;
859 					rxq->vlan_extraction = 0;
860 				}
861 
862 				for_each_txq(vi, i, txq) {
863 					txq->txcsum = 0;
864 					txq->tso_wrs = 0;
865 					txq->vlan_insertion = 0;
866 					txq->imm_wrs = 0;
867 					txq->sgl_wrs = 0;
868 					txq->txpkt_wrs = 0;
869 					txq->txpkts0_wrs = 0;
870 					txq->txpkts1_wrs = 0;
871 					txq->txpkts0_pkts = 0;
872 					txq->txpkts1_pkts = 0;
873 					mp_ring_reset_stats(txq->r);
874 				}
875 			}
876 		}
877 		break;
878 	}
879 	case CHELSIO_T4_SCHED_CLASS:
880 		rc = t4_set_sched_class(sc, (struct t4_sched_params *)data);
881 		break;
882 	case CHELSIO_T4_SCHED_QUEUE:
883 		rc = t4_set_sched_queue(sc, (struct t4_sched_queue *)data);
884 		break;
885 	default:
886 		rc = ENOTTY;
887 	}
888 
889 	return (rc);
890 }
891 
892 static device_method_t t4vf_methods[] = {
893 	DEVMETHOD(device_probe,		t4vf_probe),
894 	DEVMETHOD(device_attach,	t4vf_attach),
895 	DEVMETHOD(device_detach,	t4_detach_common),
896 
897 	DEVMETHOD_END
898 };
899 
900 static driver_t t4vf_driver = {
901 	"t4vf",
902 	t4vf_methods,
903 	sizeof(struct adapter)
904 };
905 
906 static device_method_t t5vf_methods[] = {
907 	DEVMETHOD(device_probe,		t5vf_probe),
908 	DEVMETHOD(device_attach,	t4vf_attach),
909 	DEVMETHOD(device_detach,	t4_detach_common),
910 
911 	DEVMETHOD_END
912 };
913 
914 static driver_t t5vf_driver = {
915 	"t5vf",
916 	t5vf_methods,
917 	sizeof(struct adapter)
918 };
919 
920 static device_method_t t6vf_methods[] = {
921 	DEVMETHOD(device_probe,		t6vf_probe),
922 	DEVMETHOD(device_attach,	t4vf_attach),
923 	DEVMETHOD(device_detach,	t4_detach_common),
924 
925 	DEVMETHOD_END
926 };
927 
928 static driver_t t6vf_driver = {
929 	"t6vf",
930 	t6vf_methods,
931 	sizeof(struct adapter)
932 };
933 
934 static driver_t cxgbev_driver = {
935 	"cxgbev",
936 	cxgbe_methods,
937 	sizeof(struct port_info)
938 };
939 
940 static driver_t cxlv_driver = {
941 	"cxlv",
942 	cxgbe_methods,
943 	sizeof(struct port_info)
944 };
945 
946 static driver_t ccv_driver = {
947 	"ccv",
948 	cxgbe_methods,
949 	sizeof(struct port_info)
950 };
951 
952 static devclass_t t4vf_devclass, t5vf_devclass, t6vf_devclass;
953 static devclass_t cxgbev_devclass, cxlv_devclass, ccv_devclass;
954 
955 DRIVER_MODULE(t4vf, pci, t4vf_driver, t4vf_devclass, 0, 0);
956 MODULE_VERSION(t4vf, 1);
957 MODULE_DEPEND(t4vf, t4nex, 1, 1, 1);
958 
959 DRIVER_MODULE(t5vf, pci, t5vf_driver, t5vf_devclass, 0, 0);
960 MODULE_VERSION(t5vf, 1);
961 MODULE_DEPEND(t5vf, t5nex, 1, 1, 1);
962 
963 DRIVER_MODULE(t6vf, pci, t6vf_driver, t6vf_devclass, 0, 0);
964 MODULE_VERSION(t6vf, 1);
965 MODULE_DEPEND(t6vf, t6nex, 1, 1, 1);
966 
967 DRIVER_MODULE(cxgbev, t4vf, cxgbev_driver, cxgbev_devclass, 0, 0);
968 MODULE_VERSION(cxgbev, 1);
969 
970 DRIVER_MODULE(cxlv, t5vf, cxlv_driver, cxlv_devclass, 0, 0);
971 MODULE_VERSION(cxlv, 1);
972 
973 DRIVER_MODULE(ccv, t6vf, ccv_driver, ccv_devclass, 0, 0);
974 MODULE_VERSION(ccv, 1);
975