xref: /freebsd/sys/dev/mlx5/mlx5_en/mlx5_en_main.c (revision 3332f1b444d4a73238e9f59cca27bfc95fe936bd)
1 /*-
2  * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  *
25  * $FreeBSD$
26  */
27 
28 #include "opt_kern_tls.h"
29 
30 #include "en.h"
31 
32 #include <sys/eventhandler.h>
33 #include <sys/sockio.h>
34 #include <machine/atomic.h>
35 
36 #include <net/debugnet.h>
37 
38 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
39 static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
40 static if_snd_tag_free_t mlx5e_ul_snd_tag_free;
41 
42 struct mlx5e_channel_param {
43 	struct mlx5e_rq_param rq;
44 	struct mlx5e_sq_param sq;
45 	struct mlx5e_cq_param rx_cq;
46 	struct mlx5e_cq_param tx_cq;
47 };
48 
49 struct media {
50 	u32	subtype;
51 	u64	baudrate;
52 };
53 
54 static const struct media mlx5e_mode_table[MLX5E_LINK_SPEEDS_NUMBER] =
55 {
56 	[MLX5E_1000BASE_CX_SGMII] = {
57 		.subtype = IFM_1000_CX_SGMII,
58 		.baudrate = IF_Mbps(1000ULL),
59 	},
60 	[MLX5E_1000BASE_KX] = {
61 		.subtype = IFM_1000_KX,
62 		.baudrate = IF_Mbps(1000ULL),
63 	},
64 	[MLX5E_10GBASE_CX4] = {
65 		.subtype = IFM_10G_CX4,
66 		.baudrate = IF_Gbps(10ULL),
67 	},
68 	[MLX5E_10GBASE_KX4] = {
69 		.subtype = IFM_10G_KX4,
70 		.baudrate = IF_Gbps(10ULL),
71 	},
72 	[MLX5E_10GBASE_KR] = {
73 		.subtype = IFM_10G_KR,
74 		.baudrate = IF_Gbps(10ULL),
75 	},
76 	[MLX5E_20GBASE_KR2] = {
77 		.subtype = IFM_20G_KR2,
78 		.baudrate = IF_Gbps(20ULL),
79 	},
80 	[MLX5E_40GBASE_CR4] = {
81 		.subtype = IFM_40G_CR4,
82 		.baudrate = IF_Gbps(40ULL),
83 	},
84 	[MLX5E_40GBASE_KR4] = {
85 		.subtype = IFM_40G_KR4,
86 		.baudrate = IF_Gbps(40ULL),
87 	},
88 	[MLX5E_56GBASE_R4] = {
89 		.subtype = IFM_56G_R4,
90 		.baudrate = IF_Gbps(56ULL),
91 	},
92 	[MLX5E_10GBASE_CR] = {
93 		.subtype = IFM_10G_CR1,
94 		.baudrate = IF_Gbps(10ULL),
95 	},
96 	[MLX5E_10GBASE_SR] = {
97 		.subtype = IFM_10G_SR,
98 		.baudrate = IF_Gbps(10ULL),
99 	},
100 	[MLX5E_10GBASE_ER_LR] = {
101 		.subtype = IFM_10G_ER,
102 		.baudrate = IF_Gbps(10ULL),
103 	},
104 	[MLX5E_40GBASE_SR4] = {
105 		.subtype = IFM_40G_SR4,
106 		.baudrate = IF_Gbps(40ULL),
107 	},
108 	[MLX5E_40GBASE_LR4_ER4] = {
109 		.subtype = IFM_40G_LR4,
110 		.baudrate = IF_Gbps(40ULL),
111 	},
112 	[MLX5E_100GBASE_CR4] = {
113 		.subtype = IFM_100G_CR4,
114 		.baudrate = IF_Gbps(100ULL),
115 	},
116 	[MLX5E_100GBASE_SR4] = {
117 		.subtype = IFM_100G_SR4,
118 		.baudrate = IF_Gbps(100ULL),
119 	},
120 	[MLX5E_100GBASE_KR4] = {
121 		.subtype = IFM_100G_KR4,
122 		.baudrate = IF_Gbps(100ULL),
123 	},
124 	[MLX5E_100GBASE_LR4] = {
125 		.subtype = IFM_100G_LR4,
126 		.baudrate = IF_Gbps(100ULL),
127 	},
128 	[MLX5E_100BASE_TX] = {
129 		.subtype = IFM_100_TX,
130 		.baudrate = IF_Mbps(100ULL),
131 	},
132 	[MLX5E_1000BASE_T] = {
133 		.subtype = IFM_1000_T,
134 		.baudrate = IF_Mbps(1000ULL),
135 	},
136 	[MLX5E_10GBASE_T] = {
137 		.subtype = IFM_10G_T,
138 		.baudrate = IF_Gbps(10ULL),
139 	},
140 	[MLX5E_25GBASE_CR] = {
141 		.subtype = IFM_25G_CR,
142 		.baudrate = IF_Gbps(25ULL),
143 	},
144 	[MLX5E_25GBASE_KR] = {
145 		.subtype = IFM_25G_KR,
146 		.baudrate = IF_Gbps(25ULL),
147 	},
148 	[MLX5E_25GBASE_SR] = {
149 		.subtype = IFM_25G_SR,
150 		.baudrate = IF_Gbps(25ULL),
151 	},
152 	[MLX5E_50GBASE_CR2] = {
153 		.subtype = IFM_50G_CR2,
154 		.baudrate = IF_Gbps(50ULL),
155 	},
156 	[MLX5E_50GBASE_KR2] = {
157 		.subtype = IFM_50G_KR2,
158 		.baudrate = IF_Gbps(50ULL),
159 	},
160 	[MLX5E_50GBASE_KR4] = {
161 		.subtype = IFM_50G_KR4,
162 		.baudrate = IF_Gbps(50ULL),
163 	},
164 };
165 
166 static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX5E_CABLE_TYPE_NUMBER] =
167 {
168 	/**/
169 	[MLX5E_SGMII_100M][MLX5E_CABLE_TYPE_UNKNOWN] = {
170 		.subtype = IFM_100_SGMII,
171 		.baudrate = IF_Mbps(100),
172 	},
173 
174 	/**/
175 	[MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_UNKNOWN] = {
176 		.subtype = IFM_1000_CX,
177 		.baudrate = IF_Mbps(1000),
178 	},
179 	[MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
180 		.subtype = IFM_1000_SX,
181 		.baudrate = IF_Mbps(1000),
182 	},
183 
184 	/**/
185 	[MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_UNKNOWN] = {
186 		.subtype = IFM_5000_KR,
187 		.baudrate = IF_Mbps(5000),
188 	},
189 	[MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
190 		.subtype = IFM_5000_T,
191 		.baudrate = IF_Mbps(5000),
192 	},
193 
194 	/**/
195 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_UNKNOWN] = {
196 		.subtype = IFM_10G_KR,
197 		.baudrate = IF_Gbps(10ULL),
198 	},
199 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
200 		.subtype = IFM_10G_CR1,
201 		.baudrate = IF_Gbps(10ULL),
202 	},
203 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
204 		.subtype = IFM_10G_SR,
205 		.baudrate = IF_Gbps(10ULL),
206 	},
207 
208 	/**/
209 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_UNKNOWN] = {
210 		.subtype = IFM_40G_KR4,
211 		.baudrate = IF_Gbps(40ULL),
212 	},
213 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
214 		.subtype = IFM_40G_CR4,
215 		.baudrate = IF_Gbps(40ULL),
216 	},
217 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
218 		.subtype = IFM_40G_SR4,
219 		.baudrate = IF_Gbps(40ULL),
220 	},
221 
222 	/**/
223 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
224 		.subtype = IFM_25G_KR,
225 		.baudrate = IF_Gbps(25ULL),
226 	},
227 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
228 		.subtype = IFM_25G_CR,
229 		.baudrate = IF_Gbps(25ULL),
230 	},
231 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
232 		.subtype = IFM_25G_SR,
233 		.baudrate = IF_Gbps(25ULL),
234 	},
235 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
236 		.subtype = IFM_25G_T,
237 		.baudrate = IF_Gbps(25ULL),
238 	},
239 
240 	/**/
241 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
242 		.subtype = IFM_50G_KR2,
243 		.baudrate = IF_Gbps(50ULL),
244 	},
245 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
246 		.subtype = IFM_50G_CR2,
247 		.baudrate = IF_Gbps(50ULL),
248 	},
249 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
250 		.subtype = IFM_50G_SR2,
251 		.baudrate = IF_Gbps(50ULL),
252 	},
253 
254 	/**/
255 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
256 		.subtype = IFM_50G_KR_PAM4,
257 		.baudrate = IF_Gbps(50ULL),
258 	},
259 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
260 		.subtype = IFM_50G_CP,
261 		.baudrate = IF_Gbps(50ULL),
262 	},
263 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
264 		.subtype = IFM_50G_SR,
265 		.baudrate = IF_Gbps(50ULL),
266 	},
267 
268 	/**/
269 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
270 		.subtype = IFM_100G_KR4,
271 		.baudrate = IF_Gbps(100ULL),
272 	},
273 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
274 		.subtype = IFM_100G_CR4,
275 		.baudrate = IF_Gbps(100ULL),
276 	},
277 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
278 		.subtype = IFM_100G_SR4,
279 		.baudrate = IF_Gbps(100ULL),
280 	},
281 
282 	/**/
283 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
284 		.subtype = IFM_100G_KR_PAM4,
285 		.baudrate = IF_Gbps(100ULL),
286 	},
287 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
288 		.subtype = IFM_100G_CR_PAM4,
289 		.baudrate = IF_Gbps(100ULL),
290 	},
291 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
292 		.subtype = IFM_100G_SR2,	/* XXX */
293 		.baudrate = IF_Gbps(100ULL),
294 	},
295 
296 	/**/
297 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
298 		.subtype = IFM_100G_KR4,
299 		.baudrate = IF_Gbps(100ULL),
300 	},
301 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
302 		.subtype = IFM_100G_CP2,
303 		.baudrate = IF_Gbps(100ULL),
304 	},
305 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
306 		.subtype = IFM_100G_SR2,
307 		.baudrate = IF_Gbps(100ULL),
308 	},
309 
310 	/**/
311 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
312 		.subtype = IFM_200G_KR4_PAM4,	/* XXX */
313 		.baudrate = IF_Gbps(200ULL),
314 	},
315 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
316 		.subtype = IFM_200G_CR4_PAM4,	/* XXX */
317 		.baudrate = IF_Gbps(200ULL),
318 	},
319 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
320 		.subtype = IFM_200G_SR4,	/* XXX */
321 		.baudrate = IF_Gbps(200ULL),
322 	},
323 
324 	/**/
325 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
326 		.subtype = IFM_200G_KR4_PAM4,
327 		.baudrate = IF_Gbps(200ULL),
328 	},
329 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
330 		.subtype = IFM_200G_CR4_PAM4,
331 		.baudrate = IF_Gbps(200ULL),
332 	},
333 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
334 		.subtype = IFM_200G_SR4,
335 		.baudrate = IF_Gbps(200ULL),
336 	},
337 
338 	/**/
339 	[MLX5E_400GAUI_8][MLX5E_CABLE_TYPE_UNKNOWN] = {
340 		.subtype = IFM_400G_LR8,	/* XXX */
341 		.baudrate = IF_Gbps(400ULL),
342 	},
343 
344 	/**/
345 	[MLX5E_400GAUI_4_400GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
346 		.subtype = IFM_400G_LR8,	/* XXX */
347 		.baudrate = IF_Gbps(400ULL),
348 	},
349 };
350 
351 static const struct if_snd_tag_sw mlx5e_ul_snd_tag_sw = {
352 	.snd_tag_query = mlx5e_ul_snd_tag_query,
353 	.snd_tag_free = mlx5e_ul_snd_tag_free,
354 	.type = IF_SND_TAG_TYPE_UNLIMITED
355 };
356 
357 DEBUGNET_DEFINE(mlx5_en);
358 
359 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
360 
361 static void
362 mlx5e_update_carrier(struct mlx5e_priv *priv)
363 {
364 	struct mlx5_core_dev *mdev = priv->mdev;
365 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
366 	u32 eth_proto_oper;
367 	int error;
368 	u8 i;
369 	u8 cable_type;
370 	u8 port_state;
371 	u8 is_er_type;
372 	bool ext;
373 	struct media media_entry = {};
374 
375 	port_state = mlx5_query_vport_state(mdev,
376 	    MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
377 
378 	if (port_state == VPORT_STATE_UP) {
379 		priv->media_status_last |= IFM_ACTIVE;
380 	} else {
381 		priv->media_status_last &= ~IFM_ACTIVE;
382 		priv->media_active_last = IFM_ETHER;
383 		if_link_state_change(priv->ifp, LINK_STATE_DOWN);
384 		return;
385 	}
386 
387 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
388 	    MLX5_PTYS_EN, 1);
389 	if (error) {
390 		priv->media_active_last = IFM_ETHER;
391 		priv->ifp->if_baudrate = 1;
392 		mlx5_en_err(priv->ifp, "query port ptys failed: 0x%x\n",
393 		    error);
394 		return;
395 	}
396 
397 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
398 	eth_proto_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
399 	    eth_proto_oper);
400 
401 	i = ilog2(eth_proto_oper);
402 
403 	if (ext) {
404 		error = mlx5_query_pddr_cable_type(mdev, 1, &cable_type);
405 		if (error != 0) {
406 			/* use fallback entry */
407 			media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
408 
409 			mlx5_en_err(priv->ifp,
410 			    "query port pddr failed: %d\n", error);
411 		} else {
412 			media_entry = mlx5e_ext_mode_table[i][cable_type];
413 
414 			/* check if we should use fallback entry */
415 			if (media_entry.subtype == 0)
416 				media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
417 		}
418 	} else {
419 		media_entry = mlx5e_mode_table[i];
420 	}
421 
422 	if (media_entry.subtype == 0) {
423 		mlx5_en_err(priv->ifp,
424 		    "Could not find operational media subtype\n");
425 		return;
426 	}
427 
428 	switch (media_entry.subtype) {
429 	case IFM_10G_ER:
430 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
431 		if (error != 0) {
432 			mlx5_en_err(priv->ifp,
433 			    "query port pddr failed: %d\n", error);
434 		}
435 		if (error != 0 || is_er_type == 0)
436 			media_entry.subtype = IFM_10G_LR;
437 		break;
438 	case IFM_40G_LR4:
439 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
440 		if (error != 0) {
441 			mlx5_en_err(priv->ifp,
442 			    "query port pddr failed: %d\n", error);
443 		}
444 		if (error == 0 && is_er_type != 0)
445 			media_entry.subtype = IFM_40G_ER4;
446 		break;
447 	}
448 	priv->media_active_last = media_entry.subtype | IFM_ETHER | IFM_FDX;
449 	priv->ifp->if_baudrate = media_entry.baudrate;
450 
451 	if_link_state_change(priv->ifp, LINK_STATE_UP);
452 }
453 
454 static void
455 mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr)
456 {
457 	struct mlx5e_priv *priv = dev->if_softc;
458 
459 	ifmr->ifm_status = priv->media_status_last;
460 	ifmr->ifm_current = ifmr->ifm_active = priv->media_active_last |
461 	    (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
462 	    (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
463 
464 }
465 
466 static u32
467 mlx5e_find_link_mode(u32 subtype, bool ext)
468 {
469 	u32 link_mode = 0;
470 
471 	switch (subtype) {
472 	case 0:
473 		goto done;
474 	case IFM_10G_LR:
475 		subtype = IFM_10G_ER;
476 		break;
477 	case IFM_40G_ER4:
478 		subtype = IFM_40G_LR4;
479 		break;
480 	default:
481 		break;
482 	}
483 
484 	if (ext) {
485 		for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
486 			for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
487 				if (mlx5e_ext_mode_table[i][j].subtype == subtype)
488 					link_mode |= MLX5E_PROT_MASK(i);
489 			}
490 		}
491 	} else {
492 		for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
493 			if (mlx5e_mode_table[i].subtype == subtype)
494 				link_mode |= MLX5E_PROT_MASK(i);
495 		}
496 	}
497 done:
498 	return (link_mode);
499 }
500 
501 static int
502 mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
503 {
504 	return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
505 	    priv->params.rx_pauseframe_control,
506 	    priv->params.tx_pauseframe_control,
507 	    priv->params.rx_priority_flow_control,
508 	    priv->params.tx_priority_flow_control));
509 }
510 
511 static int
512 mlx5e_set_port_pfc(struct mlx5e_priv *priv)
513 {
514 	int error;
515 
516 	if (priv->gone != 0) {
517 		error = -ENXIO;
518 	} else if (priv->params.rx_pauseframe_control ||
519 	    priv->params.tx_pauseframe_control) {
520 		mlx5_en_err(priv->ifp,
521 		    "Global pauseframes must be disabled before enabling PFC.\n");
522 		error = -EINVAL;
523 	} else {
524 		error = mlx5e_set_port_pause_and_pfc(priv);
525 	}
526 	return (error);
527 }
528 
529 static int
530 mlx5e_media_change(struct ifnet *dev)
531 {
532 	struct mlx5e_priv *priv = dev->if_softc;
533 	struct mlx5_core_dev *mdev = priv->mdev;
534 	u32 eth_proto_cap;
535 	u32 link_mode;
536 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
537 	int was_opened;
538 	int locked;
539 	int error;
540 	bool ext;
541 
542 	locked = PRIV_LOCKED(priv);
543 	if (!locked)
544 		PRIV_LOCK(priv);
545 
546 	if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
547 		error = EINVAL;
548 		goto done;
549 	}
550 
551 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
552 	    MLX5_PTYS_EN, 1);
553 	if (error != 0) {
554 		mlx5_en_err(dev, "Query port media capability failed\n");
555 		goto done;
556 	}
557 
558 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
559 	link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media), ext);
560 
561 	/* query supported capabilities */
562 	eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
563 	    eth_proto_capability);
564 
565 	/* check for autoselect */
566 	if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
567 		link_mode = eth_proto_cap;
568 		if (link_mode == 0) {
569 			mlx5_en_err(dev, "Port media capability is zero\n");
570 			error = EINVAL;
571 			goto done;
572 		}
573 	} else {
574 		link_mode = link_mode & eth_proto_cap;
575 		if (link_mode == 0) {
576 			mlx5_en_err(dev, "Not supported link mode requested\n");
577 			error = EINVAL;
578 			goto done;
579 		}
580 	}
581 	if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
582 		/* check if PFC is enabled */
583 		if (priv->params.rx_priority_flow_control ||
584 		    priv->params.tx_priority_flow_control) {
585 			mlx5_en_err(dev, "PFC must be disabled before enabling global pauseframes.\n");
586 			error = EINVAL;
587 			goto done;
588 		}
589 	}
590 	/* update pauseframe control bits */
591 	priv->params.rx_pauseframe_control =
592 	    (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
593 	priv->params.tx_pauseframe_control =
594 	    (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
595 
596 	/* check if device is opened */
597 	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
598 
599 	/* reconfigure the hardware */
600 	mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
601 	mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN, ext);
602 	error = -mlx5e_set_port_pause_and_pfc(priv);
603 	if (was_opened)
604 		mlx5_set_port_status(mdev, MLX5_PORT_UP);
605 
606 done:
607 	if (!locked)
608 		PRIV_UNLOCK(priv);
609 	return (error);
610 }
611 
612 static void
613 mlx5e_update_carrier_work(struct work_struct *work)
614 {
615 	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
616 	    update_carrier_work);
617 
618 	PRIV_LOCK(priv);
619 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
620 		mlx5e_update_carrier(priv);
621 	PRIV_UNLOCK(priv);
622 }
623 
624 #define	MLX5E_PCIE_PERF_GET_64(a,b,c,d,e,f)    \
625 	s_debug->c = MLX5_GET64(mpcnt_reg, out, counter_set.f.c);
626 
627 #define	MLX5E_PCIE_PERF_GET_32(a,b,c,d,e,f)    \
628 	s_debug->c = MLX5_GET(mpcnt_reg, out, counter_set.f.c);
629 
630 static void
631 mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
632 {
633 	struct mlx5_core_dev *mdev = priv->mdev;
634 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
635 	const unsigned sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
636 	void *out;
637 	void *in;
638 	int err;
639 
640 	/* allocate firmware request structures */
641 	in = mlx5_vzalloc(sz);
642 	out = mlx5_vzalloc(sz);
643 	if (in == NULL || out == NULL)
644 		goto free_out;
645 
646 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
647 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
648 	if (err != 0)
649 		goto free_out;
650 
651 	MLX5E_PCIE_PERFORMANCE_COUNTERS_64(MLX5E_PCIE_PERF_GET_64)
652 	MLX5E_PCIE_PERFORMANCE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
653 
654 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP);
655 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
656 	if (err != 0)
657 		goto free_out;
658 
659 	MLX5E_PCIE_TIMERS_AND_STATES_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
660 
661 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_LANE_COUNTERS_GROUP);
662 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
663 	if (err != 0)
664 		goto free_out;
665 
666 	MLX5E_PCIE_LANE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
667 
668 free_out:
669 	/* free firmware request structures */
670 	kvfree(in);
671 	kvfree(out);
672 }
673 
674 /*
675  * This function reads the physical port counters from the firmware
676  * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
677  * macros. The output is converted from big-endian 64-bit values into
678  * host endian ones and stored in the "priv->stats.pport" structure.
679  */
680 static void
681 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
682 {
683 	struct mlx5_core_dev *mdev = priv->mdev;
684 	struct mlx5e_pport_stats *s = &priv->stats.pport;
685 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
686 	u32 *in;
687 	u32 *out;
688 	const u64 *ptr;
689 	unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
690 	unsigned x;
691 	unsigned y;
692 	unsigned z;
693 
694 	/* allocate firmware request structures */
695 	in = mlx5_vzalloc(sz);
696 	out = mlx5_vzalloc(sz);
697 	if (in == NULL || out == NULL)
698 		goto free_out;
699 
700 	/*
701 	 * Get pointer to the 64-bit counter set which is located at a
702 	 * fixed offset in the output firmware request structure:
703 	 */
704 	ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
705 
706 	MLX5_SET(ppcnt_reg, in, local_port, 1);
707 
708 	/* read IEEE802_3 counter group using predefined counter layout */
709 	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
710 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
711 	for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
712 	     x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
713 		s->arg[y] = be64toh(ptr[x]);
714 
715 	/* read RFC2819 counter group using predefined counter layout */
716 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
717 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
718 	for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
719 		s->arg[y] = be64toh(ptr[x]);
720 
721 	for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
722 	    MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
723 		s_debug->arg[y] = be64toh(ptr[x]);
724 
725 	/* read RFC2863 counter group using predefined counter layout */
726 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
727 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
728 	for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
729 		s_debug->arg[y] = be64toh(ptr[x]);
730 
731 	/* read physical layer stats counter group using predefined counter layout */
732 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
733 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
734 	for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
735 		s_debug->arg[y] = be64toh(ptr[x]);
736 
737 	/* read Extended Ethernet counter group using predefined counter layout */
738 	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
739 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
740 	for (x = 0; x != MLX5E_PPORT_ETHERNET_EXTENDED_STATS_DEBUG_NUM; x++, y++)
741 		s_debug->arg[y] = be64toh(ptr[x]);
742 
743 	/* read Extended Statistical Group */
744 	if (MLX5_CAP_GEN(mdev, pcam_reg) &&
745 	    MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) &&
746 	    MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) {
747 		/* read Extended Statistical counter group using predefined counter layout */
748 		MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
749 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
750 
751 		for (x = 0; x != MLX5E_PPORT_STATISTICAL_DEBUG_NUM; x++, y++)
752 			s_debug->arg[y] = be64toh(ptr[x]);
753 	}
754 
755 	/* read PCIE counters */
756 	mlx5e_update_pcie_counters(priv);
757 
758 	/* read per-priority counters */
759 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
760 
761 	/* iterate all the priorities */
762 	for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
763 		MLX5_SET(ppcnt_reg, in, prio_tc, z);
764 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
765 
766 		/* read per priority stats counter group using predefined counter layout */
767 		for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
768 		    MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
769 			s->arg[y] = be64toh(ptr[x]);
770 	}
771 
772 free_out:
773 	/* free firmware request structures */
774 	kvfree(in);
775 	kvfree(out);
776 }
777 
778 static void
779 mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
780 {
781 	u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
782 	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
783 
784 	if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
785 		return;
786 
787 	MLX5_SET(query_vnic_env_in, in, opcode,
788 	    MLX5_CMD_OP_QUERY_VNIC_ENV);
789 	MLX5_SET(query_vnic_env_in, in, op_mod, 0);
790 	MLX5_SET(query_vnic_env_in, in, other_vport, 0);
791 
792 	if (mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out)) != 0)
793 		return;
794 
795 	priv->stats.vport.rx_steer_missed_packets =
796 	    MLX5_GET64(query_vnic_env_out, out,
797 	    vport_env.nic_receive_steering_discard);
798 }
799 
800 /*
801  * This function is called regularly to collect all statistics
802  * counters from the firmware. The values can be viewed through the
803  * sysctl interface. Execution is serialized using the priv's global
804  * configuration lock.
805  */
806 static void
807 mlx5e_update_stats_locked(struct mlx5e_priv *priv)
808 {
809 	struct mlx5_core_dev *mdev = priv->mdev;
810 	struct mlx5e_vport_stats *s = &priv->stats.vport;
811 	struct mlx5e_sq_stats *sq_stats;
812 #if (__FreeBSD_version < 1100000)
813 	struct ifnet *ifp = priv->ifp;
814 #endif
815 
816 	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
817 	u32 *out;
818 	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
819 	u64 tso_packets = 0;
820 	u64 tso_bytes = 0;
821 	u64 tx_queue_dropped = 0;
822 	u64 tx_defragged = 0;
823 	u64 tx_offload_none = 0;
824 	u64 lro_packets = 0;
825 	u64 lro_bytes = 0;
826 	u64 sw_lro_queued = 0;
827 	u64 sw_lro_flushed = 0;
828 	u64 rx_csum_none = 0;
829 	u64 rx_wqe_err = 0;
830 	u64 rx_packets = 0;
831 	u64 rx_bytes = 0;
832 	u32 rx_out_of_buffer = 0;
833 	int error;
834 	int i;
835 	int j;
836 
837 	out = mlx5_vzalloc(outlen);
838 	if (out == NULL)
839 		goto free_out;
840 
841 	/* Collect firts the SW counters and then HW for consistency */
842 	for (i = 0; i < priv->params.num_channels; i++) {
843 		struct mlx5e_channel *pch = priv->channel + i;
844 		struct mlx5e_rq *rq = &pch->rq;
845 		struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
846 
847 		/* collect stats from LRO */
848 		rq_stats->sw_lro_queued = rq->lro.lro_queued;
849 		rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
850 		sw_lro_queued += rq_stats->sw_lro_queued;
851 		sw_lro_flushed += rq_stats->sw_lro_flushed;
852 		lro_packets += rq_stats->lro_packets;
853 		lro_bytes += rq_stats->lro_bytes;
854 		rx_csum_none += rq_stats->csum_none;
855 		rx_wqe_err += rq_stats->wqe_err;
856 		rx_packets += rq_stats->packets;
857 		rx_bytes += rq_stats->bytes;
858 
859 		for (j = 0; j < priv->num_tc; j++) {
860 			sq_stats = &pch->sq[j].stats;
861 
862 			tso_packets += sq_stats->tso_packets;
863 			tso_bytes += sq_stats->tso_bytes;
864 			tx_queue_dropped += sq_stats->dropped;
865 			tx_queue_dropped += sq_stats->enobuf;
866 			tx_defragged += sq_stats->defragged;
867 			tx_offload_none += sq_stats->csum_offload_none;
868 		}
869 	}
870 
871 #ifdef RATELIMIT
872 	/* Collect statistics from all rate-limit queues */
873 	for (j = 0; j < priv->rl.param.tx_worker_threads_def; j++) {
874 		struct mlx5e_rl_worker *rlw = priv->rl.workers + j;
875 
876 		for (i = 0; i < priv->rl.param.tx_channels_per_worker_def; i++) {
877 			struct mlx5e_rl_channel *channel = rlw->channels + i;
878 			struct mlx5e_sq *sq = channel->sq;
879 
880 			if (sq == NULL)
881 				continue;
882 
883 			sq_stats = &sq->stats;
884 
885 			tso_packets += sq_stats->tso_packets;
886 			tso_bytes += sq_stats->tso_bytes;
887 			tx_queue_dropped += sq_stats->dropped;
888 			tx_queue_dropped += sq_stats->enobuf;
889 			tx_defragged += sq_stats->defragged;
890 			tx_offload_none += sq_stats->csum_offload_none;
891 		}
892 	}
893 #endif
894 
895 	/* update counters */
896 	s->tso_packets = tso_packets;
897 	s->tso_bytes = tso_bytes;
898 	s->tx_queue_dropped = tx_queue_dropped;
899 	s->tx_defragged = tx_defragged;
900 	s->lro_packets = lro_packets;
901 	s->lro_bytes = lro_bytes;
902 	s->sw_lro_queued = sw_lro_queued;
903 	s->sw_lro_flushed = sw_lro_flushed;
904 	s->rx_csum_none = rx_csum_none;
905 	s->rx_wqe_err = rx_wqe_err;
906 	s->rx_packets = rx_packets;
907 	s->rx_bytes = rx_bytes;
908 
909 	mlx5e_grp_vnic_env_update_stats(priv);
910 
911 	/* HW counters */
912 	memset(in, 0, sizeof(in));
913 
914 	MLX5_SET(query_vport_counter_in, in, opcode,
915 	    MLX5_CMD_OP_QUERY_VPORT_COUNTER);
916 	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
917 	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
918 
919 	memset(out, 0, outlen);
920 
921 	/* get number of out-of-buffer drops first */
922 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
923 	    mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
924 	    &rx_out_of_buffer) == 0) {
925 		s->rx_out_of_buffer = rx_out_of_buffer;
926 	}
927 
928 	/* get port statistics */
929 	if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen) == 0) {
930 #define	MLX5_GET_CTR(out, x) \
931 	MLX5_GET64(query_vport_counter_out, out, x)
932 
933 		s->rx_error_packets =
934 		    MLX5_GET_CTR(out, received_errors.packets);
935 		s->rx_error_bytes =
936 		    MLX5_GET_CTR(out, received_errors.octets);
937 		s->tx_error_packets =
938 		    MLX5_GET_CTR(out, transmit_errors.packets);
939 		s->tx_error_bytes =
940 		    MLX5_GET_CTR(out, transmit_errors.octets);
941 
942 		s->rx_unicast_packets =
943 		    MLX5_GET_CTR(out, received_eth_unicast.packets);
944 		s->rx_unicast_bytes =
945 		    MLX5_GET_CTR(out, received_eth_unicast.octets);
946 		s->tx_unicast_packets =
947 		    MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
948 		s->tx_unicast_bytes =
949 		    MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
950 
951 		s->rx_multicast_packets =
952 		    MLX5_GET_CTR(out, received_eth_multicast.packets);
953 		s->rx_multicast_bytes =
954 		    MLX5_GET_CTR(out, received_eth_multicast.octets);
955 		s->tx_multicast_packets =
956 		    MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
957 		s->tx_multicast_bytes =
958 		    MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
959 
960 		s->rx_broadcast_packets =
961 		    MLX5_GET_CTR(out, received_eth_broadcast.packets);
962 		s->rx_broadcast_bytes =
963 		    MLX5_GET_CTR(out, received_eth_broadcast.octets);
964 		s->tx_broadcast_packets =
965 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
966 		s->tx_broadcast_bytes =
967 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
968 
969 		s->tx_packets = s->tx_unicast_packets +
970 		    s->tx_multicast_packets + s->tx_broadcast_packets;
971 		s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes +
972 		    s->tx_broadcast_bytes;
973 
974 		/* Update calculated offload counters */
975 		s->tx_csum_offload = s->tx_packets - tx_offload_none;
976 		s->rx_csum_good = s->rx_packets - s->rx_csum_none;
977 	}
978 
979 	/* Get physical port counters */
980 	mlx5e_update_pport_counters(priv);
981 
982 	s->tx_jumbo_packets =
983 	    priv->stats.port_stats_debug.tx_stat_p1519to2047octets +
984 	    priv->stats.port_stats_debug.tx_stat_p2048to4095octets +
985 	    priv->stats.port_stats_debug.tx_stat_p4096to8191octets +
986 	    priv->stats.port_stats_debug.tx_stat_p8192to10239octets;
987 
988 #if (__FreeBSD_version < 1100000)
989 	/* no get_counters interface in fbsd 10 */
990 	ifp->if_ipackets = s->rx_packets;
991 	ifp->if_ierrors = priv->stats.pport.in_range_len_errors +
992 	    priv->stats.pport.out_of_range_len +
993 	    priv->stats.pport.too_long_errors +
994 	    priv->stats.pport.check_seq_err +
995 	    priv->stats.pport.alignment_err;
996 	ifp->if_iqdrops = s->rx_out_of_buffer;
997 	ifp->if_opackets = s->tx_packets;
998 	ifp->if_oerrors = priv->stats.port_stats_debug.out_discards;
999 	ifp->if_snd.ifq_drops = s->tx_queue_dropped;
1000 	ifp->if_ibytes = s->rx_bytes;
1001 	ifp->if_obytes = s->tx_bytes;
1002 	ifp->if_collisions =
1003 	    priv->stats.pport.collisions;
1004 #endif
1005 
1006 free_out:
1007 	kvfree(out);
1008 
1009 	/* Update diagnostics, if any */
1010 	if (priv->params_ethtool.diag_pci_enable ||
1011 	    priv->params_ethtool.diag_general_enable) {
1012 		error = mlx5_core_get_diagnostics_full(mdev,
1013 		    priv->params_ethtool.diag_pci_enable ? &priv->params_pci : NULL,
1014 		    priv->params_ethtool.diag_general_enable ? &priv->params_general : NULL);
1015 		if (error != 0)
1016 			mlx5_en_err(priv->ifp,
1017 			    "Failed reading diagnostics: %d\n", error);
1018 	}
1019 
1020 	/* Update FEC, if any */
1021 	error = mlx5e_fec_update(priv);
1022 	if (error != 0 && error != EOPNOTSUPP) {
1023 		mlx5_en_err(priv->ifp,
1024 		    "Updating FEC failed: %d\n", error);
1025 	}
1026 
1027 	/* Update temperature, if any */
1028 	if (priv->params_ethtool.hw_num_temp != 0) {
1029 		error = mlx5e_hw_temperature_update(priv);
1030 		if (error != 0 && error != EOPNOTSUPP) {
1031 			mlx5_en_err(priv->ifp,
1032 			    "Updating temperature failed: %d\n", error);
1033 		}
1034 	}
1035 }
1036 
1037 static void
1038 mlx5e_update_stats_work(struct work_struct *work)
1039 {
1040 	struct mlx5e_priv *priv;
1041 
1042 	priv = container_of(work, struct mlx5e_priv, update_stats_work);
1043 	PRIV_LOCK(priv);
1044 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
1045 	    !test_bit(MLX5_INTERFACE_STATE_TEARDOWN, &priv->mdev->intf_state))
1046 		mlx5e_update_stats_locked(priv);
1047 	PRIV_UNLOCK(priv);
1048 }
1049 
1050 static void
1051 mlx5e_update_stats(void *arg)
1052 {
1053 	struct mlx5e_priv *priv = arg;
1054 
1055 	queue_work(priv->wq, &priv->update_stats_work);
1056 
1057 	callout_reset(&priv->watchdog, hz / 4, &mlx5e_update_stats, priv);
1058 }
1059 
1060 static void
1061 mlx5e_async_event_sub(struct mlx5e_priv *priv,
1062     enum mlx5_dev_event event)
1063 {
1064 	switch (event) {
1065 	case MLX5_DEV_EVENT_PORT_UP:
1066 	case MLX5_DEV_EVENT_PORT_DOWN:
1067 		queue_work(priv->wq, &priv->update_carrier_work);
1068 		break;
1069 
1070 	default:
1071 		break;
1072 	}
1073 }
1074 
1075 static void
1076 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
1077     enum mlx5_dev_event event, unsigned long param)
1078 {
1079 	struct mlx5e_priv *priv = vpriv;
1080 
1081 	mtx_lock(&priv->async_events_mtx);
1082 	if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
1083 		mlx5e_async_event_sub(priv, event);
1084 	mtx_unlock(&priv->async_events_mtx);
1085 }
1086 
1087 static void
1088 mlx5e_enable_async_events(struct mlx5e_priv *priv)
1089 {
1090 	set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1091 }
1092 
1093 static void
1094 mlx5e_disable_async_events(struct mlx5e_priv *priv)
1095 {
1096 	mtx_lock(&priv->async_events_mtx);
1097 	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1098 	mtx_unlock(&priv->async_events_mtx);
1099 }
1100 
1101 static void mlx5e_calibration_callout(void *arg);
1102 static int mlx5e_calibration_duration = 20;
1103 static int mlx5e_fast_calibration = 1;
1104 static int mlx5e_normal_calibration = 30;
1105 
1106 static SYSCTL_NODE(_hw_mlx5, OID_AUTO, calibr, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1107     "MLX5 timestamp calibration parameteres");
1108 
1109 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, duration, CTLFLAG_RWTUN,
1110     &mlx5e_calibration_duration, 0,
1111     "Duration of initial calibration");
1112 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, fast, CTLFLAG_RWTUN,
1113     &mlx5e_fast_calibration, 0,
1114     "Recalibration interval during initial calibration");
1115 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, normal, CTLFLAG_RWTUN,
1116     &mlx5e_normal_calibration, 0,
1117     "Recalibration interval during normal operations");
1118 
1119 /*
1120  * Ignites the calibration process.
1121  */
1122 static void
1123 mlx5e_reset_calibration_callout(struct mlx5e_priv *priv)
1124 {
1125 
1126 	if (priv->clbr_done == 0)
1127 		mlx5e_calibration_callout(priv);
1128 	else
1129 		callout_reset_sbt_curcpu(&priv->tstmp_clbr, (priv->clbr_done <
1130 		    mlx5e_calibration_duration ? mlx5e_fast_calibration :
1131 		    mlx5e_normal_calibration) * SBT_1S, 0,
1132 		    mlx5e_calibration_callout, priv, C_DIRECT_EXEC);
1133 }
1134 
1135 static uint64_t
1136 mlx5e_timespec2usec(const struct timespec *ts)
1137 {
1138 
1139 	return ((uint64_t)ts->tv_sec * 1000000000 + ts->tv_nsec);
1140 }
1141 
1142 static uint64_t
1143 mlx5e_hw_clock(struct mlx5e_priv *priv)
1144 {
1145 	struct mlx5_init_seg *iseg;
1146 	uint32_t hw_h, hw_h1, hw_l;
1147 
1148 	iseg = priv->mdev->iseg;
1149 	do {
1150 		hw_h = ioread32be(&iseg->internal_timer_h);
1151 		hw_l = ioread32be(&iseg->internal_timer_l);
1152 		hw_h1 = ioread32be(&iseg->internal_timer_h);
1153 	} while (hw_h1 != hw_h);
1154 	return (((uint64_t)hw_h << 32) | hw_l);
1155 }
1156 
1157 /*
1158  * The calibration callout, it runs either in the context of the
1159  * thread which enables calibration, or in callout.  It takes the
1160  * snapshot of system and adapter clocks, then advances the pointers to
1161  * the calibration point to allow rx path to read the consistent data
1162  * lockless.
1163  */
1164 static void
1165 mlx5e_calibration_callout(void *arg)
1166 {
1167 	struct mlx5e_priv *priv;
1168 	struct mlx5e_clbr_point *next, *curr;
1169 	struct timespec ts;
1170 	int clbr_curr_next;
1171 
1172 	priv = arg;
1173 	curr = &priv->clbr_points[priv->clbr_curr];
1174 	clbr_curr_next = priv->clbr_curr + 1;
1175 	if (clbr_curr_next >= nitems(priv->clbr_points))
1176 		clbr_curr_next = 0;
1177 	next = &priv->clbr_points[clbr_curr_next];
1178 
1179 	next->base_prev = curr->base_curr;
1180 	next->clbr_hw_prev = curr->clbr_hw_curr;
1181 
1182 	next->clbr_hw_curr = mlx5e_hw_clock(priv);
1183 	if (((next->clbr_hw_curr - curr->clbr_hw_curr) >> MLX5E_TSTMP_PREC) ==
1184 	    0) {
1185 		if (priv->clbr_done != 0) {
1186 			mlx5_en_err(priv->ifp,
1187 			    "HW failed tstmp frozen %#jx %#jx, disabling\n",
1188 			     next->clbr_hw_curr, curr->clbr_hw_prev);
1189 			priv->clbr_done = 0;
1190 		}
1191 		atomic_store_rel_int(&curr->clbr_gen, 0);
1192 		return;
1193 	}
1194 
1195 	nanouptime(&ts);
1196 	next->base_curr = mlx5e_timespec2usec(&ts);
1197 
1198 	curr->clbr_gen = 0;
1199 	atomic_thread_fence_rel();
1200 	priv->clbr_curr = clbr_curr_next;
1201 	atomic_store_rel_int(&next->clbr_gen, ++(priv->clbr_gen));
1202 
1203 	if (priv->clbr_done < mlx5e_calibration_duration)
1204 		priv->clbr_done++;
1205 	mlx5e_reset_calibration_callout(priv);
1206 }
1207 
1208 static const char *mlx5e_rq_stats_desc[] = {
1209 	MLX5E_RQ_STATS(MLX5E_STATS_DESC)
1210 };
1211 
1212 static int
1213 mlx5e_create_rq(struct mlx5e_channel *c,
1214     struct mlx5e_rq_param *param,
1215     struct mlx5e_rq *rq)
1216 {
1217 	struct mlx5e_priv *priv = c->priv;
1218 	struct mlx5_core_dev *mdev = priv->mdev;
1219 	char buffer[16];
1220 	void *rqc = param->rqc;
1221 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1222 	int wq_sz;
1223 	int err;
1224 	int i;
1225 	u32 nsegs, wqe_sz;
1226 
1227 	err = mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
1228 	if (err != 0)
1229 		goto done;
1230 
1231 	/* Create DMA descriptor TAG */
1232 	if ((err = -bus_dma_tag_create(
1233 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1234 	    1,				/* any alignment */
1235 	    0,				/* no boundary */
1236 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1237 	    BUS_SPACE_MAXADDR,		/* highaddr */
1238 	    NULL, NULL,			/* filter, filterarg */
1239 	    nsegs * MLX5E_MAX_RX_BYTES,	/* maxsize */
1240 	    nsegs,			/* nsegments */
1241 	    nsegs * MLX5E_MAX_RX_BYTES,	/* maxsegsize */
1242 	    0,				/* flags */
1243 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1244 	    &rq->dma_tag)))
1245 		goto done;
1246 
1247 	err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1248 	    &rq->wq_ctrl);
1249 	if (err)
1250 		goto err_free_dma_tag;
1251 
1252 	rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1253 
1254 	err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1255 	if (err != 0)
1256 		goto err_rq_wq_destroy;
1257 
1258 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1259 
1260 	err = -tcp_lro_init_args(&rq->lro, priv->ifp, TCP_LRO_ENTRIES, wq_sz);
1261 	if (err)
1262 		goto err_rq_wq_destroy;
1263 
1264 	rq->mbuf = malloc_domainset(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN,
1265 	    mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
1266 	for (i = 0; i != wq_sz; i++) {
1267 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1268 		int j;
1269 
1270 		err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1271 		if (err != 0) {
1272 			while (i--)
1273 				bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1274 			goto err_rq_mbuf_free;
1275 		}
1276 
1277 		/* set value for constant fields */
1278 		for (j = 0; j < rq->nsegs; j++)
1279 			wqe->data[j].lkey = cpu_to_be32(priv->mr.key);
1280 	}
1281 
1282 	INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1283 	if (priv->params.rx_cq_moderation_mode < 2) {
1284 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1285 	} else {
1286 		void *cqc = container_of(param,
1287 		    struct mlx5e_channel_param, rq)->rx_cq.cqc;
1288 
1289 		switch (MLX5_GET(cqc, cqc, cq_period_mode)) {
1290 		case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
1291 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1292 			break;
1293 		case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
1294 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1295 			break;
1296 		default:
1297 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1298 			break;
1299 		}
1300 	}
1301 
1302 	rq->ifp = priv->ifp;
1303 	rq->channel = c;
1304 	rq->ix = c->ix;
1305 
1306 	snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
1307 	mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1308 	    buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
1309 	    rq->stats.arg);
1310 	return (0);
1311 
1312 err_rq_mbuf_free:
1313 	free(rq->mbuf, M_MLX5EN);
1314 	tcp_lro_free(&rq->lro);
1315 err_rq_wq_destroy:
1316 	mlx5_wq_destroy(&rq->wq_ctrl);
1317 err_free_dma_tag:
1318 	bus_dma_tag_destroy(rq->dma_tag);
1319 done:
1320 	return (err);
1321 }
1322 
1323 static void
1324 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1325 {
1326 	int wq_sz;
1327 	int i;
1328 
1329 	/* destroy all sysctl nodes */
1330 	sysctl_ctx_free(&rq->stats.ctx);
1331 
1332 	/* free leftover LRO packets, if any */
1333 	tcp_lro_free(&rq->lro);
1334 
1335 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1336 	for (i = 0; i != wq_sz; i++) {
1337 		if (rq->mbuf[i].mbuf != NULL) {
1338 			bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1339 			m_freem(rq->mbuf[i].mbuf);
1340 		}
1341 		bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1342 	}
1343 	free(rq->mbuf, M_MLX5EN);
1344 	mlx5_wq_destroy(&rq->wq_ctrl);
1345 	bus_dma_tag_destroy(rq->dma_tag);
1346 }
1347 
1348 static int
1349 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1350 {
1351 	struct mlx5e_channel *c = rq->channel;
1352 	struct mlx5e_priv *priv = c->priv;
1353 	struct mlx5_core_dev *mdev = priv->mdev;
1354 	void *in;
1355 	void *rqc;
1356 	void *wq;
1357 	int inlen;
1358 	int err;
1359 	u8 ts_format;
1360 
1361 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1362 	    sizeof(u64) * rq->wq_ctrl.buf.npages;
1363 	in = mlx5_vzalloc(inlen);
1364 	if (in == NULL)
1365 		return (-ENOMEM);
1366 
1367 	ts_format = mlx5_get_rq_default_ts(mdev);
1368 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1369 	wq = MLX5_ADDR_OF(rqc, rqc, wq);
1370 
1371 	memcpy(rqc, param->rqc, sizeof(param->rqc));
1372 
1373 	MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
1374 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1375 	MLX5_SET(rqc, rqc, ts_format, ts_format);
1376 	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1377 	if (priv->counter_set_id >= 0)
1378 		MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
1379 	MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1380 	    PAGE_SHIFT);
1381 	MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1382 
1383 	mlx5_fill_page_array(&rq->wq_ctrl.buf,
1384 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1385 
1386 	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1387 
1388 	kvfree(in);
1389 
1390 	return (err);
1391 }
1392 
1393 static int
1394 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1395 {
1396 	struct mlx5e_channel *c = rq->channel;
1397 	struct mlx5e_priv *priv = c->priv;
1398 	struct mlx5_core_dev *mdev = priv->mdev;
1399 
1400 	void *in;
1401 	void *rqc;
1402 	int inlen;
1403 	int err;
1404 
1405 	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1406 	in = mlx5_vzalloc(inlen);
1407 	if (in == NULL)
1408 		return (-ENOMEM);
1409 
1410 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1411 
1412 	MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1413 	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1414 	MLX5_SET(rqc, rqc, state, next_state);
1415 
1416 	err = mlx5_core_modify_rq(mdev, in, inlen);
1417 
1418 	kvfree(in);
1419 
1420 	return (err);
1421 }
1422 
1423 static void
1424 mlx5e_disable_rq(struct mlx5e_rq *rq)
1425 {
1426 	struct mlx5e_channel *c = rq->channel;
1427 	struct mlx5e_priv *priv = c->priv;
1428 	struct mlx5_core_dev *mdev = priv->mdev;
1429 
1430 	mlx5_core_destroy_rq(mdev, rq->rqn);
1431 }
1432 
1433 static int
1434 mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
1435 {
1436 	struct mlx5e_channel *c = rq->channel;
1437 	struct mlx5e_priv *priv = c->priv;
1438 	struct mlx5_wq_ll *wq = &rq->wq;
1439 	int i;
1440 
1441 	for (i = 0; i < 1000; i++) {
1442 		if (wq->cur_sz >= priv->params.min_rx_wqes)
1443 			return (0);
1444 
1445 		msleep(4);
1446 	}
1447 	return (-ETIMEDOUT);
1448 }
1449 
1450 static int
1451 mlx5e_open_rq(struct mlx5e_channel *c,
1452     struct mlx5e_rq_param *param,
1453     struct mlx5e_rq *rq)
1454 {
1455 	int err;
1456 
1457 	err = mlx5e_create_rq(c, param, rq);
1458 	if (err)
1459 		return (err);
1460 
1461 	err = mlx5e_enable_rq(rq, param);
1462 	if (err)
1463 		goto err_destroy_rq;
1464 
1465 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1466 	if (err)
1467 		goto err_disable_rq;
1468 
1469 	c->rq.enabled = 1;
1470 
1471 	return (0);
1472 
1473 err_disable_rq:
1474 	mlx5e_disable_rq(rq);
1475 err_destroy_rq:
1476 	mlx5e_destroy_rq(rq);
1477 
1478 	return (err);
1479 }
1480 
1481 static void
1482 mlx5e_close_rq(struct mlx5e_rq *rq)
1483 {
1484 	mtx_lock(&rq->mtx);
1485 	rq->enabled = 0;
1486 	callout_stop(&rq->watchdog);
1487 	mtx_unlock(&rq->mtx);
1488 
1489 	mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1490 }
1491 
1492 static void
1493 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1494 {
1495 
1496 	mlx5e_disable_rq(rq);
1497 	mlx5e_close_cq(&rq->cq);
1498 	cancel_work_sync(&rq->dim.work);
1499 	mlx5e_destroy_rq(rq);
1500 }
1501 
1502 void
1503 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1504 {
1505 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1506 	int x;
1507 
1508 	for (x = 0; x != wq_sz; x++) {
1509 		if (unlikely(sq->mbuf[x].p_refcount != NULL)) {
1510 			atomic_add_int(sq->mbuf[x].p_refcount, -1);
1511 			sq->mbuf[x].p_refcount = NULL;
1512 		}
1513 		if (sq->mbuf[x].mbuf != NULL) {
1514 			bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1515 			m_freem(sq->mbuf[x].mbuf);
1516 		}
1517 		bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1518 	}
1519 	free(sq->mbuf, M_MLX5EN);
1520 }
1521 
1522 int
1523 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1524 {
1525 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1526 	int err;
1527 	int x;
1528 
1529 	sq->mbuf = malloc_domainset(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN,
1530 	    mlx5_dev_domainset(sq->priv->mdev), M_WAITOK | M_ZERO);
1531 
1532 	/* Create DMA descriptor MAPs */
1533 	for (x = 0; x != wq_sz; x++) {
1534 		err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1535 		if (err != 0) {
1536 			while (x--)
1537 				bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1538 			free(sq->mbuf, M_MLX5EN);
1539 			return (err);
1540 		}
1541 	}
1542 	return (0);
1543 }
1544 
1545 static const char *mlx5e_sq_stats_desc[] = {
1546 	MLX5E_SQ_STATS(MLX5E_STATS_DESC)
1547 };
1548 
1549 void
1550 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1551 {
1552 	sq->max_inline = sq->priv->params.tx_max_inline;
1553 	sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1554 
1555 	/*
1556 	 * Check if trust state is DSCP or if inline mode is NONE which
1557 	 * indicates CX-5 or newer hardware.
1558 	 */
1559 	if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1560 	    sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1561 		if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1562 			sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1563 		else
1564 			sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1565 	} else {
1566 		sq->min_insert_caps = 0;
1567 	}
1568 }
1569 
1570 static void
1571 mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1572 {
1573 	int i;
1574 
1575 	for (i = 0; i != priv->num_tc; i++) {
1576 		mtx_lock(&c->sq[i].lock);
1577 		mlx5e_update_sq_inline(&c->sq[i]);
1578 		mtx_unlock(&c->sq[i].lock);
1579 	}
1580 }
1581 
1582 void
1583 mlx5e_refresh_sq_inline(struct mlx5e_priv *priv)
1584 {
1585 	int i;
1586 
1587 	/* check if channels are closed */
1588 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
1589 		return;
1590 
1591 	for (i = 0; i < priv->params.num_channels; i++)
1592 		mlx5e_refresh_sq_inline_sub(priv, &priv->channel[i]);
1593 }
1594 
1595 static int
1596 mlx5e_create_sq(struct mlx5e_channel *c,
1597     int tc,
1598     struct mlx5e_sq_param *param,
1599     struct mlx5e_sq *sq)
1600 {
1601 	struct mlx5e_priv *priv = c->priv;
1602 	struct mlx5_core_dev *mdev = priv->mdev;
1603 	char buffer[16];
1604 	void *sqc = param->sqc;
1605 	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
1606 	int err;
1607 
1608 	/* Create DMA descriptor TAG */
1609 	if ((err = -bus_dma_tag_create(
1610 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1611 	    1,				/* any alignment */
1612 	    0,				/* no boundary */
1613 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1614 	    BUS_SPACE_MAXADDR,		/* highaddr */
1615 	    NULL, NULL,			/* filter, filterarg */
1616 	    MLX5E_MAX_TX_PAYLOAD_SIZE,	/* maxsize */
1617 	    MLX5E_MAX_TX_MBUF_FRAGS,	/* nsegments */
1618 	    MLX5E_MAX_TX_MBUF_SIZE,	/* maxsegsize */
1619 	    0,				/* flags */
1620 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1621 	    &sq->dma_tag)))
1622 		goto done;
1623 
1624 	sq->mkey_be = cpu_to_be32(priv->mr.key);
1625 	sq->ifp = priv->ifp;
1626 	sq->priv = priv;
1627 	sq->tc = tc;
1628 
1629 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
1630 	    &sq->wq_ctrl);
1631 	if (err)
1632 		goto err_free_dma_tag;
1633 
1634 	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1635 
1636 	err = mlx5e_alloc_sq_db(sq);
1637 	if (err)
1638 		goto err_sq_wq_destroy;
1639 
1640 	mlx5e_update_sq_inline(sq);
1641 
1642 	snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1643 	mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1644 	    buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1645 	    sq->stats.arg);
1646 
1647 	return (0);
1648 
1649 err_sq_wq_destroy:
1650 	mlx5_wq_destroy(&sq->wq_ctrl);
1651 
1652 err_free_dma_tag:
1653 	bus_dma_tag_destroy(sq->dma_tag);
1654 done:
1655 	return (err);
1656 }
1657 
1658 static void
1659 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1660 {
1661 	/* destroy all sysctl nodes */
1662 	sysctl_ctx_free(&sq->stats.ctx);
1663 
1664 	mlx5e_free_sq_db(sq);
1665 	mlx5_wq_destroy(&sq->wq_ctrl);
1666 	bus_dma_tag_destroy(sq->dma_tag);
1667 }
1668 
1669 int
1670 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1671     const struct mlx5_sq_bfreg *bfreg, int tis_num)
1672 {
1673 	void *in;
1674 	void *sqc;
1675 	void *wq;
1676 	int inlen;
1677 	int err;
1678 	u8 ts_format;
1679 
1680 	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1681 	    sizeof(u64) * sq->wq_ctrl.buf.npages;
1682 	in = mlx5_vzalloc(inlen);
1683 	if (in == NULL)
1684 		return (-ENOMEM);
1685 
1686 	sq->uar_map = bfreg->map;
1687 
1688 	ts_format = mlx5_get_sq_default_ts(sq->priv->mdev);
1689 	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1690 	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1691 
1692 	memcpy(sqc, param->sqc, sizeof(param->sqc));
1693 
1694 	MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1695 	MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1696 	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1697 	MLX5_SET(sqc, sqc, ts_format, ts_format);
1698 	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1699 	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1700 	MLX5_SET(sqc, sqc, allow_swp, 1);
1701 
1702 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1703 	MLX5_SET(wq, wq, uar_page, bfreg->index);
1704 	MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1705 	    PAGE_SHIFT);
1706 	MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1707 
1708 	mlx5_fill_page_array(&sq->wq_ctrl.buf,
1709 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1710 
1711 	err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1712 
1713 	kvfree(in);
1714 
1715 	return (err);
1716 }
1717 
1718 int
1719 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1720 {
1721 	void *in;
1722 	void *sqc;
1723 	int inlen;
1724 	int err;
1725 
1726 	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1727 	in = mlx5_vzalloc(inlen);
1728 	if (in == NULL)
1729 		return (-ENOMEM);
1730 
1731 	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1732 
1733 	MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1734 	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1735 	MLX5_SET(sqc, sqc, state, next_state);
1736 
1737 	err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1738 
1739 	kvfree(in);
1740 
1741 	return (err);
1742 }
1743 
1744 void
1745 mlx5e_disable_sq(struct mlx5e_sq *sq)
1746 {
1747 
1748 	mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1749 }
1750 
1751 static int
1752 mlx5e_open_sq(struct mlx5e_channel *c,
1753     int tc,
1754     struct mlx5e_sq_param *param,
1755     struct mlx5e_sq *sq)
1756 {
1757 	int err;
1758 
1759 	sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1760 
1761 	/* ensure the TX completion event factor is not zero */
1762 	if (sq->cev_factor == 0)
1763 		sq->cev_factor = 1;
1764 
1765 	err = mlx5e_create_sq(c, tc, param, sq);
1766 	if (err)
1767 		return (err);
1768 
1769 	err = mlx5e_enable_sq(sq, param, &c->bfreg, c->priv->tisn[tc]);
1770 	if (err)
1771 		goto err_destroy_sq;
1772 
1773 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1774 	if (err)
1775 		goto err_disable_sq;
1776 
1777 	WRITE_ONCE(sq->running, 1);
1778 
1779 	return (0);
1780 
1781 err_disable_sq:
1782 	mlx5e_disable_sq(sq);
1783 err_destroy_sq:
1784 	mlx5e_destroy_sq(sq);
1785 
1786 	return (err);
1787 }
1788 
1789 static void
1790 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1791 {
1792 	/* fill up remainder with NOPs */
1793 	while (sq->cev_counter != 0) {
1794 		while (!mlx5e_sq_has_room_for(sq, 1)) {
1795 			if (can_sleep != 0) {
1796 				mtx_unlock(&sq->lock);
1797 				msleep(4);
1798 				mtx_lock(&sq->lock);
1799 			} else {
1800 				goto done;
1801 			}
1802 		}
1803 		/* send a single NOP */
1804 		mlx5e_send_nop(sq, 1);
1805 		atomic_thread_fence_rel();
1806 	}
1807 done:
1808 	/* Check if we need to write the doorbell */
1809 	if (likely(sq->doorbell.d64 != 0)) {
1810 		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
1811 		sq->doorbell.d64 = 0;
1812 	}
1813 }
1814 
1815 void
1816 mlx5e_sq_cev_timeout(void *arg)
1817 {
1818 	struct mlx5e_sq *sq = arg;
1819 
1820 	mtx_assert(&sq->lock, MA_OWNED);
1821 
1822 	/* check next state */
1823 	switch (sq->cev_next_state) {
1824 	case MLX5E_CEV_STATE_SEND_NOPS:
1825 		/* fill TX ring with NOPs, if any */
1826 		mlx5e_sq_send_nops_locked(sq, 0);
1827 
1828 		/* check if completed */
1829 		if (sq->cev_counter == 0) {
1830 			sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1831 			return;
1832 		}
1833 		break;
1834 	default:
1835 		/* send NOPs on next timeout */
1836 		sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1837 		break;
1838 	}
1839 
1840 	/* restart timer */
1841 	callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1842 }
1843 
1844 void
1845 mlx5e_drain_sq(struct mlx5e_sq *sq)
1846 {
1847 	int error;
1848 	struct mlx5_core_dev *mdev= sq->priv->mdev;
1849 
1850 	/*
1851 	 * Check if already stopped.
1852 	 *
1853 	 * NOTE: Serialization of this function is managed by the
1854 	 * caller ensuring the priv's state lock is locked or in case
1855 	 * of rate limit support, a single thread manages drain and
1856 	 * resume of SQs. The "running" variable can therefore safely
1857 	 * be read without any locks.
1858 	 */
1859 	if (READ_ONCE(sq->running) == 0)
1860 		return;
1861 
1862 	/* don't put more packets into the SQ */
1863 	WRITE_ONCE(sq->running, 0);
1864 
1865 	/* serialize access to DMA rings */
1866 	mtx_lock(&sq->lock);
1867 
1868 	/* teardown event factor timer, if any */
1869 	sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1870 	callout_stop(&sq->cev_callout);
1871 
1872 	/* send dummy NOPs in order to flush the transmit ring */
1873 	mlx5e_sq_send_nops_locked(sq, 1);
1874 	mtx_unlock(&sq->lock);
1875 
1876 	/* wait till SQ is empty or link is down */
1877 	mtx_lock(&sq->lock);
1878 	while (sq->cc != sq->pc &&
1879 	    (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1880 	    mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1881 	    pci_channel_offline(mdev->pdev) == 0) {
1882 		mtx_unlock(&sq->lock);
1883 		msleep(1);
1884 		sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1885 		mtx_lock(&sq->lock);
1886 	}
1887 	mtx_unlock(&sq->lock);
1888 
1889 	/* error out remaining requests */
1890 	error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1891 	if (error != 0) {
1892 		mlx5_en_err(sq->ifp,
1893 		    "mlx5e_modify_sq() from RDY to ERR failed: %d\n", error);
1894 	}
1895 
1896 	/* wait till SQ is empty */
1897 	mtx_lock(&sq->lock);
1898 	while (sq->cc != sq->pc &&
1899 	       mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1900 	       pci_channel_offline(mdev->pdev) == 0) {
1901 		mtx_unlock(&sq->lock);
1902 		msleep(1);
1903 		sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1904 		mtx_lock(&sq->lock);
1905 	}
1906 	mtx_unlock(&sq->lock);
1907 }
1908 
1909 static void
1910 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1911 {
1912 
1913 	mlx5e_drain_sq(sq);
1914 	mlx5e_disable_sq(sq);
1915 	mlx5e_destroy_sq(sq);
1916 }
1917 
1918 static int
1919 mlx5e_create_cq(struct mlx5e_priv *priv,
1920     struct mlx5e_cq_param *param,
1921     struct mlx5e_cq *cq,
1922     mlx5e_cq_comp_t *comp,
1923     int eq_ix)
1924 {
1925 	struct mlx5_core_dev *mdev = priv->mdev;
1926 	struct mlx5_core_cq *mcq = &cq->mcq;
1927 	int eqn_not_used;
1928 	int irqn;
1929 	int err;
1930 	u32 i;
1931 
1932 	err = mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1933 	if (err)
1934 		return (err);
1935 
1936 	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1937 	    &cq->wq_ctrl);
1938 	if (err)
1939 		return (err);
1940 
1941 	mcq->cqe_sz = 64;
1942 	mcq->set_ci_db = cq->wq_ctrl.db.db;
1943 	mcq->arm_db = cq->wq_ctrl.db.db + 1;
1944 	*mcq->set_ci_db = 0;
1945 	*mcq->arm_db = 0;
1946 	mcq->vector = eq_ix;
1947 	mcq->comp = comp;
1948 	mcq->event = mlx5e_cq_error_event;
1949 	mcq->irqn = irqn;
1950 
1951 	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1952 		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1953 
1954 		cqe->op_own = 0xf1;
1955 	}
1956 
1957 	cq->priv = priv;
1958 
1959 	return (0);
1960 }
1961 
1962 static void
1963 mlx5e_destroy_cq(struct mlx5e_cq *cq)
1964 {
1965 	mlx5_wq_destroy(&cq->wq_ctrl);
1966 }
1967 
1968 static int
1969 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
1970 {
1971 	struct mlx5_core_cq *mcq = &cq->mcq;
1972 	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
1973 	void *in;
1974 	void *cqc;
1975 	int inlen;
1976 	int irqn_not_used;
1977 	int eqn;
1978 	int err;
1979 
1980 	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1981 	    sizeof(u64) * cq->wq_ctrl.buf.npages;
1982 	in = mlx5_vzalloc(inlen);
1983 	if (in == NULL)
1984 		return (-ENOMEM);
1985 
1986 	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1987 
1988 	memcpy(cqc, param->cqc, sizeof(param->cqc));
1989 
1990 	mlx5_fill_page_array(&cq->wq_ctrl.buf,
1991 	    (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
1992 
1993 	mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
1994 
1995 	MLX5_SET(cqc, cqc, c_eqn, eqn);
1996 	MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1997 	    PAGE_SHIFT);
1998 	MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
1999 
2000 	err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen, out, sizeof(out));
2001 
2002 	kvfree(in);
2003 
2004 	if (err)
2005 		return (err);
2006 
2007 	mlx5e_cq_arm(cq, MLX5_GET_DOORBELL_LOCK(&cq->priv->doorbell_lock));
2008 
2009 	return (0);
2010 }
2011 
2012 static void
2013 mlx5e_disable_cq(struct mlx5e_cq *cq)
2014 {
2015 
2016 	mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
2017 }
2018 
2019 int
2020 mlx5e_open_cq(struct mlx5e_priv *priv,
2021     struct mlx5e_cq_param *param,
2022     struct mlx5e_cq *cq,
2023     mlx5e_cq_comp_t *comp,
2024     int eq_ix)
2025 {
2026 	int err;
2027 
2028 	err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
2029 	if (err)
2030 		return (err);
2031 
2032 	err = mlx5e_enable_cq(cq, param, eq_ix);
2033 	if (err)
2034 		goto err_destroy_cq;
2035 
2036 	return (0);
2037 
2038 err_destroy_cq:
2039 	mlx5e_destroy_cq(cq);
2040 
2041 	return (err);
2042 }
2043 
2044 void
2045 mlx5e_close_cq(struct mlx5e_cq *cq)
2046 {
2047 	mlx5e_disable_cq(cq);
2048 	mlx5e_destroy_cq(cq);
2049 }
2050 
2051 static int
2052 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2053     struct mlx5e_channel_param *cparam)
2054 {
2055 	int err;
2056 	int tc;
2057 
2058 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2059 		/* open completion queue */
2060 		err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
2061 		    &mlx5e_tx_cq_comp, c->ix);
2062 		if (err)
2063 			goto err_close_tx_cqs;
2064 	}
2065 	return (0);
2066 
2067 err_close_tx_cqs:
2068 	for (tc--; tc >= 0; tc--)
2069 		mlx5e_close_cq(&c->sq[tc].cq);
2070 
2071 	return (err);
2072 }
2073 
2074 static void
2075 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2076 {
2077 	int tc;
2078 
2079 	for (tc = 0; tc < c->priv->num_tc; tc++)
2080 		mlx5e_close_cq(&c->sq[tc].cq);
2081 }
2082 
2083 static int
2084 mlx5e_open_sqs(struct mlx5e_channel *c,
2085     struct mlx5e_channel_param *cparam)
2086 {
2087 	int err;
2088 	int tc;
2089 
2090 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2091 		err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
2092 		if (err)
2093 			goto err_close_sqs;
2094 	}
2095 
2096 	return (0);
2097 
2098 err_close_sqs:
2099 	for (tc--; tc >= 0; tc--)
2100 		mlx5e_close_sq_wait(&c->sq[tc]);
2101 
2102 	return (err);
2103 }
2104 
2105 static void
2106 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
2107 {
2108 	int tc;
2109 
2110 	for (tc = 0; tc < c->priv->num_tc; tc++)
2111 		mlx5e_close_sq_wait(&c->sq[tc]);
2112 }
2113 
2114 static void
2115 mlx5e_chan_static_init(struct mlx5e_priv *priv, struct mlx5e_channel *c, int ix)
2116 {
2117 	int tc;
2118 
2119 	/* setup priv and channel number */
2120 	c->priv = priv;
2121 	c->ix = ix;
2122 
2123 	/* setup send tag */
2124 	m_snd_tag_init(&c->tag, c->priv->ifp, &mlx5e_ul_snd_tag_sw);
2125 
2126 	init_completion(&c->completion);
2127 
2128 	mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
2129 
2130 	callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
2131 
2132 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2133 		struct mlx5e_sq *sq = c->sq + tc;
2134 
2135 		mtx_init(&sq->lock, "mlx5tx",
2136 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2137 		mtx_init(&sq->comp_lock, "mlx5comp",
2138 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2139 
2140 		callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2141 	}
2142 }
2143 
2144 static void
2145 mlx5e_chan_wait_for_completion(struct mlx5e_channel *c)
2146 {
2147 
2148 	m_snd_tag_rele(&c->tag);
2149 	wait_for_completion(&c->completion);
2150 }
2151 
2152 static void
2153 mlx5e_priv_wait_for_completion(struct mlx5e_priv *priv, const uint32_t channels)
2154 {
2155 	uint32_t x;
2156 
2157 	for (x = 0; x != channels; x++)
2158 		mlx5e_chan_wait_for_completion(&priv->channel[x]);
2159 }
2160 
2161 static void
2162 mlx5e_chan_static_destroy(struct mlx5e_channel *c)
2163 {
2164 	int tc;
2165 
2166 	callout_drain(&c->rq.watchdog);
2167 
2168 	mtx_destroy(&c->rq.mtx);
2169 
2170 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2171 		callout_drain(&c->sq[tc].cev_callout);
2172 		mtx_destroy(&c->sq[tc].lock);
2173 		mtx_destroy(&c->sq[tc].comp_lock);
2174 	}
2175 }
2176 
2177 static int
2178 mlx5e_open_channel(struct mlx5e_priv *priv,
2179     struct mlx5e_channel_param *cparam,
2180     struct mlx5e_channel *c)
2181 {
2182 	struct epoch_tracker et;
2183 	int i, err;
2184 
2185 	/* zero non-persistant data */
2186 	MLX5E_ZERO(&c->rq, mlx5e_rq_zero_start);
2187 	for (i = 0; i != priv->num_tc; i++)
2188 		MLX5E_ZERO(&c->sq[i], mlx5e_sq_zero_start);
2189 
2190 	/* open transmit completion queue */
2191 	err = mlx5e_open_tx_cqs(c, cparam);
2192 	if (err)
2193 		goto err_free;
2194 
2195 	/* open receive completion queue */
2196 	err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2197 	    &mlx5e_rx_cq_comp, c->ix);
2198 	if (err)
2199 		goto err_close_tx_cqs;
2200 
2201 	err = mlx5e_open_sqs(c, cparam);
2202 	if (err)
2203 		goto err_close_rx_cq;
2204 
2205 	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2206 	if (err)
2207 		goto err_close_sqs;
2208 
2209 	/* poll receive queue initially */
2210 	NET_EPOCH_ENTER(et);
2211 	c->rq.cq.mcq.comp(&c->rq.cq.mcq, NULL);
2212 	NET_EPOCH_EXIT(et);
2213 
2214 	return (0);
2215 
2216 err_close_sqs:
2217 	mlx5e_close_sqs_wait(c);
2218 
2219 err_close_rx_cq:
2220 	mlx5e_close_cq(&c->rq.cq);
2221 
2222 err_close_tx_cqs:
2223 	mlx5e_close_tx_cqs(c);
2224 
2225 err_free:
2226 	return (err);
2227 }
2228 
2229 static void
2230 mlx5e_close_channel(struct mlx5e_channel *c)
2231 {
2232 	mlx5e_close_rq(&c->rq);
2233 }
2234 
2235 static void
2236 mlx5e_close_channel_wait(struct mlx5e_channel *c)
2237 {
2238 	mlx5e_close_rq_wait(&c->rq);
2239 	mlx5e_close_sqs_wait(c);
2240 	mlx5e_close_tx_cqs(c);
2241 }
2242 
2243 static int
2244 mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs)
2245 {
2246 	u32 r, n;
2247 
2248 	r = priv->params.hw_lro_en ? priv->params.lro_wqe_sz :
2249 	    MLX5E_SW2MB_MTU(priv->ifp->if_mtu);
2250 	if (r > MJUM16BYTES)
2251 		return (-ENOMEM);
2252 
2253 	if (r > MJUM9BYTES)
2254 		r = MJUM16BYTES;
2255 	else if (r > MJUMPAGESIZE)
2256 		r = MJUM9BYTES;
2257 	else if (r > MCLBYTES)
2258 		r = MJUMPAGESIZE;
2259 	else
2260 		r = MCLBYTES;
2261 
2262 	/*
2263 	 * n + 1 must be a power of two, because stride size must be.
2264 	 * Stride size is 16 * (n + 1), as the first segment is
2265 	 * control.
2266 	 */
2267 	for (n = howmany(r, MLX5E_MAX_RX_BYTES); !powerof2(n + 1); n++)
2268 		;
2269 
2270 	if (n > MLX5E_MAX_BUSDMA_RX_SEGS)
2271 		return (-ENOMEM);
2272 
2273 	*wqe_sz = r;
2274 	*nsegs = n;
2275 	return (0);
2276 }
2277 
2278 static void
2279 mlx5e_build_rq_param(struct mlx5e_priv *priv,
2280     struct mlx5e_rq_param *param)
2281 {
2282 	void *rqc = param->rqc;
2283 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2284 	u32 wqe_sz, nsegs;
2285 
2286 	mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
2287 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
2288 	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2289 	MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) +
2290 	    nsegs * sizeof(struct mlx5_wqe_data_seg)));
2291 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
2292 	MLX5_SET(wq, wq, pd, priv->pdn);
2293 
2294 	param->wq.linear = 1;
2295 }
2296 
2297 static void
2298 mlx5e_build_sq_param(struct mlx5e_priv *priv,
2299     struct mlx5e_sq_param *param)
2300 {
2301 	void *sqc = param->sqc;
2302 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2303 
2304 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
2305 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2306 	MLX5_SET(wq, wq, pd, priv->pdn);
2307 
2308 	param->wq.linear = 1;
2309 }
2310 
2311 static void
2312 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2313     struct mlx5e_cq_param *param)
2314 {
2315 	void *cqc = param->cqc;
2316 
2317 	MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
2318 }
2319 
2320 static void
2321 mlx5e_get_default_profile(struct mlx5e_priv *priv, int mode, struct net_dim_cq_moder *ptr)
2322 {
2323 
2324 	*ptr = net_dim_get_profile(mode, MLX5E_DIM_DEFAULT_PROFILE);
2325 
2326 	/* apply LRO restrictions */
2327 	if (priv->params.hw_lro_en &&
2328 	    ptr->pkts > MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO) {
2329 		ptr->pkts = MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO;
2330 	}
2331 }
2332 
2333 static void
2334 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2335     struct mlx5e_cq_param *param)
2336 {
2337 	struct net_dim_cq_moder curr;
2338 	void *cqc = param->cqc;
2339 
2340 	/*
2341 	 * We use MLX5_CQE_FORMAT_HASH because the RX hash mini CQE
2342 	 * format is more beneficial for FreeBSD use case.
2343 	 *
2344 	 * Adding support for MLX5_CQE_FORMAT_CSUM will require changes
2345 	 * in mlx5e_decompress_cqe.
2346 	 */
2347 	if (priv->params.cqe_zipping_en) {
2348 		MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_HASH);
2349 		MLX5_SET(cqc, cqc, cqe_compression_en, 1);
2350 	}
2351 
2352 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
2353 
2354 	switch (priv->params.rx_cq_moderation_mode) {
2355 	case 0:
2356 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2357 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2358 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2359 		break;
2360 	case 1:
2361 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2362 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2363 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2364 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2365 		else
2366 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2367 		break;
2368 	case 2:
2369 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE, &curr);
2370 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2371 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2372 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2373 		break;
2374 	case 3:
2375 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE, &curr);
2376 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2377 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2378 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2379 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2380 		else
2381 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2382 		break;
2383 	default:
2384 		break;
2385 	}
2386 
2387 	mlx5e_dim_build_cq_param(priv, param);
2388 
2389 	mlx5e_build_common_cq_param(priv, param);
2390 }
2391 
2392 static void
2393 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2394     struct mlx5e_cq_param *param)
2395 {
2396 	void *cqc = param->cqc;
2397 
2398 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
2399 	MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
2400 	MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
2401 
2402 	switch (priv->params.tx_cq_moderation_mode) {
2403 	case 0:
2404 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2405 		break;
2406 	default:
2407 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2408 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2409 		else
2410 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2411 		break;
2412 	}
2413 
2414 	mlx5e_build_common_cq_param(priv, param);
2415 }
2416 
2417 static void
2418 mlx5e_build_channel_param(struct mlx5e_priv *priv,
2419     struct mlx5e_channel_param *cparam)
2420 {
2421 	memset(cparam, 0, sizeof(*cparam));
2422 
2423 	mlx5e_build_rq_param(priv, &cparam->rq);
2424 	mlx5e_build_sq_param(priv, &cparam->sq);
2425 	mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
2426 	mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
2427 }
2428 
2429 static int
2430 mlx5e_open_channels(struct mlx5e_priv *priv)
2431 {
2432 	struct mlx5e_channel_param *cparam;
2433 	int err;
2434 	int i;
2435 	int j;
2436 
2437 	cparam = malloc(sizeof(*cparam), M_MLX5EN, M_WAITOK);
2438 
2439 	mlx5e_build_channel_param(priv, cparam);
2440 	for (i = 0; i < priv->params.num_channels; i++) {
2441 		err = mlx5e_open_channel(priv, cparam, &priv->channel[i]);
2442 		if (err)
2443 			goto err_close_channels;
2444 
2445 		/* Bind interrupt vectors, if any. */
2446 		if (priv->params_ethtool.irq_cpu_base > -1) {
2447 			cpuset_t cpuset;
2448 			int cpu;
2449 			int irq;
2450 			int eqn;
2451 			int nirq;
2452 
2453 			err = mlx5_vector2eqn(priv->mdev, i,
2454 			    &eqn, &nirq);
2455 
2456 			/* error here is non-fatal */
2457 			if (err != 0)
2458 				continue;
2459 
2460 			irq = priv->mdev->priv.msix_arr[nirq].vector;
2461 			cpu = (unsigned)(priv->params_ethtool.irq_cpu_base +
2462 			    i * priv->params_ethtool.irq_cpu_stride) % (unsigned)mp_ncpus;
2463 
2464 			CPU_ZERO(&cpuset);
2465 			CPU_SET(cpu, &cpuset);
2466 			intr_setaffinity(irq, CPU_WHICH_INTRHANDLER, &cpuset);
2467 		}
2468 	}
2469 
2470 	for (j = 0; j < priv->params.num_channels; j++) {
2471 		err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j].rq);
2472 		if (err)
2473 			goto err_close_channels;
2474 	}
2475 	free(cparam, M_MLX5EN);
2476 	return (0);
2477 
2478 err_close_channels:
2479 	while (i--) {
2480 		mlx5e_close_channel(&priv->channel[i]);
2481 		mlx5e_close_channel_wait(&priv->channel[i]);
2482 	}
2483 	free(cparam, M_MLX5EN);
2484 	return (err);
2485 }
2486 
2487 static void
2488 mlx5e_close_channels(struct mlx5e_priv *priv)
2489 {
2490 	int i;
2491 
2492 	for (i = 0; i < priv->params.num_channels; i++)
2493 		mlx5e_close_channel(&priv->channel[i]);
2494 	for (i = 0; i < priv->params.num_channels; i++)
2495 		mlx5e_close_channel_wait(&priv->channel[i]);
2496 }
2497 
2498 static int
2499 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2500 {
2501 
2502 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2503 		uint8_t cq_mode;
2504 
2505 		switch (priv->params.tx_cq_moderation_mode) {
2506 		case 0:
2507 		case 2:
2508 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2509 			break;
2510 		default:
2511 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2512 			break;
2513 		}
2514 
2515 		return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2516 		    priv->params.tx_cq_moderation_usec,
2517 		    priv->params.tx_cq_moderation_pkts,
2518 		    cq_mode));
2519 	}
2520 
2521 	return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2522 	    priv->params.tx_cq_moderation_usec,
2523 	    priv->params.tx_cq_moderation_pkts));
2524 }
2525 
2526 static int
2527 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2528 {
2529 
2530 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2531 		uint8_t cq_mode;
2532 		uint8_t dim_mode;
2533 		int retval;
2534 
2535 		switch (priv->params.rx_cq_moderation_mode) {
2536 		case 0:
2537 		case 2:
2538 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2539 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2540 			break;
2541 		default:
2542 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2543 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2544 			break;
2545 		}
2546 
2547 		/* tear down dynamic interrupt moderation */
2548 		mtx_lock(&rq->mtx);
2549 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2550 		mtx_unlock(&rq->mtx);
2551 
2552 		/* wait for dynamic interrupt moderation work task, if any */
2553 		cancel_work_sync(&rq->dim.work);
2554 
2555 		if (priv->params.rx_cq_moderation_mode >= 2) {
2556 			struct net_dim_cq_moder curr;
2557 
2558 			mlx5e_get_default_profile(priv, dim_mode, &curr);
2559 
2560 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2561 			    curr.usec, curr.pkts, cq_mode);
2562 
2563 			/* set dynamic interrupt moderation mode and zero defaults */
2564 			mtx_lock(&rq->mtx);
2565 			rq->dim.mode = dim_mode;
2566 			rq->dim.state = 0;
2567 			rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2568 			mtx_unlock(&rq->mtx);
2569 		} else {
2570 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2571 			    priv->params.rx_cq_moderation_usec,
2572 			    priv->params.rx_cq_moderation_pkts,
2573 			    cq_mode);
2574 		}
2575 		return (retval);
2576 	}
2577 
2578 	return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2579 	    priv->params.rx_cq_moderation_usec,
2580 	    priv->params.rx_cq_moderation_pkts));
2581 }
2582 
2583 static int
2584 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
2585 {
2586 	int err;
2587 	int i;
2588 
2589 	err = mlx5e_refresh_rq_params(priv, &c->rq);
2590 	if (err)
2591 		goto done;
2592 
2593 	for (i = 0; i != priv->num_tc; i++) {
2594 		err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
2595 		if (err)
2596 			goto done;
2597 	}
2598 done:
2599 	return (err);
2600 }
2601 
2602 int
2603 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
2604 {
2605 	int i;
2606 
2607 	/* check if channels are closed */
2608 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2609 		return (EINVAL);
2610 
2611 	for (i = 0; i < priv->params.num_channels; i++) {
2612 		int err;
2613 
2614 		err = mlx5e_refresh_channel_params_sub(priv, &priv->channel[i]);
2615 		if (err)
2616 			return (err);
2617 	}
2618 	return (0);
2619 }
2620 
2621 static int
2622 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
2623 {
2624 	struct mlx5_core_dev *mdev = priv->mdev;
2625 	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
2626 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2627 
2628 	memset(in, 0, sizeof(in));
2629 
2630 	MLX5_SET(tisc, tisc, prio, tc);
2631 	MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
2632 
2633 	return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
2634 }
2635 
2636 static void
2637 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
2638 {
2639 	mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc], 0);
2640 }
2641 
2642 static int
2643 mlx5e_open_tises(struct mlx5e_priv *priv)
2644 {
2645 	int num_tc = priv->num_tc;
2646 	int err;
2647 	int tc;
2648 
2649 	for (tc = 0; tc < num_tc; tc++) {
2650 		err = mlx5e_open_tis(priv, tc);
2651 		if (err)
2652 			goto err_close_tises;
2653 	}
2654 
2655 	return (0);
2656 
2657 err_close_tises:
2658 	for (tc--; tc >= 0; tc--)
2659 		mlx5e_close_tis(priv, tc);
2660 
2661 	return (err);
2662 }
2663 
2664 static void
2665 mlx5e_close_tises(struct mlx5e_priv *priv)
2666 {
2667 	int num_tc = priv->num_tc;
2668 	int tc;
2669 
2670 	for (tc = 0; tc < num_tc; tc++)
2671 		mlx5e_close_tis(priv, tc);
2672 }
2673 
2674 static int
2675 mlx5e_open_rqt(struct mlx5e_priv *priv)
2676 {
2677 	struct mlx5_core_dev *mdev = priv->mdev;
2678 	u32 *in;
2679 	u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
2680 	void *rqtc;
2681 	int inlen;
2682 	int err;
2683 	int sz;
2684 	int i;
2685 
2686 	sz = 1 << priv->params.rx_hash_log_tbl_sz;
2687 
2688 	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2689 	in = mlx5_vzalloc(inlen);
2690 	if (in == NULL)
2691 		return (-ENOMEM);
2692 	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2693 
2694 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2695 	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2696 
2697 	for (i = 0; i < sz; i++) {
2698 		int ix = i;
2699 #ifdef RSS
2700 		ix = rss_get_indirection_to_bucket(ix);
2701 #endif
2702 		/* ensure we don't overflow */
2703 		ix %= priv->params.num_channels;
2704 
2705 		/* apply receive side scaling stride, if any */
2706 		ix -= ix % (int)priv->params.channels_rsss;
2707 
2708 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2709 	}
2710 
2711 	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
2712 
2713 	err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
2714 	if (!err)
2715 		priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn);
2716 
2717 	kvfree(in);
2718 
2719 	return (err);
2720 }
2721 
2722 static void
2723 mlx5e_close_rqt(struct mlx5e_priv *priv)
2724 {
2725 	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0};
2726 	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
2727 
2728 	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
2729 	MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn);
2730 
2731 	mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out));
2732 }
2733 
2734 #define	MLX5E_RSS_KEY_SIZE (10 * 4)	/* bytes */
2735 
2736 static void
2737 mlx5e_get_rss_key(void *key_ptr)
2738 {
2739 #ifdef RSS
2740 	rss_getkey(key_ptr);
2741 #else
2742 	static const u32 rsskey[] = {
2743 	    cpu_to_be32(0xD181C62C),
2744 	    cpu_to_be32(0xF7F4DB5B),
2745 	    cpu_to_be32(0x1983A2FC),
2746 	    cpu_to_be32(0x943E1ADB),
2747 	    cpu_to_be32(0xD9389E6B),
2748 	    cpu_to_be32(0xD1039C2C),
2749 	    cpu_to_be32(0xA74499AD),
2750 	    cpu_to_be32(0x593D56D9),
2751 	    cpu_to_be32(0xF3253C06),
2752 	    cpu_to_be32(0x2ADC1FFC),
2753 	};
2754 	CTASSERT(sizeof(rsskey) == MLX5E_RSS_KEY_SIZE);
2755 	memcpy(key_ptr, rsskey, MLX5E_RSS_KEY_SIZE);
2756 #endif
2757 }
2758 
2759 static void
2760 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt, bool inner_vxlan)
2761 {
2762 	void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2763 	void *hfsi = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
2764 	void *hfs = inner_vxlan ? hfsi : hfso;
2765 	__be32 *hkey;
2766 
2767 	MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2768 
2769 #define	ROUGH_MAX_L2_L3_HDR_SZ 256
2770 
2771 #define	MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2772 			  MLX5_HASH_FIELD_SEL_DST_IP)
2773 
2774 #define	MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2775 			  MLX5_HASH_FIELD_SEL_DST_IP   |\
2776 			  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2777 			  MLX5_HASH_FIELD_SEL_L4_DPORT)
2778 
2779 #define	MLX5_HASH_IP_IPSEC_SPI	(MLX5_HASH_FIELD_SEL_SRC_IP   |\
2780 				 MLX5_HASH_FIELD_SEL_DST_IP   |\
2781 				 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2782 
2783 	if (priv->params.hw_lro_en) {
2784 		MLX5_SET(tirc, tirc, lro_enable_mask,
2785 		    MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2786 		    MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2787 		MLX5_SET(tirc, tirc, lro_max_msg_sz,
2788 		    (priv->params.lro_wqe_sz -
2789 		    ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2790 		/* TODO: add the option to choose timer value dynamically */
2791 		MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
2792 		    MLX5_CAP_ETH(priv->mdev,
2793 		    lro_timer_supported_periods[2]));
2794 	}
2795 
2796 	if (inner_vxlan)
2797 		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
2798 
2799 	/* setup parameters for hashing TIR type, if any */
2800 	switch (tt) {
2801 	case MLX5E_TT_ANY:
2802 		MLX5_SET(tirc, tirc, disp_type,
2803 		    MLX5_TIRC_DISP_TYPE_DIRECT);
2804 		MLX5_SET(tirc, tirc, inline_rqn,
2805 		    priv->channel[0].rq.rqn);
2806 		break;
2807 	default:
2808 		MLX5_SET(tirc, tirc, disp_type,
2809 		    MLX5_TIRC_DISP_TYPE_INDIRECT);
2810 		MLX5_SET(tirc, tirc, indirect_table,
2811 		    priv->rqtn);
2812 		MLX5_SET(tirc, tirc, rx_hash_fn,
2813 		    MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
2814 		hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
2815 
2816 		CTASSERT(MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key) >=
2817 		    MLX5E_RSS_KEY_SIZE);
2818 #ifdef RSS
2819 		/*
2820 		 * The FreeBSD RSS implementation does currently not
2821 		 * support symmetric Toeplitz hashes:
2822 		 */
2823 		MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
2824 #else
2825 		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2826 #endif
2827 		mlx5e_get_rss_key(hkey);
2828 		break;
2829 	}
2830 
2831 	switch (tt) {
2832 	case MLX5E_TT_IPV4_TCP:
2833 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2834 		    MLX5_L3_PROT_TYPE_IPV4);
2835 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
2836 		    MLX5_L4_PROT_TYPE_TCP);
2837 #ifdef RSS
2838 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
2839 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2840 			    MLX5_HASH_IP);
2841 		} else
2842 #endif
2843 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2844 		    MLX5_HASH_ALL);
2845 		break;
2846 
2847 	case MLX5E_TT_IPV6_TCP:
2848 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2849 		    MLX5_L3_PROT_TYPE_IPV6);
2850 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
2851 		    MLX5_L4_PROT_TYPE_TCP);
2852 #ifdef RSS
2853 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
2854 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2855 			    MLX5_HASH_IP);
2856 		} else
2857 #endif
2858 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2859 		    MLX5_HASH_ALL);
2860 		break;
2861 
2862 	case MLX5E_TT_IPV4_UDP:
2863 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2864 		    MLX5_L3_PROT_TYPE_IPV4);
2865 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
2866 		    MLX5_L4_PROT_TYPE_UDP);
2867 #ifdef RSS
2868 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
2869 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2870 			    MLX5_HASH_IP);
2871 		} else
2872 #endif
2873 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2874 		    MLX5_HASH_ALL);
2875 		break;
2876 
2877 	case MLX5E_TT_IPV6_UDP:
2878 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2879 		    MLX5_L3_PROT_TYPE_IPV6);
2880 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
2881 		    MLX5_L4_PROT_TYPE_UDP);
2882 #ifdef RSS
2883 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
2884 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2885 			    MLX5_HASH_IP);
2886 		} else
2887 #endif
2888 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2889 		    MLX5_HASH_ALL);
2890 		break;
2891 
2892 	case MLX5E_TT_IPV4_IPSEC_AH:
2893 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2894 		    MLX5_L3_PROT_TYPE_IPV4);
2895 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2896 		    MLX5_HASH_IP_IPSEC_SPI);
2897 		break;
2898 
2899 	case MLX5E_TT_IPV6_IPSEC_AH:
2900 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2901 		    MLX5_L3_PROT_TYPE_IPV6);
2902 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2903 		    MLX5_HASH_IP_IPSEC_SPI);
2904 		break;
2905 
2906 	case MLX5E_TT_IPV4_IPSEC_ESP:
2907 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2908 		    MLX5_L3_PROT_TYPE_IPV4);
2909 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2910 		    MLX5_HASH_IP_IPSEC_SPI);
2911 		break;
2912 
2913 	case MLX5E_TT_IPV6_IPSEC_ESP:
2914 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2915 		    MLX5_L3_PROT_TYPE_IPV6);
2916 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2917 		    MLX5_HASH_IP_IPSEC_SPI);
2918 		break;
2919 
2920 	case MLX5E_TT_IPV4:
2921 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2922 		    MLX5_L3_PROT_TYPE_IPV4);
2923 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2924 		    MLX5_HASH_IP);
2925 		break;
2926 
2927 	case MLX5E_TT_IPV6:
2928 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
2929 		    MLX5_L3_PROT_TYPE_IPV6);
2930 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
2931 		    MLX5_HASH_IP);
2932 		break;
2933 
2934 	default:
2935 		break;
2936 	}
2937 }
2938 
2939 static int
2940 mlx5e_open_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
2941 {
2942 	struct mlx5_core_dev *mdev = priv->mdev;
2943 	u32 *in;
2944 	void *tirc;
2945 	int inlen;
2946 	int err;
2947 
2948 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2949 	in = mlx5_vzalloc(inlen);
2950 	if (in == NULL)
2951 		return (-ENOMEM);
2952 	tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
2953 
2954 	mlx5e_build_tir_ctx(priv, tirc, tt, inner_vxlan);
2955 
2956 	err = mlx5_core_create_tir(mdev, in, inlen, inner_vxlan ?
2957 	    &priv->tirn_inner_vxlan[tt] : &priv->tirn[tt]);
2958 
2959 	kvfree(in);
2960 
2961 	return (err);
2962 }
2963 
2964 static void
2965 mlx5e_close_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
2966 {
2967 	mlx5_core_destroy_tir(priv->mdev, inner_vxlan ?
2968 	    priv->tirn_inner_vxlan[tt] : priv->tirn[tt], 0);
2969 }
2970 
2971 static int
2972 mlx5e_open_tirs(struct mlx5e_priv *priv, bool inner_vxlan)
2973 {
2974 	int err;
2975 	int i;
2976 
2977 	for (i = 0; i < MLX5E_NUM_TT; i++) {
2978 		err = mlx5e_open_tir(priv, i, inner_vxlan);
2979 		if (err)
2980 			goto err_close_tirs;
2981 	}
2982 
2983 	return (0);
2984 
2985 err_close_tirs:
2986 	for (i--; i >= 0; i--)
2987 		mlx5e_close_tir(priv, i, inner_vxlan);
2988 
2989 	return (err);
2990 }
2991 
2992 static void
2993 mlx5e_close_tirs(struct mlx5e_priv *priv, bool inner_vxlan)
2994 {
2995 	int i;
2996 
2997 	for (i = 0; i < MLX5E_NUM_TT; i++)
2998 		mlx5e_close_tir(priv, i, inner_vxlan);
2999 }
3000 
3001 /*
3002  * SW MTU does not include headers,
3003  * HW MTU includes all headers and checksums.
3004  */
3005 static int
3006 mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu)
3007 {
3008 	struct mlx5e_priv *priv = ifp->if_softc;
3009 	struct mlx5_core_dev *mdev = priv->mdev;
3010 	int hw_mtu;
3011 	int err;
3012 
3013 	hw_mtu = MLX5E_SW2HW_MTU(sw_mtu);
3014 
3015 	err = mlx5_set_port_mtu(mdev, hw_mtu);
3016 	if (err) {
3017 		mlx5_en_err(ifp, "mlx5_set_port_mtu failed setting %d, err=%d\n",
3018 		    sw_mtu, err);
3019 		return (err);
3020 	}
3021 
3022 	/* Update vport context MTU */
3023 	err = mlx5_set_vport_mtu(mdev, hw_mtu);
3024 	if (err) {
3025 		mlx5_en_err(ifp,
3026 		    "Failed updating vport context with MTU size, err=%d\n",
3027 		    err);
3028 	}
3029 
3030 	ifp->if_mtu = sw_mtu;
3031 
3032 	err = mlx5_query_vport_mtu(mdev, &hw_mtu);
3033 	if (err || !hw_mtu) {
3034 		/* fallback to port oper mtu */
3035 		err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
3036 	}
3037 	if (err) {
3038 		mlx5_en_err(ifp,
3039 		    "Query port MTU, after setting new MTU value, failed\n");
3040 		return (err);
3041 	} else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
3042 		err = -E2BIG,
3043 		mlx5_en_err(ifp,
3044 		    "Port MTU %d is smaller than ifp mtu %d\n",
3045 		    hw_mtu, sw_mtu);
3046 	} else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
3047 		err = -EINVAL;
3048                 mlx5_en_err(ifp,
3049 		    "Port MTU %d is bigger than ifp mtu %d\n",
3050 		    hw_mtu, sw_mtu);
3051 	}
3052 	priv->params_ethtool.hw_mtu = hw_mtu;
3053 
3054 	/* compute MSB */
3055 	while (hw_mtu & (hw_mtu - 1))
3056 		hw_mtu &= (hw_mtu - 1);
3057 	priv->params_ethtool.hw_mtu_msb = hw_mtu;
3058 
3059 	return (err);
3060 }
3061 
3062 int
3063 mlx5e_open_locked(struct ifnet *ifp)
3064 {
3065 	struct mlx5e_priv *priv = ifp->if_softc;
3066 	int err;
3067 	u16 set_id;
3068 
3069 	/* check if already opened */
3070 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3071 		return (0);
3072 
3073 #ifdef RSS
3074 	if (rss_getnumbuckets() > priv->params.num_channels) {
3075 		mlx5_en_info(ifp,
3076 		    "NOTE: There are more RSS buckets(%u) than channels(%u) available\n",
3077 		    rss_getnumbuckets(), priv->params.num_channels);
3078 	}
3079 #endif
3080 	err = mlx5e_open_tises(priv);
3081 	if (err) {
3082 		mlx5_en_err(ifp, "mlx5e_open_tises failed, %d\n", err);
3083 		return (err);
3084 	}
3085 	err = mlx5_vport_alloc_q_counter(priv->mdev,
3086 	    MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
3087 	if (err) {
3088 		mlx5_en_err(priv->ifp,
3089 		    "mlx5_vport_alloc_q_counter failed: %d\n", err);
3090 		goto err_close_tises;
3091 	}
3092 	/* store counter set ID */
3093 	priv->counter_set_id = set_id;
3094 
3095 	err = mlx5e_open_channels(priv);
3096 	if (err) {
3097 		mlx5_en_err(ifp,
3098 		    "mlx5e_open_channels failed, %d\n", err);
3099 		goto err_dalloc_q_counter;
3100 	}
3101 	err = mlx5e_open_rqt(priv);
3102 	if (err) {
3103 		mlx5_en_err(ifp, "mlx5e_open_rqt failed, %d\n", err);
3104 		goto err_close_channels;
3105 	}
3106 	err = mlx5e_open_tirs(priv, false);
3107 	if (err) {
3108 		mlx5_en_err(ifp, "mlx5e_open_tir(main) failed, %d\n", err);
3109 		goto err_close_rqls;
3110 	}
3111 	if ((ifp->if_capenable & IFCAP_VXLAN_HWCSUM) != 0) {
3112 		err = mlx5e_open_tirs(priv, true);
3113 		if (err) {
3114 			mlx5_en_err(ifp, "mlx5e_open_tir(inner) failed, %d\n",
3115 			    err);
3116 			goto err_close_tirs;
3117 		}
3118 	}
3119 	err = mlx5e_open_flow_table(priv);
3120 	if (err) {
3121 		mlx5_en_err(ifp,
3122 		    "mlx5e_open_flow_table failed, %d\n", err);
3123 		goto err_close_tirs_inner;
3124 	}
3125 	err = mlx5e_add_all_vlan_rules(priv);
3126 	if (err) {
3127 		mlx5_en_err(ifp,
3128 		    "mlx5e_add_all_vlan_rules failed, %d\n", err);
3129 		goto err_close_flow_table;
3130 	}
3131 	if ((ifp->if_capenable & IFCAP_VXLAN_HWCSUM) != 0) {
3132 		err = mlx5e_add_all_vxlan_rules(priv);
3133 		if (err) {
3134 			mlx5_en_err(ifp,
3135 			    "mlx5e_add_all_vxlan_rules failed, %d\n", err);
3136 			goto err_del_vlan_rules;
3137 		}
3138 	}
3139 	set_bit(MLX5E_STATE_OPENED, &priv->state);
3140 
3141 	mlx5e_update_carrier(priv);
3142 	mlx5e_set_rx_mode_core(priv);
3143 
3144 	return (0);
3145 
3146 err_del_vlan_rules:
3147 	mlx5e_del_all_vlan_rules(priv);
3148 
3149 err_close_flow_table:
3150 	mlx5e_close_flow_table(priv);
3151 
3152 err_close_tirs_inner:
3153 	if ((ifp->if_capenable & IFCAP_VXLAN_HWCSUM) != 0)
3154 		mlx5e_close_tirs(priv, true);
3155 
3156 err_close_tirs:
3157 	mlx5e_close_tirs(priv, false);
3158 
3159 err_close_rqls:
3160 	mlx5e_close_rqt(priv);
3161 
3162 err_close_channels:
3163 	mlx5e_close_channels(priv);
3164 
3165 err_dalloc_q_counter:
3166 	mlx5_vport_dealloc_q_counter(priv->mdev,
3167 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3168 
3169 err_close_tises:
3170 	mlx5e_close_tises(priv);
3171 
3172 	return (err);
3173 }
3174 
3175 static void
3176 mlx5e_open(void *arg)
3177 {
3178 	struct mlx5e_priv *priv = arg;
3179 
3180 	PRIV_LOCK(priv);
3181 	if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
3182 		mlx5_en_err(priv->ifp,
3183 		    "Setting port status to up failed\n");
3184 
3185 	mlx5e_open_locked(priv->ifp);
3186 	priv->ifp->if_drv_flags |= IFF_DRV_RUNNING;
3187 	PRIV_UNLOCK(priv);
3188 }
3189 
3190 int
3191 mlx5e_close_locked(struct ifnet *ifp)
3192 {
3193 	struct mlx5e_priv *priv = ifp->if_softc;
3194 
3195 	/* check if already closed */
3196 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3197 		return (0);
3198 
3199 	clear_bit(MLX5E_STATE_OPENED, &priv->state);
3200 
3201 	mlx5e_set_rx_mode_core(priv);
3202 	mlx5e_del_all_vlan_rules(priv);
3203 	if ((ifp->if_capenable & IFCAP_VXLAN_HWCSUM) != 0)
3204 		mlx5e_del_all_vxlan_rules(priv);
3205 	if_link_state_change(priv->ifp, LINK_STATE_DOWN);
3206 	mlx5e_close_flow_table(priv);
3207 	if ((ifp->if_capenable & IFCAP_VXLAN_HWCSUM) != 0)
3208 		mlx5e_close_tirs(priv, true);
3209 	mlx5e_close_tirs(priv, false);
3210 	mlx5e_close_rqt(priv);
3211 	mlx5e_close_channels(priv);
3212 	mlx5_vport_dealloc_q_counter(priv->mdev,
3213 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3214 	mlx5e_close_tises(priv);
3215 
3216 	return (0);
3217 }
3218 
3219 #if (__FreeBSD_version >= 1100000)
3220 static uint64_t
3221 mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt)
3222 {
3223 	struct mlx5e_priv *priv = ifp->if_softc;
3224 	u64 retval;
3225 
3226 	/* PRIV_LOCK(priv); XXX not allowed */
3227 	switch (cnt) {
3228 	case IFCOUNTER_IPACKETS:
3229 		retval = priv->stats.vport.rx_packets;
3230 		break;
3231 	case IFCOUNTER_IERRORS:
3232 		retval = priv->stats.pport.in_range_len_errors +
3233 		    priv->stats.pport.out_of_range_len +
3234 		    priv->stats.pport.too_long_errors +
3235 		    priv->stats.pport.check_seq_err +
3236 		    priv->stats.pport.alignment_err;
3237 		break;
3238 	case IFCOUNTER_IQDROPS:
3239 		retval = priv->stats.vport.rx_out_of_buffer;
3240 		break;
3241 	case IFCOUNTER_OPACKETS:
3242 		retval = priv->stats.vport.tx_packets;
3243 		break;
3244 	case IFCOUNTER_OERRORS:
3245 		retval = priv->stats.port_stats_debug.out_discards;
3246 		break;
3247 	case IFCOUNTER_IBYTES:
3248 		retval = priv->stats.vport.rx_bytes;
3249 		break;
3250 	case IFCOUNTER_OBYTES:
3251 		retval = priv->stats.vport.tx_bytes;
3252 		break;
3253 	case IFCOUNTER_IMCASTS:
3254 		retval = priv->stats.vport.rx_multicast_packets;
3255 		break;
3256 	case IFCOUNTER_OMCASTS:
3257 		retval = priv->stats.vport.tx_multicast_packets;
3258 		break;
3259 	case IFCOUNTER_OQDROPS:
3260 		retval = priv->stats.vport.tx_queue_dropped;
3261 		break;
3262 	case IFCOUNTER_COLLISIONS:
3263 		retval = priv->stats.pport.collisions;
3264 		break;
3265 	default:
3266 		retval = if_get_counter_default(ifp, cnt);
3267 		break;
3268 	}
3269 	/* PRIV_UNLOCK(priv); XXX not allowed */
3270 	return (retval);
3271 }
3272 #endif
3273 
3274 static void
3275 mlx5e_set_rx_mode(struct ifnet *ifp)
3276 {
3277 	struct mlx5e_priv *priv = ifp->if_softc;
3278 
3279 	queue_work(priv->wq, &priv->set_rx_mode_work);
3280 }
3281 
3282 static int
3283 mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
3284 {
3285 	struct mlx5e_priv *priv;
3286 	struct ifreq *ifr;
3287 	struct ifdownreason *ifdr;
3288 	struct ifi2creq i2c;
3289 	struct ifrsskey *ifrk;
3290 	struct ifrsshash *ifrh;
3291 	int error = 0;
3292 	int mask = 0;
3293 	int size_read = 0;
3294 	int module_status;
3295 	int module_num;
3296 	int max_mtu;
3297 	uint8_t read_addr;
3298 
3299 	priv = ifp->if_softc;
3300 
3301 	/* check if detaching */
3302 	if (priv == NULL || priv->gone != 0)
3303 		return (ENXIO);
3304 
3305 	switch (command) {
3306 	case SIOCSIFMTU:
3307 		ifr = (struct ifreq *)data;
3308 
3309 		PRIV_LOCK(priv);
3310 		mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
3311 
3312 		if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
3313 		    ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
3314 			int was_opened;
3315 
3316 			was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3317 			if (was_opened)
3318 				mlx5e_close_locked(ifp);
3319 
3320 			/* set new MTU */
3321 			mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
3322 
3323 			if (was_opened)
3324 				mlx5e_open_locked(ifp);
3325 		} else {
3326 			error = EINVAL;
3327 			mlx5_en_err(ifp,
3328 			    "Invalid MTU value. Min val: %d, Max val: %d\n",
3329 			    MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
3330 		}
3331 		PRIV_UNLOCK(priv);
3332 		break;
3333 	case SIOCSIFFLAGS:
3334 		if ((ifp->if_flags & IFF_UP) &&
3335 		    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3336 			mlx5e_set_rx_mode(ifp);
3337 			break;
3338 		}
3339 		PRIV_LOCK(priv);
3340 		if (ifp->if_flags & IFF_UP) {
3341 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
3342 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3343 					mlx5e_open_locked(ifp);
3344 				ifp->if_drv_flags |= IFF_DRV_RUNNING;
3345 				mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
3346 			}
3347 		} else {
3348 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3349 				mlx5_set_port_status(priv->mdev,
3350 				    MLX5_PORT_DOWN);
3351 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3352 					mlx5e_close_locked(ifp);
3353 				mlx5e_update_carrier(priv);
3354 				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3355 			}
3356 		}
3357 		PRIV_UNLOCK(priv);
3358 		break;
3359 	case SIOCADDMULTI:
3360 	case SIOCDELMULTI:
3361 		mlx5e_set_rx_mode(ifp);
3362 		break;
3363 	case SIOCSIFMEDIA:
3364 	case SIOCGIFMEDIA:
3365 	case SIOCGIFXMEDIA:
3366 		ifr = (struct ifreq *)data;
3367 		error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
3368 		break;
3369 	case SIOCSIFCAP:
3370 		ifr = (struct ifreq *)data;
3371 		PRIV_LOCK(priv);
3372 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3373 
3374 		if (mask & IFCAP_TXCSUM) {
3375 			ifp->if_capenable ^= IFCAP_TXCSUM;
3376 			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
3377 
3378 			if (IFCAP_TSO4 & ifp->if_capenable &&
3379 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
3380 				mask &= ~IFCAP_TSO4;
3381 				ifp->if_capenable &= ~IFCAP_TSO4;
3382 				ifp->if_hwassist &= ~CSUM_IP_TSO;
3383 				mlx5_en_err(ifp,
3384 				    "tso4 disabled due to -txcsum.\n");
3385 			}
3386 		}
3387 		if (mask & IFCAP_TXCSUM_IPV6) {
3388 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3389 			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
3390 
3391 			if (IFCAP_TSO6 & ifp->if_capenable &&
3392 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3393 				mask &= ~IFCAP_TSO6;
3394 				ifp->if_capenable &= ~IFCAP_TSO6;
3395 				ifp->if_hwassist &= ~CSUM_IP6_TSO;
3396 				mlx5_en_err(ifp,
3397 				    "tso6 disabled due to -txcsum6.\n");
3398 			}
3399 		}
3400 		if (mask & IFCAP_MEXTPG)
3401 			ifp->if_capenable ^= IFCAP_MEXTPG;
3402 		if (mask & IFCAP_TXTLS4)
3403 			ifp->if_capenable ^= IFCAP_TXTLS4;
3404 		if (mask & IFCAP_TXTLS6)
3405 			ifp->if_capenable ^= IFCAP_TXTLS6;
3406 #ifdef RATELIMIT
3407 		if (mask & IFCAP_TXTLS_RTLMT)
3408 			ifp->if_capenable ^= IFCAP_TXTLS_RTLMT;
3409 #endif
3410 		if (mask & IFCAP_RXCSUM)
3411 			ifp->if_capenable ^= IFCAP_RXCSUM;
3412 		if (mask & IFCAP_RXCSUM_IPV6)
3413 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3414 		if (mask & IFCAP_TSO4) {
3415 			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
3416 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
3417 				mlx5_en_err(ifp, "enable txcsum first.\n");
3418 				error = EAGAIN;
3419 				goto out;
3420 			}
3421 			ifp->if_capenable ^= IFCAP_TSO4;
3422 			ifp->if_hwassist ^= CSUM_IP_TSO;
3423 		}
3424 		if (mask & IFCAP_TSO6) {
3425 			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
3426 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3427 				mlx5_en_err(ifp, "enable txcsum6 first.\n");
3428 				error = EAGAIN;
3429 				goto out;
3430 			}
3431 			ifp->if_capenable ^= IFCAP_TSO6;
3432 			ifp->if_hwassist ^= CSUM_IP6_TSO;
3433 		}
3434 		if (mask & IFCAP_VLAN_HWTSO)
3435 			ifp->if_capenable ^= IFCAP_VLAN_HWTSO;
3436 		if (mask & IFCAP_VLAN_HWFILTER) {
3437 			if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
3438 				mlx5e_disable_vlan_filter(priv);
3439 			else
3440 				mlx5e_enable_vlan_filter(priv);
3441 
3442 			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3443 		}
3444 		if (mask & IFCAP_VLAN_HWTAGGING)
3445 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3446 		if (mask & IFCAP_WOL_MAGIC)
3447 			ifp->if_capenable ^= IFCAP_WOL_MAGIC;
3448 		if (mask & IFCAP_VXLAN_HWCSUM) {
3449 			int was_opened = test_bit(MLX5E_STATE_OPENED,
3450 			    &priv->state);
3451 			if (was_opened)
3452 				mlx5e_close_locked(ifp);
3453 			ifp->if_capenable ^= IFCAP_VXLAN_HWCSUM;
3454 			ifp->if_hwassist ^= CSUM_INNER_IP | CSUM_INNER_IP_UDP |
3455 			    CSUM_INNER_IP_TCP | CSUM_INNER_IP6_UDP |
3456 			    CSUM_INNER_IP6_TCP;
3457 			if (was_opened)
3458 				mlx5e_open_locked(ifp);
3459 		}
3460 		if (mask & IFCAP_VXLAN_HWTSO) {
3461 			ifp->if_capenable ^= IFCAP_VXLAN_HWTSO;
3462 			ifp->if_hwassist ^= CSUM_INNER_IP_TSO |
3463 			    CSUM_INNER_IP6_TSO;
3464 		}
3465 
3466 		VLAN_CAPABILITIES(ifp);
3467 		/* turn off LRO means also turn of HW LRO - if it's on */
3468 		if (mask & IFCAP_LRO) {
3469 			int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3470 			bool need_restart = false;
3471 
3472 			ifp->if_capenable ^= IFCAP_LRO;
3473 
3474 			/* figure out if updating HW LRO is needed */
3475 			if (!(ifp->if_capenable & IFCAP_LRO)) {
3476 				if (priv->params.hw_lro_en) {
3477 					priv->params.hw_lro_en = false;
3478 					need_restart = true;
3479 				}
3480 			} else {
3481 				if (priv->params.hw_lro_en == false &&
3482 				    priv->params_ethtool.hw_lro != 0) {
3483 					priv->params.hw_lro_en = true;
3484 					need_restart = true;
3485 				}
3486 			}
3487 			if (was_opened && need_restart) {
3488 				mlx5e_close_locked(ifp);
3489 				mlx5e_open_locked(ifp);
3490 			}
3491 		}
3492 		if (mask & IFCAP_HWRXTSTMP) {
3493 			ifp->if_capenable ^= IFCAP_HWRXTSTMP;
3494 			if (ifp->if_capenable & IFCAP_HWRXTSTMP) {
3495 				if (priv->clbr_done == 0)
3496 					mlx5e_reset_calibration_callout(priv);
3497 			} else {
3498 				callout_drain(&priv->tstmp_clbr);
3499 				priv->clbr_done = 0;
3500 			}
3501 		}
3502 out:
3503 		PRIV_UNLOCK(priv);
3504 		break;
3505 
3506 	case SIOCGI2C:
3507 		ifr = (struct ifreq *)data;
3508 
3509 		/*
3510 		 * Copy from the user-space address ifr_data to the
3511 		 * kernel-space address i2c
3512 		 */
3513 		error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3514 		if (error)
3515 			break;
3516 
3517 		if (i2c.len > sizeof(i2c.data)) {
3518 			error = EINVAL;
3519 			break;
3520 		}
3521 
3522 		PRIV_LOCK(priv);
3523 		/* Get module_num which is required for the query_eeprom */
3524 		error = mlx5_query_module_num(priv->mdev, &module_num);
3525 		if (error) {
3526 			mlx5_en_err(ifp,
3527 			    "Query module num failed, eeprom reading is not supported\n");
3528 			error = EINVAL;
3529 			goto err_i2c;
3530 		}
3531 		/* Check if module is present before doing an access */
3532 		module_status = mlx5_query_module_status(priv->mdev, module_num);
3533 		if (module_status != MLX5_MODULE_STATUS_PLUGGED_ENABLED) {
3534 			error = EINVAL;
3535 			goto err_i2c;
3536 		}
3537 		/*
3538 		 * Currently 0XA0 and 0xA2 are the only addresses permitted.
3539 		 * The internal conversion is as follows:
3540 		 */
3541 		if (i2c.dev_addr == 0xA0)
3542 			read_addr = MLX5_I2C_ADDR_LOW;
3543 		else if (i2c.dev_addr == 0xA2)
3544 			read_addr = MLX5_I2C_ADDR_HIGH;
3545 		else {
3546 			mlx5_en_err(ifp,
3547 			    "Query eeprom failed, Invalid Address: %X\n",
3548 			    i2c.dev_addr);
3549 			error = EINVAL;
3550 			goto err_i2c;
3551 		}
3552 		error = mlx5_query_eeprom(priv->mdev,
3553 		    read_addr, MLX5_EEPROM_LOW_PAGE,
3554 		    (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
3555 		    (uint32_t *)i2c.data, &size_read);
3556 		if (error) {
3557 			mlx5_en_err(ifp,
3558 			    "Query eeprom failed, eeprom reading is not supported\n");
3559 			error = EINVAL;
3560 			goto err_i2c;
3561 		}
3562 
3563 		if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
3564 			error = mlx5_query_eeprom(priv->mdev,
3565 			    read_addr, MLX5_EEPROM_LOW_PAGE,
3566 			    (uint32_t)(i2c.offset + size_read),
3567 			    (uint32_t)(i2c.len - size_read), module_num,
3568 			    (uint32_t *)(i2c.data + size_read), &size_read);
3569 		}
3570 		if (error) {
3571 			mlx5_en_err(ifp,
3572 			    "Query eeprom failed, eeprom reading is not supported\n");
3573 			error = EINVAL;
3574 			goto err_i2c;
3575 		}
3576 
3577 		error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3578 err_i2c:
3579 		PRIV_UNLOCK(priv);
3580 		break;
3581 	case SIOCGIFDOWNREASON:
3582 		ifdr = (struct ifdownreason *)data;
3583 		bzero(ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3584 		PRIV_LOCK(priv);
3585 		error = -mlx5_query_pddr_troubleshooting_info(priv->mdev, NULL,
3586 		    ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3587 		PRIV_UNLOCK(priv);
3588 		if (error == 0)
3589 			ifdr->ifdr_reason = IFDR_REASON_MSG;
3590 		break;
3591 
3592 	case SIOCGIFRSSKEY:
3593 		ifrk = (struct ifrsskey *)data;
3594 		ifrk->ifrk_func = RSS_FUNC_TOEPLITZ;
3595 		ifrk->ifrk_keylen = MLX5E_RSS_KEY_SIZE;
3596 		CTASSERT(sizeof(ifrk->ifrk_key) >= MLX5E_RSS_KEY_SIZE);
3597 		mlx5e_get_rss_key(ifrk->ifrk_key);
3598 		break;
3599 
3600 	case SIOCGIFRSSHASH:
3601 		ifrh = (struct ifrsshash *)data;
3602 		ifrh->ifrh_func = RSS_FUNC_TOEPLITZ;
3603 		ifrh->ifrh_types =
3604 		    RSS_TYPE_IPV4 |
3605 		    RSS_TYPE_TCP_IPV4 |
3606 		    RSS_TYPE_UDP_IPV4 |
3607 		    RSS_TYPE_IPV6 |
3608 		    RSS_TYPE_TCP_IPV6 |
3609 		    RSS_TYPE_UDP_IPV6;
3610 		break;
3611 
3612 	default:
3613 		error = ether_ioctl(ifp, command, data);
3614 		break;
3615 	}
3616 	return (error);
3617 }
3618 
3619 static int
3620 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3621 {
3622 	/*
3623 	 * TODO: uncoment once FW really sets all these bits if
3624 	 * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
3625 	 * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
3626 	 * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
3627 	 * -ENOTSUPP;
3628 	 */
3629 
3630 	/* TODO: add more must-to-have features */
3631 
3632 	if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3633 		return (-ENODEV);
3634 
3635 	return (0);
3636 }
3637 
3638 static u16
3639 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3640 {
3641 	const int min_size = ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN;
3642 	const int max_size = MLX5E_MAX_TX_INLINE;
3643 	const int bf_buf_size =
3644 	    ((1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U) -
3645 	    (sizeof(struct mlx5e_tx_wqe) - 2);
3646 
3647 	/* verify against driver limits */
3648 	if (bf_buf_size > max_size)
3649 		return (max_size);
3650 	else if (bf_buf_size < min_size)
3651 		return (min_size);
3652 	else
3653 		return (bf_buf_size);
3654 }
3655 
3656 static int
3657 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
3658     struct mlx5e_priv *priv,
3659     int num_comp_vectors)
3660 {
3661 	int err;
3662 
3663 	/*
3664 	 * TODO: Consider link speed for setting "log_sq_size",
3665 	 * "log_rq_size" and "cq_moderation_xxx":
3666 	 */
3667 	priv->params.log_sq_size =
3668 	    MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3669 	priv->params.log_rq_size =
3670 	    MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3671 	priv->params.rx_cq_moderation_usec =
3672 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3673 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
3674 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3675 	priv->params.rx_cq_moderation_mode =
3676 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
3677 	priv->params.rx_cq_moderation_pkts =
3678 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3679 	priv->params.tx_cq_moderation_usec =
3680 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3681 	priv->params.tx_cq_moderation_pkts =
3682 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3683 	priv->params.min_rx_wqes =
3684 	    MLX5E_PARAMS_DEFAULT_MIN_RX_WQES;
3685 	priv->params.rx_hash_log_tbl_sz =
3686 	    (order_base_2(num_comp_vectors) >
3687 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
3688 	    order_base_2(num_comp_vectors) :
3689 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
3690 	priv->params.num_tc = 1;
3691 	priv->params.default_vlan_prio = 0;
3692 	priv->counter_set_id = -1;
3693 	priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3694 
3695 	err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3696 	if (err)
3697 		return (err);
3698 
3699 	/*
3700 	 * hw lro is currently defaulted to off. when it won't anymore we
3701 	 * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
3702 	 */
3703 	priv->params.hw_lro_en = false;
3704 	priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3705 
3706 	/*
3707 	 * CQE zipping is currently defaulted to off. when it won't
3708 	 * anymore we will consider the HW capability:
3709 	 * "!!MLX5_CAP_GEN(mdev, cqe_compression)"
3710 	 */
3711 	priv->params.cqe_zipping_en = false;
3712 
3713 	priv->mdev = mdev;
3714 	priv->params.num_channels = num_comp_vectors;
3715 	priv->params.channels_rsss = 1;
3716 	priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
3717 	priv->queue_mapping_channel_mask =
3718 	    roundup_pow_of_two(num_comp_vectors) - 1;
3719 	priv->num_tc = priv->params.num_tc;
3720 	priv->default_vlan_prio = priv->params.default_vlan_prio;
3721 
3722 	INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3723 	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3724 	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3725 
3726 	return (0);
3727 }
3728 
3729 static void
3730 mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
3731 {
3732 	bool ro_pci_enable =
3733 	    pci_get_relaxed_ordering_enabled(mdev->pdev->dev.bsddev);
3734 	bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
3735 	bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read);
3736 
3737 	MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read);
3738 	MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write);
3739 }
3740 
3741 static int
3742 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3743 		  struct mlx5_core_mkey *mkey)
3744 {
3745 	struct ifnet *ifp = priv->ifp;
3746 	struct mlx5_core_dev *mdev = priv->mdev;
3747 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3748 	void *mkc;
3749 	u32 *in;
3750 	int err;
3751 
3752 	in = mlx5_vzalloc(inlen);
3753 	if (in == NULL) {
3754 		mlx5_en_err(ifp, "failed to allocate inbox\n");
3755 		return (-ENOMEM);
3756 	}
3757 
3758 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3759 	MLX5_SET(mkc, mkc, access_mode, MLX5_ACCESS_MODE_PA);
3760 	MLX5_SET(mkc, mkc, umr_en, 1);	/* used by HW TLS */
3761 	MLX5_SET(mkc, mkc, lw, 1);
3762 	MLX5_SET(mkc, mkc, lr, 1);
3763 	mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
3764 	MLX5_SET(mkc, mkc, pd, pdn);
3765 	MLX5_SET(mkc, mkc, length64, 1);
3766 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
3767 
3768 	err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
3769 	if (err)
3770 		mlx5_en_err(ifp, "mlx5_core_create_mkey failed, %d\n",
3771 		    err);
3772 
3773 	kvfree(in);
3774 	return (err);
3775 }
3776 
3777 static const char *mlx5e_vport_stats_desc[] = {
3778 	MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
3779 };
3780 
3781 static const char *mlx5e_pport_stats_desc[] = {
3782 	MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
3783 };
3784 
3785 static int
3786 mlx5e_priv_static_init(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
3787     const uint32_t channels)
3788 {
3789 	uint32_t x;
3790 	int err;
3791 
3792 	mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
3793 	sx_init(&priv->state_lock, "mlx5state");
3794 	callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
3795 	MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
3796 	for (x = 0; x != channels; x++)
3797 		mlx5e_chan_static_init(priv, &priv->channel[x], x);
3798 
3799 	for (x = 0; x != channels; x++) {
3800 		err = mlx5_alloc_bfreg(mdev, &priv->channel[x].bfreg, false, false);
3801 		if (err)
3802 			goto err_alloc_bfreg;
3803 	}
3804 	return (0);
3805 
3806 err_alloc_bfreg:
3807 	while (x--)
3808 		mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
3809 
3810 	for (x = 0; x != channels; x++)
3811 		mlx5e_chan_static_destroy(&priv->channel[x]);
3812 	callout_drain(&priv->watchdog);
3813 	mtx_destroy(&priv->async_events_mtx);
3814 	sx_destroy(&priv->state_lock);
3815 	return (err);
3816 }
3817 
3818 static void
3819 mlx5e_priv_static_destroy(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
3820     const uint32_t channels)
3821 {
3822 	uint32_t x;
3823 
3824 	for (x = 0; x != channels; x++)
3825 		mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
3826 	for (x = 0; x != channels; x++)
3827 		mlx5e_chan_static_destroy(&priv->channel[x]);
3828 	callout_drain(&priv->watchdog);
3829 	mtx_destroy(&priv->async_events_mtx);
3830 	sx_destroy(&priv->state_lock);
3831 }
3832 
3833 static int
3834 sysctl_firmware(SYSCTL_HANDLER_ARGS)
3835 {
3836 	/*
3837 	 * %d.%d%.d the string format.
3838 	 * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
3839 	 * We need at most 5 chars to store that.
3840 	 * It also has: two "." and NULL at the end, which means we need 18
3841 	 * (5*3 + 3) chars at most.
3842 	 */
3843 	char fw[18];
3844 	struct mlx5e_priv *priv = arg1;
3845 	int error;
3846 
3847 	snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
3848 	    fw_rev_sub(priv->mdev));
3849 	error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
3850 	return (error);
3851 }
3852 
3853 static void
3854 mlx5e_disable_tx_dma(struct mlx5e_channel *ch)
3855 {
3856 	int i;
3857 
3858 	for (i = 0; i < ch->priv->num_tc; i++)
3859 		mlx5e_drain_sq(&ch->sq[i]);
3860 }
3861 
3862 static void
3863 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
3864 {
3865 
3866 	sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
3867 	sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
3868 	mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
3869 	sq->doorbell.d64 = 0;
3870 }
3871 
3872 void
3873 mlx5e_resume_sq(struct mlx5e_sq *sq)
3874 {
3875 	int err;
3876 
3877 	/* check if already enabled */
3878 	if (READ_ONCE(sq->running) != 0)
3879 		return;
3880 
3881 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
3882 	    MLX5_SQC_STATE_RST);
3883 	if (err != 0) {
3884 		mlx5_en_err(sq->ifp,
3885 		    "mlx5e_modify_sq() from ERR to RST failed: %d\n", err);
3886 	}
3887 
3888 	sq->cc = 0;
3889 	sq->pc = 0;
3890 
3891 	/* reset doorbell prior to moving from RST to RDY */
3892 	mlx5e_reset_sq_doorbell_record(sq);
3893 
3894 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
3895 	    MLX5_SQC_STATE_RDY);
3896 	if (err != 0) {
3897 		mlx5_en_err(sq->ifp,
3898 		    "mlx5e_modify_sq() from RST to RDY failed: %d\n", err);
3899 	}
3900 
3901 	sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
3902 	WRITE_ONCE(sq->running, 1);
3903 }
3904 
3905 static void
3906 mlx5e_enable_tx_dma(struct mlx5e_channel *ch)
3907 {
3908         int i;
3909 
3910 	for (i = 0; i < ch->priv->num_tc; i++)
3911 		mlx5e_resume_sq(&ch->sq[i]);
3912 }
3913 
3914 static void
3915 mlx5e_disable_rx_dma(struct mlx5e_channel *ch)
3916 {
3917 	struct mlx5e_rq *rq = &ch->rq;
3918 	struct epoch_tracker et;
3919 	int err;
3920 
3921 	mtx_lock(&rq->mtx);
3922 	rq->enabled = 0;
3923 	callout_stop(&rq->watchdog);
3924 	mtx_unlock(&rq->mtx);
3925 
3926 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
3927 	if (err != 0) {
3928 		mlx5_en_err(rq->ifp,
3929 		    "mlx5e_modify_rq() from RDY to RST failed: %d\n", err);
3930 	}
3931 
3932 	while (!mlx5_wq_ll_is_empty(&rq->wq)) {
3933 		msleep(1);
3934 		NET_EPOCH_ENTER(et);
3935 		rq->cq.mcq.comp(&rq->cq.mcq, NULL);
3936 		NET_EPOCH_EXIT(et);
3937 	}
3938 
3939 	/*
3940 	 * Transitioning into RST state will allow the FW to track less ERR state queues,
3941 	 * thus reducing the recv queue flushing time
3942 	 */
3943 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
3944 	if (err != 0) {
3945 		mlx5_en_err(rq->ifp,
3946 		    "mlx5e_modify_rq() from ERR to RST failed: %d\n", err);
3947 	}
3948 }
3949 
3950 static void
3951 mlx5e_enable_rx_dma(struct mlx5e_channel *ch)
3952 {
3953 	struct mlx5e_rq *rq = &ch->rq;
3954 	struct epoch_tracker et;
3955 	int err;
3956 
3957 	rq->wq.wqe_ctr = 0;
3958 	mlx5_wq_ll_update_db_record(&rq->wq);
3959 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3960 	if (err != 0) {
3961 		mlx5_en_err(rq->ifp,
3962 		    "mlx5e_modify_rq() from RST to RDY failed: %d\n", err);
3963         }
3964 
3965 	rq->enabled = 1;
3966 
3967 	NET_EPOCH_ENTER(et);
3968 	rq->cq.mcq.comp(&rq->cq.mcq, NULL);
3969 	NET_EPOCH_EXIT(et);
3970 }
3971 
3972 void
3973 mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value)
3974 {
3975 	int i;
3976 
3977 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3978 		return;
3979 
3980 	for (i = 0; i < priv->params.num_channels; i++) {
3981 		if (value)
3982 			mlx5e_disable_tx_dma(&priv->channel[i]);
3983 		else
3984 			mlx5e_enable_tx_dma(&priv->channel[i]);
3985 	}
3986 }
3987 
3988 void
3989 mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value)
3990 {
3991 	int i;
3992 
3993 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3994 		return;
3995 
3996 	for (i = 0; i < priv->params.num_channels; i++) {
3997 		if (value)
3998 			mlx5e_disable_rx_dma(&priv->channel[i]);
3999 		else
4000 			mlx5e_enable_rx_dma(&priv->channel[i]);
4001 	}
4002 }
4003 
4004 static void
4005 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
4006 {
4007 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4008 	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
4009 	    priv, 0, sysctl_firmware, "A", "HCA firmware version");
4010 
4011 	SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4012 	    OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
4013 	    "Board ID");
4014 }
4015 
4016 static int
4017 mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4018 {
4019 	struct mlx5e_priv *priv = arg1;
4020 	uint8_t temp[MLX5E_MAX_PRIORITY];
4021 	uint32_t tx_pfc;
4022 	int err;
4023 	int i;
4024 
4025 	PRIV_LOCK(priv);
4026 
4027 	tx_pfc = priv->params.tx_priority_flow_control;
4028 
4029 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4030 		temp[i] = (tx_pfc >> i) & 1;
4031 
4032 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4033 	if (err || !req->newptr)
4034 		goto done;
4035 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4036 	if (err)
4037 		goto done;
4038 
4039 	priv->params.tx_priority_flow_control = 0;
4040 
4041 	/* range check input value */
4042 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4043 		if (temp[i] > 1) {
4044 			err = ERANGE;
4045 			goto done;
4046 		}
4047 		priv->params.tx_priority_flow_control |= (temp[i] << i);
4048 	}
4049 
4050 	/* check if update is required */
4051 	if (tx_pfc != priv->params.tx_priority_flow_control)
4052 		err = -mlx5e_set_port_pfc(priv);
4053 done:
4054 	if (err != 0)
4055 		priv->params.tx_priority_flow_control= tx_pfc;
4056 	PRIV_UNLOCK(priv);
4057 
4058 	return (err);
4059 }
4060 
4061 static int
4062 mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4063 {
4064 	struct mlx5e_priv *priv = arg1;
4065 	uint8_t temp[MLX5E_MAX_PRIORITY];
4066 	uint32_t rx_pfc;
4067 	int err;
4068 	int i;
4069 
4070 	PRIV_LOCK(priv);
4071 
4072 	rx_pfc = priv->params.rx_priority_flow_control;
4073 
4074 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4075 		temp[i] = (rx_pfc >> i) & 1;
4076 
4077 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4078 	if (err || !req->newptr)
4079 		goto done;
4080 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4081 	if (err)
4082 		goto done;
4083 
4084 	priv->params.rx_priority_flow_control = 0;
4085 
4086 	/* range check input value */
4087 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4088 		if (temp[i] > 1) {
4089 			err = ERANGE;
4090 			goto done;
4091 		}
4092 		priv->params.rx_priority_flow_control |= (temp[i] << i);
4093 	}
4094 
4095 	/* check if update is required */
4096 	if (rx_pfc != priv->params.rx_priority_flow_control) {
4097 		err = -mlx5e_set_port_pfc(priv);
4098 		if (err == 0 && priv->sw_is_port_buf_owner)
4099 			err = mlx5e_update_buf_lossy(priv);
4100 	}
4101 done:
4102 	if (err != 0)
4103 		priv->params.rx_priority_flow_control= rx_pfc;
4104 	PRIV_UNLOCK(priv);
4105 
4106 	return (err);
4107 }
4108 
4109 static void
4110 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
4111 {
4112 #if (__FreeBSD_version < 1100000)
4113 	char path[96];
4114 #endif
4115 	int error;
4116 
4117 	/* enable pauseframes by default */
4118 	priv->params.tx_pauseframe_control = 1;
4119 	priv->params.rx_pauseframe_control = 1;
4120 
4121 	/* disable ports flow control, PFC, by default */
4122 	priv->params.tx_priority_flow_control = 0;
4123 	priv->params.rx_priority_flow_control = 0;
4124 
4125 #if (__FreeBSD_version < 1100000)
4126 	/* compute path for sysctl */
4127 	snprintf(path, sizeof(path), "dev.mce.%d.tx_pauseframe_control",
4128 	    device_get_unit(priv->mdev->pdev->dev.bsddev));
4129 
4130 	/* try to fetch tunable, if any */
4131 	TUNABLE_INT_FETCH(path, &priv->params.tx_pauseframe_control);
4132 
4133 	/* compute path for sysctl */
4134 	snprintf(path, sizeof(path), "dev.mce.%d.rx_pauseframe_control",
4135 	    device_get_unit(priv->mdev->pdev->dev.bsddev));
4136 
4137 	/* try to fetch tunable, if any */
4138 	TUNABLE_INT_FETCH(path, &priv->params.rx_pauseframe_control);
4139 #endif
4140 
4141 	/* register pauseframe SYSCTLs */
4142 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4143 	    OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
4144 	    &priv->params.tx_pauseframe_control, 0,
4145 	    "Set to enable TX pause frames. Clear to disable.");
4146 
4147 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4148 	    OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
4149 	    &priv->params.rx_pauseframe_control, 0,
4150 	    "Set to enable RX pause frames. Clear to disable.");
4151 
4152 	/* register priority flow control, PFC, SYSCTLs */
4153 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4154 	    OID_AUTO, "tx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4155 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_tx_priority_flow_control, "CU",
4156 	    "Set to enable TX ports flow control frames for priorities 0..7. Clear to disable.");
4157 
4158 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4159 	    OID_AUTO, "rx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4160 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_rx_priority_flow_control, "CU",
4161 	    "Set to enable RX ports flow control frames for priorities 0..7. Clear to disable.");
4162 
4163 	PRIV_LOCK(priv);
4164 
4165 	/* range check */
4166 	priv->params.tx_pauseframe_control =
4167 	    priv->params.tx_pauseframe_control ? 1 : 0;
4168 	priv->params.rx_pauseframe_control =
4169 	    priv->params.rx_pauseframe_control ? 1 : 0;
4170 
4171 	/* update firmware */
4172 	error = mlx5e_set_port_pause_and_pfc(priv);
4173 	if (error == -EINVAL) {
4174 		mlx5_en_err(priv->ifp,
4175 		    "Global pauseframes must be disabled before enabling PFC.\n");
4176 		priv->params.rx_priority_flow_control = 0;
4177 		priv->params.tx_priority_flow_control = 0;
4178 
4179 		/* update firmware */
4180 		(void) mlx5e_set_port_pause_and_pfc(priv);
4181 	}
4182 	PRIV_UNLOCK(priv);
4183 }
4184 
4185 static int
4186 mlx5e_ul_snd_tag_alloc(struct ifnet *ifp,
4187     union if_snd_tag_alloc_params *params,
4188     struct m_snd_tag **ppmt)
4189 {
4190 	struct mlx5e_priv *priv;
4191 	struct mlx5e_channel *pch;
4192 
4193 	priv = ifp->if_softc;
4194 
4195 	if (unlikely(priv->gone || params->hdr.flowtype == M_HASHTYPE_NONE)) {
4196 		return (EOPNOTSUPP);
4197 	} else {
4198 		/* keep this code synced with mlx5e_select_queue() */
4199 		u32 ch = priv->params.num_channels;
4200 #ifdef RSS
4201 		u32 temp;
4202 
4203 		if (rss_hash2bucket(params->hdr.flowid,
4204 		    params->hdr.flowtype, &temp) == 0)
4205 			ch = temp % ch;
4206 		else
4207 #endif
4208 			ch = (params->hdr.flowid % 128) % ch;
4209 
4210 		/*
4211 		 * NOTE: The channels array is only freed at detach
4212 		 * and it safe to return a pointer to the send tag
4213 		 * inside the channels structure as long as we
4214 		 * reference the priv.
4215 		 */
4216 		pch = priv->channel + ch;
4217 
4218 		/* check if send queue is not running */
4219 		if (unlikely(pch->sq[0].running == 0))
4220 			return (ENXIO);
4221 		m_snd_tag_ref(&pch->tag);
4222 		*ppmt = &pch->tag;
4223 		return (0);
4224 	}
4225 }
4226 
4227 static int
4228 mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4229 {
4230 	struct mlx5e_channel *pch =
4231 	    container_of(pmt, struct mlx5e_channel, tag);
4232 
4233 	params->unlimited.max_rate = -1ULL;
4234 	params->unlimited.queue_level = mlx5e_sq_queue_level(&pch->sq[0]);
4235 	return (0);
4236 }
4237 
4238 static void
4239 mlx5e_ul_snd_tag_free(struct m_snd_tag *pmt)
4240 {
4241 	struct mlx5e_channel *pch =
4242 	    container_of(pmt, struct mlx5e_channel, tag);
4243 
4244 	complete(&pch->completion);
4245 }
4246 
4247 static int
4248 mlx5e_snd_tag_alloc(struct ifnet *ifp,
4249     union if_snd_tag_alloc_params *params,
4250     struct m_snd_tag **ppmt)
4251 {
4252 
4253 	switch (params->hdr.type) {
4254 #ifdef RATELIMIT
4255 	case IF_SND_TAG_TYPE_RATE_LIMIT:
4256 		return (mlx5e_rl_snd_tag_alloc(ifp, params, ppmt));
4257 #ifdef KERN_TLS
4258 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4259 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4260 #endif
4261 #endif
4262 	case IF_SND_TAG_TYPE_UNLIMITED:
4263 		return (mlx5e_ul_snd_tag_alloc(ifp, params, ppmt));
4264 #ifdef KERN_TLS
4265 	case IF_SND_TAG_TYPE_TLS:
4266 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4267 #endif
4268 	default:
4269 		return (EOPNOTSUPP);
4270 	}
4271 }
4272 
4273 #ifdef RATELIMIT
4274 #define NUM_HDWR_RATES_MLX 13
4275 static const uint64_t adapter_rates_mlx[NUM_HDWR_RATES_MLX] = {
4276 	135375,			/* 1,083,000 */
4277 	180500,			/* 1,444,000 */
4278 	270750,			/* 2,166,000 */
4279 	361000,			/* 2,888,000 */
4280 	541500,			/* 4,332,000 */
4281 	721875,			/* 5,775,000 */
4282 	1082875,		/* 8,663,000 */
4283 	1443875,		/* 11,551,000 */
4284 	2165750,		/* 17,326,000 */
4285 	2887750,		/* 23,102,000 */
4286 	4331625,		/* 34,653,000 */
4287 	5775500,		/* 46,204,000 */
4288 	8663125			/* 69,305,000 */
4289 };
4290 
4291 static void
4292 mlx5e_ratelimit_query(struct ifnet *ifp __unused, struct if_ratelimit_query_results *q)
4293 {
4294 	/*
4295 	 * This function needs updating by the driver maintainer!
4296 	 * For the MLX card there are currently (ConectX-4?) 13
4297 	 * pre-set rates and others i.e. ConnectX-5, 6, 7??
4298 	 *
4299 	 * This will change based on later adapters
4300 	 * and this code should be updated to look at ifp
4301 	 * and figure out the specific adapter type
4302 	 * settings i.e. how many rates as well
4303 	 * as if they are fixed (as is shown here) or
4304 	 * if they are dynamic (example chelsio t4). Also if there
4305 	 * is a maximum number of flows that the adapter
4306 	 * can handle that too needs to be updated in
4307 	 * the max_flows field.
4308 	 */
4309 	q->rate_table = adapter_rates_mlx;
4310 	q->flags = RT_IS_FIXED_TABLE;
4311 	q->max_flows = 0;	/* mlx has no limit */
4312 	q->number_of_rates = NUM_HDWR_RATES_MLX;
4313 	q->min_segment_burst = 1;
4314 }
4315 #endif
4316 
4317 static void
4318 mlx5e_ifm_add(struct mlx5e_priv *priv, int type)
4319 {
4320 	ifmedia_add(&priv->media, type | IFM_ETHER, 0, NULL);
4321 	ifmedia_add(&priv->media, type | IFM_ETHER |
4322 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4323 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_RXPAUSE, 0, NULL);
4324 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_TXPAUSE, 0, NULL);
4325 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX, 0, NULL);
4326 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4327 	    IFM_ETH_RXPAUSE, 0, NULL);
4328 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4329 	    IFM_ETH_TXPAUSE, 0, NULL);
4330 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4331 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4332 }
4333 
4334 static void *
4335 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
4336 {
4337 	struct ifnet *ifp;
4338 	struct mlx5e_priv *priv;
4339 	u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
4340 	struct sysctl_oid_list *child;
4341 	int ncv = mdev->priv.eq_table.num_comp_vectors;
4342 	char unit[16];
4343 	struct pfil_head_args pa;
4344 	int err;
4345 	u32 eth_proto_cap;
4346 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
4347 	bool ext;
4348 	struct media media_entry = {};
4349 
4350 	if (mlx5e_check_required_hca_cap(mdev)) {
4351 		mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
4352 		return (NULL);
4353 	}
4354 
4355 	/*
4356 	 * Try to allocate the priv and make room for worst-case
4357 	 * number of channel structures:
4358 	 */
4359 	priv = malloc_domainset(sizeof(*priv) +
4360 	    (sizeof(priv->channel[0]) * mdev->priv.eq_table.num_comp_vectors),
4361 	    M_MLX5EN, mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
4362 
4363 	ifp = priv->ifp = if_alloc_dev(IFT_ETHER, mdev->pdev->dev.bsddev);
4364 	if (ifp == NULL) {
4365 		mlx5_core_err(mdev, "if_alloc() failed\n");
4366 		goto err_free_priv;
4367 	}
4368 	/* setup all static fields */
4369 	if (mlx5e_priv_static_init(priv, mdev, mdev->priv.eq_table.num_comp_vectors)) {
4370 		mlx5_core_err(mdev, "mlx5e_priv_static_init() failed\n");
4371 		goto err_free_ifp;
4372 	}
4373 
4374 	ifp->if_softc = priv;
4375 	if_initname(ifp, "mce", device_get_unit(mdev->pdev->dev.bsddev));
4376 	ifp->if_mtu = ETHERMTU;
4377 	ifp->if_init = mlx5e_open;
4378 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST |
4379 	    IFF_KNOWSEPOCH;
4380 	ifp->if_ioctl = mlx5e_ioctl;
4381 	ifp->if_transmit = mlx5e_xmit;
4382 	ifp->if_qflush = if_qflush;
4383 #if (__FreeBSD_version >= 1100000)
4384 	ifp->if_get_counter = mlx5e_get_counter;
4385 #endif
4386 	ifp->if_snd.ifq_maxlen = ifqmaxlen;
4387 	/*
4388          * Set driver features
4389          */
4390 	ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6;
4391 	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
4392 	ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER;
4393 	ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU;
4394 	ifp->if_capabilities |= IFCAP_LRO;
4395 	ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO;
4396 	ifp->if_capabilities |= IFCAP_HWSTATS | IFCAP_HWRXTSTMP;
4397 	ifp->if_capabilities |= IFCAP_MEXTPG;
4398 	ifp->if_capabilities |= IFCAP_TXTLS4 | IFCAP_TXTLS6;
4399 #ifdef RATELIMIT
4400 	ifp->if_capabilities |= IFCAP_TXRTLMT | IFCAP_TXTLS_RTLMT;
4401 #endif
4402 	ifp->if_capabilities |= IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO;
4403 	ifp->if_snd_tag_alloc = mlx5e_snd_tag_alloc;
4404 #ifdef RATELIMIT
4405 	ifp->if_ratelimit_query = mlx5e_ratelimit_query;
4406 #endif
4407 	/* set TSO limits so that we don't have to drop TX packets */
4408 	ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
4409 	ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */;
4410 	ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE;
4411 
4412 	ifp->if_capenable = ifp->if_capabilities;
4413 	ifp->if_hwassist = 0;
4414 	if (ifp->if_capenable & IFCAP_TSO)
4415 		ifp->if_hwassist |= CSUM_TSO;
4416 	if (ifp->if_capenable & IFCAP_TXCSUM)
4417 		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP);
4418 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
4419 		ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
4420 	if (ifp->if_capabilities & IFCAP_VXLAN_HWCSUM)
4421 		ifp->if_hwassist |= CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP |
4422 		    CSUM_INNER_IP | CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP |
4423 		    CSUM_ENCAP_VXLAN;
4424 	if (ifp->if_capabilities  & IFCAP_VXLAN_HWTSO)
4425 		ifp->if_hwassist |= CSUM_INNER_IP6_TSO | CSUM_INNER_IP_TSO;
4426 
4427 	/* ifnet sysctl tree */
4428 	sysctl_ctx_init(&priv->sysctl_ctx);
4429 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
4430 	    OID_AUTO, ifp->if_dname, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4431 	    "MLX5 ethernet - interface name");
4432 	if (priv->sysctl_ifnet == NULL) {
4433 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4434 		goto err_free_sysctl;
4435 	}
4436 	snprintf(unit, sizeof(unit), "%d", ifp->if_dunit);
4437 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4438 	    OID_AUTO, unit, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4439 	    "MLX5 ethernet - interface unit");
4440 	if (priv->sysctl_ifnet == NULL) {
4441 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4442 		goto err_free_sysctl;
4443 	}
4444 
4445 	/* HW sysctl tree */
4446 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
4447 	priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
4448 	    OID_AUTO, "hw", CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4449 	    "MLX5 ethernet dev hw");
4450 	if (priv->sysctl_hw == NULL) {
4451 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4452 		goto err_free_sysctl;
4453 	}
4454 
4455 	err = mlx5e_build_ifp_priv(mdev, priv, ncv);
4456 	if (err) {
4457 		mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err);
4458 		goto err_free_sysctl;
4459 	}
4460 
4461 	/* reuse mlx5core's watchdog workqueue */
4462 	priv->wq = mdev->priv.health.wq_watchdog;
4463 
4464 	err = mlx5_core_alloc_pd(mdev, &priv->pdn, 0);
4465 	if (err) {
4466 		mlx5_en_err(ifp, "mlx5_core_alloc_pd failed, %d\n", err);
4467 		goto err_free_wq;
4468 	}
4469 	err = mlx5_alloc_transport_domain(mdev, &priv->tdn, 0);
4470 	if (err) {
4471 		mlx5_en_err(ifp,
4472 		    "mlx5_alloc_transport_domain failed, %d\n", err);
4473 		goto err_dealloc_pd;
4474 	}
4475 	err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
4476 	if (err) {
4477 		mlx5_en_err(ifp, "mlx5e_create_mkey failed, %d\n", err);
4478 		goto err_dealloc_transport_domain;
4479 	}
4480 	mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
4481 
4482 	/* check if we should generate a random MAC address */
4483 	if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
4484 	    is_zero_ether_addr(dev_addr)) {
4485 		random_ether_addr(dev_addr);
4486 		mlx5_en_err(ifp, "Assigned random MAC address\n");
4487 	}
4488 
4489 	err = mlx5e_rl_init(priv);
4490 	if (err) {
4491 		mlx5_en_err(ifp, "mlx5e_rl_init failed, %d\n", err);
4492 		goto err_create_mkey;
4493 	}
4494 
4495 	err = mlx5e_tls_init(priv);
4496 	if (err) {
4497 		if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
4498 		goto err_rl_init;
4499 	}
4500 
4501 	/* set default MTU */
4502 	mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
4503 
4504 	/* Set default media status */
4505 	priv->media_status_last = IFM_AVALID;
4506 	priv->media_active_last = IFM_ETHER | IFM_AUTO | IFM_FDX;
4507 
4508 	/* setup default pauseframes configuration */
4509 	mlx5e_setup_pauseframes(priv);
4510 
4511 	/* Setup supported medias */
4512 	if (!mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1)) {
4513 		ext = MLX5_CAP_PCAM_FEATURE(mdev,
4514 		    ptys_extended_ethernet);
4515 		eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
4516 		    eth_proto_capability);
4517 	} else {
4518 		ext = false;
4519 		eth_proto_cap = 0;
4520 		mlx5_en_err(ifp, "Query port media capability failed, %d\n", err);
4521 	}
4522 
4523 	ifmedia_init(&priv->media, IFM_IMASK,
4524 	    mlx5e_media_change, mlx5e_media_status);
4525 
4526 	if (ext) {
4527 		for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
4528 			/* check if hardware has the right capability */
4529 			if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4530 				continue;
4531 			for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
4532 				media_entry = mlx5e_ext_mode_table[i][j];
4533 				if (media_entry.subtype == 0)
4534 					continue;
4535 				/* check if this subtype was already added */
4536 				for (unsigned k = 0; k != i; k++) {
4537 					/* check if hardware has the right capability */
4538 					if (MLX5E_PROT_MASK(k) & ~eth_proto_cap)
4539 						continue;
4540 					for (unsigned m = 0; m != MLX5E_CABLE_TYPE_NUMBER; m++) {
4541 						if (media_entry.subtype == mlx5e_ext_mode_table[k][m].subtype)
4542 							goto skip_ext_media;
4543 					}
4544 				}
4545 				mlx5e_ifm_add(priv, media_entry.subtype);
4546 			skip_ext_media:;
4547 			}
4548 		}
4549 	} else {
4550 		for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
4551 			media_entry = mlx5e_mode_table[i];
4552 			if (media_entry.subtype == 0)
4553 				continue;
4554 			if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4555 				continue;
4556 			/* check if this subtype was already added */
4557 			for (unsigned k = 0; k != i; k++) {
4558 				if (media_entry.subtype == mlx5e_mode_table[k].subtype)
4559 					goto skip_media;
4560 			}
4561 			mlx5e_ifm_add(priv, media_entry.subtype);
4562 
4563 			/* NOTE: 10G ER and LR shares the same entry */
4564 			if (media_entry.subtype == IFM_10G_ER)
4565 				mlx5e_ifm_add(priv, IFM_10G_LR);
4566 		skip_media:;
4567 		}
4568 	}
4569 
4570 	mlx5e_ifm_add(priv, IFM_AUTO);
4571 
4572 	/* Set autoselect by default */
4573 	ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4574 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
4575 
4576 	DEBUGNET_SET(ifp, mlx5_en);
4577 
4578 	ether_ifattach(ifp, dev_addr);
4579 
4580 	/* Register for VLAN events */
4581 	priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
4582 	    mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
4583 	priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
4584 	    mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
4585 
4586 	/* Register for VxLAN events */
4587 	priv->vxlan_start = EVENTHANDLER_REGISTER(vxlan_start,
4588 	    mlx5e_vxlan_start, priv, EVENTHANDLER_PRI_ANY);
4589 	priv->vxlan_stop = EVENTHANDLER_REGISTER(vxlan_stop,
4590 	    mlx5e_vxlan_stop, priv, EVENTHANDLER_PRI_ANY);
4591 
4592 	/* Link is down by default */
4593 	if_link_state_change(ifp, LINK_STATE_DOWN);
4594 
4595 	mlx5e_enable_async_events(priv);
4596 
4597 	mlx5e_add_hw_stats(priv);
4598 
4599 	mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4600 	    "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
4601 	    priv->stats.vport.arg);
4602 
4603 	mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4604 	    "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
4605 	    priv->stats.pport.arg);
4606 
4607 	mlx5e_create_ethtool(priv);
4608 
4609 	mtx_lock(&priv->async_events_mtx);
4610 	mlx5e_update_stats(priv);
4611 	mtx_unlock(&priv->async_events_mtx);
4612 
4613 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4614 	    OID_AUTO, "rx_clbr_done", CTLFLAG_RD,
4615 	    &priv->clbr_done, 0,
4616 	    "RX timestamps calibration state");
4617 	callout_init(&priv->tstmp_clbr, 1);
4618 	mlx5e_reset_calibration_callout(priv);
4619 
4620 	pa.pa_version = PFIL_VERSION;
4621 	pa.pa_flags = PFIL_IN;
4622 	pa.pa_type = PFIL_TYPE_ETHERNET;
4623 	pa.pa_headname = ifp->if_xname;
4624 	priv->pfil = pfil_head_register(&pa);
4625 
4626 	return (priv);
4627 
4628 err_rl_init:
4629 	mlx5e_rl_cleanup(priv);
4630 
4631 err_create_mkey:
4632 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4633 
4634 err_dealloc_transport_domain:
4635 	mlx5_dealloc_transport_domain(mdev, priv->tdn, 0);
4636 
4637 err_dealloc_pd:
4638 	mlx5_core_dealloc_pd(mdev, priv->pdn, 0);
4639 
4640 err_free_wq:
4641 	flush_workqueue(priv->wq);
4642 
4643 err_free_sysctl:
4644 	sysctl_ctx_free(&priv->sysctl_ctx);
4645 	if (priv->sysctl_debug)
4646 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4647 	mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
4648 
4649 err_free_ifp:
4650 	if_free(ifp);
4651 
4652 err_free_priv:
4653 	free(priv, M_MLX5EN);
4654 	return (NULL);
4655 }
4656 
4657 static void
4658 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
4659 {
4660 	struct mlx5e_priv *priv = vpriv;
4661 	struct ifnet *ifp = priv->ifp;
4662 
4663 	/* don't allow more IOCTLs */
4664 	priv->gone = 1;
4665 
4666 	/* XXX wait a bit to allow IOCTL handlers to complete */
4667 	pause("W", hz);
4668 
4669 #ifdef RATELIMIT
4670 	/*
4671 	 * The kernel can have reference(s) via the m_snd_tag's into
4672 	 * the ratelimit channels, and these must go away before
4673 	 * detaching:
4674 	 */
4675 	while (READ_ONCE(priv->rl.stats.tx_active_connections) != 0) {
4676 		mlx5_en_err(priv->ifp,
4677 		    "Waiting for all ratelimit connections to terminate\n");
4678 		pause("W", hz);
4679 	}
4680 #endif
4681 
4682 #ifdef KERN_TLS
4683 	/* wait for all TLS tags to get freed */
4684 	while (priv->tls.init != 0 &&
4685 	    uma_zone_get_cur(priv->tls.zone) != 0)  {
4686 		mlx5_en_err(priv->ifp,
4687 		    "Waiting for all TLS connections to terminate\n");
4688 		pause("W", hz);
4689 	}
4690 #endif
4691 	/* wait for all unlimited send tags to complete */
4692 	mlx5e_priv_wait_for_completion(priv, mdev->priv.eq_table.num_comp_vectors);
4693 
4694 	/* stop watchdog timer */
4695 	callout_drain(&priv->watchdog);
4696 
4697 	callout_drain(&priv->tstmp_clbr);
4698 
4699 	if (priv->vlan_attach != NULL)
4700 		EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
4701 	if (priv->vlan_detach != NULL)
4702 		EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
4703 	if (priv->vxlan_start != NULL)
4704 		EVENTHANDLER_DEREGISTER(vxlan_start, priv->vxlan_start);
4705 	if (priv->vxlan_stop != NULL)
4706 		EVENTHANDLER_DEREGISTER(vxlan_stop, priv->vxlan_stop);
4707 
4708 	/* make sure device gets closed */
4709 	PRIV_LOCK(priv);
4710 	mlx5e_close_locked(ifp);
4711 	PRIV_UNLOCK(priv);
4712 
4713 	/* deregister pfil */
4714 	if (priv->pfil != NULL) {
4715 		pfil_head_unregister(priv->pfil);
4716 		priv->pfil = NULL;
4717 	}
4718 
4719 	/* unregister device */
4720 	ifmedia_removeall(&priv->media);
4721 	ether_ifdetach(ifp);
4722 
4723 	mlx5e_tls_cleanup(priv);
4724 	mlx5e_rl_cleanup(priv);
4725 
4726 	/* destroy all remaining sysctl nodes */
4727 	sysctl_ctx_free(&priv->stats.vport.ctx);
4728 	sysctl_ctx_free(&priv->stats.pport.ctx);
4729 	if (priv->sysctl_debug)
4730 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4731 	sysctl_ctx_free(&priv->sysctl_ctx);
4732 
4733 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4734 	mlx5_dealloc_transport_domain(priv->mdev, priv->tdn, 0);
4735 	mlx5_core_dealloc_pd(priv->mdev, priv->pdn, 0);
4736 	mlx5e_disable_async_events(priv);
4737 	flush_workqueue(priv->wq);
4738 	mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
4739 	if_free(ifp);
4740 	free(priv, M_MLX5EN);
4741 }
4742 
4743 #ifdef DEBUGNET
4744 static void
4745 mlx5_en_debugnet_init(struct ifnet *dev, int *nrxr, int *ncl, int *clsize)
4746 {
4747 	struct mlx5e_priv *priv = if_getsoftc(dev);
4748 
4749 	PRIV_LOCK(priv);
4750 	*nrxr = priv->params.num_channels;
4751 	*ncl = DEBUGNET_MAX_IN_FLIGHT;
4752 	*clsize = MLX5E_MAX_RX_BYTES;
4753 	PRIV_UNLOCK(priv);
4754 }
4755 
4756 static void
4757 mlx5_en_debugnet_event(struct ifnet *dev, enum debugnet_ev event)
4758 {
4759 }
4760 
4761 static int
4762 mlx5_en_debugnet_transmit(struct ifnet *dev, struct mbuf *m)
4763 {
4764 	struct mlx5e_priv *priv = if_getsoftc(dev);
4765 	struct mlx5e_sq *sq;
4766 	int err;
4767 
4768 	if ((if_getdrvflags(dev) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
4769 	    IFF_DRV_RUNNING || (priv->media_status_last & IFM_ACTIVE) == 0)
4770 		return (ENOENT);
4771 
4772 	sq = &priv->channel[0].sq[0];
4773 
4774 	if (sq->running == 0) {
4775 		m_freem(m);
4776 		return (ENOENT);
4777 	}
4778 
4779 	if (mlx5e_sq_xmit(sq, &m) != 0) {
4780 		m_freem(m);
4781 		err = ENOBUFS;
4782 	} else {
4783 		err = 0;
4784 	}
4785 
4786 	if (likely(sq->doorbell.d64 != 0)) {
4787 		mlx5e_tx_notify_hw(sq, sq->doorbell.d32);
4788 		sq->doorbell.d64 = 0;
4789 	}
4790 	return (err);
4791 }
4792 
4793 static int
4794 mlx5_en_debugnet_poll(struct ifnet *dev, int count)
4795 {
4796 	struct mlx5e_priv *priv = if_getsoftc(dev);
4797 
4798 	if ((if_getdrvflags(dev) & IFF_DRV_RUNNING) == 0 ||
4799 	    (priv->media_status_last & IFM_ACTIVE) == 0)
4800 		return (ENOENT);
4801 
4802 	mlx5_poll_interrupts(priv->mdev);
4803 
4804 	return (0);
4805 }
4806 #endif /* DEBUGNET */
4807 
4808 static void *
4809 mlx5e_get_ifp(void *vpriv)
4810 {
4811 	struct mlx5e_priv *priv = vpriv;
4812 
4813 	return (priv->ifp);
4814 }
4815 
4816 static struct mlx5_interface mlx5e_interface = {
4817 	.add = mlx5e_create_ifp,
4818 	.remove = mlx5e_destroy_ifp,
4819 	.event = mlx5e_async_event,
4820 	.protocol = MLX5_INTERFACE_PROTOCOL_ETH,
4821 	.get_dev = mlx5e_get_ifp,
4822 };
4823 
4824 void
4825 mlx5e_init(void)
4826 {
4827 	mlx5_register_interface(&mlx5e_interface);
4828 }
4829 
4830 void
4831 mlx5e_cleanup(void)
4832 {
4833 	mlx5_unregister_interface(&mlx5e_interface);
4834 }
4835 
4836 module_init_order(mlx5e_init, SI_ORDER_SIXTH);
4837 module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);
4838 
4839 #if (__FreeBSD_version >= 1100000)
4840 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
4841 #endif
4842 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
4843 MODULE_VERSION(mlx5en, 1);
4844