xref: /freebsd/sys/dev/mlx5/mlx5_en/mlx5_en_main.c (revision 718519f4efc71096422fc71dab90b2a3369871ff)
1 /*-
2  * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
3  * Copyright (c) 2022 NVIDIA corporation & affiliates.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include "opt_ipsec.h"
28 #include "opt_kern_tls.h"
29 #include "opt_rss.h"
30 #include "opt_ratelimit.h"
31 
32 #include <dev/mlx5/mlx5_en/en.h>
33 #include <dev/mlx5/mlx5_accel/ipsec.h>
34 
35 #include <sys/eventhandler.h>
36 #include <sys/sockio.h>
37 #include <machine/atomic.h>
38 
39 #include <net/debugnet.h>
40 #include <netinet/tcp_ratelimit.h>
41 #include <netipsec/keydb.h>
42 #include <netipsec/ipsec_offload.h>
43 
44 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
45 static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
46 static if_snd_tag_free_t mlx5e_ul_snd_tag_free;
47 
48 struct mlx5e_channel_param {
49 	struct mlx5e_rq_param rq;
50 	struct mlx5e_sq_param sq;
51 	struct mlx5e_cq_param rx_cq;
52 	struct mlx5e_cq_param tx_cq;
53 };
54 
55 struct media {
56 	u32	subtype;
57 	u64	baudrate;
58 };
59 
60 static const struct media mlx5e_mode_table[MLX5E_LINK_SPEEDS_NUMBER] =
61 {
62 	[MLX5E_1000BASE_CX_SGMII] = {
63 		.subtype = IFM_1000_CX_SGMII,
64 		.baudrate = IF_Mbps(1000ULL),
65 	},
66 	[MLX5E_1000BASE_KX] = {
67 		.subtype = IFM_1000_KX,
68 		.baudrate = IF_Mbps(1000ULL),
69 	},
70 	[MLX5E_10GBASE_CX4] = {
71 		.subtype = IFM_10G_CX4,
72 		.baudrate = IF_Gbps(10ULL),
73 	},
74 	[MLX5E_10GBASE_KX4] = {
75 		.subtype = IFM_10G_KX4,
76 		.baudrate = IF_Gbps(10ULL),
77 	},
78 	[MLX5E_10GBASE_KR] = {
79 		.subtype = IFM_10G_KR,
80 		.baudrate = IF_Gbps(10ULL),
81 	},
82 	[MLX5E_20GBASE_KR2] = {
83 		.subtype = IFM_20G_KR2,
84 		.baudrate = IF_Gbps(20ULL),
85 	},
86 	[MLX5E_40GBASE_CR4] = {
87 		.subtype = IFM_40G_CR4,
88 		.baudrate = IF_Gbps(40ULL),
89 	},
90 	[MLX5E_40GBASE_KR4] = {
91 		.subtype = IFM_40G_KR4,
92 		.baudrate = IF_Gbps(40ULL),
93 	},
94 	[MLX5E_56GBASE_R4] = {
95 		.subtype = IFM_56G_R4,
96 		.baudrate = IF_Gbps(56ULL),
97 	},
98 	[MLX5E_10GBASE_CR] = {
99 		.subtype = IFM_10G_CR1,
100 		.baudrate = IF_Gbps(10ULL),
101 	},
102 	[MLX5E_10GBASE_SR] = {
103 		.subtype = IFM_10G_SR,
104 		.baudrate = IF_Gbps(10ULL),
105 	},
106 	[MLX5E_10GBASE_ER_LR] = {
107 		.subtype = IFM_10G_ER,
108 		.baudrate = IF_Gbps(10ULL),
109 	},
110 	[MLX5E_40GBASE_SR4] = {
111 		.subtype = IFM_40G_SR4,
112 		.baudrate = IF_Gbps(40ULL),
113 	},
114 	[MLX5E_40GBASE_LR4_ER4] = {
115 		.subtype = IFM_40G_LR4,
116 		.baudrate = IF_Gbps(40ULL),
117 	},
118 	[MLX5E_100GBASE_CR4] = {
119 		.subtype = IFM_100G_CR4,
120 		.baudrate = IF_Gbps(100ULL),
121 	},
122 	[MLX5E_100GBASE_SR4] = {
123 		.subtype = IFM_100G_SR4,
124 		.baudrate = IF_Gbps(100ULL),
125 	},
126 	[MLX5E_100GBASE_KR4] = {
127 		.subtype = IFM_100G_KR4,
128 		.baudrate = IF_Gbps(100ULL),
129 	},
130 	[MLX5E_100GBASE_LR4] = {
131 		.subtype = IFM_100G_LR4,
132 		.baudrate = IF_Gbps(100ULL),
133 	},
134 	[MLX5E_100BASE_TX] = {
135 		.subtype = IFM_100_TX,
136 		.baudrate = IF_Mbps(100ULL),
137 	},
138 	[MLX5E_1000BASE_T] = {
139 		.subtype = IFM_1000_T,
140 		.baudrate = IF_Mbps(1000ULL),
141 	},
142 	[MLX5E_10GBASE_T] = {
143 		.subtype = IFM_10G_T,
144 		.baudrate = IF_Gbps(10ULL),
145 	},
146 	[MLX5E_25GBASE_CR] = {
147 		.subtype = IFM_25G_CR,
148 		.baudrate = IF_Gbps(25ULL),
149 	},
150 	[MLX5E_25GBASE_KR] = {
151 		.subtype = IFM_25G_KR,
152 		.baudrate = IF_Gbps(25ULL),
153 	},
154 	[MLX5E_25GBASE_SR] = {
155 		.subtype = IFM_25G_SR,
156 		.baudrate = IF_Gbps(25ULL),
157 	},
158 	[MLX5E_50GBASE_CR2] = {
159 		.subtype = IFM_50G_CR2,
160 		.baudrate = IF_Gbps(50ULL),
161 	},
162 	[MLX5E_50GBASE_KR2] = {
163 		.subtype = IFM_50G_KR2,
164 		.baudrate = IF_Gbps(50ULL),
165 	},
166 	[MLX5E_50GBASE_KR4] = {
167 		.subtype = IFM_50G_KR4,
168 		.baudrate = IF_Gbps(50ULL),
169 	},
170 };
171 
172 static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX5E_CABLE_TYPE_NUMBER] =
173 {
174 	/**/
175 	[MLX5E_SGMII_100M][MLX5E_CABLE_TYPE_UNKNOWN] = {
176 		.subtype = IFM_100_SGMII,
177 		.baudrate = IF_Mbps(100),
178 	},
179 
180 	/**/
181 	[MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_UNKNOWN] = {
182 		.subtype = IFM_1000_CX,
183 		.baudrate = IF_Mbps(1000),
184 	},
185 	[MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
186 		.subtype = IFM_1000_SX,
187 		.baudrate = IF_Mbps(1000),
188 	},
189 
190 	/**/
191 	[MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_UNKNOWN] = {
192 		.subtype = IFM_5000_KR,
193 		.baudrate = IF_Mbps(5000),
194 	},
195 	[MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
196 		.subtype = IFM_5000_T,
197 		.baudrate = IF_Mbps(5000),
198 	},
199 
200 	/**/
201 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_UNKNOWN] = {
202 		.subtype = IFM_10G_KR,
203 		.baudrate = IF_Gbps(10ULL),
204 	},
205 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
206 		.subtype = IFM_10G_CR1,
207 		.baudrate = IF_Gbps(10ULL),
208 	},
209 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
210 		.subtype = IFM_10G_SR,
211 		.baudrate = IF_Gbps(10ULL),
212 	},
213 
214 	/**/
215 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_UNKNOWN] = {
216 		.subtype = IFM_40G_KR4,
217 		.baudrate = IF_Gbps(40ULL),
218 	},
219 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
220 		.subtype = IFM_40G_CR4,
221 		.baudrate = IF_Gbps(40ULL),
222 	},
223 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
224 		.subtype = IFM_40G_SR4,
225 		.baudrate = IF_Gbps(40ULL),
226 	},
227 
228 	/**/
229 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
230 		.subtype = IFM_25G_KR,
231 		.baudrate = IF_Gbps(25ULL),
232 	},
233 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
234 		.subtype = IFM_25G_CR,
235 		.baudrate = IF_Gbps(25ULL),
236 	},
237 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
238 		.subtype = IFM_25G_SR,
239 		.baudrate = IF_Gbps(25ULL),
240 	},
241 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
242 		.subtype = IFM_25G_T,
243 		.baudrate = IF_Gbps(25ULL),
244 	},
245 
246 	/**/
247 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
248 		.subtype = IFM_50G_KR2,
249 		.baudrate = IF_Gbps(50ULL),
250 	},
251 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
252 		.subtype = IFM_50G_CR2,
253 		.baudrate = IF_Gbps(50ULL),
254 	},
255 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
256 		.subtype = IFM_50G_SR2,
257 		.baudrate = IF_Gbps(50ULL),
258 	},
259 
260 	/**/
261 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
262 		.subtype = IFM_50G_KR_PAM4,
263 		.baudrate = IF_Gbps(50ULL),
264 	},
265 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
266 		.subtype = IFM_50G_CP,
267 		.baudrate = IF_Gbps(50ULL),
268 	},
269 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
270 		.subtype = IFM_50G_SR,
271 		.baudrate = IF_Gbps(50ULL),
272 	},
273 
274 	/**/
275 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
276 		.subtype = IFM_100G_KR4,
277 		.baudrate = IF_Gbps(100ULL),
278 	},
279 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
280 		.subtype = IFM_100G_CR4,
281 		.baudrate = IF_Gbps(100ULL),
282 	},
283 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
284 		.subtype = IFM_100G_SR4,
285 		.baudrate = IF_Gbps(100ULL),
286 	},
287 
288 	/**/
289 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
290 		.subtype = IFM_100G_KR_PAM4,
291 		.baudrate = IF_Gbps(100ULL),
292 	},
293 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
294 		.subtype = IFM_100G_CR_PAM4,
295 		.baudrate = IF_Gbps(100ULL),
296 	},
297 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
298 		.subtype = IFM_100G_SR2,	/* XXX */
299 		.baudrate = IF_Gbps(100ULL),
300 	},
301 
302 	/**/
303 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
304 		.subtype = IFM_100G_KR4,
305 		.baudrate = IF_Gbps(100ULL),
306 	},
307 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
308 		.subtype = IFM_100G_CP2,
309 		.baudrate = IF_Gbps(100ULL),
310 	},
311 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
312 		.subtype = IFM_100G_SR2,
313 		.baudrate = IF_Gbps(100ULL),
314 	},
315 
316 	/**/
317 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
318 		.subtype = IFM_200G_KR4_PAM4,	/* XXX */
319 		.baudrate = IF_Gbps(200ULL),
320 	},
321 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
322 		.subtype = IFM_200G_CR4_PAM4,	/* XXX */
323 		.baudrate = IF_Gbps(200ULL),
324 	},
325 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
326 		.subtype = IFM_200G_SR4,	/* XXX */
327 		.baudrate = IF_Gbps(200ULL),
328 	},
329 
330 	/**/
331 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
332 		.subtype = IFM_200G_KR4_PAM4,
333 		.baudrate = IF_Gbps(200ULL),
334 	},
335 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
336 		.subtype = IFM_200G_CR4_PAM4,
337 		.baudrate = IF_Gbps(200ULL),
338 	},
339 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
340 		.subtype = IFM_200G_SR4,
341 		.baudrate = IF_Gbps(200ULL),
342 	},
343 
344 	/**/
345 	[MLX5E_400GAUI_8][MLX5E_CABLE_TYPE_UNKNOWN] = {
346 		.subtype = IFM_400G_LR8,	/* XXX */
347 		.baudrate = IF_Gbps(400ULL),
348 	},
349 
350 	/**/
351 	[MLX5E_400GAUI_4_400GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
352 		.subtype = IFM_400G_LR8,	/* XXX */
353 		.baudrate = IF_Gbps(400ULL),
354 	},
355 };
356 
357 static const struct if_snd_tag_sw mlx5e_ul_snd_tag_sw = {
358 	.snd_tag_query = mlx5e_ul_snd_tag_query,
359 	.snd_tag_free = mlx5e_ul_snd_tag_free,
360 	.type = IF_SND_TAG_TYPE_UNLIMITED
361 };
362 
363 DEBUGNET_DEFINE(mlx5_en);
364 
365 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
366 
367 static void
368 mlx5e_update_carrier(struct mlx5e_priv *priv)
369 {
370 	struct mlx5_core_dev *mdev = priv->mdev;
371 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
372 	u32 eth_proto_oper;
373 	int error;
374 	u8 i;
375 	u8 cable_type;
376 	u8 port_state;
377 	u8 is_er_type;
378 	bool ext;
379 	struct media media_entry = {};
380 
381 	port_state = mlx5_query_vport_state(mdev,
382 	    MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
383 
384 	if (port_state == VPORT_STATE_UP) {
385 		priv->media_status_last |= IFM_ACTIVE;
386 	} else {
387 		priv->media_status_last &= ~IFM_ACTIVE;
388 		priv->media_active_last = IFM_ETHER;
389 		if_link_state_change(priv->ifp, LINK_STATE_DOWN);
390 		return;
391 	}
392 
393 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
394 	    MLX5_PTYS_EN, 1);
395 	if (error) {
396 		priv->media_active_last = IFM_ETHER;
397 		if_setbaudrate(priv->ifp, 1);
398 		mlx5_en_err(priv->ifp, "query port ptys failed: 0x%x\n",
399 		    error);
400 		return;
401 	}
402 
403 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
404 	eth_proto_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
405 	    eth_proto_oper);
406 
407 	i = ilog2(eth_proto_oper);
408 
409 	if (ext) {
410 		error = mlx5_query_pddr_cable_type(mdev, 1, &cable_type);
411 		if (error != 0) {
412 			/* use fallback entry */
413 			media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
414 
415 			mlx5_en_err(priv->ifp,
416 			    "query port pddr failed: %d\n", error);
417 		} else {
418 			media_entry = mlx5e_ext_mode_table[i][cable_type];
419 
420 			/* check if we should use fallback entry */
421 			if (media_entry.subtype == 0)
422 				media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
423 		}
424 	} else {
425 		media_entry = mlx5e_mode_table[i];
426 	}
427 
428 	if (media_entry.subtype == 0) {
429 		mlx5_en_err(priv->ifp,
430 		    "Could not find operational media subtype\n");
431 		return;
432 	}
433 
434 	switch (media_entry.subtype) {
435 	case IFM_10G_ER:
436 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
437 		if (error != 0) {
438 			mlx5_en_err(priv->ifp,
439 			    "query port pddr failed: %d\n", error);
440 		}
441 		if (error != 0 || is_er_type == 0)
442 			media_entry.subtype = IFM_10G_LR;
443 		break;
444 	case IFM_40G_LR4:
445 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
446 		if (error != 0) {
447 			mlx5_en_err(priv->ifp,
448 			    "query port pddr failed: %d\n", error);
449 		}
450 		if (error == 0 && is_er_type != 0)
451 			media_entry.subtype = IFM_40G_ER4;
452 		break;
453 	}
454 	priv->media_active_last = media_entry.subtype | IFM_ETHER | IFM_FDX;
455 	if_setbaudrate(priv->ifp, media_entry.baudrate);
456 
457 	if_link_state_change(priv->ifp, LINK_STATE_UP);
458 }
459 
460 static void
461 mlx5e_media_status(if_t dev, struct ifmediareq *ifmr)
462 {
463 	struct mlx5e_priv *priv = if_getsoftc(dev);
464 
465 	ifmr->ifm_status = priv->media_status_last;
466 	ifmr->ifm_current = ifmr->ifm_active = priv->media_active_last |
467 	    (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
468 	    (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
469 
470 }
471 
472 static u32
473 mlx5e_find_link_mode(u32 subtype, bool ext)
474 {
475 	u32 link_mode = 0;
476 
477 	switch (subtype) {
478 	case 0:
479 		goto done;
480 	case IFM_10G_LR:
481 		subtype = IFM_10G_ER;
482 		break;
483 	case IFM_40G_ER4:
484 		subtype = IFM_40G_LR4;
485 		break;
486 	default:
487 		break;
488 	}
489 
490 	if (ext) {
491 		for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
492 			for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
493 				if (mlx5e_ext_mode_table[i][j].subtype == subtype)
494 					link_mode |= MLX5E_PROT_MASK(i);
495 			}
496 		}
497 	} else {
498 		for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
499 			if (mlx5e_mode_table[i].subtype == subtype)
500 				link_mode |= MLX5E_PROT_MASK(i);
501 		}
502 	}
503 done:
504 	return (link_mode);
505 }
506 
507 static int
508 mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
509 {
510 	return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
511 	    priv->params.rx_pauseframe_control,
512 	    priv->params.tx_pauseframe_control,
513 	    priv->params.rx_priority_flow_control,
514 	    priv->params.tx_priority_flow_control));
515 }
516 
517 static int
518 mlx5e_set_port_pfc(struct mlx5e_priv *priv)
519 {
520 	int error;
521 
522 	if (priv->gone != 0) {
523 		error = -ENXIO;
524 	} else if (priv->params.rx_pauseframe_control ||
525 	    priv->params.tx_pauseframe_control) {
526 		mlx5_en_err(priv->ifp,
527 		    "Global pauseframes must be disabled before enabling PFC.\n");
528 		error = -EINVAL;
529 	} else {
530 		error = mlx5e_set_port_pause_and_pfc(priv);
531 	}
532 	return (error);
533 }
534 
535 static int
536 mlx5e_media_change(if_t dev)
537 {
538 	struct mlx5e_priv *priv = if_getsoftc(dev);
539 	struct mlx5_core_dev *mdev = priv->mdev;
540 	u32 eth_proto_cap;
541 	u32 link_mode;
542 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
543 	int was_opened;
544 	int locked;
545 	int error;
546 	bool ext;
547 
548 	locked = PRIV_LOCKED(priv);
549 	if (!locked)
550 		PRIV_LOCK(priv);
551 
552 	if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
553 		error = EINVAL;
554 		goto done;
555 	}
556 
557 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
558 	    MLX5_PTYS_EN, 1);
559 	if (error != 0) {
560 		mlx5_en_err(dev, "Query port media capability failed\n");
561 		goto done;
562 	}
563 
564 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
565 	link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media), ext);
566 
567 	/* query supported capabilities */
568 	eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
569 	    eth_proto_capability);
570 
571 	/* check for autoselect */
572 	if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
573 		link_mode = eth_proto_cap;
574 		if (link_mode == 0) {
575 			mlx5_en_err(dev, "Port media capability is zero\n");
576 			error = EINVAL;
577 			goto done;
578 		}
579 	} else {
580 		link_mode = link_mode & eth_proto_cap;
581 		if (link_mode == 0) {
582 			mlx5_en_err(dev, "Not supported link mode requested\n");
583 			error = EINVAL;
584 			goto done;
585 		}
586 	}
587 	if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
588 		/* check if PFC is enabled */
589 		if (priv->params.rx_priority_flow_control ||
590 		    priv->params.tx_priority_flow_control) {
591 			mlx5_en_err(dev, "PFC must be disabled before enabling global pauseframes.\n");
592 			error = EINVAL;
593 			goto done;
594 		}
595 	}
596 	/* update pauseframe control bits */
597 	priv->params.rx_pauseframe_control =
598 	    (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
599 	priv->params.tx_pauseframe_control =
600 	    (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
601 
602 	/* check if device is opened */
603 	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
604 
605 	/* reconfigure the hardware */
606 	mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
607 	mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN, ext);
608 	error = -mlx5e_set_port_pause_and_pfc(priv);
609 	if (was_opened)
610 		mlx5_set_port_status(mdev, MLX5_PORT_UP);
611 
612 done:
613 	if (!locked)
614 		PRIV_UNLOCK(priv);
615 	return (error);
616 }
617 
618 static void
619 mlx5e_update_carrier_work(struct work_struct *work)
620 {
621 	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
622 	    update_carrier_work);
623 
624 	PRIV_LOCK(priv);
625 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
626 		mlx5e_update_carrier(priv);
627 	PRIV_UNLOCK(priv);
628 }
629 
630 #define	MLX5E_PCIE_PERF_GET_64(a,b,c,d,e,f)    \
631 	s_debug->c = MLX5_GET64(mpcnt_reg, out, counter_set.f.c);
632 
633 #define	MLX5E_PCIE_PERF_GET_32(a,b,c,d,e,f)    \
634 	s_debug->c = MLX5_GET(mpcnt_reg, out, counter_set.f.c);
635 
636 static void
637 mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
638 {
639 	struct mlx5_core_dev *mdev = priv->mdev;
640 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
641 	const unsigned sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
642 	void *out;
643 	void *in;
644 	int err;
645 
646 	/* allocate firmware request structures */
647 	in = mlx5_vzalloc(sz);
648 	out = mlx5_vzalloc(sz);
649 	if (in == NULL || out == NULL)
650 		goto free_out;
651 
652 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
653 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
654 	if (err != 0)
655 		goto free_out;
656 
657 	MLX5E_PCIE_PERFORMANCE_COUNTERS_64(MLX5E_PCIE_PERF_GET_64)
658 	MLX5E_PCIE_PERFORMANCE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
659 
660 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP);
661 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
662 	if (err != 0)
663 		goto free_out;
664 
665 	MLX5E_PCIE_TIMERS_AND_STATES_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
666 
667 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_LANE_COUNTERS_GROUP);
668 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
669 	if (err != 0)
670 		goto free_out;
671 
672 	MLX5E_PCIE_LANE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
673 
674 free_out:
675 	/* free firmware request structures */
676 	kvfree(in);
677 	kvfree(out);
678 }
679 
680 /*
681  * This function reads the physical port counters from the firmware
682  * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
683  * macros. The output is converted from big-endian 64-bit values into
684  * host endian ones and stored in the "priv->stats.pport" structure.
685  */
686 static void
687 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
688 {
689 	struct mlx5_core_dev *mdev = priv->mdev;
690 	struct mlx5e_pport_stats *s = &priv->stats.pport;
691 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
692 	u32 *in;
693 	u32 *out;
694 	const u64 *ptr;
695 	unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
696 	unsigned x;
697 	unsigned y;
698 	unsigned z;
699 
700 	/* allocate firmware request structures */
701 	in = mlx5_vzalloc(sz);
702 	out = mlx5_vzalloc(sz);
703 	if (in == NULL || out == NULL)
704 		goto free_out;
705 
706 	/*
707 	 * Get pointer to the 64-bit counter set which is located at a
708 	 * fixed offset in the output firmware request structure:
709 	 */
710 	ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
711 
712 	MLX5_SET(ppcnt_reg, in, local_port, 1);
713 
714 	/* read IEEE802_3 counter group using predefined counter layout */
715 	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
716 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
717 	for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
718 	     x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
719 		s->arg[y] = be64toh(ptr[x]);
720 
721 	/* read RFC2819 counter group using predefined counter layout */
722 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
723 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
724 	for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
725 		s->arg[y] = be64toh(ptr[x]);
726 
727 	for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
728 	    MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
729 		s_debug->arg[y] = be64toh(ptr[x]);
730 
731 	/* read RFC2863 counter group using predefined counter layout */
732 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
733 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
734 	for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
735 		s_debug->arg[y] = be64toh(ptr[x]);
736 
737 	/* read physical layer stats counter group using predefined counter layout */
738 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
739 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
740 	for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
741 		s_debug->arg[y] = be64toh(ptr[x]);
742 
743 	/* read Extended Ethernet counter group using predefined counter layout */
744 	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
745 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
746 	for (x = 0; x != MLX5E_PPORT_ETHERNET_EXTENDED_STATS_DEBUG_NUM; x++, y++)
747 		s_debug->arg[y] = be64toh(ptr[x]);
748 
749 	/* read Extended Statistical Group */
750 	if (MLX5_CAP_GEN(mdev, pcam_reg) &&
751 	    MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) &&
752 	    MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) {
753 		/* read Extended Statistical counter group using predefined counter layout */
754 		MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
755 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
756 
757 		for (x = 0; x != MLX5E_PPORT_STATISTICAL_DEBUG_NUM; x++, y++)
758 			s_debug->arg[y] = be64toh(ptr[x]);
759 	}
760 
761 	/* read PCIE counters */
762 	mlx5e_update_pcie_counters(priv);
763 
764 	/* read per-priority counters */
765 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
766 
767 	/* iterate all the priorities */
768 	for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
769 		MLX5_SET(ppcnt_reg, in, prio_tc, z);
770 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
771 
772 		/* read per priority stats counter group using predefined counter layout */
773 		for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
774 		    MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
775 			s->arg[y] = be64toh(ptr[x]);
776 	}
777 
778 free_out:
779 	/* free firmware request structures */
780 	kvfree(in);
781 	kvfree(out);
782 }
783 
784 static void
785 mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
786 {
787 	u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
788 	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
789 
790 	if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
791 		return;
792 
793 	MLX5_SET(query_vnic_env_in, in, opcode,
794 	    MLX5_CMD_OP_QUERY_VNIC_ENV);
795 	MLX5_SET(query_vnic_env_in, in, op_mod, 0);
796 	MLX5_SET(query_vnic_env_in, in, other_vport, 0);
797 
798 	if (mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out)) != 0)
799 		return;
800 
801 	priv->stats.vport.rx_steer_missed_packets =
802 	    MLX5_GET64(query_vnic_env_out, out,
803 	    vport_env.nic_receive_steering_discard);
804 }
805 
806 /*
807  * This function is called regularly to collect all statistics
808  * counters from the firmware. The values can be viewed through the
809  * sysctl interface. Execution is serialized using the priv's global
810  * configuration lock.
811  */
812 static void
813 mlx5e_update_stats_locked(struct mlx5e_priv *priv)
814 {
815 	struct mlx5_core_dev *mdev = priv->mdev;
816 	struct mlx5e_vport_stats *s = &priv->stats.vport;
817 	struct mlx5e_sq_stats *sq_stats;
818 	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
819 	u32 *out;
820 	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
821 	u64 tso_packets = 0;
822 	u64 tso_bytes = 0;
823 	u64 tx_queue_dropped = 0;
824 	u64 tx_defragged = 0;
825 	u64 tx_offload_none = 0;
826 	u64 lro_packets = 0;
827 	u64 lro_bytes = 0;
828 	u64 sw_lro_queued = 0;
829 	u64 sw_lro_flushed = 0;
830 	u64 rx_csum_none = 0;
831 	u64 rx_wqe_err = 0;
832 	u64 rx_packets = 0;
833 	u64 rx_bytes = 0;
834 	u64 rx_decrypted_error = 0;
835 	u64 rx_decrypted_ok = 0;
836 	u32 rx_out_of_buffer = 0;
837 	int error;
838 	int i;
839 	int j;
840 
841 	out = mlx5_vzalloc(outlen);
842 	if (out == NULL)
843 		goto free_out;
844 
845 	/* Collect firts the SW counters and then HW for consistency */
846 	for (i = 0; i < priv->params.num_channels; i++) {
847 		struct mlx5e_channel *pch = priv->channel + i;
848 		struct mlx5e_rq *rq = &pch->rq;
849 		struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
850 
851 		/* collect stats from LRO */
852 		rq_stats->sw_lro_queued = rq->lro.lro_queued;
853 		rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
854 		sw_lro_queued += rq_stats->sw_lro_queued;
855 		sw_lro_flushed += rq_stats->sw_lro_flushed;
856 		lro_packets += rq_stats->lro_packets;
857 		lro_bytes += rq_stats->lro_bytes;
858 		rx_csum_none += rq_stats->csum_none;
859 		rx_wqe_err += rq_stats->wqe_err;
860 		rx_packets += rq_stats->packets;
861 		rx_bytes += rq_stats->bytes;
862 		rx_decrypted_error += rq_stats->decrypted_error_packets;
863 		rx_decrypted_ok += rq_stats->decrypted_ok_packets;
864 
865 		for (j = 0; j < priv->num_tc; j++) {
866 			sq_stats = &pch->sq[j].stats;
867 
868 			tso_packets += sq_stats->tso_packets;
869 			tso_bytes += sq_stats->tso_bytes;
870 			tx_queue_dropped += sq_stats->dropped;
871 			tx_queue_dropped += sq_stats->enobuf;
872 			tx_defragged += sq_stats->defragged;
873 			tx_offload_none += sq_stats->csum_offload_none;
874 		}
875 	}
876 
877 #ifdef RATELIMIT
878 	/* Collect statistics from all rate-limit queues */
879 	for (j = 0; j < priv->rl.param.tx_worker_threads_def; j++) {
880 		struct mlx5e_rl_worker *rlw = priv->rl.workers + j;
881 
882 		for (i = 0; i < priv->rl.param.tx_channels_per_worker_def; i++) {
883 			struct mlx5e_rl_channel *channel = rlw->channels + i;
884 			struct mlx5e_sq *sq = channel->sq;
885 
886 			if (sq == NULL)
887 				continue;
888 
889 			sq_stats = &sq->stats;
890 
891 			tso_packets += sq_stats->tso_packets;
892 			tso_bytes += sq_stats->tso_bytes;
893 			tx_queue_dropped += sq_stats->dropped;
894 			tx_queue_dropped += sq_stats->enobuf;
895 			tx_defragged += sq_stats->defragged;
896 			tx_offload_none += sq_stats->csum_offload_none;
897 		}
898 	}
899 #endif
900 
901 	/* update counters */
902 	s->tso_packets = tso_packets;
903 	s->tso_bytes = tso_bytes;
904 	s->tx_queue_dropped = tx_queue_dropped;
905 	s->tx_defragged = tx_defragged;
906 	s->lro_packets = lro_packets;
907 	s->lro_bytes = lro_bytes;
908 	s->sw_lro_queued = sw_lro_queued;
909 	s->sw_lro_flushed = sw_lro_flushed;
910 	s->rx_csum_none = rx_csum_none;
911 	s->rx_wqe_err = rx_wqe_err;
912 	s->rx_packets = rx_packets;
913 	s->rx_bytes = rx_bytes;
914 	s->rx_decrypted_error_packets = rx_decrypted_error;
915 	s->rx_decrypted_ok_packets = rx_decrypted_ok;
916 
917 	mlx5e_grp_vnic_env_update_stats(priv);
918 
919 	/* HW counters */
920 	memset(in, 0, sizeof(in));
921 
922 	MLX5_SET(query_vport_counter_in, in, opcode,
923 	    MLX5_CMD_OP_QUERY_VPORT_COUNTER);
924 	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
925 	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
926 
927 	memset(out, 0, outlen);
928 
929 	/* get number of out-of-buffer drops first */
930 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
931 	    mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
932 	    &rx_out_of_buffer) == 0) {
933 		s->rx_out_of_buffer = rx_out_of_buffer;
934 	}
935 
936 	/* get port statistics */
937 	if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen) == 0) {
938 #define	MLX5_GET_CTR(out, x) \
939 	MLX5_GET64(query_vport_counter_out, out, x)
940 
941 		s->rx_error_packets =
942 		    MLX5_GET_CTR(out, received_errors.packets);
943 		s->rx_error_bytes =
944 		    MLX5_GET_CTR(out, received_errors.octets);
945 		s->tx_error_packets =
946 		    MLX5_GET_CTR(out, transmit_errors.packets);
947 		s->tx_error_bytes =
948 		    MLX5_GET_CTR(out, transmit_errors.octets);
949 
950 		s->rx_unicast_packets =
951 		    MLX5_GET_CTR(out, received_eth_unicast.packets);
952 		s->rx_unicast_bytes =
953 		    MLX5_GET_CTR(out, received_eth_unicast.octets);
954 		s->tx_unicast_packets =
955 		    MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
956 		s->tx_unicast_bytes =
957 		    MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
958 
959 		s->rx_multicast_packets =
960 		    MLX5_GET_CTR(out, received_eth_multicast.packets);
961 		s->rx_multicast_bytes =
962 		    MLX5_GET_CTR(out, received_eth_multicast.octets);
963 		s->tx_multicast_packets =
964 		    MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
965 		s->tx_multicast_bytes =
966 		    MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
967 
968 		s->rx_broadcast_packets =
969 		    MLX5_GET_CTR(out, received_eth_broadcast.packets);
970 		s->rx_broadcast_bytes =
971 		    MLX5_GET_CTR(out, received_eth_broadcast.octets);
972 		s->tx_broadcast_packets =
973 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
974 		s->tx_broadcast_bytes =
975 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
976 
977 		s->tx_packets = s->tx_unicast_packets +
978 		    s->tx_multicast_packets + s->tx_broadcast_packets;
979 		s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes +
980 		    s->tx_broadcast_bytes;
981 
982 		/* Update calculated offload counters */
983 		s->tx_csum_offload = s->tx_packets - tx_offload_none;
984 		s->rx_csum_good = s->rx_packets - s->rx_csum_none;
985 	}
986 
987 	/* Get physical port counters */
988 	mlx5e_update_pport_counters(priv);
989 
990 	s->tx_jumbo_packets =
991 	    priv->stats.port_stats_debug.tx_stat_p1519to2047octets +
992 	    priv->stats.port_stats_debug.tx_stat_p2048to4095octets +
993 	    priv->stats.port_stats_debug.tx_stat_p4096to8191octets +
994 	    priv->stats.port_stats_debug.tx_stat_p8192to10239octets;
995 
996 free_out:
997 	kvfree(out);
998 
999 	/* Update diagnostics, if any */
1000 	if (priv->params_ethtool.diag_pci_enable ||
1001 	    priv->params_ethtool.diag_general_enable) {
1002 		error = mlx5_core_get_diagnostics_full(mdev,
1003 		    priv->params_ethtool.diag_pci_enable ? &priv->params_pci : NULL,
1004 		    priv->params_ethtool.diag_general_enable ? &priv->params_general : NULL);
1005 		if (error != 0)
1006 			mlx5_en_err(priv->ifp,
1007 			    "Failed reading diagnostics: %d\n", error);
1008 	}
1009 
1010 	/* Update FEC, if any */
1011 	error = mlx5e_fec_update(priv);
1012 	if (error != 0 && error != EOPNOTSUPP) {
1013 		mlx5_en_err(priv->ifp,
1014 		    "Updating FEC failed: %d\n", error);
1015 	}
1016 
1017 	/* Update temperature, if any */
1018 	if (priv->params_ethtool.hw_num_temp != 0) {
1019 		error = mlx5e_hw_temperature_update(priv);
1020 		if (error != 0 && error != EOPNOTSUPP) {
1021 			mlx5_en_err(priv->ifp,
1022 			    "Updating temperature failed: %d\n", error);
1023 		}
1024 	}
1025 }
1026 
1027 static void
1028 mlx5e_update_stats_work(struct work_struct *work)
1029 {
1030 	struct mlx5e_priv *priv;
1031 
1032 	priv = container_of(work, struct mlx5e_priv, update_stats_work);
1033 	PRIV_LOCK(priv);
1034 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
1035 	    !test_bit(MLX5_INTERFACE_STATE_TEARDOWN, &priv->mdev->intf_state))
1036 		mlx5e_update_stats_locked(priv);
1037 	PRIV_UNLOCK(priv);
1038 }
1039 
1040 static void
1041 mlx5e_update_stats(void *arg)
1042 {
1043 	struct mlx5e_priv *priv = arg;
1044 
1045 	queue_work(priv->wq, &priv->update_stats_work);
1046 
1047 	callout_reset(&priv->watchdog, hz / 4, &mlx5e_update_stats, priv);
1048 }
1049 
1050 static void
1051 mlx5e_async_event_sub(struct mlx5e_priv *priv,
1052     enum mlx5_dev_event event)
1053 {
1054 	switch (event) {
1055 	case MLX5_DEV_EVENT_PORT_UP:
1056 	case MLX5_DEV_EVENT_PORT_DOWN:
1057 		queue_work(priv->wq, &priv->update_carrier_work);
1058 		break;
1059 
1060 	default:
1061 		break;
1062 	}
1063 }
1064 
1065 static void
1066 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
1067     enum mlx5_dev_event event, unsigned long param)
1068 {
1069 	struct mlx5e_priv *priv = vpriv;
1070 
1071 	mtx_lock(&priv->async_events_mtx);
1072 	if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
1073 		mlx5e_async_event_sub(priv, event);
1074 	mtx_unlock(&priv->async_events_mtx);
1075 }
1076 
1077 static void
1078 mlx5e_enable_async_events(struct mlx5e_priv *priv)
1079 {
1080 	set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1081 }
1082 
1083 static void
1084 mlx5e_disable_async_events(struct mlx5e_priv *priv)
1085 {
1086 	mtx_lock(&priv->async_events_mtx);
1087 	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1088 	mtx_unlock(&priv->async_events_mtx);
1089 }
1090 
1091 static void mlx5e_calibration_callout(void *arg);
1092 static int mlx5e_calibration_duration = 20;
1093 static int mlx5e_fast_calibration = 1;
1094 static int mlx5e_normal_calibration = 30;
1095 
1096 static SYSCTL_NODE(_hw_mlx5, OID_AUTO, calibr, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1097     "MLX5 timestamp calibration parameters");
1098 
1099 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, duration, CTLFLAG_RWTUN,
1100     &mlx5e_calibration_duration, 0,
1101     "Duration of initial calibration");
1102 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, fast, CTLFLAG_RWTUN,
1103     &mlx5e_fast_calibration, 0,
1104     "Recalibration interval during initial calibration");
1105 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, normal, CTLFLAG_RWTUN,
1106     &mlx5e_normal_calibration, 0,
1107     "Recalibration interval during normal operations");
1108 
1109 /*
1110  * Ignites the calibration process.
1111  */
1112 static void
1113 mlx5e_reset_calibration_callout(struct mlx5e_priv *priv)
1114 {
1115 
1116 	if (priv->clbr_done == 0)
1117 		mlx5e_calibration_callout(priv);
1118 	else
1119 		callout_reset_sbt_curcpu(&priv->tstmp_clbr, (priv->clbr_done <
1120 		    mlx5e_calibration_duration ? mlx5e_fast_calibration :
1121 		    mlx5e_normal_calibration) * SBT_1S, 0,
1122 		    mlx5e_calibration_callout, priv, C_DIRECT_EXEC);
1123 }
1124 
1125 static uint64_t
1126 mlx5e_timespec2usec(const struct timespec *ts)
1127 {
1128 
1129 	return ((uint64_t)ts->tv_sec * 1000000000 + ts->tv_nsec);
1130 }
1131 
1132 static uint64_t
1133 mlx5e_hw_clock(struct mlx5e_priv *priv)
1134 {
1135 	struct mlx5_init_seg *iseg;
1136 	uint32_t hw_h, hw_h1, hw_l;
1137 
1138 	iseg = priv->mdev->iseg;
1139 	do {
1140 		hw_h = ioread32be(&iseg->internal_timer_h);
1141 		hw_l = ioread32be(&iseg->internal_timer_l);
1142 		hw_h1 = ioread32be(&iseg->internal_timer_h);
1143 	} while (hw_h1 != hw_h);
1144 	return (((uint64_t)hw_h << 32) | hw_l);
1145 }
1146 
1147 /*
1148  * The calibration callout, it runs either in the context of the
1149  * thread which enables calibration, or in callout.  It takes the
1150  * snapshot of system and adapter clocks, then advances the pointers to
1151  * the calibration point to allow rx path to read the consistent data
1152  * lockless.
1153  */
1154 static void
1155 mlx5e_calibration_callout(void *arg)
1156 {
1157 	struct mlx5e_priv *priv;
1158 	struct mlx5e_clbr_point *next, *curr;
1159 	struct timespec ts;
1160 	int clbr_curr_next;
1161 
1162 	priv = arg;
1163 	curr = &priv->clbr_points[priv->clbr_curr];
1164 	clbr_curr_next = priv->clbr_curr + 1;
1165 	if (clbr_curr_next >= nitems(priv->clbr_points))
1166 		clbr_curr_next = 0;
1167 	next = &priv->clbr_points[clbr_curr_next];
1168 
1169 	next->base_prev = curr->base_curr;
1170 	next->clbr_hw_prev = curr->clbr_hw_curr;
1171 
1172 	next->clbr_hw_curr = mlx5e_hw_clock(priv);
1173 	if (((next->clbr_hw_curr - curr->clbr_hw_curr) >> MLX5E_TSTMP_PREC) ==
1174 	    0) {
1175 		if (priv->clbr_done != 0) {
1176 			mlx5_en_err(priv->ifp,
1177 			    "HW failed tstmp frozen %#jx %#jx, disabling\n",
1178 			     next->clbr_hw_curr, curr->clbr_hw_prev);
1179 			priv->clbr_done = 0;
1180 		}
1181 		atomic_store_rel_int(&curr->clbr_gen, 0);
1182 		return;
1183 	}
1184 
1185 	nanouptime(&ts);
1186 	next->base_curr = mlx5e_timespec2usec(&ts);
1187 
1188 	curr->clbr_gen = 0;
1189 	atomic_thread_fence_rel();
1190 	priv->clbr_curr = clbr_curr_next;
1191 	atomic_store_rel_int(&next->clbr_gen, ++(priv->clbr_gen));
1192 
1193 	if (priv->clbr_done < mlx5e_calibration_duration)
1194 		priv->clbr_done++;
1195 	mlx5e_reset_calibration_callout(priv);
1196 }
1197 
1198 static const char *mlx5e_rq_stats_desc[] = {
1199 	MLX5E_RQ_STATS(MLX5E_STATS_DESC)
1200 };
1201 
1202 static int
1203 mlx5e_create_rq(struct mlx5e_channel *c,
1204     struct mlx5e_rq_param *param,
1205     struct mlx5e_rq *rq)
1206 {
1207 	struct mlx5e_priv *priv = c->priv;
1208 	struct mlx5_core_dev *mdev = priv->mdev;
1209 	char buffer[16];
1210 	void *rqc = param->rqc;
1211 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1212 	int wq_sz;
1213 	int err;
1214 	int i;
1215 	u32 nsegs, wqe_sz;
1216 
1217 	err = mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
1218 	if (err != 0)
1219 		goto done;
1220 
1221 	/* Create DMA descriptor TAG */
1222 	if ((err = -bus_dma_tag_create(
1223 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1224 	    1,				/* any alignment */
1225 	    0,				/* no boundary */
1226 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1227 	    BUS_SPACE_MAXADDR,		/* highaddr */
1228 	    NULL, NULL,			/* filter, filterarg */
1229 	    nsegs * MLX5E_MAX_RX_BYTES,	/* maxsize */
1230 	    nsegs,			/* nsegments */
1231 	    nsegs * MLX5E_MAX_RX_BYTES,	/* maxsegsize */
1232 	    0,				/* flags */
1233 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1234 	    &rq->dma_tag)))
1235 		goto done;
1236 
1237 	err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1238 	    &rq->wq_ctrl);
1239 	if (err)
1240 		goto err_free_dma_tag;
1241 
1242 	rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1243 
1244 	err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1245 	if (err != 0)
1246 		goto err_rq_wq_destroy;
1247 
1248 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1249 
1250 	err = -tcp_lro_init_args(&rq->lro, priv->ifp, TCP_LRO_ENTRIES, wq_sz);
1251 	if (err)
1252 		goto err_rq_wq_destroy;
1253 
1254 	rq->mbuf = malloc_domainset(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN,
1255 	    mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
1256 	for (i = 0; i != wq_sz; i++) {
1257 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1258 		int j;
1259 
1260 		err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1261 		if (err != 0) {
1262 			while (i--)
1263 				bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1264 			goto err_rq_mbuf_free;
1265 		}
1266 
1267 		/* set value for constant fields */
1268 		for (j = 0; j < rq->nsegs; j++)
1269 			wqe->data[j].lkey = cpu_to_be32(priv->mr.key);
1270 	}
1271 
1272 	INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1273 	if (priv->params.rx_cq_moderation_mode < 2) {
1274 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1275 	} else {
1276 		void *cqc = container_of(param,
1277 		    struct mlx5e_channel_param, rq)->rx_cq.cqc;
1278 
1279 		switch (MLX5_GET(cqc, cqc, cq_period_mode)) {
1280 		case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
1281 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1282 			break;
1283 		case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
1284 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1285 			break;
1286 		default:
1287 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1288 			break;
1289 		}
1290 	}
1291 
1292 	rq->ifp = priv->ifp;
1293 	rq->channel = c;
1294 	rq->ix = c->ix;
1295 
1296 	snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
1297 	mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1298 	    buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
1299 	    rq->stats.arg);
1300 	return (0);
1301 
1302 err_rq_mbuf_free:
1303 	free(rq->mbuf, M_MLX5EN);
1304 	tcp_lro_free(&rq->lro);
1305 err_rq_wq_destroy:
1306 	mlx5_wq_destroy(&rq->wq_ctrl);
1307 err_free_dma_tag:
1308 	bus_dma_tag_destroy(rq->dma_tag);
1309 done:
1310 	return (err);
1311 }
1312 
1313 static void
1314 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1315 {
1316 	int wq_sz;
1317 	int i;
1318 
1319 	/* destroy all sysctl nodes */
1320 	sysctl_ctx_free(&rq->stats.ctx);
1321 
1322 	/* free leftover LRO packets, if any */
1323 	tcp_lro_free(&rq->lro);
1324 
1325 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1326 	for (i = 0; i != wq_sz; i++) {
1327 		if (rq->mbuf[i].mbuf != NULL) {
1328 			if (rq->mbuf[i].ipsec_mtag != NULL)
1329 				m_tag_free(&rq->mbuf[i].ipsec_mtag->tag);
1330 			bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1331 			m_freem(rq->mbuf[i].mbuf);
1332 		}
1333 		bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1334 	}
1335 	free(rq->mbuf, M_MLX5EN);
1336 	mlx5_wq_destroy(&rq->wq_ctrl);
1337 	bus_dma_tag_destroy(rq->dma_tag);
1338 }
1339 
1340 static int
1341 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1342 {
1343 	struct mlx5e_channel *c = rq->channel;
1344 	struct mlx5e_priv *priv = c->priv;
1345 	struct mlx5_core_dev *mdev = priv->mdev;
1346 	void *in;
1347 	void *rqc;
1348 	void *wq;
1349 	int inlen;
1350 	int err;
1351 	u8 ts_format;
1352 
1353 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1354 	    sizeof(u64) * rq->wq_ctrl.buf.npages;
1355 	in = mlx5_vzalloc(inlen);
1356 	if (in == NULL)
1357 		return (-ENOMEM);
1358 
1359 	ts_format = mlx5_get_rq_default_ts(mdev);
1360 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1361 	wq = MLX5_ADDR_OF(rqc, rqc, wq);
1362 
1363 	memcpy(rqc, param->rqc, sizeof(param->rqc));
1364 
1365 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1366 	MLX5_SET(rqc, rqc, ts_format, ts_format);
1367 	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1368 	if (priv->counter_set_id >= 0)
1369 		MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
1370 	MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1371 	    MLX5_ADAPTER_PAGE_SHIFT);
1372 	MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1373 
1374 	mlx5_fill_page_array(&rq->wq_ctrl.buf,
1375 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1376 
1377 	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1378 
1379 	kvfree(in);
1380 
1381 	return (err);
1382 }
1383 
1384 static int
1385 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1386 {
1387 	struct mlx5e_channel *c = rq->channel;
1388 	struct mlx5e_priv *priv = c->priv;
1389 	struct mlx5_core_dev *mdev = priv->mdev;
1390 
1391 	void *in;
1392 	void *rqc;
1393 	int inlen;
1394 	int err;
1395 
1396 	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1397 	in = mlx5_vzalloc(inlen);
1398 	if (in == NULL)
1399 		return (-ENOMEM);
1400 
1401 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1402 
1403 	MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1404 	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1405 	MLX5_SET(rqc, rqc, state, next_state);
1406 
1407 	err = mlx5_core_modify_rq(mdev, in, inlen);
1408 
1409 	kvfree(in);
1410 
1411 	return (err);
1412 }
1413 
1414 static void
1415 mlx5e_disable_rq(struct mlx5e_rq *rq)
1416 {
1417 	struct mlx5e_channel *c = rq->channel;
1418 	struct mlx5e_priv *priv = c->priv;
1419 	struct mlx5_core_dev *mdev = priv->mdev;
1420 
1421 	mlx5_core_destroy_rq(mdev, rq->rqn);
1422 }
1423 
1424 static int
1425 mlx5e_open_rq(struct mlx5e_channel *c,
1426     struct mlx5e_rq_param *param,
1427     struct mlx5e_rq *rq)
1428 {
1429 	int err;
1430 
1431 	err = mlx5e_create_rq(c, param, rq);
1432 	if (err)
1433 		return (err);
1434 
1435 	/* set CQN in RQ parameters */
1436 	MLX5_SET(rqc, param->rqc, cqn, c->rq.cq.mcq.cqn);
1437 
1438 	err = mlx5e_enable_rq(rq, param);
1439 	if (err)
1440 		goto err_destroy_rq;
1441 
1442 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1443 	if (err)
1444 		goto err_disable_rq;
1445 
1446 	c->rq.enabled = 1;
1447 
1448 	return (0);
1449 
1450 err_disable_rq:
1451 	mlx5e_disable_rq(rq);
1452 err_destroy_rq:
1453 	mlx5e_destroy_rq(rq);
1454 
1455 	return (err);
1456 }
1457 
1458 static void
1459 mlx5e_close_rq(struct mlx5e_rq *rq)
1460 {
1461 	mtx_lock(&rq->mtx);
1462 	rq->enabled = 0;
1463 	callout_stop(&rq->watchdog);
1464 	mtx_unlock(&rq->mtx);
1465 
1466 	mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1467 }
1468 
1469 static void
1470 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1471 {
1472 
1473 	mlx5e_disable_rq(rq);
1474 	mlx5e_close_cq(&rq->cq);
1475 	cancel_work_sync(&rq->dim.work);
1476 	mlx5e_destroy_rq(rq);
1477 }
1478 
1479 /*
1480  * What is a drop RQ and why is it needed?
1481  *
1482  * The RSS indirection table, also called the RQT, selects the
1483  * destination RQ based on the receive queue number, RQN. The RQT is
1484  * frequently referred to by flow steering rules to distribute traffic
1485  * among multiple RQs. The problem is that the RQs cannot be destroyed
1486  * before the RQT referring them is destroyed too. Further, TLS RX
1487  * rules may still be referring to the RQT even if the link went
1488  * down. Because there is no magic RQN for dropping packets, we create
1489  * a dummy RQ, also called drop RQ, which sole purpose is to drop all
1490  * received packets. When the link goes down this RQN is filled in all
1491  * RQT entries, of the main RQT, so the real RQs which are about to be
1492  * destroyed can be released and the TLS RX rules can be sustained.
1493  */
1494 static void
1495 mlx5e_open_drop_rq_comp(struct mlx5_core_cq *mcq __unused, struct mlx5_eqe *eqe __unused)
1496 {
1497 }
1498 
1499 static int
1500 mlx5e_open_drop_rq(struct mlx5e_priv *priv,
1501     struct mlx5e_rq *drop_rq)
1502 {
1503 	struct mlx5e_cq_param param_cq = {};
1504 	struct mlx5e_rq_param param_rq = {};
1505 	void *rqc_wq = MLX5_ADDR_OF(rqc, param_rq.rqc, wq);
1506 	int err;
1507 
1508 	/* set channel pointer */
1509 	drop_rq->channel = priv->channel;
1510 
1511 	/* set basic CQ parameters needed */
1512 	MLX5_SET(cqc, param_cq.cqc, log_cq_size, 0);
1513 	MLX5_SET(cqc, param_cq.cqc, uar_page, priv->mdev->priv.uar->index);
1514 
1515 	/* open receive completion queue */
1516 	err = mlx5e_open_cq(priv, &param_cq, &drop_rq->cq,
1517 	    &mlx5e_open_drop_rq_comp, 0);
1518 	if (err)
1519 		goto err_done;
1520 
1521 	/* set basic WQ parameters needed */
1522 	MLX5_SET(wq, rqc_wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1523 	MLX5_SET(wq, rqc_wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1524 	MLX5_SET(wq, rqc_wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) + sizeof(struct mlx5_wqe_data_seg)));
1525 	MLX5_SET(wq, rqc_wq, log_wq_sz, 0);
1526 	MLX5_SET(wq, rqc_wq, pd, priv->pdn);
1527 
1528 	param_rq.wq.linear = 1;
1529 
1530 	err = mlx5_wq_ll_create(priv->mdev, &param_rq.wq, rqc_wq, &drop_rq->wq,
1531 	    &drop_rq->wq_ctrl);
1532 	if (err)
1533 		goto err_close_cq;
1534 
1535 	/* set CQN in RQ parameters */
1536 	MLX5_SET(rqc, param_rq.rqc, cqn, drop_rq->cq.mcq.cqn);
1537 
1538 	err = mlx5e_enable_rq(drop_rq, &param_rq);
1539 	if (err)
1540 		goto err_wq_destroy;
1541 
1542 	err = mlx5e_modify_rq(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1543 	if (err)
1544 		goto err_disable_rq;
1545 
1546 	return (err);
1547 
1548 err_disable_rq:
1549 	mlx5e_disable_rq(drop_rq);
1550 err_wq_destroy:
1551 	mlx5_wq_destroy(&drop_rq->wq_ctrl);
1552 err_close_cq:
1553 	mlx5e_close_cq(&drop_rq->cq);
1554 err_done:
1555 	return (err);
1556 }
1557 
1558 static void
1559 mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
1560 {
1561 	mlx5e_modify_rq(drop_rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1562 	mlx5e_disable_rq(drop_rq);
1563 	mlx5_wq_destroy(&drop_rq->wq_ctrl);
1564 	mlx5e_close_cq(&drop_rq->cq);
1565 }
1566 
1567 void
1568 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1569 {
1570 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1571 	int x;
1572 
1573 	for (x = 0; x != wq_sz; x++) {
1574 		if (sq->mbuf[x].mbuf != NULL) {
1575 			bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1576 			m_freem(sq->mbuf[x].mbuf);
1577 		}
1578 		if (sq->mbuf[x].mst != NULL) {
1579 			m_snd_tag_rele(sq->mbuf[x].mst);
1580 			sq->mbuf[x].mst = NULL;
1581 		}
1582 		bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1583 	}
1584 	free(sq->mbuf, M_MLX5EN);
1585 }
1586 
1587 int
1588 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1589 {
1590 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1591 	int err;
1592 	int x;
1593 
1594 	sq->mbuf = malloc_domainset(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN,
1595 	    mlx5_dev_domainset(sq->priv->mdev), M_WAITOK | M_ZERO);
1596 
1597 	/* Create DMA descriptor MAPs */
1598 	for (x = 0; x != wq_sz; x++) {
1599 		err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1600 		if (err != 0) {
1601 			while (x--)
1602 				bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1603 			free(sq->mbuf, M_MLX5EN);
1604 			return (err);
1605 		}
1606 	}
1607 	return (0);
1608 }
1609 
1610 static const char *mlx5e_sq_stats_desc[] = {
1611 	MLX5E_SQ_STATS(MLX5E_STATS_DESC)
1612 };
1613 
1614 void
1615 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1616 {
1617 	sq->max_inline = sq->priv->params.tx_max_inline;
1618 	sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1619 
1620 	/*
1621 	 * Check if trust state is DSCP or if inline mode is NONE which
1622 	 * indicates CX-5 or newer hardware.
1623 	 */
1624 	if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1625 	    sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1626 		if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1627 			sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1628 		else
1629 			sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1630 	} else {
1631 		sq->min_insert_caps = 0;
1632 	}
1633 }
1634 
1635 static void
1636 mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1637 {
1638 	int i;
1639 
1640 	for (i = 0; i != priv->num_tc; i++) {
1641 		mtx_lock(&c->sq[i].lock);
1642 		mlx5e_update_sq_inline(&c->sq[i]);
1643 		mtx_unlock(&c->sq[i].lock);
1644 	}
1645 }
1646 
1647 void
1648 mlx5e_refresh_sq_inline(struct mlx5e_priv *priv)
1649 {
1650 	int i;
1651 
1652 	/* check if channels are closed */
1653 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
1654 		return;
1655 
1656 	for (i = 0; i < priv->params.num_channels; i++)
1657 		mlx5e_refresh_sq_inline_sub(priv, &priv->channel[i]);
1658 }
1659 
1660 static int
1661 mlx5e_create_sq(struct mlx5e_channel *c,
1662     int tc,
1663     struct mlx5e_sq_param *param,
1664     struct mlx5e_sq *sq)
1665 {
1666 	struct mlx5e_priv *priv = c->priv;
1667 	struct mlx5_core_dev *mdev = priv->mdev;
1668 	char buffer[16];
1669 	void *sqc = param->sqc;
1670 	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
1671 	int err;
1672 
1673 	/* Create DMA descriptor TAG */
1674 	if ((err = -bus_dma_tag_create(
1675 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1676 	    1,				/* any alignment */
1677 	    0,				/* no boundary */
1678 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1679 	    BUS_SPACE_MAXADDR,		/* highaddr */
1680 	    NULL, NULL,			/* filter, filterarg */
1681 	    MLX5E_MAX_TX_PAYLOAD_SIZE,	/* maxsize */
1682 	    MLX5E_MAX_TX_MBUF_FRAGS,	/* nsegments */
1683 	    MLX5E_MAX_TX_MBUF_SIZE,	/* maxsegsize */
1684 	    0,				/* flags */
1685 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1686 	    &sq->dma_tag)))
1687 		goto done;
1688 
1689 	sq->mkey_be = cpu_to_be32(priv->mr.key);
1690 	sq->ifp = priv->ifp;
1691 	sq->priv = priv;
1692 	sq->tc = tc;
1693 
1694 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
1695 	    &sq->wq_ctrl);
1696 	if (err)
1697 		goto err_free_dma_tag;
1698 
1699 	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1700 
1701 	err = mlx5e_alloc_sq_db(sq);
1702 	if (err)
1703 		goto err_sq_wq_destroy;
1704 
1705 	mlx5e_update_sq_inline(sq);
1706 
1707 	snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1708 	mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1709 	    buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1710 	    sq->stats.arg);
1711 
1712 	return (0);
1713 
1714 err_sq_wq_destroy:
1715 	mlx5_wq_destroy(&sq->wq_ctrl);
1716 
1717 err_free_dma_tag:
1718 	bus_dma_tag_destroy(sq->dma_tag);
1719 done:
1720 	return (err);
1721 }
1722 
1723 static void
1724 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1725 {
1726 	/* destroy all sysctl nodes */
1727 	sysctl_ctx_free(&sq->stats.ctx);
1728 
1729 	mlx5e_free_sq_db(sq);
1730 	mlx5_wq_destroy(&sq->wq_ctrl);
1731 	bus_dma_tag_destroy(sq->dma_tag);
1732 }
1733 
1734 int
1735 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1736     const struct mlx5_sq_bfreg *bfreg, int tis_num)
1737 {
1738 	void *in;
1739 	void *sqc;
1740 	void *wq;
1741 	int inlen;
1742 	int err;
1743 	u8 ts_format;
1744 
1745 	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1746 	    sizeof(u64) * sq->wq_ctrl.buf.npages;
1747 	in = mlx5_vzalloc(inlen);
1748 	if (in == NULL)
1749 		return (-ENOMEM);
1750 
1751 	sq->uar_map = bfreg->map;
1752 
1753 	ts_format = mlx5_get_sq_default_ts(sq->priv->mdev);
1754 	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1755 	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1756 
1757 	memcpy(sqc, param->sqc, sizeof(param->sqc));
1758 
1759 	MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1760 	MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1761 	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1762 	MLX5_SET(sqc, sqc, ts_format, ts_format);
1763 	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1764 	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1765 	MLX5_SET(sqc, sqc, allow_swp, 1);
1766 
1767 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1768 	MLX5_SET(wq, wq, uar_page, bfreg->index);
1769 	MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1770 	    MLX5_ADAPTER_PAGE_SHIFT);
1771 	MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1772 
1773 	mlx5_fill_page_array(&sq->wq_ctrl.buf,
1774 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1775 
1776 	err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1777 
1778 	kvfree(in);
1779 
1780 	return (err);
1781 }
1782 
1783 int
1784 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1785 {
1786 	void *in;
1787 	void *sqc;
1788 	int inlen;
1789 	int err;
1790 
1791 	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1792 	in = mlx5_vzalloc(inlen);
1793 	if (in == NULL)
1794 		return (-ENOMEM);
1795 
1796 	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1797 
1798 	MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1799 	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1800 	MLX5_SET(sqc, sqc, state, next_state);
1801 
1802 	err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1803 
1804 	kvfree(in);
1805 
1806 	return (err);
1807 }
1808 
1809 void
1810 mlx5e_disable_sq(struct mlx5e_sq *sq)
1811 {
1812 
1813 	mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1814 }
1815 
1816 static int
1817 mlx5e_open_sq(struct mlx5e_channel *c,
1818     int tc,
1819     struct mlx5e_sq_param *param,
1820     struct mlx5e_sq *sq)
1821 {
1822 	int err;
1823 
1824 	sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1825 
1826 	/* ensure the TX completion event factor is not zero */
1827 	if (sq->cev_factor == 0)
1828 		sq->cev_factor = 1;
1829 
1830 	err = mlx5e_create_sq(c, tc, param, sq);
1831 	if (err)
1832 		return (err);
1833 
1834 	err = mlx5e_enable_sq(sq, param, &c->bfreg, c->priv->tisn[tc]);
1835 	if (err)
1836 		goto err_destroy_sq;
1837 
1838 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1839 	if (err)
1840 		goto err_disable_sq;
1841 
1842 	WRITE_ONCE(sq->running, 1);
1843 
1844 	return (0);
1845 
1846 err_disable_sq:
1847 	mlx5e_disable_sq(sq);
1848 err_destroy_sq:
1849 	mlx5e_destroy_sq(sq);
1850 
1851 	return (err);
1852 }
1853 
1854 static void
1855 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1856 {
1857 	/* fill up remainder with NOPs */
1858 	while (sq->cev_counter != 0) {
1859 		while (!mlx5e_sq_has_room_for(sq, 1)) {
1860 			if (can_sleep != 0) {
1861 				mtx_unlock(&sq->lock);
1862 				msleep(4);
1863 				mtx_lock(&sq->lock);
1864 			} else {
1865 				goto done;
1866 			}
1867 		}
1868 		/* send a single NOP */
1869 		mlx5e_send_nop(sq, 1);
1870 		atomic_thread_fence_rel();
1871 	}
1872 done:
1873 	mlx5e_tx_notify_hw(sq, false);
1874 }
1875 
1876 void
1877 mlx5e_sq_cev_timeout(void *arg)
1878 {
1879 	struct mlx5e_sq *sq = arg;
1880 
1881 	mtx_assert(&sq->lock, MA_OWNED);
1882 
1883 	/* check next state */
1884 	switch (sq->cev_next_state) {
1885 	case MLX5E_CEV_STATE_SEND_NOPS:
1886 		/* fill TX ring with NOPs, if any */
1887 		mlx5e_sq_send_nops_locked(sq, 0);
1888 
1889 		/* check if completed */
1890 		if (sq->cev_counter == 0) {
1891 			sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1892 			return;
1893 		}
1894 		break;
1895 	default:
1896 		/* send NOPs on next timeout */
1897 		sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1898 		break;
1899 	}
1900 
1901 	/* restart timer */
1902 	callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1903 }
1904 
1905 void
1906 mlx5e_drain_sq(struct mlx5e_sq *sq)
1907 {
1908 	int error;
1909 	struct mlx5_core_dev *mdev= sq->priv->mdev;
1910 
1911 	/*
1912 	 * Check if already stopped.
1913 	 *
1914 	 * NOTE: Serialization of this function is managed by the
1915 	 * caller ensuring the priv's state lock is locked or in case
1916 	 * of rate limit support, a single thread manages drain and
1917 	 * resume of SQs. The "running" variable can therefore safely
1918 	 * be read without any locks.
1919 	 */
1920 	if (READ_ONCE(sq->running) == 0)
1921 		return;
1922 
1923 	/* don't put more packets into the SQ */
1924 	WRITE_ONCE(sq->running, 0);
1925 
1926 	/* serialize access to DMA rings */
1927 	mtx_lock(&sq->lock);
1928 
1929 	/* teardown event factor timer, if any */
1930 	sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1931 	callout_stop(&sq->cev_callout);
1932 
1933 	/* send dummy NOPs in order to flush the transmit ring */
1934 	mlx5e_sq_send_nops_locked(sq, 1);
1935 	mtx_unlock(&sq->lock);
1936 
1937 	/* wait till SQ is empty or link is down */
1938 	mtx_lock(&sq->lock);
1939 	while (sq->cc != sq->pc &&
1940 	    (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1941 	    mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1942 	    pci_channel_offline(mdev->pdev) == 0) {
1943 		mtx_unlock(&sq->lock);
1944 		msleep(1);
1945 		sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1946 		mtx_lock(&sq->lock);
1947 	}
1948 	mtx_unlock(&sq->lock);
1949 
1950 	/* error out remaining requests */
1951 	error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1952 	if (error != 0) {
1953 		mlx5_en_err(sq->ifp,
1954 		    "mlx5e_modify_sq() from RDY to ERR failed: %d\n", error);
1955 	}
1956 
1957 	/* wait till SQ is empty */
1958 	mtx_lock(&sq->lock);
1959 	while (sq->cc != sq->pc &&
1960 	       mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1961 	       pci_channel_offline(mdev->pdev) == 0) {
1962 		mtx_unlock(&sq->lock);
1963 		msleep(1);
1964 		sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1965 		mtx_lock(&sq->lock);
1966 	}
1967 	mtx_unlock(&sq->lock);
1968 }
1969 
1970 static void
1971 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1972 {
1973 
1974 	mlx5e_drain_sq(sq);
1975 	mlx5e_disable_sq(sq);
1976 	mlx5e_destroy_sq(sq);
1977 }
1978 
1979 static int
1980 mlx5e_create_cq(struct mlx5e_priv *priv,
1981     struct mlx5e_cq_param *param,
1982     struct mlx5e_cq *cq,
1983     mlx5e_cq_comp_t *comp,
1984     int eq_ix)
1985 {
1986 	struct mlx5_core_dev *mdev = priv->mdev;
1987 	struct mlx5_core_cq *mcq = &cq->mcq;
1988 	int eqn_not_used;
1989 	int irqn;
1990 	int err;
1991 	u32 i;
1992 
1993 	err = mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1994 	if (err)
1995 		return (err);
1996 
1997 	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1998 	    &cq->wq_ctrl);
1999 	if (err)
2000 		return (err);
2001 
2002 	mcq->cqe_sz = 64;
2003 	mcq->set_ci_db = cq->wq_ctrl.db.db;
2004 	mcq->arm_db = cq->wq_ctrl.db.db + 1;
2005 	*mcq->set_ci_db = 0;
2006 	*mcq->arm_db = 0;
2007 	mcq->vector = eq_ix;
2008 	mcq->comp = comp;
2009 	mcq->event = mlx5e_cq_error_event;
2010 	mcq->irqn = irqn;
2011 
2012 	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
2013 		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
2014 
2015 		cqe->op_own = 0xf1;
2016 	}
2017 
2018 	cq->priv = priv;
2019 
2020 	return (0);
2021 }
2022 
2023 static void
2024 mlx5e_destroy_cq(struct mlx5e_cq *cq)
2025 {
2026 	mlx5_wq_destroy(&cq->wq_ctrl);
2027 }
2028 
2029 static int
2030 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
2031 {
2032 	struct mlx5_core_cq *mcq = &cq->mcq;
2033 	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
2034 	void *in;
2035 	void *cqc;
2036 	int inlen;
2037 	int irqn_not_used;
2038 	int eqn;
2039 	int err;
2040 
2041 	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
2042 	    sizeof(u64) * cq->wq_ctrl.buf.npages;
2043 	in = mlx5_vzalloc(inlen);
2044 	if (in == NULL)
2045 		return (-ENOMEM);
2046 
2047 	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
2048 
2049 	memcpy(cqc, param->cqc, sizeof(param->cqc));
2050 
2051 	mlx5_fill_page_array(&cq->wq_ctrl.buf,
2052 	    (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
2053 
2054 	mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
2055 
2056 	MLX5_SET(cqc, cqc, c_eqn, eqn);
2057 	MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
2058 	    MLX5_ADAPTER_PAGE_SHIFT);
2059 	MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
2060 
2061 	err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen, out, sizeof(out));
2062 
2063 	kvfree(in);
2064 
2065 	if (err)
2066 		return (err);
2067 
2068 	mlx5e_cq_arm(cq, MLX5_GET_DOORBELL_LOCK(&cq->priv->doorbell_lock));
2069 
2070 	return (0);
2071 }
2072 
2073 static void
2074 mlx5e_disable_cq(struct mlx5e_cq *cq)
2075 {
2076 
2077 	mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
2078 }
2079 
2080 int
2081 mlx5e_open_cq(struct mlx5e_priv *priv,
2082     struct mlx5e_cq_param *param,
2083     struct mlx5e_cq *cq,
2084     mlx5e_cq_comp_t *comp,
2085     int eq_ix)
2086 {
2087 	int err;
2088 
2089 	err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
2090 	if (err)
2091 		return (err);
2092 
2093 	err = mlx5e_enable_cq(cq, param, eq_ix);
2094 	if (err)
2095 		goto err_destroy_cq;
2096 
2097 	return (0);
2098 
2099 err_destroy_cq:
2100 	mlx5e_destroy_cq(cq);
2101 
2102 	return (err);
2103 }
2104 
2105 void
2106 mlx5e_close_cq(struct mlx5e_cq *cq)
2107 {
2108 	mlx5e_disable_cq(cq);
2109 	mlx5e_destroy_cq(cq);
2110 }
2111 
2112 static int
2113 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2114     struct mlx5e_channel_param *cparam)
2115 {
2116 	int err;
2117 	int tc;
2118 
2119 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2120 		/* open completion queue */
2121 		err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
2122 		    &mlx5e_tx_cq_comp, c->ix);
2123 		if (err)
2124 			goto err_close_tx_cqs;
2125 	}
2126 	return (0);
2127 
2128 err_close_tx_cqs:
2129 	for (tc--; tc >= 0; tc--)
2130 		mlx5e_close_cq(&c->sq[tc].cq);
2131 
2132 	return (err);
2133 }
2134 
2135 static void
2136 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2137 {
2138 	int tc;
2139 
2140 	for (tc = 0; tc < c->priv->num_tc; tc++)
2141 		mlx5e_close_cq(&c->sq[tc].cq);
2142 }
2143 
2144 static int
2145 mlx5e_open_sqs(struct mlx5e_channel *c,
2146     struct mlx5e_channel_param *cparam)
2147 {
2148 	int err;
2149 	int tc;
2150 
2151 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2152 		err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
2153 		if (err)
2154 			goto err_close_sqs;
2155 	}
2156 
2157 	return (0);
2158 
2159 err_close_sqs:
2160 	for (tc--; tc >= 0; tc--)
2161 		mlx5e_close_sq_wait(&c->sq[tc]);
2162 
2163 	return (err);
2164 }
2165 
2166 static void
2167 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
2168 {
2169 	int tc;
2170 
2171 	for (tc = 0; tc < c->priv->num_tc; tc++)
2172 		mlx5e_close_sq_wait(&c->sq[tc]);
2173 }
2174 
2175 static void
2176 mlx5e_chan_static_init(struct mlx5e_priv *priv, struct mlx5e_channel *c, int ix)
2177 {
2178 	int tc;
2179 
2180 	/* setup priv and channel number */
2181 	c->priv = priv;
2182 	c->ix = ix;
2183 
2184 	/* setup send tag */
2185 	m_snd_tag_init(&c->tag, c->priv->ifp, &mlx5e_ul_snd_tag_sw);
2186 
2187 	init_completion(&c->completion);
2188 
2189 	mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
2190 
2191 	callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
2192 
2193 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2194 		struct mlx5e_sq *sq = c->sq + tc;
2195 
2196 		mtx_init(&sq->lock, "mlx5tx",
2197 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2198 		mtx_init(&sq->comp_lock, "mlx5comp",
2199 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2200 
2201 		callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2202 	}
2203 
2204 	mlx5e_iq_static_init(&c->iq);
2205 }
2206 
2207 static void
2208 mlx5e_chan_wait_for_completion(struct mlx5e_channel *c)
2209 {
2210 
2211 	m_snd_tag_rele(&c->tag);
2212 	wait_for_completion(&c->completion);
2213 }
2214 
2215 static void
2216 mlx5e_priv_wait_for_completion(struct mlx5e_priv *priv, const uint32_t channels)
2217 {
2218 	uint32_t x;
2219 
2220 	for (x = 0; x != channels; x++)
2221 		mlx5e_chan_wait_for_completion(&priv->channel[x]);
2222 }
2223 
2224 static void
2225 mlx5e_chan_static_destroy(struct mlx5e_channel *c)
2226 {
2227 	int tc;
2228 
2229 	callout_drain(&c->rq.watchdog);
2230 
2231 	mtx_destroy(&c->rq.mtx);
2232 
2233 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2234 		callout_drain(&c->sq[tc].cev_callout);
2235 		mtx_destroy(&c->sq[tc].lock);
2236 		mtx_destroy(&c->sq[tc].comp_lock);
2237 	}
2238 
2239 	mlx5e_iq_static_destroy(&c->iq);
2240 }
2241 
2242 static int
2243 mlx5e_open_channel(struct mlx5e_priv *priv,
2244     struct mlx5e_channel_param *cparam,
2245     struct mlx5e_channel *c)
2246 {
2247 	struct epoch_tracker et;
2248 	int i, err;
2249 
2250 	/* zero non-persistent data */
2251 	MLX5E_ZERO(&c->rq, mlx5e_rq_zero_start);
2252 	for (i = 0; i != priv->num_tc; i++)
2253 		MLX5E_ZERO(&c->sq[i], mlx5e_sq_zero_start);
2254 	MLX5E_ZERO(&c->iq, mlx5e_iq_zero_start);
2255 
2256 	/* open transmit completion queue */
2257 	err = mlx5e_open_tx_cqs(c, cparam);
2258 	if (err)
2259 		goto err_free;
2260 
2261 	/* open receive completion queue */
2262 	err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2263 	    &mlx5e_rx_cq_comp, c->ix);
2264 	if (err)
2265 		goto err_close_tx_cqs;
2266 
2267 	err = mlx5e_open_sqs(c, cparam);
2268 	if (err)
2269 		goto err_close_rx_cq;
2270 
2271 	err = mlx5e_iq_open(c, &cparam->sq, &cparam->tx_cq, &c->iq);
2272 	if (err)
2273 		goto err_close_sqs;
2274 
2275 	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2276 	if (err)
2277 		goto err_close_iq;
2278 
2279 	/* poll receive queue initially */
2280 	NET_EPOCH_ENTER(et);
2281 	c->rq.cq.mcq.comp(&c->rq.cq.mcq, NULL);
2282 	NET_EPOCH_EXIT(et);
2283 
2284 	return (0);
2285 
2286 err_close_iq:
2287 	mlx5e_iq_close(&c->iq);
2288 
2289 err_close_sqs:
2290 	mlx5e_close_sqs_wait(c);
2291 
2292 err_close_rx_cq:
2293 	mlx5e_close_cq(&c->rq.cq);
2294 
2295 err_close_tx_cqs:
2296 	mlx5e_close_tx_cqs(c);
2297 
2298 err_free:
2299 	return (err);
2300 }
2301 
2302 static void
2303 mlx5e_close_channel(struct mlx5e_channel *c)
2304 {
2305 	mlx5e_close_rq(&c->rq);
2306 }
2307 
2308 static void
2309 mlx5e_close_channel_wait(struct mlx5e_channel *c)
2310 {
2311 	mlx5e_close_rq_wait(&c->rq);
2312 	mlx5e_iq_close(&c->iq);
2313 	mlx5e_close_sqs_wait(c);
2314 	mlx5e_close_tx_cqs(c);
2315 }
2316 
2317 static int
2318 mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs)
2319 {
2320 	u32 r, n;
2321 
2322 	r = priv->params.hw_lro_en ? priv->params.lro_wqe_sz :
2323 	    MLX5E_SW2MB_MTU(if_getmtu(priv->ifp));
2324 	if (r > MJUM16BYTES)
2325 		return (-ENOMEM);
2326 
2327 	if (r > MJUM9BYTES)
2328 		r = MJUM16BYTES;
2329 	else if (r > MJUMPAGESIZE)
2330 		r = MJUM9BYTES;
2331 	else if (r > MCLBYTES)
2332 		r = MJUMPAGESIZE;
2333 	else
2334 		r = MCLBYTES;
2335 
2336 	/*
2337 	 * n + 1 must be a power of two, because stride size must be.
2338 	 * Stride size is 16 * (n + 1), as the first segment is
2339 	 * control.
2340 	 */
2341 	n = roundup_pow_of_two(1 + howmany(r, MLX5E_MAX_RX_BYTES)) - 1;
2342 	if (n > MLX5E_MAX_BUSDMA_RX_SEGS)
2343 		return (-ENOMEM);
2344 
2345 	*wqe_sz = r;
2346 	*nsegs = n;
2347 	return (0);
2348 }
2349 
2350 static void
2351 mlx5e_build_rq_param(struct mlx5e_priv *priv,
2352     struct mlx5e_rq_param *param)
2353 {
2354 	void *rqc = param->rqc;
2355 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2356 	u32 wqe_sz, nsegs;
2357 
2358 	mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
2359 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
2360 	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2361 	MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) +
2362 	    nsegs * sizeof(struct mlx5_wqe_data_seg)));
2363 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
2364 	MLX5_SET(wq, wq, pd, priv->pdn);
2365 
2366 	param->wq.linear = 1;
2367 }
2368 
2369 static void
2370 mlx5e_build_sq_param(struct mlx5e_priv *priv,
2371     struct mlx5e_sq_param *param)
2372 {
2373 	void *sqc = param->sqc;
2374 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2375 
2376 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
2377 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2378 	MLX5_SET(wq, wq, pd, priv->pdn);
2379 
2380 	param->wq.linear = 1;
2381 }
2382 
2383 static void
2384 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2385     struct mlx5e_cq_param *param)
2386 {
2387 	void *cqc = param->cqc;
2388 
2389 	MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
2390 }
2391 
2392 static void
2393 mlx5e_get_default_profile(struct mlx5e_priv *priv, int mode, struct net_dim_cq_moder *ptr)
2394 {
2395 
2396 	*ptr = net_dim_get_profile(mode, MLX5E_DIM_DEFAULT_PROFILE);
2397 
2398 	/* apply LRO restrictions */
2399 	if (priv->params.hw_lro_en &&
2400 	    ptr->pkts > MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO) {
2401 		ptr->pkts = MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO;
2402 	}
2403 }
2404 
2405 static void
2406 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2407     struct mlx5e_cq_param *param)
2408 {
2409 	struct net_dim_cq_moder curr;
2410 	void *cqc = param->cqc;
2411 
2412 	/*
2413 	 * We use MLX5_CQE_FORMAT_HASH because the RX hash mini CQE
2414 	 * format is more beneficial for FreeBSD use case.
2415 	 *
2416 	 * Adding support for MLX5_CQE_FORMAT_CSUM will require changes
2417 	 * in mlx5e_decompress_cqe.
2418 	 */
2419 	if (priv->params.cqe_zipping_en) {
2420 		MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_HASH);
2421 		MLX5_SET(cqc, cqc, cqe_compression_en, 1);
2422 	}
2423 
2424 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
2425 
2426 	switch (priv->params.rx_cq_moderation_mode) {
2427 	case 0:
2428 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2429 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2430 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2431 		break;
2432 	case 1:
2433 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2434 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2435 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2436 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2437 		else
2438 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2439 		break;
2440 	case 2:
2441 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE, &curr);
2442 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2443 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2444 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2445 		break;
2446 	case 3:
2447 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE, &curr);
2448 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2449 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2450 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2451 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2452 		else
2453 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2454 		break;
2455 	default:
2456 		break;
2457 	}
2458 
2459 	mlx5e_dim_build_cq_param(priv, param);
2460 
2461 	mlx5e_build_common_cq_param(priv, param);
2462 }
2463 
2464 static void
2465 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2466     struct mlx5e_cq_param *param)
2467 {
2468 	void *cqc = param->cqc;
2469 
2470 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
2471 	MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
2472 	MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
2473 
2474 	switch (priv->params.tx_cq_moderation_mode) {
2475 	case 0:
2476 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2477 		break;
2478 	default:
2479 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2480 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2481 		else
2482 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2483 		break;
2484 	}
2485 
2486 	mlx5e_build_common_cq_param(priv, param);
2487 }
2488 
2489 static void
2490 mlx5e_build_channel_param(struct mlx5e_priv *priv,
2491     struct mlx5e_channel_param *cparam)
2492 {
2493 	memset(cparam, 0, sizeof(*cparam));
2494 
2495 	mlx5e_build_rq_param(priv, &cparam->rq);
2496 	mlx5e_build_sq_param(priv, &cparam->sq);
2497 	mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
2498 	mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
2499 }
2500 
2501 static int
2502 mlx5e_open_channels(struct mlx5e_priv *priv)
2503 {
2504 	struct mlx5e_channel_param *cparam;
2505 	int err;
2506 	int i;
2507 
2508 	cparam = malloc(sizeof(*cparam), M_MLX5EN, M_WAITOK);
2509 
2510 	mlx5e_build_channel_param(priv, cparam);
2511 	for (i = 0; i < priv->params.num_channels; i++) {
2512 		err = mlx5e_open_channel(priv, cparam, &priv->channel[i]);
2513 		if (err)
2514 			goto err_close_channels;
2515 
2516 		/* Bind interrupt vectors, if any. */
2517 		if (priv->params_ethtool.irq_cpu_base > -1) {
2518 			cpuset_t cpuset;
2519 			int cpu;
2520 			int irq;
2521 			int eqn;
2522 			int nirq;
2523 
2524 			err = mlx5_vector2eqn(priv->mdev, i,
2525 			    &eqn, &nirq);
2526 
2527 			/* error here is non-fatal */
2528 			if (err != 0)
2529 				continue;
2530 
2531 			irq = priv->mdev->priv.msix_arr[nirq].vector;
2532 			cpu = (unsigned)(priv->params_ethtool.irq_cpu_base +
2533 			    i * priv->params_ethtool.irq_cpu_stride) % (unsigned)mp_ncpus;
2534 
2535 			CPU_ZERO(&cpuset);
2536 			CPU_SET(cpu, &cpuset);
2537 			intr_setaffinity(irq, CPU_WHICH_INTRHANDLER, &cpuset);
2538 		}
2539 	}
2540 	free(cparam, M_MLX5EN);
2541 	return (0);
2542 
2543 err_close_channels:
2544 	while (i--) {
2545 		mlx5e_close_channel(&priv->channel[i]);
2546 		mlx5e_close_channel_wait(&priv->channel[i]);
2547 	}
2548 	free(cparam, M_MLX5EN);
2549 	return (err);
2550 }
2551 
2552 static void
2553 mlx5e_close_channels(struct mlx5e_priv *priv)
2554 {
2555 	int i;
2556 
2557 	for (i = 0; i < priv->params.num_channels; i++)
2558 		mlx5e_close_channel(&priv->channel[i]);
2559 	for (i = 0; i < priv->params.num_channels; i++)
2560 		mlx5e_close_channel_wait(&priv->channel[i]);
2561 }
2562 
2563 static int
2564 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2565 {
2566 
2567 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2568 		uint8_t cq_mode;
2569 
2570 		switch (priv->params.tx_cq_moderation_mode) {
2571 		case 0:
2572 		case 2:
2573 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2574 			break;
2575 		default:
2576 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2577 			break;
2578 		}
2579 
2580 		return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2581 		    priv->params.tx_cq_moderation_usec,
2582 		    priv->params.tx_cq_moderation_pkts,
2583 		    cq_mode));
2584 	}
2585 
2586 	return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2587 	    priv->params.tx_cq_moderation_usec,
2588 	    priv->params.tx_cq_moderation_pkts));
2589 }
2590 
2591 static int
2592 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2593 {
2594 
2595 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2596 		uint8_t cq_mode;
2597 		uint8_t dim_mode;
2598 		int retval;
2599 
2600 		switch (priv->params.rx_cq_moderation_mode) {
2601 		case 0:
2602 		case 2:
2603 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2604 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2605 			break;
2606 		default:
2607 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2608 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2609 			break;
2610 		}
2611 
2612 		/* tear down dynamic interrupt moderation */
2613 		mtx_lock(&rq->mtx);
2614 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2615 		mtx_unlock(&rq->mtx);
2616 
2617 		/* wait for dynamic interrupt moderation work task, if any */
2618 		cancel_work_sync(&rq->dim.work);
2619 
2620 		if (priv->params.rx_cq_moderation_mode >= 2) {
2621 			struct net_dim_cq_moder curr;
2622 
2623 			mlx5e_get_default_profile(priv, dim_mode, &curr);
2624 
2625 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2626 			    curr.usec, curr.pkts, cq_mode);
2627 
2628 			/* set dynamic interrupt moderation mode and zero defaults */
2629 			mtx_lock(&rq->mtx);
2630 			rq->dim.mode = dim_mode;
2631 			rq->dim.state = 0;
2632 			rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2633 			mtx_unlock(&rq->mtx);
2634 		} else {
2635 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2636 			    priv->params.rx_cq_moderation_usec,
2637 			    priv->params.rx_cq_moderation_pkts,
2638 			    cq_mode);
2639 		}
2640 		return (retval);
2641 	}
2642 
2643 	return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2644 	    priv->params.rx_cq_moderation_usec,
2645 	    priv->params.rx_cq_moderation_pkts));
2646 }
2647 
2648 static int
2649 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
2650 {
2651 	int err;
2652 	int i;
2653 
2654 	err = mlx5e_refresh_rq_params(priv, &c->rq);
2655 	if (err)
2656 		goto done;
2657 
2658 	for (i = 0; i != priv->num_tc; i++) {
2659 		err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
2660 		if (err)
2661 			goto done;
2662 	}
2663 done:
2664 	return (err);
2665 }
2666 
2667 int
2668 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
2669 {
2670 	int i;
2671 
2672 	/* check if channels are closed */
2673 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2674 		return (EINVAL);
2675 
2676 	for (i = 0; i < priv->params.num_channels; i++) {
2677 		int err;
2678 
2679 		err = mlx5e_refresh_channel_params_sub(priv, &priv->channel[i]);
2680 		if (err)
2681 			return (err);
2682 	}
2683 	return (0);
2684 }
2685 
2686 static int
2687 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
2688 {
2689 	struct mlx5_core_dev *mdev = priv->mdev;
2690 	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
2691 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2692 
2693 	memset(in, 0, sizeof(in));
2694 
2695 	MLX5_SET(tisc, tisc, prio, tc);
2696 	MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
2697 
2698 	return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
2699 }
2700 
2701 static void
2702 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
2703 {
2704 	mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc], 0);
2705 }
2706 
2707 static int
2708 mlx5e_open_tises(struct mlx5e_priv *priv)
2709 {
2710 	int num_tc = priv->num_tc;
2711 	int err;
2712 	int tc;
2713 
2714 	for (tc = 0; tc < num_tc; tc++) {
2715 		err = mlx5e_open_tis(priv, tc);
2716 		if (err)
2717 			goto err_close_tises;
2718 	}
2719 
2720 	return (0);
2721 
2722 err_close_tises:
2723 	for (tc--; tc >= 0; tc--)
2724 		mlx5e_close_tis(priv, tc);
2725 
2726 	return (err);
2727 }
2728 
2729 static void
2730 mlx5e_close_tises(struct mlx5e_priv *priv)
2731 {
2732 	int num_tc = priv->num_tc;
2733 	int tc;
2734 
2735 	for (tc = 0; tc < num_tc; tc++)
2736 		mlx5e_close_tis(priv, tc);
2737 }
2738 
2739 static int
2740 mlx5e_open_default_rqt(struct mlx5e_priv *priv, u32 *prqtn, int sz)
2741 {
2742 	u32 *in;
2743 	void *rqtc;
2744 	int inlen;
2745 	int err;
2746 	int i;
2747 
2748 	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2749 	in = mlx5_vzalloc(inlen);
2750 	if (in == NULL)
2751 		return (-ENOMEM);
2752 	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2753 
2754 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2755 	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2756 
2757 	for (i = 0; i != sz; i++)
2758 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2759 
2760 	err = mlx5_core_create_rqt(priv->mdev, in, inlen, prqtn);
2761 	kvfree(in);
2762 
2763 	return (err);
2764 }
2765 
2766 static int
2767 mlx5e_open_rqts(struct mlx5e_priv *priv)
2768 {
2769 	int err;
2770 	int i;
2771 
2772 	err = mlx5e_open_default_rqt(priv, &priv->rqtn,
2773 	    1 << priv->params.rx_hash_log_tbl_sz);
2774 	if (err)
2775 		goto err_default;
2776 
2777 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2778 		err = mlx5e_open_default_rqt(priv, &priv->channel[i].rqtn, 1);
2779 		if (err)
2780 			goto err_channel;
2781 	}
2782 	return (0);
2783 
2784 err_channel:
2785 	while (i--)
2786 		mlx5_core_destroy_rqt(priv->mdev, priv->channel[i].rqtn, 0);
2787 
2788 	mlx5_core_destroy_rqt(priv->mdev, priv->rqtn, 0);
2789 
2790 err_default:
2791 	return (err);
2792 }
2793 
2794 static void
2795 mlx5e_close_rqts(struct mlx5e_priv *priv)
2796 {
2797 	int i;
2798 
2799 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++)
2800 		mlx5_core_destroy_rqt(priv->mdev, priv->channel[i].rqtn, 0);
2801 
2802 	mlx5_core_destroy_rqt(priv->mdev, priv->rqtn, 0);
2803 }
2804 
2805 static int
2806 mlx5e_activate_rqt(struct mlx5e_priv *priv)
2807 {
2808 	u32 *in;
2809 	void *rqtc;
2810 	int inlen;
2811 	int err;
2812 	int sz;
2813 	int i;
2814 
2815 	sz = 1 << priv->params.rx_hash_log_tbl_sz;
2816 
2817 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2818 	in = mlx5_vzalloc(inlen);
2819 	if (in == NULL)
2820 		return (-ENOMEM);
2821 
2822 	rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2823 
2824 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2825 	MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2826 
2827 	for (i = 0; i != sz; i++) {
2828 		int ix;
2829 #ifdef RSS
2830 		ix = rss_get_indirection_to_bucket(i);
2831 #else
2832 		ix = i;
2833 #endif
2834 		/* ensure we don't overflow */
2835 		ix %= priv->params.num_channels;
2836 
2837 		/* apply receive side scaling stride, if any */
2838 		ix -= ix % (int)priv->params.channels_rsss;
2839 
2840 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2841 	}
2842 
2843 	err = mlx5_core_modify_rqt(priv->mdev, priv->rqtn, in, inlen);
2844 	if (err)
2845 		goto err_modify;
2846 
2847 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32);
2848 
2849 	MLX5_SET(rqtc, rqtc, rqt_actual_size, 1);
2850 
2851 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2852 		int ix;
2853 #ifdef RSS
2854 		ix = rss_get_indirection_to_bucket(i);
2855 #else
2856 		ix = i;
2857 #endif
2858 		/* ensure we don't overflow */
2859 		ix %= priv->params.num_channels;
2860 
2861 		/* apply receive side scaling stride, if any */
2862 		ix -= ix % (int)priv->params.channels_rsss;
2863 
2864 		MLX5_SET(rqtc, rqtc, rq_num[0], priv->channel[ix].rq.rqn);
2865 
2866 		err = mlx5_core_modify_rqt(priv->mdev, priv->channel[i].rqtn, in, inlen);
2867 		if (err)
2868 			goto err_modify;
2869 	}
2870 
2871 err_modify:
2872 	kvfree(in);
2873 	return (err);
2874 }
2875 
2876 static int
2877 mlx5e_deactivate_rqt(struct mlx5e_priv *priv)
2878 {
2879 	u32 *in;
2880 	void *rqtc;
2881 	int inlen;
2882 	int err;
2883 	int sz;
2884 	int i;
2885 
2886 	sz = 1 << priv->params.rx_hash_log_tbl_sz;
2887 
2888 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2889 	in = mlx5_vzalloc(inlen);
2890 	if (in == NULL)
2891 		return (-ENOMEM);
2892 
2893 	rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2894 
2895 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2896 	MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2897 
2898 	for (i = 0; i != sz; i++)
2899 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2900 
2901 	err = mlx5_core_modify_rqt(priv->mdev, priv->rqtn, in, inlen);
2902 	if (err)
2903 		goto err_modify;
2904 
2905 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32);
2906 
2907 	MLX5_SET(rqtc, rqtc, rqt_actual_size, 1);
2908 
2909 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2910 		MLX5_SET(rqtc, rqtc, rq_num[0], priv->drop_rq.rqn);
2911 
2912 		err = mlx5_core_modify_rqt(priv->mdev, priv->channel[i].rqtn, in, inlen);
2913 		if (err)
2914 			goto err_modify;
2915 	}
2916 
2917 err_modify:
2918 	kvfree(in);
2919 	return (err);
2920 }
2921 
2922 #define	MLX5E_RSS_KEY_SIZE (10 * 4)	/* bytes */
2923 
2924 static void
2925 mlx5e_get_rss_key(void *key_ptr)
2926 {
2927 #ifdef RSS
2928 	rss_getkey(key_ptr);
2929 #else
2930 	static const u32 rsskey[] = {
2931 	    cpu_to_be32(0xD181C62C),
2932 	    cpu_to_be32(0xF7F4DB5B),
2933 	    cpu_to_be32(0x1983A2FC),
2934 	    cpu_to_be32(0x943E1ADB),
2935 	    cpu_to_be32(0xD9389E6B),
2936 	    cpu_to_be32(0xD1039C2C),
2937 	    cpu_to_be32(0xA74499AD),
2938 	    cpu_to_be32(0x593D56D9),
2939 	    cpu_to_be32(0xF3253C06),
2940 	    cpu_to_be32(0x2ADC1FFC),
2941 	};
2942 	CTASSERT(sizeof(rsskey) == MLX5E_RSS_KEY_SIZE);
2943 	memcpy(key_ptr, rsskey, MLX5E_RSS_KEY_SIZE);
2944 #endif
2945 }
2946 
2947 static void
2948 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt, bool inner_vxlan)
2949 {
2950 	void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2951 	void *hfsi = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
2952 	void *hfs = inner_vxlan ? hfsi : hfso;
2953 	__be32 *hkey;
2954 
2955 	MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2956 
2957 #define	ROUGH_MAX_L2_L3_HDR_SZ 256
2958 
2959 #define	MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2960 			  MLX5_HASH_FIELD_SEL_DST_IP)
2961 
2962 #define	MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2963 			  MLX5_HASH_FIELD_SEL_DST_IP   |\
2964 			  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2965 			  MLX5_HASH_FIELD_SEL_L4_DPORT)
2966 
2967 #define	MLX5_HASH_IP_IPSEC_SPI	(MLX5_HASH_FIELD_SEL_SRC_IP   |\
2968 				 MLX5_HASH_FIELD_SEL_DST_IP   |\
2969 				 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2970 
2971 	if (priv->params.hw_lro_en) {
2972 		MLX5_SET(tirc, tirc, lro_enable_mask,
2973 		    MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2974 		    MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2975 		MLX5_SET(tirc, tirc, lro_max_msg_sz,
2976 		    (priv->params.lro_wqe_sz -
2977 		    ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2978 		/* TODO: add the option to choose timer value dynamically */
2979 		MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
2980 		    MLX5_CAP_ETH(priv->mdev,
2981 		    lro_timer_supported_periods[2]));
2982 	}
2983 
2984 	if (inner_vxlan)
2985 		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
2986 
2987 	/*
2988 	 * All packets must go through the indirection table, RQT,
2989 	 * because it is not possible to modify the RQN of the TIR
2990 	 * for direct dispatchment after it is created, typically
2991 	 * when the link goes up and down.
2992 	 */
2993 	MLX5_SET(tirc, tirc, disp_type,
2994 	    MLX5_TIRC_DISP_TYPE_INDIRECT);
2995 	MLX5_SET(tirc, tirc, indirect_table,
2996 	    priv->rqtn);
2997 	MLX5_SET(tirc, tirc, rx_hash_fn,
2998 		 MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
2999 	hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
3000 
3001 	CTASSERT(MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key) >=
3002 		 MLX5E_RSS_KEY_SIZE);
3003 #ifdef RSS
3004 	/*
3005 	 * The FreeBSD RSS implementation does currently not
3006 	 * support symmetric Toeplitz hashes:
3007 	 */
3008 	MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
3009 #else
3010 	MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
3011 #endif
3012 	mlx5e_get_rss_key(hkey);
3013 
3014 	switch (tt) {
3015 	case MLX5E_TT_IPV4_TCP:
3016 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3017 		    MLX5_L3_PROT_TYPE_IPV4);
3018 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3019 		    MLX5_L4_PROT_TYPE_TCP);
3020 #ifdef RSS
3021 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
3022 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3023 			    MLX5_HASH_IP);
3024 		} else
3025 #endif
3026 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3027 		    MLX5_HASH_ALL);
3028 		break;
3029 
3030 	case MLX5E_TT_IPV6_TCP:
3031 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3032 		    MLX5_L3_PROT_TYPE_IPV6);
3033 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3034 		    MLX5_L4_PROT_TYPE_TCP);
3035 #ifdef RSS
3036 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
3037 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3038 			    MLX5_HASH_IP);
3039 		} else
3040 #endif
3041 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3042 		    MLX5_HASH_ALL);
3043 		break;
3044 
3045 	case MLX5E_TT_IPV4_UDP:
3046 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3047 		    MLX5_L3_PROT_TYPE_IPV4);
3048 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3049 		    MLX5_L4_PROT_TYPE_UDP);
3050 #ifdef RSS
3051 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
3052 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3053 			    MLX5_HASH_IP);
3054 		} else
3055 #endif
3056 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3057 		    MLX5_HASH_ALL);
3058 		break;
3059 
3060 	case MLX5E_TT_IPV6_UDP:
3061 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3062 		    MLX5_L3_PROT_TYPE_IPV6);
3063 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3064 		    MLX5_L4_PROT_TYPE_UDP);
3065 #ifdef RSS
3066 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
3067 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3068 			    MLX5_HASH_IP);
3069 		} else
3070 #endif
3071 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3072 		    MLX5_HASH_ALL);
3073 		break;
3074 
3075 	case MLX5E_TT_IPV4_IPSEC_AH:
3076 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3077 		    MLX5_L3_PROT_TYPE_IPV4);
3078 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3079 		    MLX5_HASH_IP_IPSEC_SPI);
3080 		break;
3081 
3082 	case MLX5E_TT_IPV6_IPSEC_AH:
3083 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3084 		    MLX5_L3_PROT_TYPE_IPV6);
3085 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3086 		    MLX5_HASH_IP_IPSEC_SPI);
3087 		break;
3088 
3089 	case MLX5E_TT_IPV4_IPSEC_ESP:
3090 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3091 		    MLX5_L3_PROT_TYPE_IPV4);
3092 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3093 		    MLX5_HASH_IP_IPSEC_SPI);
3094 		break;
3095 
3096 	case MLX5E_TT_IPV6_IPSEC_ESP:
3097 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3098 		    MLX5_L3_PROT_TYPE_IPV6);
3099 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3100 		    MLX5_HASH_IP_IPSEC_SPI);
3101 		break;
3102 
3103 	case MLX5E_TT_IPV4:
3104 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3105 		    MLX5_L3_PROT_TYPE_IPV4);
3106 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3107 		    MLX5_HASH_IP);
3108 		break;
3109 
3110 	case MLX5E_TT_IPV6:
3111 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3112 		    MLX5_L3_PROT_TYPE_IPV6);
3113 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3114 		    MLX5_HASH_IP);
3115 		break;
3116 
3117 	default:
3118 		break;
3119 	}
3120 }
3121 
3122 static int
3123 mlx5e_open_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
3124 {
3125 	struct mlx5_core_dev *mdev = priv->mdev;
3126 	u32 *in;
3127 	void *tirc;
3128 	int inlen;
3129 	int err;
3130 
3131 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3132 	in = mlx5_vzalloc(inlen);
3133 	if (in == NULL)
3134 		return (-ENOMEM);
3135 	tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
3136 
3137 	mlx5e_build_tir_ctx(priv, tirc, tt, inner_vxlan);
3138 
3139 	err = mlx5_core_create_tir(mdev, in, inlen, inner_vxlan ?
3140 	    &priv->tirn_inner_vxlan[tt] : &priv->tirn[tt]);
3141 
3142 	kvfree(in);
3143 
3144 	return (err);
3145 }
3146 
3147 static void
3148 mlx5e_close_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
3149 {
3150 	mlx5_core_destroy_tir(priv->mdev, inner_vxlan ?
3151 	    priv->tirn_inner_vxlan[tt] : priv->tirn[tt], 0);
3152 }
3153 
3154 static int
3155 mlx5e_open_tirs(struct mlx5e_priv *priv)
3156 {
3157 	int err;
3158 	int i;
3159 
3160 	for (i = 0; i != 2 * MLX5E_NUM_TT; i++) {
3161 		err = mlx5e_open_tir(priv, i / 2, (i % 2) ? true : false);
3162 		if (err)
3163 			goto err_close_tirs;
3164 	}
3165 
3166 	return (0);
3167 
3168 err_close_tirs:
3169 	for (i--; i >= 0; i--)
3170 		mlx5e_close_tir(priv, i / 2, (i % 2) ? true : false);
3171 
3172 	return (err);
3173 }
3174 
3175 static void
3176 mlx5e_close_tirs(struct mlx5e_priv *priv)
3177 {
3178 	int i;
3179 
3180 	for (i = 0; i != 2 * MLX5E_NUM_TT; i++)
3181 		mlx5e_close_tir(priv, i / 2, (i % 2) ? true : false);
3182 }
3183 
3184 /*
3185  * SW MTU does not include headers,
3186  * HW MTU includes all headers and checksums.
3187  */
3188 static int
3189 mlx5e_set_dev_port_mtu(if_t ifp, int sw_mtu)
3190 {
3191 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3192 	struct mlx5_core_dev *mdev = priv->mdev;
3193 	int hw_mtu;
3194 	int err;
3195 
3196 	hw_mtu = MLX5E_SW2HW_MTU(sw_mtu);
3197 
3198 	err = mlx5_set_port_mtu(mdev, hw_mtu);
3199 	if (err) {
3200 		mlx5_en_err(ifp, "mlx5_set_port_mtu failed setting %d, err=%d\n",
3201 		    sw_mtu, err);
3202 		return (err);
3203 	}
3204 
3205 	/* Update vport context MTU */
3206 	err = mlx5_set_vport_mtu(mdev, hw_mtu);
3207 	if (err) {
3208 		mlx5_en_err(ifp,
3209 		    "Failed updating vport context with MTU size, err=%d\n",
3210 		    err);
3211 	}
3212 
3213 	if_setmtu(ifp, sw_mtu);
3214 
3215 	err = mlx5_query_vport_mtu(mdev, &hw_mtu);
3216 	if (err || !hw_mtu) {
3217 		/* fallback to port oper mtu */
3218 		err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
3219 	}
3220 	if (err) {
3221 		mlx5_en_err(ifp,
3222 		    "Query port MTU, after setting new MTU value, failed\n");
3223 		return (err);
3224 	} else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
3225 		err = -E2BIG,
3226 		mlx5_en_err(ifp,
3227 		    "Port MTU %d is smaller than ifp mtu %d\n",
3228 		    hw_mtu, sw_mtu);
3229 	} else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
3230 		err = -EINVAL;
3231                 mlx5_en_err(ifp,
3232 		    "Port MTU %d is bigger than ifp mtu %d\n",
3233 		    hw_mtu, sw_mtu);
3234 	}
3235 	priv->params_ethtool.hw_mtu = hw_mtu;
3236 
3237 	/* compute MSB */
3238 	while (hw_mtu & (hw_mtu - 1))
3239 		hw_mtu &= (hw_mtu - 1);
3240 	priv->params_ethtool.hw_mtu_msb = hw_mtu;
3241 
3242 	return (err);
3243 }
3244 
3245 int
3246 mlx5e_open_locked(if_t ifp)
3247 {
3248 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3249 	int err;
3250 	u16 set_id;
3251 
3252 	/* check if already opened */
3253 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3254 		return (0);
3255 
3256 #ifdef RSS
3257 	if (rss_getnumbuckets() > priv->params.num_channels) {
3258 		mlx5_en_info(ifp,
3259 		    "NOTE: There are more RSS buckets(%u) than channels(%u) available\n",
3260 		    rss_getnumbuckets(), priv->params.num_channels);
3261 	}
3262 #endif
3263 	err = mlx5e_open_tises(priv);
3264 	if (err) {
3265 		mlx5_en_err(ifp, "mlx5e_open_tises failed, %d\n", err);
3266 		return (err);
3267 	}
3268 	err = mlx5_vport_alloc_q_counter(priv->mdev,
3269 	    MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
3270 	if (err) {
3271 		mlx5_en_err(priv->ifp,
3272 		    "mlx5_vport_alloc_q_counter failed: %d\n", err);
3273 		goto err_close_tises;
3274 	}
3275 	/* store counter set ID */
3276 	priv->counter_set_id = set_id;
3277 
3278 	err = mlx5e_open_channels(priv);
3279 	if (err) {
3280 		mlx5_en_err(ifp,
3281 		    "mlx5e_open_channels failed, %d\n", err);
3282 		goto err_dalloc_q_counter;
3283 	}
3284 	err = mlx5e_activate_rqt(priv);
3285 	if (err) {
3286 		mlx5_en_err(ifp, "mlx5e_activate_rqt failed, %d\n", err);
3287 		goto err_close_channels;
3288 	}
3289 
3290 	set_bit(MLX5E_STATE_OPENED, &priv->state);
3291 
3292 	mlx5e_update_carrier(priv);
3293 
3294 	return (0);
3295 
3296 err_close_channels:
3297 	mlx5e_close_channels(priv);
3298 
3299 err_dalloc_q_counter:
3300 	mlx5_vport_dealloc_q_counter(priv->mdev,
3301 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3302 
3303 err_close_tises:
3304 	mlx5e_close_tises(priv);
3305 
3306 	return (err);
3307 }
3308 
3309 static void
3310 mlx5e_open(void *arg)
3311 {
3312 	struct mlx5e_priv *priv = arg;
3313 
3314 	PRIV_LOCK(priv);
3315 	if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
3316 		mlx5_en_err(priv->ifp,
3317 		    "Setting port status to up failed\n");
3318 
3319 	mlx5e_open_locked(priv->ifp);
3320 	if_setdrvflagbits(priv->ifp, IFF_DRV_RUNNING, 0);
3321 	PRIV_UNLOCK(priv);
3322 }
3323 
3324 int
3325 mlx5e_close_locked(if_t ifp)
3326 {
3327 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3328 
3329 	/* check if already closed */
3330 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3331 		return (0);
3332 
3333 	clear_bit(MLX5E_STATE_OPENED, &priv->state);
3334 
3335 	if_link_state_change(priv->ifp, LINK_STATE_DOWN);
3336 
3337 	mlx5e_deactivate_rqt(priv);
3338 	mlx5e_close_channels(priv);
3339 	mlx5_vport_dealloc_q_counter(priv->mdev,
3340 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3341 	mlx5e_close_tises(priv);
3342 
3343 	return (0);
3344 }
3345 
3346 static uint64_t
3347 mlx5e_get_counter(if_t ifp, ift_counter cnt)
3348 {
3349 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3350 	u64 retval;
3351 
3352 	/* PRIV_LOCK(priv); XXX not allowed */
3353 	switch (cnt) {
3354 	case IFCOUNTER_IPACKETS:
3355 		retval = priv->stats.vport.rx_packets;
3356 		break;
3357 	case IFCOUNTER_IERRORS:
3358 		retval = priv->stats.pport.in_range_len_errors +
3359 		    priv->stats.pport.out_of_range_len +
3360 		    priv->stats.pport.too_long_errors +
3361 		    priv->stats.pport.check_seq_err +
3362 		    priv->stats.pport.alignment_err;
3363 		break;
3364 	case IFCOUNTER_IQDROPS:
3365 		retval = priv->stats.vport.rx_out_of_buffer;
3366 		break;
3367 	case IFCOUNTER_OPACKETS:
3368 		retval = priv->stats.vport.tx_packets;
3369 		break;
3370 	case IFCOUNTER_OERRORS:
3371 		retval = priv->stats.port_stats_debug.out_discards;
3372 		break;
3373 	case IFCOUNTER_IBYTES:
3374 		retval = priv->stats.vport.rx_bytes;
3375 		break;
3376 	case IFCOUNTER_OBYTES:
3377 		retval = priv->stats.vport.tx_bytes;
3378 		break;
3379 	case IFCOUNTER_IMCASTS:
3380 		retval = priv->stats.vport.rx_multicast_packets;
3381 		break;
3382 	case IFCOUNTER_OMCASTS:
3383 		retval = priv->stats.vport.tx_multicast_packets;
3384 		break;
3385 	case IFCOUNTER_OQDROPS:
3386 		retval = priv->stats.vport.tx_queue_dropped;
3387 		break;
3388 	case IFCOUNTER_COLLISIONS:
3389 		retval = priv->stats.pport.collisions;
3390 		break;
3391 	default:
3392 		retval = if_get_counter_default(ifp, cnt);
3393 		break;
3394 	}
3395 	/* PRIV_UNLOCK(priv); XXX not allowed */
3396 	return (retval);
3397 }
3398 
3399 static void
3400 mlx5e_set_rx_mode(if_t ifp)
3401 {
3402 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3403 
3404 	queue_work(priv->wq, &priv->set_rx_mode_work);
3405 }
3406 
3407 static int
3408 mlx5e_ioctl(if_t ifp, u_long command, caddr_t data)
3409 {
3410 	struct mlx5e_priv *priv;
3411 	struct ifreq *ifr;
3412 	struct ifdownreason *ifdr;
3413 	struct ifi2creq i2c;
3414 	struct ifrsskey *ifrk;
3415 	struct ifrsshash *ifrh;
3416 	struct siocsifcapnv_driver_data *drv_ioctl_data, drv_ioctl_data_d;
3417 	int error = 0;
3418 	int mask;
3419 	int size_read = 0;
3420 	int module_status;
3421 	int module_num;
3422 	int max_mtu;
3423 	uint8_t read_addr;
3424 
3425 	priv = if_getsoftc(ifp);
3426 
3427 	/* check if detaching */
3428 	if (priv == NULL || priv->gone != 0)
3429 		return (ENXIO);
3430 
3431 	switch (command) {
3432 	case SIOCSIFMTU:
3433 		ifr = (struct ifreq *)data;
3434 
3435 		PRIV_LOCK(priv);
3436 		mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
3437 
3438 		if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
3439 		    ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
3440 			int was_opened;
3441 
3442 			was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3443 			if (was_opened)
3444 				mlx5e_close_locked(ifp);
3445 
3446 			/* set new MTU */
3447 			mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
3448 
3449 			if (was_opened)
3450 				mlx5e_open_locked(ifp);
3451 		} else {
3452 			error = EINVAL;
3453 			mlx5_en_err(ifp,
3454 			    "Invalid MTU value. Min val: %d, Max val: %d\n",
3455 			    MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
3456 		}
3457 		PRIV_UNLOCK(priv);
3458 		break;
3459 	case SIOCSIFFLAGS:
3460 		if ((if_getflags(ifp) & IFF_UP) &&
3461 		    (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
3462 			mlx5e_set_rx_mode(ifp);
3463 			break;
3464 		}
3465 		PRIV_LOCK(priv);
3466 		if (if_getflags(ifp) & IFF_UP) {
3467 			if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
3468 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3469 					mlx5e_open_locked(ifp);
3470 				if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
3471 				mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
3472 			}
3473 		} else {
3474 			if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3475 				mlx5_set_port_status(priv->mdev,
3476 				    MLX5_PORT_DOWN);
3477 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3478 					mlx5e_close_locked(ifp);
3479 				mlx5e_update_carrier(priv);
3480 				if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
3481 			}
3482 		}
3483 		PRIV_UNLOCK(priv);
3484 		break;
3485 	case SIOCADDMULTI:
3486 	case SIOCDELMULTI:
3487 		mlx5e_set_rx_mode(ifp);
3488 		break;
3489 	case SIOCSIFMEDIA:
3490 	case SIOCGIFMEDIA:
3491 	case SIOCGIFXMEDIA:
3492 		ifr = (struct ifreq *)data;
3493 		error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
3494 		break;
3495 	case SIOCGIFCAPNV:
3496 		error = 0;
3497 		break;
3498 	case SIOCSIFCAP:
3499 		ifr = (struct ifreq *)data;
3500 		drv_ioctl_data = &drv_ioctl_data_d;
3501 		drv_ioctl_data->reqcap = ifr->ifr_reqcap;
3502 		PRIV_LOCK(priv);
3503 		drv_ioctl_data->reqcap2 = if_getcapenable2(ifp);
3504 		drv_ioctl_data->nvcap = NULL;
3505 		goto siocsifcap_driver;
3506 	case SIOCSIFCAPNV:
3507 		drv_ioctl_data = (struct siocsifcapnv_driver_data *)data;
3508 		PRIV_LOCK(priv);
3509 siocsifcap_driver:
3510 		mask = drv_ioctl_data->reqcap ^ if_getcapenable(ifp);
3511 
3512 		if (mask & IFCAP_TXCSUM) {
3513 			if_togglecapenable(ifp, IFCAP_TXCSUM);
3514 			if_togglehwassist(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP));
3515 
3516 			if (IFCAP_TSO4 & if_getcapenable(ifp) &&
3517 			    !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
3518 				mask &= ~IFCAP_TSO4;
3519 				if_setcapenablebit(ifp, 0, IFCAP_TSO4);
3520 				if_sethwassistbits(ifp, 0, CSUM_IP_TSO);
3521 				mlx5_en_err(ifp,
3522 				    "tso4 disabled due to -txcsum.\n");
3523 			}
3524 		}
3525 		if (mask & IFCAP_TXCSUM_IPV6) {
3526 			if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6);
3527 			if_togglehwassist(ifp, (CSUM_UDP_IPV6 | CSUM_TCP_IPV6));
3528 
3529 			if (IFCAP_TSO6 & if_getcapenable(ifp) &&
3530 			    !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
3531 				mask &= ~IFCAP_TSO6;
3532 				if_setcapenablebit(ifp, 0, IFCAP_TSO6);
3533 				if_sethwassistbits(ifp, 0, CSUM_IP6_TSO);
3534 				mlx5_en_err(ifp,
3535 				    "tso6 disabled due to -txcsum6.\n");
3536 			}
3537 		}
3538 		if (mask & IFCAP_MEXTPG)
3539 			if_togglecapenable(ifp, IFCAP_MEXTPG);
3540 		if (mask & IFCAP_TXTLS4)
3541 			if_togglecapenable(ifp, IFCAP_TXTLS4);
3542 		if (mask & IFCAP_TXTLS6)
3543 			if_togglecapenable(ifp, IFCAP_TXTLS6);
3544 #ifdef RATELIMIT
3545 		if (mask & IFCAP_TXTLS_RTLMT)
3546 			if_togglecapenable(ifp, IFCAP_TXTLS_RTLMT);
3547 #endif
3548 		if (mask & IFCAP_RXCSUM)
3549 			if_togglecapenable(ifp, IFCAP_RXCSUM);
3550 		if (mask & IFCAP_RXCSUM_IPV6)
3551 			if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
3552 		if (mask & IFCAP_TSO4) {
3553 			if (!(IFCAP_TSO4 & if_getcapenable(ifp)) &&
3554 			    !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
3555 				mlx5_en_err(ifp, "enable txcsum first.\n");
3556 				error = EAGAIN;
3557 				goto out;
3558 			}
3559 			if_togglecapenable(ifp, IFCAP_TSO4);
3560 			if_togglehwassist(ifp, CSUM_IP_TSO);
3561 		}
3562 		if (mask & IFCAP_TSO6) {
3563 			if (!(IFCAP_TSO6 & if_getcapenable(ifp)) &&
3564 			    !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
3565 				mlx5_en_err(ifp, "enable txcsum6 first.\n");
3566 				error = EAGAIN;
3567 				goto out;
3568 			}
3569 			if_togglecapenable(ifp, IFCAP_TSO6);
3570 			if_togglehwassist(ifp, CSUM_IP6_TSO);
3571 		}
3572 		if (mask & IFCAP_VLAN_HWTSO)
3573 			if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
3574 		if (mask & IFCAP_VLAN_HWFILTER) {
3575 			if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3576 				mlx5e_disable_vlan_filter(priv);
3577 			else
3578 				mlx5e_enable_vlan_filter(priv);
3579 
3580 			if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
3581 		}
3582 		if (mask & IFCAP_VLAN_HWTAGGING)
3583 			if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
3584 		if (mask & IFCAP_WOL_MAGIC)
3585 			if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
3586 		if (mask & IFCAP_VXLAN_HWCSUM) {
3587 			const bool was_enabled =
3588 			    (if_getcapenable(ifp) & IFCAP_VXLAN_HWCSUM) != 0;
3589 			if (was_enabled)
3590 				mlx5e_del_all_vxlan_rules(priv);
3591 			if_togglecapenable(ifp, IFCAP_VXLAN_HWCSUM);
3592 			if_togglehwassist(ifp, CSUM_INNER_IP | CSUM_INNER_IP_UDP |
3593 			    CSUM_INNER_IP_TCP | CSUM_INNER_IP6_UDP |
3594 			    CSUM_INNER_IP6_TCP);
3595 			if (!was_enabled) {
3596 				int err = mlx5e_add_all_vxlan_rules(priv);
3597 				if (err != 0) {
3598 					mlx5_en_err(ifp,
3599 					    "mlx5e_add_all_vxlan_rules() failed, %d (ignored)\n", err);
3600 				}
3601 			}
3602 		}
3603 		if (mask & IFCAP_VXLAN_HWTSO) {
3604 			if_togglecapenable(ifp, IFCAP_VXLAN_HWTSO);
3605 			if_togglehwassist(ifp, CSUM_INNER_IP_TSO |
3606 			    CSUM_INNER_IP6_TSO);
3607 		}
3608 
3609 		VLAN_CAPABILITIES(ifp);
3610 		/* turn off LRO means also turn of HW LRO - if it's on */
3611 		if (mask & IFCAP_LRO) {
3612 			int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3613 			bool need_restart = false;
3614 
3615 			if_togglecapenable(ifp, IFCAP_LRO);
3616 
3617 			/* figure out if updating HW LRO is needed */
3618 			if (!(if_getcapenable(ifp) & IFCAP_LRO)) {
3619 				if (priv->params.hw_lro_en) {
3620 					priv->params.hw_lro_en = false;
3621 					need_restart = true;
3622 				}
3623 			} else {
3624 				if (priv->params.hw_lro_en == false &&
3625 				    priv->params_ethtool.hw_lro != 0) {
3626 					priv->params.hw_lro_en = true;
3627 					need_restart = true;
3628 				}
3629 			}
3630 			if (was_opened && need_restart) {
3631 				mlx5e_close_locked(ifp);
3632 				mlx5e_open_locked(ifp);
3633 			}
3634 		}
3635 		if (mask & IFCAP_HWRXTSTMP) {
3636 			if_togglecapenable(ifp, IFCAP_HWRXTSTMP);
3637 			if (if_getcapenable(ifp) & IFCAP_HWRXTSTMP) {
3638 				if (priv->clbr_done == 0)
3639 					mlx5e_reset_calibration_callout(priv);
3640 			} else {
3641 				callout_drain(&priv->tstmp_clbr);
3642 				priv->clbr_done = 0;
3643 			}
3644 		}
3645 		mask = drv_ioctl_data->reqcap2 ^ if_getcapenable2(ifp);
3646 		if ((mask & IFCAP2_BIT(IFCAP2_RXTLS4)) != 0)
3647 			if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS4));
3648 		if ((mask & IFCAP2_BIT(IFCAP2_RXTLS6)) != 0)
3649 			if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS6));
3650 #ifdef IPSEC_OFFLOAD
3651 		if ((mask & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0) {
3652 			bool was_enabled = (if_getcapenable2(ifp) &
3653 			    IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0;
3654 			mlx5e_close_locked(ifp);
3655 			if (was_enabled)
3656 				ipsec_accel_on_ifdown(priv->ifp);
3657 			if_togglecapenable2(ifp,
3658 			    IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD));
3659 			mlx5e_open_locked(ifp);
3660 		}
3661 #endif
3662 out:
3663 		PRIV_UNLOCK(priv);
3664 		break;
3665 
3666 	case SIOCGI2C:
3667 		ifr = (struct ifreq *)data;
3668 
3669 		/*
3670 		 * Copy from the user-space address ifr_data to the
3671 		 * kernel-space address i2c
3672 		 */
3673 		error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3674 		if (error)
3675 			break;
3676 
3677 		if (i2c.len > sizeof(i2c.data)) {
3678 			error = EINVAL;
3679 			break;
3680 		}
3681 
3682 		PRIV_LOCK(priv);
3683 		/* Get module_num which is required for the query_eeprom */
3684 		error = mlx5_query_module_num(priv->mdev, &module_num);
3685 		if (error) {
3686 			mlx5_en_err(ifp,
3687 			    "Query module num failed, eeprom reading is not supported\n");
3688 			error = EINVAL;
3689 			goto err_i2c;
3690 		}
3691 		/* Check if module is present before doing an access */
3692 		module_status = mlx5_query_module_status(priv->mdev, module_num);
3693 		if (module_status != MLX5_MODULE_STATUS_PLUGGED_ENABLED) {
3694 			if (bootverbose)
3695 				mlx5_en_err(ifp,
3696 				    "Query module %d status: not plugged (%d), "
3697 				    "eeprom reading is not supported\n",
3698 				    module_num, module_status);
3699 			error = EINVAL;
3700 			goto err_i2c;
3701 		}
3702 		/*
3703 		 * Currently 0XA0 and 0xA2 are the only addresses permitted.
3704 		 * The internal conversion is as follows:
3705 		 */
3706 		if (i2c.dev_addr == 0xA0)
3707 			read_addr = MLX5_I2C_ADDR_LOW;
3708 		else if (i2c.dev_addr == 0xA2)
3709 			read_addr = MLX5_I2C_ADDR_HIGH;
3710 		else {
3711 			mlx5_en_err(ifp,
3712 			    "Query eeprom failed, Invalid Address: %X\n",
3713 			    i2c.dev_addr);
3714 			error = EINVAL;
3715 			goto err_i2c;
3716 		}
3717 		error = mlx5_query_eeprom(priv->mdev,
3718 		    read_addr, MLX5_EEPROM_LOW_PAGE,
3719 		    (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
3720 		    (uint32_t *)i2c.data, &size_read);
3721 		if (error) {
3722 			mlx5_en_err(ifp,
3723 			    "Query eeprom failed, eeprom reading is not supported\n");
3724 			error = EINVAL;
3725 			goto err_i2c;
3726 		}
3727 
3728 		if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
3729 			error = mlx5_query_eeprom(priv->mdev,
3730 			    read_addr, MLX5_EEPROM_LOW_PAGE,
3731 			    (uint32_t)(i2c.offset + size_read),
3732 			    (uint32_t)(i2c.len - size_read), module_num,
3733 			    (uint32_t *)(i2c.data + size_read), &size_read);
3734 		}
3735 		if (error) {
3736 			mlx5_en_err(ifp,
3737 			    "Query eeprom failed, eeprom reading is not supported\n");
3738 			error = EINVAL;
3739 			goto err_i2c;
3740 		}
3741 
3742 		error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3743 err_i2c:
3744 		PRIV_UNLOCK(priv);
3745 		break;
3746 	case SIOCGIFDOWNREASON:
3747 		ifdr = (struct ifdownreason *)data;
3748 		bzero(ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3749 		PRIV_LOCK(priv);
3750 		error = -mlx5_query_pddr_troubleshooting_info(priv->mdev, NULL,
3751 		    ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3752 		PRIV_UNLOCK(priv);
3753 		if (error == 0)
3754 			ifdr->ifdr_reason = IFDR_REASON_MSG;
3755 		break;
3756 
3757 	case SIOCGIFRSSKEY:
3758 		ifrk = (struct ifrsskey *)data;
3759 		ifrk->ifrk_func = RSS_FUNC_TOEPLITZ;
3760 		ifrk->ifrk_keylen = MLX5E_RSS_KEY_SIZE;
3761 		CTASSERT(sizeof(ifrk->ifrk_key) >= MLX5E_RSS_KEY_SIZE);
3762 		mlx5e_get_rss_key(ifrk->ifrk_key);
3763 		break;
3764 
3765 	case SIOCGIFRSSHASH:
3766 		ifrh = (struct ifrsshash *)data;
3767 		ifrh->ifrh_func = RSS_FUNC_TOEPLITZ;
3768 		ifrh->ifrh_types =
3769 		    RSS_TYPE_IPV4 |
3770 		    RSS_TYPE_TCP_IPV4 |
3771 		    RSS_TYPE_UDP_IPV4 |
3772 		    RSS_TYPE_IPV6 |
3773 		    RSS_TYPE_TCP_IPV6 |
3774 		    RSS_TYPE_UDP_IPV6;
3775 		break;
3776 
3777 	default:
3778 		error = ether_ioctl(ifp, command, data);
3779 		break;
3780 	}
3781 	return (error);
3782 }
3783 
3784 static int
3785 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3786 {
3787 	/*
3788 	 * TODO: uncoment once FW really sets all these bits if
3789 	 * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
3790 	 * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
3791 	 * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
3792 	 * -ENOTSUPP;
3793 	 */
3794 
3795 	/* TODO: add more must-to-have features */
3796 
3797 	if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3798 		return (-ENODEV);
3799 
3800 	return (0);
3801 }
3802 
3803 static u16
3804 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3805 {
3806 	const int min_size = ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN;
3807 	const int max_size = MLX5E_MAX_TX_INLINE;
3808 	const int bf_buf_size =
3809 	    ((1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U) -
3810 	    (sizeof(struct mlx5e_tx_wqe) - 2);
3811 
3812 	/* verify against driver limits */
3813 	if (bf_buf_size > max_size)
3814 		return (max_size);
3815 	else if (bf_buf_size < min_size)
3816 		return (min_size);
3817 	else
3818 		return (bf_buf_size);
3819 }
3820 
3821 static int
3822 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
3823     struct mlx5e_priv *priv,
3824     int num_comp_vectors)
3825 {
3826 	int err;
3827 
3828 	/*
3829 	 * TODO: Consider link speed for setting "log_sq_size",
3830 	 * "log_rq_size" and "cq_moderation_xxx":
3831 	 */
3832 	priv->params.log_sq_size =
3833 	    MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3834 	priv->params.log_rq_size =
3835 	    MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3836 	priv->params.rx_cq_moderation_usec =
3837 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3838 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
3839 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3840 	priv->params.rx_cq_moderation_mode =
3841 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
3842 	priv->params.rx_cq_moderation_pkts =
3843 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3844 	priv->params.tx_cq_moderation_usec =
3845 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3846 	priv->params.tx_cq_moderation_pkts =
3847 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3848 	priv->params.rx_hash_log_tbl_sz =
3849 	    (order_base_2(num_comp_vectors) >
3850 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
3851 	    order_base_2(num_comp_vectors) :
3852 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
3853 	priv->params.num_tc = 1;
3854 	priv->params.default_vlan_prio = 0;
3855 	priv->counter_set_id = -1;
3856 	priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3857 
3858 	err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3859 	if (err)
3860 		return (err);
3861 
3862 	/*
3863 	 * hw lro is currently defaulted to off. when it won't anymore we
3864 	 * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
3865 	 */
3866 	priv->params.hw_lro_en = false;
3867 	priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3868 
3869 	/*
3870 	 * CQE zipping is off, because the per-packet 32-bit Toeplitz hash
3871 	 * is then not supported. The 32-bit Toeplitz hash is needed to
3872 	 * correctly demultiplex incoming traffic into the expected
3873 	 * network queues.
3874 	 */
3875 	priv->params.cqe_zipping_en = false;
3876 
3877 	priv->mdev = mdev;
3878 	priv->params.num_channels = num_comp_vectors;
3879 	priv->params.channels_rsss = 1;
3880 	priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
3881 	priv->queue_mapping_channel_mask =
3882 	    roundup_pow_of_two(num_comp_vectors) - 1;
3883 	priv->num_tc = priv->params.num_tc;
3884 	priv->default_vlan_prio = priv->params.default_vlan_prio;
3885 
3886 	INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3887 	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3888 	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3889 
3890 	return (0);
3891 }
3892 
3893 static void
3894 mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
3895 {
3896 	bool ro_pci_enable =
3897 	    pci_get_relaxed_ordering_enabled(mdev->pdev->dev.bsddev);
3898 	bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
3899 	bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read);
3900 
3901 	MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read);
3902 	MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write);
3903 }
3904 
3905 static int
3906 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3907 		  struct mlx5_core_mkey *mkey)
3908 {
3909 	if_t ifp = priv->ifp;
3910 	struct mlx5_core_dev *mdev = priv->mdev;
3911 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3912 	void *mkc;
3913 	u32 *in;
3914 	int err;
3915 
3916 	in = mlx5_vzalloc(inlen);
3917 	if (in == NULL) {
3918 		mlx5_en_err(ifp, "failed to allocate inbox\n");
3919 		return (-ENOMEM);
3920 	}
3921 
3922 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3923 	MLX5_SET(mkc, mkc, access_mode, MLX5_ACCESS_MODE_PA);
3924 	MLX5_SET(mkc, mkc, umr_en, 1);	/* used by HW TLS */
3925 	MLX5_SET(mkc, mkc, lw, 1);
3926 	MLX5_SET(mkc, mkc, lr, 1);
3927 	mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
3928 	MLX5_SET(mkc, mkc, pd, pdn);
3929 	MLX5_SET(mkc, mkc, length64, 1);
3930 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
3931 
3932 	err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
3933 	if (err)
3934 		mlx5_en_err(ifp, "mlx5_core_create_mkey failed, %d\n",
3935 		    err);
3936 
3937 	kvfree(in);
3938 	return (err);
3939 }
3940 
3941 static const char *mlx5e_vport_stats_desc[] = {
3942 	MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
3943 };
3944 
3945 static const char *mlx5e_pport_stats_desc[] = {
3946 	MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
3947 };
3948 
3949 static int
3950 mlx5e_priv_static_init(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
3951     const uint32_t channels)
3952 {
3953 	uint32_t x;
3954 	int err;
3955 
3956 	mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
3957 	sx_init(&priv->state_lock, "mlx5state");
3958 	callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
3959 	MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
3960 	for (x = 0; x != channels; x++)
3961 		mlx5e_chan_static_init(priv, &priv->channel[x], x);
3962 
3963 	for (x = 0; x != channels; x++) {
3964 		err = mlx5_alloc_bfreg(mdev, &priv->channel[x].bfreg, false, false);
3965 		if (err)
3966 			goto err_alloc_bfreg;
3967 	}
3968 	return (0);
3969 
3970 err_alloc_bfreg:
3971 	while (x--)
3972 		mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
3973 
3974 	for (x = 0; x != channels; x++)
3975 		mlx5e_chan_static_destroy(&priv->channel[x]);
3976 	callout_drain(&priv->watchdog);
3977 	mtx_destroy(&priv->async_events_mtx);
3978 	sx_destroy(&priv->state_lock);
3979 	return (err);
3980 }
3981 
3982 static void
3983 mlx5e_priv_static_destroy(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
3984     const uint32_t channels)
3985 {
3986 	uint32_t x;
3987 
3988 	for (x = 0; x != channels; x++)
3989 		mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
3990 	for (x = 0; x != channels; x++)
3991 		mlx5e_chan_static_destroy(&priv->channel[x]);
3992 	callout_drain(&priv->watchdog);
3993 	mtx_destroy(&priv->async_events_mtx);
3994 	sx_destroy(&priv->state_lock);
3995 }
3996 
3997 static int
3998 sysctl_firmware(SYSCTL_HANDLER_ARGS)
3999 {
4000 	/*
4001 	 * %d.%d%.d the string format.
4002 	 * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
4003 	 * We need at most 5 chars to store that.
4004 	 * It also has: two "." and NULL at the end, which means we need 18
4005 	 * (5*3 + 3) chars at most.
4006 	 */
4007 	char fw[18];
4008 	struct mlx5e_priv *priv = arg1;
4009 	int error;
4010 
4011 	snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
4012 	    fw_rev_sub(priv->mdev));
4013 	error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
4014 	return (error);
4015 }
4016 
4017 static void
4018 mlx5e_disable_tx_dma(struct mlx5e_channel *ch)
4019 {
4020 	int i;
4021 
4022 	for (i = 0; i < ch->priv->num_tc; i++)
4023 		mlx5e_drain_sq(&ch->sq[i]);
4024 }
4025 
4026 static void
4027 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
4028 {
4029 
4030 	sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
4031 	sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
4032 	mlx5e_tx_notify_hw(sq, true);
4033 }
4034 
4035 void
4036 mlx5e_resume_sq(struct mlx5e_sq *sq)
4037 {
4038 	int err;
4039 
4040 	/* check if already enabled */
4041 	if (READ_ONCE(sq->running) != 0)
4042 		return;
4043 
4044 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
4045 	    MLX5_SQC_STATE_RST);
4046 	if (err != 0) {
4047 		mlx5_en_err(sq->ifp,
4048 		    "mlx5e_modify_sq() from ERR to RST failed: %d\n", err);
4049 	}
4050 
4051 	sq->cc = 0;
4052 	sq->pc = 0;
4053 
4054 	/* reset doorbell prior to moving from RST to RDY */
4055 	mlx5e_reset_sq_doorbell_record(sq);
4056 
4057 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
4058 	    MLX5_SQC_STATE_RDY);
4059 	if (err != 0) {
4060 		mlx5_en_err(sq->ifp,
4061 		    "mlx5e_modify_sq() from RST to RDY failed: %d\n", err);
4062 	}
4063 
4064 	sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
4065 	WRITE_ONCE(sq->running, 1);
4066 }
4067 
4068 static void
4069 mlx5e_enable_tx_dma(struct mlx5e_channel *ch)
4070 {
4071         int i;
4072 
4073 	for (i = 0; i < ch->priv->num_tc; i++)
4074 		mlx5e_resume_sq(&ch->sq[i]);
4075 }
4076 
4077 static void
4078 mlx5e_disable_rx_dma(struct mlx5e_channel *ch)
4079 {
4080 	struct mlx5e_rq *rq = &ch->rq;
4081 	struct epoch_tracker et;
4082 	int err;
4083 
4084 	mtx_lock(&rq->mtx);
4085 	rq->enabled = 0;
4086 	callout_stop(&rq->watchdog);
4087 	mtx_unlock(&rq->mtx);
4088 
4089 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
4090 	if (err != 0) {
4091 		mlx5_en_err(rq->ifp,
4092 		    "mlx5e_modify_rq() from RDY to RST failed: %d\n", err);
4093 	}
4094 
4095 	while (!mlx5_wq_ll_is_empty(&rq->wq)) {
4096 		msleep(1);
4097 		NET_EPOCH_ENTER(et);
4098 		rq->cq.mcq.comp(&rq->cq.mcq, NULL);
4099 		NET_EPOCH_EXIT(et);
4100 	}
4101 
4102 	/*
4103 	 * Transitioning into RST state will allow the FW to track less ERR state queues,
4104 	 * thus reducing the recv queue flushing time
4105 	 */
4106 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
4107 	if (err != 0) {
4108 		mlx5_en_err(rq->ifp,
4109 		    "mlx5e_modify_rq() from ERR to RST failed: %d\n", err);
4110 	}
4111 }
4112 
4113 static void
4114 mlx5e_enable_rx_dma(struct mlx5e_channel *ch)
4115 {
4116 	struct mlx5e_rq *rq = &ch->rq;
4117 	struct epoch_tracker et;
4118 	int err;
4119 
4120 	rq->wq.wqe_ctr = 0;
4121 	mlx5_wq_ll_update_db_record(&rq->wq);
4122 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
4123 	if (err != 0) {
4124 		mlx5_en_err(rq->ifp,
4125 		    "mlx5e_modify_rq() from RST to RDY failed: %d\n", err);
4126         }
4127 
4128 	rq->enabled = 1;
4129 
4130 	NET_EPOCH_ENTER(et);
4131 	rq->cq.mcq.comp(&rq->cq.mcq, NULL);
4132 	NET_EPOCH_EXIT(et);
4133 }
4134 
4135 void
4136 mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value)
4137 {
4138 	int i;
4139 
4140 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
4141 		return;
4142 
4143 	for (i = 0; i < priv->params.num_channels; i++) {
4144 		if (value)
4145 			mlx5e_disable_tx_dma(&priv->channel[i]);
4146 		else
4147 			mlx5e_enable_tx_dma(&priv->channel[i]);
4148 	}
4149 }
4150 
4151 void
4152 mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value)
4153 {
4154 	int i;
4155 
4156 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
4157 		return;
4158 
4159 	for (i = 0; i < priv->params.num_channels; i++) {
4160 		if (value)
4161 			mlx5e_disable_rx_dma(&priv->channel[i]);
4162 		else
4163 			mlx5e_enable_rx_dma(&priv->channel[i]);
4164 	}
4165 }
4166 
4167 static void
4168 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
4169 {
4170 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4171 	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
4172 	    priv, 0, sysctl_firmware, "A", "HCA firmware version");
4173 
4174 	SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4175 	    OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
4176 	    "Board ID");
4177 }
4178 
4179 static int
4180 mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4181 {
4182 	struct mlx5e_priv *priv = arg1;
4183 	uint8_t temp[MLX5E_MAX_PRIORITY];
4184 	uint32_t tx_pfc;
4185 	int err;
4186 	int i;
4187 
4188 	PRIV_LOCK(priv);
4189 
4190 	tx_pfc = priv->params.tx_priority_flow_control;
4191 
4192 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4193 		temp[i] = (tx_pfc >> i) & 1;
4194 
4195 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4196 	if (err || !req->newptr)
4197 		goto done;
4198 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4199 	if (err)
4200 		goto done;
4201 
4202 	priv->params.tx_priority_flow_control = 0;
4203 
4204 	/* range check input value */
4205 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4206 		if (temp[i] > 1) {
4207 			err = ERANGE;
4208 			goto done;
4209 		}
4210 		priv->params.tx_priority_flow_control |= (temp[i] << i);
4211 	}
4212 
4213 	/* check if update is required */
4214 	if (tx_pfc != priv->params.tx_priority_flow_control)
4215 		err = -mlx5e_set_port_pfc(priv);
4216 done:
4217 	if (err != 0)
4218 		priv->params.tx_priority_flow_control= tx_pfc;
4219 	PRIV_UNLOCK(priv);
4220 
4221 	return (err);
4222 }
4223 
4224 static int
4225 mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4226 {
4227 	struct mlx5e_priv *priv = arg1;
4228 	uint8_t temp[MLX5E_MAX_PRIORITY];
4229 	uint32_t rx_pfc;
4230 	int err;
4231 	int i;
4232 
4233 	PRIV_LOCK(priv);
4234 
4235 	rx_pfc = priv->params.rx_priority_flow_control;
4236 
4237 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4238 		temp[i] = (rx_pfc >> i) & 1;
4239 
4240 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4241 	if (err || !req->newptr)
4242 		goto done;
4243 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4244 	if (err)
4245 		goto done;
4246 
4247 	priv->params.rx_priority_flow_control = 0;
4248 
4249 	/* range check input value */
4250 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4251 		if (temp[i] > 1) {
4252 			err = ERANGE;
4253 			goto done;
4254 		}
4255 		priv->params.rx_priority_flow_control |= (temp[i] << i);
4256 	}
4257 
4258 	/* check if update is required */
4259 	if (rx_pfc != priv->params.rx_priority_flow_control) {
4260 		err = -mlx5e_set_port_pfc(priv);
4261 		if (err == 0 && priv->sw_is_port_buf_owner)
4262 			err = mlx5e_update_buf_lossy(priv);
4263 	}
4264 done:
4265 	if (err != 0)
4266 		priv->params.rx_priority_flow_control= rx_pfc;
4267 	PRIV_UNLOCK(priv);
4268 
4269 	return (err);
4270 }
4271 
4272 static void
4273 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
4274 {
4275 	int error;
4276 
4277 	/* enable pauseframes by default */
4278 	priv->params.tx_pauseframe_control = 1;
4279 	priv->params.rx_pauseframe_control = 1;
4280 
4281 	/* disable ports flow control, PFC, by default */
4282 	priv->params.tx_priority_flow_control = 0;
4283 	priv->params.rx_priority_flow_control = 0;
4284 
4285 	/* register pauseframe SYSCTLs */
4286 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4287 	    OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
4288 	    &priv->params.tx_pauseframe_control, 0,
4289 	    "Set to enable TX pause frames. Clear to disable.");
4290 
4291 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4292 	    OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
4293 	    &priv->params.rx_pauseframe_control, 0,
4294 	    "Set to enable RX pause frames. Clear to disable.");
4295 
4296 	/* register priority flow control, PFC, SYSCTLs */
4297 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4298 	    OID_AUTO, "tx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4299 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_tx_priority_flow_control, "CU",
4300 	    "Set to enable TX ports flow control frames for priorities 0..7. Clear to disable.");
4301 
4302 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4303 	    OID_AUTO, "rx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4304 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_rx_priority_flow_control, "CU",
4305 	    "Set to enable RX ports flow control frames for priorities 0..7. Clear to disable.");
4306 
4307 	PRIV_LOCK(priv);
4308 
4309 	/* range check */
4310 	priv->params.tx_pauseframe_control =
4311 	    priv->params.tx_pauseframe_control ? 1 : 0;
4312 	priv->params.rx_pauseframe_control =
4313 	    priv->params.rx_pauseframe_control ? 1 : 0;
4314 
4315 	/* update firmware */
4316 	error = mlx5e_set_port_pause_and_pfc(priv);
4317 	if (error == -EINVAL) {
4318 		mlx5_en_err(priv->ifp,
4319 		    "Global pauseframes must be disabled before enabling PFC.\n");
4320 		priv->params.rx_priority_flow_control = 0;
4321 		priv->params.tx_priority_flow_control = 0;
4322 
4323 		/* update firmware */
4324 		(void) mlx5e_set_port_pause_and_pfc(priv);
4325 	}
4326 	PRIV_UNLOCK(priv);
4327 }
4328 
4329 static int
4330 mlx5e_ul_snd_tag_alloc(if_t ifp,
4331     union if_snd_tag_alloc_params *params,
4332     struct m_snd_tag **ppmt)
4333 {
4334 	struct mlx5e_priv *priv;
4335 	struct mlx5e_channel *pch;
4336 
4337 	priv = if_getsoftc(ifp);
4338 
4339 	if (unlikely(priv->gone || params->hdr.flowtype == M_HASHTYPE_NONE)) {
4340 		return (EOPNOTSUPP);
4341 	} else {
4342 		/* keep this code synced with mlx5e_select_queue() */
4343 		u32 ch = priv->params.num_channels;
4344 #ifdef RSS
4345 		u32 temp;
4346 
4347 		if (rss_hash2bucket(params->hdr.flowid,
4348 		    params->hdr.flowtype, &temp) == 0)
4349 			ch = temp % ch;
4350 		else
4351 #endif
4352 			ch = (params->hdr.flowid % 128) % ch;
4353 
4354 		/*
4355 		 * NOTE: The channels array is only freed at detach
4356 		 * and it safe to return a pointer to the send tag
4357 		 * inside the channels structure as long as we
4358 		 * reference the priv.
4359 		 */
4360 		pch = priv->channel + ch;
4361 
4362 		/* check if send queue is not running */
4363 		if (unlikely(pch->sq[0].running == 0))
4364 			return (ENXIO);
4365 		m_snd_tag_ref(&pch->tag);
4366 		*ppmt = &pch->tag;
4367 		return (0);
4368 	}
4369 }
4370 
4371 static int
4372 mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4373 {
4374 	struct mlx5e_channel *pch =
4375 	    container_of(pmt, struct mlx5e_channel, tag);
4376 
4377 	params->unlimited.max_rate = -1ULL;
4378 	params->unlimited.queue_level = mlx5e_sq_queue_level(&pch->sq[0]);
4379 	return (0);
4380 }
4381 
4382 static void
4383 mlx5e_ul_snd_tag_free(struct m_snd_tag *pmt)
4384 {
4385 	struct mlx5e_channel *pch =
4386 	    container_of(pmt, struct mlx5e_channel, tag);
4387 
4388 	complete(&pch->completion);
4389 }
4390 
4391 static int
4392 mlx5e_snd_tag_alloc(if_t ifp,
4393     union if_snd_tag_alloc_params *params,
4394     struct m_snd_tag **ppmt)
4395 {
4396 
4397 	switch (params->hdr.type) {
4398 #ifdef RATELIMIT
4399 	case IF_SND_TAG_TYPE_RATE_LIMIT:
4400 		return (mlx5e_rl_snd_tag_alloc(ifp, params, ppmt));
4401 #ifdef KERN_TLS
4402 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4403 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4404 #endif
4405 #endif
4406 	case IF_SND_TAG_TYPE_UNLIMITED:
4407 		return (mlx5e_ul_snd_tag_alloc(ifp, params, ppmt));
4408 #ifdef KERN_TLS
4409 	case IF_SND_TAG_TYPE_TLS:
4410 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4411 	case IF_SND_TAG_TYPE_TLS_RX:
4412 		return (mlx5e_tls_rx_snd_tag_alloc(ifp, params, ppmt));
4413 #endif
4414 	default:
4415 		return (EOPNOTSUPP);
4416 	}
4417 }
4418 
4419 #ifdef RATELIMIT
4420 #define NUM_HDWR_RATES_MLX 13
4421 static const uint64_t adapter_rates_mlx[NUM_HDWR_RATES_MLX] = {
4422 	135375,			/* 1,083,000 */
4423 	180500,			/* 1,444,000 */
4424 	270750,			/* 2,166,000 */
4425 	361000,			/* 2,888,000 */
4426 	541500,			/* 4,332,000 */
4427 	721875,			/* 5,775,000 */
4428 	1082875,		/* 8,663,000 */
4429 	1443875,		/* 11,551,000 */
4430 	2165750,		/* 17,326,000 */
4431 	2887750,		/* 23,102,000 */
4432 	4331625,		/* 34,653,000 */
4433 	5775500,		/* 46,204,000 */
4434 	8663125			/* 69,305,000 */
4435 };
4436 
4437 static void
4438 mlx5e_ratelimit_query(if_t ifp __unused, struct if_ratelimit_query_results *q)
4439 {
4440 	/*
4441 	 * This function needs updating by the driver maintainer!
4442 	 * For the MLX card there are currently (ConectX-4?) 13
4443 	 * pre-set rates and others i.e. ConnectX-5, 6, 7??
4444 	 *
4445 	 * This will change based on later adapters
4446 	 * and this code should be updated to look at ifp
4447 	 * and figure out the specific adapter type
4448 	 * settings i.e. how many rates as well
4449 	 * as if they are fixed (as is shown here) or
4450 	 * if they are dynamic (example chelsio t4). Also if there
4451 	 * is a maximum number of flows that the adapter
4452 	 * can handle that too needs to be updated in
4453 	 * the max_flows field.
4454 	 */
4455 	q->rate_table = adapter_rates_mlx;
4456 	q->flags = RT_IS_FIXED_TABLE;
4457 	q->max_flows = 0;	/* mlx has no limit */
4458 	q->number_of_rates = NUM_HDWR_RATES_MLX;
4459 	q->min_segment_burst = 1;
4460 }
4461 #endif
4462 
4463 static void
4464 mlx5e_ifm_add(struct mlx5e_priv *priv, int type)
4465 {
4466 	ifmedia_add(&priv->media, type | IFM_ETHER, 0, NULL);
4467 	ifmedia_add(&priv->media, type | IFM_ETHER |
4468 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4469 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_RXPAUSE, 0, NULL);
4470 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_TXPAUSE, 0, NULL);
4471 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX, 0, NULL);
4472 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4473 	    IFM_ETH_RXPAUSE, 0, NULL);
4474 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4475 	    IFM_ETH_TXPAUSE, 0, NULL);
4476 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4477 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4478 }
4479 
4480 static void *
4481 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
4482 {
4483 	if_t ifp;
4484 	struct mlx5e_priv *priv;
4485 	u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
4486 	struct sysctl_oid_list *child;
4487 	int ncv = mdev->priv.eq_table.num_comp_vectors;
4488 	char unit[16];
4489 	struct pfil_head_args pa;
4490 	int err;
4491 	u32 eth_proto_cap;
4492 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
4493 	bool ext;
4494 	struct media media_entry = {};
4495 
4496 	if (mlx5e_check_required_hca_cap(mdev)) {
4497 		mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
4498 		return (NULL);
4499 	}
4500 
4501 	/*
4502 	 * Try to allocate the priv and make room for worst-case
4503 	 * number of channel structures:
4504 	 */
4505 	priv = malloc_domainset(sizeof(*priv) +
4506 	    (sizeof(priv->channel[0]) * mdev->priv.eq_table.num_comp_vectors),
4507 	    M_MLX5EN, mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
4508 
4509 	ifp = priv->ifp = if_alloc_dev(IFT_ETHER, mdev->pdev->dev.bsddev);
4510 	/* setup all static fields */
4511 	if (mlx5e_priv_static_init(priv, mdev, mdev->priv.eq_table.num_comp_vectors)) {
4512 		mlx5_core_err(mdev, "mlx5e_priv_static_init() failed\n");
4513 		goto err_free_ifp;
4514 	}
4515 
4516 	if_setsoftc(ifp, priv);
4517 	if_initname(ifp, "mce", device_get_unit(mdev->pdev->dev.bsddev));
4518 	if_setmtu(ifp, ETHERMTU);
4519 	if_setinitfn(ifp, mlx5e_open);
4520 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
4521 	if_setioctlfn(ifp, mlx5e_ioctl);
4522 	if_settransmitfn(ifp, mlx5e_xmit);
4523 	if_setqflushfn(ifp, if_qflush);
4524 	if_setgetcounterfn(ifp, mlx5e_get_counter);
4525 	if_setsendqlen(ifp, ifqmaxlen);
4526 	/*
4527          * Set driver features
4528          */
4529 	if_setcapabilities(ifp, IFCAP_NV);
4530 	if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6, 0);
4531 	if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING, 0);
4532 	if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER, 0);
4533 	if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE | IFCAP_JUMBO_MTU, 0);
4534 	if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
4535 	if_setcapabilitiesbit(ifp, IFCAP_TSO | IFCAP_VLAN_HWTSO, 0);
4536 	if_setcapabilitiesbit(ifp, IFCAP_HWSTATS | IFCAP_HWRXTSTMP, 0);
4537 	if_setcapabilitiesbit(ifp, IFCAP_MEXTPG, 0);
4538 #ifdef KERN_TLS
4539 	if (MLX5_CAP_GEN(mdev, tls_tx) != 0 &&
4540 	    MLX5_CAP_GEN(mdev, log_max_dek) != 0)
4541 		if_setcapabilitiesbit(ifp, IFCAP_TXTLS4 | IFCAP_TXTLS6, 0);
4542 	if (MLX5_CAP_GEN(mdev, tls_rx) != 0 &&
4543 	    MLX5_CAP_GEN(mdev, log_max_dek) != 0 &&
4544 	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
4545 	    ft_field_support.outer_ip_version) != 0)
4546 		if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_RXTLS4) |
4547 		    IFCAP2_BIT(IFCAP2_RXTLS6), 0);
4548 #endif
4549 #ifdef RATELIMIT
4550 	if (MLX5_CAP_GEN(mdev, qos) &&
4551 	    MLX5_CAP_QOS(mdev, packet_pacing))
4552 		if_setcapabilitiesbit(ifp, IFCAP_TXRTLMT | IFCAP_TXTLS_RTLMT,
4553 		    0);
4554 #endif
4555 	if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0);
4556 #ifdef IPSEC_OFFLOAD
4557 	if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD)
4558 		if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD),
4559 		    0);
4560 #endif
4561 
4562 	if_setsndtagallocfn(ifp, mlx5e_snd_tag_alloc);
4563 #ifdef RATELIMIT
4564 	if_setratelimitqueryfn(ifp, mlx5e_ratelimit_query);
4565 #endif
4566 	/* set TSO limits so that we don't have to drop TX packets */
4567 	if_sethwtsomax(ifp, MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
4568 	if_sethwtsomaxsegcount(ifp, MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */);
4569 	if_sethwtsomaxsegsize(ifp, MLX5E_MAX_TX_MBUF_SIZE);
4570 
4571 	if_setcapenable(ifp, if_getcapabilities(ifp));
4572 	if_setcapenable2(ifp, if_getcapabilities2(ifp));
4573 	if_sethwassist(ifp, 0);
4574 	if (if_getcapenable(ifp) & IFCAP_TSO)
4575 		if_sethwassistbits(ifp, CSUM_TSO, 0);
4576 	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
4577 		if_sethwassistbits(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP), 0);
4578 	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
4579 		if_sethwassistbits(ifp, (CSUM_UDP_IPV6 | CSUM_TCP_IPV6), 0);
4580 	if (if_getcapabilities(ifp) & IFCAP_VXLAN_HWCSUM)
4581 		if_sethwassistbits(ifp, CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP |
4582 		    CSUM_INNER_IP | CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP |
4583 		    CSUM_ENCAP_VXLAN, 0);
4584 	if (if_getcapabilities(ifp) & IFCAP_VXLAN_HWTSO)
4585 		if_sethwassistbits(ifp, CSUM_INNER_IP6_TSO | CSUM_INNER_IP_TSO, 0);
4586 
4587 	/* ifnet sysctl tree */
4588 	sysctl_ctx_init(&priv->sysctl_ctx);
4589 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
4590 	    OID_AUTO, if_getdname(ifp), CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4591 	    "MLX5 ethernet - interface name");
4592 	if (priv->sysctl_ifnet == NULL) {
4593 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4594 		goto err_free_sysctl;
4595 	}
4596 	snprintf(unit, sizeof(unit), "%d", if_getdunit(ifp));
4597 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4598 	    OID_AUTO, unit, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4599 	    "MLX5 ethernet - interface unit");
4600 	if (priv->sysctl_ifnet == NULL) {
4601 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4602 		goto err_free_sysctl;
4603 	}
4604 
4605 	/* HW sysctl tree */
4606 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
4607 	priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
4608 	    OID_AUTO, "hw", CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4609 	    "MLX5 ethernet dev hw");
4610 	if (priv->sysctl_hw == NULL) {
4611 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4612 		goto err_free_sysctl;
4613 	}
4614 
4615 	err = mlx5e_build_ifp_priv(mdev, priv, ncv);
4616 	if (err) {
4617 		mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err);
4618 		goto err_free_sysctl;
4619 	}
4620 
4621 	/* reuse mlx5core's watchdog workqueue */
4622 	priv->wq = mdev->priv.health.wq_watchdog;
4623 
4624 	err = mlx5_core_alloc_pd(mdev, &priv->pdn, 0);
4625 	if (err) {
4626 		mlx5_en_err(ifp, "mlx5_core_alloc_pd failed, %d\n", err);
4627 		goto err_free_wq;
4628 	}
4629 	err = mlx5_alloc_transport_domain(mdev, &priv->tdn, 0);
4630 	if (err) {
4631 		mlx5_en_err(ifp,
4632 		    "mlx5_alloc_transport_domain failed, %d\n", err);
4633 		goto err_dealloc_pd;
4634 	}
4635 	err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
4636 	if (err) {
4637 		mlx5_en_err(ifp, "mlx5e_create_mkey failed, %d\n", err);
4638 		goto err_dealloc_transport_domain;
4639 	}
4640 	mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
4641 
4642 	/* check if we should generate a random MAC address */
4643 	if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
4644 	    is_zero_ether_addr(dev_addr)) {
4645 		random_ether_addr(dev_addr);
4646 		mlx5_en_err(ifp, "Assigned random MAC address\n");
4647 	}
4648 
4649 	err = mlx5e_rl_init(priv);
4650 	if (err) {
4651 		mlx5_en_err(ifp, "mlx5e_rl_init failed, %d\n", err);
4652 		goto err_create_mkey;
4653 	}
4654 
4655 	err = mlx5e_tls_init(priv);
4656 	if (err) {
4657 		if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
4658 		goto err_rl_init;
4659 	}
4660 
4661 	if ((if_getcapenable2(ifp) & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0) {
4662 		err = mlx5e_ipsec_init(priv);
4663 		if (err) {
4664 			if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
4665 			goto err_tls_init;
4666 		}
4667 	}
4668 
4669 	err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
4670 	if (err) {
4671 		if_printf(ifp, "%s: mlx5e_open_drop_rq failed (%d)\n", __func__, err);
4672 		goto err_ipsec_init;
4673 	}
4674 
4675 	err = mlx5e_open_rqts(priv);
4676 	if (err) {
4677 		if_printf(ifp, "%s: mlx5e_open_rqts failed (%d)\n", __func__, err);
4678 		goto err_open_drop_rq;
4679 	}
4680 
4681 	err = mlx5e_open_tirs(priv);
4682 	if (err) {
4683 		mlx5_en_err(ifp, "mlx5e_open_tirs() failed, %d\n", err);
4684 		goto err_open_rqts;
4685 	}
4686 
4687 	err = mlx5e_open_flow_tables(priv);
4688 	if (err) {
4689 		if_printf(ifp, "%s: mlx5e_open_flow_tables failed (%d)\n", __func__, err);
4690 		goto err_open_tirs;
4691 	}
4692 
4693 	err = mlx5e_tls_rx_init(priv);
4694 	if (err) {
4695 		if_printf(ifp, "%s: mlx5e_tls_rx_init() failed, %d\n", __func__, err);
4696 		goto err_open_flow_tables;
4697 	}
4698 
4699 	/* set default MTU */
4700 	mlx5e_set_dev_port_mtu(ifp, if_getmtu(ifp));
4701 
4702 	/* Set default media status */
4703 	priv->media_status_last = IFM_AVALID;
4704 	priv->media_active_last = IFM_ETHER | IFM_AUTO | IFM_FDX;
4705 
4706 	/* setup default pauseframes configuration */
4707 	mlx5e_setup_pauseframes(priv);
4708 
4709 	/* Setup supported medias */
4710 	if (!mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1)) {
4711 		ext = MLX5_CAP_PCAM_FEATURE(mdev,
4712 		    ptys_extended_ethernet);
4713 		eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
4714 		    eth_proto_capability);
4715 	} else {
4716 		ext = false;
4717 		eth_proto_cap = 0;
4718 		mlx5_en_err(ifp, "Query port media capability failed, %d\n", err);
4719 	}
4720 
4721 	ifmedia_init(&priv->media, IFM_IMASK,
4722 	    mlx5e_media_change, mlx5e_media_status);
4723 
4724 	if (ext) {
4725 		for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
4726 			/* check if hardware has the right capability */
4727 			if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4728 				continue;
4729 			for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
4730 				media_entry = mlx5e_ext_mode_table[i][j];
4731 				if (media_entry.subtype == 0)
4732 					continue;
4733 				/* check if this subtype was already added */
4734 				for (unsigned k = 0; k != i; k++) {
4735 					/* check if hardware has the right capability */
4736 					if (MLX5E_PROT_MASK(k) & ~eth_proto_cap)
4737 						continue;
4738 					for (unsigned m = 0; m != MLX5E_CABLE_TYPE_NUMBER; m++) {
4739 						if (media_entry.subtype == mlx5e_ext_mode_table[k][m].subtype)
4740 							goto skip_ext_media;
4741 					}
4742 				}
4743 				mlx5e_ifm_add(priv, media_entry.subtype);
4744 			skip_ext_media:;
4745 			}
4746 		}
4747 	} else {
4748 		for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
4749 			media_entry = mlx5e_mode_table[i];
4750 			if (media_entry.subtype == 0)
4751 				continue;
4752 			if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4753 				continue;
4754 			/* check if this subtype was already added */
4755 			for (unsigned k = 0; k != i; k++) {
4756 				if (media_entry.subtype == mlx5e_mode_table[k].subtype)
4757 					goto skip_media;
4758 			}
4759 			mlx5e_ifm_add(priv, media_entry.subtype);
4760 
4761 			/* NOTE: 10G ER and LR shares the same entry */
4762 			if (media_entry.subtype == IFM_10G_ER)
4763 				mlx5e_ifm_add(priv, IFM_10G_LR);
4764 		skip_media:;
4765 		}
4766 	}
4767 
4768 	mlx5e_ifm_add(priv, IFM_AUTO);
4769 
4770 	/* Set autoselect by default */
4771 	ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4772 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
4773 
4774 	DEBUGNET_SET(ifp, mlx5_en);
4775 
4776 	ether_ifattach(ifp, dev_addr);
4777 
4778 	/* Register for VLAN events */
4779 	priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
4780 	    mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
4781 	priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
4782 	    mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
4783 
4784 	/* Register for VxLAN events */
4785 	priv->vxlan_start = EVENTHANDLER_REGISTER(vxlan_start,
4786 	    mlx5e_vxlan_start, priv, EVENTHANDLER_PRI_ANY);
4787 	priv->vxlan_stop = EVENTHANDLER_REGISTER(vxlan_stop,
4788 	    mlx5e_vxlan_stop, priv, EVENTHANDLER_PRI_ANY);
4789 
4790 	/* Link is down by default */
4791 	if_link_state_change(ifp, LINK_STATE_DOWN);
4792 
4793 	mlx5e_enable_async_events(priv);
4794 
4795 	mlx5e_add_hw_stats(priv);
4796 
4797 	mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4798 	    "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
4799 	    priv->stats.vport.arg);
4800 
4801 	mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4802 	    "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
4803 	    priv->stats.pport.arg);
4804 
4805 	mlx5e_create_ethtool(priv);
4806 
4807 	mtx_lock(&priv->async_events_mtx);
4808 	mlx5e_update_stats(priv);
4809 	mtx_unlock(&priv->async_events_mtx);
4810 
4811 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4812 	    OID_AUTO, "rx_clbr_done", CTLFLAG_RD,
4813 	    &priv->clbr_done, 0,
4814 	    "RX timestamps calibration state");
4815 	callout_init(&priv->tstmp_clbr, 1);
4816 	/* Pull out the frequency of the clock in hz */
4817 	priv->cclk = (uint64_t)MLX5_CAP_GEN(mdev, device_frequency_khz) * 1000ULL;
4818 	mlx5e_reset_calibration_callout(priv);
4819 
4820 	pa.pa_version = PFIL_VERSION;
4821 	pa.pa_flags = PFIL_IN;
4822 	pa.pa_type = PFIL_TYPE_ETHERNET;
4823 	pa.pa_headname = if_name(ifp);
4824 	priv->pfil = pfil_head_register(&pa);
4825 
4826 	PRIV_LOCK(priv);
4827 	err = mlx5e_open_flow_rules(priv);
4828 	if (err) {
4829 		mlx5_en_err(ifp,
4830 		    "mlx5e_open_flow_rules() failed, %d (ignored)\n", err);
4831 	}
4832 	PRIV_UNLOCK(priv);
4833 
4834 	return (priv);
4835 
4836 err_open_flow_tables:
4837 	mlx5e_close_flow_tables(priv);
4838 
4839 err_open_tirs:
4840 	mlx5e_close_tirs(priv);
4841 
4842 err_open_rqts:
4843 	mlx5e_close_rqts(priv);
4844 
4845 err_open_drop_rq:
4846 	mlx5e_close_drop_rq(&priv->drop_rq);
4847 
4848 err_ipsec_init:
4849 	mlx5e_ipsec_cleanup(priv);
4850 
4851 err_tls_init:
4852 	mlx5e_tls_cleanup(priv);
4853 
4854 err_rl_init:
4855 	mlx5e_rl_cleanup(priv);
4856 
4857 err_create_mkey:
4858 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4859 
4860 err_dealloc_transport_domain:
4861 	mlx5_dealloc_transport_domain(mdev, priv->tdn, 0);
4862 
4863 err_dealloc_pd:
4864 	mlx5_core_dealloc_pd(mdev, priv->pdn, 0);
4865 
4866 err_free_wq:
4867 	flush_workqueue(priv->wq);
4868 
4869 err_free_sysctl:
4870 	sysctl_ctx_free(&priv->sysctl_ctx);
4871 	if (priv->sysctl_debug)
4872 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4873 	mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
4874 
4875 err_free_ifp:
4876 	if_free(ifp);
4877 	free(priv, M_MLX5EN);
4878 	return (NULL);
4879 }
4880 
4881 static void
4882 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
4883 {
4884 	struct mlx5e_priv *priv = vpriv;
4885 	if_t ifp = priv->ifp;
4886 
4887 	/* don't allow more IOCTLs */
4888 	priv->gone = 1;
4889 
4890 	/* XXX wait a bit to allow IOCTL handlers to complete */
4891 	pause("W", hz);
4892 
4893 #ifdef RATELIMIT
4894 	/*
4895 	 * Tell the TCP ratelimit code to release the rate-sets attached
4896 	 * to our ifnet.
4897 	 */
4898 	tcp_rl_release_ifnet(ifp);
4899 	/*
4900 	 * The kernel can still have reference(s) via the m_snd_tag's into
4901 	 * the ratelimit channels, and these must go away before
4902 	 * detaching:
4903 	 */
4904 	while (READ_ONCE(priv->rl.stats.tx_active_connections) != 0) {
4905 		mlx5_en_err(priv->ifp,
4906 		    "Waiting for all ratelimit connections to terminate\n");
4907 		pause("W", hz);
4908 	}
4909 #endif
4910 
4911 #ifdef KERN_TLS
4912 	/* wait for all TLS tags to get freed */
4913 	while (priv->tls.init != 0 &&
4914 	    uma_zone_get_cur(priv->tls.zone) != 0)  {
4915 		mlx5_en_err(priv->ifp,
4916 		    "Waiting for all TLS connections to terminate\n");
4917 		pause("W", hz);
4918 	}
4919 
4920 	/* wait for all TLS RX tags to get freed */
4921 	while (priv->tls_rx.init != 0 &&
4922 	    uma_zone_get_cur(priv->tls_rx.zone) != 0)  {
4923 		mlx5_en_err(priv->ifp,
4924 		    "Waiting for all TLS RX connections to terminate\n");
4925 		pause("W", hz);
4926 	}
4927 #endif
4928 	/* wait for all unlimited send tags to complete */
4929 	mlx5e_priv_wait_for_completion(priv, mdev->priv.eq_table.num_comp_vectors);
4930 
4931 	/* stop watchdog timer */
4932 	callout_drain(&priv->watchdog);
4933 
4934 	callout_drain(&priv->tstmp_clbr);
4935 
4936 	if (priv->vlan_attach != NULL)
4937 		EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
4938 	if (priv->vlan_detach != NULL)
4939 		EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
4940 	if (priv->vxlan_start != NULL)
4941 		EVENTHANDLER_DEREGISTER(vxlan_start, priv->vxlan_start);
4942 	if (priv->vxlan_stop != NULL)
4943 		EVENTHANDLER_DEREGISTER(vxlan_stop, priv->vxlan_stop);
4944 
4945 	/* make sure device gets closed */
4946 	PRIV_LOCK(priv);
4947 	mlx5e_close_locked(ifp);
4948 	mlx5e_close_flow_rules(priv);
4949 	PRIV_UNLOCK(priv);
4950 
4951 	/* deregister pfil */
4952 	if (priv->pfil != NULL) {
4953 		pfil_head_unregister(priv->pfil);
4954 		priv->pfil = NULL;
4955 	}
4956 
4957 	/* unregister device */
4958 	ifmedia_removeall(&priv->media);
4959 	ether_ifdetach(ifp);
4960 
4961 	mlx5e_tls_rx_cleanup(priv);
4962 #ifdef IPSEC_OFFLOAD
4963 	ipsec_accel_on_ifdown(priv->ifp);
4964 #endif
4965 	mlx5e_close_flow_tables(priv);
4966 	mlx5e_close_tirs(priv);
4967 	mlx5e_close_rqts(priv);
4968 	mlx5e_close_drop_rq(&priv->drop_rq);
4969 	mlx5e_ipsec_cleanup(priv);
4970 	mlx5e_tls_cleanup(priv);
4971 	mlx5e_rl_cleanup(priv);
4972 
4973 	/* destroy all remaining sysctl nodes */
4974 	sysctl_ctx_free(&priv->stats.vport.ctx);
4975 	sysctl_ctx_free(&priv->stats.pport.ctx);
4976 	if (priv->sysctl_debug)
4977 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4978 	sysctl_ctx_free(&priv->sysctl_ctx);
4979 
4980 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4981 	mlx5_dealloc_transport_domain(priv->mdev, priv->tdn, 0);
4982 	mlx5_core_dealloc_pd(priv->mdev, priv->pdn, 0);
4983 	mlx5e_disable_async_events(priv);
4984 	flush_workqueue(priv->wq);
4985 	mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
4986 	if_free(ifp);
4987 	free(priv, M_MLX5EN);
4988 }
4989 
4990 #ifdef DEBUGNET
4991 static void
4992 mlx5_en_debugnet_init(if_t dev, int *nrxr, int *ncl, int *clsize)
4993 {
4994 	struct mlx5e_priv *priv = if_getsoftc(dev);
4995 
4996 	PRIV_LOCK(priv);
4997 	*nrxr = priv->params.num_channels;
4998 	*ncl = DEBUGNET_MAX_IN_FLIGHT;
4999 	*clsize = MLX5E_MAX_RX_BYTES;
5000 	PRIV_UNLOCK(priv);
5001 }
5002 
5003 static void
5004 mlx5_en_debugnet_event(if_t dev, enum debugnet_ev event)
5005 {
5006 }
5007 
5008 static int
5009 mlx5_en_debugnet_transmit(if_t dev, struct mbuf *m)
5010 {
5011 	struct mlx5e_priv *priv = if_getsoftc(dev);
5012 	struct mlx5e_sq *sq;
5013 	int err;
5014 
5015 	if ((if_getdrvflags(dev) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
5016 	    IFF_DRV_RUNNING || (priv->media_status_last & IFM_ACTIVE) == 0)
5017 		return (ENOENT);
5018 
5019 	sq = &priv->channel[0].sq[0];
5020 
5021 	if (sq->running == 0) {
5022 		m_freem(m);
5023 		return (ENOENT);
5024 	}
5025 
5026 	if (mlx5e_sq_xmit(sq, &m) != 0) {
5027 		m_freem(m);
5028 		err = ENOBUFS;
5029 	} else {
5030 		err = 0;
5031 	}
5032 
5033 	mlx5e_tx_notify_hw(sq, true);
5034 
5035 	return (err);
5036 }
5037 
5038 static int
5039 mlx5_en_debugnet_poll(if_t dev, int count)
5040 {
5041 	struct mlx5e_priv *priv = if_getsoftc(dev);
5042 
5043 	if ((if_getdrvflags(dev) & IFF_DRV_RUNNING) == 0 ||
5044 	    (priv->media_status_last & IFM_ACTIVE) == 0)
5045 		return (ENOENT);
5046 
5047 	mlx5_poll_interrupts(priv->mdev);
5048 
5049 	return (0);
5050 }
5051 #endif /* DEBUGNET */
5052 
5053 static void *
5054 mlx5e_get_ifp(void *vpriv)
5055 {
5056 	struct mlx5e_priv *priv = vpriv;
5057 
5058 	return (priv->ifp);
5059 }
5060 
5061 static struct mlx5_interface mlx5e_interface = {
5062 	.add = mlx5e_create_ifp,
5063 	.remove = mlx5e_destroy_ifp,
5064 	.event = mlx5e_async_event,
5065 	.protocol = MLX5_INTERFACE_PROTOCOL_ETH,
5066 	.get_dev = mlx5e_get_ifp,
5067 };
5068 
5069 void
5070 mlx5e_init(void)
5071 {
5072 	mlx5_register_interface(&mlx5e_interface);
5073 }
5074 
5075 void
5076 mlx5e_cleanup(void)
5077 {
5078 	mlx5_unregister_interface(&mlx5e_interface);
5079 }
5080 
5081 module_init_order(mlx5e_init, SI_ORDER_SIXTH);
5082 module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);
5083 
5084 MODULE_DEPEND(mlx5en, ipsec, 1, 1, 1);
5085 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
5086 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
5087 MODULE_VERSION(mlx5en, 1);
5088