xref: /freebsd/sys/dev/mlx5/mlx5_en/mlx5_en_main.c (revision eac7052fdebb90caf2f653e06187bdbca837b9c7)
1 /*-
2  * Copyright (c) 2015-2018 Mellanox Technologies. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  *
25  * $FreeBSD$
26  */
27 
28 #include "opt_kern_tls.h"
29 
30 #include "en.h"
31 
32 #include <sys/eventhandler.h>
33 #include <sys/sockio.h>
34 #include <machine/atomic.h>
35 
36 #include <net/debugnet.h>
37 
38 #ifndef ETH_DRIVER_VERSION
39 #define	ETH_DRIVER_VERSION	"3.5.2"
40 #endif
41 #define DRIVER_RELDATE	"September 2019"
42 
43 static const char mlx5e_version[] = "mlx5en: Mellanox Ethernet driver "
44 	ETH_DRIVER_VERSION " (" DRIVER_RELDATE ")\n";
45 
46 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
47 
48 struct mlx5e_channel_param {
49 	struct mlx5e_rq_param rq;
50 	struct mlx5e_sq_param sq;
51 	struct mlx5e_cq_param rx_cq;
52 	struct mlx5e_cq_param tx_cq;
53 };
54 
55 struct media {
56 	u32	subtype;
57 	u64	baudrate;
58 };
59 
60 static const struct media mlx5e_mode_table[MLX5E_LINK_SPEEDS_NUMBER][MLX5E_LINK_MODES_NUMBER] = {
61 
62 	[MLX5E_1000BASE_CX_SGMII][MLX5E_SGMII] = {
63 		.subtype = IFM_1000_CX_SGMII,
64 		.baudrate = IF_Mbps(1000ULL),
65 	},
66 	[MLX5E_1000BASE_KX][MLX5E_KX] = {
67 		.subtype = IFM_1000_KX,
68 		.baudrate = IF_Mbps(1000ULL),
69 	},
70 	[MLX5E_10GBASE_CX4][MLX5E_CX4] = {
71 		.subtype = IFM_10G_CX4,
72 		.baudrate = IF_Gbps(10ULL),
73 	},
74 	[MLX5E_10GBASE_KX4][MLX5E_KX4] = {
75 		.subtype = IFM_10G_KX4,
76 		.baudrate = IF_Gbps(10ULL),
77 	},
78 	[MLX5E_10GBASE_KR][MLX5E_KR] = {
79 		.subtype = IFM_10G_KR,
80 		.baudrate = IF_Gbps(10ULL),
81 	},
82 	[MLX5E_20GBASE_KR2][MLX5E_KR2] = {
83 		.subtype = IFM_20G_KR2,
84 		.baudrate = IF_Gbps(20ULL),
85 	},
86 	[MLX5E_40GBASE_CR4][MLX5E_CR4] = {
87 		.subtype = IFM_40G_CR4,
88 		.baudrate = IF_Gbps(40ULL),
89 	},
90 	[MLX5E_40GBASE_KR4][MLX5E_KR4] = {
91 		.subtype = IFM_40G_KR4,
92 		.baudrate = IF_Gbps(40ULL),
93 	},
94 	[MLX5E_56GBASE_R4][MLX5E_R] = {
95 		.subtype = IFM_56G_R4,
96 		.baudrate = IF_Gbps(56ULL),
97 	},
98 	[MLX5E_10GBASE_CR][MLX5E_CR1] = {
99 		.subtype = IFM_10G_CR1,
100 		.baudrate = IF_Gbps(10ULL),
101 	},
102 	[MLX5E_10GBASE_SR][MLX5E_SR] = {
103 		.subtype = IFM_10G_SR,
104 		.baudrate = IF_Gbps(10ULL),
105 	},
106 	[MLX5E_10GBASE_ER_LR][MLX5E_ER] = {
107 		.subtype = IFM_10G_ER,
108 		.baudrate = IF_Gbps(10ULL),
109 	},
110 	[MLX5E_10GBASE_ER_LR][MLX5E_LR] = {
111 		.subtype = IFM_10G_LR,
112 		.baudrate = IF_Gbps(10ULL),
113 	},
114 	[MLX5E_40GBASE_SR4][MLX5E_SR4] = {
115 		.subtype = IFM_40G_SR4,
116 		.baudrate = IF_Gbps(40ULL),
117 	},
118 	[MLX5E_40GBASE_LR4_ER4][MLX5E_LR4] = {
119 		.subtype = IFM_40G_LR4,
120 		.baudrate = IF_Gbps(40ULL),
121 	},
122 	[MLX5E_40GBASE_LR4_ER4][MLX5E_ER4] = {
123 		.subtype = IFM_40G_ER4,
124 		.baudrate = IF_Gbps(40ULL),
125 	},
126 	[MLX5E_100GBASE_CR4][MLX5E_CR4] = {
127 		.subtype = IFM_100G_CR4,
128 		.baudrate = IF_Gbps(100ULL),
129 	},
130 	[MLX5E_100GBASE_SR4][MLX5E_SR4] = {
131 		.subtype = IFM_100G_SR4,
132 		.baudrate = IF_Gbps(100ULL),
133 	},
134 	[MLX5E_100GBASE_KR4][MLX5E_KR4] = {
135 		.subtype = IFM_100G_KR4,
136 		.baudrate = IF_Gbps(100ULL),
137 	},
138 	[MLX5E_100GBASE_LR4][MLX5E_LR4] = {
139 		.subtype = IFM_100G_LR4,
140 		.baudrate = IF_Gbps(100ULL),
141 	},
142 	[MLX5E_100BASE_TX][MLX5E_TX] = {
143 		.subtype = IFM_100_TX,
144 		.baudrate = IF_Mbps(100ULL),
145 	},
146 	[MLX5E_1000BASE_T][MLX5E_T] = {
147 		.subtype = IFM_1000_T,
148 		.baudrate = IF_Mbps(1000ULL),
149 	},
150 	[MLX5E_10GBASE_T][MLX5E_T] = {
151 		.subtype = IFM_10G_T,
152 		.baudrate = IF_Gbps(10ULL),
153 	},
154 	[MLX5E_25GBASE_CR][MLX5E_CR] = {
155 		.subtype = IFM_25G_CR,
156 		.baudrate = IF_Gbps(25ULL),
157 	},
158 	[MLX5E_25GBASE_KR][MLX5E_KR] = {
159 		.subtype = IFM_25G_KR,
160 		.baudrate = IF_Gbps(25ULL),
161 	},
162 	[MLX5E_25GBASE_SR][MLX5E_SR] = {
163 		.subtype = IFM_25G_SR,
164 		.baudrate = IF_Gbps(25ULL),
165 	},
166 	[MLX5E_50GBASE_CR2][MLX5E_CR2] = {
167 		.subtype = IFM_50G_CR2,
168 		.baudrate = IF_Gbps(50ULL),
169 	},
170 	[MLX5E_50GBASE_KR2][MLX5E_KR2] = {
171 		.subtype = IFM_50G_KR2,
172 		.baudrate = IF_Gbps(50ULL),
173 	},
174 	[MLX5E_50GBASE_KR4][MLX5E_KR4] = {
175 		.subtype = IFM_50G_KR4,
176 		.baudrate = IF_Gbps(50ULL),
177 	},
178 };
179 
180 static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX5E_LINK_MODES_NUMBER] = {
181 	[MLX5E_SGMII_100M][MLX5E_SGMII] = {
182 		.subtype = IFM_100_SGMII,
183 		.baudrate = IF_Mbps(100),
184 	},
185 	[MLX5E_1000BASE_X_SGMII][MLX5E_KX] = {
186 		.subtype = IFM_1000_KX,
187 		.baudrate = IF_Mbps(1000),
188 	},
189 	[MLX5E_1000BASE_X_SGMII][MLX5E_CX_SGMII] = {
190 		.subtype = IFM_1000_CX_SGMII,
191 		.baudrate = IF_Mbps(1000),
192 	},
193 	[MLX5E_1000BASE_X_SGMII][MLX5E_CX] = {
194 		.subtype = IFM_1000_CX,
195 		.baudrate = IF_Mbps(1000),
196 	},
197 	[MLX5E_1000BASE_X_SGMII][MLX5E_LX] = {
198 		.subtype = IFM_1000_LX,
199 		.baudrate = IF_Mbps(1000),
200 	},
201 	[MLX5E_1000BASE_X_SGMII][MLX5E_SX] = {
202 		.subtype = IFM_1000_SX,
203 		.baudrate = IF_Mbps(1000),
204 	},
205 	[MLX5E_1000BASE_X_SGMII][MLX5E_T] = {
206 		.subtype = IFM_1000_T,
207 		.baudrate = IF_Mbps(1000),
208 	},
209 	[MLX5E_5GBASE_R][MLX5E_T] = {
210 		.subtype = IFM_5000_T,
211 		.baudrate = IF_Mbps(5000),
212 	},
213 	[MLX5E_5GBASE_R][MLX5E_KR] = {
214 		.subtype = IFM_5000_KR,
215 		.baudrate = IF_Mbps(5000),
216 	},
217 	[MLX5E_5GBASE_R][MLX5E_KR1] = {
218 		.subtype = IFM_5000_KR1,
219 		.baudrate = IF_Mbps(5000),
220 	},
221 	[MLX5E_5GBASE_R][MLX5E_KR_S] = {
222 		.subtype = IFM_5000_KR_S,
223 		.baudrate = IF_Mbps(5000),
224 	},
225 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_ER] = {
226 		.subtype = IFM_10G_ER,
227 		.baudrate = IF_Gbps(10ULL),
228 	},
229 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_KR] = {
230 		.subtype = IFM_10G_KR,
231 		.baudrate = IF_Gbps(10ULL),
232 	},
233 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_LR] = {
234 		.subtype = IFM_10G_LR,
235 		.baudrate = IF_Gbps(10ULL),
236 	},
237 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_SR] = {
238 		.subtype = IFM_10G_SR,
239 		.baudrate = IF_Gbps(10ULL),
240 	},
241 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_T] = {
242 		.subtype = IFM_10G_T,
243 		.baudrate = IF_Gbps(10ULL),
244 	},
245 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_AOC] = {
246 		.subtype = IFM_10G_AOC,
247 		.baudrate = IF_Gbps(10ULL),
248 	},
249 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CR1] = {
250 		.subtype = IFM_10G_CR1,
251 		.baudrate = IF_Gbps(10ULL),
252 	},
253 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CR4] = {
254 		.subtype = IFM_40G_CR4,
255 		.baudrate = IF_Gbps(40ULL),
256 	},
257 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_KR4] = {
258 		.subtype = IFM_40G_KR4,
259 		.baudrate = IF_Gbps(40ULL),
260 	},
261 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_LR4] = {
262 		.subtype = IFM_40G_LR4,
263 		.baudrate = IF_Gbps(40ULL),
264 	},
265 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_SR4] = {
266 		.subtype = IFM_40G_SR4,
267 		.baudrate = IF_Gbps(40ULL),
268 	},
269 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_ER4] = {
270 		.subtype = IFM_40G_ER4,
271 		.baudrate = IF_Gbps(40ULL),
272 	},
273 
274 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR] = {
275 		.subtype = IFM_25G_CR,
276 		.baudrate = IF_Gbps(25ULL),
277 	},
278 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR] = {
279 		.subtype = IFM_25G_KR,
280 		.baudrate = IF_Gbps(25ULL),
281 	},
282 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_SR] = {
283 		.subtype = IFM_25G_SR,
284 		.baudrate = IF_Gbps(25ULL),
285 	},
286 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_ACC] = {
287 		.subtype = IFM_25G_ACC,
288 		.baudrate = IF_Gbps(25ULL),
289 	},
290 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_AOC] = {
291 		.subtype = IFM_25G_AOC,
292 		.baudrate = IF_Gbps(25ULL),
293 	},
294 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR1] = {
295 		.subtype = IFM_25G_CR1,
296 		.baudrate = IF_Gbps(25ULL),
297 	},
298 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CR_S] = {
299 		.subtype = IFM_25G_CR_S,
300 		.baudrate = IF_Gbps(25ULL),
301 	},
302 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR1] = {
303 		.subtype = IFM_5000_KR1,
304 		.baudrate = IF_Gbps(25ULL),
305 	},
306 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_KR_S] = {
307 		.subtype = IFM_25G_KR_S,
308 		.baudrate = IF_Gbps(25ULL),
309 	},
310 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_LR] = {
311 		.subtype = IFM_25G_LR,
312 		.baudrate = IF_Gbps(25ULL),
313 	},
314 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_T] = {
315 		.subtype = IFM_25G_T,
316 		.baudrate = IF_Gbps(25ULL),
317 	},
318 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CR2] = {
319 		.subtype = IFM_50G_CR2,
320 		.baudrate = IF_Gbps(50ULL),
321 	},
322 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_KR2] = {
323 		.subtype = IFM_50G_KR2,
324 		.baudrate = IF_Gbps(50ULL),
325 	},
326 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_KR4] = {
327 		.subtype = IFM_50G_KR4,
328 		.baudrate = IF_Gbps(50ULL),
329 	},
330 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_SR2] = {
331 		.subtype = IFM_50G_SR2,
332 		.baudrate = IF_Gbps(50ULL),
333 	},
334 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_LR2] = {
335 		.subtype = IFM_50G_LR2,
336 		.baudrate = IF_Gbps(50ULL),
337 	},
338 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_LR] = {
339 		.subtype = IFM_50G_LR,
340 		.baudrate = IF_Gbps(50ULL),
341 	},
342 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_SR] = {
343 		.subtype = IFM_50G_SR,
344 		.baudrate = IF_Gbps(50ULL),
345 	},
346 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CP] = {
347 		.subtype = IFM_50G_CP,
348 		.baudrate = IF_Gbps(50ULL),
349 	},
350 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_FR] = {
351 		.subtype = IFM_50G_FR,
352 		.baudrate = IF_Gbps(50ULL),
353 	},
354 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_KR_PAM4] = {
355 		.subtype = IFM_50G_KR_PAM4,
356 		.baudrate = IF_Gbps(50ULL),
357 	},
358 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CR4] = {
359 		.subtype = IFM_100G_CR4,
360 		.baudrate = IF_Gbps(100ULL),
361 	},
362 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_KR4] = {
363 		.subtype = IFM_100G_KR4,
364 		.baudrate = IF_Gbps(100ULL),
365 	},
366 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_LR4] = {
367 		.subtype = IFM_100G_LR4,
368 		.baudrate = IF_Gbps(100ULL),
369 	},
370 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_SR4] = {
371 		.subtype = IFM_100G_SR4,
372 		.baudrate = IF_Gbps(100ULL),
373 	},
374 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_SR2] = {
375 		.subtype = IFM_100G_SR2,
376 		.baudrate = IF_Gbps(100ULL),
377 	},
378 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CP2] = {
379 		.subtype = IFM_100G_CP2,
380 		.baudrate = IF_Gbps(100ULL),
381 	},
382 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_KR2_PAM4] = {
383 		.subtype = IFM_100G_KR2_PAM4,
384 		.baudrate = IF_Gbps(100ULL),
385 	},
386 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_DR4] = {
387 		.subtype = IFM_200G_DR4,
388 		.baudrate = IF_Gbps(200ULL),
389 	},
390 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_LR4] = {
391 		.subtype = IFM_200G_LR4,
392 		.baudrate = IF_Gbps(200ULL),
393 	},
394 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_SR4] = {
395 		.subtype = IFM_200G_SR4,
396 		.baudrate = IF_Gbps(200ULL),
397 	},
398 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_FR4] = {
399 		.subtype = IFM_200G_FR4,
400 		.baudrate = IF_Gbps(200ULL),
401 	},
402 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CR4_PAM4] = {
403 		.subtype = IFM_200G_CR4_PAM4,
404 		.baudrate = IF_Gbps(200ULL),
405 	},
406 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_KR4_PAM4] = {
407 		.subtype = IFM_200G_KR4_PAM4,
408 		.baudrate = IF_Gbps(200ULL),
409 	},
410 };
411 
412 DEBUGNET_DEFINE(mlx5_en);
413 
414 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
415 
416 static void
417 mlx5e_update_carrier(struct mlx5e_priv *priv)
418 {
419 	struct mlx5_core_dev *mdev = priv->mdev;
420 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
421 	u32 eth_proto_oper;
422 	int error;
423 	u8 port_state;
424 	u8 is_er_type;
425 	u8 i, j;
426 	bool ext;
427 	struct media media_entry = {};
428 
429 	port_state = mlx5_query_vport_state(mdev,
430 	    MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
431 
432 	if (port_state == VPORT_STATE_UP) {
433 		priv->media_status_last |= IFM_ACTIVE;
434 	} else {
435 		priv->media_status_last &= ~IFM_ACTIVE;
436 		priv->media_active_last = IFM_ETHER;
437 		if_link_state_change(priv->ifp, LINK_STATE_DOWN);
438 		return;
439 	}
440 
441 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
442 	    MLX5_PTYS_EN, 1);
443 	if (error) {
444 		priv->media_active_last = IFM_ETHER;
445 		priv->ifp->if_baudrate = 1;
446 		mlx5_en_err(priv->ifp, "query port ptys failed: 0x%x\n",
447 		    error);
448 		return;
449 	}
450 
451 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
452 	eth_proto_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
453 	    eth_proto_oper);
454 
455 	i = ilog2(eth_proto_oper);
456 
457 	for (j = 0; j != MLX5E_LINK_MODES_NUMBER; j++) {
458 		media_entry = ext ? mlx5e_ext_mode_table[i][j] :
459 		    mlx5e_mode_table[i][j];
460 		if (media_entry.baudrate != 0)
461 			break;
462 	}
463 
464 	if (media_entry.subtype == 0) {
465 		mlx5_en_err(priv->ifp,
466 		    "Could not find operational media subtype\n");
467 		return;
468 	}
469 
470 	switch (media_entry.subtype) {
471 	case IFM_10G_ER:
472 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
473 		if (error != 0) {
474 			mlx5_en_err(priv->ifp,
475 			    "query port pddr failed: %d\n", error);
476 		}
477 		if (error != 0 || is_er_type == 0)
478 			media_entry.subtype = IFM_10G_LR;
479 		break;
480 	case IFM_40G_LR4:
481 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
482 		if (error != 0) {
483 			mlx5_en_err(priv->ifp,
484 			    "query port pddr failed: %d\n", error);
485 		}
486 		if (error == 0 && is_er_type != 0)
487 			media_entry.subtype = IFM_40G_ER4;
488 		break;
489 	}
490 	priv->media_active_last = media_entry.subtype | IFM_ETHER | IFM_FDX;
491 	priv->ifp->if_baudrate = media_entry.baudrate;
492 
493 	if_link_state_change(priv->ifp, LINK_STATE_UP);
494 }
495 
496 static void
497 mlx5e_media_status(struct ifnet *dev, struct ifmediareq *ifmr)
498 {
499 	struct mlx5e_priv *priv = dev->if_softc;
500 
501 	ifmr->ifm_status = priv->media_status_last;
502 	ifmr->ifm_active = priv->media_active_last |
503 	    (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
504 	    (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
505 
506 }
507 
508 static u32
509 mlx5e_find_link_mode(u32 subtype, bool ext)
510 {
511 	u32 i;
512 	u32 j;
513 	u32 link_mode = 0;
514 	u32 speeds_num = 0;
515 	struct media media_entry = {};
516 
517 	switch (subtype) {
518 	case IFM_10G_LR:
519 		subtype = IFM_10G_ER;
520 		break;
521 	case IFM_40G_ER4:
522 		subtype = IFM_40G_LR4;
523 		break;
524 	}
525 
526 	speeds_num = ext ? MLX5E_EXT_LINK_SPEEDS_NUMBER :
527 	    MLX5E_LINK_SPEEDS_NUMBER;
528 
529 	for (i = 0; i != speeds_num; i++) {
530 		for (j = 0; j < MLX5E_LINK_MODES_NUMBER ; ++j) {
531 			media_entry = ext ? mlx5e_ext_mode_table[i][j] :
532 			    mlx5e_mode_table[i][j];
533 			if (media_entry.baudrate == 0)
534 				continue;
535 			if (media_entry.subtype == subtype) {
536 				link_mode |= MLX5E_PROT_MASK(i);
537 			}
538 		}
539 	}
540 
541 	return (link_mode);
542 }
543 
544 static int
545 mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
546 {
547 	return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
548 	    priv->params.rx_pauseframe_control,
549 	    priv->params.tx_pauseframe_control,
550 	    priv->params.rx_priority_flow_control,
551 	    priv->params.tx_priority_flow_control));
552 }
553 
554 static int
555 mlx5e_set_port_pfc(struct mlx5e_priv *priv)
556 {
557 	int error;
558 
559 	if (priv->gone != 0) {
560 		error = -ENXIO;
561 	} else if (priv->params.rx_pauseframe_control ||
562 	    priv->params.tx_pauseframe_control) {
563 		mlx5_en_err(priv->ifp,
564 		    "Global pauseframes must be disabled before enabling PFC.\n");
565 		error = -EINVAL;
566 	} else {
567 		error = mlx5e_set_port_pause_and_pfc(priv);
568 	}
569 	return (error);
570 }
571 
572 static int
573 mlx5e_media_change(struct ifnet *dev)
574 {
575 	struct mlx5e_priv *priv = dev->if_softc;
576 	struct mlx5_core_dev *mdev = priv->mdev;
577 	u32 eth_proto_cap;
578 	u32 link_mode;
579 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
580 	int was_opened;
581 	int locked;
582 	int error;
583 	bool ext;
584 
585 	locked = PRIV_LOCKED(priv);
586 	if (!locked)
587 		PRIV_LOCK(priv);
588 
589 	if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
590 		error = EINVAL;
591 		goto done;
592 	}
593 
594 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
595 	    MLX5_PTYS_EN, 1);
596 	if (error != 0) {
597 		mlx5_en_err(dev, "Query port media capability failed\n");
598 		goto done;
599 	}
600 
601 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
602 	link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media), ext);
603 
604 	/* query supported capabilities */
605 	eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
606 	    eth_proto_capability);
607 
608 	/* check for autoselect */
609 	if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
610 		link_mode = eth_proto_cap;
611 		if (link_mode == 0) {
612 			mlx5_en_err(dev, "Port media capability is zero\n");
613 			error = EINVAL;
614 			goto done;
615 		}
616 	} else {
617 		link_mode = link_mode & eth_proto_cap;
618 		if (link_mode == 0) {
619 			mlx5_en_err(dev, "Not supported link mode requested\n");
620 			error = EINVAL;
621 			goto done;
622 		}
623 	}
624 	if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
625 		/* check if PFC is enabled */
626 		if (priv->params.rx_priority_flow_control ||
627 		    priv->params.tx_priority_flow_control) {
628 			mlx5_en_err(dev, "PFC must be disabled before enabling global pauseframes.\n");
629 			error = EINVAL;
630 			goto done;
631 		}
632 	}
633 	/* update pauseframe control bits */
634 	priv->params.rx_pauseframe_control =
635 	    (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
636 	priv->params.tx_pauseframe_control =
637 	    (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
638 
639 	/* check if device is opened */
640 	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
641 
642 	/* reconfigure the hardware */
643 	mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
644 	mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN, ext);
645 	error = -mlx5e_set_port_pause_and_pfc(priv);
646 	if (was_opened)
647 		mlx5_set_port_status(mdev, MLX5_PORT_UP);
648 
649 done:
650 	if (!locked)
651 		PRIV_UNLOCK(priv);
652 	return (error);
653 }
654 
655 static void
656 mlx5e_update_carrier_work(struct work_struct *work)
657 {
658 	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
659 	    update_carrier_work);
660 
661 	PRIV_LOCK(priv);
662 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
663 		mlx5e_update_carrier(priv);
664 	PRIV_UNLOCK(priv);
665 }
666 
667 #define	MLX5E_PCIE_PERF_GET_64(a,b,c,d,e,f)    \
668 	s_debug->c = MLX5_GET64(mpcnt_reg, out, counter_set.f.c);
669 
670 #define	MLX5E_PCIE_PERF_GET_32(a,b,c,d,e,f)    \
671 	s_debug->c = MLX5_GET(mpcnt_reg, out, counter_set.f.c);
672 
673 static void
674 mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
675 {
676 	struct mlx5_core_dev *mdev = priv->mdev;
677 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
678 	const unsigned sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
679 	void *out;
680 	void *in;
681 	int err;
682 
683 	/* allocate firmware request structures */
684 	in = mlx5_vzalloc(sz);
685 	out = mlx5_vzalloc(sz);
686 	if (in == NULL || out == NULL)
687 		goto free_out;
688 
689 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
690 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
691 	if (err != 0)
692 		goto free_out;
693 
694 	MLX5E_PCIE_PERFORMANCE_COUNTERS_64(MLX5E_PCIE_PERF_GET_64)
695 	MLX5E_PCIE_PERFORMANCE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
696 
697 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP);
698 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
699 	if (err != 0)
700 		goto free_out;
701 
702 	MLX5E_PCIE_TIMERS_AND_STATES_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
703 
704 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_LANE_COUNTERS_GROUP);
705 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
706 	if (err != 0)
707 		goto free_out;
708 
709 	MLX5E_PCIE_LANE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
710 
711 free_out:
712 	/* free firmware request structures */
713 	kvfree(in);
714 	kvfree(out);
715 }
716 
717 /*
718  * This function reads the physical port counters from the firmware
719  * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
720  * macros. The output is converted from big-endian 64-bit values into
721  * host endian ones and stored in the "priv->stats.pport" structure.
722  */
723 static void
724 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
725 {
726 	struct mlx5_core_dev *mdev = priv->mdev;
727 	struct mlx5e_pport_stats *s = &priv->stats.pport;
728 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
729 	u32 *in;
730 	u32 *out;
731 	const u64 *ptr;
732 	unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
733 	unsigned x;
734 	unsigned y;
735 	unsigned z;
736 
737 	/* allocate firmware request structures */
738 	in = mlx5_vzalloc(sz);
739 	out = mlx5_vzalloc(sz);
740 	if (in == NULL || out == NULL)
741 		goto free_out;
742 
743 	/*
744 	 * Get pointer to the 64-bit counter set which is located at a
745 	 * fixed offset in the output firmware request structure:
746 	 */
747 	ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
748 
749 	MLX5_SET(ppcnt_reg, in, local_port, 1);
750 
751 	/* read IEEE802_3 counter group using predefined counter layout */
752 	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
753 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
754 	for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
755 	     x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
756 		s->arg[y] = be64toh(ptr[x]);
757 
758 	/* read RFC2819 counter group using predefined counter layout */
759 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
760 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
761 	for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
762 		s->arg[y] = be64toh(ptr[x]);
763 
764 	for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
765 	    MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
766 		s_debug->arg[y] = be64toh(ptr[x]);
767 
768 	/* read RFC2863 counter group using predefined counter layout */
769 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
770 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
771 	for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
772 		s_debug->arg[y] = be64toh(ptr[x]);
773 
774 	/* read physical layer stats counter group using predefined counter layout */
775 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
776 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
777 	for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
778 		s_debug->arg[y] = be64toh(ptr[x]);
779 
780 	/* read Extended Ethernet counter group using predefined counter layout */
781 	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
782 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
783 	for (x = 0; x != MLX5E_PPORT_ETHERNET_EXTENDED_STATS_DEBUG_NUM; x++, y++)
784 		s_debug->arg[y] = be64toh(ptr[x]);
785 
786 	/* read Extended Statistical Group */
787 	if (MLX5_CAP_GEN(mdev, pcam_reg) &&
788 	    MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) &&
789 	    MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) {
790 		/* read Extended Statistical counter group using predefined counter layout */
791 		MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
792 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
793 
794 		for (x = 0; x != MLX5E_PPORT_STATISTICAL_DEBUG_NUM; x++, y++)
795 			s_debug->arg[y] = be64toh(ptr[x]);
796 	}
797 
798 	/* read PCIE counters */
799 	mlx5e_update_pcie_counters(priv);
800 
801 	/* read per-priority counters */
802 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
803 
804 	/* iterate all the priorities */
805 	for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
806 		MLX5_SET(ppcnt_reg, in, prio_tc, z);
807 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
808 
809 		/* read per priority stats counter group using predefined counter layout */
810 		for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
811 		    MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
812 			s->arg[y] = be64toh(ptr[x]);
813 	}
814 
815 free_out:
816 	/* free firmware request structures */
817 	kvfree(in);
818 	kvfree(out);
819 }
820 
821 static void
822 mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
823 {
824 	u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
825 	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
826 
827 	if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
828 		return;
829 
830 	MLX5_SET(query_vnic_env_in, in, opcode,
831 	    MLX5_CMD_OP_QUERY_VNIC_ENV);
832 	MLX5_SET(query_vnic_env_in, in, op_mod, 0);
833 	MLX5_SET(query_vnic_env_in, in, other_vport, 0);
834 
835 	if (mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out)) != 0)
836 		return;
837 
838 	priv->stats.vport.rx_steer_missed_packets =
839 	    MLX5_GET64(query_vnic_env_out, out,
840 	    vport_env.nic_receive_steering_discard);
841 }
842 
843 /*
844  * This function is called regularly to collect all statistics
845  * counters from the firmware. The values can be viewed through the
846  * sysctl interface. Execution is serialized using the priv's global
847  * configuration lock.
848  */
849 static void
850 mlx5e_update_stats_locked(struct mlx5e_priv *priv)
851 {
852 	struct mlx5_core_dev *mdev = priv->mdev;
853 	struct mlx5e_vport_stats *s = &priv->stats.vport;
854 	struct mlx5e_sq_stats *sq_stats;
855 #if (__FreeBSD_version < 1100000)
856 	struct ifnet *ifp = priv->ifp;
857 #endif
858 
859 	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
860 	u32 *out;
861 	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
862 	u64 tso_packets = 0;
863 	u64 tso_bytes = 0;
864 	u64 tx_queue_dropped = 0;
865 	u64 tx_defragged = 0;
866 	u64 tx_offload_none = 0;
867 	u64 lro_packets = 0;
868 	u64 lro_bytes = 0;
869 	u64 sw_lro_queued = 0;
870 	u64 sw_lro_flushed = 0;
871 	u64 rx_csum_none = 0;
872 	u64 rx_wqe_err = 0;
873 	u64 rx_packets = 0;
874 	u64 rx_bytes = 0;
875 	u32 rx_out_of_buffer = 0;
876 	int error;
877 	int i;
878 	int j;
879 
880 	out = mlx5_vzalloc(outlen);
881 	if (out == NULL)
882 		goto free_out;
883 
884 	/* Collect firts the SW counters and then HW for consistency */
885 	for (i = 0; i < priv->params.num_channels; i++) {
886 		struct mlx5e_channel *pch = priv->channel + i;
887 		struct mlx5e_rq *rq = &pch->rq;
888 		struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
889 
890 		/* collect stats from LRO */
891 		rq_stats->sw_lro_queued = rq->lro.lro_queued;
892 		rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
893 		sw_lro_queued += rq_stats->sw_lro_queued;
894 		sw_lro_flushed += rq_stats->sw_lro_flushed;
895 		lro_packets += rq_stats->lro_packets;
896 		lro_bytes += rq_stats->lro_bytes;
897 		rx_csum_none += rq_stats->csum_none;
898 		rx_wqe_err += rq_stats->wqe_err;
899 		rx_packets += rq_stats->packets;
900 		rx_bytes += rq_stats->bytes;
901 
902 		for (j = 0; j < priv->num_tc; j++) {
903 			sq_stats = &pch->sq[j].stats;
904 
905 			tso_packets += sq_stats->tso_packets;
906 			tso_bytes += sq_stats->tso_bytes;
907 			tx_queue_dropped += sq_stats->dropped;
908 			tx_queue_dropped += sq_stats->enobuf;
909 			tx_defragged += sq_stats->defragged;
910 			tx_offload_none += sq_stats->csum_offload_none;
911 		}
912 	}
913 
914 	/* update counters */
915 	s->tso_packets = tso_packets;
916 	s->tso_bytes = tso_bytes;
917 	s->tx_queue_dropped = tx_queue_dropped;
918 	s->tx_defragged = tx_defragged;
919 	s->lro_packets = lro_packets;
920 	s->lro_bytes = lro_bytes;
921 	s->sw_lro_queued = sw_lro_queued;
922 	s->sw_lro_flushed = sw_lro_flushed;
923 	s->rx_csum_none = rx_csum_none;
924 	s->rx_wqe_err = rx_wqe_err;
925 	s->rx_packets = rx_packets;
926 	s->rx_bytes = rx_bytes;
927 
928 	mlx5e_grp_vnic_env_update_stats(priv);
929 
930 	/* HW counters */
931 	memset(in, 0, sizeof(in));
932 
933 	MLX5_SET(query_vport_counter_in, in, opcode,
934 	    MLX5_CMD_OP_QUERY_VPORT_COUNTER);
935 	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
936 	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
937 
938 	memset(out, 0, outlen);
939 
940 	/* get number of out-of-buffer drops first */
941 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
942 	    mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
943 	    &rx_out_of_buffer) == 0) {
944 		s->rx_out_of_buffer = rx_out_of_buffer;
945 	}
946 
947 	/* get port statistics */
948 	if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen) == 0) {
949 #define	MLX5_GET_CTR(out, x) \
950 	MLX5_GET64(query_vport_counter_out, out, x)
951 
952 		s->rx_error_packets =
953 		    MLX5_GET_CTR(out, received_errors.packets);
954 		s->rx_error_bytes =
955 		    MLX5_GET_CTR(out, received_errors.octets);
956 		s->tx_error_packets =
957 		    MLX5_GET_CTR(out, transmit_errors.packets);
958 		s->tx_error_bytes =
959 		    MLX5_GET_CTR(out, transmit_errors.octets);
960 
961 		s->rx_unicast_packets =
962 		    MLX5_GET_CTR(out, received_eth_unicast.packets);
963 		s->rx_unicast_bytes =
964 		    MLX5_GET_CTR(out, received_eth_unicast.octets);
965 		s->tx_unicast_packets =
966 		    MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
967 		s->tx_unicast_bytes =
968 		    MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
969 
970 		s->rx_multicast_packets =
971 		    MLX5_GET_CTR(out, received_eth_multicast.packets);
972 		s->rx_multicast_bytes =
973 		    MLX5_GET_CTR(out, received_eth_multicast.octets);
974 		s->tx_multicast_packets =
975 		    MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
976 		s->tx_multicast_bytes =
977 		    MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
978 
979 		s->rx_broadcast_packets =
980 		    MLX5_GET_CTR(out, received_eth_broadcast.packets);
981 		s->rx_broadcast_bytes =
982 		    MLX5_GET_CTR(out, received_eth_broadcast.octets);
983 		s->tx_broadcast_packets =
984 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
985 		s->tx_broadcast_bytes =
986 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
987 
988 		s->tx_packets = s->tx_unicast_packets +
989 		    s->tx_multicast_packets + s->tx_broadcast_packets;
990 		s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes +
991 		    s->tx_broadcast_bytes;
992 
993 		/* Update calculated offload counters */
994 		s->tx_csum_offload = s->tx_packets - tx_offload_none;
995 		s->rx_csum_good = s->rx_packets - s->rx_csum_none;
996 	}
997 
998 	/* Get physical port counters */
999 	mlx5e_update_pport_counters(priv);
1000 
1001 	s->tx_jumbo_packets =
1002 	    priv->stats.port_stats_debug.tx_stat_p1519to2047octets +
1003 	    priv->stats.port_stats_debug.tx_stat_p2048to4095octets +
1004 	    priv->stats.port_stats_debug.tx_stat_p4096to8191octets +
1005 	    priv->stats.port_stats_debug.tx_stat_p8192to10239octets;
1006 
1007 #if (__FreeBSD_version < 1100000)
1008 	/* no get_counters interface in fbsd 10 */
1009 	ifp->if_ipackets = s->rx_packets;
1010 	ifp->if_ierrors = priv->stats.pport.in_range_len_errors +
1011 	    priv->stats.pport.out_of_range_len +
1012 	    priv->stats.pport.too_long_errors +
1013 	    priv->stats.pport.check_seq_err +
1014 	    priv->stats.pport.alignment_err;
1015 	ifp->if_iqdrops = s->rx_out_of_buffer;
1016 	ifp->if_opackets = s->tx_packets;
1017 	ifp->if_oerrors = priv->stats.port_stats_debug.out_discards;
1018 	ifp->if_snd.ifq_drops = s->tx_queue_dropped;
1019 	ifp->if_ibytes = s->rx_bytes;
1020 	ifp->if_obytes = s->tx_bytes;
1021 	ifp->if_collisions =
1022 	    priv->stats.pport.collisions;
1023 #endif
1024 
1025 free_out:
1026 	kvfree(out);
1027 
1028 	/* Update diagnostics, if any */
1029 	if (priv->params_ethtool.diag_pci_enable ||
1030 	    priv->params_ethtool.diag_general_enable) {
1031 		error = mlx5_core_get_diagnostics_full(mdev,
1032 		    priv->params_ethtool.diag_pci_enable ? &priv->params_pci : NULL,
1033 		    priv->params_ethtool.diag_general_enable ? &priv->params_general : NULL);
1034 		if (error != 0)
1035 			mlx5_en_err(priv->ifp,
1036 			    "Failed reading diagnostics: %d\n", error);
1037 	}
1038 
1039 	/* Update FEC, if any */
1040 	error = mlx5e_fec_update(priv);
1041 	if (error != 0 && error != EOPNOTSUPP) {
1042 		mlx5_en_err(priv->ifp,
1043 		    "Updating FEC failed: %d\n", error);
1044 	}
1045 
1046 	/* Update temperature, if any */
1047 	if (priv->params_ethtool.hw_num_temp != 0) {
1048 		error = mlx5e_hw_temperature_update(priv);
1049 		if (error != 0 && error != EOPNOTSUPP) {
1050 			mlx5_en_err(priv->ifp,
1051 			    "Updating temperature failed: %d\n", error);
1052 		}
1053 	}
1054 }
1055 
1056 static void
1057 mlx5e_update_stats_work(struct work_struct *work)
1058 {
1059 	struct mlx5e_priv *priv;
1060 
1061 	priv = container_of(work, struct mlx5e_priv, update_stats_work);
1062 	PRIV_LOCK(priv);
1063 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
1064 	    !test_bit(MLX5_INTERFACE_STATE_TEARDOWN, &priv->mdev->intf_state))
1065 		mlx5e_update_stats_locked(priv);
1066 	PRIV_UNLOCK(priv);
1067 }
1068 
1069 static void
1070 mlx5e_update_stats(void *arg)
1071 {
1072 	struct mlx5e_priv *priv = arg;
1073 
1074 	queue_work(priv->wq, &priv->update_stats_work);
1075 
1076 	callout_reset(&priv->watchdog, hz, &mlx5e_update_stats, priv);
1077 }
1078 
1079 static void
1080 mlx5e_async_event_sub(struct mlx5e_priv *priv,
1081     enum mlx5_dev_event event)
1082 {
1083 	switch (event) {
1084 	case MLX5_DEV_EVENT_PORT_UP:
1085 	case MLX5_DEV_EVENT_PORT_DOWN:
1086 		queue_work(priv->wq, &priv->update_carrier_work);
1087 		break;
1088 
1089 	default:
1090 		break;
1091 	}
1092 }
1093 
1094 static void
1095 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
1096     enum mlx5_dev_event event, unsigned long param)
1097 {
1098 	struct mlx5e_priv *priv = vpriv;
1099 
1100 	mtx_lock(&priv->async_events_mtx);
1101 	if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
1102 		mlx5e_async_event_sub(priv, event);
1103 	mtx_unlock(&priv->async_events_mtx);
1104 }
1105 
1106 static void
1107 mlx5e_enable_async_events(struct mlx5e_priv *priv)
1108 {
1109 	set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1110 }
1111 
1112 static void
1113 mlx5e_disable_async_events(struct mlx5e_priv *priv)
1114 {
1115 	mtx_lock(&priv->async_events_mtx);
1116 	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1117 	mtx_unlock(&priv->async_events_mtx);
1118 }
1119 
1120 static void mlx5e_calibration_callout(void *arg);
1121 static int mlx5e_calibration_duration = 20;
1122 static int mlx5e_fast_calibration = 1;
1123 static int mlx5e_normal_calibration = 30;
1124 
1125 static SYSCTL_NODE(_hw_mlx5, OID_AUTO, calibr, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1126     "MLX5 timestamp calibration parameteres");
1127 
1128 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, duration, CTLFLAG_RWTUN,
1129     &mlx5e_calibration_duration, 0,
1130     "Duration of initial calibration");
1131 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, fast, CTLFLAG_RWTUN,
1132     &mlx5e_fast_calibration, 0,
1133     "Recalibration interval during initial calibration");
1134 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, normal, CTLFLAG_RWTUN,
1135     &mlx5e_normal_calibration, 0,
1136     "Recalibration interval during normal operations");
1137 
1138 /*
1139  * Ignites the calibration process.
1140  */
1141 static void
1142 mlx5e_reset_calibration_callout(struct mlx5e_priv *priv)
1143 {
1144 
1145 	if (priv->clbr_done == 0)
1146 		mlx5e_calibration_callout(priv);
1147 	else
1148 		callout_reset_curcpu(&priv->tstmp_clbr, (priv->clbr_done <
1149 		    mlx5e_calibration_duration ? mlx5e_fast_calibration :
1150 		    mlx5e_normal_calibration) * hz, mlx5e_calibration_callout,
1151 		    priv);
1152 }
1153 
1154 static uint64_t
1155 mlx5e_timespec2usec(const struct timespec *ts)
1156 {
1157 
1158 	return ((uint64_t)ts->tv_sec * 1000000000 + ts->tv_nsec);
1159 }
1160 
1161 static uint64_t
1162 mlx5e_hw_clock(struct mlx5e_priv *priv)
1163 {
1164 	struct mlx5_init_seg *iseg;
1165 	uint32_t hw_h, hw_h1, hw_l;
1166 
1167 	iseg = priv->mdev->iseg;
1168 	do {
1169 		hw_h = ioread32be(&iseg->internal_timer_h);
1170 		hw_l = ioread32be(&iseg->internal_timer_l);
1171 		hw_h1 = ioread32be(&iseg->internal_timer_h);
1172 	} while (hw_h1 != hw_h);
1173 	return (((uint64_t)hw_h << 32) | hw_l);
1174 }
1175 
1176 /*
1177  * The calibration callout, it runs either in the context of the
1178  * thread which enables calibration, or in callout.  It takes the
1179  * snapshot of system and adapter clocks, then advances the pointers to
1180  * the calibration point to allow rx path to read the consistent data
1181  * lockless.
1182  */
1183 static void
1184 mlx5e_calibration_callout(void *arg)
1185 {
1186 	struct mlx5e_priv *priv;
1187 	struct mlx5e_clbr_point *next, *curr;
1188 	struct timespec ts;
1189 	int clbr_curr_next;
1190 
1191 	priv = arg;
1192 	curr = &priv->clbr_points[priv->clbr_curr];
1193 	clbr_curr_next = priv->clbr_curr + 1;
1194 	if (clbr_curr_next >= nitems(priv->clbr_points))
1195 		clbr_curr_next = 0;
1196 	next = &priv->clbr_points[clbr_curr_next];
1197 
1198 	next->base_prev = curr->base_curr;
1199 	next->clbr_hw_prev = curr->clbr_hw_curr;
1200 
1201 	next->clbr_hw_curr = mlx5e_hw_clock(priv);
1202 	if (((next->clbr_hw_curr - curr->clbr_hw_curr) >> MLX5E_TSTMP_PREC) ==
1203 	    0) {
1204 		if (priv->clbr_done != 0) {
1205 			mlx5_en_err(priv->ifp,
1206 			    "HW failed tstmp frozen %#jx %#jx, disabling\n",
1207 			     next->clbr_hw_curr, curr->clbr_hw_prev);
1208 			priv->clbr_done = 0;
1209 		}
1210 		atomic_store_rel_int(&curr->clbr_gen, 0);
1211 		return;
1212 	}
1213 
1214 	nanouptime(&ts);
1215 	next->base_curr = mlx5e_timespec2usec(&ts);
1216 
1217 	curr->clbr_gen = 0;
1218 	atomic_thread_fence_rel();
1219 	priv->clbr_curr = clbr_curr_next;
1220 	atomic_store_rel_int(&next->clbr_gen, ++(priv->clbr_gen));
1221 
1222 	if (priv->clbr_done < mlx5e_calibration_duration)
1223 		priv->clbr_done++;
1224 	mlx5e_reset_calibration_callout(priv);
1225 }
1226 
1227 static const char *mlx5e_rq_stats_desc[] = {
1228 	MLX5E_RQ_STATS(MLX5E_STATS_DESC)
1229 };
1230 
1231 static int
1232 mlx5e_create_rq(struct mlx5e_channel *c,
1233     struct mlx5e_rq_param *param,
1234     struct mlx5e_rq *rq)
1235 {
1236 	struct mlx5e_priv *priv = c->priv;
1237 	struct mlx5_core_dev *mdev = priv->mdev;
1238 	char buffer[16];
1239 	void *rqc = param->rqc;
1240 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1241 	int wq_sz;
1242 	int err;
1243 	int i;
1244 	u32 nsegs, wqe_sz;
1245 
1246 	err = mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
1247 	if (err != 0)
1248 		goto done;
1249 
1250 	/* Create DMA descriptor TAG */
1251 	if ((err = -bus_dma_tag_create(
1252 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1253 	    1,				/* any alignment */
1254 	    0,				/* no boundary */
1255 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1256 	    BUS_SPACE_MAXADDR,		/* highaddr */
1257 	    NULL, NULL,			/* filter, filterarg */
1258 	    nsegs * MLX5E_MAX_RX_BYTES,	/* maxsize */
1259 	    nsegs,			/* nsegments */
1260 	    nsegs * MLX5E_MAX_RX_BYTES,	/* maxsegsize */
1261 	    0,				/* flags */
1262 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1263 	    &rq->dma_tag)))
1264 		goto done;
1265 
1266 	err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1267 	    &rq->wq_ctrl);
1268 	if (err)
1269 		goto err_free_dma_tag;
1270 
1271 	rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1272 
1273 	err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1274 	if (err != 0)
1275 		goto err_rq_wq_destroy;
1276 
1277 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1278 
1279 	err = -tcp_lro_init_args(&rq->lro, priv->ifp, TCP_LRO_ENTRIES, wq_sz);
1280 	if (err)
1281 		goto err_rq_wq_destroy;
1282 
1283 	rq->mbuf = malloc(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
1284 	for (i = 0; i != wq_sz; i++) {
1285 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1286 		int j;
1287 
1288 		err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1289 		if (err != 0) {
1290 			while (i--)
1291 				bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1292 			goto err_rq_mbuf_free;
1293 		}
1294 
1295 		/* set value for constant fields */
1296 		for (j = 0; j < rq->nsegs; j++)
1297 			wqe->data[j].lkey = cpu_to_be32(priv->mr.key);
1298 	}
1299 
1300 	INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1301 	if (priv->params.rx_cq_moderation_mode < 2) {
1302 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1303 	} else {
1304 		void *cqc = container_of(param,
1305 		    struct mlx5e_channel_param, rq)->rx_cq.cqc;
1306 
1307 		switch (MLX5_GET(cqc, cqc, cq_period_mode)) {
1308 		case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
1309 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1310 			break;
1311 		case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
1312 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1313 			break;
1314 		default:
1315 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1316 			break;
1317 		}
1318 	}
1319 
1320 	rq->ifp = priv->ifp;
1321 	rq->channel = c;
1322 	rq->ix = c->ix;
1323 
1324 	snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
1325 	mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1326 	    buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
1327 	    rq->stats.arg);
1328 	return (0);
1329 
1330 err_rq_mbuf_free:
1331 	free(rq->mbuf, M_MLX5EN);
1332 	tcp_lro_free(&rq->lro);
1333 err_rq_wq_destroy:
1334 	mlx5_wq_destroy(&rq->wq_ctrl);
1335 err_free_dma_tag:
1336 	bus_dma_tag_destroy(rq->dma_tag);
1337 done:
1338 	return (err);
1339 }
1340 
1341 static void
1342 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1343 {
1344 	int wq_sz;
1345 	int i;
1346 
1347 	/* destroy all sysctl nodes */
1348 	sysctl_ctx_free(&rq->stats.ctx);
1349 
1350 	/* free leftover LRO packets, if any */
1351 	tcp_lro_free(&rq->lro);
1352 
1353 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1354 	for (i = 0; i != wq_sz; i++) {
1355 		if (rq->mbuf[i].mbuf != NULL) {
1356 			bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1357 			m_freem(rq->mbuf[i].mbuf);
1358 		}
1359 		bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1360 	}
1361 	free(rq->mbuf, M_MLX5EN);
1362 	mlx5_wq_destroy(&rq->wq_ctrl);
1363 	bus_dma_tag_destroy(rq->dma_tag);
1364 }
1365 
1366 static int
1367 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1368 {
1369 	struct mlx5e_channel *c = rq->channel;
1370 	struct mlx5e_priv *priv = c->priv;
1371 	struct mlx5_core_dev *mdev = priv->mdev;
1372 
1373 	void *in;
1374 	void *rqc;
1375 	void *wq;
1376 	int inlen;
1377 	int err;
1378 
1379 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1380 	    sizeof(u64) * rq->wq_ctrl.buf.npages;
1381 	in = mlx5_vzalloc(inlen);
1382 	if (in == NULL)
1383 		return (-ENOMEM);
1384 
1385 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1386 	wq = MLX5_ADDR_OF(rqc, rqc, wq);
1387 
1388 	memcpy(rqc, param->rqc, sizeof(param->rqc));
1389 
1390 	MLX5_SET(rqc, rqc, cqn, c->rq.cq.mcq.cqn);
1391 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1392 	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1393 	if (priv->counter_set_id >= 0)
1394 		MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
1395 	MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1396 	    PAGE_SHIFT);
1397 	MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1398 
1399 	mlx5_fill_page_array(&rq->wq_ctrl.buf,
1400 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1401 
1402 	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1403 
1404 	kvfree(in);
1405 
1406 	return (err);
1407 }
1408 
1409 static int
1410 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1411 {
1412 	struct mlx5e_channel *c = rq->channel;
1413 	struct mlx5e_priv *priv = c->priv;
1414 	struct mlx5_core_dev *mdev = priv->mdev;
1415 
1416 	void *in;
1417 	void *rqc;
1418 	int inlen;
1419 	int err;
1420 
1421 	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1422 	in = mlx5_vzalloc(inlen);
1423 	if (in == NULL)
1424 		return (-ENOMEM);
1425 
1426 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1427 
1428 	MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1429 	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1430 	MLX5_SET(rqc, rqc, state, next_state);
1431 
1432 	err = mlx5_core_modify_rq(mdev, in, inlen);
1433 
1434 	kvfree(in);
1435 
1436 	return (err);
1437 }
1438 
1439 static void
1440 mlx5e_disable_rq(struct mlx5e_rq *rq)
1441 {
1442 	struct mlx5e_channel *c = rq->channel;
1443 	struct mlx5e_priv *priv = c->priv;
1444 	struct mlx5_core_dev *mdev = priv->mdev;
1445 
1446 	mlx5_core_destroy_rq(mdev, rq->rqn);
1447 }
1448 
1449 static int
1450 mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq)
1451 {
1452 	struct mlx5e_channel *c = rq->channel;
1453 	struct mlx5e_priv *priv = c->priv;
1454 	struct mlx5_wq_ll *wq = &rq->wq;
1455 	int i;
1456 
1457 	for (i = 0; i < 1000; i++) {
1458 		if (wq->cur_sz >= priv->params.min_rx_wqes)
1459 			return (0);
1460 
1461 		msleep(4);
1462 	}
1463 	return (-ETIMEDOUT);
1464 }
1465 
1466 static int
1467 mlx5e_open_rq(struct mlx5e_channel *c,
1468     struct mlx5e_rq_param *param,
1469     struct mlx5e_rq *rq)
1470 {
1471 	int err;
1472 
1473 	err = mlx5e_create_rq(c, param, rq);
1474 	if (err)
1475 		return (err);
1476 
1477 	err = mlx5e_enable_rq(rq, param);
1478 	if (err)
1479 		goto err_destroy_rq;
1480 
1481 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1482 	if (err)
1483 		goto err_disable_rq;
1484 
1485 	c->rq.enabled = 1;
1486 
1487 	return (0);
1488 
1489 err_disable_rq:
1490 	mlx5e_disable_rq(rq);
1491 err_destroy_rq:
1492 	mlx5e_destroy_rq(rq);
1493 
1494 	return (err);
1495 }
1496 
1497 static void
1498 mlx5e_close_rq(struct mlx5e_rq *rq)
1499 {
1500 	mtx_lock(&rq->mtx);
1501 	rq->enabled = 0;
1502 	callout_stop(&rq->watchdog);
1503 	mtx_unlock(&rq->mtx);
1504 
1505 	mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1506 }
1507 
1508 static void
1509 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1510 {
1511 
1512 	mlx5e_disable_rq(rq);
1513 	mlx5e_close_cq(&rq->cq);
1514 	cancel_work_sync(&rq->dim.work);
1515 	mlx5e_destroy_rq(rq);
1516 }
1517 
1518 void
1519 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1520 {
1521 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1522 	int x;
1523 
1524 	for (x = 0; x != wq_sz; x++) {
1525 		if (unlikely(sq->mbuf[x].p_refcount != NULL)) {
1526 			atomic_add_int(sq->mbuf[x].p_refcount, -1);
1527 			sq->mbuf[x].p_refcount = NULL;
1528 		}
1529 		if (sq->mbuf[x].mbuf != NULL) {
1530 			bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1531 			m_freem(sq->mbuf[x].mbuf);
1532 		}
1533 		bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1534 	}
1535 	free(sq->mbuf, M_MLX5EN);
1536 }
1537 
1538 int
1539 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1540 {
1541 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1542 	int err;
1543 	int x;
1544 
1545 	sq->mbuf = malloc(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN, M_WAITOK | M_ZERO);
1546 
1547 	/* Create DMA descriptor MAPs */
1548 	for (x = 0; x != wq_sz; x++) {
1549 		err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1550 		if (err != 0) {
1551 			while (x--)
1552 				bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1553 			free(sq->mbuf, M_MLX5EN);
1554 			return (err);
1555 		}
1556 	}
1557 	return (0);
1558 }
1559 
1560 static const char *mlx5e_sq_stats_desc[] = {
1561 	MLX5E_SQ_STATS(MLX5E_STATS_DESC)
1562 };
1563 
1564 void
1565 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1566 {
1567 	sq->max_inline = sq->priv->params.tx_max_inline;
1568 	sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1569 
1570 	/*
1571 	 * Check if trust state is DSCP or if inline mode is NONE which
1572 	 * indicates CX-5 or newer hardware.
1573 	 */
1574 	if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1575 	    sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1576 		if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1577 			sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1578 		else
1579 			sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1580 	} else {
1581 		sq->min_insert_caps = 0;
1582 	}
1583 }
1584 
1585 static void
1586 mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1587 {
1588 	int i;
1589 
1590 	for (i = 0; i != priv->num_tc; i++) {
1591 		mtx_lock(&c->sq[i].lock);
1592 		mlx5e_update_sq_inline(&c->sq[i]);
1593 		mtx_unlock(&c->sq[i].lock);
1594 	}
1595 }
1596 
1597 void
1598 mlx5e_refresh_sq_inline(struct mlx5e_priv *priv)
1599 {
1600 	int i;
1601 
1602 	/* check if channels are closed */
1603 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
1604 		return;
1605 
1606 	for (i = 0; i < priv->params.num_channels; i++)
1607 		mlx5e_refresh_sq_inline_sub(priv, &priv->channel[i]);
1608 }
1609 
1610 static int
1611 mlx5e_create_sq(struct mlx5e_channel *c,
1612     int tc,
1613     struct mlx5e_sq_param *param,
1614     struct mlx5e_sq *sq)
1615 {
1616 	struct mlx5e_priv *priv = c->priv;
1617 	struct mlx5_core_dev *mdev = priv->mdev;
1618 	char buffer[16];
1619 	void *sqc = param->sqc;
1620 	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
1621 	int err;
1622 
1623 	/* Create DMA descriptor TAG */
1624 	if ((err = -bus_dma_tag_create(
1625 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1626 	    1,				/* any alignment */
1627 	    0,				/* no boundary */
1628 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1629 	    BUS_SPACE_MAXADDR,		/* highaddr */
1630 	    NULL, NULL,			/* filter, filterarg */
1631 	    MLX5E_MAX_TX_PAYLOAD_SIZE,	/* maxsize */
1632 	    MLX5E_MAX_TX_MBUF_FRAGS,	/* nsegments */
1633 	    MLX5E_MAX_TX_MBUF_SIZE,	/* maxsegsize */
1634 	    0,				/* flags */
1635 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1636 	    &sq->dma_tag)))
1637 		goto done;
1638 
1639 	err = mlx5_alloc_map_uar(mdev, &sq->uar);
1640 	if (err)
1641 		goto err_free_dma_tag;
1642 
1643 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
1644 	    &sq->wq_ctrl);
1645 	if (err)
1646 		goto err_unmap_free_uar;
1647 
1648 	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1649 	sq->bf_buf_size = (1 << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2;
1650 
1651 	err = mlx5e_alloc_sq_db(sq);
1652 	if (err)
1653 		goto err_sq_wq_destroy;
1654 
1655 	sq->mkey_be = cpu_to_be32(priv->mr.key);
1656 	sq->ifp = priv->ifp;
1657 	sq->priv = priv;
1658 	sq->tc = tc;
1659 
1660 	mlx5e_update_sq_inline(sq);
1661 
1662 	snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1663 	mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1664 	    buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1665 	    sq->stats.arg);
1666 
1667 	return (0);
1668 
1669 err_sq_wq_destroy:
1670 	mlx5_wq_destroy(&sq->wq_ctrl);
1671 
1672 err_unmap_free_uar:
1673 	mlx5_unmap_free_uar(mdev, &sq->uar);
1674 
1675 err_free_dma_tag:
1676 	bus_dma_tag_destroy(sq->dma_tag);
1677 done:
1678 	return (err);
1679 }
1680 
1681 static void
1682 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1683 {
1684 	/* destroy all sysctl nodes */
1685 	sysctl_ctx_free(&sq->stats.ctx);
1686 
1687 	mlx5e_free_sq_db(sq);
1688 	mlx5_wq_destroy(&sq->wq_ctrl);
1689 	mlx5_unmap_free_uar(sq->priv->mdev, &sq->uar);
1690 	bus_dma_tag_destroy(sq->dma_tag);
1691 }
1692 
1693 int
1694 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1695     int tis_num)
1696 {
1697 	void *in;
1698 	void *sqc;
1699 	void *wq;
1700 	int inlen;
1701 	int err;
1702 
1703 	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1704 	    sizeof(u64) * sq->wq_ctrl.buf.npages;
1705 	in = mlx5_vzalloc(inlen);
1706 	if (in == NULL)
1707 		return (-ENOMEM);
1708 
1709 	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1710 	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1711 
1712 	memcpy(sqc, param->sqc, sizeof(param->sqc));
1713 
1714 	MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1715 	MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1716 	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1717 	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1718 	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1719 
1720 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1721 	MLX5_SET(wq, wq, uar_page, sq->uar.index);
1722 	MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1723 	    PAGE_SHIFT);
1724 	MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1725 
1726 	mlx5_fill_page_array(&sq->wq_ctrl.buf,
1727 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1728 
1729 	err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1730 
1731 	kvfree(in);
1732 
1733 	return (err);
1734 }
1735 
1736 int
1737 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1738 {
1739 	void *in;
1740 	void *sqc;
1741 	int inlen;
1742 	int err;
1743 
1744 	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1745 	in = mlx5_vzalloc(inlen);
1746 	if (in == NULL)
1747 		return (-ENOMEM);
1748 
1749 	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1750 
1751 	MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1752 	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1753 	MLX5_SET(sqc, sqc, state, next_state);
1754 
1755 	err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1756 
1757 	kvfree(in);
1758 
1759 	return (err);
1760 }
1761 
1762 void
1763 mlx5e_disable_sq(struct mlx5e_sq *sq)
1764 {
1765 
1766 	mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1767 }
1768 
1769 static int
1770 mlx5e_open_sq(struct mlx5e_channel *c,
1771     int tc,
1772     struct mlx5e_sq_param *param,
1773     struct mlx5e_sq *sq)
1774 {
1775 	int err;
1776 
1777 	sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1778 
1779 	/* ensure the TX completion event factor is not zero */
1780 	if (sq->cev_factor == 0)
1781 		sq->cev_factor = 1;
1782 
1783 	err = mlx5e_create_sq(c, tc, param, sq);
1784 	if (err)
1785 		return (err);
1786 
1787 	err = mlx5e_enable_sq(sq, param, c->priv->tisn[tc]);
1788 	if (err)
1789 		goto err_destroy_sq;
1790 
1791 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1792 	if (err)
1793 		goto err_disable_sq;
1794 
1795 	WRITE_ONCE(sq->running, 1);
1796 
1797 	return (0);
1798 
1799 err_disable_sq:
1800 	mlx5e_disable_sq(sq);
1801 err_destroy_sq:
1802 	mlx5e_destroy_sq(sq);
1803 
1804 	return (err);
1805 }
1806 
1807 static void
1808 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1809 {
1810 	/* fill up remainder with NOPs */
1811 	while (sq->cev_counter != 0) {
1812 		while (!mlx5e_sq_has_room_for(sq, 1)) {
1813 			if (can_sleep != 0) {
1814 				mtx_unlock(&sq->lock);
1815 				msleep(4);
1816 				mtx_lock(&sq->lock);
1817 			} else {
1818 				goto done;
1819 			}
1820 		}
1821 		/* send a single NOP */
1822 		mlx5e_send_nop(sq, 1);
1823 		atomic_thread_fence_rel();
1824 	}
1825 done:
1826 	/* Check if we need to write the doorbell */
1827 	if (likely(sq->doorbell.d64 != 0)) {
1828 		mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
1829 		sq->doorbell.d64 = 0;
1830 	}
1831 }
1832 
1833 void
1834 mlx5e_sq_cev_timeout(void *arg)
1835 {
1836 	struct mlx5e_sq *sq = arg;
1837 
1838 	mtx_assert(&sq->lock, MA_OWNED);
1839 
1840 	/* check next state */
1841 	switch (sq->cev_next_state) {
1842 	case MLX5E_CEV_STATE_SEND_NOPS:
1843 		/* fill TX ring with NOPs, if any */
1844 		mlx5e_sq_send_nops_locked(sq, 0);
1845 
1846 		/* check if completed */
1847 		if (sq->cev_counter == 0) {
1848 			sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1849 			return;
1850 		}
1851 		break;
1852 	default:
1853 		/* send NOPs on next timeout */
1854 		sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1855 		break;
1856 	}
1857 
1858 	/* restart timer */
1859 	callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1860 }
1861 
1862 void
1863 mlx5e_drain_sq(struct mlx5e_sq *sq)
1864 {
1865 	int error;
1866 	struct mlx5_core_dev *mdev= sq->priv->mdev;
1867 
1868 	/*
1869 	 * Check if already stopped.
1870 	 *
1871 	 * NOTE: Serialization of this function is managed by the
1872 	 * caller ensuring the priv's state lock is locked or in case
1873 	 * of rate limit support, a single thread manages drain and
1874 	 * resume of SQs. The "running" variable can therefore safely
1875 	 * be read without any locks.
1876 	 */
1877 	if (READ_ONCE(sq->running) == 0)
1878 		return;
1879 
1880 	/* don't put more packets into the SQ */
1881 	WRITE_ONCE(sq->running, 0);
1882 
1883 	/* serialize access to DMA rings */
1884 	mtx_lock(&sq->lock);
1885 
1886 	/* teardown event factor timer, if any */
1887 	sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1888 	callout_stop(&sq->cev_callout);
1889 
1890 	/* send dummy NOPs in order to flush the transmit ring */
1891 	mlx5e_sq_send_nops_locked(sq, 1);
1892 	mtx_unlock(&sq->lock);
1893 
1894 	/* wait till SQ is empty or link is down */
1895 	mtx_lock(&sq->lock);
1896 	while (sq->cc != sq->pc &&
1897 	    (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1898 	    mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
1899 		mtx_unlock(&sq->lock);
1900 		msleep(1);
1901 		sq->cq.mcq.comp(&sq->cq.mcq);
1902 		mtx_lock(&sq->lock);
1903 	}
1904 	mtx_unlock(&sq->lock);
1905 
1906 	/* error out remaining requests */
1907 	error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1908 	if (error != 0) {
1909 		mlx5_en_err(sq->ifp,
1910 		    "mlx5e_modify_sq() from RDY to ERR failed: %d\n", error);
1911 	}
1912 
1913 	/* wait till SQ is empty */
1914 	mtx_lock(&sq->lock);
1915 	while (sq->cc != sq->pc &&
1916 	       mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
1917 		mtx_unlock(&sq->lock);
1918 		msleep(1);
1919 		sq->cq.mcq.comp(&sq->cq.mcq);
1920 		mtx_lock(&sq->lock);
1921 	}
1922 	mtx_unlock(&sq->lock);
1923 }
1924 
1925 static void
1926 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1927 {
1928 
1929 	mlx5e_drain_sq(sq);
1930 	mlx5e_disable_sq(sq);
1931 	mlx5e_destroy_sq(sq);
1932 }
1933 
1934 static int
1935 mlx5e_create_cq(struct mlx5e_priv *priv,
1936     struct mlx5e_cq_param *param,
1937     struct mlx5e_cq *cq,
1938     mlx5e_cq_comp_t *comp,
1939     int eq_ix)
1940 {
1941 	struct mlx5_core_dev *mdev = priv->mdev;
1942 	struct mlx5_core_cq *mcq = &cq->mcq;
1943 	int eqn_not_used;
1944 	int irqn;
1945 	int err;
1946 	u32 i;
1947 
1948 	param->wq.buf_numa_node = 0;
1949 	param->wq.db_numa_node = 0;
1950 
1951 	err = mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1952 	if (err)
1953 		return (err);
1954 
1955 	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1956 	    &cq->wq_ctrl);
1957 	if (err)
1958 		return (err);
1959 
1960 	mcq->cqe_sz = 64;
1961 	mcq->set_ci_db = cq->wq_ctrl.db.db;
1962 	mcq->arm_db = cq->wq_ctrl.db.db + 1;
1963 	*mcq->set_ci_db = 0;
1964 	*mcq->arm_db = 0;
1965 	mcq->vector = eq_ix;
1966 	mcq->comp = comp;
1967 	mcq->event = mlx5e_cq_error_event;
1968 	mcq->irqn = irqn;
1969 	mcq->uar = &priv->cq_uar;
1970 
1971 	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1972 		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1973 
1974 		cqe->op_own = 0xf1;
1975 	}
1976 
1977 	cq->priv = priv;
1978 
1979 	return (0);
1980 }
1981 
1982 static void
1983 mlx5e_destroy_cq(struct mlx5e_cq *cq)
1984 {
1985 	mlx5_wq_destroy(&cq->wq_ctrl);
1986 }
1987 
1988 static int
1989 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
1990 {
1991 	struct mlx5_core_cq *mcq = &cq->mcq;
1992 	void *in;
1993 	void *cqc;
1994 	int inlen;
1995 	int irqn_not_used;
1996 	int eqn;
1997 	int err;
1998 
1999 	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
2000 	    sizeof(u64) * cq->wq_ctrl.buf.npages;
2001 	in = mlx5_vzalloc(inlen);
2002 	if (in == NULL)
2003 		return (-ENOMEM);
2004 
2005 	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
2006 
2007 	memcpy(cqc, param->cqc, sizeof(param->cqc));
2008 
2009 	mlx5_fill_page_array(&cq->wq_ctrl.buf,
2010 	    (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
2011 
2012 	mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
2013 
2014 	MLX5_SET(cqc, cqc, c_eqn, eqn);
2015 	MLX5_SET(cqc, cqc, uar_page, mcq->uar->index);
2016 	MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
2017 	    PAGE_SHIFT);
2018 	MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
2019 
2020 	err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen);
2021 
2022 	kvfree(in);
2023 
2024 	if (err)
2025 		return (err);
2026 
2027 	mlx5e_cq_arm(cq, MLX5_GET_DOORBELL_LOCK(&cq->priv->doorbell_lock));
2028 
2029 	return (0);
2030 }
2031 
2032 static void
2033 mlx5e_disable_cq(struct mlx5e_cq *cq)
2034 {
2035 
2036 	mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
2037 }
2038 
2039 int
2040 mlx5e_open_cq(struct mlx5e_priv *priv,
2041     struct mlx5e_cq_param *param,
2042     struct mlx5e_cq *cq,
2043     mlx5e_cq_comp_t *comp,
2044     int eq_ix)
2045 {
2046 	int err;
2047 
2048 	err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
2049 	if (err)
2050 		return (err);
2051 
2052 	err = mlx5e_enable_cq(cq, param, eq_ix);
2053 	if (err)
2054 		goto err_destroy_cq;
2055 
2056 	return (0);
2057 
2058 err_destroy_cq:
2059 	mlx5e_destroy_cq(cq);
2060 
2061 	return (err);
2062 }
2063 
2064 void
2065 mlx5e_close_cq(struct mlx5e_cq *cq)
2066 {
2067 	mlx5e_disable_cq(cq);
2068 	mlx5e_destroy_cq(cq);
2069 }
2070 
2071 static int
2072 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2073     struct mlx5e_channel_param *cparam)
2074 {
2075 	int err;
2076 	int tc;
2077 
2078 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2079 		/* open completion queue */
2080 		err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
2081 		    &mlx5e_tx_cq_comp, c->ix);
2082 		if (err)
2083 			goto err_close_tx_cqs;
2084 	}
2085 	return (0);
2086 
2087 err_close_tx_cqs:
2088 	for (tc--; tc >= 0; tc--)
2089 		mlx5e_close_cq(&c->sq[tc].cq);
2090 
2091 	return (err);
2092 }
2093 
2094 static void
2095 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2096 {
2097 	int tc;
2098 
2099 	for (tc = 0; tc < c->priv->num_tc; tc++)
2100 		mlx5e_close_cq(&c->sq[tc].cq);
2101 }
2102 
2103 static int
2104 mlx5e_open_sqs(struct mlx5e_channel *c,
2105     struct mlx5e_channel_param *cparam)
2106 {
2107 	int err;
2108 	int tc;
2109 
2110 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2111 		err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
2112 		if (err)
2113 			goto err_close_sqs;
2114 	}
2115 
2116 	return (0);
2117 
2118 err_close_sqs:
2119 	for (tc--; tc >= 0; tc--)
2120 		mlx5e_close_sq_wait(&c->sq[tc]);
2121 
2122 	return (err);
2123 }
2124 
2125 static void
2126 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
2127 {
2128 	int tc;
2129 
2130 	for (tc = 0; tc < c->priv->num_tc; tc++)
2131 		mlx5e_close_sq_wait(&c->sq[tc]);
2132 }
2133 
2134 static void
2135 mlx5e_chan_static_init(struct mlx5e_priv *priv, struct mlx5e_channel *c, int ix)
2136 {
2137 	int tc;
2138 
2139 	/* setup priv and channel number */
2140 	c->priv = priv;
2141 	c->ix = ix;
2142 
2143 	/* setup send tag */
2144 	c->tag.type = IF_SND_TAG_TYPE_UNLIMITED;
2145 	m_snd_tag_init(&c->tag.m_snd_tag, c->priv->ifp);
2146 
2147 	init_completion(&c->completion);
2148 
2149 	mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
2150 
2151 	callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
2152 
2153 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2154 		struct mlx5e_sq *sq = c->sq + tc;
2155 
2156 		mtx_init(&sq->lock, "mlx5tx",
2157 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2158 		mtx_init(&sq->comp_lock, "mlx5comp",
2159 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2160 
2161 		callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2162 	}
2163 }
2164 
2165 static void
2166 mlx5e_chan_wait_for_completion(struct mlx5e_channel *c)
2167 {
2168 
2169 	m_snd_tag_rele(&c->tag.m_snd_tag);
2170 	wait_for_completion(&c->completion);
2171 }
2172 
2173 static void
2174 mlx5e_priv_wait_for_completion(struct mlx5e_priv *priv, const uint32_t channels)
2175 {
2176 	uint32_t x;
2177 
2178 	for (x = 0; x != channels; x++)
2179 		mlx5e_chan_wait_for_completion(&priv->channel[x]);
2180 }
2181 
2182 static void
2183 mlx5e_chan_static_destroy(struct mlx5e_channel *c)
2184 {
2185 	int tc;
2186 
2187 	callout_drain(&c->rq.watchdog);
2188 
2189 	mtx_destroy(&c->rq.mtx);
2190 
2191 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2192 		callout_drain(&c->sq[tc].cev_callout);
2193 		mtx_destroy(&c->sq[tc].lock);
2194 		mtx_destroy(&c->sq[tc].comp_lock);
2195 	}
2196 }
2197 
2198 static int
2199 mlx5e_open_channel(struct mlx5e_priv *priv,
2200     struct mlx5e_channel_param *cparam,
2201     struct mlx5e_channel *c)
2202 {
2203 	struct epoch_tracker et;
2204 	int i, err;
2205 
2206 	/* zero non-persistant data */
2207 	MLX5E_ZERO(&c->rq, mlx5e_rq_zero_start);
2208 	for (i = 0; i != priv->num_tc; i++)
2209 		MLX5E_ZERO(&c->sq[i], mlx5e_sq_zero_start);
2210 
2211 	/* open transmit completion queue */
2212 	err = mlx5e_open_tx_cqs(c, cparam);
2213 	if (err)
2214 		goto err_free;
2215 
2216 	/* open receive completion queue */
2217 	err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2218 	    &mlx5e_rx_cq_comp, c->ix);
2219 	if (err)
2220 		goto err_close_tx_cqs;
2221 
2222 	err = mlx5e_open_sqs(c, cparam);
2223 	if (err)
2224 		goto err_close_rx_cq;
2225 
2226 	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2227 	if (err)
2228 		goto err_close_sqs;
2229 
2230 	/* poll receive queue initially */
2231 	NET_EPOCH_ENTER(et);
2232 	c->rq.cq.mcq.comp(&c->rq.cq.mcq);
2233 	NET_EPOCH_EXIT(et);
2234 
2235 	return (0);
2236 
2237 err_close_sqs:
2238 	mlx5e_close_sqs_wait(c);
2239 
2240 err_close_rx_cq:
2241 	mlx5e_close_cq(&c->rq.cq);
2242 
2243 err_close_tx_cqs:
2244 	mlx5e_close_tx_cqs(c);
2245 
2246 err_free:
2247 	return (err);
2248 }
2249 
2250 static void
2251 mlx5e_close_channel(struct mlx5e_channel *c)
2252 {
2253 	mlx5e_close_rq(&c->rq);
2254 }
2255 
2256 static void
2257 mlx5e_close_channel_wait(struct mlx5e_channel *c)
2258 {
2259 	mlx5e_close_rq_wait(&c->rq);
2260 	mlx5e_close_sqs_wait(c);
2261 	mlx5e_close_tx_cqs(c);
2262 }
2263 
2264 static int
2265 mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs)
2266 {
2267 	u32 r, n;
2268 
2269 	r = priv->params.hw_lro_en ? priv->params.lro_wqe_sz :
2270 	    MLX5E_SW2MB_MTU(priv->ifp->if_mtu);
2271 	if (r > MJUM16BYTES)
2272 		return (-ENOMEM);
2273 
2274 	if (r > MJUM9BYTES)
2275 		r = MJUM16BYTES;
2276 	else if (r > MJUMPAGESIZE)
2277 		r = MJUM9BYTES;
2278 	else if (r > MCLBYTES)
2279 		r = MJUMPAGESIZE;
2280 	else
2281 		r = MCLBYTES;
2282 
2283 	/*
2284 	 * n + 1 must be a power of two, because stride size must be.
2285 	 * Stride size is 16 * (n + 1), as the first segment is
2286 	 * control.
2287 	 */
2288 	for (n = howmany(r, MLX5E_MAX_RX_BYTES); !powerof2(n + 1); n++)
2289 		;
2290 
2291 	if (n > MLX5E_MAX_BUSDMA_RX_SEGS)
2292 		return (-ENOMEM);
2293 
2294 	*wqe_sz = r;
2295 	*nsegs = n;
2296 	return (0);
2297 }
2298 
2299 static void
2300 mlx5e_build_rq_param(struct mlx5e_priv *priv,
2301     struct mlx5e_rq_param *param)
2302 {
2303 	void *rqc = param->rqc;
2304 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2305 	u32 wqe_sz, nsegs;
2306 
2307 	mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
2308 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
2309 	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2310 	MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) +
2311 	    nsegs * sizeof(struct mlx5_wqe_data_seg)));
2312 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
2313 	MLX5_SET(wq, wq, pd, priv->pdn);
2314 
2315 	param->wq.buf_numa_node = 0;
2316 	param->wq.db_numa_node = 0;
2317 	param->wq.linear = 1;
2318 }
2319 
2320 static void
2321 mlx5e_build_sq_param(struct mlx5e_priv *priv,
2322     struct mlx5e_sq_param *param)
2323 {
2324 	void *sqc = param->sqc;
2325 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2326 
2327 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
2328 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2329 	MLX5_SET(wq, wq, pd, priv->pdn);
2330 
2331 	param->wq.buf_numa_node = 0;
2332 	param->wq.db_numa_node = 0;
2333 	param->wq.linear = 1;
2334 }
2335 
2336 static void
2337 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2338     struct mlx5e_cq_param *param)
2339 {
2340 	void *cqc = param->cqc;
2341 
2342 	MLX5_SET(cqc, cqc, uar_page, priv->cq_uar.index);
2343 }
2344 
2345 static void
2346 mlx5e_get_default_profile(struct mlx5e_priv *priv, int mode, struct net_dim_cq_moder *ptr)
2347 {
2348 
2349 	*ptr = net_dim_get_profile(mode, MLX5E_DIM_DEFAULT_PROFILE);
2350 
2351 	/* apply LRO restrictions */
2352 	if (priv->params.hw_lro_en &&
2353 	    ptr->pkts > MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO) {
2354 		ptr->pkts = MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO;
2355 	}
2356 }
2357 
2358 static void
2359 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2360     struct mlx5e_cq_param *param)
2361 {
2362 	struct net_dim_cq_moder curr;
2363 	void *cqc = param->cqc;
2364 
2365 	/*
2366 	 * We use MLX5_CQE_FORMAT_HASH because the RX hash mini CQE
2367 	 * format is more beneficial for FreeBSD use case.
2368 	 *
2369 	 * Adding support for MLX5_CQE_FORMAT_CSUM will require changes
2370 	 * in mlx5e_decompress_cqe.
2371 	 */
2372 	if (priv->params.cqe_zipping_en) {
2373 		MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_HASH);
2374 		MLX5_SET(cqc, cqc, cqe_compression_en, 1);
2375 	}
2376 
2377 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
2378 
2379 	switch (priv->params.rx_cq_moderation_mode) {
2380 	case 0:
2381 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2382 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2383 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2384 		break;
2385 	case 1:
2386 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2387 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2388 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2389 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2390 		else
2391 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2392 		break;
2393 	case 2:
2394 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE, &curr);
2395 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2396 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2397 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2398 		break;
2399 	case 3:
2400 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE, &curr);
2401 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2402 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2403 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2404 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2405 		else
2406 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2407 		break;
2408 	default:
2409 		break;
2410 	}
2411 
2412 	mlx5e_dim_build_cq_param(priv, param);
2413 
2414 	mlx5e_build_common_cq_param(priv, param);
2415 }
2416 
2417 static void
2418 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2419     struct mlx5e_cq_param *param)
2420 {
2421 	void *cqc = param->cqc;
2422 
2423 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
2424 	MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
2425 	MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
2426 
2427 	switch (priv->params.tx_cq_moderation_mode) {
2428 	case 0:
2429 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2430 		break;
2431 	default:
2432 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2433 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2434 		else
2435 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2436 		break;
2437 	}
2438 
2439 	mlx5e_build_common_cq_param(priv, param);
2440 }
2441 
2442 static void
2443 mlx5e_build_channel_param(struct mlx5e_priv *priv,
2444     struct mlx5e_channel_param *cparam)
2445 {
2446 	memset(cparam, 0, sizeof(*cparam));
2447 
2448 	mlx5e_build_rq_param(priv, &cparam->rq);
2449 	mlx5e_build_sq_param(priv, &cparam->sq);
2450 	mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
2451 	mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
2452 }
2453 
2454 static int
2455 mlx5e_open_channels(struct mlx5e_priv *priv)
2456 {
2457 	struct mlx5e_channel_param *cparam;
2458 	int err;
2459 	int i;
2460 	int j;
2461 
2462 	cparam = malloc(sizeof(*cparam), M_MLX5EN, M_WAITOK);
2463 
2464 	mlx5e_build_channel_param(priv, cparam);
2465 	for (i = 0; i < priv->params.num_channels; i++) {
2466 		err = mlx5e_open_channel(priv, cparam, &priv->channel[i]);
2467 		if (err)
2468 			goto err_close_channels;
2469 	}
2470 
2471 	for (j = 0; j < priv->params.num_channels; j++) {
2472 		err = mlx5e_wait_for_min_rx_wqes(&priv->channel[j].rq);
2473 		if (err)
2474 			goto err_close_channels;
2475 	}
2476 	free(cparam, M_MLX5EN);
2477 	return (0);
2478 
2479 err_close_channels:
2480 	while (i--) {
2481 		mlx5e_close_channel(&priv->channel[i]);
2482 		mlx5e_close_channel_wait(&priv->channel[i]);
2483 	}
2484 	free(cparam, M_MLX5EN);
2485 	return (err);
2486 }
2487 
2488 static void
2489 mlx5e_close_channels(struct mlx5e_priv *priv)
2490 {
2491 	int i;
2492 
2493 	for (i = 0; i < priv->params.num_channels; i++)
2494 		mlx5e_close_channel(&priv->channel[i]);
2495 	for (i = 0; i < priv->params.num_channels; i++)
2496 		mlx5e_close_channel_wait(&priv->channel[i]);
2497 }
2498 
2499 static int
2500 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2501 {
2502 
2503 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2504 		uint8_t cq_mode;
2505 
2506 		switch (priv->params.tx_cq_moderation_mode) {
2507 		case 0:
2508 		case 2:
2509 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2510 			break;
2511 		default:
2512 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2513 			break;
2514 		}
2515 
2516 		return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2517 		    priv->params.tx_cq_moderation_usec,
2518 		    priv->params.tx_cq_moderation_pkts,
2519 		    cq_mode));
2520 	}
2521 
2522 	return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2523 	    priv->params.tx_cq_moderation_usec,
2524 	    priv->params.tx_cq_moderation_pkts));
2525 }
2526 
2527 static int
2528 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2529 {
2530 
2531 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2532 		uint8_t cq_mode;
2533 		uint8_t dim_mode;
2534 		int retval;
2535 
2536 		switch (priv->params.rx_cq_moderation_mode) {
2537 		case 0:
2538 		case 2:
2539 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2540 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2541 			break;
2542 		default:
2543 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2544 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2545 			break;
2546 		}
2547 
2548 		/* tear down dynamic interrupt moderation */
2549 		mtx_lock(&rq->mtx);
2550 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2551 		mtx_unlock(&rq->mtx);
2552 
2553 		/* wait for dynamic interrupt moderation work task, if any */
2554 		cancel_work_sync(&rq->dim.work);
2555 
2556 		if (priv->params.rx_cq_moderation_mode >= 2) {
2557 			struct net_dim_cq_moder curr;
2558 
2559 			mlx5e_get_default_profile(priv, dim_mode, &curr);
2560 
2561 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2562 			    curr.usec, curr.pkts, cq_mode);
2563 
2564 			/* set dynamic interrupt moderation mode and zero defaults */
2565 			mtx_lock(&rq->mtx);
2566 			rq->dim.mode = dim_mode;
2567 			rq->dim.state = 0;
2568 			rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2569 			mtx_unlock(&rq->mtx);
2570 		} else {
2571 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2572 			    priv->params.rx_cq_moderation_usec,
2573 			    priv->params.rx_cq_moderation_pkts,
2574 			    cq_mode);
2575 		}
2576 		return (retval);
2577 	}
2578 
2579 	return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2580 	    priv->params.rx_cq_moderation_usec,
2581 	    priv->params.rx_cq_moderation_pkts));
2582 }
2583 
2584 static int
2585 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
2586 {
2587 	int err;
2588 	int i;
2589 
2590 	err = mlx5e_refresh_rq_params(priv, &c->rq);
2591 	if (err)
2592 		goto done;
2593 
2594 	for (i = 0; i != priv->num_tc; i++) {
2595 		err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
2596 		if (err)
2597 			goto done;
2598 	}
2599 done:
2600 	return (err);
2601 }
2602 
2603 int
2604 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
2605 {
2606 	int i;
2607 
2608 	/* check if channels are closed */
2609 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2610 		return (EINVAL);
2611 
2612 	for (i = 0; i < priv->params.num_channels; i++) {
2613 		int err;
2614 
2615 		err = mlx5e_refresh_channel_params_sub(priv, &priv->channel[i]);
2616 		if (err)
2617 			return (err);
2618 	}
2619 	return (0);
2620 }
2621 
2622 static int
2623 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
2624 {
2625 	struct mlx5_core_dev *mdev = priv->mdev;
2626 	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
2627 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2628 
2629 	memset(in, 0, sizeof(in));
2630 
2631 	MLX5_SET(tisc, tisc, prio, tc);
2632 	MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
2633 
2634 	return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
2635 }
2636 
2637 static void
2638 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
2639 {
2640 	mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc]);
2641 }
2642 
2643 static int
2644 mlx5e_open_tises(struct mlx5e_priv *priv)
2645 {
2646 	int num_tc = priv->num_tc;
2647 	int err;
2648 	int tc;
2649 
2650 	for (tc = 0; tc < num_tc; tc++) {
2651 		err = mlx5e_open_tis(priv, tc);
2652 		if (err)
2653 			goto err_close_tises;
2654 	}
2655 
2656 	return (0);
2657 
2658 err_close_tises:
2659 	for (tc--; tc >= 0; tc--)
2660 		mlx5e_close_tis(priv, tc);
2661 
2662 	return (err);
2663 }
2664 
2665 static void
2666 mlx5e_close_tises(struct mlx5e_priv *priv)
2667 {
2668 	int num_tc = priv->num_tc;
2669 	int tc;
2670 
2671 	for (tc = 0; tc < num_tc; tc++)
2672 		mlx5e_close_tis(priv, tc);
2673 }
2674 
2675 static int
2676 mlx5e_open_rqt(struct mlx5e_priv *priv)
2677 {
2678 	struct mlx5_core_dev *mdev = priv->mdev;
2679 	u32 *in;
2680 	u32 out[MLX5_ST_SZ_DW(create_rqt_out)] = {0};
2681 	void *rqtc;
2682 	int inlen;
2683 	int err;
2684 	int sz;
2685 	int i;
2686 
2687 	sz = 1 << priv->params.rx_hash_log_tbl_sz;
2688 
2689 	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2690 	in = mlx5_vzalloc(inlen);
2691 	if (in == NULL)
2692 		return (-ENOMEM);
2693 	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2694 
2695 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2696 	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2697 
2698 	for (i = 0; i < sz; i++) {
2699 		int ix = i;
2700 #ifdef RSS
2701 		ix = rss_get_indirection_to_bucket(ix);
2702 #endif
2703 		/* ensure we don't overflow */
2704 		ix %= priv->params.num_channels;
2705 
2706 		/* apply receive side scaling stride, if any */
2707 		ix -= ix % (int)priv->params.channels_rsss;
2708 
2709 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2710 	}
2711 
2712 	MLX5_SET(create_rqt_in, in, opcode, MLX5_CMD_OP_CREATE_RQT);
2713 
2714 	err = mlx5_cmd_exec(mdev, in, inlen, out, sizeof(out));
2715 	if (!err)
2716 		priv->rqtn = MLX5_GET(create_rqt_out, out, rqtn);
2717 
2718 	kvfree(in);
2719 
2720 	return (err);
2721 }
2722 
2723 static void
2724 mlx5e_close_rqt(struct mlx5e_priv *priv)
2725 {
2726 	u32 in[MLX5_ST_SZ_DW(destroy_rqt_in)] = {0};
2727 	u32 out[MLX5_ST_SZ_DW(destroy_rqt_out)] = {0};
2728 
2729 	MLX5_SET(destroy_rqt_in, in, opcode, MLX5_CMD_OP_DESTROY_RQT);
2730 	MLX5_SET(destroy_rqt_in, in, rqtn, priv->rqtn);
2731 
2732 	mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out));
2733 }
2734 
2735 static void
2736 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt)
2737 {
2738 	void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2739 	__be32 *hkey;
2740 
2741 	MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
2742 
2743 #define	ROUGH_MAX_L2_L3_HDR_SZ 256
2744 
2745 #define	MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2746 			  MLX5_HASH_FIELD_SEL_DST_IP)
2747 
2748 #define	MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2749 			  MLX5_HASH_FIELD_SEL_DST_IP   |\
2750 			  MLX5_HASH_FIELD_SEL_L4_SPORT |\
2751 			  MLX5_HASH_FIELD_SEL_L4_DPORT)
2752 
2753 #define	MLX5_HASH_IP_IPSEC_SPI	(MLX5_HASH_FIELD_SEL_SRC_IP   |\
2754 				 MLX5_HASH_FIELD_SEL_DST_IP   |\
2755 				 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2756 
2757 	if (priv->params.hw_lro_en) {
2758 		MLX5_SET(tirc, tirc, lro_enable_mask,
2759 		    MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2760 		    MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2761 		MLX5_SET(tirc, tirc, lro_max_msg_sz,
2762 		    (priv->params.lro_wqe_sz -
2763 		    ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2764 		/* TODO: add the option to choose timer value dynamically */
2765 		MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
2766 		    MLX5_CAP_ETH(priv->mdev,
2767 		    lro_timer_supported_periods[2]));
2768 	}
2769 
2770 	/* setup parameters for hashing TIR type, if any */
2771 	switch (tt) {
2772 	case MLX5E_TT_ANY:
2773 		MLX5_SET(tirc, tirc, disp_type,
2774 		    MLX5_TIRC_DISP_TYPE_DIRECT);
2775 		MLX5_SET(tirc, tirc, inline_rqn,
2776 		    priv->channel[0].rq.rqn);
2777 		break;
2778 	default:
2779 		MLX5_SET(tirc, tirc, disp_type,
2780 		    MLX5_TIRC_DISP_TYPE_INDIRECT);
2781 		MLX5_SET(tirc, tirc, indirect_table,
2782 		    priv->rqtn);
2783 		MLX5_SET(tirc, tirc, rx_hash_fn,
2784 		    MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
2785 		hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
2786 #ifdef RSS
2787 		/*
2788 		 * The FreeBSD RSS implementation does currently not
2789 		 * support symmetric Toeplitz hashes:
2790 		 */
2791 		MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
2792 		rss_getkey((uint8_t *)hkey);
2793 #else
2794 		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2795 		hkey[0] = cpu_to_be32(0xD181C62C);
2796 		hkey[1] = cpu_to_be32(0xF7F4DB5B);
2797 		hkey[2] = cpu_to_be32(0x1983A2FC);
2798 		hkey[3] = cpu_to_be32(0x943E1ADB);
2799 		hkey[4] = cpu_to_be32(0xD9389E6B);
2800 		hkey[5] = cpu_to_be32(0xD1039C2C);
2801 		hkey[6] = cpu_to_be32(0xA74499AD);
2802 		hkey[7] = cpu_to_be32(0x593D56D9);
2803 		hkey[8] = cpu_to_be32(0xF3253C06);
2804 		hkey[9] = cpu_to_be32(0x2ADC1FFC);
2805 #endif
2806 		break;
2807 	}
2808 
2809 	switch (tt) {
2810 	case MLX5E_TT_IPV4_TCP:
2811 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2812 		    MLX5_L3_PROT_TYPE_IPV4);
2813 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2814 		    MLX5_L4_PROT_TYPE_TCP);
2815 #ifdef RSS
2816 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
2817 			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2818 			    MLX5_HASH_IP);
2819 		} else
2820 #endif
2821 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2822 		    MLX5_HASH_ALL);
2823 		break;
2824 
2825 	case MLX5E_TT_IPV6_TCP:
2826 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2827 		    MLX5_L3_PROT_TYPE_IPV6);
2828 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2829 		    MLX5_L4_PROT_TYPE_TCP);
2830 #ifdef RSS
2831 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
2832 			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2833 			    MLX5_HASH_IP);
2834 		} else
2835 #endif
2836 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2837 		    MLX5_HASH_ALL);
2838 		break;
2839 
2840 	case MLX5E_TT_IPV4_UDP:
2841 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2842 		    MLX5_L3_PROT_TYPE_IPV4);
2843 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2844 		    MLX5_L4_PROT_TYPE_UDP);
2845 #ifdef RSS
2846 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
2847 			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2848 			    MLX5_HASH_IP);
2849 		} else
2850 #endif
2851 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2852 		    MLX5_HASH_ALL);
2853 		break;
2854 
2855 	case MLX5E_TT_IPV6_UDP:
2856 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2857 		    MLX5_L3_PROT_TYPE_IPV6);
2858 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2859 		    MLX5_L4_PROT_TYPE_UDP);
2860 #ifdef RSS
2861 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
2862 			MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2863 			    MLX5_HASH_IP);
2864 		} else
2865 #endif
2866 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2867 		    MLX5_HASH_ALL);
2868 		break;
2869 
2870 	case MLX5E_TT_IPV4_IPSEC_AH:
2871 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2872 		    MLX5_L3_PROT_TYPE_IPV4);
2873 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2874 		    MLX5_HASH_IP_IPSEC_SPI);
2875 		break;
2876 
2877 	case MLX5E_TT_IPV6_IPSEC_AH:
2878 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2879 		    MLX5_L3_PROT_TYPE_IPV6);
2880 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2881 		    MLX5_HASH_IP_IPSEC_SPI);
2882 		break;
2883 
2884 	case MLX5E_TT_IPV4_IPSEC_ESP:
2885 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2886 		    MLX5_L3_PROT_TYPE_IPV4);
2887 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2888 		    MLX5_HASH_IP_IPSEC_SPI);
2889 		break;
2890 
2891 	case MLX5E_TT_IPV6_IPSEC_ESP:
2892 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2893 		    MLX5_L3_PROT_TYPE_IPV6);
2894 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2895 		    MLX5_HASH_IP_IPSEC_SPI);
2896 		break;
2897 
2898 	case MLX5E_TT_IPV4:
2899 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2900 		    MLX5_L3_PROT_TYPE_IPV4);
2901 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2902 		    MLX5_HASH_IP);
2903 		break;
2904 
2905 	case MLX5E_TT_IPV6:
2906 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2907 		    MLX5_L3_PROT_TYPE_IPV6);
2908 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2909 		    MLX5_HASH_IP);
2910 		break;
2911 
2912 	default:
2913 		break;
2914 	}
2915 }
2916 
2917 static int
2918 mlx5e_open_tir(struct mlx5e_priv *priv, int tt)
2919 {
2920 	struct mlx5_core_dev *mdev = priv->mdev;
2921 	u32 *in;
2922 	void *tirc;
2923 	int inlen;
2924 	int err;
2925 
2926 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
2927 	in = mlx5_vzalloc(inlen);
2928 	if (in == NULL)
2929 		return (-ENOMEM);
2930 	tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
2931 
2932 	mlx5e_build_tir_ctx(priv, tirc, tt);
2933 
2934 	err = mlx5_core_create_tir(mdev, in, inlen, &priv->tirn[tt]);
2935 
2936 	kvfree(in);
2937 
2938 	return (err);
2939 }
2940 
2941 static void
2942 mlx5e_close_tir(struct mlx5e_priv *priv, int tt)
2943 {
2944 	mlx5_core_destroy_tir(priv->mdev, priv->tirn[tt]);
2945 }
2946 
2947 static int
2948 mlx5e_open_tirs(struct mlx5e_priv *priv)
2949 {
2950 	int err;
2951 	int i;
2952 
2953 	for (i = 0; i < MLX5E_NUM_TT; i++) {
2954 		err = mlx5e_open_tir(priv, i);
2955 		if (err)
2956 			goto err_close_tirs;
2957 	}
2958 
2959 	return (0);
2960 
2961 err_close_tirs:
2962 	for (i--; i >= 0; i--)
2963 		mlx5e_close_tir(priv, i);
2964 
2965 	return (err);
2966 }
2967 
2968 static void
2969 mlx5e_close_tirs(struct mlx5e_priv *priv)
2970 {
2971 	int i;
2972 
2973 	for (i = 0; i < MLX5E_NUM_TT; i++)
2974 		mlx5e_close_tir(priv, i);
2975 }
2976 
2977 /*
2978  * SW MTU does not include headers,
2979  * HW MTU includes all headers and checksums.
2980  */
2981 static int
2982 mlx5e_set_dev_port_mtu(struct ifnet *ifp, int sw_mtu)
2983 {
2984 	struct mlx5e_priv *priv = ifp->if_softc;
2985 	struct mlx5_core_dev *mdev = priv->mdev;
2986 	int hw_mtu;
2987 	int err;
2988 
2989 	hw_mtu = MLX5E_SW2HW_MTU(sw_mtu);
2990 
2991 	err = mlx5_set_port_mtu(mdev, hw_mtu);
2992 	if (err) {
2993 		mlx5_en_err(ifp, "mlx5_set_port_mtu failed setting %d, err=%d\n",
2994 		    sw_mtu, err);
2995 		return (err);
2996 	}
2997 
2998 	/* Update vport context MTU */
2999 	err = mlx5_set_vport_mtu(mdev, hw_mtu);
3000 	if (err) {
3001 		mlx5_en_err(ifp,
3002 		    "Failed updating vport context with MTU size, err=%d\n",
3003 		    err);
3004 	}
3005 
3006 	ifp->if_mtu = sw_mtu;
3007 
3008 	err = mlx5_query_vport_mtu(mdev, &hw_mtu);
3009 	if (err || !hw_mtu) {
3010 		/* fallback to port oper mtu */
3011 		err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
3012 	}
3013 	if (err) {
3014 		mlx5_en_err(ifp,
3015 		    "Query port MTU, after setting new MTU value, failed\n");
3016 		return (err);
3017 	} else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
3018 		err = -E2BIG,
3019 		mlx5_en_err(ifp,
3020 		    "Port MTU %d is smaller than ifp mtu %d\n",
3021 		    hw_mtu, sw_mtu);
3022 	} else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
3023 		err = -EINVAL;
3024                 mlx5_en_err(ifp,
3025 		    "Port MTU %d is bigger than ifp mtu %d\n",
3026 		    hw_mtu, sw_mtu);
3027 	}
3028 	priv->params_ethtool.hw_mtu = hw_mtu;
3029 
3030 	/* compute MSB */
3031 	while (hw_mtu & (hw_mtu - 1))
3032 		hw_mtu &= (hw_mtu - 1);
3033 	priv->params_ethtool.hw_mtu_msb = hw_mtu;
3034 
3035 	return (err);
3036 }
3037 
3038 int
3039 mlx5e_open_locked(struct ifnet *ifp)
3040 {
3041 	struct mlx5e_priv *priv = ifp->if_softc;
3042 	int err;
3043 	u16 set_id;
3044 
3045 	/* check if already opened */
3046 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3047 		return (0);
3048 
3049 #ifdef RSS
3050 	if (rss_getnumbuckets() > priv->params.num_channels) {
3051 		mlx5_en_info(ifp,
3052 		    "NOTE: There are more RSS buckets(%u) than channels(%u) available\n",
3053 		    rss_getnumbuckets(), priv->params.num_channels);
3054 	}
3055 #endif
3056 	err = mlx5e_open_tises(priv);
3057 	if (err) {
3058 		mlx5_en_err(ifp, "mlx5e_open_tises failed, %d\n", err);
3059 		return (err);
3060 	}
3061 	err = mlx5_vport_alloc_q_counter(priv->mdev,
3062 	    MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
3063 	if (err) {
3064 		mlx5_en_err(priv->ifp,
3065 		    "mlx5_vport_alloc_q_counter failed: %d\n", err);
3066 		goto err_close_tises;
3067 	}
3068 	/* store counter set ID */
3069 	priv->counter_set_id = set_id;
3070 
3071 	err = mlx5e_open_channels(priv);
3072 	if (err) {
3073 		mlx5_en_err(ifp,
3074 		    "mlx5e_open_channels failed, %d\n", err);
3075 		goto err_dalloc_q_counter;
3076 	}
3077 	err = mlx5e_open_rqt(priv);
3078 	if (err) {
3079 		mlx5_en_err(ifp, "mlx5e_open_rqt failed, %d\n", err);
3080 		goto err_close_channels;
3081 	}
3082 	err = mlx5e_open_tirs(priv);
3083 	if (err) {
3084 		mlx5_en_err(ifp, "mlx5e_open_tir failed, %d\n", err);
3085 		goto err_close_rqls;
3086 	}
3087 	err = mlx5e_open_flow_table(priv);
3088 	if (err) {
3089 		mlx5_en_err(ifp,
3090 		    "mlx5e_open_flow_table failed, %d\n", err);
3091 		goto err_close_tirs;
3092 	}
3093 	err = mlx5e_add_all_vlan_rules(priv);
3094 	if (err) {
3095 		mlx5_en_err(ifp,
3096 		    "mlx5e_add_all_vlan_rules failed, %d\n", err);
3097 		goto err_close_flow_table;
3098 	}
3099 	set_bit(MLX5E_STATE_OPENED, &priv->state);
3100 
3101 	mlx5e_update_carrier(priv);
3102 	mlx5e_set_rx_mode_core(priv);
3103 
3104 	return (0);
3105 
3106 err_close_flow_table:
3107 	mlx5e_close_flow_table(priv);
3108 
3109 err_close_tirs:
3110 	mlx5e_close_tirs(priv);
3111 
3112 err_close_rqls:
3113 	mlx5e_close_rqt(priv);
3114 
3115 err_close_channels:
3116 	mlx5e_close_channels(priv);
3117 
3118 err_dalloc_q_counter:
3119 	mlx5_vport_dealloc_q_counter(priv->mdev,
3120 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3121 
3122 err_close_tises:
3123 	mlx5e_close_tises(priv);
3124 
3125 	return (err);
3126 }
3127 
3128 static void
3129 mlx5e_open(void *arg)
3130 {
3131 	struct mlx5e_priv *priv = arg;
3132 
3133 	PRIV_LOCK(priv);
3134 	if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
3135 		mlx5_en_err(priv->ifp,
3136 		    "Setting port status to up failed\n");
3137 
3138 	mlx5e_open_locked(priv->ifp);
3139 	priv->ifp->if_drv_flags |= IFF_DRV_RUNNING;
3140 	PRIV_UNLOCK(priv);
3141 }
3142 
3143 int
3144 mlx5e_close_locked(struct ifnet *ifp)
3145 {
3146 	struct mlx5e_priv *priv = ifp->if_softc;
3147 
3148 	/* check if already closed */
3149 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3150 		return (0);
3151 
3152 	clear_bit(MLX5E_STATE_OPENED, &priv->state);
3153 
3154 	mlx5e_set_rx_mode_core(priv);
3155 	mlx5e_del_all_vlan_rules(priv);
3156 	if_link_state_change(priv->ifp, LINK_STATE_DOWN);
3157 	mlx5e_close_flow_table(priv);
3158 	mlx5e_close_tirs(priv);
3159 	mlx5e_close_rqt(priv);
3160 	mlx5e_close_channels(priv);
3161 	mlx5_vport_dealloc_q_counter(priv->mdev,
3162 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3163 	mlx5e_close_tises(priv);
3164 
3165 	return (0);
3166 }
3167 
3168 #if (__FreeBSD_version >= 1100000)
3169 static uint64_t
3170 mlx5e_get_counter(struct ifnet *ifp, ift_counter cnt)
3171 {
3172 	struct mlx5e_priv *priv = ifp->if_softc;
3173 	u64 retval;
3174 
3175 	/* PRIV_LOCK(priv); XXX not allowed */
3176 	switch (cnt) {
3177 	case IFCOUNTER_IPACKETS:
3178 		retval = priv->stats.vport.rx_packets;
3179 		break;
3180 	case IFCOUNTER_IERRORS:
3181 		retval = priv->stats.pport.in_range_len_errors +
3182 		    priv->stats.pport.out_of_range_len +
3183 		    priv->stats.pport.too_long_errors +
3184 		    priv->stats.pport.check_seq_err +
3185 		    priv->stats.pport.alignment_err;
3186 		break;
3187 	case IFCOUNTER_IQDROPS:
3188 		retval = priv->stats.vport.rx_out_of_buffer;
3189 		break;
3190 	case IFCOUNTER_OPACKETS:
3191 		retval = priv->stats.vport.tx_packets;
3192 		break;
3193 	case IFCOUNTER_OERRORS:
3194 		retval = priv->stats.port_stats_debug.out_discards;
3195 		break;
3196 	case IFCOUNTER_IBYTES:
3197 		retval = priv->stats.vport.rx_bytes;
3198 		break;
3199 	case IFCOUNTER_OBYTES:
3200 		retval = priv->stats.vport.tx_bytes;
3201 		break;
3202 	case IFCOUNTER_IMCASTS:
3203 		retval = priv->stats.vport.rx_multicast_packets;
3204 		break;
3205 	case IFCOUNTER_OMCASTS:
3206 		retval = priv->stats.vport.tx_multicast_packets;
3207 		break;
3208 	case IFCOUNTER_OQDROPS:
3209 		retval = priv->stats.vport.tx_queue_dropped;
3210 		break;
3211 	case IFCOUNTER_COLLISIONS:
3212 		retval = priv->stats.pport.collisions;
3213 		break;
3214 	default:
3215 		retval = if_get_counter_default(ifp, cnt);
3216 		break;
3217 	}
3218 	/* PRIV_UNLOCK(priv); XXX not allowed */
3219 	return (retval);
3220 }
3221 #endif
3222 
3223 static void
3224 mlx5e_set_rx_mode(struct ifnet *ifp)
3225 {
3226 	struct mlx5e_priv *priv = ifp->if_softc;
3227 
3228 	queue_work(priv->wq, &priv->set_rx_mode_work);
3229 }
3230 
3231 static int
3232 mlx5e_ioctl(struct ifnet *ifp, u_long command, caddr_t data)
3233 {
3234 	struct mlx5e_priv *priv;
3235 	struct ifreq *ifr;
3236 	struct ifdownreason *ifdr;
3237 	struct ifi2creq i2c;
3238 	int error = 0;
3239 	int mask = 0;
3240 	int size_read = 0;
3241 	int module_status;
3242 	int module_num;
3243 	int max_mtu;
3244 	uint8_t read_addr;
3245 
3246 	priv = ifp->if_softc;
3247 
3248 	/* check if detaching */
3249 	if (priv == NULL || priv->gone != 0)
3250 		return (ENXIO);
3251 
3252 	switch (command) {
3253 	case SIOCSIFMTU:
3254 		ifr = (struct ifreq *)data;
3255 
3256 		PRIV_LOCK(priv);
3257 		mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
3258 
3259 		if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
3260 		    ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
3261 			int was_opened;
3262 
3263 			was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3264 			if (was_opened)
3265 				mlx5e_close_locked(ifp);
3266 
3267 			/* set new MTU */
3268 			mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
3269 
3270 			if (was_opened)
3271 				mlx5e_open_locked(ifp);
3272 		} else {
3273 			error = EINVAL;
3274 			mlx5_en_err(ifp,
3275 			    "Invalid MTU value. Min val: %d, Max val: %d\n",
3276 			    MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
3277 		}
3278 		PRIV_UNLOCK(priv);
3279 		break;
3280 	case SIOCSIFFLAGS:
3281 		if ((ifp->if_flags & IFF_UP) &&
3282 		    (ifp->if_drv_flags & IFF_DRV_RUNNING)) {
3283 			mlx5e_set_rx_mode(ifp);
3284 			break;
3285 		}
3286 		PRIV_LOCK(priv);
3287 		if (ifp->if_flags & IFF_UP) {
3288 			if ((ifp->if_drv_flags & IFF_DRV_RUNNING) == 0) {
3289 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3290 					mlx5e_open_locked(ifp);
3291 				ifp->if_drv_flags |= IFF_DRV_RUNNING;
3292 				mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
3293 			}
3294 		} else {
3295 			if (ifp->if_drv_flags & IFF_DRV_RUNNING) {
3296 				mlx5_set_port_status(priv->mdev,
3297 				    MLX5_PORT_DOWN);
3298 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3299 					mlx5e_close_locked(ifp);
3300 				mlx5e_update_carrier(priv);
3301 				ifp->if_drv_flags &= ~IFF_DRV_RUNNING;
3302 			}
3303 		}
3304 		PRIV_UNLOCK(priv);
3305 		break;
3306 	case SIOCADDMULTI:
3307 	case SIOCDELMULTI:
3308 		mlx5e_set_rx_mode(ifp);
3309 		break;
3310 	case SIOCSIFMEDIA:
3311 	case SIOCGIFMEDIA:
3312 	case SIOCGIFXMEDIA:
3313 		ifr = (struct ifreq *)data;
3314 		error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
3315 		break;
3316 	case SIOCSIFCAP:
3317 		ifr = (struct ifreq *)data;
3318 		PRIV_LOCK(priv);
3319 		mask = ifr->ifr_reqcap ^ ifp->if_capenable;
3320 
3321 		if (mask & IFCAP_TXCSUM) {
3322 			ifp->if_capenable ^= IFCAP_TXCSUM;
3323 			ifp->if_hwassist ^= (CSUM_TCP | CSUM_UDP | CSUM_IP);
3324 
3325 			if (IFCAP_TSO4 & ifp->if_capenable &&
3326 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
3327 				mask &= ~IFCAP_TSO4;
3328 				ifp->if_capenable &= ~IFCAP_TSO4;
3329 				ifp->if_hwassist &= ~CSUM_IP_TSO;
3330 				mlx5_en_err(ifp,
3331 				    "tso4 disabled due to -txcsum.\n");
3332 			}
3333 		}
3334 		if (mask & IFCAP_TXCSUM_IPV6) {
3335 			ifp->if_capenable ^= IFCAP_TXCSUM_IPV6;
3336 			ifp->if_hwassist ^= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
3337 
3338 			if (IFCAP_TSO6 & ifp->if_capenable &&
3339 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3340 				mask &= ~IFCAP_TSO6;
3341 				ifp->if_capenable &= ~IFCAP_TSO6;
3342 				ifp->if_hwassist &= ~CSUM_IP6_TSO;
3343 				mlx5_en_err(ifp,
3344 				    "tso6 disabled due to -txcsum6.\n");
3345 			}
3346 		}
3347 		if (mask & IFCAP_NOMAP)
3348 			ifp->if_capenable ^= IFCAP_NOMAP;
3349 		if (mask & IFCAP_TXTLS4)
3350 			ifp->if_capenable ^= IFCAP_TXTLS4;
3351 		if (mask & IFCAP_TXTLS6)
3352 			ifp->if_capenable ^= IFCAP_TXTLS6;
3353 		if (mask & IFCAP_RXCSUM)
3354 			ifp->if_capenable ^= IFCAP_RXCSUM;
3355 		if (mask & IFCAP_RXCSUM_IPV6)
3356 			ifp->if_capenable ^= IFCAP_RXCSUM_IPV6;
3357 		if (mask & IFCAP_TSO4) {
3358 			if (!(IFCAP_TSO4 & ifp->if_capenable) &&
3359 			    !(IFCAP_TXCSUM & ifp->if_capenable)) {
3360 				mlx5_en_err(ifp, "enable txcsum first.\n");
3361 				error = EAGAIN;
3362 				goto out;
3363 			}
3364 			ifp->if_capenable ^= IFCAP_TSO4;
3365 			ifp->if_hwassist ^= CSUM_IP_TSO;
3366 		}
3367 		if (mask & IFCAP_TSO6) {
3368 			if (!(IFCAP_TSO6 & ifp->if_capenable) &&
3369 			    !(IFCAP_TXCSUM_IPV6 & ifp->if_capenable)) {
3370 				mlx5_en_err(ifp, "enable txcsum6 first.\n");
3371 				error = EAGAIN;
3372 				goto out;
3373 			}
3374 			ifp->if_capenable ^= IFCAP_TSO6;
3375 			ifp->if_hwassist ^= CSUM_IP6_TSO;
3376 		}
3377 		if (mask & IFCAP_VLAN_HWFILTER) {
3378 			if (ifp->if_capenable & IFCAP_VLAN_HWFILTER)
3379 				mlx5e_disable_vlan_filter(priv);
3380 			else
3381 				mlx5e_enable_vlan_filter(priv);
3382 
3383 			ifp->if_capenable ^= IFCAP_VLAN_HWFILTER;
3384 		}
3385 		if (mask & IFCAP_VLAN_HWTAGGING)
3386 			ifp->if_capenable ^= IFCAP_VLAN_HWTAGGING;
3387 		if (mask & IFCAP_WOL_MAGIC)
3388 			ifp->if_capenable ^= IFCAP_WOL_MAGIC;
3389 
3390 		VLAN_CAPABILITIES(ifp);
3391 		/* turn off LRO means also turn of HW LRO - if it's on */
3392 		if (mask & IFCAP_LRO) {
3393 			int was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3394 			bool need_restart = false;
3395 
3396 			ifp->if_capenable ^= IFCAP_LRO;
3397 
3398 			/* figure out if updating HW LRO is needed */
3399 			if (!(ifp->if_capenable & IFCAP_LRO)) {
3400 				if (priv->params.hw_lro_en) {
3401 					priv->params.hw_lro_en = false;
3402 					need_restart = true;
3403 				}
3404 			} else {
3405 				if (priv->params.hw_lro_en == false &&
3406 				    priv->params_ethtool.hw_lro != 0) {
3407 					priv->params.hw_lro_en = true;
3408 					need_restart = true;
3409 				}
3410 			}
3411 			if (was_opened && need_restart) {
3412 				mlx5e_close_locked(ifp);
3413 				mlx5e_open_locked(ifp);
3414 			}
3415 		}
3416 		if (mask & IFCAP_HWRXTSTMP) {
3417 			ifp->if_capenable ^= IFCAP_HWRXTSTMP;
3418 			if (ifp->if_capenable & IFCAP_HWRXTSTMP) {
3419 				if (priv->clbr_done == 0)
3420 					mlx5e_reset_calibration_callout(priv);
3421 			} else {
3422 				callout_drain(&priv->tstmp_clbr);
3423 				priv->clbr_done = 0;
3424 			}
3425 		}
3426 out:
3427 		PRIV_UNLOCK(priv);
3428 		break;
3429 
3430 	case SIOCGI2C:
3431 		ifr = (struct ifreq *)data;
3432 
3433 		/*
3434 		 * Copy from the user-space address ifr_data to the
3435 		 * kernel-space address i2c
3436 		 */
3437 		error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3438 		if (error)
3439 			break;
3440 
3441 		if (i2c.len > sizeof(i2c.data)) {
3442 			error = EINVAL;
3443 			break;
3444 		}
3445 
3446 		PRIV_LOCK(priv);
3447 		/* Get module_num which is required for the query_eeprom */
3448 		error = mlx5_query_module_num(priv->mdev, &module_num);
3449 		if (error) {
3450 			mlx5_en_err(ifp,
3451 			    "Query module num failed, eeprom reading is not supported\n");
3452 			error = EINVAL;
3453 			goto err_i2c;
3454 		}
3455 		/* Check if module is present before doing an access */
3456 		module_status = mlx5_query_module_status(priv->mdev, module_num);
3457 		if (module_status != MLX5_MODULE_STATUS_PLUGGED_ENABLED) {
3458 			error = EINVAL;
3459 			goto err_i2c;
3460 		}
3461 		/*
3462 		 * Currently 0XA0 and 0xA2 are the only addresses permitted.
3463 		 * The internal conversion is as follows:
3464 		 */
3465 		if (i2c.dev_addr == 0xA0)
3466 			read_addr = MLX5_I2C_ADDR_LOW;
3467 		else if (i2c.dev_addr == 0xA2)
3468 			read_addr = MLX5_I2C_ADDR_HIGH;
3469 		else {
3470 			mlx5_en_err(ifp,
3471 			    "Query eeprom failed, Invalid Address: %X\n",
3472 			    i2c.dev_addr);
3473 			error = EINVAL;
3474 			goto err_i2c;
3475 		}
3476 		error = mlx5_query_eeprom(priv->mdev,
3477 		    read_addr, MLX5_EEPROM_LOW_PAGE,
3478 		    (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
3479 		    (uint32_t *)i2c.data, &size_read);
3480 		if (error) {
3481 			mlx5_en_err(ifp,
3482 			    "Query eeprom failed, eeprom reading is not supported\n");
3483 			error = EINVAL;
3484 			goto err_i2c;
3485 		}
3486 
3487 		if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
3488 			error = mlx5_query_eeprom(priv->mdev,
3489 			    read_addr, MLX5_EEPROM_LOW_PAGE,
3490 			    (uint32_t)(i2c.offset + size_read),
3491 			    (uint32_t)(i2c.len - size_read), module_num,
3492 			    (uint32_t *)(i2c.data + size_read), &size_read);
3493 		}
3494 		if (error) {
3495 			mlx5_en_err(ifp,
3496 			    "Query eeprom failed, eeprom reading is not supported\n");
3497 			error = EINVAL;
3498 			goto err_i2c;
3499 		}
3500 
3501 		error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3502 err_i2c:
3503 		PRIV_UNLOCK(priv);
3504 		break;
3505 	case SIOCGIFDOWNREASON:
3506 		ifdr = (struct ifdownreason *)data;
3507 		bzero(ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3508 		PRIV_LOCK(priv);
3509 		error = -mlx5_query_pddr_troubleshooting_info(priv->mdev, NULL,
3510 		    ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3511 		PRIV_UNLOCK(priv);
3512 		if (error == 0)
3513 			ifdr->ifdr_reason = IFDR_REASON_MSG;
3514 		break;
3515 
3516 	default:
3517 		error = ether_ioctl(ifp, command, data);
3518 		break;
3519 	}
3520 	return (error);
3521 }
3522 
3523 static int
3524 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3525 {
3526 	/*
3527 	 * TODO: uncoment once FW really sets all these bits if
3528 	 * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
3529 	 * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
3530 	 * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
3531 	 * -ENOTSUPP;
3532 	 */
3533 
3534 	/* TODO: add more must-to-have features */
3535 
3536 	if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3537 		return (-ENODEV);
3538 
3539 	return (0);
3540 }
3541 
3542 static u16
3543 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3544 {
3545 	const int min_size = ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN;
3546 	const int max_size = MLX5E_MAX_TX_INLINE;
3547 	const int bf_buf_size =
3548 	    ((1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U) -
3549 	    (sizeof(struct mlx5e_tx_wqe) - 2);
3550 
3551 	/* verify against driver limits */
3552 	if (bf_buf_size > max_size)
3553 		return (max_size);
3554 	else if (bf_buf_size < min_size)
3555 		return (min_size);
3556 	else
3557 		return (bf_buf_size);
3558 }
3559 
3560 static int
3561 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
3562     struct mlx5e_priv *priv,
3563     int num_comp_vectors)
3564 {
3565 	int err;
3566 
3567 	/*
3568 	 * TODO: Consider link speed for setting "log_sq_size",
3569 	 * "log_rq_size" and "cq_moderation_xxx":
3570 	 */
3571 	priv->params.log_sq_size =
3572 	    MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3573 	priv->params.log_rq_size =
3574 	    MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3575 	priv->params.rx_cq_moderation_usec =
3576 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3577 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
3578 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3579 	priv->params.rx_cq_moderation_mode =
3580 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
3581 	priv->params.rx_cq_moderation_pkts =
3582 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3583 	priv->params.tx_cq_moderation_usec =
3584 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3585 	priv->params.tx_cq_moderation_pkts =
3586 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3587 	priv->params.min_rx_wqes =
3588 	    MLX5E_PARAMS_DEFAULT_MIN_RX_WQES;
3589 	priv->params.rx_hash_log_tbl_sz =
3590 	    (order_base_2(num_comp_vectors) >
3591 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
3592 	    order_base_2(num_comp_vectors) :
3593 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
3594 	priv->params.num_tc = 1;
3595 	priv->params.default_vlan_prio = 0;
3596 	priv->counter_set_id = -1;
3597 	priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3598 
3599 	err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3600 	if (err)
3601 		return (err);
3602 
3603 	/*
3604 	 * hw lro is currently defaulted to off. when it won't anymore we
3605 	 * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
3606 	 */
3607 	priv->params.hw_lro_en = false;
3608 	priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3609 
3610 	/*
3611 	 * CQE zipping is currently defaulted to off. when it won't
3612 	 * anymore we will consider the HW capability:
3613 	 * "!!MLX5_CAP_GEN(mdev, cqe_compression)"
3614 	 */
3615 	priv->params.cqe_zipping_en = false;
3616 
3617 	priv->mdev = mdev;
3618 	priv->params.num_channels = num_comp_vectors;
3619 	priv->params.channels_rsss = 1;
3620 	priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
3621 	priv->queue_mapping_channel_mask =
3622 	    roundup_pow_of_two(num_comp_vectors) - 1;
3623 	priv->num_tc = priv->params.num_tc;
3624 	priv->default_vlan_prio = priv->params.default_vlan_prio;
3625 
3626 	INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3627 	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3628 	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3629 
3630 	return (0);
3631 }
3632 
3633 static int
3634 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3635 		  struct mlx5_core_mr *mkey)
3636 {
3637 	struct ifnet *ifp = priv->ifp;
3638 	struct mlx5_core_dev *mdev = priv->mdev;
3639 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3640 	void *mkc;
3641 	u32 *in;
3642 	int err;
3643 
3644 	in = mlx5_vzalloc(inlen);
3645 	if (in == NULL) {
3646 		mlx5_en_err(ifp, "failed to allocate inbox\n");
3647 		return (-ENOMEM);
3648 	}
3649 
3650 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
3651 	MLX5_SET(mkc, mkc, access_mode, MLX5_ACCESS_MODE_PA);
3652 	MLX5_SET(mkc, mkc, umr_en, 1);	/* used by HW TLS */
3653 	MLX5_SET(mkc, mkc, lw, 1);
3654 	MLX5_SET(mkc, mkc, lr, 1);
3655 
3656 	MLX5_SET(mkc, mkc, pd, pdn);
3657 	MLX5_SET(mkc, mkc, length64, 1);
3658 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
3659 
3660 	err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
3661 	if (err)
3662 		mlx5_en_err(ifp, "mlx5_core_create_mkey failed, %d\n",
3663 		    err);
3664 
3665 	kvfree(in);
3666 	return (err);
3667 }
3668 
3669 static const char *mlx5e_vport_stats_desc[] = {
3670 	MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
3671 };
3672 
3673 static const char *mlx5e_pport_stats_desc[] = {
3674 	MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
3675 };
3676 
3677 static void
3678 mlx5e_priv_static_init(struct mlx5e_priv *priv, const uint32_t channels)
3679 {
3680 	uint32_t x;
3681 
3682 	mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
3683 	sx_init(&priv->state_lock, "mlx5state");
3684 	callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
3685 	MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
3686 	for (x = 0; x != channels; x++)
3687 		mlx5e_chan_static_init(priv, &priv->channel[x], x);
3688 }
3689 
3690 static void
3691 mlx5e_priv_static_destroy(struct mlx5e_priv *priv, const uint32_t channels)
3692 {
3693 	uint32_t x;
3694 
3695 	for (x = 0; x != channels; x++)
3696 		mlx5e_chan_static_destroy(&priv->channel[x]);
3697 	callout_drain(&priv->watchdog);
3698 	mtx_destroy(&priv->async_events_mtx);
3699 	sx_destroy(&priv->state_lock);
3700 }
3701 
3702 static int
3703 sysctl_firmware(SYSCTL_HANDLER_ARGS)
3704 {
3705 	/*
3706 	 * %d.%d%.d the string format.
3707 	 * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
3708 	 * We need at most 5 chars to store that.
3709 	 * It also has: two "." and NULL at the end, which means we need 18
3710 	 * (5*3 + 3) chars at most.
3711 	 */
3712 	char fw[18];
3713 	struct mlx5e_priv *priv = arg1;
3714 	int error;
3715 
3716 	snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
3717 	    fw_rev_sub(priv->mdev));
3718 	error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
3719 	return (error);
3720 }
3721 
3722 static void
3723 mlx5e_disable_tx_dma(struct mlx5e_channel *ch)
3724 {
3725 	int i;
3726 
3727 	for (i = 0; i < ch->priv->num_tc; i++)
3728 		mlx5e_drain_sq(&ch->sq[i]);
3729 }
3730 
3731 static void
3732 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
3733 {
3734 
3735 	sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
3736 	sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
3737 	mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
3738 	sq->doorbell.d64 = 0;
3739 }
3740 
3741 void
3742 mlx5e_resume_sq(struct mlx5e_sq *sq)
3743 {
3744 	int err;
3745 
3746 	/* check if already enabled */
3747 	if (READ_ONCE(sq->running) != 0)
3748 		return;
3749 
3750 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
3751 	    MLX5_SQC_STATE_RST);
3752 	if (err != 0) {
3753 		mlx5_en_err(sq->ifp,
3754 		    "mlx5e_modify_sq() from ERR to RST failed: %d\n", err);
3755 	}
3756 
3757 	sq->cc = 0;
3758 	sq->pc = 0;
3759 
3760 	/* reset doorbell prior to moving from RST to RDY */
3761 	mlx5e_reset_sq_doorbell_record(sq);
3762 
3763 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
3764 	    MLX5_SQC_STATE_RDY);
3765 	if (err != 0) {
3766 		mlx5_en_err(sq->ifp,
3767 		    "mlx5e_modify_sq() from RST to RDY failed: %d\n", err);
3768 	}
3769 
3770 	sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
3771 	WRITE_ONCE(sq->running, 1);
3772 }
3773 
3774 static void
3775 mlx5e_enable_tx_dma(struct mlx5e_channel *ch)
3776 {
3777         int i;
3778 
3779 	for (i = 0; i < ch->priv->num_tc; i++)
3780 		mlx5e_resume_sq(&ch->sq[i]);
3781 }
3782 
3783 static void
3784 mlx5e_disable_rx_dma(struct mlx5e_channel *ch)
3785 {
3786 	struct mlx5e_rq *rq = &ch->rq;
3787 	struct epoch_tracker et;
3788 	int err;
3789 
3790 	mtx_lock(&rq->mtx);
3791 	rq->enabled = 0;
3792 	callout_stop(&rq->watchdog);
3793 	mtx_unlock(&rq->mtx);
3794 
3795 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
3796 	if (err != 0) {
3797 		mlx5_en_err(rq->ifp,
3798 		    "mlx5e_modify_rq() from RDY to RST failed: %d\n", err);
3799 	}
3800 
3801 	while (!mlx5_wq_ll_is_empty(&rq->wq)) {
3802 		msleep(1);
3803 		NET_EPOCH_ENTER(et);
3804 		rq->cq.mcq.comp(&rq->cq.mcq);
3805 		NET_EPOCH_EXIT(et);
3806 	}
3807 
3808 	/*
3809 	 * Transitioning into RST state will allow the FW to track less ERR state queues,
3810 	 * thus reducing the recv queue flushing time
3811 	 */
3812 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
3813 	if (err != 0) {
3814 		mlx5_en_err(rq->ifp,
3815 		    "mlx5e_modify_rq() from ERR to RST failed: %d\n", err);
3816 	}
3817 }
3818 
3819 static void
3820 mlx5e_enable_rx_dma(struct mlx5e_channel *ch)
3821 {
3822 	struct mlx5e_rq *rq = &ch->rq;
3823 	struct epoch_tracker et;
3824 	int err;
3825 
3826 	rq->wq.wqe_ctr = 0;
3827 	mlx5_wq_ll_update_db_record(&rq->wq);
3828 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3829 	if (err != 0) {
3830 		mlx5_en_err(rq->ifp,
3831 		    "mlx5e_modify_rq() from RST to RDY failed: %d\n", err);
3832         }
3833 
3834 	rq->enabled = 1;
3835 
3836 	NET_EPOCH_ENTER(et);
3837 	rq->cq.mcq.comp(&rq->cq.mcq);
3838 	NET_EPOCH_EXIT(et);
3839 }
3840 
3841 void
3842 mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value)
3843 {
3844 	int i;
3845 
3846 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3847 		return;
3848 
3849 	for (i = 0; i < priv->params.num_channels; i++) {
3850 		if (value)
3851 			mlx5e_disable_tx_dma(&priv->channel[i]);
3852 		else
3853 			mlx5e_enable_tx_dma(&priv->channel[i]);
3854 	}
3855 }
3856 
3857 void
3858 mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value)
3859 {
3860 	int i;
3861 
3862 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3863 		return;
3864 
3865 	for (i = 0; i < priv->params.num_channels; i++) {
3866 		if (value)
3867 			mlx5e_disable_rx_dma(&priv->channel[i]);
3868 		else
3869 			mlx5e_enable_rx_dma(&priv->channel[i]);
3870 	}
3871 }
3872 
3873 static void
3874 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
3875 {
3876 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
3877 	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
3878 	    priv, 0, sysctl_firmware, "A", "HCA firmware version");
3879 
3880 	SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
3881 	    OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
3882 	    "Board ID");
3883 }
3884 
3885 static int
3886 mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
3887 {
3888 	struct mlx5e_priv *priv = arg1;
3889 	uint8_t temp[MLX5E_MAX_PRIORITY];
3890 	uint32_t tx_pfc;
3891 	int err;
3892 	int i;
3893 
3894 	PRIV_LOCK(priv);
3895 
3896 	tx_pfc = priv->params.tx_priority_flow_control;
3897 
3898 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
3899 		temp[i] = (tx_pfc >> i) & 1;
3900 
3901 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
3902 	if (err || !req->newptr)
3903 		goto done;
3904 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
3905 	if (err)
3906 		goto done;
3907 
3908 	priv->params.tx_priority_flow_control = 0;
3909 
3910 	/* range check input value */
3911 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
3912 		if (temp[i] > 1) {
3913 			err = ERANGE;
3914 			goto done;
3915 		}
3916 		priv->params.tx_priority_flow_control |= (temp[i] << i);
3917 	}
3918 
3919 	/* check if update is required */
3920 	if (tx_pfc != priv->params.tx_priority_flow_control)
3921 		err = -mlx5e_set_port_pfc(priv);
3922 done:
3923 	if (err != 0)
3924 		priv->params.tx_priority_flow_control= tx_pfc;
3925 	PRIV_UNLOCK(priv);
3926 
3927 	return (err);
3928 }
3929 
3930 static int
3931 mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
3932 {
3933 	struct mlx5e_priv *priv = arg1;
3934 	uint8_t temp[MLX5E_MAX_PRIORITY];
3935 	uint32_t rx_pfc;
3936 	int err;
3937 	int i;
3938 
3939 	PRIV_LOCK(priv);
3940 
3941 	rx_pfc = priv->params.rx_priority_flow_control;
3942 
3943 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
3944 		temp[i] = (rx_pfc >> i) & 1;
3945 
3946 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
3947 	if (err || !req->newptr)
3948 		goto done;
3949 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
3950 	if (err)
3951 		goto done;
3952 
3953 	priv->params.rx_priority_flow_control = 0;
3954 
3955 	/* range check input value */
3956 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
3957 		if (temp[i] > 1) {
3958 			err = ERANGE;
3959 			goto done;
3960 		}
3961 		priv->params.rx_priority_flow_control |= (temp[i] << i);
3962 	}
3963 
3964 	/* check if update is required */
3965 	if (rx_pfc != priv->params.rx_priority_flow_control) {
3966 		err = -mlx5e_set_port_pfc(priv);
3967 		if (err == 0 && priv->sw_is_port_buf_owner)
3968 			err = mlx5e_update_buf_lossy(priv);
3969 	}
3970 done:
3971 	if (err != 0)
3972 		priv->params.rx_priority_flow_control= rx_pfc;
3973 	PRIV_UNLOCK(priv);
3974 
3975 	return (err);
3976 }
3977 
3978 static void
3979 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
3980 {
3981 #if (__FreeBSD_version < 1100000)
3982 	char path[96];
3983 #endif
3984 	int error;
3985 
3986 	/* enable pauseframes by default */
3987 	priv->params.tx_pauseframe_control = 1;
3988 	priv->params.rx_pauseframe_control = 1;
3989 
3990 	/* disable ports flow control, PFC, by default */
3991 	priv->params.tx_priority_flow_control = 0;
3992 	priv->params.rx_priority_flow_control = 0;
3993 
3994 #if (__FreeBSD_version < 1100000)
3995 	/* compute path for sysctl */
3996 	snprintf(path, sizeof(path), "dev.mce.%d.tx_pauseframe_control",
3997 	    device_get_unit(priv->mdev->pdev->dev.bsddev));
3998 
3999 	/* try to fetch tunable, if any */
4000 	TUNABLE_INT_FETCH(path, &priv->params.tx_pauseframe_control);
4001 
4002 	/* compute path for sysctl */
4003 	snprintf(path, sizeof(path), "dev.mce.%d.rx_pauseframe_control",
4004 	    device_get_unit(priv->mdev->pdev->dev.bsddev));
4005 
4006 	/* try to fetch tunable, if any */
4007 	TUNABLE_INT_FETCH(path, &priv->params.rx_pauseframe_control);
4008 #endif
4009 
4010 	/* register pauseframe SYSCTLs */
4011 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4012 	    OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
4013 	    &priv->params.tx_pauseframe_control, 0,
4014 	    "Set to enable TX pause frames. Clear to disable.");
4015 
4016 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4017 	    OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
4018 	    &priv->params.rx_pauseframe_control, 0,
4019 	    "Set to enable RX pause frames. Clear to disable.");
4020 
4021 	/* register priority flow control, PFC, SYSCTLs */
4022 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4023 	    OID_AUTO, "tx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4024 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_tx_priority_flow_control, "CU",
4025 	    "Set to enable TX ports flow control frames for priorities 0..7. Clear to disable.");
4026 
4027 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4028 	    OID_AUTO, "rx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4029 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_rx_priority_flow_control, "CU",
4030 	    "Set to enable RX ports flow control frames for priorities 0..7. Clear to disable.");
4031 
4032 	PRIV_LOCK(priv);
4033 
4034 	/* range check */
4035 	priv->params.tx_pauseframe_control =
4036 	    priv->params.tx_pauseframe_control ? 1 : 0;
4037 	priv->params.rx_pauseframe_control =
4038 	    priv->params.rx_pauseframe_control ? 1 : 0;
4039 
4040 	/* update firmware */
4041 	error = mlx5e_set_port_pause_and_pfc(priv);
4042 	if (error == -EINVAL) {
4043 		mlx5_en_err(priv->ifp,
4044 		    "Global pauseframes must be disabled before enabling PFC.\n");
4045 		priv->params.rx_priority_flow_control = 0;
4046 		priv->params.tx_priority_flow_control = 0;
4047 
4048 		/* update firmware */
4049 		(void) mlx5e_set_port_pause_and_pfc(priv);
4050 	}
4051 	PRIV_UNLOCK(priv);
4052 }
4053 
4054 int
4055 mlx5e_ul_snd_tag_alloc(struct ifnet *ifp,
4056     union if_snd_tag_alloc_params *params,
4057     struct m_snd_tag **ppmt)
4058 {
4059 	struct mlx5e_priv *priv;
4060 	struct mlx5e_channel *pch;
4061 
4062 	priv = ifp->if_softc;
4063 
4064 	if (unlikely(priv->gone || params->hdr.flowtype == M_HASHTYPE_NONE)) {
4065 		return (EOPNOTSUPP);
4066 	} else {
4067 		/* keep this code synced with mlx5e_select_queue() */
4068 		u32 ch = priv->params.num_channels;
4069 #ifdef RSS
4070 		u32 temp;
4071 
4072 		if (rss_hash2bucket(params->hdr.flowid,
4073 		    params->hdr.flowtype, &temp) == 0)
4074 			ch = temp % ch;
4075 		else
4076 #endif
4077 			ch = (params->hdr.flowid % 128) % ch;
4078 
4079 		/*
4080 		 * NOTE: The channels array is only freed at detach
4081 		 * and it safe to return a pointer to the send tag
4082 		 * inside the channels structure as long as we
4083 		 * reference the priv.
4084 		 */
4085 		pch = priv->channel + ch;
4086 
4087 		/* check if send queue is not running */
4088 		if (unlikely(pch->sq[0].running == 0))
4089 			return (ENXIO);
4090 		m_snd_tag_ref(&pch->tag.m_snd_tag);
4091 		*ppmt = &pch->tag.m_snd_tag;
4092 		return (0);
4093 	}
4094 }
4095 
4096 int
4097 mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4098 {
4099 	struct mlx5e_channel *pch =
4100 	    container_of(pmt, struct mlx5e_channel, tag.m_snd_tag);
4101 
4102 	params->unlimited.max_rate = -1ULL;
4103 	params->unlimited.queue_level = mlx5e_sq_queue_level(&pch->sq[0]);
4104 	return (0);
4105 }
4106 
4107 void
4108 mlx5e_ul_snd_tag_free(struct m_snd_tag *pmt)
4109 {
4110 	struct mlx5e_channel *pch =
4111 	    container_of(pmt, struct mlx5e_channel, tag.m_snd_tag);
4112 
4113 	complete(&pch->completion);
4114 }
4115 
4116 static int
4117 mlx5e_snd_tag_alloc(struct ifnet *ifp,
4118     union if_snd_tag_alloc_params *params,
4119     struct m_snd_tag **ppmt)
4120 {
4121 
4122 	switch (params->hdr.type) {
4123 #ifdef RATELIMIT
4124 	case IF_SND_TAG_TYPE_RATE_LIMIT:
4125 		return (mlx5e_rl_snd_tag_alloc(ifp, params, ppmt));
4126 #if defined(KERN_TLS) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
4127 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4128 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4129 #endif
4130 #endif
4131 	case IF_SND_TAG_TYPE_UNLIMITED:
4132 		return (mlx5e_ul_snd_tag_alloc(ifp, params, ppmt));
4133 #ifdef KERN_TLS
4134 	case IF_SND_TAG_TYPE_TLS:
4135 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4136 #endif
4137 	default:
4138 		return (EOPNOTSUPP);
4139 	}
4140 }
4141 
4142 static int
4143 mlx5e_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
4144 {
4145 	struct mlx5e_snd_tag *tag =
4146 	    container_of(pmt, struct mlx5e_snd_tag, m_snd_tag);
4147 
4148 	switch (tag->type) {
4149 #ifdef RATELIMIT
4150 	case IF_SND_TAG_TYPE_RATE_LIMIT:
4151 		return (mlx5e_rl_snd_tag_modify(pmt, params));
4152 #if defined(KERN_TLS) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
4153 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4154 		return (mlx5e_tls_snd_tag_modify(pmt, params));
4155 #endif
4156 #endif
4157 	case IF_SND_TAG_TYPE_UNLIMITED:
4158 #ifdef KERN_TLS
4159 	case IF_SND_TAG_TYPE_TLS:
4160 #endif
4161 	default:
4162 		return (EOPNOTSUPP);
4163 	}
4164 }
4165 
4166 static int
4167 mlx5e_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4168 {
4169 	struct mlx5e_snd_tag *tag =
4170 	    container_of(pmt, struct mlx5e_snd_tag, m_snd_tag);
4171 
4172 	switch (tag->type) {
4173 #ifdef RATELIMIT
4174 	case IF_SND_TAG_TYPE_RATE_LIMIT:
4175 		return (mlx5e_rl_snd_tag_query(pmt, params));
4176 #if defined(KERN_TLS) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
4177 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4178 		return (mlx5e_tls_snd_tag_query(pmt, params));
4179 #endif
4180 #endif
4181 	case IF_SND_TAG_TYPE_UNLIMITED:
4182 		return (mlx5e_ul_snd_tag_query(pmt, params));
4183 #ifdef KERN_TLS
4184 	case IF_SND_TAG_TYPE_TLS:
4185 		return (mlx5e_tls_snd_tag_query(pmt, params));
4186 #endif
4187 	default:
4188 		return (EOPNOTSUPP);
4189 	}
4190 }
4191 
4192 #ifdef RATELIMIT
4193 #define NUM_HDWR_RATES_MLX 13
4194 static const uint64_t adapter_rates_mlx[NUM_HDWR_RATES_MLX] = {
4195 	135375,			/* 1,083,000 */
4196 	180500,			/* 1,444,000 */
4197 	270750,			/* 2,166,000 */
4198 	361000,			/* 2,888,000 */
4199 	541500,			/* 4,332,000 */
4200 	721875,			/* 5,775,000 */
4201 	1082875,		/* 8,663,000 */
4202 	1443875,		/* 11,551,000 */
4203 	2165750,		/* 17,326,000 */
4204 	2887750,		/* 23,102,000 */
4205 	4331625,		/* 34,653,000 */
4206 	5775500,		/* 46,204,000 */
4207 	8663125			/* 69,305,000 */
4208 };
4209 
4210 static void
4211 mlx5e_ratelimit_query(struct ifnet *ifp __unused, struct if_ratelimit_query_results *q)
4212 {
4213 	/*
4214 	 * This function needs updating by the driver maintainer!
4215 	 * For the MLX card there are currently (ConectX-4?) 13
4216 	 * pre-set rates and others i.e. ConnectX-5, 6, 7??
4217 	 *
4218 	 * This will change based on later adapters
4219 	 * and this code should be updated to look at ifp
4220 	 * and figure out the specific adapter type
4221 	 * settings i.e. how many rates as well
4222 	 * as if they are fixed (as is shown here) or
4223 	 * if they are dynamic (example chelsio t4). Also if there
4224 	 * is a maximum number of flows that the adapter
4225 	 * can handle that too needs to be updated in
4226 	 * the max_flows field.
4227 	 */
4228 	q->rate_table = adapter_rates_mlx;
4229 	q->flags = RT_IS_FIXED_TABLE;
4230 	q->max_flows = 0;	/* mlx has no limit */
4231 	q->number_of_rates = NUM_HDWR_RATES_MLX;
4232 	q->min_segment_burst = 1;
4233 }
4234 #endif
4235 
4236 static void
4237 mlx5e_snd_tag_free(struct m_snd_tag *pmt)
4238 {
4239 	struct mlx5e_snd_tag *tag =
4240 	    container_of(pmt, struct mlx5e_snd_tag, m_snd_tag);
4241 
4242 	switch (tag->type) {
4243 #ifdef RATELIMIT
4244 	case IF_SND_TAG_TYPE_RATE_LIMIT:
4245 		mlx5e_rl_snd_tag_free(pmt);
4246 		break;
4247 #if defined(KERN_TLS) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
4248 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4249 		mlx5e_tls_snd_tag_free(pmt);
4250 		break;
4251 #endif
4252 #endif
4253 	case IF_SND_TAG_TYPE_UNLIMITED:
4254 		mlx5e_ul_snd_tag_free(pmt);
4255 		break;
4256 #ifdef KERN_TLS
4257 	case IF_SND_TAG_TYPE_TLS:
4258 		mlx5e_tls_snd_tag_free(pmt);
4259 		break;
4260 #endif
4261 	default:
4262 		break;
4263 	}
4264 }
4265 
4266 static void *
4267 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
4268 {
4269 	struct ifnet *ifp;
4270 	struct mlx5e_priv *priv;
4271 	u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
4272 	u8 connector_type;
4273 	struct sysctl_oid_list *child;
4274 	int ncv = mdev->priv.eq_table.num_comp_vectors;
4275 	char unit[16];
4276 	struct pfil_head_args pa;
4277 	int err;
4278 	int i,j;
4279 	u32 eth_proto_cap;
4280 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
4281 	bool ext = 0;
4282 	u32 speeds_num;
4283 	struct media media_entry = {};
4284 
4285 	if (mlx5e_check_required_hca_cap(mdev)) {
4286 		mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
4287 		return (NULL);
4288 	}
4289 	/*
4290 	 * Try to allocate the priv and make room for worst-case
4291 	 * number of channel structures:
4292 	 */
4293 	priv = malloc(sizeof(*priv) +
4294 	    (sizeof(priv->channel[0]) * mdev->priv.eq_table.num_comp_vectors),
4295 	    M_MLX5EN, M_WAITOK | M_ZERO);
4296 
4297 	ifp = priv->ifp = if_alloc_dev(IFT_ETHER, mdev->pdev->dev.bsddev);
4298 	if (ifp == NULL) {
4299 		mlx5_core_err(mdev, "if_alloc() failed\n");
4300 		goto err_free_priv;
4301 	}
4302 	/* setup all static fields */
4303 	mlx5e_priv_static_init(priv, mdev->priv.eq_table.num_comp_vectors);
4304 
4305 	ifp->if_softc = priv;
4306 	if_initname(ifp, "mce", device_get_unit(mdev->pdev->dev.bsddev));
4307 	ifp->if_mtu = ETHERMTU;
4308 	ifp->if_init = mlx5e_open;
4309 	ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST |
4310 	    IFF_KNOWSEPOCH;
4311 	ifp->if_ioctl = mlx5e_ioctl;
4312 	ifp->if_transmit = mlx5e_xmit;
4313 	ifp->if_qflush = if_qflush;
4314 #if (__FreeBSD_version >= 1100000)
4315 	ifp->if_get_counter = mlx5e_get_counter;
4316 #endif
4317 	ifp->if_snd.ifq_maxlen = ifqmaxlen;
4318 	/*
4319          * Set driver features
4320          */
4321 	ifp->if_capabilities |= IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6;
4322 	ifp->if_capabilities |= IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING;
4323 	ifp->if_capabilities |= IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER;
4324 	ifp->if_capabilities |= IFCAP_LINKSTATE | IFCAP_JUMBO_MTU;
4325 	ifp->if_capabilities |= IFCAP_LRO;
4326 	ifp->if_capabilities |= IFCAP_TSO | IFCAP_VLAN_HWTSO;
4327 	ifp->if_capabilities |= IFCAP_HWSTATS | IFCAP_HWRXTSTMP;
4328 	ifp->if_capabilities |= IFCAP_NOMAP;
4329 	ifp->if_capabilities |= IFCAP_TXTLS4 | IFCAP_TXTLS6;
4330 	ifp->if_capabilities |= IFCAP_TXRTLMT;
4331 	ifp->if_snd_tag_alloc = mlx5e_snd_tag_alloc;
4332 	ifp->if_snd_tag_free = mlx5e_snd_tag_free;
4333 	ifp->if_snd_tag_modify = mlx5e_snd_tag_modify;
4334 	ifp->if_snd_tag_query = mlx5e_snd_tag_query;
4335 #ifdef RATELIMIT
4336 	ifp->if_ratelimit_query = mlx5e_ratelimit_query;
4337 #endif
4338 	/* set TSO limits so that we don't have to drop TX packets */
4339 	ifp->if_hw_tsomax = MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
4340 	ifp->if_hw_tsomaxsegcount = MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */;
4341 	ifp->if_hw_tsomaxsegsize = MLX5E_MAX_TX_MBUF_SIZE;
4342 
4343 	ifp->if_capenable = ifp->if_capabilities;
4344 	ifp->if_hwassist = 0;
4345 	if (ifp->if_capenable & IFCAP_TSO)
4346 		ifp->if_hwassist |= CSUM_TSO;
4347 	if (ifp->if_capenable & IFCAP_TXCSUM)
4348 		ifp->if_hwassist |= (CSUM_TCP | CSUM_UDP | CSUM_IP);
4349 	if (ifp->if_capenable & IFCAP_TXCSUM_IPV6)
4350 		ifp->if_hwassist |= (CSUM_UDP_IPV6 | CSUM_TCP_IPV6);
4351 
4352 	/* ifnet sysctl tree */
4353 	sysctl_ctx_init(&priv->sysctl_ctx);
4354 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
4355 	    OID_AUTO, ifp->if_dname, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4356 	    "MLX5 ethernet - interface name");
4357 	if (priv->sysctl_ifnet == NULL) {
4358 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4359 		goto err_free_sysctl;
4360 	}
4361 	snprintf(unit, sizeof(unit), "%d", ifp->if_dunit);
4362 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4363 	    OID_AUTO, unit, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4364 	    "MLX5 ethernet - interface unit");
4365 	if (priv->sysctl_ifnet == NULL) {
4366 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4367 		goto err_free_sysctl;
4368 	}
4369 
4370 	/* HW sysctl tree */
4371 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
4372 	priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
4373 	    OID_AUTO, "hw", CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4374 	    "MLX5 ethernet dev hw");
4375 	if (priv->sysctl_hw == NULL) {
4376 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4377 		goto err_free_sysctl;
4378 	}
4379 
4380 	err = mlx5e_build_ifp_priv(mdev, priv, ncv);
4381 	if (err) {
4382 		mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err);
4383 		goto err_free_sysctl;
4384 	}
4385 
4386 	/* reuse mlx5core's watchdog workqueue */
4387 	priv->wq = mdev->priv.health.wq_watchdog;
4388 
4389 	err = mlx5_alloc_map_uar(mdev, &priv->cq_uar);
4390 	if (err) {
4391 		mlx5_en_err(ifp, "mlx5_alloc_map_uar failed, %d\n", err);
4392 		goto err_free_wq;
4393 	}
4394 	err = mlx5_core_alloc_pd(mdev, &priv->pdn);
4395 	if (err) {
4396 		mlx5_en_err(ifp, "mlx5_core_alloc_pd failed, %d\n", err);
4397 		goto err_unmap_free_uar;
4398 	}
4399 	err = mlx5_alloc_transport_domain(mdev, &priv->tdn);
4400 	if (err) {
4401 		mlx5_en_err(ifp,
4402 		    "mlx5_alloc_transport_domain failed, %d\n", err);
4403 		goto err_dealloc_pd;
4404 	}
4405 	err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
4406 	if (err) {
4407 		mlx5_en_err(ifp, "mlx5e_create_mkey failed, %d\n", err);
4408 		goto err_dealloc_transport_domain;
4409 	}
4410 	mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
4411 
4412 	/* check if we should generate a random MAC address */
4413 	if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
4414 	    is_zero_ether_addr(dev_addr)) {
4415 		random_ether_addr(dev_addr);
4416 		mlx5_en_err(ifp, "Assigned random MAC address\n");
4417 	}
4418 
4419 	err = mlx5e_rl_init(priv);
4420 	if (err) {
4421 		mlx5_en_err(ifp, "mlx5e_rl_init failed, %d\n", err);
4422 		goto err_create_mkey;
4423 	}
4424 
4425 	err = mlx5e_tls_init(priv);
4426 	if (err) {
4427 		if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
4428 		goto err_rl_init;
4429 	}
4430 
4431 	/* set default MTU */
4432 	mlx5e_set_dev_port_mtu(ifp, ifp->if_mtu);
4433 
4434 	/* Set default media status */
4435 	priv->media_status_last = IFM_AVALID;
4436 	priv->media_active_last = IFM_ETHER | IFM_AUTO |
4437 	    IFM_ETH_RXPAUSE | IFM_FDX;
4438 
4439 	/* setup default pauseframes configuration */
4440 	mlx5e_setup_pauseframes(priv);
4441 
4442 	/* Setup supported medias */
4443 	//TODO: If we failed to query ptys is it ok to proceed??
4444 	if (!mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1)) {
4445 		ext = MLX5_CAP_PCAM_FEATURE(mdev,
4446 		    ptys_extended_ethernet);
4447 		eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
4448 		    eth_proto_capability);
4449 		if (MLX5_CAP_PCAM_FEATURE(mdev, ptys_connector_type))
4450 			connector_type = MLX5_GET(ptys_reg, out,
4451 			    connector_type);
4452 	} else {
4453 		eth_proto_cap = 0;
4454 		mlx5_en_err(ifp, "Query port media capability failed, %d\n", err);
4455 	}
4456 
4457 	ifmedia_init(&priv->media, IFM_IMASK | IFM_ETH_FMASK,
4458 	    mlx5e_media_change, mlx5e_media_status);
4459 
4460 	speeds_num = ext ? MLX5E_EXT_LINK_SPEEDS_NUMBER : MLX5E_LINK_SPEEDS_NUMBER;
4461 	for (i = 0; i != speeds_num; i++) {
4462 		for (j = 0; j < MLX5E_LINK_MODES_NUMBER ; ++j) {
4463 			media_entry = ext ? mlx5e_ext_mode_table[i][j] :
4464 			    mlx5e_mode_table[i][j];
4465 			if (media_entry.baudrate == 0)
4466 				continue;
4467 			if (MLX5E_PROT_MASK(i) & eth_proto_cap) {
4468 				ifmedia_add(&priv->media,
4469 				    media_entry.subtype |
4470 				    IFM_ETHER, 0, NULL);
4471 				ifmedia_add(&priv->media,
4472 				    media_entry.subtype |
4473 				    IFM_ETHER | IFM_FDX |
4474 				    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4475 			}
4476 		}
4477 	}
4478 
4479 	ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO, 0, NULL);
4480 	ifmedia_add(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4481 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4482 
4483 	/* Set autoselect by default */
4484 	ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4485 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
4486 
4487 	DEBUGNET_SET(ifp, mlx5_en);
4488 
4489 	ether_ifattach(ifp, dev_addr);
4490 
4491 	/* Register for VLAN events */
4492 	priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
4493 	    mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
4494 	priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
4495 	    mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
4496 
4497 	/* Link is down by default */
4498 	if_link_state_change(ifp, LINK_STATE_DOWN);
4499 
4500 	mlx5e_enable_async_events(priv);
4501 
4502 	mlx5e_add_hw_stats(priv);
4503 
4504 	mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4505 	    "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
4506 	    priv->stats.vport.arg);
4507 
4508 	mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4509 	    "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
4510 	    priv->stats.pport.arg);
4511 
4512 	mlx5e_create_ethtool(priv);
4513 
4514 	mtx_lock(&priv->async_events_mtx);
4515 	mlx5e_update_stats(priv);
4516 	mtx_unlock(&priv->async_events_mtx);
4517 
4518 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4519 	    OID_AUTO, "rx_clbr_done", CTLFLAG_RD,
4520 	    &priv->clbr_done, 0,
4521 	    "RX timestamps calibration state");
4522 	callout_init(&priv->tstmp_clbr, CALLOUT_DIRECT);
4523 	mlx5e_reset_calibration_callout(priv);
4524 
4525 	pa.pa_version = PFIL_VERSION;
4526 	pa.pa_flags = PFIL_IN;
4527 	pa.pa_type = PFIL_TYPE_ETHERNET;
4528 	pa.pa_headname = ifp->if_xname;
4529 	priv->pfil = pfil_head_register(&pa);
4530 
4531 	return (priv);
4532 
4533 err_rl_init:
4534 	mlx5e_rl_cleanup(priv);
4535 
4536 err_create_mkey:
4537 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4538 
4539 err_dealloc_transport_domain:
4540 	mlx5_dealloc_transport_domain(mdev, priv->tdn);
4541 
4542 err_dealloc_pd:
4543 	mlx5_core_dealloc_pd(mdev, priv->pdn);
4544 
4545 err_unmap_free_uar:
4546 	mlx5_unmap_free_uar(mdev, &priv->cq_uar);
4547 
4548 err_free_wq:
4549 	flush_workqueue(priv->wq);
4550 
4551 err_free_sysctl:
4552 	sysctl_ctx_free(&priv->sysctl_ctx);
4553 	if (priv->sysctl_debug)
4554 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4555 	mlx5e_priv_static_destroy(priv, mdev->priv.eq_table.num_comp_vectors);
4556 	if_free(ifp);
4557 
4558 err_free_priv:
4559 	free(priv, M_MLX5EN);
4560 	return (NULL);
4561 }
4562 
4563 static void
4564 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
4565 {
4566 	struct mlx5e_priv *priv = vpriv;
4567 	struct ifnet *ifp = priv->ifp;
4568 
4569 	/* don't allow more IOCTLs */
4570 	priv->gone = 1;
4571 
4572 	/* XXX wait a bit to allow IOCTL handlers to complete */
4573 	pause("W", hz);
4574 
4575 #ifdef RATELIMIT
4576 	/*
4577 	 * The kernel can have reference(s) via the m_snd_tag's into
4578 	 * the ratelimit channels, and these must go away before
4579 	 * detaching:
4580 	 */
4581 	while (READ_ONCE(priv->rl.stats.tx_active_connections) != 0) {
4582 		mlx5_en_err(priv->ifp,
4583 		    "Waiting for all ratelimit connections to terminate\n");
4584 		pause("W", hz);
4585 	}
4586 #endif
4587 	/* wait for all unlimited send tags to complete */
4588 	mlx5e_priv_wait_for_completion(priv, mdev->priv.eq_table.num_comp_vectors);
4589 
4590 	/* stop watchdog timer */
4591 	callout_drain(&priv->watchdog);
4592 
4593 	callout_drain(&priv->tstmp_clbr);
4594 
4595 	if (priv->vlan_attach != NULL)
4596 		EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
4597 	if (priv->vlan_detach != NULL)
4598 		EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
4599 
4600 	/* make sure device gets closed */
4601 	PRIV_LOCK(priv);
4602 	mlx5e_close_locked(ifp);
4603 	PRIV_UNLOCK(priv);
4604 
4605 	/* deregister pfil */
4606 	if (priv->pfil != NULL) {
4607 		pfil_head_unregister(priv->pfil);
4608 		priv->pfil = NULL;
4609 	}
4610 
4611 	/* unregister device */
4612 	ifmedia_removeall(&priv->media);
4613 	ether_ifdetach(ifp);
4614 
4615 	mlx5e_tls_cleanup(priv);
4616 	mlx5e_rl_cleanup(priv);
4617 
4618 	/* destroy all remaining sysctl nodes */
4619 	sysctl_ctx_free(&priv->stats.vport.ctx);
4620 	sysctl_ctx_free(&priv->stats.pport.ctx);
4621 	if (priv->sysctl_debug)
4622 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4623 	sysctl_ctx_free(&priv->sysctl_ctx);
4624 
4625 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4626 	mlx5_dealloc_transport_domain(priv->mdev, priv->tdn);
4627 	mlx5_core_dealloc_pd(priv->mdev, priv->pdn);
4628 	mlx5_unmap_free_uar(priv->mdev, &priv->cq_uar);
4629 	mlx5e_disable_async_events(priv);
4630 	flush_workqueue(priv->wq);
4631 	mlx5e_priv_static_destroy(priv, mdev->priv.eq_table.num_comp_vectors);
4632 	if_free(ifp);
4633 	free(priv, M_MLX5EN);
4634 }
4635 
4636 #ifdef DEBUGNET
4637 static void
4638 mlx5_en_debugnet_init(struct ifnet *dev, int *nrxr, int *ncl, int *clsize)
4639 {
4640 	struct mlx5e_priv *priv = if_getsoftc(dev);
4641 
4642 	PRIV_LOCK(priv);
4643 	*nrxr = priv->params.num_channels;
4644 	*ncl = DEBUGNET_MAX_IN_FLIGHT;
4645 	*clsize = MLX5E_MAX_RX_BYTES;
4646 	PRIV_UNLOCK(priv);
4647 }
4648 
4649 static void
4650 mlx5_en_debugnet_event(struct ifnet *dev, enum debugnet_ev event)
4651 {
4652 }
4653 
4654 static int
4655 mlx5_en_debugnet_transmit(struct ifnet *dev, struct mbuf *m)
4656 {
4657 	struct mlx5e_priv *priv = if_getsoftc(dev);
4658 	struct mlx5e_sq *sq;
4659 	int err;
4660 
4661 	if ((if_getdrvflags(dev) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
4662 	    IFF_DRV_RUNNING || (priv->media_status_last & IFM_ACTIVE) == 0)
4663 		return (ENOENT);
4664 
4665 	sq = &priv->channel[0].sq[0];
4666 
4667 	if (sq->running == 0) {
4668 		m_freem(m);
4669 		return (ENOENT);
4670 	}
4671 
4672 	if (mlx5e_sq_xmit(sq, &m) != 0) {
4673 		m_freem(m);
4674 		err = ENOBUFS;
4675 	} else {
4676 		err = 0;
4677 	}
4678 
4679 	if (likely(sq->doorbell.d64 != 0)) {
4680 		mlx5e_tx_notify_hw(sq, sq->doorbell.d32, 0);
4681 		sq->doorbell.d64 = 0;
4682 	}
4683 	return (err);
4684 }
4685 
4686 static int
4687 mlx5_en_debugnet_poll(struct ifnet *dev, int count)
4688 {
4689 	struct mlx5e_priv *priv = if_getsoftc(dev);
4690 
4691 	if ((if_getdrvflags(dev) & IFF_DRV_RUNNING) == 0 ||
4692 	    (priv->media_status_last & IFM_ACTIVE) == 0)
4693 		return (ENOENT);
4694 
4695 	mlx5_poll_interrupts(priv->mdev);
4696 
4697 	return (0);
4698 }
4699 #endif /* DEBUGNET */
4700 
4701 static void *
4702 mlx5e_get_ifp(void *vpriv)
4703 {
4704 	struct mlx5e_priv *priv = vpriv;
4705 
4706 	return (priv->ifp);
4707 }
4708 
4709 static struct mlx5_interface mlx5e_interface = {
4710 	.add = mlx5e_create_ifp,
4711 	.remove = mlx5e_destroy_ifp,
4712 	.event = mlx5e_async_event,
4713 	.protocol = MLX5_INTERFACE_PROTOCOL_ETH,
4714 	.get_dev = mlx5e_get_ifp,
4715 };
4716 
4717 void
4718 mlx5e_init(void)
4719 {
4720 	mlx5_register_interface(&mlx5e_interface);
4721 }
4722 
4723 void
4724 mlx5e_cleanup(void)
4725 {
4726 	mlx5_unregister_interface(&mlx5e_interface);
4727 }
4728 
4729 static void
4730 mlx5e_show_version(void __unused *arg)
4731 {
4732 
4733 	printf("%s", mlx5e_version);
4734 }
4735 SYSINIT(mlx5e_show_version, SI_SUB_DRIVERS, SI_ORDER_ANY, mlx5e_show_version, NULL);
4736 
4737 module_init_order(mlx5e_init, SI_ORDER_SIXTH);
4738 module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);
4739 
4740 #if (__FreeBSD_version >= 1100000)
4741 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
4742 #endif
4743 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
4744 MODULE_VERSION(mlx5en, 1);
4745