xref: /freebsd/sys/dev/mlx5/mlx5_en/mlx5_en_main.c (revision 016f40466afc029df28d699650237b182aa88211)
1 /*-
2  * Copyright (c) 2015-2021 Mellanox Technologies. All rights reserved.
3  * Copyright (c) 2022 NVIDIA corporation & affiliates.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  */
26 
27 #include "opt_ipsec.h"
28 #include "opt_kern_tls.h"
29 #include "opt_rss.h"
30 #include "opt_ratelimit.h"
31 
32 #include <dev/mlx5/mlx5_en/en.h>
33 #include <dev/mlx5/mlx5_accel/ipsec.h>
34 
35 #include <sys/eventhandler.h>
36 #include <sys/sockio.h>
37 #include <machine/atomic.h>
38 
39 #include <net/debugnet.h>
40 #include <netinet/tcp_ratelimit.h>
41 #include <netipsec/keydb.h>
42 #include <netipsec/ipsec_offload.h>
43 
44 static int mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs);
45 static if_snd_tag_query_t mlx5e_ul_snd_tag_query;
46 static if_snd_tag_free_t mlx5e_ul_snd_tag_free;
47 
48 struct mlx5e_channel_param {
49 	struct mlx5e_rq_param rq;
50 	struct mlx5e_sq_param sq;
51 	struct mlx5e_cq_param rx_cq;
52 	struct mlx5e_cq_param tx_cq;
53 };
54 
55 struct media {
56 	u32	subtype;
57 	u64	baudrate;
58 };
59 
60 static const struct media mlx5e_mode_table[MLX5E_LINK_SPEEDS_NUMBER] =
61 {
62 	[MLX5E_1000BASE_CX_SGMII] = {
63 		.subtype = IFM_1000_CX_SGMII,
64 		.baudrate = IF_Mbps(1000ULL),
65 	},
66 	[MLX5E_1000BASE_KX] = {
67 		.subtype = IFM_1000_KX,
68 		.baudrate = IF_Mbps(1000ULL),
69 	},
70 	[MLX5E_10GBASE_CX4] = {
71 		.subtype = IFM_10G_CX4,
72 		.baudrate = IF_Gbps(10ULL),
73 	},
74 	[MLX5E_10GBASE_KX4] = {
75 		.subtype = IFM_10G_KX4,
76 		.baudrate = IF_Gbps(10ULL),
77 	},
78 	[MLX5E_10GBASE_KR] = {
79 		.subtype = IFM_10G_KR,
80 		.baudrate = IF_Gbps(10ULL),
81 	},
82 	[MLX5E_20GBASE_KR2] = {
83 		.subtype = IFM_20G_KR2,
84 		.baudrate = IF_Gbps(20ULL),
85 	},
86 	[MLX5E_40GBASE_CR4] = {
87 		.subtype = IFM_40G_CR4,
88 		.baudrate = IF_Gbps(40ULL),
89 	},
90 	[MLX5E_40GBASE_KR4] = {
91 		.subtype = IFM_40G_KR4,
92 		.baudrate = IF_Gbps(40ULL),
93 	},
94 	[MLX5E_56GBASE_R4] = {
95 		.subtype = IFM_56G_R4,
96 		.baudrate = IF_Gbps(56ULL),
97 	},
98 	[MLX5E_10GBASE_CR] = {
99 		.subtype = IFM_10G_CR1,
100 		.baudrate = IF_Gbps(10ULL),
101 	},
102 	[MLX5E_10GBASE_SR] = {
103 		.subtype = IFM_10G_SR,
104 		.baudrate = IF_Gbps(10ULL),
105 	},
106 	[MLX5E_10GBASE_ER_LR] = {
107 		.subtype = IFM_10G_ER,
108 		.baudrate = IF_Gbps(10ULL),
109 	},
110 	[MLX5E_40GBASE_SR4] = {
111 		.subtype = IFM_40G_SR4,
112 		.baudrate = IF_Gbps(40ULL),
113 	},
114 	[MLX5E_40GBASE_LR4_ER4] = {
115 		.subtype = IFM_40G_LR4,
116 		.baudrate = IF_Gbps(40ULL),
117 	},
118 	[MLX5E_100GBASE_CR4] = {
119 		.subtype = IFM_100G_CR4,
120 		.baudrate = IF_Gbps(100ULL),
121 	},
122 	[MLX5E_100GBASE_SR4] = {
123 		.subtype = IFM_100G_SR4,
124 		.baudrate = IF_Gbps(100ULL),
125 	},
126 	[MLX5E_100GBASE_KR4] = {
127 		.subtype = IFM_100G_KR4,
128 		.baudrate = IF_Gbps(100ULL),
129 	},
130 	[MLX5E_100GBASE_LR4] = {
131 		.subtype = IFM_100G_LR4,
132 		.baudrate = IF_Gbps(100ULL),
133 	},
134 	[MLX5E_100BASE_TX] = {
135 		.subtype = IFM_100_TX,
136 		.baudrate = IF_Mbps(100ULL),
137 	},
138 	[MLX5E_1000BASE_T] = {
139 		.subtype = IFM_1000_T,
140 		.baudrate = IF_Mbps(1000ULL),
141 	},
142 	[MLX5E_10GBASE_T] = {
143 		.subtype = IFM_10G_T,
144 		.baudrate = IF_Gbps(10ULL),
145 	},
146 	[MLX5E_25GBASE_CR] = {
147 		.subtype = IFM_25G_CR,
148 		.baudrate = IF_Gbps(25ULL),
149 	},
150 	[MLX5E_25GBASE_KR] = {
151 		.subtype = IFM_25G_KR,
152 		.baudrate = IF_Gbps(25ULL),
153 	},
154 	[MLX5E_25GBASE_SR] = {
155 		.subtype = IFM_25G_SR,
156 		.baudrate = IF_Gbps(25ULL),
157 	},
158 	[MLX5E_50GBASE_CR2] = {
159 		.subtype = IFM_50G_CR2,
160 		.baudrate = IF_Gbps(50ULL),
161 	},
162 	[MLX5E_50GBASE_KR2] = {
163 		.subtype = IFM_50G_KR2,
164 		.baudrate = IF_Gbps(50ULL),
165 	},
166 	[MLX5E_50GBASE_KR4] = {
167 		.subtype = IFM_50G_KR4,
168 		.baudrate = IF_Gbps(50ULL),
169 	},
170 };
171 
172 static const struct media mlx5e_ext_mode_table[MLX5E_EXT_LINK_SPEEDS_NUMBER][MLX5E_CABLE_TYPE_NUMBER] =
173 {
174 	/**/
175 	[MLX5E_SGMII_100M][MLX5E_CABLE_TYPE_UNKNOWN] = {
176 		.subtype = IFM_100_SGMII,
177 		.baudrate = IF_Mbps(100),
178 	},
179 
180 	/**/
181 	[MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_UNKNOWN] = {
182 		.subtype = IFM_1000_CX,
183 		.baudrate = IF_Mbps(1000),
184 	},
185 	[MLX5E_1000BASE_X_SGMII][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
186 		.subtype = IFM_1000_SX,
187 		.baudrate = IF_Mbps(1000),
188 	},
189 
190 	/**/
191 	[MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_UNKNOWN] = {
192 		.subtype = IFM_5000_KR,
193 		.baudrate = IF_Mbps(5000),
194 	},
195 	[MLX5E_5GBASE_R][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
196 		.subtype = IFM_5000_T,
197 		.baudrate = IF_Mbps(5000),
198 	},
199 
200 	/**/
201 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_UNKNOWN] = {
202 		.subtype = IFM_10G_KR,
203 		.baudrate = IF_Gbps(10ULL),
204 	},
205 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
206 		.subtype = IFM_10G_CR1,
207 		.baudrate = IF_Gbps(10ULL),
208 	},
209 	[MLX5E_10GBASE_XFI_XAUI_1][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
210 		.subtype = IFM_10G_SR,
211 		.baudrate = IF_Gbps(10ULL),
212 	},
213 
214 	/**/
215 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_UNKNOWN] = {
216 		.subtype = IFM_40G_KR4,
217 		.baudrate = IF_Gbps(40ULL),
218 	},
219 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
220 		.subtype = IFM_40G_CR4,
221 		.baudrate = IF_Gbps(40ULL),
222 	},
223 	[MLX5E_40GBASE_XLAUI_4_XLPPI_4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
224 		.subtype = IFM_40G_SR4,
225 		.baudrate = IF_Gbps(40ULL),
226 	},
227 
228 	/**/
229 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
230 		.subtype = IFM_25G_KR,
231 		.baudrate = IF_Gbps(25ULL),
232 	},
233 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
234 		.subtype = IFM_25G_CR,
235 		.baudrate = IF_Gbps(25ULL),
236 	},
237 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
238 		.subtype = IFM_25G_SR,
239 		.baudrate = IF_Gbps(25ULL),
240 	},
241 	[MLX5E_25GAUI_1_25GBASE_CR_KR][MLX5E_CABLE_TYPE_TWISTED_PAIR] = {
242 		.subtype = IFM_25G_T,
243 		.baudrate = IF_Gbps(25ULL),
244 	},
245 
246 	/**/
247 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
248 		.subtype = IFM_50G_KR2,
249 		.baudrate = IF_Gbps(50ULL),
250 	},
251 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
252 		.subtype = IFM_50G_CR2,
253 		.baudrate = IF_Gbps(50ULL),
254 	},
255 	[MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
256 		.subtype = IFM_50G_SR2,
257 		.baudrate = IF_Gbps(50ULL),
258 	},
259 
260 	/**/
261 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
262 		.subtype = IFM_50G_KR_PAM4,
263 		.baudrate = IF_Gbps(50ULL),
264 	},
265 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
266 		.subtype = IFM_50G_CP,
267 		.baudrate = IF_Gbps(50ULL),
268 	},
269 	[MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
270 		.subtype = IFM_50G_SR,
271 		.baudrate = IF_Gbps(50ULL),
272 	},
273 
274 	/**/
275 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
276 		.subtype = IFM_100G_KR4,
277 		.baudrate = IF_Gbps(100ULL),
278 	},
279 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
280 		.subtype = IFM_100G_CR4,
281 		.baudrate = IF_Gbps(100ULL),
282 	},
283 	[MLX5E_CAUI_4_100GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
284 		.subtype = IFM_100G_SR4,
285 		.baudrate = IF_Gbps(100ULL),
286 	},
287 
288 	/**/
289 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_UNKNOWN] = {
290 		.subtype = IFM_100G_KR_PAM4,
291 		.baudrate = IF_Gbps(100ULL),
292 	},
293 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
294 		.subtype = IFM_100G_CR_PAM4,
295 		.baudrate = IF_Gbps(100ULL),
296 	},
297 	[MLX5E_100GAUI_1_100GBASE_CR_KR][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
298 		.subtype = IFM_100G_SR2,	/* XXX */
299 		.baudrate = IF_Gbps(100ULL),
300 	},
301 
302 	/**/
303 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
304 		.subtype = IFM_100G_KR4,
305 		.baudrate = IF_Gbps(100ULL),
306 	},
307 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
308 		.subtype = IFM_100G_CP2,
309 		.baudrate = IF_Gbps(100ULL),
310 	},
311 	[MLX5E_100GAUI_2_100GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
312 		.subtype = IFM_100G_SR2,
313 		.baudrate = IF_Gbps(100ULL),
314 	},
315 
316 	/**/
317 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_UNKNOWN] = {
318 		.subtype = IFM_200G_KR4_PAM4,	/* XXX */
319 		.baudrate = IF_Gbps(200ULL),
320 	},
321 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
322 		.subtype = IFM_200G_CR4_PAM4,	/* XXX */
323 		.baudrate = IF_Gbps(200ULL),
324 	},
325 	[MLX5E_200GAUI_2_200GBASE_CR2_KR2][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
326 		.subtype = IFM_200G_SR4,	/* XXX */
327 		.baudrate = IF_Gbps(200ULL),
328 	},
329 
330 	/**/
331 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
332 		.subtype = IFM_200G_KR4_PAM4,
333 		.baudrate = IF_Gbps(200ULL),
334 	},
335 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_PASSIVE_COPPER] = {
336 		.subtype = IFM_200G_CR4_PAM4,
337 		.baudrate = IF_Gbps(200ULL),
338 	},
339 	[MLX5E_200GAUI_4_200GBASE_CR4_KR4][MLX5E_CABLE_TYPE_OPTICAL_MODULE] = {
340 		.subtype = IFM_200G_SR4,
341 		.baudrate = IF_Gbps(200ULL),
342 	},
343 
344 	/**/
345 	[MLX5E_400GAUI_8][MLX5E_CABLE_TYPE_UNKNOWN] = {
346 		.subtype = IFM_400G_LR8,	/* XXX */
347 		.baudrate = IF_Gbps(400ULL),
348 	},
349 
350 	/**/
351 	[MLX5E_400GAUI_4_400GBASE_CR4_KR4][MLX5E_CABLE_TYPE_UNKNOWN] = {
352 		.subtype = IFM_400G_LR8,	/* XXX */
353 		.baudrate = IF_Gbps(400ULL),
354 	},
355 };
356 
357 static const struct if_snd_tag_sw mlx5e_ul_snd_tag_sw = {
358 	.snd_tag_query = mlx5e_ul_snd_tag_query,
359 	.snd_tag_free = mlx5e_ul_snd_tag_free,
360 	.type = IF_SND_TAG_TYPE_UNLIMITED
361 };
362 
363 DEBUGNET_DEFINE(mlx5_en);
364 
365 MALLOC_DEFINE(M_MLX5EN, "MLX5EN", "MLX5 Ethernet");
366 
367 static void
mlx5e_update_carrier(struct mlx5e_priv * priv)368 mlx5e_update_carrier(struct mlx5e_priv *priv)
369 {
370 	struct mlx5_core_dev *mdev = priv->mdev;
371 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
372 	u32 eth_proto_oper;
373 	int error;
374 	u8 i;
375 	u8 cable_type;
376 	u8 port_state;
377 	u8 is_er_type;
378 	bool ext;
379 	struct media media_entry = {};
380 
381 	port_state = mlx5_query_vport_state(mdev,
382 	    MLX5_QUERY_VPORT_STATE_IN_OP_MOD_VNIC_VPORT, 0);
383 
384 	if (port_state == VPORT_STATE_UP) {
385 		priv->media_status_last |= IFM_ACTIVE;
386 	} else {
387 		priv->media_status_last &= ~IFM_ACTIVE;
388 		priv->media_active_last = IFM_ETHER;
389 		if_link_state_change(priv->ifp, LINK_STATE_DOWN);
390 		return;
391 	}
392 
393 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
394 	    MLX5_PTYS_EN, 1);
395 	if (error) {
396 		priv->media_active_last = IFM_ETHER;
397 		if_setbaudrate(priv->ifp, 1);
398 		mlx5_en_err(priv->ifp, "query port ptys failed: 0x%x\n",
399 		    error);
400 		return;
401 	}
402 
403 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
404 	eth_proto_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
405 	    eth_proto_oper);
406 
407 	i = ilog2(eth_proto_oper);
408 
409 	if (ext) {
410 		error = mlx5_query_pddr_cable_type(mdev, 1, &cable_type);
411 		if (error != 0) {
412 			/* use fallback entry */
413 			media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
414 
415 			mlx5_en_err(priv->ifp,
416 			    "query port pddr failed: %d\n", error);
417 		} else {
418 			media_entry = mlx5e_ext_mode_table[i][cable_type];
419 
420 			/* check if we should use fallback entry */
421 			if (media_entry.subtype == 0)
422 				media_entry = mlx5e_ext_mode_table[i][MLX5E_CABLE_TYPE_UNKNOWN];
423 		}
424 	} else {
425 		media_entry = mlx5e_mode_table[i];
426 	}
427 
428 	if (media_entry.subtype == 0) {
429 		mlx5_en_err(priv->ifp,
430 		    "Could not find operational media subtype\n");
431 		return;
432 	}
433 
434 	switch (media_entry.subtype) {
435 	case IFM_10G_ER:
436 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
437 		if (error != 0) {
438 			mlx5_en_err(priv->ifp,
439 			    "query port pddr failed: %d\n", error);
440 		}
441 		if (error != 0 || is_er_type == 0)
442 			media_entry.subtype = IFM_10G_LR;
443 		break;
444 	case IFM_40G_LR4:
445 		error = mlx5_query_pddr_range_info(mdev, 1, &is_er_type);
446 		if (error != 0) {
447 			mlx5_en_err(priv->ifp,
448 			    "query port pddr failed: %d\n", error);
449 		}
450 		if (error == 0 && is_er_type != 0)
451 			media_entry.subtype = IFM_40G_ER4;
452 		break;
453 	}
454 	priv->media_active_last = media_entry.subtype | IFM_ETHER | IFM_FDX;
455 	if_setbaudrate(priv->ifp, media_entry.baudrate);
456 
457 	if_link_state_change(priv->ifp, LINK_STATE_UP);
458 }
459 
460 static void
mlx5e_media_status(if_t dev,struct ifmediareq * ifmr)461 mlx5e_media_status(if_t dev, struct ifmediareq *ifmr)
462 {
463 	struct mlx5e_priv *priv = if_getsoftc(dev);
464 
465 	ifmr->ifm_status = priv->media_status_last;
466 	ifmr->ifm_current = ifmr->ifm_active = priv->media_active_last |
467 	    (priv->params.rx_pauseframe_control ? IFM_ETH_RXPAUSE : 0) |
468 	    (priv->params.tx_pauseframe_control ? IFM_ETH_TXPAUSE : 0);
469 
470 }
471 
472 static u32
mlx5e_find_link_mode(u32 subtype,bool ext)473 mlx5e_find_link_mode(u32 subtype, bool ext)
474 {
475 	u32 link_mode = 0;
476 
477 	switch (subtype) {
478 	case 0:
479 		goto done;
480 	case IFM_10G_LR:
481 		subtype = IFM_10G_ER;
482 		break;
483 	case IFM_40G_ER4:
484 		subtype = IFM_40G_LR4;
485 		break;
486 	default:
487 		break;
488 	}
489 
490 	if (ext) {
491 		for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
492 			for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
493 				if (mlx5e_ext_mode_table[i][j].subtype == subtype)
494 					link_mode |= MLX5E_PROT_MASK(i);
495 			}
496 		}
497 	} else {
498 		for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
499 			if (mlx5e_mode_table[i].subtype == subtype)
500 				link_mode |= MLX5E_PROT_MASK(i);
501 		}
502 	}
503 done:
504 	return (link_mode);
505 }
506 
507 static int
mlx5e_set_port_pause_and_pfc(struct mlx5e_priv * priv)508 mlx5e_set_port_pause_and_pfc(struct mlx5e_priv *priv)
509 {
510 	return (mlx5_set_port_pause_and_pfc(priv->mdev, 1,
511 	    priv->params.rx_pauseframe_control,
512 	    priv->params.tx_pauseframe_control,
513 	    priv->params.rx_priority_flow_control,
514 	    priv->params.tx_priority_flow_control));
515 }
516 
517 static int
mlx5e_set_port_pfc(struct mlx5e_priv * priv)518 mlx5e_set_port_pfc(struct mlx5e_priv *priv)
519 {
520 	int error;
521 
522 	if (priv->gone != 0) {
523 		error = -ENXIO;
524 	} else if (priv->params.rx_pauseframe_control ||
525 	    priv->params.tx_pauseframe_control) {
526 		mlx5_en_err(priv->ifp,
527 		    "Global pauseframes must be disabled before enabling PFC.\n");
528 		error = -EINVAL;
529 	} else {
530 		error = mlx5e_set_port_pause_and_pfc(priv);
531 	}
532 	return (error);
533 }
534 
535 static int
mlx5e_media_change(if_t dev)536 mlx5e_media_change(if_t dev)
537 {
538 	struct mlx5e_priv *priv = if_getsoftc(dev);
539 	struct mlx5_core_dev *mdev = priv->mdev;
540 	u32 eth_proto_cap;
541 	u32 link_mode;
542 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
543 	int was_opened;
544 	int locked;
545 	int error;
546 	bool ext;
547 
548 	locked = PRIV_LOCKED(priv);
549 	if (!locked)
550 		PRIV_LOCK(priv);
551 
552 	if (IFM_TYPE(priv->media.ifm_media) != IFM_ETHER) {
553 		error = EINVAL;
554 		goto done;
555 	}
556 
557 	error = mlx5_query_port_ptys(mdev, out, sizeof(out),
558 	    MLX5_PTYS_EN, 1);
559 	if (error != 0) {
560 		mlx5_en_err(dev, "Query port media capability failed\n");
561 		goto done;
562 	}
563 
564 	ext = MLX5_CAP_PCAM_FEATURE(mdev, ptys_extended_ethernet);
565 	link_mode = mlx5e_find_link_mode(IFM_SUBTYPE(priv->media.ifm_media), ext);
566 
567 	/* query supported capabilities */
568 	eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
569 	    eth_proto_capability);
570 
571 	/* check for autoselect */
572 	if (IFM_SUBTYPE(priv->media.ifm_media) == IFM_AUTO) {
573 		link_mode = eth_proto_cap;
574 		if (link_mode == 0) {
575 			mlx5_en_err(dev, "Port media capability is zero\n");
576 			error = EINVAL;
577 			goto done;
578 		}
579 	} else {
580 		link_mode = link_mode & eth_proto_cap;
581 		if (link_mode == 0) {
582 			mlx5_en_err(dev, "Not supported link mode requested\n");
583 			error = EINVAL;
584 			goto done;
585 		}
586 	}
587 	if (priv->media.ifm_media & (IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE)) {
588 		/* check if PFC is enabled */
589 		if (priv->params.rx_priority_flow_control ||
590 		    priv->params.tx_priority_flow_control) {
591 			mlx5_en_err(dev, "PFC must be disabled before enabling global pauseframes.\n");
592 			error = EINVAL;
593 			goto done;
594 		}
595 	}
596 	/* update pauseframe control bits */
597 	priv->params.rx_pauseframe_control =
598 	    (priv->media.ifm_media & IFM_ETH_RXPAUSE) ? 1 : 0;
599 	priv->params.tx_pauseframe_control =
600 	    (priv->media.ifm_media & IFM_ETH_TXPAUSE) ? 1 : 0;
601 
602 	/* check if device is opened */
603 	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
604 
605 	/* reconfigure the hardware */
606 	mlx5_set_port_status(mdev, MLX5_PORT_DOWN);
607 	mlx5_set_port_proto(mdev, link_mode, MLX5_PTYS_EN, ext);
608 	error = -mlx5e_set_port_pause_and_pfc(priv);
609 	if (was_opened)
610 		mlx5_set_port_status(mdev, MLX5_PORT_UP);
611 
612 done:
613 	if (!locked)
614 		PRIV_UNLOCK(priv);
615 	return (error);
616 }
617 
618 static void
mlx5e_update_carrier_work(struct work_struct * work)619 mlx5e_update_carrier_work(struct work_struct *work)
620 {
621 	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
622 	    update_carrier_work);
623 
624 	PRIV_LOCK(priv);
625 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
626 		mlx5e_update_carrier(priv);
627 	PRIV_UNLOCK(priv);
628 }
629 
630 #define	MLX5E_PCIE_PERF_GET_64(a,b,c,d,e,f)    \
631 	s_debug->c = MLX5_GET64(mpcnt_reg, out, counter_set.f.c);
632 
633 #define	MLX5E_PCIE_PERF_GET_32(a,b,c,d,e,f)    \
634 	s_debug->c = MLX5_GET(mpcnt_reg, out, counter_set.f.c);
635 
636 static void
mlx5e_update_pcie_counters(struct mlx5e_priv * priv)637 mlx5e_update_pcie_counters(struct mlx5e_priv *priv)
638 {
639 	struct mlx5_core_dev *mdev = priv->mdev;
640 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
641 	const unsigned sz = MLX5_ST_SZ_BYTES(mpcnt_reg);
642 	void *out;
643 	void *in;
644 	int err;
645 
646 	/* allocate firmware request structures */
647 	in = mlx5_vzalloc(sz);
648 	out = mlx5_vzalloc(sz);
649 	if (in == NULL || out == NULL)
650 		goto free_out;
651 
652 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_PERFORMANCE_COUNTERS_GROUP);
653 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
654 	if (err != 0)
655 		goto free_out;
656 
657 	MLX5E_PCIE_PERFORMANCE_COUNTERS_64(MLX5E_PCIE_PERF_GET_64)
658 	MLX5E_PCIE_PERFORMANCE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
659 
660 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_TIMERS_AND_STATES_COUNTERS_GROUP);
661 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
662 	if (err != 0)
663 		goto free_out;
664 
665 	MLX5E_PCIE_TIMERS_AND_STATES_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
666 
667 	MLX5_SET(mpcnt_reg, in, grp, MLX5_PCIE_LANE_COUNTERS_GROUP);
668 	err = mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_MPCNT, 0, 0);
669 	if (err != 0)
670 		goto free_out;
671 
672 	MLX5E_PCIE_LANE_COUNTERS_32(MLX5E_PCIE_PERF_GET_32)
673 
674 free_out:
675 	/* free firmware request structures */
676 	kvfree(in);
677 	kvfree(out);
678 }
679 
680 /*
681  * This function reads the physical port counters from the firmware
682  * using a pre-defined layout defined by various MLX5E_PPORT_XXX()
683  * macros. The output is converted from big-endian 64-bit values into
684  * host endian ones and stored in the "priv->stats.pport" structure.
685  */
686 static void
mlx5e_update_pport_counters(struct mlx5e_priv * priv)687 mlx5e_update_pport_counters(struct mlx5e_priv *priv)
688 {
689 	struct mlx5_core_dev *mdev = priv->mdev;
690 	struct mlx5e_pport_stats *s = &priv->stats.pport;
691 	struct mlx5e_port_stats_debug *s_debug = &priv->stats.port_stats_debug;
692 	u32 *in;
693 	u32 *out;
694 	const u64 *ptr;
695 	unsigned sz = MLX5_ST_SZ_BYTES(ppcnt_reg);
696 	unsigned x;
697 	unsigned y;
698 	unsigned z;
699 
700 	/* allocate firmware request structures */
701 	in = mlx5_vzalloc(sz);
702 	out = mlx5_vzalloc(sz);
703 	if (in == NULL || out == NULL)
704 		goto free_out;
705 
706 	/*
707 	 * Get pointer to the 64-bit counter set which is located at a
708 	 * fixed offset in the output firmware request structure:
709 	 */
710 	ptr = (const uint64_t *)MLX5_ADDR_OF(ppcnt_reg, out, counter_set);
711 
712 	MLX5_SET(ppcnt_reg, in, local_port, 1);
713 
714 	/* read IEEE802_3 counter group using predefined counter layout */
715 	MLX5_SET(ppcnt_reg, in, grp, MLX5_IEEE_802_3_COUNTERS_GROUP);
716 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
717 	for (x = 0, y = MLX5E_PPORT_PER_PRIO_STATS_NUM;
718 	     x != MLX5E_PPORT_IEEE802_3_STATS_NUM; x++, y++)
719 		s->arg[y] = be64toh(ptr[x]);
720 
721 	/* read RFC2819 counter group using predefined counter layout */
722 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2819_COUNTERS_GROUP);
723 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
724 	for (x = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM; x++, y++)
725 		s->arg[y] = be64toh(ptr[x]);
726 
727 	for (y = 0; x != MLX5E_PPORT_RFC2819_STATS_NUM +
728 	    MLX5E_PPORT_RFC2819_STATS_DEBUG_NUM; x++, y++)
729 		s_debug->arg[y] = be64toh(ptr[x]);
730 
731 	/* read RFC2863 counter group using predefined counter layout */
732 	MLX5_SET(ppcnt_reg, in, grp, MLX5_RFC_2863_COUNTERS_GROUP);
733 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
734 	for (x = 0; x != MLX5E_PPORT_RFC2863_STATS_DEBUG_NUM; x++, y++)
735 		s_debug->arg[y] = be64toh(ptr[x]);
736 
737 	/* read physical layer stats counter group using predefined counter layout */
738 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_COUNTERS_GROUP);
739 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
740 	for (x = 0; x != MLX5E_PPORT_PHYSICAL_LAYER_STATS_DEBUG_NUM; x++, y++)
741 		s_debug->arg[y] = be64toh(ptr[x]);
742 
743 	/* read Extended Ethernet counter group using predefined counter layout */
744 	MLX5_SET(ppcnt_reg, in, grp, MLX5_ETHERNET_EXTENDED_COUNTERS_GROUP);
745 	mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
746 	for (x = 0; x != MLX5E_PPORT_ETHERNET_EXTENDED_STATS_DEBUG_NUM; x++, y++)
747 		s_debug->arg[y] = be64toh(ptr[x]);
748 
749 	/* read Extended Statistical Group */
750 	if (MLX5_CAP_GEN(mdev, pcam_reg) &&
751 	    MLX5_CAP_PCAM_FEATURE(mdev, ppcnt_statistical_group) &&
752 	    MLX5_CAP_PCAM_FEATURE(mdev, per_lane_error_counters)) {
753 		/* read Extended Statistical counter group using predefined counter layout */
754 		MLX5_SET(ppcnt_reg, in, grp, MLX5_PHYSICAL_LAYER_STATISTICAL_GROUP);
755 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
756 
757 		for (x = 0; x != MLX5E_PPORT_STATISTICAL_DEBUG_NUM; x++, y++)
758 			s_debug->arg[y] = be64toh(ptr[x]);
759 	}
760 
761 	/* read PCIE counters */
762 	mlx5e_update_pcie_counters(priv);
763 
764 	/* read per-priority counters */
765 	MLX5_SET(ppcnt_reg, in, grp, MLX5_PER_PRIORITY_COUNTERS_GROUP);
766 
767 	/* iterate all the priorities */
768 	for (y = z = 0; z != MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO; z++) {
769 		MLX5_SET(ppcnt_reg, in, prio_tc, z);
770 		mlx5_core_access_reg(mdev, in, sz, out, sz, MLX5_REG_PPCNT, 0, 0);
771 
772 		/* read per priority stats counter group using predefined counter layout */
773 		for (x = 0; x != (MLX5E_PPORT_PER_PRIO_STATS_NUM /
774 		    MLX5E_PPORT_PER_PRIO_STATS_NUM_PRIO); x++, y++)
775 			s->arg[y] = be64toh(ptr[x]);
776 	}
777 
778 free_out:
779 	/* free firmware request structures */
780 	kvfree(in);
781 	kvfree(out);
782 }
783 
784 static void
mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv * priv)785 mlx5e_grp_vnic_env_update_stats(struct mlx5e_priv *priv)
786 {
787 	u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {};
788 	u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {};
789 
790 	if (!MLX5_CAP_GEN(priv->mdev, nic_receive_steering_discard))
791 		return;
792 
793 	MLX5_SET(query_vnic_env_in, in, opcode,
794 	    MLX5_CMD_OP_QUERY_VNIC_ENV);
795 	MLX5_SET(query_vnic_env_in, in, op_mod, 0);
796 	MLX5_SET(query_vnic_env_in, in, other_vport, 0);
797 
798 	if (mlx5_cmd_exec(priv->mdev, in, sizeof(in), out, sizeof(out)) != 0)
799 		return;
800 
801 	priv->stats.vport.rx_steer_missed_packets =
802 	    MLX5_GET64(query_vnic_env_out, out,
803 	    vport_env.nic_receive_steering_discard);
804 }
805 
806 /*
807  * This function is called regularly to collect all statistics
808  * counters from the firmware. The values can be viewed through the
809  * sysctl interface. Execution is serialized using the priv's global
810  * configuration lock.
811  */
812 static void
mlx5e_update_stats_locked(struct mlx5e_priv * priv)813 mlx5e_update_stats_locked(struct mlx5e_priv *priv)
814 {
815 	struct mlx5_core_dev *mdev = priv->mdev;
816 	struct mlx5e_vport_stats *s = &priv->stats.vport;
817 	struct mlx5e_sq_stats *sq_stats;
818 	u32 in[MLX5_ST_SZ_DW(query_vport_counter_in)];
819 	u32 *out;
820 	int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out);
821 	u64 tso_packets = 0;
822 	u64 tso_bytes = 0;
823 	u64 tx_queue_dropped = 0;
824 	u64 tx_defragged = 0;
825 	u64 tx_offload_none = 0;
826 	u64 lro_packets = 0;
827 	u64 lro_bytes = 0;
828 	u64 sw_lro_queued = 0;
829 	u64 sw_lro_flushed = 0;
830 	u64 rx_csum_none = 0;
831 	u64 rx_wqe_err = 0;
832 	u64 rx_packets = 0;
833 	u64 rx_bytes = 0;
834 	u64 rx_decrypted_error = 0;
835 	u64 rx_decrypted_ok = 0;
836 	u32 rx_out_of_buffer = 0;
837 	int error;
838 	int i;
839 	int j;
840 
841 	out = mlx5_vzalloc(outlen);
842 	if (out == NULL)
843 		goto free_out;
844 
845 	/* Collect firts the SW counters and then HW for consistency */
846 	for (i = 0; i < priv->params.num_channels; i++) {
847 		struct mlx5e_channel *pch = priv->channel + i;
848 		struct mlx5e_rq *rq = &pch->rq;
849 		struct mlx5e_rq_stats *rq_stats = &pch->rq.stats;
850 
851 		/* collect stats from LRO */
852 		rq_stats->sw_lro_queued = rq->lro.lro_queued;
853 		rq_stats->sw_lro_flushed = rq->lro.lro_flushed;
854 		sw_lro_queued += rq_stats->sw_lro_queued;
855 		sw_lro_flushed += rq_stats->sw_lro_flushed;
856 		lro_packets += rq_stats->lro_packets;
857 		lro_bytes += rq_stats->lro_bytes;
858 		rx_csum_none += rq_stats->csum_none;
859 		rx_wqe_err += rq_stats->wqe_err;
860 		rx_packets += rq_stats->packets;
861 		rx_bytes += rq_stats->bytes;
862 		rx_decrypted_error += rq_stats->decrypted_error_packets;
863 		rx_decrypted_ok += rq_stats->decrypted_ok_packets;
864 
865 		for (j = 0; j < priv->num_tc; j++) {
866 			sq_stats = &pch->sq[j].stats;
867 
868 			tso_packets += sq_stats->tso_packets;
869 			tso_bytes += sq_stats->tso_bytes;
870 			tx_queue_dropped += sq_stats->dropped;
871 			tx_queue_dropped += sq_stats->enobuf;
872 			tx_defragged += sq_stats->defragged;
873 			tx_offload_none += sq_stats->csum_offload_none;
874 		}
875 	}
876 
877 #ifdef RATELIMIT
878 	/* Collect statistics from all rate-limit queues */
879 	for (j = 0; j < priv->rl.param.tx_worker_threads_def; j++) {
880 		struct mlx5e_rl_worker *rlw = priv->rl.workers + j;
881 
882 		for (i = 0; i < priv->rl.param.tx_channels_per_worker_def; i++) {
883 			struct mlx5e_rl_channel *channel = rlw->channels + i;
884 			struct mlx5e_sq *sq = channel->sq;
885 
886 			if (sq == NULL)
887 				continue;
888 
889 			sq_stats = &sq->stats;
890 
891 			tso_packets += sq_stats->tso_packets;
892 			tso_bytes += sq_stats->tso_bytes;
893 			tx_queue_dropped += sq_stats->dropped;
894 			tx_queue_dropped += sq_stats->enobuf;
895 			tx_defragged += sq_stats->defragged;
896 			tx_offload_none += sq_stats->csum_offload_none;
897 		}
898 	}
899 #endif
900 
901 	/* update counters */
902 	s->tso_packets = tso_packets;
903 	s->tso_bytes = tso_bytes;
904 	s->tx_queue_dropped = tx_queue_dropped;
905 	s->tx_defragged = tx_defragged;
906 	s->lro_packets = lro_packets;
907 	s->lro_bytes = lro_bytes;
908 	s->sw_lro_queued = sw_lro_queued;
909 	s->sw_lro_flushed = sw_lro_flushed;
910 	s->rx_csum_none = rx_csum_none;
911 	s->rx_wqe_err = rx_wqe_err;
912 	s->rx_packets = rx_packets;
913 	s->rx_bytes = rx_bytes;
914 	s->rx_decrypted_error_packets = rx_decrypted_error;
915 	s->rx_decrypted_ok_packets = rx_decrypted_ok;
916 
917 	mlx5e_grp_vnic_env_update_stats(priv);
918 
919 	/* HW counters */
920 	memset(in, 0, sizeof(in));
921 
922 	MLX5_SET(query_vport_counter_in, in, opcode,
923 	    MLX5_CMD_OP_QUERY_VPORT_COUNTER);
924 	MLX5_SET(query_vport_counter_in, in, op_mod, 0);
925 	MLX5_SET(query_vport_counter_in, in, other_vport, 0);
926 
927 	memset(out, 0, outlen);
928 
929 	/* get number of out-of-buffer drops first */
930 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
931 	    mlx5_vport_query_out_of_rx_buffer(mdev, priv->counter_set_id,
932 	    &rx_out_of_buffer) == 0) {
933 		s->rx_out_of_buffer = rx_out_of_buffer;
934 	}
935 
936 	/* get port statistics */
937 	if (mlx5_cmd_exec(mdev, in, sizeof(in), out, outlen) == 0) {
938 #define	MLX5_GET_CTR(out, x) \
939 	MLX5_GET64(query_vport_counter_out, out, x)
940 
941 		s->rx_error_packets =
942 		    MLX5_GET_CTR(out, received_errors.packets);
943 		s->rx_error_bytes =
944 		    MLX5_GET_CTR(out, received_errors.octets);
945 		s->tx_error_packets =
946 		    MLX5_GET_CTR(out, transmit_errors.packets);
947 		s->tx_error_bytes =
948 		    MLX5_GET_CTR(out, transmit_errors.octets);
949 
950 		s->rx_unicast_packets =
951 		    MLX5_GET_CTR(out, received_eth_unicast.packets);
952 		s->rx_unicast_bytes =
953 		    MLX5_GET_CTR(out, received_eth_unicast.octets);
954 		s->tx_unicast_packets =
955 		    MLX5_GET_CTR(out, transmitted_eth_unicast.packets);
956 		s->tx_unicast_bytes =
957 		    MLX5_GET_CTR(out, transmitted_eth_unicast.octets);
958 
959 		s->rx_multicast_packets =
960 		    MLX5_GET_CTR(out, received_eth_multicast.packets);
961 		s->rx_multicast_bytes =
962 		    MLX5_GET_CTR(out, received_eth_multicast.octets);
963 		s->tx_multicast_packets =
964 		    MLX5_GET_CTR(out, transmitted_eth_multicast.packets);
965 		s->tx_multicast_bytes =
966 		    MLX5_GET_CTR(out, transmitted_eth_multicast.octets);
967 
968 		s->rx_broadcast_packets =
969 		    MLX5_GET_CTR(out, received_eth_broadcast.packets);
970 		s->rx_broadcast_bytes =
971 		    MLX5_GET_CTR(out, received_eth_broadcast.octets);
972 		s->tx_broadcast_packets =
973 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.packets);
974 		s->tx_broadcast_bytes =
975 		    MLX5_GET_CTR(out, transmitted_eth_broadcast.octets);
976 
977 		s->tx_packets = s->tx_unicast_packets +
978 		    s->tx_multicast_packets + s->tx_broadcast_packets;
979 		s->tx_bytes = s->tx_unicast_bytes + s->tx_multicast_bytes +
980 		    s->tx_broadcast_bytes;
981 
982 		/* Update calculated offload counters */
983 		s->tx_csum_offload = s->tx_packets - tx_offload_none;
984 		s->rx_csum_good = s->rx_packets - s->rx_csum_none;
985 	}
986 
987 	/* Get physical port counters */
988 	mlx5e_update_pport_counters(priv);
989 
990 	s->tx_jumbo_packets =
991 	    priv->stats.port_stats_debug.tx_stat_p1519to2047octets +
992 	    priv->stats.port_stats_debug.tx_stat_p2048to4095octets +
993 	    priv->stats.port_stats_debug.tx_stat_p4096to8191octets +
994 	    priv->stats.port_stats_debug.tx_stat_p8192to10239octets;
995 
996 free_out:
997 	kvfree(out);
998 
999 	/* Update diagnostics, if any */
1000 	if (priv->params_ethtool.diag_pci_enable ||
1001 	    priv->params_ethtool.diag_general_enable) {
1002 		error = mlx5_core_get_diagnostics_full(mdev,
1003 		    priv->params_ethtool.diag_pci_enable ? &priv->params_pci : NULL,
1004 		    priv->params_ethtool.diag_general_enable ? &priv->params_general : NULL);
1005 		if (error != 0)
1006 			mlx5_en_err(priv->ifp,
1007 			    "Failed reading diagnostics: %d\n", error);
1008 	}
1009 
1010 	/* Update FEC, if any */
1011 	error = mlx5e_fec_update(priv);
1012 	if (error != 0 && error != EOPNOTSUPP) {
1013 		mlx5_en_err(priv->ifp,
1014 		    "Updating FEC failed: %d\n", error);
1015 	}
1016 
1017 	/* Update temperature, if any */
1018 	if (priv->params_ethtool.hw_num_temp != 0) {
1019 		error = mlx5e_hw_temperature_update(priv);
1020 		if (error != 0 && error != EOPNOTSUPP) {
1021 			mlx5_en_err(priv->ifp,
1022 			    "Updating temperature failed: %d\n", error);
1023 		}
1024 	}
1025 }
1026 
1027 static void
mlx5e_update_stats_work(struct work_struct * work)1028 mlx5e_update_stats_work(struct work_struct *work)
1029 {
1030 	struct mlx5e_priv *priv;
1031 
1032 	priv = container_of(work, struct mlx5e_priv, update_stats_work);
1033 	PRIV_LOCK(priv);
1034 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0 &&
1035 	    !test_bit(MLX5_INTERFACE_STATE_TEARDOWN, &priv->mdev->intf_state))
1036 		mlx5e_update_stats_locked(priv);
1037 	PRIV_UNLOCK(priv);
1038 }
1039 
1040 static void
mlx5e_update_stats(void * arg)1041 mlx5e_update_stats(void *arg)
1042 {
1043 	struct mlx5e_priv *priv = arg;
1044 
1045 	queue_work(priv->wq, &priv->update_stats_work);
1046 
1047 	callout_reset(&priv->watchdog, hz / 4, &mlx5e_update_stats, priv);
1048 }
1049 
1050 static void
mlx5e_async_event_sub(struct mlx5e_priv * priv,enum mlx5_dev_event event)1051 mlx5e_async_event_sub(struct mlx5e_priv *priv,
1052     enum mlx5_dev_event event)
1053 {
1054 	switch (event) {
1055 	case MLX5_DEV_EVENT_PORT_UP:
1056 	case MLX5_DEV_EVENT_PORT_DOWN:
1057 		queue_work(priv->wq, &priv->update_carrier_work);
1058 		break;
1059 
1060 	default:
1061 		break;
1062 	}
1063 }
1064 
1065 static void
mlx5e_async_event(struct mlx5_core_dev * mdev,void * vpriv,enum mlx5_dev_event event,unsigned long param)1066 mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
1067     enum mlx5_dev_event event, unsigned long param)
1068 {
1069 	struct mlx5e_priv *priv = vpriv;
1070 
1071 	mtx_lock(&priv->async_events_mtx);
1072 	if (test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state))
1073 		mlx5e_async_event_sub(priv, event);
1074 	mtx_unlock(&priv->async_events_mtx);
1075 }
1076 
1077 static void
mlx5e_enable_async_events(struct mlx5e_priv * priv)1078 mlx5e_enable_async_events(struct mlx5e_priv *priv)
1079 {
1080 	set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1081 }
1082 
1083 static void
mlx5e_disable_async_events(struct mlx5e_priv * priv)1084 mlx5e_disable_async_events(struct mlx5e_priv *priv)
1085 {
1086 	mtx_lock(&priv->async_events_mtx);
1087 	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLE, &priv->state);
1088 	mtx_unlock(&priv->async_events_mtx);
1089 }
1090 
1091 static void mlx5e_calibration_callout(void *arg);
1092 static int mlx5e_calibration_duration = 20;
1093 static int mlx5e_fast_calibration = 1;
1094 static int mlx5e_normal_calibration = 30;
1095 
1096 static SYSCTL_NODE(_hw_mlx5, OID_AUTO, calibr, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
1097     "MLX5 timestamp calibration parameters");
1098 
1099 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, duration, CTLFLAG_RWTUN,
1100     &mlx5e_calibration_duration, 0,
1101     "Duration of initial calibration");
1102 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, fast, CTLFLAG_RWTUN,
1103     &mlx5e_fast_calibration, 0,
1104     "Recalibration interval during initial calibration");
1105 SYSCTL_INT(_hw_mlx5_calibr, OID_AUTO, normal, CTLFLAG_RWTUN,
1106     &mlx5e_normal_calibration, 0,
1107     "Recalibration interval during normal operations");
1108 
1109 /*
1110  * Ignites the calibration process.
1111  */
1112 static void
mlx5e_reset_calibration_callout(struct mlx5e_priv * priv)1113 mlx5e_reset_calibration_callout(struct mlx5e_priv *priv)
1114 {
1115 
1116 	if (priv->clbr_done == 0)
1117 		mlx5e_calibration_callout(priv);
1118 	else
1119 		callout_reset_sbt_curcpu(&priv->tstmp_clbr, (priv->clbr_done <
1120 		    mlx5e_calibration_duration ? mlx5e_fast_calibration :
1121 		    mlx5e_normal_calibration) * SBT_1S, 0,
1122 		    mlx5e_calibration_callout, priv, C_DIRECT_EXEC);
1123 }
1124 
1125 static uint64_t
mlx5e_timespec2usec(const struct timespec * ts)1126 mlx5e_timespec2usec(const struct timespec *ts)
1127 {
1128 
1129 	return ((uint64_t)ts->tv_sec * 1000000000 + ts->tv_nsec);
1130 }
1131 
1132 static uint64_t
mlx5e_hw_clock(struct mlx5e_priv * priv)1133 mlx5e_hw_clock(struct mlx5e_priv *priv)
1134 {
1135 	struct mlx5_init_seg *iseg;
1136 	uint32_t hw_h, hw_h1, hw_l;
1137 
1138 	iseg = priv->mdev->iseg;
1139 	do {
1140 		hw_h = ioread32be(&iseg->internal_timer_h);
1141 		hw_l = ioread32be(&iseg->internal_timer_l);
1142 		hw_h1 = ioread32be(&iseg->internal_timer_h);
1143 	} while (hw_h1 != hw_h);
1144 	return (((uint64_t)hw_h << 32) | hw_l);
1145 }
1146 
1147 /*
1148  * The calibration callout, it runs either in the context of the
1149  * thread which enables calibration, or in callout.  It takes the
1150  * snapshot of system and adapter clocks, then advances the pointers to
1151  * the calibration point to allow rx path to read the consistent data
1152  * lockless.
1153  */
1154 static void
mlx5e_calibration_callout(void * arg)1155 mlx5e_calibration_callout(void *arg)
1156 {
1157 	struct mlx5e_priv *priv;
1158 	struct mlx5e_clbr_point *next, *curr;
1159 	struct timespec ts;
1160 	int clbr_curr_next;
1161 
1162 	priv = arg;
1163 	curr = &priv->clbr_points[priv->clbr_curr];
1164 	clbr_curr_next = priv->clbr_curr + 1;
1165 	if (clbr_curr_next >= nitems(priv->clbr_points))
1166 		clbr_curr_next = 0;
1167 	next = &priv->clbr_points[clbr_curr_next];
1168 
1169 	next->base_prev = curr->base_curr;
1170 	next->clbr_hw_prev = curr->clbr_hw_curr;
1171 
1172 	next->clbr_hw_curr = mlx5e_hw_clock(priv);
1173 	if (((next->clbr_hw_curr - curr->clbr_hw_curr) >> MLX5E_TSTMP_PREC) ==
1174 	    0) {
1175 		if (priv->clbr_done != 0) {
1176 			mlx5_en_err(priv->ifp,
1177 			    "HW failed tstmp frozen %#jx %#jx, disabling\n",
1178 			     next->clbr_hw_curr, curr->clbr_hw_prev);
1179 			priv->clbr_done = 0;
1180 		}
1181 		atomic_store_rel_int(&curr->clbr_gen, 0);
1182 		return;
1183 	}
1184 
1185 	nanouptime(&ts);
1186 	next->base_curr = mlx5e_timespec2usec(&ts);
1187 
1188 	curr->clbr_gen = 0;
1189 	atomic_thread_fence_rel();
1190 	priv->clbr_curr = clbr_curr_next;
1191 	atomic_store_rel_int(&next->clbr_gen, ++(priv->clbr_gen));
1192 
1193 	if (priv->clbr_done < mlx5e_calibration_duration)
1194 		priv->clbr_done++;
1195 	mlx5e_reset_calibration_callout(priv);
1196 }
1197 
1198 static const char *mlx5e_rq_stats_desc[] = {
1199 	MLX5E_RQ_STATS(MLX5E_STATS_DESC)
1200 };
1201 
1202 static int
mlx5e_create_rq(struct mlx5e_channel * c,struct mlx5e_rq_param * param,struct mlx5e_rq * rq)1203 mlx5e_create_rq(struct mlx5e_channel *c,
1204     struct mlx5e_rq_param *param,
1205     struct mlx5e_rq *rq)
1206 {
1207 	struct mlx5e_priv *priv = c->priv;
1208 	struct mlx5_core_dev *mdev = priv->mdev;
1209 	char buffer[16];
1210 	void *rqc = param->rqc;
1211 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
1212 	int wq_sz;
1213 	int err;
1214 	int i;
1215 	u32 nsegs, wqe_sz;
1216 
1217 	err = mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
1218 	if (err != 0)
1219 		goto done;
1220 
1221 	/* Create DMA descriptor TAG */
1222 	if ((err = -bus_dma_tag_create(
1223 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1224 	    1,				/* any alignment */
1225 	    0,				/* no boundary */
1226 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1227 	    BUS_SPACE_MAXADDR,		/* highaddr */
1228 	    NULL, NULL,			/* filter, filterarg */
1229 	    nsegs * wqe_sz,		/* maxsize */
1230 	    nsegs,			/* nsegments */
1231 	    nsegs * wqe_sz,		/* maxsegsize */
1232 	    0,				/* flags */
1233 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1234 	    &rq->dma_tag)))
1235 		goto done;
1236 
1237 	err = mlx5_wq_ll_create(mdev, &param->wq, rqc_wq, &rq->wq,
1238 	    &rq->wq_ctrl);
1239 	if (err)
1240 		goto err_free_dma_tag;
1241 
1242 	rq->wq.db = &rq->wq.db[MLX5_RCV_DBR];
1243 
1244 	err = mlx5e_get_wqe_sz(priv, &rq->wqe_sz, &rq->nsegs);
1245 	if (err != 0)
1246 		goto err_rq_wq_destroy;
1247 
1248 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1249 
1250 	err = -tcp_lro_init_args(&rq->lro, priv->ifp, TCP_LRO_ENTRIES, wq_sz);
1251 	if (err)
1252 		goto err_rq_wq_destroy;
1253 
1254 	rq->mbuf = malloc_domainset(wq_sz * sizeof(rq->mbuf[0]), M_MLX5EN,
1255 	    mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
1256 	for (i = 0; i != wq_sz; i++) {
1257 		struct mlx5e_rx_wqe *wqe = mlx5_wq_ll_get_wqe(&rq->wq, i);
1258 		int j;
1259 
1260 		err = -bus_dmamap_create(rq->dma_tag, 0, &rq->mbuf[i].dma_map);
1261 		if (err != 0) {
1262 			while (i--)
1263 				bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1264 			goto err_rq_mbuf_free;
1265 		}
1266 
1267 		/* set value for constant fields */
1268 		for (j = 0; j < rq->nsegs; j++)
1269 			wqe->data[j].lkey = cpu_to_be32(priv->mr.key);
1270 	}
1271 
1272 	INIT_WORK(&rq->dim.work, mlx5e_dim_work);
1273 	if (priv->params.rx_cq_moderation_mode < 2) {
1274 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1275 	} else {
1276 		void *cqc = container_of(param,
1277 		    struct mlx5e_channel_param, rq)->rx_cq.cqc;
1278 
1279 		switch (MLX5_GET(cqc, cqc, cq_period_mode)) {
1280 		case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
1281 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
1282 			break;
1283 		case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
1284 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
1285 			break;
1286 		default:
1287 			rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
1288 			break;
1289 		}
1290 	}
1291 
1292 	rq->ifp = priv->ifp;
1293 	rq->channel = c;
1294 	rq->ix = c->ix;
1295 
1296 	snprintf(buffer, sizeof(buffer), "rxstat%d", c->ix);
1297 	mlx5e_create_stats(&rq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1298 	    buffer, mlx5e_rq_stats_desc, MLX5E_RQ_STATS_NUM,
1299 	    rq->stats.arg);
1300 	return (0);
1301 
1302 err_rq_mbuf_free:
1303 	free(rq->mbuf, M_MLX5EN);
1304 	tcp_lro_free(&rq->lro);
1305 err_rq_wq_destroy:
1306 	mlx5_wq_destroy(&rq->wq_ctrl);
1307 err_free_dma_tag:
1308 	bus_dma_tag_destroy(rq->dma_tag);
1309 done:
1310 	return (err);
1311 }
1312 
1313 static void
mlx5e_destroy_rq(struct mlx5e_rq * rq)1314 mlx5e_destroy_rq(struct mlx5e_rq *rq)
1315 {
1316 	int wq_sz;
1317 	int i;
1318 
1319 	/* destroy all sysctl nodes */
1320 	sysctl_ctx_free(&rq->stats.ctx);
1321 
1322 	/* free leftover LRO packets, if any */
1323 	tcp_lro_free(&rq->lro);
1324 
1325 	wq_sz = mlx5_wq_ll_get_size(&rq->wq);
1326 	for (i = 0; i != wq_sz; i++) {
1327 		if (rq->mbuf[i].mbuf != NULL) {
1328 			if (rq->mbuf[i].ipsec_mtag != NULL)
1329 				m_tag_free(&rq->mbuf[i].ipsec_mtag->tag);
1330 			bus_dmamap_unload(rq->dma_tag, rq->mbuf[i].dma_map);
1331 			m_freem(rq->mbuf[i].mbuf);
1332 		}
1333 		bus_dmamap_destroy(rq->dma_tag, rq->mbuf[i].dma_map);
1334 	}
1335 	free(rq->mbuf, M_MLX5EN);
1336 	mlx5_wq_destroy(&rq->wq_ctrl);
1337 	bus_dma_tag_destroy(rq->dma_tag);
1338 }
1339 
1340 static int
mlx5e_enable_rq(struct mlx5e_rq * rq,struct mlx5e_rq_param * param)1341 mlx5e_enable_rq(struct mlx5e_rq *rq, struct mlx5e_rq_param *param)
1342 {
1343 	struct mlx5e_channel *c = rq->channel;
1344 	struct mlx5e_priv *priv = c->priv;
1345 	struct mlx5_core_dev *mdev = priv->mdev;
1346 	void *in;
1347 	void *rqc;
1348 	void *wq;
1349 	int inlen;
1350 	int err;
1351 	u8 ts_format;
1352 
1353 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1354 	    sizeof(u64) * rq->wq_ctrl.buf.npages;
1355 	in = mlx5_vzalloc(inlen);
1356 	if (in == NULL)
1357 		return (-ENOMEM);
1358 
1359 	ts_format = mlx5_get_rq_default_ts(mdev);
1360 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1361 	wq = MLX5_ADDR_OF(rqc, rqc, wq);
1362 
1363 	memcpy(rqc, param->rqc, sizeof(param->rqc));
1364 
1365 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1366 	MLX5_SET(rqc, rqc, ts_format, ts_format);
1367 	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1368 	if (priv->counter_set_id >= 0)
1369 		MLX5_SET(rqc, rqc, counter_set_id, priv->counter_set_id);
1370 	MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift -
1371 	    MLX5_ADAPTER_PAGE_SHIFT);
1372 	MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma);
1373 
1374 	mlx5_fill_page_array(&rq->wq_ctrl.buf,
1375 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1376 
1377 	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
1378 
1379 	kvfree(in);
1380 
1381 	return (err);
1382 }
1383 
1384 static int
mlx5e_modify_rq(struct mlx5e_rq * rq,int curr_state,int next_state)1385 mlx5e_modify_rq(struct mlx5e_rq *rq, int curr_state, int next_state)
1386 {
1387 	struct mlx5e_channel *c = rq->channel;
1388 	struct mlx5e_priv *priv = c->priv;
1389 	struct mlx5_core_dev *mdev = priv->mdev;
1390 
1391 	void *in;
1392 	void *rqc;
1393 	int inlen;
1394 	int err;
1395 
1396 	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
1397 	in = mlx5_vzalloc(inlen);
1398 	if (in == NULL)
1399 		return (-ENOMEM);
1400 
1401 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
1402 
1403 	MLX5_SET(modify_rq_in, in, rqn, rq->rqn);
1404 	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
1405 	MLX5_SET(rqc, rqc, state, next_state);
1406 
1407 	err = mlx5_core_modify_rq(mdev, in, inlen);
1408 
1409 	kvfree(in);
1410 
1411 	return (err);
1412 }
1413 
1414 static void
mlx5e_disable_rq(struct mlx5e_rq * rq)1415 mlx5e_disable_rq(struct mlx5e_rq *rq)
1416 {
1417 	struct mlx5e_channel *c = rq->channel;
1418 	struct mlx5e_priv *priv = c->priv;
1419 	struct mlx5_core_dev *mdev = priv->mdev;
1420 
1421 	mlx5_core_destroy_rq(mdev, rq->rqn);
1422 }
1423 
1424 static int
mlx5e_open_rq(struct mlx5e_channel * c,struct mlx5e_rq_param * param,struct mlx5e_rq * rq)1425 mlx5e_open_rq(struct mlx5e_channel *c,
1426     struct mlx5e_rq_param *param,
1427     struct mlx5e_rq *rq)
1428 {
1429 	int err;
1430 
1431 	err = mlx5e_create_rq(c, param, rq);
1432 	if (err)
1433 		return (err);
1434 
1435 	/* set CQN in RQ parameters */
1436 	MLX5_SET(rqc, param->rqc, cqn, c->rq.cq.mcq.cqn);
1437 
1438 	err = mlx5e_enable_rq(rq, param);
1439 	if (err)
1440 		goto err_destroy_rq;
1441 
1442 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1443 	if (err)
1444 		goto err_disable_rq;
1445 
1446 	c->rq.enabled = 1;
1447 
1448 	return (0);
1449 
1450 err_disable_rq:
1451 	mlx5e_disable_rq(rq);
1452 err_destroy_rq:
1453 	mlx5e_destroy_rq(rq);
1454 
1455 	return (err);
1456 }
1457 
1458 static void
mlx5e_close_rq(struct mlx5e_rq * rq)1459 mlx5e_close_rq(struct mlx5e_rq *rq)
1460 {
1461 	mtx_lock(&rq->mtx);
1462 	rq->enabled = 0;
1463 	callout_stop(&rq->watchdog);
1464 	mtx_unlock(&rq->mtx);
1465 
1466 	mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1467 }
1468 
1469 static void
mlx5e_close_rq_wait(struct mlx5e_rq * rq)1470 mlx5e_close_rq_wait(struct mlx5e_rq *rq)
1471 {
1472 
1473 	mlx5e_disable_rq(rq);
1474 	mlx5e_close_cq(&rq->cq);
1475 	cancel_work_sync(&rq->dim.work);
1476 	mlx5e_destroy_rq(rq);
1477 }
1478 
1479 /*
1480  * What is a drop RQ and why is it needed?
1481  *
1482  * The RSS indirection table, also called the RQT, selects the
1483  * destination RQ based on the receive queue number, RQN. The RQT is
1484  * frequently referred to by flow steering rules to distribute traffic
1485  * among multiple RQs. The problem is that the RQs cannot be destroyed
1486  * before the RQT referring them is destroyed too. Further, TLS RX
1487  * rules may still be referring to the RQT even if the link went
1488  * down. Because there is no magic RQN for dropping packets, we create
1489  * a dummy RQ, also called drop RQ, which sole purpose is to drop all
1490  * received packets. When the link goes down this RQN is filled in all
1491  * RQT entries, of the main RQT, so the real RQs which are about to be
1492  * destroyed can be released and the TLS RX rules can be sustained.
1493  */
1494 static void
mlx5e_open_drop_rq_comp(struct mlx5_core_cq * mcq __unused,struct mlx5_eqe * eqe __unused)1495 mlx5e_open_drop_rq_comp(struct mlx5_core_cq *mcq __unused, struct mlx5_eqe *eqe __unused)
1496 {
1497 }
1498 
1499 static int
mlx5e_open_drop_rq(struct mlx5e_priv * priv,struct mlx5e_rq * drop_rq)1500 mlx5e_open_drop_rq(struct mlx5e_priv *priv,
1501     struct mlx5e_rq *drop_rq)
1502 {
1503 	struct mlx5e_cq_param param_cq = {};
1504 	struct mlx5e_rq_param param_rq = {};
1505 	void *rqc_wq = MLX5_ADDR_OF(rqc, param_rq.rqc, wq);
1506 	int err;
1507 
1508 	/* set channel pointer */
1509 	drop_rq->channel = priv->channel;
1510 
1511 	/* set basic CQ parameters needed */
1512 	MLX5_SET(cqc, param_cq.cqc, log_cq_size, 0);
1513 	MLX5_SET(cqc, param_cq.cqc, uar_page, priv->mdev->priv.uar->index);
1514 
1515 	/* open receive completion queue */
1516 	err = mlx5e_open_cq(priv, &param_cq, &drop_rq->cq,
1517 	    &mlx5e_open_drop_rq_comp, 0);
1518 	if (err)
1519 		goto err_done;
1520 
1521 	/* set basic WQ parameters needed */
1522 	MLX5_SET(wq, rqc_wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
1523 	MLX5_SET(wq, rqc_wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1524 	MLX5_SET(wq, rqc_wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) + sizeof(struct mlx5_wqe_data_seg)));
1525 	MLX5_SET(wq, rqc_wq, log_wq_sz, 0);
1526 	MLX5_SET(wq, rqc_wq, pd, priv->pdn);
1527 
1528 	param_rq.wq.linear = 1;
1529 
1530 	err = mlx5_wq_ll_create(priv->mdev, &param_rq.wq, rqc_wq, &drop_rq->wq,
1531 	    &drop_rq->wq_ctrl);
1532 	if (err)
1533 		goto err_close_cq;
1534 
1535 	/* set CQN in RQ parameters */
1536 	MLX5_SET(rqc, param_rq.rqc, cqn, drop_rq->cq.mcq.cqn);
1537 
1538 	err = mlx5e_enable_rq(drop_rq, &param_rq);
1539 	if (err)
1540 		goto err_wq_destroy;
1541 
1542 	err = mlx5e_modify_rq(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
1543 	if (err)
1544 		goto err_disable_rq;
1545 
1546 	return (err);
1547 
1548 err_disable_rq:
1549 	mlx5e_disable_rq(drop_rq);
1550 err_wq_destroy:
1551 	mlx5_wq_destroy(&drop_rq->wq_ctrl);
1552 err_close_cq:
1553 	mlx5e_close_cq(&drop_rq->cq);
1554 err_done:
1555 	return (err);
1556 }
1557 
1558 static void
mlx5e_close_drop_rq(struct mlx5e_rq * drop_rq)1559 mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
1560 {
1561 	mlx5e_modify_rq(drop_rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
1562 	mlx5e_disable_rq(drop_rq);
1563 	mlx5_wq_destroy(&drop_rq->wq_ctrl);
1564 	mlx5e_close_cq(&drop_rq->cq);
1565 }
1566 
1567 void
mlx5e_free_sq_db(struct mlx5e_sq * sq)1568 mlx5e_free_sq_db(struct mlx5e_sq *sq)
1569 {
1570 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1571 	int x;
1572 
1573 	for (x = 0; x != wq_sz; x++) {
1574 		if (sq->mbuf[x].mbuf != NULL) {
1575 			bus_dmamap_unload(sq->dma_tag, sq->mbuf[x].dma_map);
1576 			m_freem(sq->mbuf[x].mbuf);
1577 		}
1578 		if (sq->mbuf[x].mst != NULL) {
1579 			m_snd_tag_rele(sq->mbuf[x].mst);
1580 			sq->mbuf[x].mst = NULL;
1581 		}
1582 		bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1583 	}
1584 	free(sq->mbuf, M_MLX5EN);
1585 }
1586 
1587 int
mlx5e_alloc_sq_db(struct mlx5e_sq * sq)1588 mlx5e_alloc_sq_db(struct mlx5e_sq *sq)
1589 {
1590 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1591 	int err;
1592 	int x;
1593 
1594 	sq->mbuf = malloc_domainset(wq_sz * sizeof(sq->mbuf[0]), M_MLX5EN,
1595 	    mlx5_dev_domainset(sq->priv->mdev), M_WAITOK | M_ZERO);
1596 
1597 	/* Create DMA descriptor MAPs */
1598 	for (x = 0; x != wq_sz; x++) {
1599 		err = -bus_dmamap_create(sq->dma_tag, 0, &sq->mbuf[x].dma_map);
1600 		if (err != 0) {
1601 			while (x--)
1602 				bus_dmamap_destroy(sq->dma_tag, sq->mbuf[x].dma_map);
1603 			free(sq->mbuf, M_MLX5EN);
1604 			return (err);
1605 		}
1606 	}
1607 	return (0);
1608 }
1609 
1610 static const char *mlx5e_sq_stats_desc[] = {
1611 	MLX5E_SQ_STATS(MLX5E_STATS_DESC)
1612 };
1613 
1614 void
mlx5e_update_sq_inline(struct mlx5e_sq * sq)1615 mlx5e_update_sq_inline(struct mlx5e_sq *sq)
1616 {
1617 	sq->max_inline = sq->priv->params.tx_max_inline;
1618 	sq->min_inline_mode = sq->priv->params.tx_min_inline_mode;
1619 
1620 	/*
1621 	 * Check if trust state is DSCP or if inline mode is NONE which
1622 	 * indicates CX-5 or newer hardware.
1623 	 */
1624 	if (sq->priv->params_ethtool.trust_state != MLX5_QPTS_TRUST_PCP ||
1625 	    sq->min_inline_mode == MLX5_INLINE_MODE_NONE) {
1626 		if (MLX5_CAP_ETH(sq->priv->mdev, wqe_vlan_insert))
1627 			sq->min_insert_caps = MLX5E_INSERT_VLAN | MLX5E_INSERT_NON_VLAN;
1628 		else
1629 			sq->min_insert_caps = MLX5E_INSERT_NON_VLAN;
1630 	} else {
1631 		sq->min_insert_caps = 0;
1632 	}
1633 }
1634 
1635 static void
mlx5e_refresh_sq_inline_sub(struct mlx5e_priv * priv,struct mlx5e_channel * c)1636 mlx5e_refresh_sq_inline_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
1637 {
1638 	int i;
1639 
1640 	for (i = 0; i != priv->num_tc; i++) {
1641 		mtx_lock(&c->sq[i].lock);
1642 		mlx5e_update_sq_inline(&c->sq[i]);
1643 		mtx_unlock(&c->sq[i].lock);
1644 	}
1645 }
1646 
1647 void
mlx5e_refresh_sq_inline(struct mlx5e_priv * priv)1648 mlx5e_refresh_sq_inline(struct mlx5e_priv *priv)
1649 {
1650 	int i;
1651 
1652 	/* check if channels are closed */
1653 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
1654 		return;
1655 
1656 	for (i = 0; i < priv->params.num_channels; i++)
1657 		mlx5e_refresh_sq_inline_sub(priv, &priv->channel[i]);
1658 }
1659 
1660 static int
mlx5e_create_sq(struct mlx5e_channel * c,int tc,struct mlx5e_sq_param * param,struct mlx5e_sq * sq)1661 mlx5e_create_sq(struct mlx5e_channel *c,
1662     int tc,
1663     struct mlx5e_sq_param *param,
1664     struct mlx5e_sq *sq)
1665 {
1666 	struct mlx5e_priv *priv = c->priv;
1667 	struct mlx5_core_dev *mdev = priv->mdev;
1668 	char buffer[16];
1669 	void *sqc = param->sqc;
1670 	void *sqc_wq = MLX5_ADDR_OF(sqc, sqc, wq);
1671 	int err;
1672 
1673 	/* Create DMA descriptor TAG */
1674 	if ((err = -bus_dma_tag_create(
1675 	    bus_get_dma_tag(mdev->pdev->dev.bsddev),
1676 	    1,				/* any alignment */
1677 	    0,				/* no boundary */
1678 	    BUS_SPACE_MAXADDR,		/* lowaddr */
1679 	    BUS_SPACE_MAXADDR,		/* highaddr */
1680 	    NULL, NULL,			/* filter, filterarg */
1681 	    MLX5E_MAX_TX_PAYLOAD_SIZE,	/* maxsize */
1682 	    MLX5E_MAX_TX_MBUF_FRAGS,	/* nsegments */
1683 	    MLX5E_MAX_TX_MBUF_SIZE,	/* maxsegsize */
1684 	    0,				/* flags */
1685 	    NULL, NULL,			/* lockfunc, lockfuncarg */
1686 	    &sq->dma_tag)))
1687 		goto done;
1688 
1689 	sq->mkey_be = cpu_to_be32(priv->mr.key);
1690 	sq->ifp = priv->ifp;
1691 	sq->priv = priv;
1692 	sq->tc = tc;
1693 
1694 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, &sq->wq,
1695 	    &sq->wq_ctrl);
1696 	if (err)
1697 		goto err_free_dma_tag;
1698 
1699 	sq->wq.db = &sq->wq.db[MLX5_SND_DBR];
1700 
1701 	err = mlx5e_alloc_sq_db(sq);
1702 	if (err)
1703 		goto err_sq_wq_destroy;
1704 
1705 	mlx5e_update_sq_inline(sq);
1706 
1707 	snprintf(buffer, sizeof(buffer), "txstat%dtc%d", c->ix, tc);
1708 	mlx5e_create_stats(&sq->stats.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
1709 	    buffer, mlx5e_sq_stats_desc, MLX5E_SQ_STATS_NUM,
1710 	    sq->stats.arg);
1711 
1712 	return (0);
1713 
1714 err_sq_wq_destroy:
1715 	mlx5_wq_destroy(&sq->wq_ctrl);
1716 
1717 err_free_dma_tag:
1718 	bus_dma_tag_destroy(sq->dma_tag);
1719 done:
1720 	return (err);
1721 }
1722 
1723 static void
mlx5e_destroy_sq(struct mlx5e_sq * sq)1724 mlx5e_destroy_sq(struct mlx5e_sq *sq)
1725 {
1726 	/* destroy all sysctl nodes */
1727 	sysctl_ctx_free(&sq->stats.ctx);
1728 
1729 	mlx5e_free_sq_db(sq);
1730 	mlx5_wq_destroy(&sq->wq_ctrl);
1731 	bus_dma_tag_destroy(sq->dma_tag);
1732 }
1733 
1734 int
mlx5e_enable_sq(struct mlx5e_sq * sq,struct mlx5e_sq_param * param,const struct mlx5_sq_bfreg * bfreg,int tis_num)1735 mlx5e_enable_sq(struct mlx5e_sq *sq, struct mlx5e_sq_param *param,
1736     const struct mlx5_sq_bfreg *bfreg, int tis_num)
1737 {
1738 	void *in;
1739 	void *sqc;
1740 	void *wq;
1741 	int inlen;
1742 	int err;
1743 	u8 ts_format;
1744 
1745 	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1746 	    sizeof(u64) * sq->wq_ctrl.buf.npages;
1747 	in = mlx5_vzalloc(inlen);
1748 	if (in == NULL)
1749 		return (-ENOMEM);
1750 
1751 	sq->uar_map = bfreg->map;
1752 
1753 	ts_format = mlx5_get_sq_default_ts(sq->priv->mdev);
1754 	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1755 	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1756 
1757 	memcpy(sqc, param->sqc, sizeof(param->sqc));
1758 
1759 	MLX5_SET(sqc, sqc, tis_num_0, tis_num);
1760 	MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn);
1761 	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1762 	MLX5_SET(sqc, sqc, ts_format, ts_format);
1763 	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1764 	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1765 	MLX5_SET(sqc, sqc, allow_swp, 1);
1766 
1767 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1768 	MLX5_SET(wq, wq, uar_page, bfreg->index);
1769 	MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift -
1770 	    MLX5_ADAPTER_PAGE_SHIFT);
1771 	MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma);
1772 
1773 	mlx5_fill_page_array(&sq->wq_ctrl.buf,
1774 	    (__be64 *) MLX5_ADDR_OF(wq, wq, pas));
1775 
1776 	err = mlx5_core_create_sq(sq->priv->mdev, in, inlen, &sq->sqn);
1777 
1778 	kvfree(in);
1779 
1780 	return (err);
1781 }
1782 
1783 int
mlx5e_modify_sq(struct mlx5e_sq * sq,int curr_state,int next_state)1784 mlx5e_modify_sq(struct mlx5e_sq *sq, int curr_state, int next_state)
1785 {
1786 	void *in;
1787 	void *sqc;
1788 	int inlen;
1789 	int err;
1790 
1791 	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1792 	in = mlx5_vzalloc(inlen);
1793 	if (in == NULL)
1794 		return (-ENOMEM);
1795 
1796 	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1797 
1798 	MLX5_SET(modify_sq_in, in, sqn, sq->sqn);
1799 	MLX5_SET(modify_sq_in, in, sq_state, curr_state);
1800 	MLX5_SET(sqc, sqc, state, next_state);
1801 
1802 	err = mlx5_core_modify_sq(sq->priv->mdev, in, inlen);
1803 
1804 	kvfree(in);
1805 
1806 	return (err);
1807 }
1808 
1809 void
mlx5e_disable_sq(struct mlx5e_sq * sq)1810 mlx5e_disable_sq(struct mlx5e_sq *sq)
1811 {
1812 
1813 	mlx5_core_destroy_sq(sq->priv->mdev, sq->sqn);
1814 }
1815 
1816 static int
mlx5e_open_sq(struct mlx5e_channel * c,int tc,struct mlx5e_sq_param * param,struct mlx5e_sq * sq)1817 mlx5e_open_sq(struct mlx5e_channel *c,
1818     int tc,
1819     struct mlx5e_sq_param *param,
1820     struct mlx5e_sq *sq)
1821 {
1822 	int err;
1823 
1824 	sq->cev_factor = c->priv->params_ethtool.tx_completion_fact;
1825 
1826 	/* ensure the TX completion event factor is not zero */
1827 	if (sq->cev_factor == 0)
1828 		sq->cev_factor = 1;
1829 
1830 	err = mlx5e_create_sq(c, tc, param, sq);
1831 	if (err)
1832 		return (err);
1833 
1834 	err = mlx5e_enable_sq(sq, param, &c->bfreg, c->priv->tisn[tc]);
1835 	if (err)
1836 		goto err_destroy_sq;
1837 
1838 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST, MLX5_SQC_STATE_RDY);
1839 	if (err)
1840 		goto err_disable_sq;
1841 
1842 	WRITE_ONCE(sq->running, 1);
1843 
1844 	return (0);
1845 
1846 err_disable_sq:
1847 	mlx5e_disable_sq(sq);
1848 err_destroy_sq:
1849 	mlx5e_destroy_sq(sq);
1850 
1851 	return (err);
1852 }
1853 
1854 static void
mlx5e_sq_send_nops_locked(struct mlx5e_sq * sq,int can_sleep)1855 mlx5e_sq_send_nops_locked(struct mlx5e_sq *sq, int can_sleep)
1856 {
1857 	/* fill up remainder with NOPs */
1858 	while (sq->cev_counter != 0) {
1859 		while (!mlx5e_sq_has_room_for(sq, 1)) {
1860 			if (can_sleep != 0) {
1861 				mtx_unlock(&sq->lock);
1862 				msleep(4);
1863 				mtx_lock(&sq->lock);
1864 			} else {
1865 				goto done;
1866 			}
1867 		}
1868 		/* send a single NOP */
1869 		mlx5e_send_nop(sq, 1);
1870 		atomic_thread_fence_rel();
1871 	}
1872 done:
1873 	mlx5e_tx_notify_hw(sq, false);
1874 }
1875 
1876 void
mlx5e_sq_cev_timeout(void * arg)1877 mlx5e_sq_cev_timeout(void *arg)
1878 {
1879 	struct mlx5e_sq *sq = arg;
1880 
1881 	mtx_assert(&sq->lock, MA_OWNED);
1882 
1883 	/* check next state */
1884 	switch (sq->cev_next_state) {
1885 	case MLX5E_CEV_STATE_SEND_NOPS:
1886 		/* fill TX ring with NOPs, if any */
1887 		mlx5e_sq_send_nops_locked(sq, 0);
1888 
1889 		/* check if completed */
1890 		if (sq->cev_counter == 0) {
1891 			sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
1892 			return;
1893 		}
1894 		break;
1895 	default:
1896 		/* send NOPs on next timeout */
1897 		sq->cev_next_state = MLX5E_CEV_STATE_SEND_NOPS;
1898 		break;
1899 	}
1900 
1901 	/* restart timer */
1902 	callout_reset_curcpu(&sq->cev_callout, hz, mlx5e_sq_cev_timeout, sq);
1903 }
1904 
1905 void
mlx5e_drain_sq(struct mlx5e_sq * sq)1906 mlx5e_drain_sq(struct mlx5e_sq *sq)
1907 {
1908 	int error;
1909 	struct mlx5_core_dev *mdev= sq->priv->mdev;
1910 
1911 	/*
1912 	 * Check if already stopped.
1913 	 *
1914 	 * NOTE: Serialization of this function is managed by the
1915 	 * caller ensuring the priv's state lock is locked or in case
1916 	 * of rate limit support, a single thread manages drain and
1917 	 * resume of SQs. The "running" variable can therefore safely
1918 	 * be read without any locks.
1919 	 */
1920 	if (READ_ONCE(sq->running) == 0)
1921 		return;
1922 
1923 	/* don't put more packets into the SQ */
1924 	WRITE_ONCE(sq->running, 0);
1925 
1926 	/* serialize access to DMA rings */
1927 	mtx_lock(&sq->lock);
1928 
1929 	/* teardown event factor timer, if any */
1930 	sq->cev_next_state = MLX5E_CEV_STATE_HOLD_NOPS;
1931 	callout_stop(&sq->cev_callout);
1932 
1933 	/* send dummy NOPs in order to flush the transmit ring */
1934 	mlx5e_sq_send_nops_locked(sq, 1);
1935 	mtx_unlock(&sq->lock);
1936 
1937 	/* wait till SQ is empty or link is down */
1938 	mtx_lock(&sq->lock);
1939 	while (sq->cc != sq->pc &&
1940 	    (sq->priv->media_status_last & IFM_ACTIVE) != 0 &&
1941 	    mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1942 	    pci_channel_offline(mdev->pdev) == 0) {
1943 		mtx_unlock(&sq->lock);
1944 		msleep(1);
1945 		sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1946 		mtx_lock(&sq->lock);
1947 	}
1948 	mtx_unlock(&sq->lock);
1949 
1950 	/* error out remaining requests */
1951 	error = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RDY, MLX5_SQC_STATE_ERR);
1952 	if (error != 0) {
1953 		mlx5_en_err(sq->ifp,
1954 		    "mlx5e_modify_sq() from RDY to ERR failed: %d\n", error);
1955 	}
1956 
1957 	/* wait till SQ is empty */
1958 	mtx_lock(&sq->lock);
1959 	while (sq->cc != sq->pc &&
1960 	       mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR &&
1961 	       pci_channel_offline(mdev->pdev) == 0) {
1962 		mtx_unlock(&sq->lock);
1963 		msleep(1);
1964 		sq->cq.mcq.comp(&sq->cq.mcq, NULL);
1965 		mtx_lock(&sq->lock);
1966 	}
1967 	mtx_unlock(&sq->lock);
1968 }
1969 
1970 static void
mlx5e_close_sq_wait(struct mlx5e_sq * sq)1971 mlx5e_close_sq_wait(struct mlx5e_sq *sq)
1972 {
1973 
1974 	mlx5e_drain_sq(sq);
1975 	mlx5e_disable_sq(sq);
1976 	mlx5e_destroy_sq(sq);
1977 }
1978 
1979 static int
mlx5e_create_cq(struct mlx5e_priv * priv,struct mlx5e_cq_param * param,struct mlx5e_cq * cq,mlx5e_cq_comp_t * comp,int eq_ix)1980 mlx5e_create_cq(struct mlx5e_priv *priv,
1981     struct mlx5e_cq_param *param,
1982     struct mlx5e_cq *cq,
1983     mlx5e_cq_comp_t *comp,
1984     int eq_ix)
1985 {
1986 	struct mlx5_core_dev *mdev = priv->mdev;
1987 	struct mlx5_core_cq *mcq = &cq->mcq;
1988 	int eqn_not_used;
1989 	int irqn;
1990 	int err;
1991 	u32 i;
1992 
1993 	err = mlx5_vector2eqn(mdev, eq_ix, &eqn_not_used, &irqn);
1994 	if (err)
1995 		return (err);
1996 
1997 	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1998 	    &cq->wq_ctrl);
1999 	if (err)
2000 		return (err);
2001 
2002 	mcq->cqe_sz = 64;
2003 	mcq->set_ci_db = cq->wq_ctrl.db.db;
2004 	mcq->arm_db = cq->wq_ctrl.db.db + 1;
2005 	*mcq->set_ci_db = 0;
2006 	*mcq->arm_db = 0;
2007 	mcq->vector = eq_ix;
2008 	mcq->comp = comp;
2009 	mcq->event = mlx5e_cq_error_event;
2010 	mcq->irqn = irqn;
2011 
2012 	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
2013 		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
2014 
2015 		cqe->op_own = 0xf1;
2016 	}
2017 
2018 	cq->priv = priv;
2019 
2020 	return (0);
2021 }
2022 
2023 static void
mlx5e_destroy_cq(struct mlx5e_cq * cq)2024 mlx5e_destroy_cq(struct mlx5e_cq *cq)
2025 {
2026 	mlx5_wq_destroy(&cq->wq_ctrl);
2027 }
2028 
2029 static int
mlx5e_enable_cq(struct mlx5e_cq * cq,struct mlx5e_cq_param * param,int eq_ix)2030 mlx5e_enable_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param, int eq_ix)
2031 {
2032 	struct mlx5_core_cq *mcq = &cq->mcq;
2033 	u32 out[MLX5_ST_SZ_DW(create_cq_out)];
2034 	void *in;
2035 	void *cqc;
2036 	int inlen;
2037 	int irqn_not_used;
2038 	int eqn;
2039 	int err;
2040 
2041 	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
2042 	    sizeof(u64) * cq->wq_ctrl.buf.npages;
2043 	in = mlx5_vzalloc(inlen);
2044 	if (in == NULL)
2045 		return (-ENOMEM);
2046 
2047 	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
2048 
2049 	memcpy(cqc, param->cqc, sizeof(param->cqc));
2050 
2051 	mlx5_fill_page_array(&cq->wq_ctrl.buf,
2052 	    (__be64 *) MLX5_ADDR_OF(create_cq_in, in, pas));
2053 
2054 	mlx5_vector2eqn(cq->priv->mdev, eq_ix, &eqn, &irqn_not_used);
2055 
2056 	MLX5_SET(cqc, cqc, c_eqn, eqn);
2057 	MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
2058 	    MLX5_ADAPTER_PAGE_SHIFT);
2059 	MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma);
2060 
2061 	err = mlx5_core_create_cq(cq->priv->mdev, mcq, in, inlen, out, sizeof(out));
2062 
2063 	kvfree(in);
2064 
2065 	if (err)
2066 		return (err);
2067 
2068 	mlx5e_cq_arm(cq, MLX5_GET_DOORBELL_LOCK(&cq->priv->doorbell_lock));
2069 
2070 	return (0);
2071 }
2072 
2073 static void
mlx5e_disable_cq(struct mlx5e_cq * cq)2074 mlx5e_disable_cq(struct mlx5e_cq *cq)
2075 {
2076 
2077 	mlx5_core_destroy_cq(cq->priv->mdev, &cq->mcq);
2078 }
2079 
2080 int
mlx5e_open_cq(struct mlx5e_priv * priv,struct mlx5e_cq_param * param,struct mlx5e_cq * cq,mlx5e_cq_comp_t * comp,int eq_ix)2081 mlx5e_open_cq(struct mlx5e_priv *priv,
2082     struct mlx5e_cq_param *param,
2083     struct mlx5e_cq *cq,
2084     mlx5e_cq_comp_t *comp,
2085     int eq_ix)
2086 {
2087 	int err;
2088 
2089 	err = mlx5e_create_cq(priv, param, cq, comp, eq_ix);
2090 	if (err)
2091 		return (err);
2092 
2093 	err = mlx5e_enable_cq(cq, param, eq_ix);
2094 	if (err)
2095 		goto err_destroy_cq;
2096 
2097 	return (0);
2098 
2099 err_destroy_cq:
2100 	mlx5e_destroy_cq(cq);
2101 
2102 	return (err);
2103 }
2104 
2105 void
mlx5e_close_cq(struct mlx5e_cq * cq)2106 mlx5e_close_cq(struct mlx5e_cq *cq)
2107 {
2108 	mlx5e_disable_cq(cq);
2109 	mlx5e_destroy_cq(cq);
2110 }
2111 
2112 static int
mlx5e_open_tx_cqs(struct mlx5e_channel * c,struct mlx5e_channel_param * cparam)2113 mlx5e_open_tx_cqs(struct mlx5e_channel *c,
2114     struct mlx5e_channel_param *cparam)
2115 {
2116 	int err;
2117 	int tc;
2118 
2119 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2120 		/* open completion queue */
2121 		err = mlx5e_open_cq(c->priv, &cparam->tx_cq, &c->sq[tc].cq,
2122 		    &mlx5e_tx_cq_comp, c->ix);
2123 		if (err)
2124 			goto err_close_tx_cqs;
2125 	}
2126 	return (0);
2127 
2128 err_close_tx_cqs:
2129 	for (tc--; tc >= 0; tc--)
2130 		mlx5e_close_cq(&c->sq[tc].cq);
2131 
2132 	return (err);
2133 }
2134 
2135 static void
mlx5e_close_tx_cqs(struct mlx5e_channel * c)2136 mlx5e_close_tx_cqs(struct mlx5e_channel *c)
2137 {
2138 	int tc;
2139 
2140 	for (tc = 0; tc < c->priv->num_tc; tc++)
2141 		mlx5e_close_cq(&c->sq[tc].cq);
2142 }
2143 
2144 static int
mlx5e_open_sqs(struct mlx5e_channel * c,struct mlx5e_channel_param * cparam)2145 mlx5e_open_sqs(struct mlx5e_channel *c,
2146     struct mlx5e_channel_param *cparam)
2147 {
2148 	int err;
2149 	int tc;
2150 
2151 	for (tc = 0; tc < c->priv->num_tc; tc++) {
2152 		err = mlx5e_open_sq(c, tc, &cparam->sq, &c->sq[tc]);
2153 		if (err)
2154 			goto err_close_sqs;
2155 	}
2156 
2157 	return (0);
2158 
2159 err_close_sqs:
2160 	for (tc--; tc >= 0; tc--)
2161 		mlx5e_close_sq_wait(&c->sq[tc]);
2162 
2163 	return (err);
2164 }
2165 
2166 static void
mlx5e_close_sqs_wait(struct mlx5e_channel * c)2167 mlx5e_close_sqs_wait(struct mlx5e_channel *c)
2168 {
2169 	int tc;
2170 
2171 	for (tc = 0; tc < c->priv->num_tc; tc++)
2172 		mlx5e_close_sq_wait(&c->sq[tc]);
2173 }
2174 
2175 static void
mlx5e_chan_static_init(struct mlx5e_priv * priv,struct mlx5e_channel * c,int ix)2176 mlx5e_chan_static_init(struct mlx5e_priv *priv, struct mlx5e_channel *c, int ix)
2177 {
2178 	int tc;
2179 
2180 	/* setup priv and channel number */
2181 	c->priv = priv;
2182 	c->ix = ix;
2183 
2184 	/* setup send tag */
2185 	m_snd_tag_init(&c->tag, c->priv->ifp, &mlx5e_ul_snd_tag_sw);
2186 
2187 	init_completion(&c->completion);
2188 
2189 	mtx_init(&c->rq.mtx, "mlx5rx", MTX_NETWORK_LOCK, MTX_DEF);
2190 
2191 	callout_init_mtx(&c->rq.watchdog, &c->rq.mtx, 0);
2192 
2193 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2194 		struct mlx5e_sq *sq = c->sq + tc;
2195 
2196 		mtx_init(&sq->lock, "mlx5tx",
2197 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2198 		mtx_init(&sq->comp_lock, "mlx5comp",
2199 		    MTX_NETWORK_LOCK " TX", MTX_DEF);
2200 
2201 		callout_init_mtx(&sq->cev_callout, &sq->lock, 0);
2202 	}
2203 
2204 	mlx5e_iq_static_init(&c->iq);
2205 }
2206 
2207 static void
mlx5e_chan_wait_for_completion(struct mlx5e_channel * c)2208 mlx5e_chan_wait_for_completion(struct mlx5e_channel *c)
2209 {
2210 
2211 	m_snd_tag_rele(&c->tag);
2212 	wait_for_completion(&c->completion);
2213 }
2214 
2215 static void
mlx5e_priv_wait_for_completion(struct mlx5e_priv * priv,const uint32_t channels)2216 mlx5e_priv_wait_for_completion(struct mlx5e_priv *priv, const uint32_t channels)
2217 {
2218 	uint32_t x;
2219 
2220 	for (x = 0; x != channels; x++)
2221 		mlx5e_chan_wait_for_completion(&priv->channel[x]);
2222 }
2223 
2224 static void
mlx5e_chan_static_destroy(struct mlx5e_channel * c)2225 mlx5e_chan_static_destroy(struct mlx5e_channel *c)
2226 {
2227 	int tc;
2228 
2229 	callout_drain(&c->rq.watchdog);
2230 
2231 	mtx_destroy(&c->rq.mtx);
2232 
2233 	for (tc = 0; tc != MLX5E_MAX_TX_NUM_TC; tc++) {
2234 		callout_drain(&c->sq[tc].cev_callout);
2235 		mtx_destroy(&c->sq[tc].lock);
2236 		mtx_destroy(&c->sq[tc].comp_lock);
2237 	}
2238 
2239 	mlx5e_iq_static_destroy(&c->iq);
2240 }
2241 
2242 static int
mlx5e_open_channel(struct mlx5e_priv * priv,struct mlx5e_channel_param * cparam,struct mlx5e_channel * c)2243 mlx5e_open_channel(struct mlx5e_priv *priv,
2244     struct mlx5e_channel_param *cparam,
2245     struct mlx5e_channel *c)
2246 {
2247 	struct epoch_tracker et;
2248 	int i, err;
2249 
2250 	/* zero non-persistent data */
2251 	MLX5E_ZERO(&c->rq, mlx5e_rq_zero_start);
2252 	for (i = 0; i != priv->num_tc; i++)
2253 		MLX5E_ZERO(&c->sq[i], mlx5e_sq_zero_start);
2254 	MLX5E_ZERO(&c->iq, mlx5e_iq_zero_start);
2255 
2256 	/* open transmit completion queue */
2257 	err = mlx5e_open_tx_cqs(c, cparam);
2258 	if (err)
2259 		goto err_free;
2260 
2261 	/* open receive completion queue */
2262 	err = mlx5e_open_cq(c->priv, &cparam->rx_cq, &c->rq.cq,
2263 	    &mlx5e_rx_cq_comp, c->ix);
2264 	if (err)
2265 		goto err_close_tx_cqs;
2266 
2267 	err = mlx5e_open_sqs(c, cparam);
2268 	if (err)
2269 		goto err_close_rx_cq;
2270 
2271 	err = mlx5e_iq_open(c, &cparam->sq, &cparam->tx_cq, &c->iq);
2272 	if (err)
2273 		goto err_close_sqs;
2274 
2275 	err = mlx5e_open_rq(c, &cparam->rq, &c->rq);
2276 	if (err)
2277 		goto err_close_iq;
2278 
2279 	/* poll receive queue initially */
2280 	NET_EPOCH_ENTER(et);
2281 	c->rq.cq.mcq.comp(&c->rq.cq.mcq, NULL);
2282 	NET_EPOCH_EXIT(et);
2283 
2284 	return (0);
2285 
2286 err_close_iq:
2287 	mlx5e_iq_close(&c->iq);
2288 
2289 err_close_sqs:
2290 	mlx5e_close_sqs_wait(c);
2291 
2292 err_close_rx_cq:
2293 	mlx5e_close_cq(&c->rq.cq);
2294 
2295 err_close_tx_cqs:
2296 	mlx5e_close_tx_cqs(c);
2297 
2298 err_free:
2299 	return (err);
2300 }
2301 
2302 static void
mlx5e_close_channel(struct mlx5e_channel * c)2303 mlx5e_close_channel(struct mlx5e_channel *c)
2304 {
2305 	mlx5e_close_rq(&c->rq);
2306 }
2307 
2308 static void
mlx5e_close_channel_wait(struct mlx5e_channel * c)2309 mlx5e_close_channel_wait(struct mlx5e_channel *c)
2310 {
2311 	mlx5e_close_rq_wait(&c->rq);
2312 	mlx5e_iq_close(&c->iq);
2313 	mlx5e_close_sqs_wait(c);
2314 	mlx5e_close_tx_cqs(c);
2315 }
2316 
2317 static int
mlx5e_get_wqe_sz(struct mlx5e_priv * priv,u32 * wqe_sz,u32 * nsegs)2318 mlx5e_get_wqe_sz(struct mlx5e_priv *priv, u32 *wqe_sz, u32 *nsegs)
2319 {
2320 	u32 r, n, maxs;
2321 
2322 	maxs = priv->params.hw_lro_en ? priv->params.lro_wqe_sz :
2323 	    MLX5E_SW2MB_MTU(if_getmtu(priv->ifp));
2324 	r = maxs > MCLBYTES ? MJUMPAGESIZE : MCLBYTES;
2325 
2326 	/*
2327 	 * n + 1 must be a power of two, because stride size must be.
2328 	 * Stride size is 16 * (n + 1), as the first segment is
2329 	 * control.
2330 	 */
2331 	n = roundup_pow_of_two(1 + howmany(maxs, r)) - 1;
2332 	if (n > MLX5E_MAX_BUSDMA_RX_SEGS)
2333 		return (-ENOMEM);
2334 
2335 	*wqe_sz = r;
2336 	*nsegs = n;
2337 	return (0);
2338 }
2339 
2340 static void
mlx5e_build_rq_param(struct mlx5e_priv * priv,struct mlx5e_rq_param * param)2341 mlx5e_build_rq_param(struct mlx5e_priv *priv,
2342     struct mlx5e_rq_param *param)
2343 {
2344 	void *rqc = param->rqc;
2345 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2346 	u32 wqe_sz, nsegs;
2347 
2348 	mlx5e_get_wqe_sz(priv, &wqe_sz, &nsegs);
2349 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_LINKED_LIST);
2350 	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2351 	MLX5_SET(wq, wq, log_wq_stride, ilog2(sizeof(struct mlx5e_rx_wqe) +
2352 	    nsegs * sizeof(struct mlx5_wqe_data_seg)));
2353 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_rq_size);
2354 	MLX5_SET(wq, wq, pd, priv->pdn);
2355 
2356 	param->wq.linear = 1;
2357 }
2358 
2359 static void
mlx5e_build_sq_param(struct mlx5e_priv * priv,struct mlx5e_sq_param * param)2360 mlx5e_build_sq_param(struct mlx5e_priv *priv,
2361     struct mlx5e_sq_param *param)
2362 {
2363 	void *sqc = param->sqc;
2364 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2365 
2366 	MLX5_SET(wq, wq, log_wq_sz, priv->params.log_sq_size);
2367 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2368 	MLX5_SET(wq, wq, pd, priv->pdn);
2369 
2370 	param->wq.linear = 1;
2371 }
2372 
2373 static void
mlx5e_build_common_cq_param(struct mlx5e_priv * priv,struct mlx5e_cq_param * param)2374 mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2375     struct mlx5e_cq_param *param)
2376 {
2377 	void *cqc = param->cqc;
2378 
2379 	MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
2380 }
2381 
2382 static void
mlx5e_get_default_profile(struct mlx5e_priv * priv,int mode,struct net_dim_cq_moder * ptr)2383 mlx5e_get_default_profile(struct mlx5e_priv *priv, int mode, struct net_dim_cq_moder *ptr)
2384 {
2385 
2386 	*ptr = net_dim_get_profile(mode, MLX5E_DIM_DEFAULT_PROFILE);
2387 
2388 	/* apply LRO restrictions */
2389 	if (priv->params.hw_lro_en &&
2390 	    ptr->pkts > MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO) {
2391 		ptr->pkts = MLX5E_DIM_MAX_RX_CQ_MODERATION_PKTS_WITH_LRO;
2392 	}
2393 }
2394 
2395 static void
mlx5e_build_rx_cq_param(struct mlx5e_priv * priv,struct mlx5e_cq_param * param)2396 mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2397     struct mlx5e_cq_param *param)
2398 {
2399 	struct net_dim_cq_moder curr;
2400 	void *cqc = param->cqc;
2401 
2402 	/*
2403 	 * We use MLX5_CQE_FORMAT_HASH because the RX hash mini CQE
2404 	 * format is more beneficial for FreeBSD use case.
2405 	 *
2406 	 * Adding support for MLX5_CQE_FORMAT_CSUM will require changes
2407 	 * in mlx5e_decompress_cqe.
2408 	 */
2409 	if (priv->params.cqe_zipping_en) {
2410 		MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_HASH);
2411 		MLX5_SET(cqc, cqc, cqe_compression_en, 1);
2412 	}
2413 
2414 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_rq_size);
2415 
2416 	switch (priv->params.rx_cq_moderation_mode) {
2417 	case 0:
2418 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2419 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2420 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2421 		break;
2422 	case 1:
2423 		MLX5_SET(cqc, cqc, cq_period, priv->params.rx_cq_moderation_usec);
2424 		MLX5_SET(cqc, cqc, cq_max_count, priv->params.rx_cq_moderation_pkts);
2425 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2426 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2427 		else
2428 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2429 		break;
2430 	case 2:
2431 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE, &curr);
2432 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2433 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2434 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2435 		break;
2436 	case 3:
2437 		mlx5e_get_default_profile(priv, NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE, &curr);
2438 		MLX5_SET(cqc, cqc, cq_period, curr.usec);
2439 		MLX5_SET(cqc, cqc, cq_max_count, curr.pkts);
2440 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2441 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2442 		else
2443 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2444 		break;
2445 	default:
2446 		break;
2447 	}
2448 
2449 	mlx5e_dim_build_cq_param(priv, param);
2450 
2451 	mlx5e_build_common_cq_param(priv, param);
2452 }
2453 
2454 static void
mlx5e_build_tx_cq_param(struct mlx5e_priv * priv,struct mlx5e_cq_param * param)2455 mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2456     struct mlx5e_cq_param *param)
2457 {
2458 	void *cqc = param->cqc;
2459 
2460 	MLX5_SET(cqc, cqc, log_cq_size, priv->params.log_sq_size);
2461 	MLX5_SET(cqc, cqc, cq_period, priv->params.tx_cq_moderation_usec);
2462 	MLX5_SET(cqc, cqc, cq_max_count, priv->params.tx_cq_moderation_pkts);
2463 
2464 	switch (priv->params.tx_cq_moderation_mode) {
2465 	case 0:
2466 		MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2467 		break;
2468 	default:
2469 		if (MLX5_CAP_GEN(priv->mdev, cq_period_start_from_cqe))
2470 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
2471 		else
2472 			MLX5_SET(cqc, cqc, cq_period_mode, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
2473 		break;
2474 	}
2475 
2476 	mlx5e_build_common_cq_param(priv, param);
2477 }
2478 
2479 static void
mlx5e_build_channel_param(struct mlx5e_priv * priv,struct mlx5e_channel_param * cparam)2480 mlx5e_build_channel_param(struct mlx5e_priv *priv,
2481     struct mlx5e_channel_param *cparam)
2482 {
2483 	memset(cparam, 0, sizeof(*cparam));
2484 
2485 	mlx5e_build_rq_param(priv, &cparam->rq);
2486 	mlx5e_build_sq_param(priv, &cparam->sq);
2487 	mlx5e_build_rx_cq_param(priv, &cparam->rx_cq);
2488 	mlx5e_build_tx_cq_param(priv, &cparam->tx_cq);
2489 }
2490 
2491 static int
mlx5e_open_channels(struct mlx5e_priv * priv)2492 mlx5e_open_channels(struct mlx5e_priv *priv)
2493 {
2494 	struct mlx5e_channel_param *cparam;
2495 	int err;
2496 	int i;
2497 
2498 	cparam = malloc(sizeof(*cparam), M_MLX5EN, M_WAITOK);
2499 
2500 	mlx5e_build_channel_param(priv, cparam);
2501 	for (i = 0; i < priv->params.num_channels; i++) {
2502 		err = mlx5e_open_channel(priv, cparam, &priv->channel[i]);
2503 		if (err)
2504 			goto err_close_channels;
2505 
2506 		/* Bind interrupt vectors, if any. */
2507 		if (priv->params_ethtool.irq_cpu_base > -1) {
2508 			cpuset_t cpuset;
2509 			int cpu;
2510 			int irq;
2511 			int eqn;
2512 			int nirq;
2513 
2514 			err = mlx5_vector2eqn(priv->mdev, i,
2515 			    &eqn, &nirq);
2516 
2517 			/* error here is non-fatal */
2518 			if (err != 0)
2519 				continue;
2520 
2521 			irq = priv->mdev->priv.msix_arr[nirq].vector;
2522 			cpu = (unsigned)(priv->params_ethtool.irq_cpu_base +
2523 			    i * priv->params_ethtool.irq_cpu_stride) % (unsigned)mp_ncpus;
2524 
2525 			CPU_ZERO(&cpuset);
2526 			CPU_SET(cpu, &cpuset);
2527 			intr_setaffinity(irq, CPU_WHICH_INTRHANDLER, &cpuset);
2528 		}
2529 	}
2530 	free(cparam, M_MLX5EN);
2531 	return (0);
2532 
2533 err_close_channels:
2534 	while (i--) {
2535 		mlx5e_close_channel(&priv->channel[i]);
2536 		mlx5e_close_channel_wait(&priv->channel[i]);
2537 	}
2538 	free(cparam, M_MLX5EN);
2539 	return (err);
2540 }
2541 
2542 static void
mlx5e_close_channels(struct mlx5e_priv * priv)2543 mlx5e_close_channels(struct mlx5e_priv *priv)
2544 {
2545 	int i;
2546 
2547 	for (i = 0; i < priv->params.num_channels; i++)
2548 		mlx5e_close_channel(&priv->channel[i]);
2549 	for (i = 0; i < priv->params.num_channels; i++)
2550 		mlx5e_close_channel_wait(&priv->channel[i]);
2551 }
2552 
2553 static int
mlx5e_refresh_sq_params(struct mlx5e_priv * priv,struct mlx5e_sq * sq)2554 mlx5e_refresh_sq_params(struct mlx5e_priv *priv, struct mlx5e_sq *sq)
2555 {
2556 
2557 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2558 		uint8_t cq_mode;
2559 
2560 		switch (priv->params.tx_cq_moderation_mode) {
2561 		case 0:
2562 		case 2:
2563 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2564 			break;
2565 		default:
2566 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2567 			break;
2568 		}
2569 
2570 		return (mlx5_core_modify_cq_moderation_mode(priv->mdev, &sq->cq.mcq,
2571 		    priv->params.tx_cq_moderation_usec,
2572 		    priv->params.tx_cq_moderation_pkts,
2573 		    cq_mode));
2574 	}
2575 
2576 	return (mlx5_core_modify_cq_moderation(priv->mdev, &sq->cq.mcq,
2577 	    priv->params.tx_cq_moderation_usec,
2578 	    priv->params.tx_cq_moderation_pkts));
2579 }
2580 
2581 static int
mlx5e_refresh_rq_params(struct mlx5e_priv * priv,struct mlx5e_rq * rq)2582 mlx5e_refresh_rq_params(struct mlx5e_priv *priv, struct mlx5e_rq *rq)
2583 {
2584 
2585 	if (MLX5_CAP_GEN(priv->mdev, cq_period_mode_modify)) {
2586 		uint8_t cq_mode;
2587 		uint8_t dim_mode;
2588 		int retval;
2589 
2590 		switch (priv->params.rx_cq_moderation_mode) {
2591 		case 0:
2592 		case 2:
2593 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
2594 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2595 			break;
2596 		default:
2597 			cq_mode = MLX5_CQ_PERIOD_MODE_START_FROM_CQE;
2598 			dim_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
2599 			break;
2600 		}
2601 
2602 		/* tear down dynamic interrupt moderation */
2603 		mtx_lock(&rq->mtx);
2604 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_DISABLED;
2605 		mtx_unlock(&rq->mtx);
2606 
2607 		/* wait for dynamic interrupt moderation work task, if any */
2608 		cancel_work_sync(&rq->dim.work);
2609 
2610 		if (priv->params.rx_cq_moderation_mode >= 2) {
2611 			struct net_dim_cq_moder curr;
2612 
2613 			mlx5e_get_default_profile(priv, dim_mode, &curr);
2614 
2615 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2616 			    curr.usec, curr.pkts, cq_mode);
2617 
2618 			/* set dynamic interrupt moderation mode and zero defaults */
2619 			mtx_lock(&rq->mtx);
2620 			rq->dim.mode = dim_mode;
2621 			rq->dim.state = 0;
2622 			rq->dim.profile_ix = MLX5E_DIM_DEFAULT_PROFILE;
2623 			mtx_unlock(&rq->mtx);
2624 		} else {
2625 			retval = mlx5_core_modify_cq_moderation_mode(priv->mdev, &rq->cq.mcq,
2626 			    priv->params.rx_cq_moderation_usec,
2627 			    priv->params.rx_cq_moderation_pkts,
2628 			    cq_mode);
2629 		}
2630 		return (retval);
2631 	}
2632 
2633 	return (mlx5_core_modify_cq_moderation(priv->mdev, &rq->cq.mcq,
2634 	    priv->params.rx_cq_moderation_usec,
2635 	    priv->params.rx_cq_moderation_pkts));
2636 }
2637 
2638 static int
mlx5e_refresh_channel_params_sub(struct mlx5e_priv * priv,struct mlx5e_channel * c)2639 mlx5e_refresh_channel_params_sub(struct mlx5e_priv *priv, struct mlx5e_channel *c)
2640 {
2641 	int err;
2642 	int i;
2643 
2644 	err = mlx5e_refresh_rq_params(priv, &c->rq);
2645 	if (err)
2646 		goto done;
2647 
2648 	for (i = 0; i != priv->num_tc; i++) {
2649 		err = mlx5e_refresh_sq_params(priv, &c->sq[i]);
2650 		if (err)
2651 			goto done;
2652 	}
2653 done:
2654 	return (err);
2655 }
2656 
2657 int
mlx5e_refresh_channel_params(struct mlx5e_priv * priv)2658 mlx5e_refresh_channel_params(struct mlx5e_priv *priv)
2659 {
2660 	int i;
2661 
2662 	/* check if channels are closed */
2663 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
2664 		return (EINVAL);
2665 
2666 	for (i = 0; i < priv->params.num_channels; i++) {
2667 		int err;
2668 
2669 		err = mlx5e_refresh_channel_params_sub(priv, &priv->channel[i]);
2670 		if (err)
2671 			return (err);
2672 	}
2673 	return (0);
2674 }
2675 
2676 static int
mlx5e_open_tis(struct mlx5e_priv * priv,int tc)2677 mlx5e_open_tis(struct mlx5e_priv *priv, int tc)
2678 {
2679 	struct mlx5_core_dev *mdev = priv->mdev;
2680 	u32 in[MLX5_ST_SZ_DW(create_tis_in)];
2681 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
2682 
2683 	memset(in, 0, sizeof(in));
2684 
2685 	MLX5_SET(tisc, tisc, prio, tc);
2686 	MLX5_SET(tisc, tisc, transport_domain, priv->tdn);
2687 
2688 	return (mlx5_core_create_tis(mdev, in, sizeof(in), &priv->tisn[tc]));
2689 }
2690 
2691 static void
mlx5e_close_tis(struct mlx5e_priv * priv,int tc)2692 mlx5e_close_tis(struct mlx5e_priv *priv, int tc)
2693 {
2694 	mlx5_core_destroy_tis(priv->mdev, priv->tisn[tc], 0);
2695 }
2696 
2697 static int
mlx5e_open_tises(struct mlx5e_priv * priv)2698 mlx5e_open_tises(struct mlx5e_priv *priv)
2699 {
2700 	int num_tc = priv->num_tc;
2701 	int err;
2702 	int tc;
2703 
2704 	for (tc = 0; tc < num_tc; tc++) {
2705 		err = mlx5e_open_tis(priv, tc);
2706 		if (err)
2707 			goto err_close_tises;
2708 	}
2709 
2710 	return (0);
2711 
2712 err_close_tises:
2713 	for (tc--; tc >= 0; tc--)
2714 		mlx5e_close_tis(priv, tc);
2715 
2716 	return (err);
2717 }
2718 
2719 static void
mlx5e_close_tises(struct mlx5e_priv * priv)2720 mlx5e_close_tises(struct mlx5e_priv *priv)
2721 {
2722 	int num_tc = priv->num_tc;
2723 	int tc;
2724 
2725 	for (tc = 0; tc < num_tc; tc++)
2726 		mlx5e_close_tis(priv, tc);
2727 }
2728 
2729 static int
mlx5e_open_default_rqt(struct mlx5e_priv * priv,u32 * prqtn,int sz)2730 mlx5e_open_default_rqt(struct mlx5e_priv *priv, u32 *prqtn, int sz)
2731 {
2732 	u32 *in;
2733 	void *rqtc;
2734 	int inlen;
2735 	int err;
2736 	int i;
2737 
2738 	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2739 	in = mlx5_vzalloc(inlen);
2740 	if (in == NULL)
2741 		return (-ENOMEM);
2742 	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2743 
2744 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2745 	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2746 
2747 	for (i = 0; i != sz; i++)
2748 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2749 
2750 	err = mlx5_core_create_rqt(priv->mdev, in, inlen, prqtn);
2751 	kvfree(in);
2752 
2753 	return (err);
2754 }
2755 
2756 static int
mlx5e_open_rqts(struct mlx5e_priv * priv)2757 mlx5e_open_rqts(struct mlx5e_priv *priv)
2758 {
2759 	int err;
2760 	int i;
2761 
2762 	err = mlx5e_open_default_rqt(priv, &priv->rqtn,
2763 	    1 << priv->params.rx_hash_log_tbl_sz);
2764 	if (err)
2765 		goto err_default;
2766 
2767 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2768 		err = mlx5e_open_default_rqt(priv, &priv->channel[i].rqtn, 1);
2769 		if (err)
2770 			goto err_channel;
2771 	}
2772 	return (0);
2773 
2774 err_channel:
2775 	while (i--)
2776 		mlx5_core_destroy_rqt(priv->mdev, priv->channel[i].rqtn, 0);
2777 
2778 	mlx5_core_destroy_rqt(priv->mdev, priv->rqtn, 0);
2779 
2780 err_default:
2781 	return (err);
2782 }
2783 
2784 static void
mlx5e_close_rqts(struct mlx5e_priv * priv)2785 mlx5e_close_rqts(struct mlx5e_priv *priv)
2786 {
2787 	int i;
2788 
2789 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++)
2790 		mlx5_core_destroy_rqt(priv->mdev, priv->channel[i].rqtn, 0);
2791 
2792 	mlx5_core_destroy_rqt(priv->mdev, priv->rqtn, 0);
2793 }
2794 
2795 static int
mlx5e_activate_rqt(struct mlx5e_priv * priv)2796 mlx5e_activate_rqt(struct mlx5e_priv *priv)
2797 {
2798 	u32 *in;
2799 	void *rqtc;
2800 	int inlen;
2801 	int err;
2802 	int sz;
2803 	int i;
2804 
2805 	sz = 1 << priv->params.rx_hash_log_tbl_sz;
2806 
2807 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2808 	in = mlx5_vzalloc(inlen);
2809 	if (in == NULL)
2810 		return (-ENOMEM);
2811 
2812 	rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2813 
2814 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2815 	MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2816 
2817 	for (i = 0; i != sz; i++) {
2818 		int ix;
2819 #ifdef RSS
2820 		ix = rss_get_indirection_to_bucket(i);
2821 #else
2822 		ix = i;
2823 #endif
2824 		/* ensure we don't overflow */
2825 		ix %= priv->params.num_channels;
2826 
2827 		/* apply receive side scaling stride, if any */
2828 		ix -= ix % (int)priv->params.channels_rsss;
2829 
2830 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->channel[ix].rq.rqn);
2831 	}
2832 
2833 	err = mlx5_core_modify_rqt(priv->mdev, priv->rqtn, in, inlen);
2834 	if (err)
2835 		goto err_modify;
2836 
2837 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32);
2838 
2839 	MLX5_SET(rqtc, rqtc, rqt_actual_size, 1);
2840 
2841 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2842 		int ix;
2843 #ifdef RSS
2844 		ix = rss_get_indirection_to_bucket(i);
2845 #else
2846 		ix = i;
2847 #endif
2848 		/* ensure we don't overflow */
2849 		ix %= priv->params.num_channels;
2850 
2851 		/* apply receive side scaling stride, if any */
2852 		ix -= ix % (int)priv->params.channels_rsss;
2853 
2854 		MLX5_SET(rqtc, rqtc, rq_num[0], priv->channel[ix].rq.rqn);
2855 
2856 		err = mlx5_core_modify_rqt(priv->mdev, priv->channel[i].rqtn, in, inlen);
2857 		if (err)
2858 			goto err_modify;
2859 	}
2860 
2861 err_modify:
2862 	kvfree(in);
2863 	return (err);
2864 }
2865 
2866 static int
mlx5e_deactivate_rqt(struct mlx5e_priv * priv)2867 mlx5e_deactivate_rqt(struct mlx5e_priv *priv)
2868 {
2869 	u32 *in;
2870 	void *rqtc;
2871 	int inlen;
2872 	int err;
2873 	int sz;
2874 	int i;
2875 
2876 	sz = 1 << priv->params.rx_hash_log_tbl_sz;
2877 
2878 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2879 	in = mlx5_vzalloc(inlen);
2880 	if (in == NULL)
2881 		return (-ENOMEM);
2882 
2883 	rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2884 
2885 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2886 	MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2887 
2888 	for (i = 0; i != sz; i++)
2889 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2890 
2891 	err = mlx5_core_modify_rqt(priv->mdev, priv->rqtn, in, inlen);
2892 	if (err)
2893 		goto err_modify;
2894 
2895 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32);
2896 
2897 	MLX5_SET(rqtc, rqtc, rqt_actual_size, 1);
2898 
2899 	for (i = 0; i != priv->mdev->priv.eq_table.num_comp_vectors; i++) {
2900 		MLX5_SET(rqtc, rqtc, rq_num[0], priv->drop_rq.rqn);
2901 
2902 		err = mlx5_core_modify_rqt(priv->mdev, priv->channel[i].rqtn, in, inlen);
2903 		if (err)
2904 			goto err_modify;
2905 	}
2906 
2907 err_modify:
2908 	kvfree(in);
2909 	return (err);
2910 }
2911 
2912 #define	MLX5E_RSS_KEY_SIZE (10 * 4)	/* bytes */
2913 
2914 static void
mlx5e_get_rss_key(void * key_ptr)2915 mlx5e_get_rss_key(void *key_ptr)
2916 {
2917 #ifdef RSS
2918 	rss_getkey(key_ptr);
2919 #else
2920 	static const u32 rsskey[] = {
2921 	    cpu_to_be32(0xD181C62C),
2922 	    cpu_to_be32(0xF7F4DB5B),
2923 	    cpu_to_be32(0x1983A2FC),
2924 	    cpu_to_be32(0x943E1ADB),
2925 	    cpu_to_be32(0xD9389E6B),
2926 	    cpu_to_be32(0xD1039C2C),
2927 	    cpu_to_be32(0xA74499AD),
2928 	    cpu_to_be32(0x593D56D9),
2929 	    cpu_to_be32(0xF3253C06),
2930 	    cpu_to_be32(0x2ADC1FFC),
2931 	};
2932 	CTASSERT(sizeof(rsskey) == MLX5E_RSS_KEY_SIZE);
2933 	memcpy(key_ptr, rsskey, MLX5E_RSS_KEY_SIZE);
2934 #endif
2935 }
2936 
2937 static void
mlx5e_hw_lro_set_tir_ctx_lro_max_msg_sz(struct mlx5e_priv * priv,u32 * tirc)2938 mlx5e_hw_lro_set_tir_ctx_lro_max_msg_sz(struct mlx5e_priv *priv, u32 *tirc)
2939 {
2940 	MLX5_SET(tirc, tirc, lro_max_msg_sz, (priv->params.lro_wqe_sz >> 8) -
2941 	    (MLX5_CAP_ETH(priv->mdev, lro_max_msg_sz_mode) == 0 ? 1 : 0));
2942 }
2943 
2944 static void
mlx5e_hw_lro_set_tir_ctx(struct mlx5e_priv * priv,u32 * tirc)2945 mlx5e_hw_lro_set_tir_ctx(struct mlx5e_priv *priv, u32 *tirc)
2946 {
2947 	MLX5_SET(tirc, tirc, lro_enable_mask,
2948 	    MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2949 	    MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2950 	/* TODO: add the option to choose timer value dynamically */
2951 	MLX5_SET(tirc, tirc, lro_timeout_period_usecs,
2952 	    MLX5_CAP_ETH(priv->mdev, lro_timer_supported_periods[2]));
2953 	mlx5e_hw_lro_set_tir_ctx_lro_max_msg_sz(priv, tirc);
2954 }
2955 
2956 static int
mlx5e_hw_lro_update_tir(struct mlx5e_priv * priv,int tt,bool inner_vxlan)2957 mlx5e_hw_lro_update_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
2958 {
2959 	struct mlx5_core_dev *mdev = priv->mdev;
2960 	u32 *in;
2961 	void *tirc;
2962 	int inlen;
2963 	int err;
2964 
2965 	inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
2966 	in = mlx5_vzalloc(inlen);
2967 	if (in == NULL)
2968 		return (-ENOMEM);
2969 	tirc = MLX5_ADDR_OF(modify_tir_in, in, tir_context);
2970 
2971 	/* fill the command part */
2972 	MLX5_SET(modify_tir_in, in, tirn, inner_vxlan ?
2973 	    priv->tirn_inner_vxlan[tt] : priv->tirn[tt]);
2974 	MLX5_SET64(modify_tir_in, in, modify_bitmask,
2975 	    (1 << MLX5_MODIFY_TIR_BITMASK_LRO));
2976 
2977 	/* fill the context */
2978 	if (priv->params.hw_lro_en)
2979 		mlx5e_hw_lro_set_tir_ctx(priv, tirc);
2980 
2981 	err = mlx5_core_modify_tir(mdev, in, inlen);
2982 
2983 	kvfree(in);
2984 	return (err);
2985 }
2986 
2987 int
mlx5e_hw_lro_update_tirs(struct mlx5e_priv * priv)2988 mlx5e_hw_lro_update_tirs(struct mlx5e_priv *priv)
2989 {
2990 	int err, err1, i;
2991 
2992 	err = 0;
2993 	for (i = 0; i != 2 * MLX5E_NUM_TT; i++) {
2994 		err1 = mlx5e_hw_lro_update_tir(priv, i / 2, (i % 2) ? true :
2995 		    false);
2996 		if (err1 != 0 && err == 0)
2997 			err = err1;
2998 	}
2999 	return (-err);
3000 }
3001 
3002 static void
mlx5e_build_tir_ctx(struct mlx5e_priv * priv,u32 * tirc,int tt,bool inner_vxlan)3003 mlx5e_build_tir_ctx(struct mlx5e_priv *priv, u32 * tirc, int tt, bool inner_vxlan)
3004 {
3005 	void *hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
3006 	void *hfsi = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
3007 	void *hfs = inner_vxlan ? hfsi : hfso;
3008 	__be32 *hkey;
3009 
3010 	MLX5_SET(tirc, tirc, transport_domain, priv->tdn);
3011 
3012 #define	MLX5_HASH_IP     (MLX5_HASH_FIELD_SEL_SRC_IP   |\
3013 			  MLX5_HASH_FIELD_SEL_DST_IP)
3014 
3015 #define	MLX5_HASH_ALL    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
3016 			  MLX5_HASH_FIELD_SEL_DST_IP   |\
3017 			  MLX5_HASH_FIELD_SEL_L4_SPORT |\
3018 			  MLX5_HASH_FIELD_SEL_L4_DPORT)
3019 
3020 #define	MLX5_HASH_IP_IPSEC_SPI	(MLX5_HASH_FIELD_SEL_SRC_IP   |\
3021 				 MLX5_HASH_FIELD_SEL_DST_IP   |\
3022 				 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
3023 
3024 	if (priv->params.hw_lro_en)
3025 		mlx5e_hw_lro_set_tir_ctx(priv, tirc);
3026 
3027 	if (inner_vxlan)
3028 		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
3029 
3030 	/*
3031 	 * All packets must go through the indirection table, RQT,
3032 	 * because it is not possible to modify the RQN of the TIR
3033 	 * for direct dispatchment after it is created, typically
3034 	 * when the link goes up and down.
3035 	 */
3036 	MLX5_SET(tirc, tirc, disp_type,
3037 	    MLX5_TIRC_DISP_TYPE_INDIRECT);
3038 	MLX5_SET(tirc, tirc, indirect_table,
3039 	    priv->rqtn);
3040 	MLX5_SET(tirc, tirc, rx_hash_fn,
3041 		 MLX5_TIRC_RX_HASH_FN_HASH_TOEPLITZ);
3042 	hkey = (__be32 *) MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
3043 
3044 	CTASSERT(MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key) >=
3045 		 MLX5E_RSS_KEY_SIZE);
3046 #ifdef RSS
3047 	/*
3048 	 * The FreeBSD RSS implementation does currently not
3049 	 * support symmetric Toeplitz hashes:
3050 	 */
3051 	MLX5_SET(tirc, tirc, rx_hash_symmetric, 0);
3052 #else
3053 	MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
3054 #endif
3055 	mlx5e_get_rss_key(hkey);
3056 
3057 	switch (tt) {
3058 	case MLX5E_TT_IPV4_TCP:
3059 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3060 		    MLX5_L3_PROT_TYPE_IPV4);
3061 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3062 		    MLX5_L4_PROT_TYPE_TCP);
3063 #ifdef RSS
3064 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV4)) {
3065 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3066 			    MLX5_HASH_IP);
3067 		} else
3068 #endif
3069 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3070 		    MLX5_HASH_ALL);
3071 		break;
3072 
3073 	case MLX5E_TT_IPV6_TCP:
3074 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3075 		    MLX5_L3_PROT_TYPE_IPV6);
3076 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3077 		    MLX5_L4_PROT_TYPE_TCP);
3078 #ifdef RSS
3079 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_TCP_IPV6)) {
3080 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3081 			    MLX5_HASH_IP);
3082 		} else
3083 #endif
3084 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3085 		    MLX5_HASH_ALL);
3086 		break;
3087 
3088 	case MLX5E_TT_IPV4_UDP:
3089 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3090 		    MLX5_L3_PROT_TYPE_IPV4);
3091 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3092 		    MLX5_L4_PROT_TYPE_UDP);
3093 #ifdef RSS
3094 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV4)) {
3095 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3096 			    MLX5_HASH_IP);
3097 		} else
3098 #endif
3099 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3100 		    MLX5_HASH_ALL);
3101 		break;
3102 
3103 	case MLX5E_TT_IPV6_UDP:
3104 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3105 		    MLX5_L3_PROT_TYPE_IPV6);
3106 		MLX5_SET(rx_hash_field_select, hfs, l4_prot_type,
3107 		    MLX5_L4_PROT_TYPE_UDP);
3108 #ifdef RSS
3109 		if (!(rss_gethashconfig() & RSS_HASHTYPE_RSS_UDP_IPV6)) {
3110 			MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3111 			    MLX5_HASH_IP);
3112 		} else
3113 #endif
3114 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3115 		    MLX5_HASH_ALL);
3116 		break;
3117 
3118 	case MLX5E_TT_IPV4_IPSEC_AH:
3119 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3120 		    MLX5_L3_PROT_TYPE_IPV4);
3121 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3122 		    MLX5_HASH_IP_IPSEC_SPI);
3123 		break;
3124 
3125 	case MLX5E_TT_IPV6_IPSEC_AH:
3126 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3127 		    MLX5_L3_PROT_TYPE_IPV6);
3128 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3129 		    MLX5_HASH_IP_IPSEC_SPI);
3130 		break;
3131 
3132 	case MLX5E_TT_IPV4_IPSEC_ESP:
3133 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3134 		    MLX5_L3_PROT_TYPE_IPV4);
3135 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3136 		    MLX5_HASH_IP_IPSEC_SPI);
3137 		break;
3138 
3139 	case MLX5E_TT_IPV6_IPSEC_ESP:
3140 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3141 		    MLX5_L3_PROT_TYPE_IPV6);
3142 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3143 		    MLX5_HASH_IP_IPSEC_SPI);
3144 		break;
3145 
3146 	case MLX5E_TT_IPV4:
3147 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3148 		    MLX5_L3_PROT_TYPE_IPV4);
3149 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3150 		    MLX5_HASH_IP);
3151 		break;
3152 
3153 	case MLX5E_TT_IPV6:
3154 		MLX5_SET(rx_hash_field_select, hfs, l3_prot_type,
3155 		    MLX5_L3_PROT_TYPE_IPV6);
3156 		MLX5_SET(rx_hash_field_select, hfs, selected_fields,
3157 		    MLX5_HASH_IP);
3158 		break;
3159 
3160 	default:
3161 		break;
3162 	}
3163 }
3164 
3165 static int
mlx5e_open_tir(struct mlx5e_priv * priv,int tt,bool inner_vxlan)3166 mlx5e_open_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
3167 {
3168 	struct mlx5_core_dev *mdev = priv->mdev;
3169 	u32 *in;
3170 	void *tirc;
3171 	int inlen;
3172 	int err;
3173 
3174 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3175 	in = mlx5_vzalloc(inlen);
3176 	if (in == NULL)
3177 		return (-ENOMEM);
3178 	tirc = MLX5_ADDR_OF(create_tir_in, in, tir_context);
3179 
3180 	mlx5e_build_tir_ctx(priv, tirc, tt, inner_vxlan);
3181 
3182 	err = mlx5_core_create_tir(mdev, in, inlen, inner_vxlan ?
3183 	    &priv->tirn_inner_vxlan[tt] : &priv->tirn[tt]);
3184 
3185 	kvfree(in);
3186 
3187 	return (err);
3188 }
3189 
3190 static void
mlx5e_close_tir(struct mlx5e_priv * priv,int tt,bool inner_vxlan)3191 mlx5e_close_tir(struct mlx5e_priv *priv, int tt, bool inner_vxlan)
3192 {
3193 	mlx5_core_destroy_tir(priv->mdev, inner_vxlan ?
3194 	    priv->tirn_inner_vxlan[tt] : priv->tirn[tt], 0);
3195 }
3196 
3197 static int
mlx5e_open_tirs(struct mlx5e_priv * priv)3198 mlx5e_open_tirs(struct mlx5e_priv *priv)
3199 {
3200 	int err;
3201 	int i;
3202 
3203 	for (i = 0; i != 2 * MLX5E_NUM_TT; i++) {
3204 		err = mlx5e_open_tir(priv, i / 2, (i % 2) ? true : false);
3205 		if (err)
3206 			goto err_close_tirs;
3207 	}
3208 
3209 	return (0);
3210 
3211 err_close_tirs:
3212 	for (i--; i >= 0; i--)
3213 		mlx5e_close_tir(priv, i / 2, (i % 2) ? true : false);
3214 
3215 	return (err);
3216 }
3217 
3218 static void
mlx5e_close_tirs(struct mlx5e_priv * priv)3219 mlx5e_close_tirs(struct mlx5e_priv *priv)
3220 {
3221 	int i;
3222 
3223 	for (i = 0; i != 2 * MLX5E_NUM_TT; i++)
3224 		mlx5e_close_tir(priv, i / 2, (i % 2) ? true : false);
3225 }
3226 
3227 /*
3228  * SW MTU does not include headers,
3229  * HW MTU includes all headers and checksums.
3230  */
3231 static int
mlx5e_set_dev_port_mtu(if_t ifp,int sw_mtu)3232 mlx5e_set_dev_port_mtu(if_t ifp, int sw_mtu)
3233 {
3234 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3235 	struct mlx5_core_dev *mdev = priv->mdev;
3236 	int hw_mtu;
3237 	int err;
3238 
3239 	hw_mtu = MLX5E_SW2HW_MTU(sw_mtu);
3240 
3241 	err = mlx5_set_port_mtu(mdev, hw_mtu);
3242 	if (err) {
3243 		mlx5_en_err(ifp, "mlx5_set_port_mtu failed setting %d, err=%d\n",
3244 		    sw_mtu, err);
3245 		return (err);
3246 	}
3247 
3248 	/* Update vport context MTU */
3249 	err = mlx5_set_vport_mtu(mdev, hw_mtu);
3250 	if (err) {
3251 		mlx5_en_err(ifp,
3252 		    "Failed updating vport context with MTU size, err=%d\n",
3253 		    err);
3254 	}
3255 
3256 	if_setmtu(ifp, sw_mtu);
3257 
3258 	err = mlx5_query_vport_mtu(mdev, &hw_mtu);
3259 	if (err || !hw_mtu) {
3260 		/* fallback to port oper mtu */
3261 		err = mlx5_query_port_oper_mtu(mdev, &hw_mtu);
3262 	}
3263 	if (err) {
3264 		mlx5_en_err(ifp,
3265 		    "Query port MTU, after setting new MTU value, failed\n");
3266 		return (err);
3267 	} else if (MLX5E_HW2SW_MTU(hw_mtu) < sw_mtu) {
3268 		err = -E2BIG,
3269 		mlx5_en_err(ifp,
3270 		    "Port MTU %d is smaller than ifp mtu %d\n",
3271 		    hw_mtu, sw_mtu);
3272 	} else if (MLX5E_HW2SW_MTU(hw_mtu) > sw_mtu) {
3273 		err = -EINVAL;
3274                 mlx5_en_err(ifp,
3275 		    "Port MTU %d is bigger than ifp mtu %d\n",
3276 		    hw_mtu, sw_mtu);
3277 	}
3278 	priv->params_ethtool.hw_mtu = hw_mtu;
3279 
3280 	/* compute MSB */
3281 	while (hw_mtu & (hw_mtu - 1))
3282 		hw_mtu &= (hw_mtu - 1);
3283 	priv->params_ethtool.hw_mtu_msb = hw_mtu;
3284 
3285 	return (err);
3286 }
3287 
3288 int
mlx5e_open_locked(if_t ifp)3289 mlx5e_open_locked(if_t ifp)
3290 {
3291 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3292 	int err;
3293 	u16 set_id;
3294 
3295 	/* check if already opened */
3296 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3297 		return (0);
3298 
3299 #ifdef RSS
3300 	if (rss_getnumbuckets() > priv->params.num_channels) {
3301 		mlx5_en_info(ifp,
3302 		    "NOTE: There are more RSS buckets(%u) than channels(%u) available\n",
3303 		    rss_getnumbuckets(), priv->params.num_channels);
3304 	}
3305 #endif
3306 	err = mlx5e_open_tises(priv);
3307 	if (err) {
3308 		mlx5_en_err(ifp, "mlx5e_open_tises failed, %d\n", err);
3309 		return (err);
3310 	}
3311 	err = mlx5_vport_alloc_q_counter(priv->mdev,
3312 	    MLX5_INTERFACE_PROTOCOL_ETH, &set_id);
3313 	if (err) {
3314 		mlx5_en_err(priv->ifp,
3315 		    "mlx5_vport_alloc_q_counter failed: %d\n", err);
3316 		goto err_close_tises;
3317 	}
3318 	/* store counter set ID */
3319 	priv->counter_set_id = set_id;
3320 
3321 	err = mlx5e_open_channels(priv);
3322 	if (err) {
3323 		mlx5_en_err(ifp,
3324 		    "mlx5e_open_channels failed, %d\n", err);
3325 		goto err_dalloc_q_counter;
3326 	}
3327 	err = mlx5e_activate_rqt(priv);
3328 	if (err) {
3329 		mlx5_en_err(ifp, "mlx5e_activate_rqt failed, %d\n", err);
3330 		goto err_close_channels;
3331 	}
3332 
3333 	set_bit(MLX5E_STATE_OPENED, &priv->state);
3334 
3335 	mlx5e_update_carrier(priv);
3336 
3337 	return (0);
3338 
3339 err_close_channels:
3340 	mlx5e_close_channels(priv);
3341 
3342 err_dalloc_q_counter:
3343 	mlx5_vport_dealloc_q_counter(priv->mdev,
3344 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3345 
3346 err_close_tises:
3347 	mlx5e_close_tises(priv);
3348 
3349 	return (err);
3350 }
3351 
3352 static void
mlx5e_open(void * arg)3353 mlx5e_open(void *arg)
3354 {
3355 	struct mlx5e_priv *priv = arg;
3356 
3357 	PRIV_LOCK(priv);
3358 	if (mlx5_set_port_status(priv->mdev, MLX5_PORT_UP))
3359 		mlx5_en_err(priv->ifp,
3360 		    "Setting port status to up failed\n");
3361 
3362 	mlx5e_open_locked(priv->ifp);
3363 	if_setdrvflagbits(priv->ifp, IFF_DRV_RUNNING, 0);
3364 	PRIV_UNLOCK(priv);
3365 }
3366 
3367 int
mlx5e_close_locked(if_t ifp)3368 mlx5e_close_locked(if_t ifp)
3369 {
3370 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3371 
3372 	/* check if already closed */
3373 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3374 		return (0);
3375 
3376 	clear_bit(MLX5E_STATE_OPENED, &priv->state);
3377 
3378 	if_link_state_change(priv->ifp, LINK_STATE_DOWN);
3379 
3380 	mlx5e_deactivate_rqt(priv);
3381 	mlx5e_close_channels(priv);
3382 	mlx5_vport_dealloc_q_counter(priv->mdev,
3383 	    MLX5_INTERFACE_PROTOCOL_ETH, priv->counter_set_id);
3384 	mlx5e_close_tises(priv);
3385 
3386 	return (0);
3387 }
3388 
3389 static uint64_t
mlx5e_get_counter(if_t ifp,ift_counter cnt)3390 mlx5e_get_counter(if_t ifp, ift_counter cnt)
3391 {
3392 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3393 	u64 retval;
3394 
3395 	/* PRIV_LOCK(priv); XXX not allowed */
3396 	switch (cnt) {
3397 	case IFCOUNTER_IPACKETS:
3398 		retval = priv->stats.vport.rx_packets;
3399 		break;
3400 	case IFCOUNTER_IERRORS:
3401 		retval = priv->stats.pport.in_range_len_errors +
3402 		    priv->stats.pport.out_of_range_len +
3403 		    priv->stats.pport.too_long_errors +
3404 		    priv->stats.pport.check_seq_err +
3405 		    priv->stats.pport.alignment_err;
3406 		break;
3407 	case IFCOUNTER_IQDROPS:
3408 		retval = priv->stats.vport.rx_out_of_buffer;
3409 		break;
3410 	case IFCOUNTER_OPACKETS:
3411 		retval = priv->stats.vport.tx_packets;
3412 		break;
3413 	case IFCOUNTER_OERRORS:
3414 		retval = priv->stats.port_stats_debug.out_discards;
3415 		break;
3416 	case IFCOUNTER_IBYTES:
3417 		retval = priv->stats.vport.rx_bytes;
3418 		break;
3419 	case IFCOUNTER_OBYTES:
3420 		retval = priv->stats.vport.tx_bytes;
3421 		break;
3422 	case IFCOUNTER_IMCASTS:
3423 		retval = priv->stats.vport.rx_multicast_packets;
3424 		break;
3425 	case IFCOUNTER_OMCASTS:
3426 		retval = priv->stats.vport.tx_multicast_packets;
3427 		break;
3428 	case IFCOUNTER_OQDROPS:
3429 		retval = priv->stats.vport.tx_queue_dropped;
3430 		break;
3431 	case IFCOUNTER_COLLISIONS:
3432 		retval = priv->stats.pport.collisions;
3433 		break;
3434 	default:
3435 		retval = if_get_counter_default(ifp, cnt);
3436 		break;
3437 	}
3438 	/* PRIV_UNLOCK(priv); XXX not allowed */
3439 	return (retval);
3440 }
3441 
3442 static void
mlx5e_set_rx_mode(if_t ifp)3443 mlx5e_set_rx_mode(if_t ifp)
3444 {
3445 	struct mlx5e_priv *priv = if_getsoftc(ifp);
3446 
3447 	queue_work(priv->wq, &priv->set_rx_mode_work);
3448 }
3449 
3450 static bool
mlx5e_is_ipsec_capable(struct mlx5_core_dev * mdev)3451 mlx5e_is_ipsec_capable(struct mlx5_core_dev *mdev)
3452 {
3453 #ifdef IPSEC_OFFLOAD
3454 	if ((mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PACKET_OFFLOAD) != 0)
3455 		return (true);
3456 #endif
3457 	return (false);
3458 }
3459 
3460 static bool
mlx5e_is_ratelimit_capable(struct mlx5_core_dev * mdev)3461 mlx5e_is_ratelimit_capable(struct mlx5_core_dev *mdev)
3462 {
3463 #ifdef RATELIMIT
3464 	if (MLX5_CAP_GEN(mdev, qos) &&
3465 	    MLX5_CAP_QOS(mdev, packet_pacing))
3466 		return (true);
3467 #endif
3468 	return (false);
3469 }
3470 
3471 static bool
mlx5e_is_tlstx_capable(struct mlx5_core_dev * mdev)3472 mlx5e_is_tlstx_capable(struct mlx5_core_dev *mdev)
3473 {
3474 #ifdef KERN_TLS
3475 	if (MLX5_CAP_GEN(mdev, tls_tx) != 0 &&
3476 	    MLX5_CAP_GEN(mdev, log_max_dek) != 0)
3477 		return (true);
3478 #endif
3479 	return (false);
3480 }
3481 
3482 static bool
mlx5e_is_tlsrx_capable(struct mlx5_core_dev * mdev)3483 mlx5e_is_tlsrx_capable(struct mlx5_core_dev *mdev)
3484 {
3485 #ifdef KERN_TLS
3486 	if (MLX5_CAP_GEN(mdev, tls_rx) != 0 &&
3487 	    MLX5_CAP_GEN(mdev, log_max_dek) != 0 &&
3488 	    MLX5_CAP_FLOWTABLE_NIC_RX(mdev,
3489 	    ft_field_support.outer_ip_version) != 0)
3490 		return (true);
3491 #endif
3492 	return (false);
3493 }
3494 
3495 static int
mlx5e_ioctl(if_t ifp,u_long command,caddr_t data)3496 mlx5e_ioctl(if_t ifp, u_long command, caddr_t data)
3497 {
3498 	struct mlx5e_priv *priv;
3499 	struct ifreq *ifr;
3500 	struct ifdownreason *ifdr;
3501 	struct ifi2creq i2c;
3502 	struct ifrsskey *ifrk;
3503 	struct ifrsshash *ifrh;
3504 	struct siocsifcapnv_driver_data *drv_ioctl_data, drv_ioctl_data_d;
3505 	int error = 0;
3506 	int mask;
3507 	int size_read = 0;
3508 	int module_status;
3509 	int module_num;
3510 	int max_mtu;
3511 	uint8_t read_addr;
3512 
3513 	priv = if_getsoftc(ifp);
3514 
3515 	/* check if detaching */
3516 	if (priv == NULL || priv->gone != 0)
3517 		return (ENXIO);
3518 
3519 	switch (command) {
3520 	case SIOCSIFMTU:
3521 		ifr = (struct ifreq *)data;
3522 
3523 		PRIV_LOCK(priv);
3524 		mlx5_query_port_max_mtu(priv->mdev, &max_mtu);
3525 
3526 		if (ifr->ifr_mtu >= MLX5E_MTU_MIN &&
3527 		    ifr->ifr_mtu <= MIN(MLX5E_MTU_MAX, max_mtu)) {
3528 			int was_opened;
3529 
3530 			was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
3531 			if (was_opened)
3532 				mlx5e_close_locked(ifp);
3533 
3534 			/* set new MTU */
3535 			mlx5e_set_dev_port_mtu(ifp, ifr->ifr_mtu);
3536 
3537 			if (was_opened)
3538 				mlx5e_open_locked(ifp);
3539 		} else {
3540 			error = EINVAL;
3541 			mlx5_en_err(ifp,
3542 			    "Invalid MTU value. Min val: %d, Max val: %d\n",
3543 			    MLX5E_MTU_MIN, MIN(MLX5E_MTU_MAX, max_mtu));
3544 		}
3545 		PRIV_UNLOCK(priv);
3546 		break;
3547 	case SIOCSIFFLAGS:
3548 		if ((if_getflags(ifp) & IFF_UP) &&
3549 		    (if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
3550 			mlx5e_set_rx_mode(ifp);
3551 			break;
3552 		}
3553 		PRIV_LOCK(priv);
3554 		if (if_getflags(ifp) & IFF_UP) {
3555 			if ((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) {
3556 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
3557 					mlx5e_open_locked(ifp);
3558 				if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
3559 				mlx5_set_port_status(priv->mdev, MLX5_PORT_UP);
3560 			}
3561 		} else {
3562 			if (if_getdrvflags(ifp) & IFF_DRV_RUNNING) {
3563 				mlx5_set_port_status(priv->mdev,
3564 				    MLX5_PORT_DOWN);
3565 				if (test_bit(MLX5E_STATE_OPENED, &priv->state) != 0)
3566 					mlx5e_close_locked(ifp);
3567 				mlx5e_update_carrier(priv);
3568 				if_setdrvflagbits(ifp, 0, IFF_DRV_RUNNING);
3569 			}
3570 		}
3571 		PRIV_UNLOCK(priv);
3572 		break;
3573 	case SIOCADDMULTI:
3574 	case SIOCDELMULTI:
3575 		mlx5e_set_rx_mode(ifp);
3576 		break;
3577 	case SIOCSIFMEDIA:
3578 	case SIOCGIFMEDIA:
3579 	case SIOCGIFXMEDIA:
3580 		ifr = (struct ifreq *)data;
3581 		error = ifmedia_ioctl(ifp, ifr, &priv->media, command);
3582 		break;
3583 	case SIOCGIFCAPNV:
3584 		error = 0;
3585 		break;
3586 	case SIOCSIFCAP:
3587 		ifr = (struct ifreq *)data;
3588 		drv_ioctl_data = &drv_ioctl_data_d;
3589 		drv_ioctl_data->reqcap = ifr->ifr_reqcap;
3590 		PRIV_LOCK(priv);
3591 		drv_ioctl_data->reqcap2 = if_getcapenable2(ifp);
3592 		drv_ioctl_data->nvcap = NULL;
3593 		goto siocsifcap_driver;
3594 	case SIOCSIFCAPNV:
3595 		drv_ioctl_data = (struct siocsifcapnv_driver_data *)data;
3596 		PRIV_LOCK(priv);
3597 siocsifcap_driver:
3598 		if (!mlx5e_is_tlstx_capable(priv->mdev)) {
3599 			drv_ioctl_data->reqcap &= ~(IFCAP_TXTLS4 |
3600 			    IFCAP_TXTLS6);
3601 		}
3602 		if (!mlx5e_is_tlsrx_capable(priv->mdev)) {
3603 		        drv_ioctl_data->reqcap2 &= ~(
3604 			    IFCAP2_BIT(IFCAP2_RXTLS4) |
3605 			    IFCAP2_BIT(IFCAP2_RXTLS6));
3606 		}
3607 		if (!mlx5e_is_ipsec_capable(priv->mdev)) {
3608 			drv_ioctl_data->reqcap2 &=
3609 			    ~IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD);
3610 		}
3611 		if (!mlx5e_is_ratelimit_capable(priv->mdev)) {
3612 			drv_ioctl_data->reqcap &= ~(IFCAP_TXTLS_RTLMT |
3613 			    IFCAP_TXRTLMT);
3614 		}
3615 
3616 		mask = drv_ioctl_data->reqcap ^ if_getcapenable(ifp);
3617 
3618 		if (mask & IFCAP_TXCSUM) {
3619 			if_togglecapenable(ifp, IFCAP_TXCSUM);
3620 			if_togglehwassist(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP));
3621 
3622 			if (IFCAP_TSO4 & if_getcapenable(ifp) &&
3623 			    !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
3624 				mask &= ~IFCAP_TSO4;
3625 				if_setcapenablebit(ifp, 0, IFCAP_TSO4);
3626 				if_sethwassistbits(ifp, 0, CSUM_IP_TSO);
3627 				mlx5_en_err(ifp,
3628 				    "tso4 disabled due to -txcsum.\n");
3629 			}
3630 		}
3631 		if (mask & IFCAP_TXCSUM_IPV6) {
3632 			if_togglecapenable(ifp, IFCAP_TXCSUM_IPV6);
3633 			if_togglehwassist(ifp, (CSUM_UDP_IPV6 | CSUM_TCP_IPV6));
3634 
3635 			if (IFCAP_TSO6 & if_getcapenable(ifp) &&
3636 			    !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
3637 				mask &= ~IFCAP_TSO6;
3638 				if_setcapenablebit(ifp, 0, IFCAP_TSO6);
3639 				if_sethwassistbits(ifp, 0, CSUM_IP6_TSO);
3640 				mlx5_en_err(ifp,
3641 				    "tso6 disabled due to -txcsum6.\n");
3642 			}
3643 		}
3644 		if (mask & IFCAP_MEXTPG)
3645 			if_togglecapenable(ifp, IFCAP_MEXTPG);
3646 		if (mask & IFCAP_TXTLS4)
3647 			if_togglecapenable(ifp, IFCAP_TXTLS4);
3648 		if (mask & IFCAP_TXTLS6)
3649 			if_togglecapenable(ifp, IFCAP_TXTLS6);
3650 #ifdef RATELIMIT
3651 		if (mask & IFCAP_TXTLS_RTLMT)
3652 			if_togglecapenable(ifp, IFCAP_TXTLS_RTLMT);
3653 #endif
3654 		if (mask & IFCAP_RXCSUM)
3655 			if_togglecapenable(ifp, IFCAP_RXCSUM);
3656 		if (mask & IFCAP_RXCSUM_IPV6)
3657 			if_togglecapenable(ifp, IFCAP_RXCSUM_IPV6);
3658 		if (mask & IFCAP_TSO4) {
3659 			if (!(IFCAP_TSO4 & if_getcapenable(ifp)) &&
3660 			    !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
3661 				mlx5_en_err(ifp, "enable txcsum first.\n");
3662 				error = EAGAIN;
3663 				goto out;
3664 			}
3665 			if_togglecapenable(ifp, IFCAP_TSO4);
3666 			if_togglehwassist(ifp, CSUM_IP_TSO);
3667 		}
3668 		if (mask & IFCAP_TSO6) {
3669 			if (!(IFCAP_TSO6 & if_getcapenable(ifp)) &&
3670 			    !(IFCAP_TXCSUM_IPV6 & if_getcapenable(ifp))) {
3671 				mlx5_en_err(ifp, "enable txcsum6 first.\n");
3672 				error = EAGAIN;
3673 				goto out;
3674 			}
3675 			if_togglecapenable(ifp, IFCAP_TSO6);
3676 			if_togglehwassist(ifp, CSUM_IP6_TSO);
3677 		}
3678 		if (mask & IFCAP_VLAN_HWTSO)
3679 			if_togglecapenable(ifp, IFCAP_VLAN_HWTSO);
3680 		if (mask & IFCAP_VLAN_HWFILTER) {
3681 			if (if_getcapenable(ifp) & IFCAP_VLAN_HWFILTER)
3682 				mlx5e_disable_vlan_filter(priv);
3683 			else
3684 				mlx5e_enable_vlan_filter(priv);
3685 
3686 			if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
3687 		}
3688 		if (mask & IFCAP_VLAN_HWTAGGING)
3689 			if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
3690 		if (mask & IFCAP_WOL_MAGIC)
3691 			if_togglecapenable(ifp, IFCAP_WOL_MAGIC);
3692 		if (mask & IFCAP_VXLAN_HWCSUM) {
3693 			const bool was_enabled =
3694 			    (if_getcapenable(ifp) & IFCAP_VXLAN_HWCSUM) != 0;
3695 			if (was_enabled)
3696 				mlx5e_del_all_vxlan_rules(priv);
3697 			if_togglecapenable(ifp, IFCAP_VXLAN_HWCSUM);
3698 			if_togglehwassist(ifp, CSUM_INNER_IP | CSUM_INNER_IP_UDP |
3699 			    CSUM_INNER_IP_TCP | CSUM_INNER_IP6_UDP |
3700 			    CSUM_INNER_IP6_TCP);
3701 			if (!was_enabled) {
3702 				int err = mlx5e_add_all_vxlan_rules(priv);
3703 				if (err != 0) {
3704 					mlx5_en_err(ifp,
3705 					    "mlx5e_add_all_vxlan_rules() failed, %d (ignored)\n", err);
3706 				}
3707 			}
3708 		}
3709 		if (mask & IFCAP_VXLAN_HWTSO) {
3710 			if_togglecapenable(ifp, IFCAP_VXLAN_HWTSO);
3711 			if_togglehwassist(ifp, CSUM_INNER_IP_TSO |
3712 			    CSUM_INNER_IP6_TSO);
3713 		}
3714 
3715 		VLAN_CAPABILITIES(ifp);
3716 
3717 		/* hw_lro and IFCAP_LRO are divorsed, only toggle sw LRO. */
3718 		if (mask & IFCAP_LRO)
3719 			if_togglecapenable(ifp, IFCAP_LRO);
3720 
3721 		if (mask & IFCAP_HWRXTSTMP) {
3722 			if_togglecapenable(ifp, IFCAP_HWRXTSTMP);
3723 			if (if_getcapenable(ifp) & IFCAP_HWRXTSTMP) {
3724 				if (priv->clbr_done == 0)
3725 					mlx5e_reset_calibration_callout(priv);
3726 			} else {
3727 				callout_drain(&priv->tstmp_clbr);
3728 				priv->clbr_done = 0;
3729 			}
3730 		}
3731 		mask = drv_ioctl_data->reqcap2 ^ if_getcapenable2(ifp);
3732 		if ((mask & IFCAP2_BIT(IFCAP2_RXTLS4)) != 0)
3733 			if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS4));
3734 		if ((mask & IFCAP2_BIT(IFCAP2_RXTLS6)) != 0)
3735 			if_togglecapenable2(ifp, IFCAP2_BIT(IFCAP2_RXTLS6));
3736 #ifdef IPSEC_OFFLOAD
3737 		if ((mask & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0) {
3738 			bool was_enabled = (if_getcapenable2(ifp) &
3739 			    IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0;
3740 			mlx5e_close_locked(ifp);
3741 			if (was_enabled)
3742 				ipsec_accel_on_ifdown(priv->ifp);
3743 			if_togglecapenable2(ifp,
3744 			    IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD));
3745 			mlx5e_open_locked(ifp);
3746 		}
3747 #endif
3748 out:
3749 		PRIV_UNLOCK(priv);
3750 		break;
3751 
3752 	case SIOCGI2C:
3753 		ifr = (struct ifreq *)data;
3754 
3755 		/*
3756 		 * Copy from the user-space address ifr_data to the
3757 		 * kernel-space address i2c
3758 		 */
3759 		error = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
3760 		if (error)
3761 			break;
3762 
3763 		if (i2c.len > sizeof(i2c.data)) {
3764 			error = EINVAL;
3765 			break;
3766 		}
3767 
3768 		PRIV_LOCK(priv);
3769 		/* Get module_num which is required for the query_eeprom */
3770 		error = mlx5_query_module_num(priv->mdev, &module_num);
3771 		if (error) {
3772 			mlx5_en_err(ifp,
3773 			    "Query module num failed, eeprom reading is not supported\n");
3774 			error = EINVAL;
3775 			goto err_i2c;
3776 		}
3777 		/* Check if module is present before doing an access */
3778 		module_status = mlx5_query_module_status(priv->mdev, module_num);
3779 		if (module_status != MLX5_MODULE_STATUS_PLUGGED_ENABLED) {
3780 			if (bootverbose)
3781 				mlx5_en_err(ifp,
3782 				    "Query module %d status: not plugged (%d), "
3783 				    "eeprom reading is not supported\n",
3784 				    module_num, module_status);
3785 			error = EINVAL;
3786 			goto err_i2c;
3787 		}
3788 		/*
3789 		 * Currently 0XA0 and 0xA2 are the only addresses permitted.
3790 		 * The internal conversion is as follows:
3791 		 */
3792 		if (i2c.dev_addr == 0xA0)
3793 			read_addr = MLX5_I2C_ADDR_LOW;
3794 		else if (i2c.dev_addr == 0xA2)
3795 			read_addr = MLX5_I2C_ADDR_HIGH;
3796 		else {
3797 			mlx5_en_err(ifp,
3798 			    "Query eeprom failed, Invalid Address: %X\n",
3799 			    i2c.dev_addr);
3800 			error = EINVAL;
3801 			goto err_i2c;
3802 		}
3803 		error = mlx5_query_eeprom(priv->mdev,
3804 		    read_addr, MLX5_EEPROM_LOW_PAGE,
3805 		    (uint32_t)i2c.offset, (uint32_t)i2c.len, module_num,
3806 		    (uint32_t *)i2c.data, &size_read);
3807 		if (error) {
3808 			mlx5_en_err(ifp,
3809 			    "Query eeprom failed, eeprom reading is not supported\n");
3810 			error = EINVAL;
3811 			goto err_i2c;
3812 		}
3813 
3814 		if (i2c.len > MLX5_EEPROM_MAX_BYTES) {
3815 			error = mlx5_query_eeprom(priv->mdev,
3816 			    read_addr, MLX5_EEPROM_LOW_PAGE,
3817 			    (uint32_t)(i2c.offset + size_read),
3818 			    (uint32_t)(i2c.len - size_read), module_num,
3819 			    (uint32_t *)(i2c.data + size_read), &size_read);
3820 		}
3821 		if (error) {
3822 			mlx5_en_err(ifp,
3823 			    "Query eeprom failed, eeprom reading is not supported\n");
3824 			error = EINVAL;
3825 			goto err_i2c;
3826 		}
3827 
3828 		error = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
3829 err_i2c:
3830 		PRIV_UNLOCK(priv);
3831 		break;
3832 	case SIOCGIFDOWNREASON:
3833 		ifdr = (struct ifdownreason *)data;
3834 		bzero(ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3835 		PRIV_LOCK(priv);
3836 		error = -mlx5_query_pddr_troubleshooting_info(priv->mdev, NULL,
3837 		    ifdr->ifdr_msg, sizeof(ifdr->ifdr_msg));
3838 		PRIV_UNLOCK(priv);
3839 		if (error == 0)
3840 			ifdr->ifdr_reason = IFDR_REASON_MSG;
3841 		break;
3842 
3843 	case SIOCGIFRSSKEY:
3844 		ifrk = (struct ifrsskey *)data;
3845 		ifrk->ifrk_func = RSS_FUNC_TOEPLITZ;
3846 		ifrk->ifrk_keylen = MLX5E_RSS_KEY_SIZE;
3847 		CTASSERT(sizeof(ifrk->ifrk_key) >= MLX5E_RSS_KEY_SIZE);
3848 		mlx5e_get_rss_key(ifrk->ifrk_key);
3849 		break;
3850 
3851 	case SIOCGIFRSSHASH:
3852 		ifrh = (struct ifrsshash *)data;
3853 		ifrh->ifrh_func = RSS_FUNC_TOEPLITZ;
3854 		ifrh->ifrh_types =
3855 		    RSS_TYPE_IPV4 |
3856 		    RSS_TYPE_TCP_IPV4 |
3857 		    RSS_TYPE_UDP_IPV4 |
3858 		    RSS_TYPE_IPV6 |
3859 		    RSS_TYPE_TCP_IPV6 |
3860 		    RSS_TYPE_UDP_IPV6;
3861 		break;
3862 
3863 	default:
3864 		error = ether_ioctl(ifp, command, data);
3865 		break;
3866 	}
3867 	return (error);
3868 }
3869 
3870 static int
mlx5e_check_required_hca_cap(struct mlx5_core_dev * mdev)3871 mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
3872 {
3873 	/*
3874 	 * TODO: uncoment once FW really sets all these bits if
3875 	 * (!mdev->caps.eth.rss_ind_tbl_cap || !mdev->caps.eth.csum_cap ||
3876 	 * !mdev->caps.eth.max_lso_cap || !mdev->caps.eth.vlan_cap ||
3877 	 * !(mdev->caps.gen.flags & MLX5_DEV_CAP_FLAG_SCQE_BRK_MOD)) return
3878 	 * -ENOTSUPP;
3879 	 */
3880 
3881 	/* TODO: add more must-to-have features */
3882 
3883 	if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
3884 		return (-ENODEV);
3885 
3886 	return (0);
3887 }
3888 
3889 static u16
mlx5e_get_max_inline_cap(struct mlx5_core_dev * mdev)3890 mlx5e_get_max_inline_cap(struct mlx5_core_dev *mdev)
3891 {
3892 	const int min_size = ETHER_VLAN_ENCAP_LEN + ETHER_HDR_LEN;
3893 	const int max_size = MLX5E_MAX_TX_INLINE;
3894 	const int bf_buf_size =
3895 	    ((1U << MLX5_CAP_GEN(mdev, log_bf_reg_size)) / 2U) -
3896 	    (sizeof(struct mlx5e_tx_wqe) - 2);
3897 
3898 	/* verify against driver limits */
3899 	if (bf_buf_size > max_size)
3900 		return (max_size);
3901 	else if (bf_buf_size < min_size)
3902 		return (min_size);
3903 	else
3904 		return (bf_buf_size);
3905 }
3906 
3907 static int
mlx5e_build_ifp_priv(struct mlx5_core_dev * mdev,struct mlx5e_priv * priv,int num_comp_vectors)3908 mlx5e_build_ifp_priv(struct mlx5_core_dev *mdev,
3909     struct mlx5e_priv *priv,
3910     int num_comp_vectors)
3911 {
3912 	int err;
3913 
3914 	/*
3915 	 * TODO: Consider link speed for setting "log_sq_size",
3916 	 * "log_rq_size" and "cq_moderation_xxx":
3917 	 */
3918 	priv->params.log_sq_size =
3919 	    MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
3920 	priv->params.log_rq_size =
3921 	    MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
3922 	priv->params.rx_cq_moderation_usec =
3923 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
3924 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE :
3925 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
3926 	priv->params.rx_cq_moderation_mode =
3927 	    MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1 : 0;
3928 	priv->params.rx_cq_moderation_pkts =
3929 	    MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
3930 	priv->params.tx_cq_moderation_usec =
3931 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
3932 	priv->params.tx_cq_moderation_pkts =
3933 	    MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
3934 	priv->params.rx_hash_log_tbl_sz =
3935 	    (order_base_2(num_comp_vectors) >
3936 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ) ?
3937 	    order_base_2(num_comp_vectors) :
3938 	    MLX5E_PARAMS_DEFAULT_RX_HASH_LOG_TBL_SZ;
3939 	priv->params.num_tc = 1;
3940 	priv->params.default_vlan_prio = 0;
3941 	priv->counter_set_id = -1;
3942 	priv->params.tx_max_inline = mlx5e_get_max_inline_cap(mdev);
3943 
3944 	err = mlx5_query_min_inline(mdev, &priv->params.tx_min_inline_mode);
3945 	if (err)
3946 		return (err);
3947 
3948 	/*
3949 	 * hw lro is currently defaulted to off. when it won't anymore we
3950 	 * will consider the HW capability: "!!MLX5_CAP_ETH(mdev, lro_cap)"
3951 	 */
3952 	priv->params.hw_lro_en = false;
3953 	priv->params.lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
3954 
3955 	/*
3956 	 * CQE zipping is off, because the per-packet 32-bit Toeplitz hash
3957 	 * is then not supported. The 32-bit Toeplitz hash is needed to
3958 	 * correctly demultiplex incoming traffic into the expected
3959 	 * network queues.
3960 	 */
3961 	priv->params.cqe_zipping_en = false;
3962 
3963 	priv->mdev = mdev;
3964 	priv->params.num_channels = num_comp_vectors;
3965 	priv->params.channels_rsss = 1;
3966 	priv->order_base_2_num_channels = order_base_2(num_comp_vectors);
3967 	priv->queue_mapping_channel_mask =
3968 	    roundup_pow_of_two(num_comp_vectors) - 1;
3969 	priv->num_tc = priv->params.num_tc;
3970 	priv->default_vlan_prio = priv->params.default_vlan_prio;
3971 
3972 	INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
3973 	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
3974 	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
3975 
3976 	return (0);
3977 }
3978 
3979 static void
mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev * mdev,void * mkc)3980 mlx5e_mkey_set_relaxed_ordering(struct mlx5_core_dev *mdev, void *mkc)
3981 {
3982 	bool ro_pci_enable =
3983 	    pci_get_relaxed_ordering_enabled(mdev->pdev->dev.bsddev);
3984 	bool ro_write = MLX5_CAP_GEN(mdev, relaxed_ordering_write);
3985 	bool ro_read = MLX5_CAP_GEN(mdev, relaxed_ordering_read);
3986 
3987 	MLX5_SET(mkc, mkc, relaxed_ordering_read, ro_pci_enable && ro_read);
3988 	MLX5_SET(mkc, mkc, relaxed_ordering_write, ro_pci_enable && ro_write);
3989 }
3990 
3991 static int
mlx5e_create_mkey(struct mlx5e_priv * priv,u32 pdn,struct mlx5_core_mkey * mkey)3992 mlx5e_create_mkey(struct mlx5e_priv *priv, u32 pdn,
3993 		  struct mlx5_core_mkey *mkey)
3994 {
3995 	if_t ifp = priv->ifp;
3996 	struct mlx5_core_dev *mdev = priv->mdev;
3997 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
3998 	void *mkc;
3999 	u32 *in;
4000 	int err;
4001 
4002 	in = mlx5_vzalloc(inlen);
4003 	if (in == NULL) {
4004 		mlx5_en_err(ifp, "failed to allocate inbox\n");
4005 		return (-ENOMEM);
4006 	}
4007 
4008 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
4009 	MLX5_SET(mkc, mkc, access_mode, MLX5_ACCESS_MODE_PA);
4010 	MLX5_SET(mkc, mkc, umr_en, 1);	/* used by HW TLS */
4011 	MLX5_SET(mkc, mkc, lw, 1);
4012 	MLX5_SET(mkc, mkc, lr, 1);
4013 	mlx5e_mkey_set_relaxed_ordering(mdev, mkc);
4014 	MLX5_SET(mkc, mkc, pd, pdn);
4015 	MLX5_SET(mkc, mkc, length64, 1);
4016 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
4017 
4018 	err = mlx5_core_create_mkey(mdev, mkey, in, inlen);
4019 	if (err)
4020 		mlx5_en_err(ifp, "mlx5_core_create_mkey failed, %d\n",
4021 		    err);
4022 
4023 	kvfree(in);
4024 	return (err);
4025 }
4026 
4027 static const char *mlx5e_vport_stats_desc[] = {
4028 	MLX5E_VPORT_STATS(MLX5E_STATS_DESC)
4029 };
4030 
4031 static const char *mlx5e_pport_stats_desc[] = {
4032 	MLX5E_PPORT_STATS(MLX5E_STATS_DESC)
4033 };
4034 
4035 static int
mlx5e_priv_static_init(struct mlx5e_priv * priv,struct mlx5_core_dev * mdev,const uint32_t channels)4036 mlx5e_priv_static_init(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
4037     const uint32_t channels)
4038 {
4039 	uint32_t x;
4040 	int err;
4041 
4042 	mtx_init(&priv->async_events_mtx, "mlx5async", MTX_NETWORK_LOCK, MTX_DEF);
4043 	sx_init(&priv->state_lock, "mlx5state");
4044 	callout_init_mtx(&priv->watchdog, &priv->async_events_mtx, 0);
4045 	MLX5_INIT_DOORBELL_LOCK(&priv->doorbell_lock);
4046 	for (x = 0; x != channels; x++)
4047 		mlx5e_chan_static_init(priv, &priv->channel[x], x);
4048 
4049 	for (x = 0; x != channels; x++) {
4050 		err = mlx5_alloc_bfreg(mdev, &priv->channel[x].bfreg, false, false);
4051 		if (err)
4052 			goto err_alloc_bfreg;
4053 	}
4054 	return (0);
4055 
4056 err_alloc_bfreg:
4057 	while (x--)
4058 		mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
4059 
4060 	for (x = 0; x != channels; x++)
4061 		mlx5e_chan_static_destroy(&priv->channel[x]);
4062 	callout_drain(&priv->watchdog);
4063 	mtx_destroy(&priv->async_events_mtx);
4064 	sx_destroy(&priv->state_lock);
4065 	return (err);
4066 }
4067 
4068 static void
mlx5e_priv_static_destroy(struct mlx5e_priv * priv,struct mlx5_core_dev * mdev,const uint32_t channels)4069 mlx5e_priv_static_destroy(struct mlx5e_priv *priv, struct mlx5_core_dev *mdev,
4070     const uint32_t channels)
4071 {
4072 	uint32_t x;
4073 
4074 	for (x = 0; x != channels; x++)
4075 		mlx5_free_bfreg(mdev, &priv->channel[x].bfreg);
4076 	for (x = 0; x != channels; x++)
4077 		mlx5e_chan_static_destroy(&priv->channel[x]);
4078 	callout_drain(&priv->watchdog);
4079 	mtx_destroy(&priv->async_events_mtx);
4080 	sx_destroy(&priv->state_lock);
4081 }
4082 
4083 static int
sysctl_firmware(SYSCTL_HANDLER_ARGS)4084 sysctl_firmware(SYSCTL_HANDLER_ARGS)
4085 {
4086 	/*
4087 	 * %d.%d%.d the string format.
4088 	 * fw_rev_{maj,min,sub} return u16, 2^16 = 65536.
4089 	 * We need at most 5 chars to store that.
4090 	 * It also has: two "." and NULL at the end, which means we need 18
4091 	 * (5*3 + 3) chars at most.
4092 	 */
4093 	char fw[18];
4094 	struct mlx5e_priv *priv = arg1;
4095 	int error;
4096 
4097 	snprintf(fw, sizeof(fw), "%d.%d.%d", fw_rev_maj(priv->mdev), fw_rev_min(priv->mdev),
4098 	    fw_rev_sub(priv->mdev));
4099 	error = sysctl_handle_string(oidp, fw, sizeof(fw), req);
4100 	return (error);
4101 }
4102 
4103 static void
mlx5e_disable_tx_dma(struct mlx5e_channel * ch)4104 mlx5e_disable_tx_dma(struct mlx5e_channel *ch)
4105 {
4106 	int i;
4107 
4108 	for (i = 0; i < ch->priv->num_tc; i++)
4109 		mlx5e_drain_sq(&ch->sq[i]);
4110 }
4111 
4112 static void
mlx5e_reset_sq_doorbell_record(struct mlx5e_sq * sq)4113 mlx5e_reset_sq_doorbell_record(struct mlx5e_sq *sq)
4114 {
4115 
4116 	sq->doorbell.d32[0] = cpu_to_be32(MLX5_OPCODE_NOP);
4117 	sq->doorbell.d32[1] = cpu_to_be32(sq->sqn << 8);
4118 	mlx5e_tx_notify_hw(sq, true);
4119 }
4120 
4121 void
mlx5e_resume_sq(struct mlx5e_sq * sq)4122 mlx5e_resume_sq(struct mlx5e_sq *sq)
4123 {
4124 	int err;
4125 
4126 	/* check if already enabled */
4127 	if (READ_ONCE(sq->running) != 0)
4128 		return;
4129 
4130 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_ERR,
4131 	    MLX5_SQC_STATE_RST);
4132 	if (err != 0) {
4133 		mlx5_en_err(sq->ifp,
4134 		    "mlx5e_modify_sq() from ERR to RST failed: %d\n", err);
4135 	}
4136 
4137 	sq->cc = 0;
4138 	sq->pc = 0;
4139 
4140 	/* reset doorbell prior to moving from RST to RDY */
4141 	mlx5e_reset_sq_doorbell_record(sq);
4142 
4143 	err = mlx5e_modify_sq(sq, MLX5_SQC_STATE_RST,
4144 	    MLX5_SQC_STATE_RDY);
4145 	if (err != 0) {
4146 		mlx5_en_err(sq->ifp,
4147 		    "mlx5e_modify_sq() from RST to RDY failed: %d\n", err);
4148 	}
4149 
4150 	sq->cev_next_state = MLX5E_CEV_STATE_INITIAL;
4151 	WRITE_ONCE(sq->running, 1);
4152 }
4153 
4154 static void
mlx5e_enable_tx_dma(struct mlx5e_channel * ch)4155 mlx5e_enable_tx_dma(struct mlx5e_channel *ch)
4156 {
4157         int i;
4158 
4159 	for (i = 0; i < ch->priv->num_tc; i++)
4160 		mlx5e_resume_sq(&ch->sq[i]);
4161 }
4162 
4163 static void
mlx5e_disable_rx_dma(struct mlx5e_channel * ch)4164 mlx5e_disable_rx_dma(struct mlx5e_channel *ch)
4165 {
4166 	struct mlx5e_rq *rq = &ch->rq;
4167 	struct epoch_tracker et;
4168 	int err;
4169 
4170 	mtx_lock(&rq->mtx);
4171 	rq->enabled = 0;
4172 	callout_stop(&rq->watchdog);
4173 	mtx_unlock(&rq->mtx);
4174 
4175 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RDY, MLX5_RQC_STATE_ERR);
4176 	if (err != 0) {
4177 		mlx5_en_err(rq->ifp,
4178 		    "mlx5e_modify_rq() from RDY to RST failed: %d\n", err);
4179 	}
4180 
4181 	while (!mlx5_wq_ll_is_empty(&rq->wq)) {
4182 		msleep(1);
4183 		NET_EPOCH_ENTER(et);
4184 		rq->cq.mcq.comp(&rq->cq.mcq, NULL);
4185 		NET_EPOCH_EXIT(et);
4186 	}
4187 
4188 	/*
4189 	 * Transitioning into RST state will allow the FW to track less ERR state queues,
4190 	 * thus reducing the recv queue flushing time
4191 	 */
4192 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_ERR, MLX5_RQC_STATE_RST);
4193 	if (err != 0) {
4194 		mlx5_en_err(rq->ifp,
4195 		    "mlx5e_modify_rq() from ERR to RST failed: %d\n", err);
4196 	}
4197 }
4198 
4199 static void
mlx5e_enable_rx_dma(struct mlx5e_channel * ch)4200 mlx5e_enable_rx_dma(struct mlx5e_channel *ch)
4201 {
4202 	struct mlx5e_rq *rq = &ch->rq;
4203 	struct epoch_tracker et;
4204 	int err;
4205 
4206 	rq->wq.wqe_ctr = 0;
4207 	mlx5_wq_ll_update_db_record(&rq->wq);
4208 	err = mlx5e_modify_rq(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
4209 	if (err != 0) {
4210 		mlx5_en_err(rq->ifp,
4211 		    "mlx5e_modify_rq() from RST to RDY failed: %d\n", err);
4212         }
4213 
4214 	rq->enabled = 1;
4215 
4216 	NET_EPOCH_ENTER(et);
4217 	rq->cq.mcq.comp(&rq->cq.mcq, NULL);
4218 	NET_EPOCH_EXIT(et);
4219 }
4220 
4221 void
mlx5e_modify_tx_dma(struct mlx5e_priv * priv,uint8_t value)4222 mlx5e_modify_tx_dma(struct mlx5e_priv *priv, uint8_t value)
4223 {
4224 	int i;
4225 
4226 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
4227 		return;
4228 
4229 	for (i = 0; i < priv->params.num_channels; i++) {
4230 		if (value)
4231 			mlx5e_disable_tx_dma(&priv->channel[i]);
4232 		else
4233 			mlx5e_enable_tx_dma(&priv->channel[i]);
4234 	}
4235 }
4236 
4237 void
mlx5e_modify_rx_dma(struct mlx5e_priv * priv,uint8_t value)4238 mlx5e_modify_rx_dma(struct mlx5e_priv *priv, uint8_t value)
4239 {
4240 	int i;
4241 
4242 	if (test_bit(MLX5E_STATE_OPENED, &priv->state) == 0)
4243 		return;
4244 
4245 	for (i = 0; i < priv->params.num_channels; i++) {
4246 		if (value)
4247 			mlx5e_disable_rx_dma(&priv->channel[i]);
4248 		else
4249 			mlx5e_enable_rx_dma(&priv->channel[i]);
4250 	}
4251 }
4252 
4253 static void
mlx5e_add_hw_stats(struct mlx5e_priv * priv)4254 mlx5e_add_hw_stats(struct mlx5e_priv *priv)
4255 {
4256 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4257 	    OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE,
4258 	    priv, 0, sysctl_firmware, "A", "HCA firmware version");
4259 
4260 	SYSCTL_ADD_STRING(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_hw),
4261 	    OID_AUTO, "board_id", CTLFLAG_RD, priv->mdev->board_id, 0,
4262 	    "Board ID");
4263 }
4264 
4265 static int
mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)4266 mlx5e_sysctl_tx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4267 {
4268 	struct mlx5e_priv *priv = arg1;
4269 	uint8_t temp[MLX5E_MAX_PRIORITY];
4270 	uint32_t tx_pfc;
4271 	int err;
4272 	int i;
4273 
4274 	PRIV_LOCK(priv);
4275 
4276 	tx_pfc = priv->params.tx_priority_flow_control;
4277 
4278 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4279 		temp[i] = (tx_pfc >> i) & 1;
4280 
4281 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4282 	if (err || !req->newptr)
4283 		goto done;
4284 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4285 	if (err)
4286 		goto done;
4287 
4288 	priv->params.tx_priority_flow_control = 0;
4289 
4290 	/* range check input value */
4291 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4292 		if (temp[i] > 1) {
4293 			err = ERANGE;
4294 			goto done;
4295 		}
4296 		priv->params.tx_priority_flow_control |= (temp[i] << i);
4297 	}
4298 
4299 	/* check if update is required */
4300 	if (tx_pfc != priv->params.tx_priority_flow_control)
4301 		err = -mlx5e_set_port_pfc(priv);
4302 done:
4303 	if (err != 0)
4304 		priv->params.tx_priority_flow_control= tx_pfc;
4305 	PRIV_UNLOCK(priv);
4306 
4307 	return (err);
4308 }
4309 
4310 static int
mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)4311 mlx5e_sysctl_rx_priority_flow_control(SYSCTL_HANDLER_ARGS)
4312 {
4313 	struct mlx5e_priv *priv = arg1;
4314 	uint8_t temp[MLX5E_MAX_PRIORITY];
4315 	uint32_t rx_pfc;
4316 	int err;
4317 	int i;
4318 
4319 	PRIV_LOCK(priv);
4320 
4321 	rx_pfc = priv->params.rx_priority_flow_control;
4322 
4323 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++)
4324 		temp[i] = (rx_pfc >> i) & 1;
4325 
4326 	err = SYSCTL_OUT(req, temp, MLX5E_MAX_PRIORITY);
4327 	if (err || !req->newptr)
4328 		goto done;
4329 	err = SYSCTL_IN(req, temp, MLX5E_MAX_PRIORITY);
4330 	if (err)
4331 		goto done;
4332 
4333 	priv->params.rx_priority_flow_control = 0;
4334 
4335 	/* range check input value */
4336 	for (i = 0; i != MLX5E_MAX_PRIORITY; i++) {
4337 		if (temp[i] > 1) {
4338 			err = ERANGE;
4339 			goto done;
4340 		}
4341 		priv->params.rx_priority_flow_control |= (temp[i] << i);
4342 	}
4343 
4344 	/* check if update is required */
4345 	if (rx_pfc != priv->params.rx_priority_flow_control) {
4346 		err = -mlx5e_set_port_pfc(priv);
4347 		if (err == 0 && priv->sw_is_port_buf_owner)
4348 			err = mlx5e_update_buf_lossy(priv);
4349 	}
4350 done:
4351 	if (err != 0)
4352 		priv->params.rx_priority_flow_control= rx_pfc;
4353 	PRIV_UNLOCK(priv);
4354 
4355 	return (err);
4356 }
4357 
4358 static void
mlx5e_setup_pauseframes(struct mlx5e_priv * priv)4359 mlx5e_setup_pauseframes(struct mlx5e_priv *priv)
4360 {
4361 	int error;
4362 
4363 	/* enable pauseframes by default */
4364 	priv->params.tx_pauseframe_control = 1;
4365 	priv->params.rx_pauseframe_control = 1;
4366 
4367 	/* disable ports flow control, PFC, by default */
4368 	priv->params.tx_priority_flow_control = 0;
4369 	priv->params.rx_priority_flow_control = 0;
4370 
4371 	/* register pauseframe SYSCTLs */
4372 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4373 	    OID_AUTO, "tx_pauseframe_control", CTLFLAG_RDTUN,
4374 	    &priv->params.tx_pauseframe_control, 0,
4375 	    "Set to enable TX pause frames. Clear to disable.");
4376 
4377 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4378 	    OID_AUTO, "rx_pauseframe_control", CTLFLAG_RDTUN,
4379 	    &priv->params.rx_pauseframe_control, 0,
4380 	    "Set to enable RX pause frames. Clear to disable.");
4381 
4382 	/* register priority flow control, PFC, SYSCTLs */
4383 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4384 	    OID_AUTO, "tx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4385 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_tx_priority_flow_control, "CU",
4386 	    "Set to enable TX ports flow control frames for priorities 0..7. Clear to disable.");
4387 
4388 	SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4389 	    OID_AUTO, "rx_priority_flow_control", CTLTYPE_U8 | CTLFLAG_RWTUN |
4390 	    CTLFLAG_MPSAFE, priv, 0, &mlx5e_sysctl_rx_priority_flow_control, "CU",
4391 	    "Set to enable RX ports flow control frames for priorities 0..7. Clear to disable.");
4392 
4393 	PRIV_LOCK(priv);
4394 
4395 	/* range check */
4396 	priv->params.tx_pauseframe_control =
4397 	    priv->params.tx_pauseframe_control ? 1 : 0;
4398 	priv->params.rx_pauseframe_control =
4399 	    priv->params.rx_pauseframe_control ? 1 : 0;
4400 
4401 	/* update firmware */
4402 	error = mlx5e_set_port_pause_and_pfc(priv);
4403 	if (error == -EINVAL) {
4404 		mlx5_en_err(priv->ifp,
4405 		    "Global pauseframes must be disabled before enabling PFC.\n");
4406 		priv->params.rx_priority_flow_control = 0;
4407 		priv->params.tx_priority_flow_control = 0;
4408 
4409 		/* update firmware */
4410 		(void) mlx5e_set_port_pause_and_pfc(priv);
4411 	}
4412 	PRIV_UNLOCK(priv);
4413 }
4414 
4415 static int
mlx5e_ul_snd_tag_alloc(if_t ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)4416 mlx5e_ul_snd_tag_alloc(if_t ifp,
4417     union if_snd_tag_alloc_params *params,
4418     struct m_snd_tag **ppmt)
4419 {
4420 	struct mlx5e_priv *priv;
4421 	struct mlx5e_channel *pch;
4422 
4423 	priv = if_getsoftc(ifp);
4424 
4425 	if (unlikely(priv->gone || params->hdr.flowtype == M_HASHTYPE_NONE)) {
4426 		return (EOPNOTSUPP);
4427 	} else {
4428 		/* keep this code synced with mlx5e_select_queue() */
4429 		u32 ch = priv->params.num_channels;
4430 #ifdef RSS
4431 		u32 temp;
4432 
4433 		if (rss_hash2bucket(params->hdr.flowid,
4434 		    params->hdr.flowtype, &temp) == 0)
4435 			ch = temp % ch;
4436 		else
4437 #endif
4438 			ch = (params->hdr.flowid % 128) % ch;
4439 
4440 		/*
4441 		 * NOTE: The channels array is only freed at detach
4442 		 * and it safe to return a pointer to the send tag
4443 		 * inside the channels structure as long as we
4444 		 * reference the priv.
4445 		 */
4446 		pch = priv->channel + ch;
4447 
4448 		/* check if send queue is not running */
4449 		if (unlikely(pch->sq[0].running == 0))
4450 			return (ENXIO);
4451 		m_snd_tag_ref(&pch->tag);
4452 		*ppmt = &pch->tag;
4453 		return (0);
4454 	}
4455 }
4456 
4457 static int
mlx5e_ul_snd_tag_query(struct m_snd_tag * pmt,union if_snd_tag_query_params * params)4458 mlx5e_ul_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
4459 {
4460 	struct mlx5e_channel *pch =
4461 	    container_of(pmt, struct mlx5e_channel, tag);
4462 
4463 	params->unlimited.max_rate = -1ULL;
4464 	params->unlimited.queue_level = mlx5e_sq_queue_level(&pch->sq[0]);
4465 	return (0);
4466 }
4467 
4468 static void
mlx5e_ul_snd_tag_free(struct m_snd_tag * pmt)4469 mlx5e_ul_snd_tag_free(struct m_snd_tag *pmt)
4470 {
4471 	struct mlx5e_channel *pch =
4472 	    container_of(pmt, struct mlx5e_channel, tag);
4473 
4474 	complete(&pch->completion);
4475 }
4476 
4477 static int
mlx5e_snd_tag_alloc(if_t ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)4478 mlx5e_snd_tag_alloc(if_t ifp,
4479     union if_snd_tag_alloc_params *params,
4480     struct m_snd_tag **ppmt)
4481 {
4482 
4483 	switch (params->hdr.type) {
4484 #ifdef RATELIMIT
4485 	case IF_SND_TAG_TYPE_RATE_LIMIT:
4486 		return (mlx5e_rl_snd_tag_alloc(ifp, params, ppmt));
4487 #ifdef KERN_TLS
4488 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
4489 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4490 #endif
4491 #endif
4492 	case IF_SND_TAG_TYPE_UNLIMITED:
4493 		return (mlx5e_ul_snd_tag_alloc(ifp, params, ppmt));
4494 #ifdef KERN_TLS
4495 	case IF_SND_TAG_TYPE_TLS:
4496 		return (mlx5e_tls_snd_tag_alloc(ifp, params, ppmt));
4497 	case IF_SND_TAG_TYPE_TLS_RX:
4498 		return (mlx5e_tls_rx_snd_tag_alloc(ifp, params, ppmt));
4499 #endif
4500 	default:
4501 		return (EOPNOTSUPP);
4502 	}
4503 }
4504 
4505 #ifdef RATELIMIT
4506 #define NUM_HDWR_RATES_MLX 13
4507 static const uint64_t adapter_rates_mlx[NUM_HDWR_RATES_MLX] = {
4508 	135375,			/* 1,083,000 */
4509 	180500,			/* 1,444,000 */
4510 	270750,			/* 2,166,000 */
4511 	361000,			/* 2,888,000 */
4512 	541500,			/* 4,332,000 */
4513 	721875,			/* 5,775,000 */
4514 	1082875,		/* 8,663,000 */
4515 	1443875,		/* 11,551,000 */
4516 	2165750,		/* 17,326,000 */
4517 	2887750,		/* 23,102,000 */
4518 	4331625,		/* 34,653,000 */
4519 	5775500,		/* 46,204,000 */
4520 	8663125			/* 69,305,000 */
4521 };
4522 
4523 static void
mlx5e_ratelimit_query(if_t ifp __unused,struct if_ratelimit_query_results * q)4524 mlx5e_ratelimit_query(if_t ifp __unused, struct if_ratelimit_query_results *q)
4525 {
4526 	/*
4527 	 * This function needs updating by the driver maintainer!
4528 	 * For the MLX card there are currently (ConectX-4?) 13
4529 	 * pre-set rates and others i.e. ConnectX-5, 6, 7??
4530 	 *
4531 	 * This will change based on later adapters
4532 	 * and this code should be updated to look at ifp
4533 	 * and figure out the specific adapter type
4534 	 * settings i.e. how many rates as well
4535 	 * as if they are fixed (as is shown here) or
4536 	 * if they are dynamic (example chelsio t4). Also if there
4537 	 * is a maximum number of flows that the adapter
4538 	 * can handle that too needs to be updated in
4539 	 * the max_flows field.
4540 	 */
4541 	q->rate_table = adapter_rates_mlx;
4542 	q->flags = RT_IS_FIXED_TABLE;
4543 	q->max_flows = 0;	/* mlx has no limit */
4544 	q->number_of_rates = NUM_HDWR_RATES_MLX;
4545 	q->min_segment_burst = 1;
4546 }
4547 #endif
4548 
4549 static void
mlx5e_ifm_add(struct mlx5e_priv * priv,int type)4550 mlx5e_ifm_add(struct mlx5e_priv *priv, int type)
4551 {
4552 	ifmedia_add(&priv->media, type | IFM_ETHER, 0, NULL);
4553 	ifmedia_add(&priv->media, type | IFM_ETHER |
4554 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4555 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_RXPAUSE, 0, NULL);
4556 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_ETH_TXPAUSE, 0, NULL);
4557 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX, 0, NULL);
4558 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4559 	    IFM_ETH_RXPAUSE, 0, NULL);
4560 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4561 	    IFM_ETH_TXPAUSE, 0, NULL);
4562 	ifmedia_add(&priv->media, type | IFM_ETHER | IFM_FDX |
4563 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE, 0, NULL);
4564 }
4565 
4566 static void *
mlx5e_create_ifp(struct mlx5_core_dev * mdev)4567 mlx5e_create_ifp(struct mlx5_core_dev *mdev)
4568 {
4569 	if_t ifp;
4570 	struct mlx5e_priv *priv;
4571 	u8 dev_addr[ETHER_ADDR_LEN] __aligned(4);
4572 	struct sysctl_oid_list *child;
4573 	int ncv = mdev->priv.eq_table.num_comp_vectors;
4574 	char unit[16];
4575 	struct pfil_head_args pa;
4576 	int err;
4577 	u32 eth_proto_cap;
4578 	u32 out[MLX5_ST_SZ_DW(ptys_reg)];
4579 	bool ext;
4580 	struct media media_entry = {};
4581 
4582 	if (mlx5e_check_required_hca_cap(mdev)) {
4583 		mlx5_core_dbg(mdev, "mlx5e_check_required_hca_cap() failed\n");
4584 		return (NULL);
4585 	}
4586 
4587 	/*
4588 	 * Try to allocate the priv and make room for worst-case
4589 	 * number of channel structures:
4590 	 */
4591 	priv = malloc_domainset(sizeof(*priv) +
4592 	    (sizeof(priv->channel[0]) * mdev->priv.eq_table.num_comp_vectors),
4593 	    M_MLX5EN, mlx5_dev_domainset(mdev), M_WAITOK | M_ZERO);
4594 
4595 	ifp = priv->ifp = if_alloc_dev(IFT_ETHER, mdev->pdev->dev.bsddev);
4596 	/* setup all static fields */
4597 	if (mlx5e_priv_static_init(priv, mdev, mdev->priv.eq_table.num_comp_vectors)) {
4598 		mlx5_core_err(mdev, "mlx5e_priv_static_init() failed\n");
4599 		goto err_free_ifp;
4600 	}
4601 
4602 	if_setsoftc(ifp, priv);
4603 	if_initname(ifp, "mce", device_get_unit(mdev->pdev->dev.bsddev));
4604 	if_setmtu(ifp, ETHERMTU);
4605 	if_setinitfn(ifp, mlx5e_open);
4606 	if_setflags(ifp, IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST);
4607 	if_setioctlfn(ifp, mlx5e_ioctl);
4608 	if_settransmitfn(ifp, mlx5e_xmit);
4609 	if_setqflushfn(ifp, if_qflush);
4610 	if_setgetcounterfn(ifp, mlx5e_get_counter);
4611 	if_setsendqlen(ifp, ifqmaxlen);
4612 	/*
4613          * Set driver features
4614          */
4615 	if_setcapabilities(ifp, IFCAP_NV);
4616 	if_setcapabilitiesbit(ifp, IFCAP_HWCSUM | IFCAP_HWCSUM_IPV6, 0);
4617 	if_setcapabilitiesbit(ifp, IFCAP_VLAN_MTU | IFCAP_VLAN_HWTAGGING, 0);
4618 	if_setcapabilitiesbit(ifp, IFCAP_VLAN_HWCSUM | IFCAP_VLAN_HWFILTER, 0);
4619 	if_setcapabilitiesbit(ifp, IFCAP_LINKSTATE | IFCAP_JUMBO_MTU, 0);
4620 	if_setcapabilitiesbit(ifp, IFCAP_LRO, 0);
4621 	if_setcapabilitiesbit(ifp, IFCAP_TSO | IFCAP_VLAN_HWTSO, 0);
4622 	if_setcapabilitiesbit(ifp, IFCAP_HWSTATS | IFCAP_HWRXTSTMP, 0);
4623 	if_setcapabilitiesbit(ifp, IFCAP_MEXTPG, 0);
4624 	if (mlx5e_is_tlstx_capable(mdev))
4625 		if_setcapabilitiesbit(ifp, IFCAP_TXTLS4 | IFCAP_TXTLS6, 0);
4626 	if (mlx5e_is_tlsrx_capable(mdev))
4627 		if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_RXTLS4) |
4628 		    IFCAP2_BIT(IFCAP2_RXTLS6), 0);
4629 	if (mlx5e_is_ratelimit_capable(mdev)) {
4630 		if_setcapabilitiesbit(ifp, IFCAP_TXRTLMT, 0);
4631 		if (mlx5e_is_tlstx_capable(mdev))
4632 			if_setcapabilitiesbit(ifp, IFCAP_TXTLS_RTLMT, 0);
4633 	}
4634 	if_setcapabilitiesbit(ifp, IFCAP_VXLAN_HWCSUM | IFCAP_VXLAN_HWTSO, 0);
4635 	if (mlx5e_is_ipsec_capable(mdev))
4636 		if_setcapabilities2bit(ifp, IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD),
4637 		    0);
4638 
4639 	if_setsndtagallocfn(ifp, mlx5e_snd_tag_alloc);
4640 #ifdef RATELIMIT
4641 	if_setratelimitqueryfn(ifp, mlx5e_ratelimit_query);
4642 #endif
4643 	/* set TSO limits so that we don't have to drop TX packets */
4644 	if_sethwtsomax(ifp, MLX5E_MAX_TX_PAYLOAD_SIZE - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
4645 	if_sethwtsomaxsegcount(ifp, MLX5E_MAX_TX_MBUF_FRAGS - 1 /* hdr */);
4646 	if_sethwtsomaxsegsize(ifp, MLX5E_MAX_TX_MBUF_SIZE);
4647 
4648 	if_setcapenable(ifp, if_getcapabilities(ifp));
4649 	if_setcapenable2(ifp, if_getcapabilities2(ifp));
4650 	if_sethwassist(ifp, 0);
4651 	if (if_getcapenable(ifp) & IFCAP_TSO)
4652 		if_sethwassistbits(ifp, CSUM_TSO, 0);
4653 	if (if_getcapenable(ifp) & IFCAP_TXCSUM)
4654 		if_sethwassistbits(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP), 0);
4655 	if (if_getcapenable(ifp) & IFCAP_TXCSUM_IPV6)
4656 		if_sethwassistbits(ifp, (CSUM_UDP_IPV6 | CSUM_TCP_IPV6), 0);
4657 	if (if_getcapabilities(ifp) & IFCAP_VXLAN_HWCSUM)
4658 		if_sethwassistbits(ifp, CSUM_INNER_IP6_UDP | CSUM_INNER_IP6_TCP |
4659 		    CSUM_INNER_IP | CSUM_INNER_IP_UDP | CSUM_INNER_IP_TCP |
4660 		    CSUM_ENCAP_VXLAN, 0);
4661 	if (if_getcapabilities(ifp) & IFCAP_VXLAN_HWTSO)
4662 		if_sethwassistbits(ifp, CSUM_INNER_IP6_TSO | CSUM_INNER_IP_TSO, 0);
4663 
4664 	/* ifnet sysctl tree */
4665 	sysctl_ctx_init(&priv->sysctl_ctx);
4666 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_STATIC_CHILDREN(_dev),
4667 	    OID_AUTO, if_getdname(ifp), CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4668 	    "MLX5 ethernet - interface name");
4669 	if (priv->sysctl_ifnet == NULL) {
4670 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4671 		goto err_free_sysctl;
4672 	}
4673 	snprintf(unit, sizeof(unit), "%d", if_getdunit(ifp));
4674 	priv->sysctl_ifnet = SYSCTL_ADD_NODE(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4675 	    OID_AUTO, unit, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4676 	    "MLX5 ethernet - interface unit");
4677 	if (priv->sysctl_ifnet == NULL) {
4678 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4679 		goto err_free_sysctl;
4680 	}
4681 
4682 	/* HW sysctl tree */
4683 	child = SYSCTL_CHILDREN(device_get_sysctl_tree(mdev->pdev->dev.bsddev));
4684 	priv->sysctl_hw = SYSCTL_ADD_NODE(&priv->sysctl_ctx, child,
4685 	    OID_AUTO, "hw", CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
4686 	    "MLX5 ethernet dev hw");
4687 	if (priv->sysctl_hw == NULL) {
4688 		mlx5_core_err(mdev, "SYSCTL_ADD_NODE() failed\n");
4689 		goto err_free_sysctl;
4690 	}
4691 
4692 	err = mlx5e_build_ifp_priv(mdev, priv, ncv);
4693 	if (err) {
4694 		mlx5_core_err(mdev, "mlx5e_build_ifp_priv() failed (%d)\n", err);
4695 		goto err_free_sysctl;
4696 	}
4697 
4698 	/* reuse mlx5core's watchdog workqueue */
4699 	priv->wq = mdev->priv.health.wq_watchdog;
4700 
4701 	err = mlx5_core_alloc_pd(mdev, &priv->pdn, 0);
4702 	if (err) {
4703 		mlx5_en_err(ifp, "mlx5_core_alloc_pd failed, %d\n", err);
4704 		goto err_free_wq;
4705 	}
4706 	err = mlx5_alloc_transport_domain(mdev, &priv->tdn, 0);
4707 	if (err) {
4708 		mlx5_en_err(ifp,
4709 		    "mlx5_alloc_transport_domain failed, %d\n", err);
4710 		goto err_dealloc_pd;
4711 	}
4712 	err = mlx5e_create_mkey(priv, priv->pdn, &priv->mr);
4713 	if (err) {
4714 		mlx5_en_err(ifp, "mlx5e_create_mkey failed, %d\n", err);
4715 		goto err_dealloc_transport_domain;
4716 	}
4717 	mlx5_query_nic_vport_mac_address(priv->mdev, 0, dev_addr);
4718 
4719 	/* check if we should generate a random MAC address */
4720 	if (MLX5_CAP_GEN(priv->mdev, vport_group_manager) == 0 &&
4721 	    is_zero_ether_addr(dev_addr)) {
4722 		random_ether_addr(dev_addr);
4723 		mlx5_en_err(ifp, "Assigned random MAC address\n");
4724 	}
4725 
4726 	err = mlx5e_rl_init(priv);
4727 	if (err) {
4728 		mlx5_en_err(ifp, "mlx5e_rl_init failed, %d\n", err);
4729 		goto err_create_mkey;
4730 	}
4731 
4732 	err = mlx5e_tls_init(priv);
4733 	if (err) {
4734 		if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
4735 		goto err_rl_init;
4736 	}
4737 
4738 	if ((if_getcapenable2(ifp) & IFCAP2_BIT(IFCAP2_IPSEC_OFFLOAD)) != 0) {
4739 		err = mlx5e_ipsec_init(priv);
4740 		if (err) {
4741 			if_printf(ifp, "%s: mlx5e_tls_init failed\n", __func__);
4742 			goto err_tls_init;
4743 		}
4744 	}
4745 
4746 	err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
4747 	if (err) {
4748 		if_printf(ifp, "%s: mlx5e_open_drop_rq failed (%d)\n", __func__, err);
4749 		goto err_ipsec_init;
4750 	}
4751 
4752 	err = mlx5e_open_rqts(priv);
4753 	if (err) {
4754 		if_printf(ifp, "%s: mlx5e_open_rqts failed (%d)\n", __func__, err);
4755 		goto err_open_drop_rq;
4756 	}
4757 
4758 	err = mlx5e_open_tirs(priv);
4759 	if (err) {
4760 		mlx5_en_err(ifp, "mlx5e_open_tirs() failed, %d\n", err);
4761 		goto err_open_rqts;
4762 	}
4763 
4764 	err = mlx5e_open_flow_tables(priv);
4765 	if (err) {
4766 		if_printf(ifp, "%s: mlx5e_open_flow_tables failed (%d)\n", __func__, err);
4767 		goto err_open_tirs;
4768 	}
4769 
4770 	err = mlx5e_tls_rx_init(priv);
4771 	if (err) {
4772 		if_printf(ifp, "%s: mlx5e_tls_rx_init() failed, %d\n", __func__, err);
4773 		goto err_open_flow_tables;
4774 	}
4775 
4776 	/* set default MTU */
4777 	mlx5e_set_dev_port_mtu(ifp, if_getmtu(ifp));
4778 
4779 	/* Set default media status */
4780 	priv->media_status_last = IFM_AVALID;
4781 	priv->media_active_last = IFM_ETHER | IFM_AUTO | IFM_FDX;
4782 
4783 	/* setup default pauseframes configuration */
4784 	mlx5e_setup_pauseframes(priv);
4785 
4786 	/* Setup supported medias */
4787 	if (!mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN, 1)) {
4788 		ext = MLX5_CAP_PCAM_FEATURE(mdev,
4789 		    ptys_extended_ethernet);
4790 		eth_proto_cap = MLX5_GET_ETH_PROTO(ptys_reg, out, ext,
4791 		    eth_proto_capability);
4792 	} else {
4793 		ext = false;
4794 		eth_proto_cap = 0;
4795 		mlx5_en_err(ifp, "Query port media capability failed, %d\n", err);
4796 	}
4797 
4798 	ifmedia_init(&priv->media, IFM_IMASK,
4799 	    mlx5e_media_change, mlx5e_media_status);
4800 
4801 	if (ext) {
4802 		for (unsigned i = 0; i != MLX5E_EXT_LINK_SPEEDS_NUMBER; i++) {
4803 			/* check if hardware has the right capability */
4804 			if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4805 				continue;
4806 			for (unsigned j = 0; j != MLX5E_CABLE_TYPE_NUMBER; j++) {
4807 				media_entry = mlx5e_ext_mode_table[i][j];
4808 				if (media_entry.subtype == 0)
4809 					continue;
4810 				/* check if this subtype was already added */
4811 				for (unsigned k = 0; k != i; k++) {
4812 					/* check if hardware has the right capability */
4813 					if (MLX5E_PROT_MASK(k) & ~eth_proto_cap)
4814 						continue;
4815 					for (unsigned m = 0; m != MLX5E_CABLE_TYPE_NUMBER; m++) {
4816 						if (media_entry.subtype == mlx5e_ext_mode_table[k][m].subtype)
4817 							goto skip_ext_media;
4818 					}
4819 				}
4820 				mlx5e_ifm_add(priv, media_entry.subtype);
4821 			skip_ext_media:;
4822 			}
4823 		}
4824 	} else {
4825 		for (unsigned i = 0; i != MLX5E_LINK_SPEEDS_NUMBER; i++) {
4826 			media_entry = mlx5e_mode_table[i];
4827 			if (media_entry.subtype == 0)
4828 				continue;
4829 			if (MLX5E_PROT_MASK(i) & ~eth_proto_cap)
4830 				continue;
4831 			/* check if this subtype was already added */
4832 			for (unsigned k = 0; k != i; k++) {
4833 				if (media_entry.subtype == mlx5e_mode_table[k].subtype)
4834 					goto skip_media;
4835 			}
4836 			mlx5e_ifm_add(priv, media_entry.subtype);
4837 
4838 			/* NOTE: 10G ER and LR shares the same entry */
4839 			if (media_entry.subtype == IFM_10G_ER)
4840 				mlx5e_ifm_add(priv, IFM_10G_LR);
4841 		skip_media:;
4842 		}
4843 	}
4844 
4845 	mlx5e_ifm_add(priv, IFM_AUTO);
4846 
4847 	/* Set autoselect by default */
4848 	ifmedia_set(&priv->media, IFM_ETHER | IFM_AUTO | IFM_FDX |
4849 	    IFM_ETH_RXPAUSE | IFM_ETH_TXPAUSE);
4850 
4851 	DEBUGNET_SET(ifp, mlx5_en);
4852 
4853 	ether_ifattach(ifp, dev_addr);
4854 
4855 	/* Register for VLAN events */
4856 	priv->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
4857 	    mlx5e_vlan_rx_add_vid, priv, EVENTHANDLER_PRI_FIRST);
4858 	priv->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
4859 	    mlx5e_vlan_rx_kill_vid, priv, EVENTHANDLER_PRI_FIRST);
4860 
4861 	/* Register for VxLAN events */
4862 	priv->vxlan_start = EVENTHANDLER_REGISTER(vxlan_start,
4863 	    mlx5e_vxlan_start, priv, EVENTHANDLER_PRI_ANY);
4864 	priv->vxlan_stop = EVENTHANDLER_REGISTER(vxlan_stop,
4865 	    mlx5e_vxlan_stop, priv, EVENTHANDLER_PRI_ANY);
4866 
4867 	/* Link is down by default */
4868 	if_link_state_change(ifp, LINK_STATE_DOWN);
4869 
4870 	mlx5e_enable_async_events(priv);
4871 
4872 	mlx5e_add_hw_stats(priv);
4873 
4874 	mlx5e_create_stats(&priv->stats.vport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4875 	    "vstats", mlx5e_vport_stats_desc, MLX5E_VPORT_STATS_NUM,
4876 	    priv->stats.vport.arg);
4877 
4878 	mlx5e_create_stats(&priv->stats.pport.ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4879 	    "pstats", mlx5e_pport_stats_desc, MLX5E_PPORT_STATS_NUM,
4880 	    priv->stats.pport.arg);
4881 
4882 	mlx5e_create_ethtool(priv);
4883 
4884 	mtx_lock(&priv->async_events_mtx);
4885 	mlx5e_update_stats(priv);
4886 	mtx_unlock(&priv->async_events_mtx);
4887 
4888 	SYSCTL_ADD_INT(&priv->sysctl_ctx, SYSCTL_CHILDREN(priv->sysctl_ifnet),
4889 	    OID_AUTO, "rx_clbr_done", CTLFLAG_RD,
4890 	    &priv->clbr_done, 0,
4891 	    "RX timestamps calibration state");
4892 	callout_init(&priv->tstmp_clbr, 1);
4893 	/* Pull out the frequency of the clock in hz */
4894 	priv->cclk = (uint64_t)MLX5_CAP_GEN(mdev, device_frequency_khz) * 1000ULL;
4895 	mlx5e_reset_calibration_callout(priv);
4896 
4897 	pa.pa_version = PFIL_VERSION;
4898 	pa.pa_flags = PFIL_IN;
4899 	pa.pa_type = PFIL_TYPE_ETHERNET;
4900 	pa.pa_headname = if_name(ifp);
4901 	priv->pfil = pfil_head_register(&pa);
4902 
4903 	PRIV_LOCK(priv);
4904 	err = mlx5e_open_flow_rules(priv);
4905 	if (err) {
4906 		mlx5_en_err(ifp,
4907 		    "mlx5e_open_flow_rules() failed, %d (ignored)\n", err);
4908 	}
4909 	PRIV_UNLOCK(priv);
4910 
4911 	return (priv);
4912 
4913 err_open_flow_tables:
4914 	mlx5e_close_flow_tables(priv);
4915 
4916 err_open_tirs:
4917 	mlx5e_close_tirs(priv);
4918 
4919 err_open_rqts:
4920 	mlx5e_close_rqts(priv);
4921 
4922 err_open_drop_rq:
4923 	mlx5e_close_drop_rq(&priv->drop_rq);
4924 
4925 err_ipsec_init:
4926 	mlx5e_ipsec_cleanup(priv);
4927 
4928 err_tls_init:
4929 	mlx5e_tls_cleanup(priv);
4930 
4931 err_rl_init:
4932 	mlx5e_rl_cleanup(priv);
4933 
4934 err_create_mkey:
4935 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
4936 
4937 err_dealloc_transport_domain:
4938 	mlx5_dealloc_transport_domain(mdev, priv->tdn, 0);
4939 
4940 err_dealloc_pd:
4941 	mlx5_core_dealloc_pd(mdev, priv->pdn, 0);
4942 
4943 err_free_wq:
4944 	flush_workqueue(priv->wq);
4945 
4946 err_free_sysctl:
4947 	sysctl_ctx_free(&priv->sysctl_ctx);
4948 	if (priv->sysctl_debug)
4949 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
4950 	mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
4951 
4952 err_free_ifp:
4953 	if_free(ifp);
4954 	free(priv, M_MLX5EN);
4955 	return (NULL);
4956 }
4957 
4958 static void
mlx5e_destroy_ifp(struct mlx5_core_dev * mdev,void * vpriv)4959 mlx5e_destroy_ifp(struct mlx5_core_dev *mdev, void *vpriv)
4960 {
4961 	struct mlx5e_priv *priv = vpriv;
4962 	if_t ifp = priv->ifp;
4963 
4964 	/* don't allow more IOCTLs */
4965 	priv->gone = 1;
4966 
4967 	/* XXX wait a bit to allow IOCTL handlers to complete */
4968 	pause("W", hz);
4969 
4970 #ifdef RATELIMIT
4971 	/*
4972 	 * Tell the TCP ratelimit code to release the rate-sets attached
4973 	 * to our ifnet.
4974 	 */
4975 	tcp_rl_release_ifnet(ifp);
4976 	/*
4977 	 * The kernel can still have reference(s) via the m_snd_tag's into
4978 	 * the ratelimit channels, and these must go away before
4979 	 * detaching:
4980 	 */
4981 	while (READ_ONCE(priv->rl.stats.tx_active_connections) != 0) {
4982 		mlx5_en_err(priv->ifp,
4983 		    "Waiting for all ratelimit connections to terminate\n");
4984 		pause("W", hz);
4985 	}
4986 #endif
4987 
4988 #ifdef KERN_TLS
4989 	/* wait for all TLS tags to get freed */
4990 	while (priv->tls.init != 0 &&
4991 	    uma_zone_get_cur(priv->tls.zone) != 0)  {
4992 		mlx5_en_err(priv->ifp,
4993 		    "Waiting for all TLS connections to terminate\n");
4994 		pause("W", hz);
4995 	}
4996 
4997 	/* wait for all TLS RX tags to get freed */
4998 	while (priv->tls_rx.init != 0 &&
4999 	    uma_zone_get_cur(priv->tls_rx.zone) != 0)  {
5000 		mlx5_en_err(priv->ifp,
5001 		    "Waiting for all TLS RX connections to terminate\n");
5002 		pause("W", hz);
5003 	}
5004 #endif
5005 	/* wait for all unlimited send tags to complete */
5006 	mlx5e_priv_wait_for_completion(priv, mdev->priv.eq_table.num_comp_vectors);
5007 
5008 	/* stop watchdog timer */
5009 	callout_drain(&priv->watchdog);
5010 
5011 	callout_drain(&priv->tstmp_clbr);
5012 
5013 	if (priv->vlan_attach != NULL)
5014 		EVENTHANDLER_DEREGISTER(vlan_config, priv->vlan_attach);
5015 	if (priv->vlan_detach != NULL)
5016 		EVENTHANDLER_DEREGISTER(vlan_unconfig, priv->vlan_detach);
5017 	if (priv->vxlan_start != NULL)
5018 		EVENTHANDLER_DEREGISTER(vxlan_start, priv->vxlan_start);
5019 	if (priv->vxlan_stop != NULL)
5020 		EVENTHANDLER_DEREGISTER(vxlan_stop, priv->vxlan_stop);
5021 
5022 	/* make sure device gets closed */
5023 	PRIV_LOCK(priv);
5024 	mlx5e_close_locked(ifp);
5025 	mlx5e_close_flow_rules(priv);
5026 	PRIV_UNLOCK(priv);
5027 
5028 	/* deregister pfil */
5029 	if (priv->pfil != NULL) {
5030 		pfil_head_unregister(priv->pfil);
5031 		priv->pfil = NULL;
5032 	}
5033 
5034 	/* unregister device */
5035 	ifmedia_removeall(&priv->media);
5036 	ether_ifdetach(ifp);
5037 
5038 	mlx5e_tls_rx_cleanup(priv);
5039 #ifdef IPSEC_OFFLOAD
5040 	ipsec_accel_on_ifdown(priv->ifp);
5041 #endif
5042 	mlx5e_close_flow_tables(priv);
5043 	mlx5e_close_tirs(priv);
5044 	mlx5e_close_rqts(priv);
5045 	mlx5e_close_drop_rq(&priv->drop_rq);
5046 	mlx5e_ipsec_cleanup(priv);
5047 	mlx5e_tls_cleanup(priv);
5048 	mlx5e_rl_cleanup(priv);
5049 
5050 	/* destroy all remaining sysctl nodes */
5051 	sysctl_ctx_free(&priv->stats.vport.ctx);
5052 	sysctl_ctx_free(&priv->stats.pport.ctx);
5053 	if (priv->sysctl_debug)
5054 		sysctl_ctx_free(&priv->stats.port_stats_debug.ctx);
5055 	sysctl_ctx_free(&priv->sysctl_ctx);
5056 
5057 	mlx5_core_destroy_mkey(priv->mdev, &priv->mr);
5058 	mlx5_dealloc_transport_domain(priv->mdev, priv->tdn, 0);
5059 	mlx5_core_dealloc_pd(priv->mdev, priv->pdn, 0);
5060 	mlx5e_disable_async_events(priv);
5061 	flush_workqueue(priv->wq);
5062 	mlx5e_priv_static_destroy(priv, mdev, mdev->priv.eq_table.num_comp_vectors);
5063 	if_free(ifp);
5064 	free(priv, M_MLX5EN);
5065 }
5066 
5067 #ifdef DEBUGNET
5068 static void
mlx5_en_debugnet_init(if_t dev,int * nrxr,int * ncl,int * clsize)5069 mlx5_en_debugnet_init(if_t dev, int *nrxr, int *ncl, int *clsize)
5070 {
5071 	struct mlx5e_priv *priv = if_getsoftc(dev);
5072 
5073 	PRIV_LOCK(priv);
5074 	*nrxr = priv->params.num_channels;
5075 	*ncl = DEBUGNET_MAX_IN_FLIGHT;
5076 	*clsize = MLX5E_MAX_RX_BYTES;
5077 	PRIV_UNLOCK(priv);
5078 }
5079 
5080 static void
mlx5_en_debugnet_event(if_t dev,enum debugnet_ev event)5081 mlx5_en_debugnet_event(if_t dev, enum debugnet_ev event)
5082 {
5083 }
5084 
5085 static int
mlx5_en_debugnet_transmit(if_t dev,struct mbuf * m)5086 mlx5_en_debugnet_transmit(if_t dev, struct mbuf *m)
5087 {
5088 	struct mlx5e_priv *priv = if_getsoftc(dev);
5089 	struct mlx5e_sq *sq;
5090 	int err;
5091 
5092 	if ((if_getdrvflags(dev) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
5093 	    IFF_DRV_RUNNING || (priv->media_status_last & IFM_ACTIVE) == 0)
5094 		return (ENOENT);
5095 
5096 	sq = &priv->channel[0].sq[0];
5097 
5098 	if (sq->running == 0) {
5099 		m_freem(m);
5100 		return (ENOENT);
5101 	}
5102 
5103 	if (mlx5e_sq_xmit(sq, &m) != 0) {
5104 		m_freem(m);
5105 		err = ENOBUFS;
5106 	} else {
5107 		err = 0;
5108 	}
5109 
5110 	mlx5e_tx_notify_hw(sq, true);
5111 
5112 	return (err);
5113 }
5114 
5115 static int
mlx5_en_debugnet_poll(if_t dev,int count)5116 mlx5_en_debugnet_poll(if_t dev, int count)
5117 {
5118 	struct mlx5e_priv *priv = if_getsoftc(dev);
5119 
5120 	if ((if_getdrvflags(dev) & IFF_DRV_RUNNING) == 0 ||
5121 	    (priv->media_status_last & IFM_ACTIVE) == 0)
5122 		return (ENOENT);
5123 
5124 	mlx5_poll_interrupts(priv->mdev);
5125 
5126 	return (0);
5127 }
5128 #endif /* DEBUGNET */
5129 
5130 static void *
mlx5e_get_ifp(void * vpriv)5131 mlx5e_get_ifp(void *vpriv)
5132 {
5133 	struct mlx5e_priv *priv = vpriv;
5134 
5135 	return (priv->ifp);
5136 }
5137 
5138 static struct mlx5_interface mlx5e_interface = {
5139 	.add = mlx5e_create_ifp,
5140 	.remove = mlx5e_destroy_ifp,
5141 	.event = mlx5e_async_event,
5142 	.protocol = MLX5_INTERFACE_PROTOCOL_ETH,
5143 	.get_dev = mlx5e_get_ifp,
5144 };
5145 
5146 void
mlx5e_init(void)5147 mlx5e_init(void)
5148 {
5149 	mlx5_register_interface(&mlx5e_interface);
5150 }
5151 
5152 void
mlx5e_cleanup(void)5153 mlx5e_cleanup(void)
5154 {
5155 	mlx5_unregister_interface(&mlx5e_interface);
5156 }
5157 
5158 module_init_order(mlx5e_init, SI_ORDER_SIXTH);
5159 module_exit_order(mlx5e_cleanup, SI_ORDER_SIXTH);
5160 
5161 MODULE_DEPEND(mlx5en, ipsec, 1, 1, 1);
5162 MODULE_DEPEND(mlx5en, linuxkpi, 1, 1, 1);
5163 MODULE_DEPEND(mlx5en, mlx5, 1, 1, 1);
5164 MODULE_VERSION(mlx5en, 1);
5165