xref: /freebsd/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c (revision 6ba2210ee039f2f12878c217bcf058e9c8b26b29)
1 /*-
2  * Copyright (c) 2019 Mellanox Technologies. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  *
25  * $FreeBSD$
26  */
27 
28 #include "opt_kern_tls.h"
29 
30 #include "en.h"
31 
32 #include <dev/mlx5/tls.h>
33 
34 #include <linux/delay.h>
35 #include <sys/ktls.h>
36 #include <opencrypto/cryptodev.h>
37 
38 #ifdef KERN_TLS
39 
40 #ifdef RATELIMIT
41 static if_snd_tag_modify_t mlx5e_tls_rl_snd_tag_modify;
42 #endif
43 static if_snd_tag_query_t mlx5e_tls_snd_tag_query;
44 static if_snd_tag_free_t mlx5e_tls_snd_tag_free;
45 
46 static const struct if_snd_tag_sw mlx5e_tls_snd_tag_sw = {
47 	.snd_tag_query = mlx5e_tls_snd_tag_query,
48 	.snd_tag_free = mlx5e_tls_snd_tag_free,
49 	.type = IF_SND_TAG_TYPE_TLS
50 };
51 
52 #ifdef RATELIMIT
53 static const struct if_snd_tag_sw mlx5e_tls_rl_snd_tag_sw = {
54 	.snd_tag_modify = mlx5e_tls_rl_snd_tag_modify,
55 	.snd_tag_query = mlx5e_tls_snd_tag_query,
56 	.snd_tag_free = mlx5e_tls_snd_tag_free,
57 	.type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT
58 };
59 #endif
60 
61 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS");
62 
63 /* software TLS context */
64 struct mlx5_ifc_sw_tls_cntx_bits {
65 	struct mlx5_ifc_tls_static_params_bits param;
66 	struct mlx5_ifc_tls_progress_params_bits progress;
67 	struct {
68 		uint8_t key_data[8][0x20];
69 		uint8_t key_len[0x20];
70 	} key;
71 };
72 
73 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params));
74 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc));
75 
76 static const char *mlx5e_tls_stats_desc[] = {
77 	MLX5E_TLS_STATS(MLX5E_STATS_DESC)
78 };
79 
80 static void mlx5e_tls_work(struct work_struct *);
81 
82 static int
83 mlx5e_tls_tag_zinit(void *mem, int size, int flags)
84 {
85 	struct mlx5e_tls_tag *ptag = mem;
86 
87 	MPASS(size == sizeof(*ptag));
88 
89 	memset(ptag, 0, sizeof(*ptag));
90 	mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF);
91 	INIT_WORK(&ptag->work, mlx5e_tls_work);
92 
93 	return (0);
94 }
95 
96 static void
97 mlx5e_tls_tag_zfini(void *mem, int size)
98 {
99 	struct mlx5e_tls_tag *ptag = mem;
100 	struct mlx5e_priv *priv;
101 	struct mlx5e_tls *ptls;
102 
103 	ptls = ptag->tls;
104 	priv = container_of(ptls, struct mlx5e_priv, tls);
105 
106 	flush_work(&ptag->work);
107 
108 	if (ptag->tisn != 0) {
109 		mlx5_tls_close_tis(priv->mdev, ptag->tisn);
110 		atomic_add_32(&ptls->num_resources, -1U);
111 	}
112 
113 	mtx_destroy(&ptag->mtx);
114 }
115 
116 static void
117 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag)
118 {
119 
120 	/* reset some variables */
121 	ptag->state = MLX5E_TLS_ST_INIT;
122 	ptag->dek_index = 0;
123 	ptag->dek_index_ok = 0;
124 
125 	/* avoid leaking keys */
126 	memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
127 
128 	/* update number of TIS contexts */
129 	if (ptag->tisn == 0)
130 		atomic_add_32(&ptag->tls->num_resources, -1U);
131 
132 	/* return tag to UMA */
133 	uma_zfree(ptag->tls->zone, ptag);
134 }
135 
136 int
137 mlx5e_tls_init(struct mlx5e_priv *priv)
138 {
139 	struct mlx5e_tls *ptls = &priv->tls;
140 	struct sysctl_oid *node;
141 	uint32_t x;
142 
143 	if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0)
144 		return (0);
145 
146 	ptls->wq = create_singlethread_workqueue("mlx5-tls-wq");
147 	if (ptls->wq == NULL)
148 		return (ENOMEM);
149 
150 	sysctl_ctx_init(&ptls->ctx);
151 
152 	snprintf(ptls->zname, sizeof(ptls->zname),
153 	    "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev));
154 
155 	ptls->zone = uma_zcreate(ptls->zname, sizeof(struct mlx5e_tls_tag),
156 	    NULL, NULL, mlx5e_tls_tag_zinit, mlx5e_tls_tag_zfini, UMA_ALIGN_CACHE, 0);
157 
158 	ptls->max_resources = 1U << MLX5_CAP_GEN(priv->mdev, log_max_dek);
159 
160 	for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
161 		ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
162 
163 	ptls->init = 1;
164 
165 	node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
166 	    SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
167 	    "tls", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS offload");
168 	if (node == NULL)
169 		return (0);
170 
171 	mlx5e_create_counter_stats(&ptls->ctx,
172 	    SYSCTL_CHILDREN(node), "stats",
173 	    mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM,
174 	    ptls->stats.arg);
175 
176 	return (0);
177 }
178 
179 void
180 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
181 {
182 	struct mlx5e_tls *ptls = &priv->tls;
183 	uint32_t x;
184 
185 	if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0)
186 		return;
187 
188 	ptls->init = 0;
189 	flush_workqueue(ptls->wq);
190 	sysctl_ctx_free(&ptls->ctx);
191 	uma_zdestroy(ptls->zone);
192 	destroy_workqueue(ptls->wq);
193 
194 	/* check if all resources are freed */
195 	MPASS(priv->tls.num_resources == 0);
196 
197 	for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
198 		counter_u64_free(ptls->stats.arg[x]);
199 }
200 
201 static void
202 mlx5e_tls_work(struct work_struct *work)
203 {
204 	struct mlx5e_tls_tag *ptag;
205 	struct mlx5e_priv *priv;
206 	int err;
207 
208 	ptag = container_of(work, struct mlx5e_tls_tag, work);
209 	priv = container_of(ptag->tls, struct mlx5e_priv, tls);
210 
211 	switch (ptag->state) {
212 	case MLX5E_TLS_ST_INIT:
213 		/* try to open TIS, if not present */
214 		if (ptag->tisn == 0) {
215 			err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn,
216 			    priv->pdn, &ptag->tisn);
217 			if (err) {
218 				MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
219 				break;
220 			}
221 		}
222 		MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn);
223 
224 		/* try to allocate a DEK context ID */
225 		err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
226 		    MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
227 		    MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
228 		    &ptag->dek_index);
229 		if (err) {
230 			MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
231 			break;
232 		}
233 
234 		MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
235 
236 		ptag->dek_index_ok = 1;
237 
238 		MLX5E_TLS_TAG_LOCK(ptag);
239 		if (ptag->state == MLX5E_TLS_ST_INIT)
240 			ptag->state = MLX5E_TLS_ST_SETUP;
241 		MLX5E_TLS_TAG_UNLOCK(ptag);
242 		break;
243 
244 	case MLX5E_TLS_ST_FREED:
245 		/* wait for all refs to go away */
246 		while (ptag->refs != 0)
247 			msleep(1);
248 
249 		/* try to destroy DEK context by ID */
250 		if (ptag->dek_index_ok)
251 			err = mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
252 
253 		/* free tag */
254 		mlx5e_tls_tag_zfree(ptag);
255 		break;
256 
257 	default:
258 		break;
259 	}
260 }
261 
262 static int
263 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en)
264 {
265 
266 	MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2);
267 	if (en->tls_vminor == TLS_MINOR_VER_TWO)
268 		MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */
269 	else
270 		MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */
271 	MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1);
272 	MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */
273 
274 	/* copy the initial vector in place */
275 	switch (en->iv_len) {
276 	case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv):
277 	case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) +
278 	     MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv):
279 		memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv),
280 		    en->iv, en->iv_len);
281 		break;
282 	default:
283 		return (EINVAL);
284 	}
285 
286 	if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) {
287 		memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data),
288 		    en->cipher_key, en->cipher_key_len);
289 		MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len);
290 	} else {
291 		return (EINVAL);
292 	}
293 	return (0);
294 }
295 
296 /* Verify zero default */
297 CTASSERT(MLX5E_TLS_ST_INIT == 0);
298 
299 int
300 mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
301     union if_snd_tag_alloc_params *params,
302     struct m_snd_tag **ppmt)
303 {
304 	union if_snd_tag_alloc_params rl_params;
305 	const struct if_snd_tag_sw *snd_tag_sw;
306 	struct mlx5e_priv *priv;
307 	struct mlx5e_tls_tag *ptag;
308 	const struct tls_session_params *en;
309 	int error;
310 
311 	priv = ifp->if_softc;
312 
313 	if (priv->gone != 0 || priv->tls.init == 0)
314 		return (EOPNOTSUPP);
315 
316 	/* allocate new tag from zone, if any */
317 	ptag = uma_zalloc(priv->tls.zone, M_NOWAIT);
318 	if (ptag == NULL)
319 		return (ENOMEM);
320 
321 	/* sanity check default values */
322 	MPASS(ptag->state == MLX5E_TLS_ST_INIT);
323 	MPASS(ptag->dek_index == 0);
324 	MPASS(ptag->dek_index_ok == 0);
325 
326 	/* setup TLS tag */
327 	ptag->tls = &priv->tls;
328 
329 	/* check if there is no TIS context */
330 	if (ptag->tisn == 0) {
331 		uint32_t value;
332 
333 		value = atomic_fetchadd_32(&priv->tls.num_resources, 1U);
334 
335 		/* check resource limits */
336 		if (value >= priv->tls.max_resources) {
337 			error = ENOMEM;
338 			goto failure;
339 		}
340 	}
341 
342 	en = &params->tls.tls->params;
343 
344 	/* only TLS v1.2 and v1.3 is currently supported */
345 	if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
346 	    (en->tls_vminor != TLS_MINOR_VER_TWO
347 #ifdef TLS_MINOR_VER_THREE
348 	     && en->tls_vminor != TLS_MINOR_VER_THREE
349 #endif
350 	     )) {
351 		error = EPROTONOSUPPORT;
352 		goto failure;
353 	}
354 
355 	switch (en->cipher_algorithm) {
356 	case CRYPTO_AES_NIST_GCM_16:
357 		switch (en->cipher_key_len) {
358 		case 128 / 8:
359 			if (en->tls_vminor == TLS_MINOR_VER_TWO) {
360 				if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
361 					error = EPROTONOSUPPORT;
362 					goto failure;
363 				}
364 			} else {
365 				if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
366 					error = EPROTONOSUPPORT;
367 					goto failure;
368 				}
369 			}
370 			error = mlx5e_tls_set_params(ptag->crypto_params, en);
371 			if (error)
372 				goto failure;
373 			break;
374 
375 		case 256 / 8:
376 			if (en->tls_vminor == TLS_MINOR_VER_TWO) {
377 				if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
378 					error = EPROTONOSUPPORT;
379 					goto failure;
380 				}
381 			} else {
382 				if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
383 					error = EPROTONOSUPPORT;
384 					goto failure;
385 				}
386 			}
387 			error = mlx5e_tls_set_params(ptag->crypto_params, en);
388 			if (error)
389 				goto failure;
390 			break;
391 
392 		default:
393 			error = EINVAL;
394 			goto failure;
395 		}
396 		break;
397 	default:
398 		error = EPROTONOSUPPORT;
399 		goto failure;
400 	}
401 
402 	memset(&rl_params, 0, sizeof(rl_params));
403 	rl_params.hdr = params->hdr;
404 	switch (params->hdr.type) {
405 #ifdef RATELIMIT
406 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
407 		rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT;
408 		rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
409 		snd_tag_sw = &mlx5e_tls_rl_snd_tag_sw;
410 		break;
411 #endif
412 	case IF_SND_TAG_TYPE_TLS:
413 		rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED;
414 		snd_tag_sw = &mlx5e_tls_snd_tag_sw;
415 		break;
416 	default:
417 		error = EOPNOTSUPP;
418 		goto failure;
419 	}
420 
421 	error = m_snd_tag_alloc(ifp, &rl_params, &ptag->rl_tag);
422 	if (error)
423 		goto failure;
424 
425 	/* store pointer to mbuf tag */
426 	MPASS(ptag->tag.refcount == 0);
427 	m_snd_tag_init(&ptag->tag, ifp, snd_tag_sw);
428 	*ppmt = &ptag->tag;
429 
430 	queue_work(priv->tls.wq, &ptag->work);
431 	flush_work(&ptag->work);
432 
433 	return (0);
434 
435 failure:
436 	mlx5e_tls_tag_zfree(ptag);
437 	return (error);
438 }
439 
440 #ifdef RATELIMIT
441 static int
442 mlx5e_tls_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
443 {
444 	union if_snd_tag_modify_params rl_params;
445 	struct mlx5e_tls_tag *ptag =
446 	    container_of(pmt, struct mlx5e_tls_tag, tag);
447 	int error;
448 
449 	memset(&rl_params, 0, sizeof(rl_params));
450 	rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
451 	error = ptag->rl_tag->sw->snd_tag_modify(ptag->rl_tag, &rl_params);
452 	return (error);
453 }
454 #endif
455 
456 static int
457 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
458 {
459 	struct mlx5e_tls_tag *ptag =
460 	    container_of(pmt, struct mlx5e_tls_tag, tag);
461 
462 	return (ptag->rl_tag->sw->snd_tag_query(ptag->rl_tag, params));
463 }
464 
465 static void
466 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt)
467 {
468 	struct mlx5e_tls_tag *ptag =
469 	    container_of(pmt, struct mlx5e_tls_tag, tag);
470 	struct mlx5e_priv *priv;
471 
472 	m_snd_tag_rele(ptag->rl_tag);
473 
474 	MLX5E_TLS_TAG_LOCK(ptag);
475 	ptag->state = MLX5E_TLS_ST_FREED;
476 	MLX5E_TLS_TAG_UNLOCK(ptag);
477 
478 	priv = ptag->tag.ifp->if_softc;
479 	queue_work(priv->tls.wq, &ptag->work);
480 }
481 
482 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0);
483 
484 static void
485 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
486 {
487 	const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
488 	    MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS);
489 	struct mlx5e_tx_umr_wqe *wqe;
490 	u16 pi;
491 
492 	pi = sq->pc & sq->wq.sz_m1;
493 	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
494 
495 	memset(wqe, 0, sizeof(*wqe));
496 
497 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
498 	    MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24));
499 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
500 	wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8);
501 
502 	if (mlx5e_do_send_cqe(sq))
503 		wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
504 	else
505 		wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
506 
507 	/* fill out UMR control segment */
508 	wqe->umr.flags = 0x80;	/* inline data */
509 	wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16);
510 
511 	/* copy in the static crypto parameters */
512 	memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param),
513 	    MLX5_FLD_SZ_BYTES(sw_tls_cntx, param));
514 
515 	/* copy data for doorbell */
516 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
517 
518 	sq->mbuf[pi].mbuf = NULL;
519 	sq->mbuf[pi].num_bytes = 0;
520 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
521 	sq->mbuf[pi].p_refcount = &ptag->refs;
522 	atomic_add_int(&ptag->refs, 1);
523 	sq->pc += sq->mbuf[pi].num_wqebbs;
524 }
525 
526 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) ==
527     sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv));
528 
529 static void
530 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
531 {
532 	const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
533 	    MLX5_SEND_WQE_DS);
534 	struct mlx5e_tx_psv_wqe *wqe;
535 	u16 pi;
536 
537 	pi = sq->pc & sq->wq.sz_m1;
538 	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
539 
540 	memset(wqe, 0, sizeof(*wqe));
541 
542 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
543 	    MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24));
544 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
545 
546 	if (mlx5e_do_send_cqe(sq))
547 		wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
548 
549 	/* copy in the PSV control segment */
550 	memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress),
551 	    sizeof(wqe->psv));
552 
553 	/* copy data for doorbell */
554 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
555 
556 	sq->mbuf[pi].mbuf = NULL;
557 	sq->mbuf[pi].num_bytes = 0;
558 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
559 	sq->mbuf[pi].p_refcount = &ptag->refs;
560 	atomic_add_int(&ptag->refs, 1);
561 	sq->pc += sq->mbuf[pi].num_wqebbs;
562 }
563 
564 static void
565 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
566 {
567 	const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS;
568 	struct mlx5e_tx_wqe *wqe;
569 	u16 pi;
570 
571 	pi = sq->pc & sq->wq.sz_m1;
572 	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
573 
574 	memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
575 
576 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
577 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
578 	if (mlx5e_do_send_cqe(sq))
579 		wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
580 	else
581 		wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
582 
583 	/* Copy data for doorbell */
584 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
585 
586 	sq->mbuf[pi].mbuf = NULL;
587 	sq->mbuf[pi].num_bytes = 0;
588 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
589 	sq->mbuf[pi].p_refcount = &ptag->refs;
590 	atomic_add_int(&ptag->refs, 1);
591 	sq->pc += sq->mbuf[pi].num_wqebbs;
592 }
593 
594 #define	SBTLS_MBUF_NO_DATA ((struct mbuf *)1)
595 
596 static struct mbuf *
597 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq, bool *pis_start)
598 {
599 	struct mbuf *mr, *top;
600 	uint32_t offset;
601 	uint32_t delta;
602 
603 	/* check format of incoming mbuf */
604 	if (mb->m_next == NULL ||
605 	    (mb->m_next->m_flags & (M_EXTPG | M_EXT)) != (M_EXTPG | M_EXT)) {
606 		top = NULL;
607 		goto done;
608 	}
609 
610 	/* get unmapped data offset */
611 	offset = mtod(mb->m_next, uintptr_t);
612 
613 	/* check if we don't need to re-transmit anything */
614 	if (offset == 0) {
615 		top = SBTLS_MBUF_NO_DATA;
616 		*pis_start = true;
617 		goto done;
618 	}
619 
620 	/* try to get a new  packet header */
621 	top = m_gethdr(wait, MT_DATA);
622 	if (top == NULL)
623 		goto done;
624 
625 	mr = m_get(wait, MT_DATA);
626 	if (mr == NULL) {
627 		m_free(top);
628 		top = NULL;
629 		goto done;
630 	}
631 
632 	top->m_next = mr;
633 
634 	mb_dupcl(mr, mb->m_next);
635 
636 	/* the beginning of the TLS record */
637 	mr->m_data = NULL;
638 
639 	/* setup packet header length */
640 	top->m_pkthdr.len = mr->m_len = offset;
641 	top->m_len = 0;
642 
643 	/* check for partial re-transmit */
644 	delta = *ptcp_seq - tcp_old;
645 
646 	if (delta < offset) {
647 		m_adj(top, offset - delta);
648 		offset = delta;
649 
650 		/* continue where we left off */
651 		*pis_start = false;
652 	} else {
653 		*pis_start = true;
654 	}
655 
656 	/*
657 	 * Rewind the TCP sequence number by the amount of data
658 	 * retransmitted:
659 	 */
660 	*ptcp_seq -= offset;
661 done:
662 	return (top);
663 }
664 
665 static int
666 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq)
667 {
668 
669 	for (; mb != NULL; mb = mb->m_next) {
670 		if (!(mb->m_flags & M_EXTPG))
671 			continue;
672 		*pseq = mb->m_epg_seqno;
673 		return (1);
674 	}
675 	return (0);
676 }
677 
678 int
679 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb)
680 {
681 	struct mlx5e_tls_tag *ptls_tag;
682 	struct m_snd_tag *ptag;
683 	const struct tcphdr *th;
684 	struct mbuf *mb = *ppmb;
685 	u64 rcd_sn;
686 	u32 header_size;
687 	u32 mb_seq;
688 
689 	if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0)
690 		return (MLX5E_TLS_CONTINUE);
691 
692 	ptag = mb->m_pkthdr.snd_tag;
693 
694 	if (
695 #ifdef RATELIMIT
696 	    ptag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT &&
697 #endif
698 	    ptag->sw->type != IF_SND_TAG_TYPE_TLS)
699 		return (MLX5E_TLS_CONTINUE);
700 
701 	ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag);
702 
703 	header_size = mlx5e_get_full_header_size(mb, &th);
704 	if (unlikely(header_size == 0 || th == NULL))
705 		return (MLX5E_TLS_FAILURE);
706 
707 	/*
708 	 * Send non-TLS TCP packets AS-IS:
709 	 */
710 	if (header_size == mb->m_pkthdr.len ||
711 	    mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) {
712 		parg->tisn = 0;
713 		parg->ihs = header_size;
714 		return (MLX5E_TLS_CONTINUE);
715 	}
716 
717 	mb_seq = ntohl(th->th_seq);
718 
719 	MLX5E_TLS_TAG_LOCK(ptls_tag);
720 	switch (ptls_tag->state) {
721 	case MLX5E_TLS_ST_INIT:
722 		MLX5E_TLS_TAG_UNLOCK(ptls_tag);
723 		return (MLX5E_TLS_FAILURE);
724 	case MLX5E_TLS_ST_SETUP:
725 		ptls_tag->state = MLX5E_TLS_ST_TXRDY;
726 		ptls_tag->expected_seq = ~mb_seq;	/* force setup */
727 	default:
728 		MLX5E_TLS_TAG_UNLOCK(ptls_tag);
729 		break;
730 	}
731 
732 	if (unlikely(ptls_tag->expected_seq != mb_seq)) {
733 		bool is_start;
734 		struct mbuf *r_mb;
735 		uint32_t tcp_seq = mb_seq;
736 
737 		r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq, &is_start);
738 		if (r_mb == NULL) {
739 			MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1);
740 			return (MLX5E_TLS_FAILURE);
741 		}
742 
743 		MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1);
744 
745 		/* check if this is the first fragment of a TLS record */
746 		if (is_start) {
747 			/* setup TLS static parameters */
748 			MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params,
749 			    param.initial_record_number, rcd_sn);
750 
751 			/*
752 			 * NOTE: The sendqueue should have enough room to
753 			 * carry both the static and the progress parameters
754 			 * when we get here!
755 			 */
756 			mlx5e_tls_send_static_parameters(sq, ptls_tag);
757 			mlx5e_tls_send_progress_parameters(sq, ptls_tag);
758 
759 			if (r_mb == SBTLS_MBUF_NO_DATA) {
760 				mlx5e_tls_send_nop(sq, ptls_tag);
761 				ptls_tag->expected_seq = mb_seq;
762 				return (MLX5E_TLS_LOOP);
763 			}
764 		}
765 
766 		MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len);
767 
768 		/* setup transmit arguments */
769 		parg->tisn = ptls_tag->tisn;
770 		parg->pref = &ptls_tag->refs;
771 
772 		/* try to send DUMP data */
773 		if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) {
774 			m_freem(r_mb);
775 			ptls_tag->expected_seq = tcp_seq;
776 			return (MLX5E_TLS_FAILURE);
777 		} else {
778 			ptls_tag->expected_seq = mb_seq;
779 			return (MLX5E_TLS_LOOP);
780 		}
781 	} else {
782 		MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1);
783 		MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len);
784 	}
785 	ptls_tag->expected_seq += mb->m_pkthdr.len - header_size;
786 
787 	parg->tisn = ptls_tag->tisn;
788 	parg->ihs = header_size;
789 	parg->pref = &ptls_tag->refs;
790 	return (MLX5E_TLS_CONTINUE);
791 }
792 
793 #else
794 
795 int
796 mlx5e_tls_init(struct mlx5e_priv *priv)
797 {
798 
799 	return (0);
800 }
801 
802 void
803 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
804 {
805 	/* NOP */
806 }
807 
808 #endif		/* KERN_TLS */
809