xref: /freebsd/sys/dev/mlx5/mlx5_en/mlx5_en_hw_tls.c (revision 2f513db72b034fd5ef7f080b11be5c711c15186a)
1 /*-
2  * Copyright (c) 2019 Mellanox Technologies. All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. Redistributions in binary form must reproduce the above copyright
10  *    notice, this list of conditions and the following disclaimer in the
11  *    documentation and/or other materials provided with the distribution.
12  *
13  * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16  * ARE DISCLAIMED.  IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23  * SUCH DAMAGE.
24  *
25  * $FreeBSD$
26  */
27 
28 #include "opt_kern_tls.h"
29 
30 #include "en.h"
31 
32 #include <dev/mlx5/tls.h>
33 
34 #include <linux/delay.h>
35 #include <sys/ktls.h>
36 #include <opencrypto/cryptodev.h>
37 
38 #ifdef KERN_TLS
39 
40 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS");
41 
42 /* software TLS context */
43 struct mlx5_ifc_sw_tls_cntx_bits {
44 	struct mlx5_ifc_tls_static_params_bits param;
45 	struct mlx5_ifc_tls_progress_params_bits progress;
46 	struct {
47 		uint8_t key_data[8][0x20];
48 		uint8_t key_len[0x20];
49 	} key;
50 };
51 
52 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params));
53 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc));
54 
55 static const char *mlx5e_tls_stats_desc[] = {
56 	MLX5E_TLS_STATS(MLX5E_STATS_DESC)
57 };
58 
59 static void mlx5e_tls_work(struct work_struct *);
60 
61 static int
62 mlx5e_tls_tag_zinit(void *mem, int size, int flags)
63 {
64 	struct mlx5e_tls_tag *ptag = mem;
65 
66 	MPASS(size == sizeof(*ptag));
67 
68 	memset(ptag, 0, sizeof(*ptag));
69 	mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF);
70 	INIT_WORK(&ptag->work, mlx5e_tls_work);
71 
72 	return (0);
73 }
74 
75 static void
76 mlx5e_tls_tag_zfini(void *mem, int size)
77 {
78 	struct mlx5e_tls_tag *ptag = mem;
79 	struct mlx5e_priv *priv;
80 	struct mlx5e_tls *ptls;
81 
82 	ptls = ptag->tls;
83 	priv = container_of(ptls, struct mlx5e_priv, tls);
84 
85 	flush_work(&ptag->work);
86 
87 	if (ptag->tisn != 0) {
88 		mlx5_tls_close_tis(priv->mdev, ptag->tisn);
89 		atomic_add_32(&ptls->num_resources, -1U);
90 	}
91 
92 	mtx_destroy(&ptag->mtx);
93 }
94 
95 static void
96 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag)
97 {
98 
99 	/* reset some variables */
100 	ptag->state = MLX5E_TLS_ST_INIT;
101 	ptag->dek_index = 0;
102 	ptag->dek_index_ok = 0;
103 
104 	/* avoid leaking keys */
105 	memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
106 
107 	/* update number of TIS contexts */
108 	if (ptag->tisn == 0)
109 		atomic_add_32(&ptag->tls->num_resources, -1U);
110 
111 	/* return tag to UMA */
112 	uma_zfree(ptag->tls->zone, ptag);
113 }
114 
115 int
116 mlx5e_tls_init(struct mlx5e_priv *priv)
117 {
118 	struct mlx5e_tls *ptls = &priv->tls;
119 	struct sysctl_oid *node;
120 	uint32_t x;
121 
122 	if (MLX5_CAP_GEN(priv->mdev, tls) == 0)
123 		return (0);
124 
125 	ptls->wq = create_singlethread_workqueue("mlx5-tls-wq");
126 	if (ptls->wq == NULL)
127 		return (ENOMEM);
128 
129 	sysctl_ctx_init(&ptls->ctx);
130 
131 	snprintf(ptls->zname, sizeof(ptls->zname),
132 	    "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev));
133 
134 	ptls->zone = uma_zcreate(ptls->zname, sizeof(struct mlx5e_tls_tag),
135 	    NULL, NULL, mlx5e_tls_tag_zinit, mlx5e_tls_tag_zfini, UMA_ALIGN_CACHE, 0);
136 
137 	ptls->max_resources = 1U << MLX5_CAP_GEN(priv->mdev, log_max_dek);
138 
139 	for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
140 		ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
141 
142 	ptls->init = 1;
143 
144 	node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
145 	    SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
146 	    "tls", CTLFLAG_RW, NULL, "Hardware TLS offload");
147 	if (node == NULL)
148 		return (0);
149 
150 	mlx5e_create_counter_stats(&ptls->ctx,
151 	    SYSCTL_CHILDREN(node), "stats",
152 	    mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM,
153 	    ptls->stats.arg);
154 
155 	return (0);
156 }
157 
158 void
159 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
160 {
161 	struct mlx5e_tls *ptls = &priv->tls;
162 	uint32_t x;
163 
164 	if (MLX5_CAP_GEN(priv->mdev, tls) == 0)
165 		return;
166 
167 	ptls->init = 0;
168 	flush_workqueue(ptls->wq);
169 	sysctl_ctx_free(&ptls->ctx);
170 	uma_zdestroy(ptls->zone);
171 	destroy_workqueue(ptls->wq);
172 
173 	/* check if all resources are freed */
174 	MPASS(priv->tls.num_resources == 0);
175 
176 	for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
177 		counter_u64_free(ptls->stats.arg[x]);
178 }
179 
180 static void
181 mlx5e_tls_work(struct work_struct *work)
182 {
183 	struct mlx5e_tls_tag *ptag;
184 	struct mlx5e_priv *priv;
185 	int err;
186 
187 	ptag = container_of(work, struct mlx5e_tls_tag, work);
188 	priv = container_of(ptag->tls, struct mlx5e_priv, tls);
189 
190 	switch (ptag->state) {
191 	case MLX5E_TLS_ST_SETUP:
192 		/* try to open TIS, if not present */
193 		if (ptag->tisn == 0) {
194 			err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn,
195 			    priv->pdn, &ptag->tisn);
196 			if (err) {
197 				MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
198 				break;
199 			}
200 		}
201 		MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn);
202 
203 		/* try to allocate a DEK context ID */
204 		err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
205 		    MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
206 		    MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
207 		    &ptag->dek_index);
208 		if (err) {
209 			MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
210 			break;
211 		}
212 
213 		MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
214 
215 		ptag->dek_index_ok = 1;
216 
217 		MLX5E_TLS_TAG_LOCK(ptag);
218 		if (ptag->state == MLX5E_TLS_ST_SETUP)
219 			ptag->state = MLX5E_TLS_ST_TXRDY;
220 		MLX5E_TLS_TAG_UNLOCK(ptag);
221 		break;
222 
223 	case MLX5E_TLS_ST_FREED:
224 		/* wait for all refs to go away */
225 		while (ptag->refs != 0)
226 			msleep(1);
227 
228 		/* try to destroy DEK context by ID */
229 		if (ptag->dek_index_ok)
230 			err = mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
231 
232 		/* free tag */
233 		mlx5e_tls_tag_zfree(ptag);
234 		break;
235 
236 	default:
237 		break;
238 	}
239 }
240 
241 static int
242 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en)
243 {
244 
245 	MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2);
246 	if (en->tls_vminor == TLS_MINOR_VER_TWO)
247 		MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */
248 	else
249 		MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */
250 	MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1);
251 	MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */
252 
253 	/* copy the initial vector in place */
254 	if (en->iv_len == MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv)) {
255 		memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv),
256 		    en->iv, MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv));
257 	} else if (en->iv_len == (MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) +
258 				  MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv))) {
259 		memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv),
260 		    (char *)en->iv + MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv),
261 		    MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv));
262 		memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.implicit_iv),
263 		    en->iv,
264 		    MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv));
265 	} else {
266 		return (EINVAL);
267 	}
268 
269 	if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) {
270 		memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data),
271 		    en->cipher_key, en->cipher_key_len);
272 		MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len);
273 	} else {
274 		return (EINVAL);
275 	}
276 	return (0);
277 }
278 
279 /* Verify zero default */
280 CTASSERT(MLX5E_TLS_ST_INIT == 0);
281 
282 int
283 mlx5e_tls_snd_tag_alloc(struct ifnet *ifp,
284     union if_snd_tag_alloc_params *params,
285     struct m_snd_tag **ppmt)
286 {
287 	struct if_snd_tag_alloc_rate_limit rl_params;
288 	struct mlx5e_priv *priv;
289 	struct mlx5e_tls_tag *ptag;
290 	const struct tls_session_params *en;
291 	int error;
292 
293 	priv = ifp->if_softc;
294 
295 	if (priv->tls.init == 0)
296 		return (EOPNOTSUPP);
297 
298 	/* allocate new tag from zone, if any */
299 	ptag = uma_zalloc(priv->tls.zone, M_NOWAIT);
300 	if (ptag == NULL)
301 		return (ENOMEM);
302 
303 	/* sanity check default values */
304 	MPASS(ptag->state == MLX5E_TLS_ST_INIT);
305 	MPASS(ptag->dek_index == 0);
306 	MPASS(ptag->dek_index_ok == 0);
307 
308 	/* setup TLS tag */
309 	ptag->tls = &priv->tls;
310 	ptag->tag.type = params->hdr.type;
311 
312 	/* check if there is no TIS context */
313 	if (ptag->tisn == 0) {
314 		uint32_t value;
315 
316 		value = atomic_fetchadd_32(&priv->tls.num_resources, 1U);
317 
318 		/* check resource limits */
319 		if (value >= priv->tls.max_resources) {
320 			error = ENOMEM;
321 			goto failure;
322 		}
323 	}
324 
325 	en = &params->tls.tls->params;
326 
327 	/* only TLS v1.2 and v1.3 is currently supported */
328 	if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
329 	    (en->tls_vminor != TLS_MINOR_VER_TWO
330 #ifdef TLS_MINOR_VER_THREE
331 	     && en->tls_vminor != TLS_MINOR_VER_THREE
332 #endif
333 	     )) {
334 		error = EPROTONOSUPPORT;
335 		goto failure;
336 	}
337 
338 	switch (en->cipher_algorithm) {
339 	case CRYPTO_AES_NIST_GCM_16:
340 		switch (en->cipher_key_len) {
341 		case 128 / 8:
342 			if (en->tls_vminor == TLS_MINOR_VER_TWO) {
343 				if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
344 					error = EPROTONOSUPPORT;
345 					goto failure;
346 				}
347 			} else {
348 				if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
349 					error = EPROTONOSUPPORT;
350 					goto failure;
351 				}
352 			}
353 			error = mlx5e_tls_set_params(ptag->crypto_params, en);
354 			if (error)
355 				goto failure;
356 			break;
357 
358 		case 256 / 8:
359 			if (en->tls_vminor == TLS_MINOR_VER_TWO) {
360 				if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
361 					error = EPROTONOSUPPORT;
362 					goto failure;
363 				}
364 			} else {
365 				if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
366 					error = EPROTONOSUPPORT;
367 					goto failure;
368 				}
369 			}
370 			error = mlx5e_tls_set_params(ptag->crypto_params, en);
371 			if (error)
372 				goto failure;
373 			break;
374 
375 		default:
376 			error = EINVAL;
377 			goto failure;
378 		}
379 		break;
380 	default:
381 		error = EPROTONOSUPPORT;
382 		goto failure;
383 	}
384 
385 	switch (ptag->tag.type) {
386 #if defined(RATELIMIT) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
387 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
388 		memset(&rl_params, 0, sizeof(rl_params));
389 		rl_params.hdr = params->tls_rate_limit.hdr;
390 		rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT;
391 		rl_params.max_rate = params->tls_rate_limit.max_rate;
392 
393 		error = mlx5e_rl_snd_tag_alloc(ifp,
394 		    container_of(&rl_params, union if_snd_tag_alloc_params, rate_limit),
395 		    &ptag->rl_tag);
396 		if (error)
397 			goto failure;
398 		break;
399 #endif
400 	case IF_SND_TAG_TYPE_TLS:
401 		memset(&rl_params, 0, sizeof(rl_params));
402 		rl_params.hdr = params->tls.hdr;
403 		rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED;
404 
405 		error = mlx5e_ul_snd_tag_alloc(ifp,
406 		    container_of(&rl_params, union if_snd_tag_alloc_params, unlimited),
407 		    &ptag->rl_tag);
408 		if (error)
409 			goto failure;
410 		break;
411 	default:
412 		error = EOPNOTSUPP;
413 		goto failure;
414 	}
415 
416 	/* store pointer to mbuf tag */
417 	MPASS(ptag->tag.m_snd_tag.refcount == 0);
418 	m_snd_tag_init(&ptag->tag.m_snd_tag, ifp);
419 	*ppmt = &ptag->tag.m_snd_tag;
420 	return (0);
421 
422 failure:
423 	mlx5e_tls_tag_zfree(ptag);
424 	return (error);
425 }
426 
427 int
428 mlx5e_tls_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
429 {
430 #if defined(RATELIMIT) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
431 	struct if_snd_tag_rate_limit_params rl_params;
432 	int error;
433 #endif
434 	struct mlx5e_tls_tag *ptag =
435 	    container_of(pmt, struct mlx5e_tls_tag, tag.m_snd_tag);
436 
437 	switch (ptag->tag.type) {
438 #if defined(RATELIMIT) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
439 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
440 		memset(&rl_params, 0, sizeof(rl_params));
441 		rl_params.max_rate = params->tls_rate_limit.max_rate;
442 		error = mlx5e_rl_snd_tag_modify(ptag->rl_tag,
443 		    container_of(&rl_params, union if_snd_tag_modify_params, rate_limit));
444 		return (error);
445 #endif
446 	default:
447 		return (EOPNOTSUPP);
448 	}
449 }
450 
451 int
452 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
453 {
454 	struct mlx5e_tls_tag *ptag =
455 	    container_of(pmt, struct mlx5e_tls_tag, tag.m_snd_tag);
456 	int error;
457 
458 	switch (ptag->tag.type) {
459 #if defined(RATELIMIT) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
460 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
461 		error = mlx5e_rl_snd_tag_query(ptag->rl_tag, params);
462 		break;
463 #endif
464 	case IF_SND_TAG_TYPE_TLS:
465 		error = mlx5e_ul_snd_tag_query(ptag->rl_tag, params);
466 		break;
467 	default:
468 		error = EOPNOTSUPP;
469 		break;
470 	}
471 	return (error);
472 }
473 
474 void
475 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt)
476 {
477 	struct mlx5e_tls_tag *ptag =
478 	    container_of(pmt, struct mlx5e_tls_tag, tag.m_snd_tag);
479 	struct mlx5e_priv *priv;
480 
481 	switch (ptag->tag.type) {
482 #if defined(RATELIMIT) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
483 	case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
484 		mlx5e_rl_snd_tag_free(ptag->rl_tag);
485 		break;
486 #endif
487 	case IF_SND_TAG_TYPE_TLS:
488 		mlx5e_ul_snd_tag_free(ptag->rl_tag);
489 		break;
490 	default:
491 		break;
492 	}
493 
494 	MLX5E_TLS_TAG_LOCK(ptag);
495 	ptag->state = MLX5E_TLS_ST_FREED;
496 	MLX5E_TLS_TAG_UNLOCK(ptag);
497 
498 	priv = ptag->tag.m_snd_tag.ifp->if_softc;
499 	queue_work(priv->tls.wq, &ptag->work);
500 }
501 
502 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0);
503 
504 static void
505 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
506 {
507 	const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
508 	    MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS);
509 	struct mlx5e_tx_umr_wqe *wqe;
510 	u16 pi;
511 
512 	pi = sq->pc & sq->wq.sz_m1;
513 	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
514 
515 	memset(wqe, 0, sizeof(*wqe));
516 
517 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
518 	    MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24));
519 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
520 	wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8);
521 
522 	if (mlx5e_do_send_cqe(sq))
523 		wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
524 	else
525 		wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
526 
527 	/* fill out UMR control segment */
528 	wqe->umr.flags = 0x80;	/* inline data */
529 	wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16);
530 
531 	/* copy in the static crypto parameters */
532 	memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param),
533 	    MLX5_FLD_SZ_BYTES(sw_tls_cntx, param));
534 
535 	/* copy data for doorbell */
536 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
537 
538 	sq->mbuf[pi].mbuf = NULL;
539 	sq->mbuf[pi].num_bytes = 0;
540 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
541 	sq->mbuf[pi].p_refcount = &ptag->refs;
542 	atomic_add_int(&ptag->refs, 1);
543 	sq->pc += sq->mbuf[pi].num_wqebbs;
544 }
545 
546 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) ==
547     sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv));
548 
549 static void
550 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
551 {
552 	const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
553 	    MLX5_SEND_WQE_DS);
554 	struct mlx5e_tx_psv_wqe *wqe;
555 	u16 pi;
556 
557 	pi = sq->pc & sq->wq.sz_m1;
558 	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
559 
560 	memset(wqe, 0, sizeof(*wqe));
561 
562 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
563 	    MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24));
564 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
565 
566 	if (mlx5e_do_send_cqe(sq))
567 		wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
568 	else
569 		wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
570 
571 	/* copy in the PSV control segment */
572 	memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress),
573 	    sizeof(wqe->psv));
574 
575 	/* copy data for doorbell */
576 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
577 
578 	sq->mbuf[pi].mbuf = NULL;
579 	sq->mbuf[pi].num_bytes = 0;
580 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
581 	sq->mbuf[pi].p_refcount = &ptag->refs;
582 	atomic_add_int(&ptag->refs, 1);
583 	sq->pc += sq->mbuf[pi].num_wqebbs;
584 }
585 
586 static void
587 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
588 {
589 	const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS;
590 	struct mlx5e_tx_wqe *wqe;
591 	u16 pi;
592 
593 	pi = sq->pc & sq->wq.sz_m1;
594 	wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
595 
596 	memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
597 
598 	wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
599 	wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
600 	if (mlx5e_do_send_cqe(sq))
601 		wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
602 	else
603 		wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
604 
605 	/* Copy data for doorbell */
606 	memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
607 
608 	sq->mbuf[pi].mbuf = NULL;
609 	sq->mbuf[pi].num_bytes = 0;
610 	sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
611 	sq->mbuf[pi].p_refcount = &ptag->refs;
612 	atomic_add_int(&ptag->refs, 1);
613 	sq->pc += sq->mbuf[pi].num_wqebbs;
614 }
615 
616 #define	SBTLS_MBUF_NO_DATA ((struct mbuf *)1)
617 
618 static struct mbuf *
619 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq)
620 {
621 	struct mbuf *mr;
622 	uint32_t offset;
623 	uint32_t delta;
624 
625 	/* check format of incoming mbuf */
626 	if (mb->m_next == NULL ||
627 	    (mb->m_next->m_flags & (M_NOMAP | M_EXT)) != (M_NOMAP | M_EXT) ||
628 	    mb->m_next->m_ext.ext_buf == NULL) {
629 		mr = NULL;
630 		goto done;
631 	}
632 
633 	/* get unmapped data offset */
634 	offset = mtod(mb->m_next, uintptr_t);
635 
636 	/* check if we don't need to re-transmit anything */
637 	if (offset == 0) {
638 		mr = SBTLS_MBUF_NO_DATA;
639 		goto done;
640 	}
641 
642 	/* try to get a new mbufs with packet header */
643 	mr = m_gethdr(wait, MT_DATA);
644 	if (mr == NULL)
645 		goto done;
646 
647 	mb_dupcl(mr, mb->m_next);
648 
649 	/* the beginning of the TLS record */
650 	mr->m_data = NULL;
651 
652 	/* setup packet header length */
653 	mr->m_pkthdr.len = mr->m_len = offset;
654 
655 	/* check for partial re-transmit */
656 	delta = *ptcp_seq - tcp_old;
657 
658 	if (delta < offset) {
659 		m_adj(mr, offset - delta);
660 		offset = delta;
661 	}
662 
663 	/*
664 	 * Rewind the TCP sequence number by the amount of data
665 	 * retransmitted:
666 	 */
667 	*ptcp_seq -= offset;
668 done:
669 	return (mr);
670 }
671 
672 static int
673 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq)
674 {
675 	struct mbuf_ext_pgs *ext_pgs;
676 
677 	for (; mb != NULL; mb = mb->m_next) {
678 		if (!(mb->m_flags & M_NOMAP))
679 			continue;
680 		ext_pgs = (void *)mb->m_ext.ext_buf;
681 		*pseq = ext_pgs->seqno;
682 		return (1);
683 	}
684 	return (0);
685 }
686 
687 int
688 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb)
689 {
690 	struct mlx5e_tls_tag *ptls_tag;
691 	struct mlx5e_snd_tag *ptag;
692 	struct tcphdr *th;
693 	struct mbuf *mb = *ppmb;
694 	u64 rcd_sn;
695 	u32 header_size;
696 	u32 mb_seq;
697 
698 	if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0)
699 		return (MLX5E_TLS_CONTINUE);
700 
701 	ptag = container_of(mb->m_pkthdr.snd_tag,
702 	    struct mlx5e_snd_tag, m_snd_tag);
703 
704 	if (
705 #if defined(RATELIMIT) && defined(IF_SND_TAG_TYPE_TLS_RATE_LIMIT)
706 	    ptag->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT &&
707 #endif
708 	    ptag->type != IF_SND_TAG_TYPE_TLS)
709 		return (MLX5E_TLS_CONTINUE);
710 
711 	ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag);
712 
713 	header_size = mlx5e_get_full_header_size(mb, &th);
714 	if (unlikely(header_size == 0 || th == NULL))
715 		return (MLX5E_TLS_FAILURE);
716 
717 	/*
718 	 * Send non-TLS TCP packets AS-IS:
719 	 */
720 	if (header_size == mb->m_pkthdr.len ||
721 	    mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) {
722 		parg->tisn = 0;
723 		parg->ihs = header_size;
724 		return (MLX5E_TLS_CONTINUE);
725 	}
726 
727 	mb_seq = ntohl(th->th_seq);
728 
729 	MLX5E_TLS_TAG_LOCK(ptls_tag);
730 	switch (ptls_tag->state) {
731 	case MLX5E_TLS_ST_INIT:
732 		queue_work(sq->priv->tls.wq, &ptls_tag->work);
733 		ptls_tag->state = MLX5E_TLS_ST_SETUP;
734 		ptls_tag->expected_seq = ~mb_seq;	/* force setup */
735 		MLX5E_TLS_TAG_UNLOCK(ptls_tag);
736 		return (MLX5E_TLS_FAILURE);
737 
738 	case MLX5E_TLS_ST_SETUP:
739 		MLX5E_TLS_TAG_UNLOCK(ptls_tag);
740 		return (MLX5E_TLS_FAILURE);
741 
742 	default:
743 		MLX5E_TLS_TAG_UNLOCK(ptls_tag);
744 		break;
745 	}
746 
747 	if (unlikely(ptls_tag->expected_seq != mb_seq)) {
748 		struct mbuf *r_mb;
749 		uint32_t tcp_seq = mb_seq;
750 
751 		r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq);
752 		if (r_mb == NULL) {
753 			MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1);
754 			return (MLX5E_TLS_FAILURE);
755 		}
756 
757 		MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1);
758 
759 		/* check if this is the first fragment of a TLS record */
760 		if (r_mb == SBTLS_MBUF_NO_DATA || r_mb->m_data == NULL) {
761 			/* setup TLS static parameters */
762 			MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params,
763 			    param.initial_record_number, rcd_sn);
764 
765 			/* setup TLS progress parameters */
766 			MLX5_SET(sw_tls_cntx, ptls_tag->crypto_params,
767 			    progress.next_record_tcp_sn, tcp_seq);
768 
769 			/*
770 			 * NOTE: The sendqueue should have enough room to
771 			 * carry both the static and the progress parameters
772 			 * when we get here!
773 			 */
774 			mlx5e_tls_send_static_parameters(sq, ptls_tag);
775 			mlx5e_tls_send_progress_parameters(sq, ptls_tag);
776 
777 			if (r_mb == SBTLS_MBUF_NO_DATA) {
778 				mlx5e_tls_send_nop(sq, ptls_tag);
779 				ptls_tag->expected_seq = mb_seq;
780 				return (MLX5E_TLS_LOOP);
781 			}
782 		}
783 
784 		MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len);
785 
786 		/* setup transmit arguments */
787 		parg->tisn = ptls_tag->tisn;
788 		parg->pref = &ptls_tag->refs;
789 
790 		/* try to send DUMP data */
791 		if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) {
792 			m_freem(r_mb);
793 			ptls_tag->expected_seq = tcp_seq;
794 			return (MLX5E_TLS_FAILURE);
795 		} else {
796 			ptls_tag->expected_seq = mb_seq;
797 			return (MLX5E_TLS_LOOP);
798 		}
799 	} else {
800 		MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1);
801 		MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len);
802 	}
803 	ptls_tag->expected_seq += mb->m_pkthdr.len - header_size;
804 
805 	parg->tisn = ptls_tag->tisn;
806 	parg->ihs = header_size;
807 	parg->pref = &ptls_tag->refs;
808 	return (MLX5E_TLS_CONTINUE);
809 }
810 
811 #else
812 
813 int
814 mlx5e_tls_init(struct mlx5e_priv *priv)
815 {
816 
817 	return (0);
818 }
819 
820 void
821 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
822 {
823 	/* NOP */
824 }
825 
826 #endif		/* KERN_TLS */
827