1 /*-
2 * Copyright (c) 2019-2021 Mellanox Technologies. All rights reserved.
3 * Copyright (c) 2022 NVIDIA corporation & affiliates.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #include "opt_kern_tls.h"
28 #include "opt_rss.h"
29 #include "opt_ratelimit.h"
30
31 #include <dev/mlx5/mlx5_en/en.h>
32
33 #include <dev/mlx5/tls.h>
34 #include <dev/mlx5/crypto.h>
35
36 #include <linux/delay.h>
37 #include <sys/ktls.h>
38 #include <opencrypto/cryptodev.h>
39
40 #ifdef KERN_TLS
41
42 #ifdef RATELIMIT
43 static if_snd_tag_modify_t mlx5e_tls_rl_snd_tag_modify;
44 #endif
45 static if_snd_tag_query_t mlx5e_tls_snd_tag_query;
46 static if_snd_tag_free_t mlx5e_tls_snd_tag_free;
47
48 static const struct if_snd_tag_sw mlx5e_tls_snd_tag_sw = {
49 .snd_tag_query = mlx5e_tls_snd_tag_query,
50 .snd_tag_free = mlx5e_tls_snd_tag_free,
51 .type = IF_SND_TAG_TYPE_TLS
52 };
53
54 #ifdef RATELIMIT
55 static const struct if_snd_tag_sw mlx5e_tls_rl_snd_tag_sw = {
56 .snd_tag_modify = mlx5e_tls_rl_snd_tag_modify,
57 .snd_tag_query = mlx5e_tls_snd_tag_query,
58 .snd_tag_free = mlx5e_tls_snd_tag_free,
59 .type = IF_SND_TAG_TYPE_TLS_RATE_LIMIT
60 };
61 #endif
62
63 MALLOC_DEFINE(M_MLX5E_TLS, "MLX5E_TLS", "MLX5 ethernet HW TLS");
64
65 /* software TLS context */
66 struct mlx5_ifc_sw_tls_cntx_bits {
67 struct mlx5_ifc_tls_static_params_bits param;
68 struct mlx5_ifc_tls_progress_params_bits progress;
69 struct {
70 uint8_t key_data[8][0x20];
71 uint8_t key_len[0x20];
72 } key;
73 };
74
75 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_cntx) <= sizeof(((struct mlx5e_tls_tag *)0)->crypto_params));
76 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)0)->mkc));
77
78 static const char *mlx5e_tls_stats_desc[] = {
79 MLX5E_TLS_STATS(MLX5E_STATS_DESC)
80 };
81
82 static void mlx5e_tls_work(struct work_struct *);
83 static void mlx5e_tls_prealloc_work(struct work_struct *);
84
85 /*
86 * Expand the tls tag UMA zone in an async context
87 */
88
89 static void
mlx5e_tls_prealloc_work(struct work_struct * work)90 mlx5e_tls_prealloc_work(struct work_struct *work)
91 {
92 struct mlx5e_priv *priv;
93 struct mlx5e_tls *ptls;
94 struct mlx5e_tls_tag **tags;
95 int i, nitems;
96
97 ptls = container_of(work, struct mlx5e_tls, prealloc_work);
98 priv = container_of(ptls, struct mlx5e_priv, tls);
99 nitems = ptls->zone_max;
100
101 tags = malloc(sizeof(tags[0]) * nitems,
102 M_MLX5E_TLS, M_WAITOK | M_ZERO);
103 for (i = 0; i < nitems; i++) {
104 tags[i] = uma_zalloc(priv->tls.zone, M_NOWAIT);
105 /*
106 * If the allocation fails, its likely we are competing
107 * with real consumers of tags and the zone is full,
108 * so exit the loop, and release the tags like we would
109 * if we allocated all "nitems"
110 */
111 if (tags[i] == NULL)
112 break;
113 }
114 __compiler_membar();
115 for (i = 0; i < nitems && tags[i] != NULL; i++)
116 uma_zfree(priv->tls.zone, tags[i]);
117 free(tags, M_MLX5E_TLS);
118 }
119
120 static int
mlx5e_tls_tag_import(void * arg,void ** store,int cnt,int domain,int flags)121 mlx5e_tls_tag_import(void *arg, void **store, int cnt, int domain, int flags)
122 {
123 struct mlx5e_tls_tag *ptag;
124 struct mlx5e_priv *priv = arg;
125 int err, i;
126
127 /*
128 * mlx5_tls_open_tis() sleeps on a firmware command, so
129 * zone allocations must be done from a sleepable context.
130 * Note that the uma_zalloc() in mlx5e_tls_snd_tag_alloc()
131 * is done with M_NOWAIT so that hitting the zone limit does
132 * not cause the allocation to pause forever.
133 */
134
135 for (i = 0; i != cnt; i++) {
136 ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS,
137 mlx5_dev_domainset(priv->mdev), flags | M_ZERO);
138 if (ptag == NULL)
139 return (i);
140 ptag->tls = &priv->tls;
141 mtx_init(&ptag->mtx, "mlx5-tls-tag-mtx", NULL, MTX_DEF);
142 INIT_WORK(&ptag->work, mlx5e_tls_work);
143 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn,
144 priv->pdn, &ptag->tisn);
145 if (err) {
146 MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
147 free(ptag, M_MLX5E_TLS);
148 return (i);
149 }
150
151 store[i] = ptag;
152 }
153 return (i);
154 }
155
156 static void
mlx5e_tls_tag_release(void * arg,void ** store,int cnt)157 mlx5e_tls_tag_release(void *arg, void **store, int cnt)
158 {
159 struct mlx5e_tls_tag *ptag;
160 struct mlx5e_priv *priv;
161 struct mlx5e_tls *ptls;
162 int i;
163
164 for (i = 0; i != cnt; i++) {
165 ptag = store[i];
166 ptls = ptag->tls;
167 priv = container_of(ptls, struct mlx5e_priv, tls);
168
169 flush_work(&ptag->work);
170
171 if (ptag->tisn != 0) {
172 mlx5_tls_close_tis(priv->mdev, ptag->tisn);
173 }
174
175 mtx_destroy(&ptag->mtx);
176
177 free(ptag, M_MLX5E_TLS);
178 }
179 }
180
181 static void
mlx5e_tls_tag_zfree(struct mlx5e_tls_tag * ptag)182 mlx5e_tls_tag_zfree(struct mlx5e_tls_tag *ptag)
183 {
184 /* make sure any unhandled taskqueue events are ignored */
185 ptag->state = MLX5E_TLS_ST_FREED;
186
187 /* reset some variables */
188 ptag->dek_index = 0;
189 ptag->dek_index_ok = 0;
190
191 /* avoid leaking keys */
192 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
193
194 /* return tag to UMA */
195 uma_zfree(ptag->tls->zone, ptag);
196 }
197
198 static int
mlx5e_max_tag_proc(SYSCTL_HANDLER_ARGS)199 mlx5e_max_tag_proc(SYSCTL_HANDLER_ARGS)
200 {
201 struct mlx5e_priv *priv = (struct mlx5e_priv *)arg1;
202 struct mlx5e_tls *ptls = &priv->tls;
203 int err;
204 unsigned int max_tags;
205
206 max_tags = ptls->zone_max;
207 err = sysctl_handle_int(oidp, &max_tags, arg2, req);
208 if (err != 0 || req->newptr == NULL )
209 return err;
210 if (max_tags == ptls->zone_max)
211 return 0;
212 if (max_tags > priv->tls.max_resources || max_tags == 0)
213 return (EINVAL);
214 ptls->zone_max = max_tags;
215 uma_zone_set_max(ptls->zone, ptls->zone_max);
216 return 0;
217 }
218
219 int
mlx5e_tls_init(struct mlx5e_priv * priv)220 mlx5e_tls_init(struct mlx5e_priv *priv)
221 {
222 struct mlx5e_tls *ptls = &priv->tls;
223 struct sysctl_oid *node;
224 uint32_t max_dek, max_tis, x;
225 int zone_max = 0, prealloc_tags = 0;
226
227 if (MLX5_CAP_GEN(priv->mdev, tls_tx) == 0 ||
228 MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0)
229 return (0);
230
231 ptls->wq = create_singlethread_workqueue("mlx5-tls-wq");
232 if (ptls->wq == NULL)
233 return (ENOMEM);
234
235 sysctl_ctx_init(&ptls->ctx);
236
237 snprintf(ptls->zname, sizeof(ptls->zname),
238 "mlx5_%u_tls", device_get_unit(priv->mdev->pdev->dev.bsddev));
239
240
241 TUNABLE_INT_FETCH("hw.mlx5.tls_max_tags", &zone_max);
242 TUNABLE_INT_FETCH("hw.mlx5.tls_prealloc_tags", &prealloc_tags);
243
244 ptls->zone = uma_zcache_create(ptls->zname,
245 sizeof(struct mlx5e_tls_tag), NULL, NULL, NULL, NULL,
246 mlx5e_tls_tag_import, mlx5e_tls_tag_release, priv,
247 UMA_ZONE_UNMANAGED | (prealloc_tags ? UMA_ZONE_NOFREE : 0));
248
249 /* shared between RX and TX TLS */
250 max_dek = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1);
251 max_tis = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_tis) - 1);
252 ptls->max_resources = MIN(max_dek, max_tis);
253
254 if (zone_max != 0) {
255 ptls->zone_max = zone_max;
256 if (ptls->zone_max > priv->tls.max_resources)
257 ptls->zone_max = priv->tls.max_resources;
258 } else {
259 ptls->zone_max = priv->tls.max_resources;
260 }
261
262 uma_zone_set_max(ptls->zone, ptls->zone_max);
263
264 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
265 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
266
267 ptls->init = 1;
268
269 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
270 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
271 "tls", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS offload");
272 if (node == NULL)
273 return (0);
274
275 SYSCTL_ADD_PROC(&priv->sysctl_ctx, SYSCTL_CHILDREN(node), OID_AUTO, "tls_max_tag",
276 CTLFLAG_RW | CTLTYPE_UINT | CTLFLAG_MPSAFE, priv, 0, mlx5e_max_tag_proc,
277 "IU", "Max number of TLS offload session tags");
278
279 mlx5e_create_counter_stats(&ptls->ctx,
280 SYSCTL_CHILDREN(node), "stats",
281 mlx5e_tls_stats_desc, MLX5E_TLS_STATS_NUM,
282 ptls->stats.arg);
283
284 return (0);
285 }
286
287 void
mlx5e_tls_prealloc_tags(struct mlx5e_priv * priv)288 mlx5e_tls_prealloc_tags(struct mlx5e_priv *priv)
289 {
290 struct mlx5e_tls *ptls = &priv->tls;
291 int prealloc_tags = 0;
292
293 if (ptls->prealloc_wq != NULL)
294 return;
295
296 TUNABLE_INT_FETCH("hw.mlx5.tls_prealloc_tags", &prealloc_tags);
297 if (prealloc_tags == 0)
298 return;
299 ptls->prealloc_wq = create_singlethread_workqueue("mlx5-tls-prealloc_wq");
300 INIT_WORK(&ptls->prealloc_work, mlx5e_tls_prealloc_work);
301 queue_work(ptls->prealloc_wq, &ptls->prealloc_work);
302 }
303
304 void
mlx5e_tls_cleanup(struct mlx5e_priv * priv)305 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
306 {
307 struct mlx5e_tls *ptls = &priv->tls;
308 uint32_t x;
309
310 if (ptls->init == 0)
311 return;
312
313 ptls->init = 0;
314 if (ptls->prealloc_wq != NULL) {
315 flush_workqueue(ptls->prealloc_wq);
316 destroy_workqueue(ptls->prealloc_wq);
317 }
318 flush_workqueue(ptls->wq);
319 sysctl_ctx_free(&ptls->ctx);
320 uma_zdestroy(ptls->zone);
321 destroy_workqueue(ptls->wq);
322
323 for (x = 0; x != MLX5E_TLS_STATS_NUM; x++)
324 counter_u64_free(ptls->stats.arg[x]);
325 }
326
327
328 static int
mlx5e_tls_st_init(struct mlx5e_priv * priv,struct mlx5e_tls_tag * ptag)329 mlx5e_tls_st_init(struct mlx5e_priv *priv, struct mlx5e_tls_tag *ptag)
330 {
331 int err;
332
333 /* try to open TIS, if not present */
334 if (ptag->tisn == 0) {
335 err = mlx5_tls_open_tis(priv->mdev, 0, priv->tdn,
336 priv->pdn, &ptag->tisn);
337 if (err) {
338 MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
339 return (-err);
340 }
341 }
342 MLX5_SET(sw_tls_cntx, ptag->crypto_params, progress.pd, ptag->tisn);
343
344 /* try to allocate a DEK context ID */
345 err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
346 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
347 MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, key.key_data),
348 MLX5_GET(sw_tls_cntx, ptag->crypto_params, key.key_len),
349 &ptag->dek_index);
350 if (err) {
351 MLX5E_TLS_STAT_INC(ptag, tx_error, 1);
352 return (-err);
353 }
354
355 MLX5_SET(sw_tls_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
356
357 ptag->dek_index_ok = 1;
358
359 MLX5E_TLS_TAG_LOCK(ptag);
360 if (ptag->state == MLX5E_TLS_ST_INIT)
361 ptag->state = MLX5E_TLS_ST_SETUP;
362 MLX5E_TLS_TAG_UNLOCK(ptag);
363 return (0);
364 }
365
366 static void
mlx5e_tls_work(struct work_struct * work)367 mlx5e_tls_work(struct work_struct *work)
368 {
369 struct mlx5e_tls_tag *ptag;
370 struct mlx5e_priv *priv;
371
372 ptag = container_of(work, struct mlx5e_tls_tag, work);
373 priv = container_of(ptag->tls, struct mlx5e_priv, tls);
374
375 switch (ptag->state) {
376 case MLX5E_TLS_ST_INIT:
377 (void)mlx5e_tls_st_init(priv, ptag);
378 break;
379
380 case MLX5E_TLS_ST_RELEASE:
381 /* try to destroy DEK context by ID */
382 if (ptag->dek_index_ok)
383 (void)mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
384
385 /* free tag */
386 mlx5e_tls_tag_zfree(ptag);
387 break;
388
389 default:
390 break;
391 }
392 }
393
394 static int
mlx5e_tls_set_params(void * ctx,const struct tls_session_params * en)395 mlx5e_tls_set_params(void *ctx, const struct tls_session_params *en)
396 {
397
398 MLX5_SET(sw_tls_cntx, ctx, param.const_2, 2);
399 if (en->tls_vminor == TLS_MINOR_VER_TWO)
400 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 2); /* v1.2 */
401 else
402 MLX5_SET(sw_tls_cntx, ctx, param.tls_version, 3); /* v1.3 */
403 MLX5_SET(sw_tls_cntx, ctx, param.const_1, 1);
404 MLX5_SET(sw_tls_cntx, ctx, param.encryption_standard, 1); /* TLS */
405
406 /* copy the initial vector in place */
407 switch (en->iv_len) {
408 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv):
409 case MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.gcm_iv) +
410 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param.implicit_iv):
411 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, param.gcm_iv),
412 en->iv, en->iv_len);
413 break;
414 default:
415 return (EINVAL);
416 }
417
418 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_cntx, key.key_data)) {
419 memcpy(MLX5_ADDR_OF(sw_tls_cntx, ctx, key.key_data),
420 en->cipher_key, en->cipher_key_len);
421 MLX5_SET(sw_tls_cntx, ctx, key.key_len, en->cipher_key_len);
422 } else {
423 return (EINVAL);
424 }
425 return (0);
426 }
427
428 /* Verify zero default */
429 CTASSERT(MLX5E_TLS_ST_INIT == 0);
430
431 int
mlx5e_tls_snd_tag_alloc(if_t ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)432 mlx5e_tls_snd_tag_alloc(if_t ifp, union if_snd_tag_alloc_params *params,
433 struct m_snd_tag **ppmt)
434 {
435 union if_snd_tag_alloc_params rl_params;
436 const struct if_snd_tag_sw *snd_tag_sw;
437 struct mlx5e_priv *priv;
438 struct mlx5e_tls_tag *ptag;
439 const struct tls_session_params *en;
440 int error;
441
442 priv = if_getsoftc(ifp);
443
444 if (priv->gone != 0 || priv->tls.init == 0)
445 return (EOPNOTSUPP);
446
447 ptag = uma_zalloc(priv->tls.zone, M_NOWAIT);
448 if (ptag == NULL)
449 return (ENOMEM);
450
451 /* sanity check default values */
452 MPASS(ptag->dek_index == 0);
453 MPASS(ptag->dek_index_ok == 0);
454
455 /* check if there is no TIS context */
456 KASSERT(ptag->tisn != 0, ("ptag %p w/0 tisn", ptag));
457
458 en = ¶ms->tls.tls->params;
459
460 /* only TLS v1.2 and v1.3 is currently supported */
461 if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
462 (en->tls_vminor != TLS_MINOR_VER_TWO
463 #ifdef TLS_MINOR_VER_THREE
464 && en->tls_vminor != TLS_MINOR_VER_THREE
465 #endif
466 )) {
467 error = EPROTONOSUPPORT;
468 goto failure;
469 }
470
471 switch (en->cipher_algorithm) {
472 case CRYPTO_AES_NIST_GCM_16:
473 switch (en->cipher_key_len) {
474 case 128 / 8:
475 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
476 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
477 error = EPROTONOSUPPORT;
478 goto failure;
479 }
480 } else {
481 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
482 error = EPROTONOSUPPORT;
483 goto failure;
484 }
485 }
486 error = mlx5e_tls_set_params(ptag->crypto_params, en);
487 if (error)
488 goto failure;
489 break;
490
491 case 256 / 8:
492 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
493 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
494 error = EPROTONOSUPPORT;
495 goto failure;
496 }
497 } else {
498 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
499 error = EPROTONOSUPPORT;
500 goto failure;
501 }
502 }
503 error = mlx5e_tls_set_params(ptag->crypto_params, en);
504 if (error)
505 goto failure;
506 break;
507
508 default:
509 error = EINVAL;
510 goto failure;
511 }
512 break;
513 default:
514 error = EPROTONOSUPPORT;
515 goto failure;
516 }
517
518 memset(&rl_params, 0, sizeof(rl_params));
519 rl_params.hdr = params->hdr;
520 switch (params->hdr.type) {
521 #ifdef RATELIMIT
522 case IF_SND_TAG_TYPE_TLS_RATE_LIMIT:
523 rl_params.hdr.type = IF_SND_TAG_TYPE_RATE_LIMIT;
524 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
525 snd_tag_sw = &mlx5e_tls_rl_snd_tag_sw;
526 break;
527 #endif
528 case IF_SND_TAG_TYPE_TLS:
529 rl_params.hdr.type = IF_SND_TAG_TYPE_UNLIMITED;
530 snd_tag_sw = &mlx5e_tls_snd_tag_sw;
531 break;
532 default:
533 error = EOPNOTSUPP;
534 goto failure;
535 }
536
537 error = m_snd_tag_alloc(ifp, &rl_params, &ptag->rl_tag);
538 if (error)
539 goto failure;
540
541 /* store pointer to mbuf tag */
542 MPASS(ptag->tag.refcount == 0);
543 m_snd_tag_init(&ptag->tag, ifp, snd_tag_sw);
544 *ppmt = &ptag->tag;
545
546 /* reset state */
547 ptag->state = MLX5E_TLS_ST_INIT;
548
549 error = mlx5e_tls_st_init(priv, ptag);
550 if (error != 0)
551 goto failure;
552
553 return (0);
554
555 failure:
556 mlx5e_tls_tag_zfree(ptag);
557 return (error);
558 }
559
560 #ifdef RATELIMIT
561 static int
mlx5e_tls_rl_snd_tag_modify(struct m_snd_tag * pmt,union if_snd_tag_modify_params * params)562 mlx5e_tls_rl_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
563 {
564 union if_snd_tag_modify_params rl_params;
565 struct mlx5e_tls_tag *ptag =
566 container_of(pmt, struct mlx5e_tls_tag, tag);
567 int error;
568
569 memset(&rl_params, 0, sizeof(rl_params));
570 rl_params.rate_limit.max_rate = params->tls_rate_limit.max_rate;
571 error = ptag->rl_tag->sw->snd_tag_modify(ptag->rl_tag, &rl_params);
572 return (error);
573 }
574 #endif
575
576 static int
mlx5e_tls_snd_tag_query(struct m_snd_tag * pmt,union if_snd_tag_query_params * params)577 mlx5e_tls_snd_tag_query(struct m_snd_tag *pmt, union if_snd_tag_query_params *params)
578 {
579 struct mlx5e_tls_tag *ptag =
580 container_of(pmt, struct mlx5e_tls_tag, tag);
581
582 return (ptag->rl_tag->sw->snd_tag_query(ptag->rl_tag, params));
583 }
584
585 static void
mlx5e_tls_snd_tag_free(struct m_snd_tag * pmt)586 mlx5e_tls_snd_tag_free(struct m_snd_tag *pmt)
587 {
588 struct mlx5e_tls_tag *ptag =
589 container_of(pmt, struct mlx5e_tls_tag, tag);
590 struct mlx5e_priv *priv;
591
592 m_snd_tag_rele(ptag->rl_tag);
593
594 MLX5E_TLS_TAG_LOCK(ptag);
595 ptag->state = MLX5E_TLS_ST_RELEASE;
596 MLX5E_TLS_TAG_UNLOCK(ptag);
597
598 priv = if_getsoftc(ptag->tag.ifp);
599 queue_work(priv->tls.wq, &ptag->work);
600 }
601
602 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) % 16) == 0);
603
604 static void
mlx5e_tls_send_static_parameters(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)605 mlx5e_tls_send_static_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
606 {
607 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
608 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param), MLX5_SEND_WQE_DS);
609 struct mlx5e_tx_umr_wqe *wqe;
610 u16 pi;
611
612 pi = sq->pc & sq->wq.sz_m1;
613 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
614
615 memset(wqe, 0, sizeof(*wqe));
616
617 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
618 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIS_STATIC_PARAMS << 24));
619 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
620 wqe->ctrl.imm = cpu_to_be32(ptag->tisn << 8);
621
622 if (mlx5e_do_send_cqe(sq))
623 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
624 else
625 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
626
627 /* fill out UMR control segment */
628 wqe->umr.flags = 0x80; /* inline data */
629 wqe->umr.bsf_octowords = cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_cntx, param) / 16);
630
631 /* copy in the static crypto parameters */
632 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, param),
633 MLX5_FLD_SZ_BYTES(sw_tls_cntx, param));
634
635 /* copy data for doorbell */
636 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
637
638 sq->mbuf[pi].mbuf = NULL;
639 sq->mbuf[pi].num_bytes = 0;
640 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
641 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
642
643 sq->pc += sq->mbuf[pi].num_wqebbs;
644 }
645
646 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_cntx, progress) ==
647 sizeof(((struct mlx5e_tx_psv_wqe *)0)->psv));
648
649 static void
mlx5e_tls_send_progress_parameters(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)650 mlx5e_tls_send_progress_parameters(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
651 {
652 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
653 MLX5_SEND_WQE_DS);
654 struct mlx5e_tx_psv_wqe *wqe;
655 u16 pi;
656
657 pi = sq->pc & sq->wq.sz_m1;
658 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
659
660 memset(wqe, 0, sizeof(*wqe));
661
662 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) |
663 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIS_PROGRESS_PARAMS << 24));
664 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
665
666 if (mlx5e_do_send_cqe(sq))
667 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
668
669 /* copy in the PSV control segment */
670 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_cntx, ptag->crypto_params, progress),
671 sizeof(wqe->psv));
672
673 /* copy data for doorbell */
674 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
675
676 sq->mbuf[pi].mbuf = NULL;
677 sq->mbuf[pi].num_bytes = 0;
678 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
679 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
680
681 sq->pc += sq->mbuf[pi].num_wqebbs;
682 }
683
684 static void
mlx5e_tls_send_nop(struct mlx5e_sq * sq,struct mlx5e_tls_tag * ptag)685 mlx5e_tls_send_nop(struct mlx5e_sq *sq, struct mlx5e_tls_tag *ptag)
686 {
687 const u32 ds_cnt = MLX5_SEND_WQEBB_NUM_DS;
688 struct mlx5e_tx_wqe *wqe;
689 u16 pi;
690
691 pi = sq->pc & sq->wq.sz_m1;
692 wqe = mlx5_wq_cyc_get_wqe(&sq->wq, pi);
693
694 memset(&wqe->ctrl, 0, sizeof(wqe->ctrl));
695
696 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_NOP);
697 wqe->ctrl.qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
698 if (mlx5e_do_send_cqe(sq))
699 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
700 else
701 wqe->ctrl.fm_ce_se = MLX5_FENCE_MODE_INITIATOR_SMALL;
702
703 /* Copy data for doorbell */
704 memcpy(sq->doorbell.d32, &wqe->ctrl, sizeof(sq->doorbell.d32));
705
706 sq->mbuf[pi].mbuf = NULL;
707 sq->mbuf[pi].num_bytes = 0;
708 sq->mbuf[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
709 sq->mbuf[pi].mst = m_snd_tag_ref(&ptag->tag);
710
711 sq->pc += sq->mbuf[pi].num_wqebbs;
712 }
713
714 #define SBTLS_MBUF_NO_DATA ((struct mbuf *)1)
715
716 static struct mbuf *
sbtls_recover_record(struct mbuf * mb,int wait,uint32_t tcp_old,uint32_t * ptcp_seq,bool * pis_start)717 sbtls_recover_record(struct mbuf *mb, int wait, uint32_t tcp_old, uint32_t *ptcp_seq, bool *pis_start)
718 {
719 struct mbuf *mr, *top;
720 uint32_t offset;
721 uint32_t delta;
722
723 /* check format of incoming mbuf */
724 if (mb->m_next == NULL ||
725 (mb->m_next->m_flags & (M_EXTPG | M_EXT)) != (M_EXTPG | M_EXT)) {
726 top = NULL;
727 goto done;
728 }
729
730 /* get unmapped data offset */
731 offset = mtod(mb->m_next, uintptr_t);
732
733 /* check if we don't need to re-transmit anything */
734 if (offset == 0) {
735 top = SBTLS_MBUF_NO_DATA;
736 *pis_start = true;
737 goto done;
738 }
739
740 /* try to get a new packet header */
741 top = m_gethdr(wait, MT_DATA);
742 if (top == NULL)
743 goto done;
744
745 mr = m_get(wait, MT_DATA);
746 if (mr == NULL) {
747 m_free(top);
748 top = NULL;
749 goto done;
750 }
751
752 top->m_next = mr;
753
754 mb_dupcl(mr, mb->m_next);
755
756 /* the beginning of the TLS record */
757 mr->m_data = NULL;
758
759 /* setup packet header length */
760 top->m_pkthdr.len = mr->m_len = offset;
761 top->m_len = 0;
762
763 /* check for partial re-transmit */
764 delta = *ptcp_seq - tcp_old;
765
766 if (delta < offset) {
767 m_adj(top, offset - delta);
768 offset = delta;
769
770 /* continue where we left off */
771 *pis_start = false;
772 } else {
773 *pis_start = true;
774 }
775
776 /*
777 * Rewind the TCP sequence number by the amount of data
778 * retransmitted:
779 */
780 *ptcp_seq -= offset;
781 done:
782 return (top);
783 }
784
785 static int
mlx5e_sq_tls_populate(struct mbuf * mb,uint64_t * pseq)786 mlx5e_sq_tls_populate(struct mbuf *mb, uint64_t *pseq)
787 {
788
789 for (; mb != NULL; mb = mb->m_next) {
790 if (!(mb->m_flags & M_EXTPG))
791 continue;
792 *pseq = mb->m_epg_seqno;
793 return (1);
794 }
795 return (0);
796 }
797
798 int
mlx5e_sq_tls_xmit(struct mlx5e_sq * sq,struct mlx5e_xmit_args * parg,struct mbuf ** ppmb)799 mlx5e_sq_tls_xmit(struct mlx5e_sq *sq, struct mlx5e_xmit_args *parg, struct mbuf **ppmb)
800 {
801 struct mlx5e_tls_tag *ptls_tag;
802 struct m_snd_tag *ptag;
803 const struct tcphdr *th;
804 struct mbuf *mb = *ppmb;
805 u64 rcd_sn;
806 u32 header_size;
807 u32 mb_seq;
808
809 if ((mb->m_pkthdr.csum_flags & CSUM_SND_TAG) == 0)
810 return (MLX5E_TLS_CONTINUE);
811
812 ptag = mb->m_pkthdr.snd_tag;
813
814 if (
815 #ifdef RATELIMIT
816 ptag->sw->type != IF_SND_TAG_TYPE_TLS_RATE_LIMIT &&
817 #endif
818 ptag->sw->type != IF_SND_TAG_TYPE_TLS)
819 return (MLX5E_TLS_CONTINUE);
820
821 ptls_tag = container_of(ptag, struct mlx5e_tls_tag, tag);
822
823 header_size = mlx5e_get_full_header_size(mb, &th);
824 if (unlikely(header_size == 0 || th == NULL))
825 return (MLX5E_TLS_FAILURE);
826
827 /*
828 * Send non-TLS TCP packets AS-IS:
829 */
830 if (header_size == mb->m_pkthdr.len ||
831 mlx5e_sq_tls_populate(mb, &rcd_sn) == 0) {
832 parg->tisn = 0;
833 parg->ihs = header_size;
834 return (MLX5E_TLS_CONTINUE);
835 }
836
837 mb_seq = ntohl(th->th_seq);
838
839 MLX5E_TLS_TAG_LOCK(ptls_tag);
840 switch (ptls_tag->state) {
841 case MLX5E_TLS_ST_INIT:
842 MLX5E_TLS_TAG_UNLOCK(ptls_tag);
843 return (MLX5E_TLS_FAILURE);
844 case MLX5E_TLS_ST_SETUP:
845 ptls_tag->state = MLX5E_TLS_ST_TXRDY;
846 ptls_tag->expected_seq = ~mb_seq; /* force setup */
847 default:
848 MLX5E_TLS_TAG_UNLOCK(ptls_tag);
849 break;
850 }
851
852 if (unlikely(ptls_tag->expected_seq != mb_seq)) {
853 bool is_start;
854 struct mbuf *r_mb;
855 uint32_t tcp_seq = mb_seq;
856
857 r_mb = sbtls_recover_record(mb, M_NOWAIT, ptls_tag->expected_seq, &tcp_seq, &is_start);
858 if (r_mb == NULL) {
859 MLX5E_TLS_STAT_INC(ptls_tag, tx_error, 1);
860 return (MLX5E_TLS_FAILURE);
861 }
862
863 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets_ooo, 1);
864
865 /* check if this is the first fragment of a TLS record */
866 if (is_start) {
867 /* setup TLS static parameters */
868 MLX5_SET64(sw_tls_cntx, ptls_tag->crypto_params,
869 param.initial_record_number, rcd_sn);
870
871 /*
872 * NOTE: The sendqueue should have enough room to
873 * carry both the static and the progress parameters
874 * when we get here!
875 */
876 mlx5e_tls_send_static_parameters(sq, ptls_tag);
877 mlx5e_tls_send_progress_parameters(sq, ptls_tag);
878
879 if (r_mb == SBTLS_MBUF_NO_DATA) {
880 mlx5e_tls_send_nop(sq, ptls_tag);
881 ptls_tag->expected_seq = mb_seq;
882 return (MLX5E_TLS_LOOP);
883 }
884 }
885
886 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes_ooo, r_mb->m_pkthdr.len);
887
888 /* setup transmit arguments */
889 parg->tisn = ptls_tag->tisn;
890 parg->mst = &ptls_tag->tag;
891
892 /* try to send DUMP data */
893 if (mlx5e_sq_dump_xmit(sq, parg, &r_mb) != 0) {
894 m_freem(r_mb);
895 ptls_tag->expected_seq = tcp_seq;
896 return (MLX5E_TLS_FAILURE);
897 } else {
898 ptls_tag->expected_seq = mb_seq;
899 return (MLX5E_TLS_LOOP);
900 }
901 } else {
902 MLX5E_TLS_STAT_INC(ptls_tag, tx_packets, 1);
903 MLX5E_TLS_STAT_INC(ptls_tag, tx_bytes, mb->m_pkthdr.len);
904 }
905 ptls_tag->expected_seq += mb->m_pkthdr.len - header_size;
906
907 parg->tisn = ptls_tag->tisn;
908 parg->ihs = header_size;
909 parg->mst = &ptls_tag->tag;
910 return (MLX5E_TLS_CONTINUE);
911 }
912
913 #else
914
915 int
mlx5e_tls_init(struct mlx5e_priv * priv)916 mlx5e_tls_init(struct mlx5e_priv *priv)
917 {
918
919 return (0);
920 }
921
922 void
mlx5e_tls_cleanup(struct mlx5e_priv * priv)923 mlx5e_tls_cleanup(struct mlx5e_priv *priv)
924 {
925 /* NOP */
926 }
927
928 #endif /* KERN_TLS */
929