1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 // Copyright (c) 2019 Mellanox Technologies.
3
4 #include <net/inet6_hashtables.h>
5 #include "en_accel/en_accel.h"
6 #include "en_accel/ktls.h"
7 #include "en_accel/ktls_txrx.h"
8 #include "en_accel/ktls_utils.h"
9 #include "en_accel/fs_tcp.h"
10
11 struct accel_rule {
12 struct work_struct work;
13 struct mlx5e_priv *priv;
14 struct mlx5_flow_handle *rule;
15 };
16
17 #define PROGRESS_PARAMS_WRITE_UNIT 64
18 #define PROGRESS_PARAMS_PADDED_SIZE \
19 (ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \
20 PROGRESS_PARAMS_WRITE_UNIT))
21
22 struct mlx5e_ktls_rx_resync_buf {
23 union {
24 struct mlx5_wqe_tls_progress_params_seg progress;
25 u8 pad[PROGRESS_PARAMS_PADDED_SIZE];
26 } ____cacheline_aligned_in_smp;
27 dma_addr_t dma_addr;
28 struct mlx5e_ktls_offload_context_rx *priv_rx;
29 };
30
31 enum {
32 MLX5E_PRIV_RX_FLAG_DELETING,
33 MLX5E_NUM_PRIV_RX_FLAGS,
34 };
35
36 struct mlx5e_ktls_rx_resync_ctx {
37 struct tls_offload_resync_async core;
38 struct work_struct work;
39 struct mlx5e_priv *priv;
40 refcount_t refcnt;
41 __be64 sw_rcd_sn_be;
42 u32 seq;
43 };
44
45 struct mlx5e_ktls_offload_context_rx {
46 union mlx5e_crypto_info crypto_info;
47 struct accel_rule rule;
48 struct sock *sk;
49 struct mlx5e_rq_stats *rq_stats;
50 struct mlx5e_tls_sw_stats *sw_stats;
51 struct completion add_ctx;
52 struct mlx5e_tir tir;
53 struct mlx5_crypto_dek *dek;
54 u32 rxq;
55 DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS);
56
57 /* resync */
58 spinlock_t lock; /* protects resync fields */
59 struct mlx5e_ktls_rx_resync_ctx resync;
60 struct list_head list;
61 };
62
mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx * priv_rx)63 static bool mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx *priv_rx)
64 {
65 if (!refcount_dec_and_test(&priv_rx->resync.refcnt))
66 return false;
67
68 kfree(priv_rx);
69 return true;
70 }
71
mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx * priv_rx)72 static void mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx *priv_rx)
73 {
74 refcount_inc(&priv_rx->resync.refcnt);
75 }
76
77 struct mlx5e_ktls_resync_resp {
78 /* protects list changes */
79 spinlock_t lock;
80 struct list_head list;
81 };
82
mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp * resp_list)83 void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_list)
84 {
85 kvfree(resp_list);
86 }
87
88 struct mlx5e_ktls_resync_resp *
mlx5e_ktls_rx_resync_create_resp_list(void)89 mlx5e_ktls_rx_resync_create_resp_list(void)
90 {
91 struct mlx5e_ktls_resync_resp *resp_list;
92
93 resp_list = kvzalloc(sizeof(*resp_list), GFP_KERNEL);
94 if (!resp_list)
95 return ERR_PTR(-ENOMEM);
96
97 INIT_LIST_HEAD(&resp_list->list);
98 spin_lock_init(&resp_list->lock);
99
100 return resp_list;
101 }
102
accel_rule_handle_work(struct work_struct * work)103 static void accel_rule_handle_work(struct work_struct *work)
104 {
105 struct mlx5e_ktls_offload_context_rx *priv_rx;
106 struct accel_rule *accel_rule;
107 struct mlx5_flow_handle *rule;
108
109 accel_rule = container_of(work, struct accel_rule, work);
110 priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule);
111 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
112 goto out;
113
114 rule = mlx5e_accel_fs_add_sk(accel_rule->priv->fs, priv_rx->sk,
115 mlx5e_tir_get_tirn(&priv_rx->tir),
116 MLX5_FS_DEFAULT_FLOW_TAG);
117 if (!IS_ERR_OR_NULL(rule))
118 accel_rule->rule = rule;
119 out:
120 complete(&priv_rx->add_ctx);
121 }
122
accel_rule_init(struct accel_rule * rule,struct mlx5e_priv * priv)123 static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv)
124 {
125 INIT_WORK(&rule->work, accel_rule_handle_work);
126 rule->priv = priv;
127 }
128
icosq_fill_wi(struct mlx5e_icosq * sq,u16 pi,struct mlx5e_icosq_wqe_info * wi)129 static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi,
130 struct mlx5e_icosq_wqe_info *wi)
131 {
132 sq->db.wqe_info[pi] = *wi;
133 }
134
135 static struct mlx5_wqe_ctrl_seg *
post_static_params(struct mlx5e_icosq * sq,struct mlx5e_ktls_offload_context_rx * priv_rx)136 post_static_params(struct mlx5e_icosq *sq,
137 struct mlx5e_ktls_offload_context_rx *priv_rx)
138 {
139 struct mlx5e_set_tls_static_params_wqe *wqe;
140 struct mlx5e_icosq_wqe_info wi;
141 u16 pi, num_wqebbs;
142
143 num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS;
144 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs)))
145 return ERR_PTR(-ENOSPC);
146
147 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
148 wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi);
149 mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info,
150 mlx5e_tir_get_tirn(&priv_rx->tir),
151 mlx5_crypto_dek_get_id(priv_rx->dek),
152 priv_rx->resync.seq, false,
153 TLS_OFFLOAD_CTX_DIR_RX);
154 wi = (struct mlx5e_icosq_wqe_info) {
155 .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS,
156 .num_wqebbs = num_wqebbs,
157 .tls_set_params.priv_rx = priv_rx,
158 };
159 icosq_fill_wi(sq, pi, &wi);
160 sq->pc += num_wqebbs;
161
162 return &wqe->ctrl;
163 }
164
165 static struct mlx5_wqe_ctrl_seg *
post_progress_params(struct mlx5e_icosq * sq,struct mlx5e_ktls_offload_context_rx * priv_rx,u32 next_record_tcp_sn)166 post_progress_params(struct mlx5e_icosq *sq,
167 struct mlx5e_ktls_offload_context_rx *priv_rx,
168 u32 next_record_tcp_sn)
169 {
170 struct mlx5e_set_tls_progress_params_wqe *wqe;
171 struct mlx5e_icosq_wqe_info wi;
172 u16 pi, num_wqebbs;
173
174 num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS;
175 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs)))
176 return ERR_PTR(-ENOSPC);
177
178 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs);
179 wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi);
180 mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn,
181 mlx5e_tir_get_tirn(&priv_rx->tir),
182 false, next_record_tcp_sn,
183 TLS_OFFLOAD_CTX_DIR_RX);
184 wi = (struct mlx5e_icosq_wqe_info) {
185 .wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS,
186 .num_wqebbs = num_wqebbs,
187 .tls_set_params.priv_rx = priv_rx,
188 };
189
190 icosq_fill_wi(sq, pi, &wi);
191 sq->pc += num_wqebbs;
192
193 return &wqe->ctrl;
194 }
195
post_rx_param_wqes(struct mlx5e_channel * c,struct mlx5e_ktls_offload_context_rx * priv_rx,u32 next_record_tcp_sn)196 static int post_rx_param_wqes(struct mlx5e_channel *c,
197 struct mlx5e_ktls_offload_context_rx *priv_rx,
198 u32 next_record_tcp_sn)
199 {
200 struct mlx5_wqe_ctrl_seg *cseg;
201 struct mlx5e_icosq *sq;
202 int err;
203
204 err = 0;
205 sq = &c->async_icosq;
206 spin_lock_bh(&c->async_icosq_lock);
207
208 cseg = post_static_params(sq, priv_rx);
209 if (IS_ERR(cseg))
210 goto err_out;
211 cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn);
212 if (IS_ERR(cseg))
213 goto err_out;
214
215 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
216 unlock:
217 spin_unlock_bh(&c->async_icosq_lock);
218
219 return err;
220
221 err_out:
222 priv_rx->rq_stats->tls_resync_req_skip++;
223 err = PTR_ERR(cseg);
224 complete(&priv_rx->add_ctx);
225 goto unlock;
226 }
227
228 static void
mlx5e_set_ktls_rx_priv_ctx(struct tls_context * tls_ctx,struct mlx5e_ktls_offload_context_rx * priv_rx)229 mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx,
230 struct mlx5e_ktls_offload_context_rx *priv_rx)
231 {
232 struct mlx5e_ktls_offload_context_rx **ctx =
233 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
234
235 BUILD_BUG_ON(sizeof(priv_rx) > TLS_DRIVER_STATE_SIZE_RX);
236
237 *ctx = priv_rx;
238 }
239
240 static struct mlx5e_ktls_offload_context_rx *
mlx5e_get_ktls_rx_priv_ctx(struct tls_context * tls_ctx)241 mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx)
242 {
243 struct mlx5e_ktls_offload_context_rx **ctx =
244 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX);
245
246 return *ctx;
247 }
248
249 /* Re-sync */
250 /* Runs in work context */
251 static int
resync_post_get_progress_params(struct mlx5e_icosq * sq,struct mlx5e_ktls_offload_context_rx * priv_rx)252 resync_post_get_progress_params(struct mlx5e_icosq *sq,
253 struct mlx5e_ktls_offload_context_rx *priv_rx)
254 {
255 struct mlx5e_get_tls_progress_params_wqe *wqe;
256 struct mlx5e_ktls_rx_resync_buf *buf;
257 struct mlx5e_icosq_wqe_info wi;
258 struct mlx5_wqe_ctrl_seg *cseg;
259 struct mlx5_seg_get_psv *psv;
260 struct device *pdev;
261 int err;
262 u16 pi;
263
264 buf = kzalloc(sizeof(*buf), GFP_KERNEL);
265 if (unlikely(!buf)) {
266 err = -ENOMEM;
267 goto err_out;
268 }
269
270 pdev = mlx5_core_dma_dev(sq->channel->mdev);
271 buf->dma_addr = dma_map_single(pdev, &buf->progress,
272 PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
273 if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) {
274 err = -ENOMEM;
275 goto err_free;
276 }
277
278 buf->priv_rx = priv_rx;
279
280 spin_lock_bh(&sq->channel->async_icosq_lock);
281
282 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS))) {
283 spin_unlock_bh(&sq->channel->async_icosq_lock);
284 err = -ENOSPC;
285 goto err_dma_unmap;
286 }
287
288 pi = mlx5e_icosq_get_next_pi(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS);
289 wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi);
290
291 #define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS))
292
293 cseg = &wqe->ctrl;
294 cseg->opmod_idx_opcode =
295 cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV |
296 (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24));
297 cseg->qpn_ds =
298 cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT);
299
300 psv = &wqe->psv;
301 psv->num_psv = 1 << 4;
302 psv->l_key = sq->channel->mkey_be;
303 psv->psv_index[0] = cpu_to_be32(mlx5e_tir_get_tirn(&priv_rx->tir));
304 psv->va = cpu_to_be64(buf->dma_addr);
305
306 wi = (struct mlx5e_icosq_wqe_info) {
307 .wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS,
308 .num_wqebbs = MLX5E_KTLS_GET_PROGRESS_WQEBBS,
309 .tls_get_params.buf = buf,
310 };
311 icosq_fill_wi(sq, pi, &wi);
312 sq->pc++;
313 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg);
314 spin_unlock_bh(&sq->channel->async_icosq_lock);
315
316 return 0;
317
318 err_dma_unmap:
319 dma_unmap_single(pdev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
320 err_free:
321 kfree(buf);
322 err_out:
323 return err;
324 }
325
326 /* Function is called with elevated refcount.
327 * It decreases it only if no WQE is posted.
328 */
resync_handle_work(struct work_struct * work)329 static void resync_handle_work(struct work_struct *work)
330 {
331 struct mlx5e_ktls_offload_context_rx *priv_rx;
332 struct mlx5e_ktls_rx_resync_ctx *resync;
333 struct mlx5e_channel *c;
334 struct mlx5e_icosq *sq;
335
336 resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work);
337 priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync);
338
339 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
340 mlx5e_ktls_priv_rx_put(priv_rx);
341 priv_rx->rq_stats->tls_resync_req_skip++;
342 tls_offload_rx_resync_async_request_cancel(&resync->core);
343 return;
344 }
345
346 c = resync->priv->channels.c[priv_rx->rxq];
347 sq = &c->async_icosq;
348
349 if (resync_post_get_progress_params(sq, priv_rx)) {
350 priv_rx->rq_stats->tls_resync_req_skip++;
351 tls_offload_rx_resync_async_request_cancel(&resync->core);
352 mlx5e_ktls_priv_rx_put(priv_rx);
353 }
354 }
355
resync_init(struct mlx5e_ktls_rx_resync_ctx * resync,struct mlx5e_priv * priv)356 static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync,
357 struct mlx5e_priv *priv)
358 {
359 INIT_WORK(&resync->work, resync_handle_work);
360 resync->priv = priv;
361 refcount_set(&resync->refcnt, 1);
362 }
363
364 /* Function can be called with the refcount being either elevated or not.
365 * It does not affect the refcount.
366 */
resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx * priv_rx,struct mlx5e_channel * c)367 static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx,
368 struct mlx5e_channel *c)
369 {
370 struct mlx5e_ktls_resync_resp *ktls_resync;
371 struct mlx5e_icosq *sq;
372 bool trigger_poll;
373
374 sq = &c->async_icosq;
375 ktls_resync = sq->ktls_resync;
376 trigger_poll = false;
377
378 spin_lock_bh(&ktls_resync->lock);
379 spin_lock_bh(&priv_rx->lock);
380 switch (priv_rx->crypto_info.crypto_info.cipher_type) {
381 case TLS_CIPHER_AES_GCM_128: {
382 struct tls12_crypto_info_aes_gcm_128 *info =
383 &priv_rx->crypto_info.crypto_info_128;
384
385 memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be,
386 sizeof(info->rec_seq));
387 break;
388 }
389 case TLS_CIPHER_AES_GCM_256: {
390 struct tls12_crypto_info_aes_gcm_256 *info =
391 &priv_rx->crypto_info.crypto_info_256;
392
393 memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be,
394 sizeof(info->rec_seq));
395 break;
396 }
397 default:
398 WARN_ONCE(1, "Unsupported cipher type %u\n",
399 priv_rx->crypto_info.crypto_info.cipher_type);
400 spin_unlock_bh(&priv_rx->lock);
401 spin_unlock_bh(&ktls_resync->lock);
402 return;
403 }
404
405 if (list_empty(&priv_rx->list)) {
406 list_add_tail(&priv_rx->list, &ktls_resync->list);
407 trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
408 }
409 spin_unlock_bh(&priv_rx->lock);
410 spin_unlock_bh(&ktls_resync->lock);
411
412 if (!trigger_poll)
413 return;
414
415 if (!napi_if_scheduled_mark_missed(&c->napi)) {
416 spin_lock_bh(&c->async_icosq_lock);
417 mlx5e_trigger_irq(sq);
418 spin_unlock_bh(&c->async_icosq_lock);
419 }
420 }
421
422 /* Function can be called with the refcount being either elevated or not.
423 * It decreases the refcount and may free the kTLS priv context.
424 * Refcount is not elevated only if tls_dev_del has been called, but GET_PSV was
425 * already in flight.
426 */
mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info * wi,struct mlx5e_icosq * sq)427 void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi,
428 struct mlx5e_icosq *sq)
429 {
430 struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf;
431 struct mlx5e_ktls_offload_context_rx *priv_rx;
432 struct tls_offload_resync_async *async_resync;
433 struct tls_offload_context_rx *rx_ctx;
434 u8 tracker_state, auth_state, *ctx;
435 struct device *dev;
436 u32 hw_seq;
437
438 priv_rx = buf->priv_rx;
439 dev = mlx5_core_dma_dev(sq->channel->mdev);
440 rx_ctx = tls_offload_ctx_rx(tls_get_ctx(priv_rx->sk));
441 async_resync = rx_ctx->resync_async;
442 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
443 priv_rx->rq_stats->tls_resync_req_skip++;
444 tls_offload_rx_resync_async_request_cancel(async_resync);
445 goto out;
446 }
447
448 dma_sync_single_for_cpu(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE,
449 DMA_FROM_DEVICE);
450
451 ctx = buf->progress.ctx;
452 tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state);
453 auth_state = MLX5_GET(tls_progress_params, ctx, auth_state);
454 if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING ||
455 auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) {
456 priv_rx->rq_stats->tls_resync_req_skip++;
457 tls_offload_rx_resync_async_request_cancel(async_resync);
458 goto out;
459 }
460
461 hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn);
462 tls_offload_rx_resync_async_request_end(async_resync,
463 cpu_to_be32(hw_seq));
464 priv_rx->rq_stats->tls_resync_req_end++;
465 out:
466 mlx5e_ktls_priv_rx_put(priv_rx);
467 dma_unmap_single(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE);
468 kfree(buf);
469 }
470
471 /* Runs in NAPI.
472 * Function elevates the refcount, unless no work is queued.
473 */
resync_queue_get_psv(struct sock * sk)474 static bool resync_queue_get_psv(struct sock *sk)
475 {
476 struct mlx5e_ktls_offload_context_rx *priv_rx;
477 struct mlx5e_ktls_rx_resync_ctx *resync;
478
479 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
480 if (unlikely(!priv_rx))
481 return false;
482
483 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags)))
484 return false;
485
486 resync = &priv_rx->resync;
487 mlx5e_ktls_priv_rx_get(priv_rx);
488 if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work))) {
489 mlx5e_ktls_priv_rx_put(priv_rx);
490 return false;
491 }
492
493 return true;
494 }
495
496 /* Runs in NAPI */
resync_update_sn(struct mlx5e_rq * rq,struct sk_buff * skb)497 static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb)
498 {
499 struct ethhdr *eth = (struct ethhdr *)(skb->data);
500 struct tls_offload_resync_async *resync_async;
501 struct net_device *netdev = rq->netdev;
502 struct net *net = dev_net(netdev);
503 struct sock *sk = NULL;
504 unsigned int datalen;
505 struct iphdr *iph;
506 struct tcphdr *th;
507 __be32 seq;
508 int depth = 0;
509
510 __vlan_get_protocol(skb, eth->h_proto, &depth);
511 iph = (struct iphdr *)(skb->data + depth);
512
513 if (iph->version == 4) {
514 depth += sizeof(struct iphdr);
515 th = (void *)iph + sizeof(struct iphdr);
516
517 sk = inet_lookup_established(net, iph->saddr, th->source,
518 iph->daddr, th->dest,
519 netdev->ifindex);
520 #if IS_ENABLED(CONFIG_IPV6)
521 } else {
522 struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph;
523
524 depth += sizeof(struct ipv6hdr);
525 th = (void *)ipv6h + sizeof(struct ipv6hdr);
526
527 sk = __inet6_lookup_established(net, &ipv6h->saddr, th->source,
528 &ipv6h->daddr, ntohs(th->dest),
529 netdev->ifindex, 0);
530 #endif
531 }
532
533 depth += sizeof(struct tcphdr);
534
535 if (unlikely(!sk))
536 return;
537
538 if (unlikely(sk->sk_state == TCP_TIME_WAIT))
539 goto unref;
540
541 if (unlikely(!resync_queue_get_psv(sk)))
542 goto unref;
543
544 seq = th->seq;
545 datalen = skb->len - depth;
546 resync_async = tls_offload_ctx_rx(tls_get_ctx(sk))->resync_async;
547 tls_offload_rx_resync_async_request_start(resync_async, seq, datalen);
548 rq->stats->tls_resync_req_start++;
549
550 unref:
551 sock_gen_put(sk);
552 }
553
mlx5e_ktls_rx_resync(struct net_device * netdev,struct sock * sk,u32 seq,u8 * rcd_sn)554 void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk,
555 u32 seq, u8 *rcd_sn)
556 {
557 struct mlx5e_ktls_offload_context_rx *priv_rx;
558 struct mlx5e_ktls_rx_resync_ctx *resync;
559 struct mlx5e_priv *priv;
560 struct mlx5e_channel *c;
561
562 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk));
563 if (unlikely(!priv_rx))
564 return;
565
566 resync = &priv_rx->resync;
567 resync->sw_rcd_sn_be = *(__be64 *)rcd_sn;
568 resync->seq = seq;
569
570 priv = netdev_priv(netdev);
571 c = priv->channels.c[priv_rx->rxq];
572
573 resync_handle_seq_match(priv_rx, c);
574 }
575
576 void
mlx5e_ktls_rx_resync_async_request_cancel(struct mlx5e_icosq_wqe_info * wi)577 mlx5e_ktls_rx_resync_async_request_cancel(struct mlx5e_icosq_wqe_info *wi)
578 {
579 struct mlx5e_ktls_offload_context_rx *priv_rx;
580 struct mlx5e_ktls_rx_resync_buf *buf;
581
582 buf = wi->tls_get_params.buf;
583 priv_rx = buf->priv_rx;
584 priv_rx->rq_stats->tls_resync_req_skip++;
585 tls_offload_rx_resync_async_request_cancel(&priv_rx->resync.core);
586 }
587
588 /* End of resync section */
589
mlx5e_ktls_handle_rx_skb(struct mlx5e_rq * rq,struct sk_buff * skb,struct mlx5_cqe64 * cqe,u32 * cqe_bcnt)590 void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb,
591 struct mlx5_cqe64 *cqe, u32 *cqe_bcnt)
592 {
593 struct mlx5e_rq_stats *stats = rq->stats;
594
595 switch (get_cqe_tls_offload(cqe)) {
596 case CQE_TLS_OFFLOAD_DECRYPTED:
597 skb->decrypted = 1;
598 stats->tls_decrypted_packets++;
599 stats->tls_decrypted_bytes += *cqe_bcnt;
600 break;
601 case CQE_TLS_OFFLOAD_RESYNC:
602 stats->tls_resync_req_pkt++;
603 resync_update_sn(rq, skb);
604 break;
605 default: /* CQE_TLS_OFFLOAD_ERROR: */
606 stats->tls_err++;
607 break;
608 }
609 }
610
mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info * wi)611 void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi)
612 {
613 struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx;
614 struct accel_rule *rule = &priv_rx->rule;
615
616 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) {
617 complete(&priv_rx->add_ctx);
618 return;
619 }
620 queue_work(rule->priv->tls->rx_wq, &rule->work);
621 }
622
mlx5e_ktls_sk_get_rxq(struct sock * sk)623 static int mlx5e_ktls_sk_get_rxq(struct sock *sk)
624 {
625 int rxq = sk_rx_queue_get(sk);
626
627 if (unlikely(rxq == -1))
628 rxq = 0;
629
630 return rxq;
631 }
632
mlx5e_ktls_add_rx(struct net_device * netdev,struct sock * sk,struct tls_crypto_info * crypto_info,u32 start_offload_tcp_sn)633 int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk,
634 struct tls_crypto_info *crypto_info,
635 u32 start_offload_tcp_sn)
636 {
637 struct mlx5e_ktls_offload_context_rx *priv_rx;
638 struct mlx5e_ktls_rx_resync_ctx *resync;
639 struct tls_context *tls_ctx;
640 struct mlx5_crypto_dek *dek;
641 struct mlx5e_priv *priv;
642 int rxq, err;
643
644 tls_ctx = tls_get_ctx(sk);
645 priv = netdev_priv(netdev);
646 priv_rx = kzalloc(sizeof(*priv_rx), GFP_KERNEL);
647 if (unlikely(!priv_rx))
648 return -ENOMEM;
649
650 switch (crypto_info->cipher_type) {
651 case TLS_CIPHER_AES_GCM_128:
652 priv_rx->crypto_info.crypto_info_128 =
653 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
654 break;
655 case TLS_CIPHER_AES_GCM_256:
656 priv_rx->crypto_info.crypto_info_256 =
657 *(struct tls12_crypto_info_aes_gcm_256 *)crypto_info;
658 break;
659 default:
660 WARN_ONCE(1, "Unsupported cipher type %u\n",
661 crypto_info->cipher_type);
662 err = -EOPNOTSUPP;
663 goto err_cipher_type;
664 }
665
666 dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info);
667 if (IS_ERR(dek)) {
668 err = PTR_ERR(dek);
669 goto err_cipher_type;
670 }
671 priv_rx->dek = dek;
672
673 INIT_LIST_HEAD(&priv_rx->list);
674 spin_lock_init(&priv_rx->lock);
675
676 rxq = mlx5e_ktls_sk_get_rxq(sk);
677 priv_rx->rxq = rxq;
678 priv_rx->sk = sk;
679
680 priv_rx->rq_stats = &priv->channel_stats[rxq]->rq;
681 priv_rx->sw_stats = &priv->tls->sw_stats;
682 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx);
683
684 err = mlx5e_rx_res_tls_tir_create(priv->rx_res, rxq, &priv_rx->tir);
685 if (err)
686 goto err_create_tir;
687
688 init_completion(&priv_rx->add_ctx);
689
690 accel_rule_init(&priv_rx->rule, priv);
691 resync = &priv_rx->resync;
692 resync_init(resync, priv);
693 tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core;
694 tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC);
695
696 err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn);
697 if (err)
698 goto err_post_wqes;
699
700 atomic64_inc(&priv_rx->sw_stats->rx_tls_ctx);
701
702 return 0;
703
704 err_post_wqes:
705 mlx5e_tir_destroy(&priv_rx->tir);
706 err_create_tir:
707 mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek);
708 err_cipher_type:
709 kfree(priv_rx);
710 return err;
711 }
712
mlx5e_ktls_del_rx(struct net_device * netdev,struct tls_context * tls_ctx)713 void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx)
714 {
715 struct mlx5e_ktls_offload_context_rx *priv_rx;
716 struct mlx5e_ktls_rx_resync_ctx *resync;
717 struct mlx5e_priv *priv;
718
719 priv = netdev_priv(netdev);
720
721 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx);
722 set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags);
723 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL);
724 synchronize_net(); /* Sync with NAPI */
725 if (!cancel_work_sync(&priv_rx->rule.work))
726 /* completion is needed, as the priv_rx in the add flow
727 * is maintained on the wqe info (wi), not on the socket.
728 */
729 wait_for_completion(&priv_rx->add_ctx);
730 resync = &priv_rx->resync;
731 if (cancel_work_sync(&resync->work))
732 mlx5e_ktls_priv_rx_put(priv_rx);
733
734 atomic64_inc(&priv_rx->sw_stats->rx_tls_del);
735 if (priv_rx->rule.rule)
736 mlx5e_accel_fs_del_sk(priv_rx->rule.rule);
737
738 mlx5e_tir_destroy(&priv_rx->tir);
739 mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek);
740 /* priv_rx should normally be freed here, but if there is an outstanding
741 * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is
742 * processed.
743 */
744 mlx5e_ktls_priv_rx_put(priv_rx);
745 }
746
mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel * c,int budget)747 bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_channel *c, int budget)
748 {
749 struct mlx5e_ktls_offload_context_rx *priv_rx, *tmp;
750 struct mlx5e_ktls_resync_resp *ktls_resync;
751 struct mlx5_wqe_ctrl_seg *db_cseg;
752 struct mlx5e_icosq *sq;
753 LIST_HEAD(local_list);
754 int i, j;
755
756 sq = &c->async_icosq;
757
758 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state)))
759 return false;
760
761 ktls_resync = sq->ktls_resync;
762 db_cseg = NULL;
763 i = 0;
764
765 spin_lock(&ktls_resync->lock);
766 list_for_each_entry_safe(priv_rx, tmp, &ktls_resync->list, list) {
767 list_move(&priv_rx->list, &local_list);
768 if (++i == budget)
769 break;
770 }
771 if (list_empty(&ktls_resync->list))
772 clear_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
773 spin_unlock(&ktls_resync->lock);
774
775 spin_lock(&c->async_icosq_lock);
776 for (j = 0; j < i; j++) {
777 struct mlx5_wqe_ctrl_seg *cseg;
778
779 priv_rx = list_first_entry(&local_list,
780 struct mlx5e_ktls_offload_context_rx,
781 list);
782 spin_lock(&priv_rx->lock);
783 cseg = post_static_params(sq, priv_rx);
784 if (IS_ERR(cseg)) {
785 spin_unlock(&priv_rx->lock);
786 break;
787 }
788 list_del_init(&priv_rx->list);
789 spin_unlock(&priv_rx->lock);
790 db_cseg = cseg;
791 }
792 if (db_cseg)
793 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, db_cseg);
794 spin_unlock(&c->async_icosq_lock);
795
796 priv_rx->rq_stats->tls_resync_res_ok += j;
797
798 if (!list_empty(&local_list)) {
799 /* This happens only if ICOSQ is full.
800 * There is no need to mark busy or explicitly ask for a NAPI cycle,
801 * it will be triggered by the outstanding ICOSQ completions.
802 */
803 spin_lock(&ktls_resync->lock);
804 list_splice(&local_list, &ktls_resync->list);
805 set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state);
806 spin_unlock(&ktls_resync->lock);
807 priv_rx->rq_stats->tls_resync_res_retry++;
808 }
809
810 return i == budget;
811 }
812