1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include <net/inet6_hashtables.h> 5 #include "en_accel/en_accel.h" 6 #include "en_accel/ktls.h" 7 #include "en_accel/ktls_txrx.h" 8 #include "en_accel/ktls_utils.h" 9 #include "en_accel/fs_tcp.h" 10 11 struct accel_rule { 12 struct work_struct work; 13 struct mlx5e_priv *priv; 14 struct mlx5_flow_handle *rule; 15 }; 16 17 #define PROGRESS_PARAMS_WRITE_UNIT 64 18 #define PROGRESS_PARAMS_PADDED_SIZE \ 19 (ALIGN(sizeof(struct mlx5_wqe_tls_progress_params_seg), \ 20 PROGRESS_PARAMS_WRITE_UNIT)) 21 22 struct mlx5e_ktls_rx_resync_buf { 23 union { 24 struct mlx5_wqe_tls_progress_params_seg progress; 25 u8 pad[PROGRESS_PARAMS_PADDED_SIZE]; 26 } ____cacheline_aligned_in_smp; 27 dma_addr_t dma_addr; 28 struct mlx5e_ktls_offload_context_rx *priv_rx; 29 }; 30 31 enum { 32 MLX5E_PRIV_RX_FLAG_DELETING, 33 MLX5E_NUM_PRIV_RX_FLAGS, 34 }; 35 36 struct mlx5e_ktls_rx_resync_ctx { 37 struct tls_offload_resync_async core; 38 struct work_struct work; 39 struct mlx5e_priv *priv; 40 refcount_t refcnt; 41 __be64 sw_rcd_sn_be; 42 u32 seq; 43 }; 44 45 struct mlx5e_ktls_offload_context_rx { 46 union mlx5e_crypto_info crypto_info; 47 struct accel_rule rule; 48 struct sock *sk; 49 struct mlx5e_rq_stats *rq_stats; 50 struct mlx5e_tls_sw_stats *sw_stats; 51 struct completion add_ctx; 52 struct mlx5e_tir tir; 53 struct mlx5_crypto_dek *dek; 54 u32 rxq; 55 DECLARE_BITMAP(flags, MLX5E_NUM_PRIV_RX_FLAGS); 56 57 /* resync */ 58 spinlock_t lock; /* protects resync fields */ 59 struct mlx5e_ktls_rx_resync_ctx resync; 60 struct list_head list; 61 }; 62 63 static bool mlx5e_ktls_priv_rx_put(struct mlx5e_ktls_offload_context_rx *priv_rx) 64 { 65 if (!refcount_dec_and_test(&priv_rx->resync.refcnt)) 66 return false; 67 68 kfree(priv_rx); 69 return true; 70 } 71 72 static void mlx5e_ktls_priv_rx_get(struct mlx5e_ktls_offload_context_rx *priv_rx) 73 { 74 refcount_inc(&priv_rx->resync.refcnt); 75 } 76 77 struct mlx5e_ktls_resync_resp { 78 /* protects list changes */ 79 spinlock_t lock; 80 struct list_head list; 81 }; 82 83 void mlx5e_ktls_rx_resync_destroy_resp_list(struct mlx5e_ktls_resync_resp *resp_list) 84 { 85 kvfree(resp_list); 86 } 87 88 struct mlx5e_ktls_resync_resp * 89 mlx5e_ktls_rx_resync_create_resp_list(void) 90 { 91 struct mlx5e_ktls_resync_resp *resp_list; 92 93 resp_list = kvzalloc_obj(*resp_list); 94 if (!resp_list) 95 return ERR_PTR(-ENOMEM); 96 97 INIT_LIST_HEAD(&resp_list->list); 98 spin_lock_init(&resp_list->lock); 99 100 return resp_list; 101 } 102 103 static void accel_rule_handle_work(struct work_struct *work) 104 { 105 struct mlx5e_ktls_offload_context_rx *priv_rx; 106 struct accel_rule *accel_rule; 107 struct mlx5_flow_handle *rule; 108 109 accel_rule = container_of(work, struct accel_rule, work); 110 priv_rx = container_of(accel_rule, struct mlx5e_ktls_offload_context_rx, rule); 111 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) 112 goto out; 113 114 rule = mlx5e_accel_fs_add_sk(accel_rule->priv->fs, priv_rx->sk, 115 mlx5e_tir_get_tirn(&priv_rx->tir), 116 MLX5_FS_DEFAULT_FLOW_TAG); 117 if (!IS_ERR_OR_NULL(rule)) 118 accel_rule->rule = rule; 119 out: 120 complete(&priv_rx->add_ctx); 121 } 122 123 static void accel_rule_init(struct accel_rule *rule, struct mlx5e_priv *priv) 124 { 125 INIT_WORK(&rule->work, accel_rule_handle_work); 126 rule->priv = priv; 127 } 128 129 static void icosq_fill_wi(struct mlx5e_icosq *sq, u16 pi, 130 struct mlx5e_icosq_wqe_info *wi) 131 { 132 sq->db.wqe_info[pi] = *wi; 133 } 134 135 static struct mlx5_wqe_ctrl_seg * 136 post_static_params(struct mlx5e_icosq *sq, 137 struct mlx5e_ktls_offload_context_rx *priv_rx) 138 { 139 struct mlx5e_set_tls_static_params_wqe *wqe; 140 struct mlx5e_icosq_wqe_info wi; 141 u16 pi, num_wqebbs; 142 143 num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; 144 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs))) 145 return ERR_PTR(-ENOSPC); 146 147 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); 148 wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); 149 mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_rx->crypto_info, 150 mlx5e_tir_get_tirn(&priv_rx->tir), 151 mlx5_crypto_dek_get_id(priv_rx->dek), 152 priv_rx->resync.seq, false, 153 TLS_OFFLOAD_CTX_DIR_RX); 154 wi = (struct mlx5e_icosq_wqe_info) { 155 .wqe_type = MLX5E_ICOSQ_WQE_UMR_TLS, 156 .num_wqebbs = num_wqebbs, 157 .tls_set_params.priv_rx = priv_rx, 158 }; 159 icosq_fill_wi(sq, pi, &wi); 160 sq->pc += num_wqebbs; 161 162 return &wqe->ctrl; 163 } 164 165 static struct mlx5_wqe_ctrl_seg * 166 post_progress_params(struct mlx5e_icosq *sq, 167 struct mlx5e_ktls_offload_context_rx *priv_rx, 168 u32 next_record_tcp_sn) 169 { 170 struct mlx5e_set_tls_progress_params_wqe *wqe; 171 struct mlx5e_icosq_wqe_info wi; 172 u16 pi, num_wqebbs; 173 174 num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; 175 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, num_wqebbs))) 176 return ERR_PTR(-ENOSPC); 177 178 pi = mlx5e_icosq_get_next_pi(sq, num_wqebbs); 179 wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); 180 mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, 181 mlx5e_tir_get_tirn(&priv_rx->tir), 182 false, next_record_tcp_sn, 183 TLS_OFFLOAD_CTX_DIR_RX); 184 wi = (struct mlx5e_icosq_wqe_info) { 185 .wqe_type = MLX5E_ICOSQ_WQE_SET_PSV_TLS, 186 .num_wqebbs = num_wqebbs, 187 .tls_set_params.priv_rx = priv_rx, 188 }; 189 190 icosq_fill_wi(sq, pi, &wi); 191 sq->pc += num_wqebbs; 192 193 return &wqe->ctrl; 194 } 195 196 static int post_rx_param_wqes(struct mlx5e_channel *c, 197 struct mlx5e_ktls_offload_context_rx *priv_rx, 198 u32 next_record_tcp_sn) 199 { 200 struct mlx5_wqe_ctrl_seg *cseg; 201 struct mlx5e_icosq *sq; 202 int err; 203 204 err = 0; 205 sq = c->async_icosq; 206 spin_lock_bh(&sq->lock); 207 208 cseg = post_static_params(sq, priv_rx); 209 if (IS_ERR(cseg)) 210 goto err_out; 211 cseg = post_progress_params(sq, priv_rx, next_record_tcp_sn); 212 if (IS_ERR(cseg)) 213 goto err_out; 214 215 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 216 unlock: 217 spin_unlock_bh(&sq->lock); 218 219 return err; 220 221 err_out: 222 priv_rx->rq_stats->tls_resync_req_skip++; 223 err = PTR_ERR(cseg); 224 complete(&priv_rx->add_ctx); 225 goto unlock; 226 } 227 228 static void 229 mlx5e_set_ktls_rx_priv_ctx(struct tls_context *tls_ctx, 230 struct mlx5e_ktls_offload_context_rx *priv_rx) 231 { 232 struct mlx5e_ktls_offload_context_rx **ctx = 233 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); 234 235 BUILD_BUG_ON(sizeof(priv_rx) > TLS_DRIVER_STATE_SIZE_RX); 236 237 *ctx = priv_rx; 238 } 239 240 static struct mlx5e_ktls_offload_context_rx * 241 mlx5e_get_ktls_rx_priv_ctx(struct tls_context *tls_ctx) 242 { 243 struct mlx5e_ktls_offload_context_rx **ctx = 244 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_RX); 245 246 return *ctx; 247 } 248 249 /* Re-sync */ 250 /* Runs in work context */ 251 static int 252 resync_post_get_progress_params(struct mlx5e_icosq *sq, 253 struct mlx5e_ktls_offload_context_rx *priv_rx) 254 { 255 struct mlx5e_get_tls_progress_params_wqe *wqe; 256 struct mlx5e_ktls_rx_resync_buf *buf; 257 struct mlx5e_icosq_wqe_info wi; 258 struct mlx5_wqe_ctrl_seg *cseg; 259 struct mlx5_seg_get_psv *psv; 260 struct device *pdev; 261 int err; 262 u16 pi; 263 264 buf = kzalloc_obj(*buf); 265 if (unlikely(!buf)) { 266 err = -ENOMEM; 267 goto err_out; 268 } 269 270 pdev = mlx5_core_dma_dev(sq->channel->mdev); 271 buf->dma_addr = dma_map_single(pdev, &buf->progress, 272 PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); 273 if (unlikely(dma_mapping_error(pdev, buf->dma_addr))) { 274 err = -ENOMEM; 275 goto err_free; 276 } 277 278 buf->priv_rx = priv_rx; 279 280 spin_lock_bh(&sq->lock); 281 282 if (unlikely(!mlx5e_icosq_can_post_wqe(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS))) { 283 spin_unlock_bh(&sq->lock); 284 err = -ENOSPC; 285 goto err_dma_unmap; 286 } 287 288 pi = mlx5e_icosq_get_next_pi(sq, MLX5E_KTLS_GET_PROGRESS_WQEBBS); 289 wqe = MLX5E_TLS_FETCH_GET_PROGRESS_PARAMS_WQE(sq, pi); 290 291 #define GET_PSV_DS_CNT (DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS)) 292 293 cseg = &wqe->ctrl; 294 cseg->opmod_idx_opcode = 295 cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_GET_PSV | 296 (MLX5_OPC_MOD_TLS_TIR_PROGRESS_PARAMS << 24)); 297 cseg->qpn_ds = 298 cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | GET_PSV_DS_CNT); 299 300 psv = &wqe->psv; 301 psv->num_psv = 1 << 4; 302 psv->l_key = sq->channel->mkey_be; 303 psv->psv_index[0] = cpu_to_be32(mlx5e_tir_get_tirn(&priv_rx->tir)); 304 psv->va = cpu_to_be64(buf->dma_addr); 305 306 wi = (struct mlx5e_icosq_wqe_info) { 307 .wqe_type = MLX5E_ICOSQ_WQE_GET_PSV_TLS, 308 .num_wqebbs = MLX5E_KTLS_GET_PROGRESS_WQEBBS, 309 .tls_get_params.buf = buf, 310 }; 311 icosq_fill_wi(sq, pi, &wi); 312 sq->pc++; 313 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, cseg); 314 spin_unlock_bh(&sq->lock); 315 316 return 0; 317 318 err_dma_unmap: 319 dma_unmap_single(pdev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); 320 err_free: 321 kfree(buf); 322 err_out: 323 return err; 324 } 325 326 /* Function is called with elevated refcount. 327 * It decreases it only if no WQE is posted. 328 */ 329 static void resync_handle_work(struct work_struct *work) 330 { 331 struct mlx5e_ktls_offload_context_rx *priv_rx; 332 struct mlx5e_ktls_rx_resync_ctx *resync; 333 struct mlx5e_channel *c; 334 struct mlx5e_icosq *sq; 335 336 resync = container_of(work, struct mlx5e_ktls_rx_resync_ctx, work); 337 priv_rx = container_of(resync, struct mlx5e_ktls_offload_context_rx, resync); 338 339 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { 340 mlx5e_ktls_priv_rx_put(priv_rx); 341 priv_rx->rq_stats->tls_resync_req_skip++; 342 tls_offload_rx_resync_async_request_cancel(&resync->core); 343 return; 344 } 345 346 c = resync->priv->channels.c[priv_rx->rxq]; 347 sq = c->async_icosq; 348 349 if (resync_post_get_progress_params(sq, priv_rx)) { 350 priv_rx->rq_stats->tls_resync_req_skip++; 351 tls_offload_rx_resync_async_request_cancel(&resync->core); 352 mlx5e_ktls_priv_rx_put(priv_rx); 353 } 354 } 355 356 static void resync_init(struct mlx5e_ktls_rx_resync_ctx *resync, 357 struct mlx5e_priv *priv) 358 { 359 INIT_WORK(&resync->work, resync_handle_work); 360 resync->priv = priv; 361 refcount_set(&resync->refcnt, 1); 362 } 363 364 /* Function can be called with the refcount being either elevated or not. 365 * It does not affect the refcount. 366 */ 367 static void resync_handle_seq_match(struct mlx5e_ktls_offload_context_rx *priv_rx, 368 struct mlx5e_channel *c) 369 { 370 struct mlx5e_ktls_resync_resp *ktls_resync; 371 struct mlx5e_icosq *sq; 372 bool trigger_poll; 373 374 sq = c->async_icosq; 375 ktls_resync = sq->ktls_resync; 376 trigger_poll = false; 377 378 spin_lock_bh(&ktls_resync->lock); 379 spin_lock_bh(&priv_rx->lock); 380 switch (priv_rx->crypto_info.crypto_info.cipher_type) { 381 case TLS_CIPHER_AES_GCM_128: { 382 struct tls12_crypto_info_aes_gcm_128 *info = 383 &priv_rx->crypto_info.crypto_info_128; 384 385 memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, 386 sizeof(info->rec_seq)); 387 break; 388 } 389 case TLS_CIPHER_AES_GCM_256: { 390 struct tls12_crypto_info_aes_gcm_256 *info = 391 &priv_rx->crypto_info.crypto_info_256; 392 393 memcpy(info->rec_seq, &priv_rx->resync.sw_rcd_sn_be, 394 sizeof(info->rec_seq)); 395 break; 396 } 397 default: 398 WARN_ONCE(1, "Unsupported cipher type %u\n", 399 priv_rx->crypto_info.crypto_info.cipher_type); 400 spin_unlock_bh(&priv_rx->lock); 401 spin_unlock_bh(&ktls_resync->lock); 402 return; 403 } 404 405 if (list_empty(&priv_rx->list)) { 406 list_add_tail(&priv_rx->list, &ktls_resync->list); 407 trigger_poll = !test_and_set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); 408 } 409 spin_unlock_bh(&priv_rx->lock); 410 spin_unlock_bh(&ktls_resync->lock); 411 412 if (!trigger_poll) 413 return; 414 415 if (!napi_if_scheduled_mark_missed(&c->napi)) { 416 spin_lock_bh(&sq->lock); 417 mlx5e_trigger_irq(sq); 418 spin_unlock_bh(&sq->lock); 419 } 420 } 421 422 /* Function can be called with the refcount being either elevated or not. 423 * It decreases the refcount and may free the kTLS priv context. 424 * Refcount is not elevated only if tls_dev_del has been called, but GET_PSV was 425 * already in flight. 426 */ 427 void mlx5e_ktls_handle_get_psv_completion(struct mlx5e_icosq_wqe_info *wi, 428 struct mlx5e_icosq *sq) 429 { 430 struct mlx5e_ktls_rx_resync_buf *buf = wi->tls_get_params.buf; 431 struct mlx5e_ktls_offload_context_rx *priv_rx; 432 struct tls_offload_resync_async *async_resync; 433 struct tls_offload_context_rx *rx_ctx; 434 u8 tracker_state, auth_state, *ctx; 435 struct device *dev; 436 u32 hw_seq; 437 438 priv_rx = buf->priv_rx; 439 dev = mlx5_core_dma_dev(sq->channel->mdev); 440 rx_ctx = tls_offload_ctx_rx(tls_get_ctx(priv_rx->sk)); 441 async_resync = rx_ctx->resync_async; 442 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { 443 priv_rx->rq_stats->tls_resync_req_skip++; 444 tls_offload_rx_resync_async_request_cancel(async_resync); 445 goto out; 446 } 447 448 dma_sync_single_for_cpu(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, 449 DMA_FROM_DEVICE); 450 451 ctx = buf->progress.ctx; 452 tracker_state = MLX5_GET(tls_progress_params, ctx, record_tracker_state); 453 auth_state = MLX5_GET(tls_progress_params, ctx, auth_state); 454 if (tracker_state != MLX5E_TLS_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING || 455 auth_state != MLX5E_TLS_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD) { 456 priv_rx->rq_stats->tls_resync_req_skip++; 457 tls_offload_rx_resync_async_request_cancel(async_resync); 458 goto out; 459 } 460 461 hw_seq = MLX5_GET(tls_progress_params, ctx, hw_resync_tcp_sn); 462 tls_offload_rx_resync_async_request_end(async_resync, 463 cpu_to_be32(hw_seq)); 464 priv_rx->rq_stats->tls_resync_req_end++; 465 out: 466 mlx5e_ktls_priv_rx_put(priv_rx); 467 dma_unmap_single(dev, buf->dma_addr, PROGRESS_PARAMS_PADDED_SIZE, DMA_FROM_DEVICE); 468 kfree(buf); 469 } 470 471 /* Runs in NAPI. 472 * Function elevates the refcount, unless no work is queued. 473 */ 474 static bool resync_queue_get_psv(struct sock *sk) 475 { 476 struct mlx5e_ktls_offload_context_rx *priv_rx; 477 struct mlx5e_ktls_rx_resync_ctx *resync; 478 479 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); 480 if (unlikely(!priv_rx)) 481 return false; 482 483 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) 484 return false; 485 486 resync = &priv_rx->resync; 487 mlx5e_ktls_priv_rx_get(priv_rx); 488 if (unlikely(!queue_work(resync->priv->tls->rx_wq, &resync->work))) { 489 mlx5e_ktls_priv_rx_put(priv_rx); 490 return false; 491 } 492 493 return true; 494 } 495 496 /* Runs in NAPI */ 497 static void resync_update_sn(struct mlx5e_rq *rq, struct sk_buff *skb) 498 { 499 struct ethhdr *eth = (struct ethhdr *)(skb->data); 500 struct tls_offload_resync_async *resync_async; 501 struct net_device *netdev = rq->netdev; 502 struct net *net = dev_net(netdev); 503 struct sock *sk = NULL; 504 unsigned int datalen; 505 struct iphdr *iph; 506 struct tcphdr *th; 507 __be32 seq; 508 int depth = 0; 509 510 __vlan_get_protocol(skb, eth->h_proto, &depth); 511 iph = (struct iphdr *)(skb->data + depth); 512 513 if (iph->version == 4) { 514 depth += sizeof(struct iphdr); 515 th = (void *)iph + sizeof(struct iphdr); 516 517 sk = inet_lookup_established(net, iph->saddr, th->source, 518 iph->daddr, th->dest, 519 netdev->ifindex); 520 #if IS_ENABLED(CONFIG_IPV6) 521 } else { 522 struct ipv6hdr *ipv6h = (struct ipv6hdr *)iph; 523 524 depth += sizeof(struct ipv6hdr); 525 th = (void *)ipv6h + sizeof(struct ipv6hdr); 526 527 sk = __inet6_lookup_established(net, &ipv6h->saddr, th->source, 528 &ipv6h->daddr, ntohs(th->dest), 529 netdev->ifindex, 0); 530 #endif 531 } 532 533 depth += sizeof(struct tcphdr); 534 535 if (unlikely(!sk)) 536 return; 537 538 if (unlikely(sk->sk_state == TCP_TIME_WAIT)) 539 goto unref; 540 541 if (unlikely(!resync_queue_get_psv(sk))) 542 goto unref; 543 544 seq = th->seq; 545 datalen = skb->len - depth; 546 resync_async = tls_offload_ctx_rx(tls_get_ctx(sk))->resync_async; 547 tls_offload_rx_resync_async_request_start(resync_async, seq, datalen); 548 rq->stats->tls_resync_req_start++; 549 550 unref: 551 sock_gen_put(sk); 552 } 553 554 void mlx5e_ktls_rx_resync(struct net_device *netdev, struct sock *sk, 555 u32 seq, u8 *rcd_sn) 556 { 557 struct mlx5e_ktls_offload_context_rx *priv_rx; 558 struct mlx5e_ktls_rx_resync_ctx *resync; 559 struct mlx5e_priv *priv; 560 struct mlx5e_channel *c; 561 562 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_get_ctx(sk)); 563 if (unlikely(!priv_rx)) 564 return; 565 566 resync = &priv_rx->resync; 567 resync->sw_rcd_sn_be = *(__be64 *)rcd_sn; 568 resync->seq = seq; 569 570 priv = netdev_priv(netdev); 571 c = priv->channels.c[priv_rx->rxq]; 572 573 resync_handle_seq_match(priv_rx, c); 574 } 575 576 void 577 mlx5e_ktls_rx_resync_async_request_cancel(struct mlx5e_icosq_wqe_info *wi) 578 { 579 struct mlx5e_ktls_offload_context_rx *priv_rx; 580 struct mlx5e_ktls_rx_resync_buf *buf; 581 582 buf = wi->tls_get_params.buf; 583 priv_rx = buf->priv_rx; 584 priv_rx->rq_stats->tls_resync_req_skip++; 585 tls_offload_rx_resync_async_request_cancel(&priv_rx->resync.core); 586 } 587 588 /* End of resync section */ 589 590 void mlx5e_ktls_handle_rx_skb(struct mlx5e_rq *rq, struct sk_buff *skb, 591 struct mlx5_cqe64 *cqe, u32 *cqe_bcnt) 592 { 593 struct mlx5e_rq_stats *stats = rq->stats; 594 595 switch (get_cqe_tls_offload(cqe)) { 596 case CQE_TLS_OFFLOAD_DECRYPTED: 597 skb->decrypted = 1; 598 stats->tls_decrypted_packets++; 599 stats->tls_decrypted_bytes += *cqe_bcnt; 600 break; 601 case CQE_TLS_OFFLOAD_RESYNC: 602 stats->tls_resync_req_pkt++; 603 resync_update_sn(rq, skb); 604 break; 605 default: /* CQE_TLS_OFFLOAD_ERROR: */ 606 stats->tls_err++; 607 break; 608 } 609 } 610 611 void mlx5e_ktls_handle_ctx_completion(struct mlx5e_icosq_wqe_info *wi) 612 { 613 struct mlx5e_ktls_offload_context_rx *priv_rx = wi->tls_set_params.priv_rx; 614 struct accel_rule *rule = &priv_rx->rule; 615 616 if (unlikely(test_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags))) { 617 complete(&priv_rx->add_ctx); 618 return; 619 } 620 queue_work(rule->priv->tls->rx_wq, &rule->work); 621 } 622 623 static int mlx5e_ktls_sk_get_rxq(struct sock *sk) 624 { 625 int rxq = sk_rx_queue_get(sk); 626 627 if (unlikely(rxq == -1)) 628 rxq = 0; 629 630 return rxq; 631 } 632 633 int mlx5e_ktls_add_rx(struct net_device *netdev, struct sock *sk, 634 struct tls_crypto_info *crypto_info, 635 u32 start_offload_tcp_sn) 636 { 637 struct mlx5e_ktls_offload_context_rx *priv_rx; 638 struct mlx5e_ktls_rx_resync_ctx *resync; 639 struct tls_context *tls_ctx; 640 struct mlx5_crypto_dek *dek; 641 struct mlx5e_priv *priv; 642 int rxq, err; 643 644 tls_ctx = tls_get_ctx(sk); 645 priv = netdev_priv(netdev); 646 priv_rx = kzalloc_obj(*priv_rx); 647 if (unlikely(!priv_rx)) 648 return -ENOMEM; 649 650 switch (crypto_info->cipher_type) { 651 case TLS_CIPHER_AES_GCM_128: 652 priv_rx->crypto_info.crypto_info_128 = 653 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 654 break; 655 case TLS_CIPHER_AES_GCM_256: 656 priv_rx->crypto_info.crypto_info_256 = 657 *(struct tls12_crypto_info_aes_gcm_256 *)crypto_info; 658 break; 659 default: 660 WARN_ONCE(1, "Unsupported cipher type %u\n", 661 crypto_info->cipher_type); 662 err = -EOPNOTSUPP; 663 goto err_cipher_type; 664 } 665 666 dek = mlx5_ktls_create_key(priv->tls->dek_pool, crypto_info); 667 if (IS_ERR(dek)) { 668 err = PTR_ERR(dek); 669 goto err_cipher_type; 670 } 671 priv_rx->dek = dek; 672 673 INIT_LIST_HEAD(&priv_rx->list); 674 spin_lock_init(&priv_rx->lock); 675 676 rxq = mlx5e_ktls_sk_get_rxq(sk); 677 priv_rx->rxq = rxq; 678 priv_rx->sk = sk; 679 680 priv_rx->rq_stats = &priv->channel_stats[rxq]->rq; 681 priv_rx->sw_stats = &priv->tls->sw_stats; 682 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, priv_rx); 683 684 err = mlx5e_rx_res_tls_tir_create(priv->rx_res, rxq, &priv_rx->tir); 685 if (err) 686 goto err_create_tir; 687 688 init_completion(&priv_rx->add_ctx); 689 690 accel_rule_init(&priv_rx->rule, priv); 691 resync = &priv_rx->resync; 692 resync_init(resync, priv); 693 tls_offload_ctx_rx(tls_ctx)->resync_async = &resync->core; 694 tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_DRIVER_REQ_ASYNC); 695 696 err = post_rx_param_wqes(priv->channels.c[rxq], priv_rx, start_offload_tcp_sn); 697 if (err) 698 goto err_post_wqes; 699 700 atomic64_inc(&priv_rx->sw_stats->rx_tls_ctx); 701 702 return 0; 703 704 err_post_wqes: 705 mlx5e_tir_destroy(&priv_rx->tir); 706 err_create_tir: 707 mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek); 708 err_cipher_type: 709 kfree(priv_rx); 710 return err; 711 } 712 713 void mlx5e_ktls_del_rx(struct net_device *netdev, struct tls_context *tls_ctx) 714 { 715 struct mlx5e_ktls_offload_context_rx *priv_rx; 716 struct mlx5e_ktls_rx_resync_ctx *resync; 717 struct mlx5e_priv *priv; 718 719 priv = netdev_priv(netdev); 720 721 priv_rx = mlx5e_get_ktls_rx_priv_ctx(tls_ctx); 722 set_bit(MLX5E_PRIV_RX_FLAG_DELETING, priv_rx->flags); 723 mlx5e_set_ktls_rx_priv_ctx(tls_ctx, NULL); 724 synchronize_net(); /* Sync with NAPI */ 725 if (!cancel_work_sync(&priv_rx->rule.work)) 726 /* completion is needed, as the priv_rx in the add flow 727 * is maintained on the wqe info (wi), not on the socket. 728 */ 729 wait_for_completion(&priv_rx->add_ctx); 730 resync = &priv_rx->resync; 731 if (cancel_work_sync(&resync->work)) 732 mlx5e_ktls_priv_rx_put(priv_rx); 733 734 atomic64_inc(&priv_rx->sw_stats->rx_tls_del); 735 if (priv_rx->rule.rule) 736 mlx5e_accel_fs_del_sk(priv_rx->rule.rule); 737 738 mlx5e_tir_destroy(&priv_rx->tir); 739 mlx5_ktls_destroy_key(priv->tls->dek_pool, priv_rx->dek); 740 /* priv_rx should normally be freed here, but if there is an outstanding 741 * GET_PSV, deallocation will be delayed until the CQE for GET_PSV is 742 * processed. 743 */ 744 mlx5e_ktls_priv_rx_put(priv_rx); 745 } 746 747 bool mlx5e_ktls_rx_handle_resync_list(struct mlx5e_icosq *sq, int budget) 748 { 749 struct mlx5e_ktls_offload_context_rx *priv_rx, *tmp; 750 struct mlx5e_ktls_resync_resp *ktls_resync; 751 struct mlx5_wqe_ctrl_seg *db_cseg; 752 LIST_HEAD(local_list); 753 int i, j; 754 755 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &sq->state))) 756 return false; 757 758 ktls_resync = sq->ktls_resync; 759 db_cseg = NULL; 760 i = 0; 761 762 spin_lock(&ktls_resync->lock); 763 list_for_each_entry_safe(priv_rx, tmp, &ktls_resync->list, list) { 764 list_move(&priv_rx->list, &local_list); 765 if (++i == budget) 766 break; 767 } 768 if (list_empty(&ktls_resync->list)) 769 clear_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); 770 spin_unlock(&ktls_resync->lock); 771 772 spin_lock(&sq->lock); 773 for (j = 0; j < i; j++) { 774 struct mlx5_wqe_ctrl_seg *cseg; 775 776 priv_rx = list_first_entry(&local_list, 777 struct mlx5e_ktls_offload_context_rx, 778 list); 779 spin_lock(&priv_rx->lock); 780 cseg = post_static_params(sq, priv_rx); 781 if (IS_ERR(cseg)) { 782 spin_unlock(&priv_rx->lock); 783 break; 784 } 785 list_del_init(&priv_rx->list); 786 spin_unlock(&priv_rx->lock); 787 db_cseg = cseg; 788 } 789 if (db_cseg) 790 mlx5e_notify_hw(&sq->wq, sq->pc, sq->uar_map, db_cseg); 791 spin_unlock(&sq->lock); 792 793 priv_rx->rq_stats->tls_resync_res_ok += j; 794 795 if (!list_empty(&local_list)) { 796 /* This happens only if ICOSQ is full. 797 * There is no need to mark busy or explicitly ask for a NAPI cycle, 798 * it will be triggered by the outstanding ICOSQ completions. 799 */ 800 spin_lock(&ktls_resync->lock); 801 list_splice(&local_list, &ktls_resync->list); 802 set_bit(MLX5E_SQ_STATE_PENDING_TLS_RX_RESYNC, &sq->state); 803 spin_unlock(&ktls_resync->lock); 804 priv_rx->rq_stats->tls_resync_res_retry++; 805 } 806 807 return i == budget; 808 } 809