1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/ipv6.h> 6 #include <linux/skbuff.h> 7 #include <linux/string.h> 8 #include <net/inet6_hashtables.h> 9 #include <net/tls.h> 10 11 #include "../ccm.h" 12 #include "../nfp_net.h" 13 #include "crypto.h" 14 #include "fw.h" 15 16 #define NFP_NET_TLS_CCM_MBOX_OPS_MASK \ 17 (BIT(NFP_CCM_TYPE_CRYPTO_RESET) | \ 18 BIT(NFP_CCM_TYPE_CRYPTO_ADD) | \ 19 BIT(NFP_CCM_TYPE_CRYPTO_DEL) | \ 20 BIT(NFP_CCM_TYPE_CRYPTO_UPDATE)) 21 22 #define NFP_NET_TLS_OPCODE_MASK_RX \ 23 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC) 24 25 #define NFP_NET_TLS_OPCODE_MASK_TX \ 26 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC) 27 28 #define NFP_NET_TLS_OPCODE_MASK \ 29 (NFP_NET_TLS_OPCODE_MASK_RX | NFP_NET_TLS_OPCODE_MASK_TX) 30 31 static void nfp_net_crypto_set_op(struct nfp_net *nn, u8 opcode, bool on) 32 { 33 u32 off, val; 34 35 off = nn->tlv_caps.crypto_enable_off + round_down(opcode / 8, 4); 36 37 val = nn_readl(nn, off); 38 if (on) 39 val |= BIT(opcode & 31); 40 else 41 val &= ~BIT(opcode & 31); 42 nn_writel(nn, off, val); 43 } 44 45 static bool 46 __nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add, 47 enum tls_offload_ctx_dir direction) 48 { 49 u8 opcode; 50 int cnt; 51 52 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 53 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 54 nn->ktls_tx_conn_cnt += add; 55 cnt = nn->ktls_tx_conn_cnt; 56 nn->dp.ktls_tx = !!nn->ktls_tx_conn_cnt; 57 } else { 58 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 59 nn->ktls_rx_conn_cnt += add; 60 cnt = nn->ktls_rx_conn_cnt; 61 } 62 63 /* Care only about 0 -> 1 and 1 -> 0 transitions */ 64 if (cnt > 1) 65 return false; 66 67 nfp_net_crypto_set_op(nn, opcode, cnt); 68 return true; 69 } 70 71 static int 72 nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add, 73 enum tls_offload_ctx_dir direction) 74 { 75 int ret = 0; 76 77 /* Use the BAR lock to protect the connection counts */ 78 nn_ctrl_bar_lock(nn); 79 if (__nfp_net_tls_conn_cnt_changed(nn, add, direction)) { 80 ret = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO); 81 /* Undo the cnt adjustment if failed */ 82 if (ret) 83 __nfp_net_tls_conn_cnt_changed(nn, -add, direction); 84 } 85 nn_ctrl_bar_unlock(nn); 86 87 return ret; 88 } 89 90 static int 91 nfp_net_tls_conn_add(struct nfp_net *nn, enum tls_offload_ctx_dir direction) 92 { 93 return nfp_net_tls_conn_cnt_changed(nn, 1, direction); 94 } 95 96 static int 97 nfp_net_tls_conn_remove(struct nfp_net *nn, enum tls_offload_ctx_dir direction) 98 { 99 return nfp_net_tls_conn_cnt_changed(nn, -1, direction); 100 } 101 102 static struct sk_buff * 103 nfp_net_tls_alloc_simple(struct nfp_net *nn, size_t req_sz, gfp_t flags) 104 { 105 return nfp_ccm_mbox_msg_alloc(nn, req_sz, 106 sizeof(struct nfp_crypto_reply_simple), 107 flags); 108 } 109 110 static int 111 nfp_net_tls_communicate_simple(struct nfp_net *nn, struct sk_buff *skb, 112 const char *name, enum nfp_ccm_type type) 113 { 114 struct nfp_crypto_reply_simple *reply; 115 int err; 116 117 err = __nfp_ccm_mbox_communicate(nn, skb, type, 118 sizeof(*reply), sizeof(*reply), 119 type == NFP_CCM_TYPE_CRYPTO_DEL); 120 if (err) { 121 nn_dp_warn(&nn->dp, "failed to %s TLS: %d\n", name, err); 122 return err; 123 } 124 125 reply = (void *)skb->data; 126 err = -be32_to_cpu(reply->error); 127 if (err) 128 nn_dp_warn(&nn->dp, "failed to %s TLS, fw replied: %d\n", 129 name, err); 130 dev_consume_skb_any(skb); 131 132 return err; 133 } 134 135 static void nfp_net_tls_del_fw(struct nfp_net *nn, __be32 *fw_handle) 136 { 137 struct nfp_crypto_req_del *req; 138 struct sk_buff *skb; 139 140 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL); 141 if (!skb) 142 return; 143 144 req = (void *)skb->data; 145 req->ep_id = 0; 146 memcpy(req->handle, fw_handle, sizeof(req->handle)); 147 148 nfp_net_tls_communicate_simple(nn, skb, "delete", 149 NFP_CCM_TYPE_CRYPTO_DEL); 150 } 151 152 static void 153 nfp_net_tls_set_ipver_vlan(struct nfp_crypto_req_add_front *front, u8 ipver) 154 { 155 front->ipver_vlan = cpu_to_be16(FIELD_PREP(NFP_NET_TLS_IPVER, ipver) | 156 FIELD_PREP(NFP_NET_TLS_VLAN, 157 NFP_NET_TLS_VLAN_UNUSED)); 158 } 159 160 static void 161 nfp_net_tls_assign_conn_id(struct nfp_net *nn, 162 struct nfp_crypto_req_add_front *front) 163 { 164 u32 len; 165 u64 id; 166 167 id = atomic64_inc_return(&nn->ktls_conn_id_gen); 168 len = front->key_len - NFP_NET_TLS_NON_ADDR_KEY_LEN; 169 170 memcpy(front->l3_addrs, &id, sizeof(id)); 171 memset(front->l3_addrs + sizeof(id), 0, len - sizeof(id)); 172 } 173 174 static struct nfp_crypto_req_add_back * 175 nfp_net_tls_set_ipv4(struct nfp_net *nn, struct nfp_crypto_req_add_v4 *req, 176 struct sock *sk, int direction) 177 { 178 struct inet_sock *inet = inet_sk(sk); 179 180 req->front.key_len += sizeof(__be32) * 2; 181 182 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 183 nfp_net_tls_assign_conn_id(nn, &req->front); 184 } else { 185 req->src_ip = inet->inet_daddr; 186 req->dst_ip = inet->inet_saddr; 187 } 188 189 return &req->back; 190 } 191 192 static struct nfp_crypto_req_add_back * 193 nfp_net_tls_set_ipv6(struct nfp_net *nn, struct nfp_crypto_req_add_v6 *req, 194 struct sock *sk, int direction) 195 { 196 #if IS_ENABLED(CONFIG_IPV6) 197 struct ipv6_pinfo *np = inet6_sk(sk); 198 199 req->front.key_len += sizeof(struct in6_addr) * 2; 200 201 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 202 nfp_net_tls_assign_conn_id(nn, &req->front); 203 } else { 204 memcpy(req->src_ip, &sk->sk_v6_daddr, sizeof(req->src_ip)); 205 memcpy(req->dst_ip, &np->saddr, sizeof(req->dst_ip)); 206 } 207 208 #endif 209 return &req->back; 210 } 211 212 static void 213 nfp_net_tls_set_l4(struct nfp_crypto_req_add_front *front, 214 struct nfp_crypto_req_add_back *back, struct sock *sk, 215 int direction) 216 { 217 struct inet_sock *inet = inet_sk(sk); 218 219 front->l4_proto = IPPROTO_TCP; 220 221 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 222 back->src_port = 0; 223 back->dst_port = 0; 224 } else { 225 back->src_port = inet->inet_dport; 226 back->dst_port = inet->inet_sport; 227 } 228 } 229 230 static u8 nfp_tls_1_2_dir_to_opcode(enum tls_offload_ctx_dir direction) 231 { 232 switch (direction) { 233 case TLS_OFFLOAD_CTX_DIR_TX: 234 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 235 case TLS_OFFLOAD_CTX_DIR_RX: 236 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 237 default: 238 WARN_ON_ONCE(1); 239 return 0; 240 } 241 } 242 243 static bool 244 nfp_net_cipher_supported(struct nfp_net *nn, u16 cipher_type, 245 enum tls_offload_ctx_dir direction) 246 { 247 u8 bit; 248 249 switch (cipher_type) { 250 case TLS_CIPHER_AES_GCM_128: 251 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 252 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 253 else 254 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 255 break; 256 default: 257 return false; 258 } 259 260 return nn->tlv_caps.crypto_ops & BIT(bit); 261 } 262 263 static int 264 nfp_net_tls_add(struct net_device *netdev, struct sock *sk, 265 enum tls_offload_ctx_dir direction, 266 struct tls_crypto_info *crypto_info, 267 u32 start_offload_tcp_sn) 268 { 269 struct tls12_crypto_info_aes_gcm_128 *tls_ci; 270 struct nfp_net *nn = netdev_priv(netdev); 271 struct nfp_crypto_req_add_front *front; 272 struct nfp_net_tls_offload_ctx *ntls; 273 struct nfp_crypto_req_add_back *back; 274 struct nfp_crypto_reply_add *reply; 275 struct sk_buff *skb; 276 size_t req_sz; 277 void *req; 278 bool ipv6; 279 int err; 280 281 BUILD_BUG_ON(sizeof(struct nfp_net_tls_offload_ctx) > 282 TLS_DRIVER_STATE_SIZE_TX); 283 BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) > 284 TLS_DRIVER_STATE_SIZE_RX); 285 286 if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction)) 287 return -EOPNOTSUPP; 288 289 switch (sk->sk_family) { 290 #if IS_ENABLED(CONFIG_IPV6) 291 case AF_INET6: 292 if (ipv6_only_sock(sk) || 293 ipv6_addr_type(&sk->sk_v6_daddr) != IPV6_ADDR_MAPPED) { 294 req_sz = sizeof(struct nfp_crypto_req_add_v6); 295 ipv6 = true; 296 break; 297 } 298 fallthrough; 299 #endif 300 case AF_INET: 301 req_sz = sizeof(struct nfp_crypto_req_add_v4); 302 ipv6 = false; 303 break; 304 default: 305 return -EOPNOTSUPP; 306 } 307 308 err = nfp_net_tls_conn_add(nn, direction); 309 if (err) 310 return err; 311 312 skb = nfp_ccm_mbox_msg_alloc(nn, req_sz, sizeof(*reply), GFP_KERNEL); 313 if (!skb) { 314 err = -ENOMEM; 315 goto err_conn_remove; 316 } 317 318 front = (void *)skb->data; 319 front->ep_id = 0; 320 front->key_len = NFP_NET_TLS_NON_ADDR_KEY_LEN; 321 front->opcode = nfp_tls_1_2_dir_to_opcode(direction); 322 memset(front->resv, 0, sizeof(front->resv)); 323 324 nfp_net_tls_set_ipver_vlan(front, ipv6 ? 6 : 4); 325 326 req = (void *)skb->data; 327 if (ipv6) 328 back = nfp_net_tls_set_ipv6(nn, req, sk, direction); 329 else 330 back = nfp_net_tls_set_ipv4(nn, req, sk, direction); 331 332 nfp_net_tls_set_l4(front, back, sk, direction); 333 334 back->counter = 0; 335 back->tcp_seq = cpu_to_be32(start_offload_tcp_sn); 336 337 tls_ci = (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 338 memcpy(back->key, tls_ci->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE); 339 memset(&back->key[TLS_CIPHER_AES_GCM_128_KEY_SIZE / 4], 0, 340 sizeof(back->key) - TLS_CIPHER_AES_GCM_128_KEY_SIZE); 341 memcpy(back->iv, tls_ci->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE); 342 memcpy(&back->salt, tls_ci->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 343 memcpy(back->rec_no, tls_ci->rec_seq, sizeof(tls_ci->rec_seq)); 344 345 /* Get an extra ref on the skb so we can wipe the key after */ 346 skb_get(skb); 347 348 err = nfp_ccm_mbox_communicate(nn, skb, NFP_CCM_TYPE_CRYPTO_ADD, 349 sizeof(*reply), sizeof(*reply)); 350 reply = (void *)skb->data; 351 352 /* We depend on CCM MBOX code not reallocating skb we sent 353 * so we can clear the key material out of the memory. 354 */ 355 if (!WARN_ON_ONCE((u8 *)back < skb->head || 356 (u8 *)back > skb_end_pointer(skb)) && 357 !WARN_ON_ONCE((u8 *)&reply[1] > (u8 *)back)) 358 memzero_explicit(back, sizeof(*back)); 359 dev_consume_skb_any(skb); /* the extra ref from skb_get() above */ 360 361 if (err) { 362 nn_dp_warn(&nn->dp, "failed to add TLS: %d (%d)\n", 363 err, direction == TLS_OFFLOAD_CTX_DIR_TX); 364 /* communicate frees skb on error */ 365 goto err_conn_remove; 366 } 367 368 err = -be32_to_cpu(reply->error); 369 if (err) { 370 if (err == -ENOSPC) { 371 if (!atomic_fetch_inc(&nn->ktls_no_space)) 372 nn_info(nn, "HW TLS table full\n"); 373 } else { 374 nn_dp_warn(&nn->dp, 375 "failed to add TLS, FW replied: %d\n", err); 376 } 377 goto err_free_skb; 378 } 379 380 if (!reply->handle[0] && !reply->handle[1]) { 381 nn_dp_warn(&nn->dp, "FW returned NULL handle\n"); 382 err = -EINVAL; 383 goto err_fw_remove; 384 } 385 386 ntls = tls_driver_ctx(sk, direction); 387 memcpy(ntls->fw_handle, reply->handle, sizeof(ntls->fw_handle)); 388 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 389 ntls->next_seq = start_offload_tcp_sn; 390 dev_consume_skb_any(skb); 391 392 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 393 return 0; 394 395 if (!nn->tlv_caps.tls_resync_ss) 396 tls_offload_rx_resync_set_type(sk, TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT); 397 398 return 0; 399 400 err_fw_remove: 401 nfp_net_tls_del_fw(nn, reply->handle); 402 err_free_skb: 403 dev_consume_skb_any(skb); 404 err_conn_remove: 405 nfp_net_tls_conn_remove(nn, direction); 406 return err; 407 } 408 409 static void 410 nfp_net_tls_del(struct net_device *netdev, struct tls_context *tls_ctx, 411 enum tls_offload_ctx_dir direction) 412 { 413 struct nfp_net *nn = netdev_priv(netdev); 414 struct nfp_net_tls_offload_ctx *ntls; 415 416 nfp_net_tls_conn_remove(nn, direction); 417 418 ntls = __tls_driver_ctx(tls_ctx, direction); 419 nfp_net_tls_del_fw(nn, ntls->fw_handle); 420 } 421 422 static int 423 nfp_net_tls_resync(struct net_device *netdev, struct sock *sk, u32 seq, 424 u8 *rcd_sn, enum tls_offload_ctx_dir direction) 425 { 426 struct nfp_net *nn = netdev_priv(netdev); 427 struct nfp_net_tls_offload_ctx *ntls; 428 struct nfp_crypto_req_update *req; 429 enum nfp_ccm_type type; 430 struct sk_buff *skb; 431 gfp_t flags; 432 int err; 433 434 flags = direction == TLS_OFFLOAD_CTX_DIR_TX ? GFP_KERNEL : GFP_ATOMIC; 435 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), flags); 436 if (!skb) 437 return -ENOMEM; 438 439 ntls = tls_driver_ctx(sk, direction); 440 req = (void *)skb->data; 441 req->ep_id = 0; 442 req->opcode = nfp_tls_1_2_dir_to_opcode(direction); 443 memset(req->resv, 0, sizeof(req->resv)); 444 memcpy(req->handle, ntls->fw_handle, sizeof(ntls->fw_handle)); 445 req->tcp_seq = cpu_to_be32(seq); 446 memcpy(req->rec_no, rcd_sn, sizeof(req->rec_no)); 447 448 type = NFP_CCM_TYPE_CRYPTO_UPDATE; 449 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 450 err = nfp_net_tls_communicate_simple(nn, skb, "sync", type); 451 if (err) 452 return err; 453 ntls->next_seq = seq; 454 } else { 455 if (nn->tlv_caps.tls_resync_ss) 456 type = NFP_CCM_TYPE_CRYPTO_RESYNC; 457 nfp_ccm_mbox_post(nn, skb, type, 458 sizeof(struct nfp_crypto_reply_simple)); 459 atomic_inc(&nn->ktls_rx_resync_sent); 460 } 461 462 return 0; 463 } 464 465 static const struct tlsdev_ops nfp_net_tls_ops = { 466 .tls_dev_add = nfp_net_tls_add, 467 .tls_dev_del = nfp_net_tls_del, 468 .tls_dev_resync = nfp_net_tls_resync, 469 }; 470 471 int nfp_net_tls_rx_resync_req(struct net_device *netdev, 472 struct nfp_net_tls_resync_req *req, 473 void *pkt, unsigned int pkt_len) 474 { 475 struct nfp_net *nn = netdev_priv(netdev); 476 struct nfp_net_tls_offload_ctx *ntls; 477 struct net *net = dev_net(netdev); 478 struct ipv6hdr *ipv6h; 479 struct tcphdr *th; 480 struct iphdr *iph; 481 struct sock *sk; 482 __be32 tcp_seq; 483 int err; 484 485 iph = pkt + req->l3_offset; 486 ipv6h = pkt + req->l3_offset; 487 th = pkt + req->l4_offset; 488 489 if ((u8 *)&th[1] > (u8 *)pkt + pkt_len) { 490 netdev_warn_once(netdev, "invalid TLS RX resync request (l3_off: %hhu l4_off: %hhu pkt_len: %u)\n", 491 req->l3_offset, req->l4_offset, pkt_len); 492 err = -EINVAL; 493 goto err_cnt_ign; 494 } 495 496 switch (ipv6h->version) { 497 case 4: 498 sk = inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 499 iph->saddr, th->source, iph->daddr, 500 th->dest, netdev->ifindex); 501 break; 502 #if IS_ENABLED(CONFIG_IPV6) 503 case 6: 504 sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo, 505 &ipv6h->saddr, th->source, 506 &ipv6h->daddr, ntohs(th->dest), 507 netdev->ifindex, 0); 508 break; 509 #endif 510 default: 511 netdev_warn_once(netdev, "invalid TLS RX resync request (l3_off: %hhu l4_off: %hhu ipver: %u)\n", 512 req->l3_offset, req->l4_offset, iph->version); 513 err = -EINVAL; 514 goto err_cnt_ign; 515 } 516 517 err = 0; 518 if (!sk) 519 goto err_cnt_ign; 520 if (!tls_is_sk_rx_device_offloaded(sk) || 521 sk->sk_shutdown & RCV_SHUTDOWN) 522 goto err_put_sock; 523 524 ntls = tls_driver_ctx(sk, TLS_OFFLOAD_CTX_DIR_RX); 525 /* some FW versions can't report the handle and report 0s */ 526 if (memchr_inv(&req->fw_handle, 0, sizeof(req->fw_handle)) && 527 memcmp(&req->fw_handle, &ntls->fw_handle, sizeof(ntls->fw_handle))) 528 goto err_put_sock; 529 530 /* copy to ensure alignment */ 531 memcpy(&tcp_seq, &req->tcp_seq, sizeof(tcp_seq)); 532 tls_offload_rx_resync_request(sk, tcp_seq); 533 atomic_inc(&nn->ktls_rx_resync_req); 534 535 sock_gen_put(sk); 536 return 0; 537 538 err_put_sock: 539 sock_gen_put(sk); 540 err_cnt_ign: 541 atomic_inc(&nn->ktls_rx_resync_ign); 542 return err; 543 } 544 545 static int nfp_net_tls_reset(struct nfp_net *nn) 546 { 547 struct nfp_crypto_req_reset *req; 548 struct sk_buff *skb; 549 550 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL); 551 if (!skb) 552 return -ENOMEM; 553 554 req = (void *)skb->data; 555 req->ep_id = 0; 556 557 return nfp_net_tls_communicate_simple(nn, skb, "reset", 558 NFP_CCM_TYPE_CRYPTO_RESET); 559 } 560 561 int nfp_net_tls_init(struct nfp_net *nn) 562 { 563 struct net_device *netdev = nn->dp.netdev; 564 int err; 565 566 if (!(nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK)) 567 return 0; 568 569 if ((nn->tlv_caps.mbox_cmsg_types & NFP_NET_TLS_CCM_MBOX_OPS_MASK) != 570 NFP_NET_TLS_CCM_MBOX_OPS_MASK) 571 return 0; 572 573 if (!nfp_ccm_mbox_fits(nn, sizeof(struct nfp_crypto_req_add_v6))) { 574 nn_warn(nn, "disabling TLS offload - mbox too small: %d\n", 575 nn->tlv_caps.mbox_len); 576 return 0; 577 } 578 579 err = nfp_net_tls_reset(nn); 580 if (err) 581 return err; 582 583 nn_ctrl_bar_lock(nn); 584 nn_writel(nn, nn->tlv_caps.crypto_enable_off, 0); 585 err = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO); 586 nn_ctrl_bar_unlock(nn); 587 if (err) 588 return err; 589 590 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_RX) { 591 netdev->hw_features |= NETIF_F_HW_TLS_RX; 592 netdev->features |= NETIF_F_HW_TLS_RX; 593 } 594 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_TX) { 595 netdev->hw_features |= NETIF_F_HW_TLS_TX; 596 netdev->features |= NETIF_F_HW_TLS_TX; 597 } 598 599 netdev->tlsdev_ops = &nfp_net_tls_ops; 600 601 return 0; 602 } 603