1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2019 Netronome Systems, Inc. */ 3 4 #include <linux/bitfield.h> 5 #include <linux/ipv6.h> 6 #include <linux/skbuff.h> 7 #include <net/tls.h> 8 9 #include "../ccm.h" 10 #include "../nfp_net.h" 11 #include "crypto.h" 12 #include "fw.h" 13 14 #define NFP_NET_TLS_CCM_MBOX_OPS_MASK \ 15 (BIT(NFP_CCM_TYPE_CRYPTO_RESET) | \ 16 BIT(NFP_CCM_TYPE_CRYPTO_ADD) | \ 17 BIT(NFP_CCM_TYPE_CRYPTO_DEL) | \ 18 BIT(NFP_CCM_TYPE_CRYPTO_UPDATE)) 19 20 #define NFP_NET_TLS_OPCODE_MASK_RX \ 21 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC) 22 23 #define NFP_NET_TLS_OPCODE_MASK_TX \ 24 BIT(NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC) 25 26 #define NFP_NET_TLS_OPCODE_MASK \ 27 (NFP_NET_TLS_OPCODE_MASK_RX | NFP_NET_TLS_OPCODE_MASK_TX) 28 29 static void nfp_net_crypto_set_op(struct nfp_net *nn, u8 opcode, bool on) 30 { 31 u32 off, val; 32 33 off = nn->tlv_caps.crypto_enable_off + round_down(opcode / 8, 4); 34 35 val = nn_readl(nn, off); 36 if (on) 37 val |= BIT(opcode & 31); 38 else 39 val &= ~BIT(opcode & 31); 40 nn_writel(nn, off, val); 41 } 42 43 static bool 44 __nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add, 45 enum tls_offload_ctx_dir direction) 46 { 47 u8 opcode; 48 int cnt; 49 50 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 51 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 52 nn->ktls_tx_conn_cnt += add; 53 cnt = nn->ktls_tx_conn_cnt; 54 nn->dp.ktls_tx = !!nn->ktls_tx_conn_cnt; 55 } else { 56 opcode = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 57 nn->ktls_rx_conn_cnt += add; 58 cnt = nn->ktls_rx_conn_cnt; 59 } 60 61 /* Care only about 0 -> 1 and 1 -> 0 transitions */ 62 if (cnt > 1) 63 return false; 64 65 nfp_net_crypto_set_op(nn, opcode, cnt); 66 return true; 67 } 68 69 static int 70 nfp_net_tls_conn_cnt_changed(struct nfp_net *nn, int add, 71 enum tls_offload_ctx_dir direction) 72 { 73 int ret = 0; 74 75 /* Use the BAR lock to protect the connection counts */ 76 nn_ctrl_bar_lock(nn); 77 if (__nfp_net_tls_conn_cnt_changed(nn, add, direction)) { 78 ret = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO); 79 /* Undo the cnt adjustment if failed */ 80 if (ret) 81 __nfp_net_tls_conn_cnt_changed(nn, -add, direction); 82 } 83 nn_ctrl_bar_unlock(nn); 84 85 return ret; 86 } 87 88 static int 89 nfp_net_tls_conn_add(struct nfp_net *nn, enum tls_offload_ctx_dir direction) 90 { 91 return nfp_net_tls_conn_cnt_changed(nn, 1, direction); 92 } 93 94 static int 95 nfp_net_tls_conn_remove(struct nfp_net *nn, enum tls_offload_ctx_dir direction) 96 { 97 return nfp_net_tls_conn_cnt_changed(nn, -1, direction); 98 } 99 100 static struct sk_buff * 101 nfp_net_tls_alloc_simple(struct nfp_net *nn, size_t req_sz, gfp_t flags) 102 { 103 return nfp_ccm_mbox_msg_alloc(nn, req_sz, 104 sizeof(struct nfp_crypto_reply_simple), 105 flags); 106 } 107 108 static int 109 nfp_net_tls_communicate_simple(struct nfp_net *nn, struct sk_buff *skb, 110 const char *name, enum nfp_ccm_type type) 111 { 112 struct nfp_crypto_reply_simple *reply; 113 int err; 114 115 err = nfp_ccm_mbox_communicate(nn, skb, type, 116 sizeof(*reply), sizeof(*reply)); 117 if (err) { 118 nn_dp_warn(&nn->dp, "failed to %s TLS: %d\n", name, err); 119 return err; 120 } 121 122 reply = (void *)skb->data; 123 err = -be32_to_cpu(reply->error); 124 if (err) 125 nn_dp_warn(&nn->dp, "failed to %s TLS, fw replied: %d\n", 126 name, err); 127 dev_consume_skb_any(skb); 128 129 return err; 130 } 131 132 static void nfp_net_tls_del_fw(struct nfp_net *nn, __be32 *fw_handle) 133 { 134 struct nfp_crypto_req_del *req; 135 struct sk_buff *skb; 136 137 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL); 138 if (!skb) 139 return; 140 141 req = (void *)skb->data; 142 req->ep_id = 0; 143 memcpy(req->handle, fw_handle, sizeof(req->handle)); 144 145 nfp_net_tls_communicate_simple(nn, skb, "delete", 146 NFP_CCM_TYPE_CRYPTO_DEL); 147 } 148 149 static struct nfp_crypto_req_add_back * 150 nfp_net_tls_set_ipv4(struct nfp_crypto_req_add_v4 *req, struct sock *sk, 151 int direction) 152 { 153 struct inet_sock *inet = inet_sk(sk); 154 155 req->front.key_len += sizeof(__be32) * 2; 156 req->front.ipver_vlan = cpu_to_be16(FIELD_PREP(NFP_NET_TLS_IPVER, 4) | 157 FIELD_PREP(NFP_NET_TLS_VLAN, 158 NFP_NET_TLS_VLAN_UNUSED)); 159 160 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 161 req->src_ip = inet->inet_saddr; 162 req->dst_ip = inet->inet_daddr; 163 } else { 164 req->src_ip = inet->inet_daddr; 165 req->dst_ip = inet->inet_saddr; 166 } 167 168 return &req->back; 169 } 170 171 static struct nfp_crypto_req_add_back * 172 nfp_net_tls_set_ipv6(struct nfp_crypto_req_add_v6 *req, struct sock *sk, 173 int direction) 174 { 175 #if IS_ENABLED(CONFIG_IPV6) 176 struct ipv6_pinfo *np = inet6_sk(sk); 177 178 req->front.key_len += sizeof(struct in6_addr) * 2; 179 req->front.ipver_vlan = cpu_to_be16(FIELD_PREP(NFP_NET_TLS_IPVER, 6) | 180 FIELD_PREP(NFP_NET_TLS_VLAN, 181 NFP_NET_TLS_VLAN_UNUSED)); 182 183 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 184 memcpy(req->src_ip, &np->saddr, sizeof(req->src_ip)); 185 memcpy(req->dst_ip, &sk->sk_v6_daddr, sizeof(req->dst_ip)); 186 } else { 187 memcpy(req->src_ip, &sk->sk_v6_daddr, sizeof(req->src_ip)); 188 memcpy(req->dst_ip, &np->saddr, sizeof(req->dst_ip)); 189 } 190 191 #endif 192 return &req->back; 193 } 194 195 static void 196 nfp_net_tls_set_l4(struct nfp_crypto_req_add_front *front, 197 struct nfp_crypto_req_add_back *back, struct sock *sk, 198 int direction) 199 { 200 struct inet_sock *inet = inet_sk(sk); 201 202 front->l4_proto = IPPROTO_TCP; 203 204 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 205 back->src_port = inet->inet_sport; 206 back->dst_port = inet->inet_dport; 207 } else { 208 back->src_port = inet->inet_dport; 209 back->dst_port = inet->inet_sport; 210 } 211 } 212 213 static u8 nfp_tls_1_2_dir_to_opcode(enum tls_offload_ctx_dir direction) 214 { 215 switch (direction) { 216 case TLS_OFFLOAD_CTX_DIR_TX: 217 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 218 case TLS_OFFLOAD_CTX_DIR_RX: 219 return NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 220 default: 221 WARN_ON_ONCE(1); 222 return 0; 223 } 224 } 225 226 static bool 227 nfp_net_cipher_supported(struct nfp_net *nn, u16 cipher_type, 228 enum tls_offload_ctx_dir direction) 229 { 230 u8 bit; 231 232 switch (cipher_type) { 233 case TLS_CIPHER_AES_GCM_128: 234 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 235 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_ENC; 236 else 237 bit = NFP_NET_CRYPTO_OP_TLS_1_2_AES_GCM_128_DEC; 238 break; 239 default: 240 return false; 241 } 242 243 return nn->tlv_caps.crypto_ops & BIT(bit); 244 } 245 246 static int 247 nfp_net_tls_add(struct net_device *netdev, struct sock *sk, 248 enum tls_offload_ctx_dir direction, 249 struct tls_crypto_info *crypto_info, 250 u32 start_offload_tcp_sn) 251 { 252 struct tls12_crypto_info_aes_gcm_128 *tls_ci; 253 struct nfp_net *nn = netdev_priv(netdev); 254 struct nfp_crypto_req_add_front *front; 255 struct nfp_net_tls_offload_ctx *ntls; 256 struct nfp_crypto_req_add_back *back; 257 struct nfp_crypto_reply_add *reply; 258 struct sk_buff *skb; 259 size_t req_sz; 260 bool ipv6; 261 int err; 262 263 BUILD_BUG_ON(sizeof(struct nfp_net_tls_offload_ctx) > 264 TLS_DRIVER_STATE_SIZE_TX); 265 BUILD_BUG_ON(offsetof(struct nfp_net_tls_offload_ctx, rx_end) > 266 TLS_DRIVER_STATE_SIZE_RX); 267 268 if (!nfp_net_cipher_supported(nn, crypto_info->cipher_type, direction)) 269 return -EOPNOTSUPP; 270 271 switch (sk->sk_family) { 272 #if IS_ENABLED(CONFIG_IPV6) 273 case AF_INET6: 274 if (sk->sk_ipv6only || 275 ipv6_addr_type(&sk->sk_v6_daddr) != IPV6_ADDR_MAPPED) { 276 req_sz = sizeof(struct nfp_crypto_req_add_v6); 277 ipv6 = true; 278 break; 279 } 280 #endif 281 /* fall through */ 282 case AF_INET: 283 req_sz = sizeof(struct nfp_crypto_req_add_v4); 284 ipv6 = false; 285 break; 286 default: 287 return -EOPNOTSUPP; 288 } 289 290 err = nfp_net_tls_conn_add(nn, direction); 291 if (err) 292 return err; 293 294 skb = nfp_ccm_mbox_msg_alloc(nn, req_sz, sizeof(*reply), GFP_KERNEL); 295 if (!skb) { 296 err = -ENOMEM; 297 goto err_conn_remove; 298 } 299 300 front = (void *)skb->data; 301 front->ep_id = 0; 302 front->key_len = 8; 303 front->opcode = nfp_tls_1_2_dir_to_opcode(direction); 304 memset(front->resv, 0, sizeof(front->resv)); 305 306 if (ipv6) 307 back = nfp_net_tls_set_ipv6((void *)skb->data, sk, direction); 308 else 309 back = nfp_net_tls_set_ipv4((void *)skb->data, sk, direction); 310 311 nfp_net_tls_set_l4(front, back, sk, direction); 312 313 back->counter = 0; 314 back->tcp_seq = cpu_to_be32(start_offload_tcp_sn); 315 316 tls_ci = (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 317 memcpy(back->key, tls_ci->key, TLS_CIPHER_AES_GCM_128_KEY_SIZE); 318 memset(&back->key[TLS_CIPHER_AES_GCM_128_KEY_SIZE / 4], 0, 319 sizeof(back->key) - TLS_CIPHER_AES_GCM_128_KEY_SIZE); 320 memcpy(back->iv, tls_ci->iv, TLS_CIPHER_AES_GCM_128_IV_SIZE); 321 memcpy(&back->salt, tls_ci->salt, TLS_CIPHER_AES_GCM_128_SALT_SIZE); 322 memcpy(back->rec_no, tls_ci->rec_seq, sizeof(tls_ci->rec_seq)); 323 324 err = nfp_ccm_mbox_communicate(nn, skb, NFP_CCM_TYPE_CRYPTO_ADD, 325 sizeof(*reply), sizeof(*reply)); 326 if (err) { 327 nn_dp_warn(&nn->dp, "failed to add TLS: %d\n", err); 328 /* communicate frees skb on error */ 329 goto err_conn_remove; 330 } 331 332 reply = (void *)skb->data; 333 err = -be32_to_cpu(reply->error); 334 if (err) { 335 if (err == -ENOSPC) { 336 if (!atomic_fetch_inc(&nn->ktls_no_space)) 337 nn_info(nn, "HW TLS table full\n"); 338 } else { 339 nn_dp_warn(&nn->dp, 340 "failed to add TLS, FW replied: %d\n", err); 341 } 342 goto err_free_skb; 343 } 344 345 if (!reply->handle[0] && !reply->handle[1]) { 346 nn_dp_warn(&nn->dp, "FW returned NULL handle\n"); 347 err = -EINVAL; 348 goto err_fw_remove; 349 } 350 351 ntls = tls_driver_ctx(sk, direction); 352 memcpy(ntls->fw_handle, reply->handle, sizeof(ntls->fw_handle)); 353 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 354 ntls->next_seq = start_offload_tcp_sn; 355 dev_consume_skb_any(skb); 356 357 if (direction == TLS_OFFLOAD_CTX_DIR_TX) 358 return 0; 359 360 tls_offload_rx_resync_set_type(sk, 361 TLS_OFFLOAD_SYNC_TYPE_CORE_NEXT_HINT); 362 return 0; 363 364 err_fw_remove: 365 nfp_net_tls_del_fw(nn, reply->handle); 366 err_free_skb: 367 dev_consume_skb_any(skb); 368 err_conn_remove: 369 nfp_net_tls_conn_remove(nn, direction); 370 return err; 371 } 372 373 static void 374 nfp_net_tls_del(struct net_device *netdev, struct tls_context *tls_ctx, 375 enum tls_offload_ctx_dir direction) 376 { 377 struct nfp_net *nn = netdev_priv(netdev); 378 struct nfp_net_tls_offload_ctx *ntls; 379 380 nfp_net_tls_conn_remove(nn, direction); 381 382 ntls = __tls_driver_ctx(tls_ctx, direction); 383 nfp_net_tls_del_fw(nn, ntls->fw_handle); 384 } 385 386 static void 387 nfp_net_tls_resync(struct net_device *netdev, struct sock *sk, u32 seq, 388 u8 *rcd_sn, enum tls_offload_ctx_dir direction) 389 { 390 struct nfp_net *nn = netdev_priv(netdev); 391 struct nfp_net_tls_offload_ctx *ntls; 392 struct nfp_crypto_req_update *req; 393 struct sk_buff *skb; 394 gfp_t flags; 395 396 flags = direction == TLS_OFFLOAD_CTX_DIR_TX ? GFP_KERNEL : GFP_ATOMIC; 397 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), flags); 398 if (!skb) 399 return; 400 401 ntls = tls_driver_ctx(sk, direction); 402 req = (void *)skb->data; 403 req->ep_id = 0; 404 req->opcode = nfp_tls_1_2_dir_to_opcode(direction); 405 memset(req->resv, 0, sizeof(req->resv)); 406 memcpy(req->handle, ntls->fw_handle, sizeof(ntls->fw_handle)); 407 req->tcp_seq = cpu_to_be32(seq); 408 memcpy(req->rec_no, rcd_sn, sizeof(req->rec_no)); 409 410 if (direction == TLS_OFFLOAD_CTX_DIR_TX) { 411 nfp_net_tls_communicate_simple(nn, skb, "sync", 412 NFP_CCM_TYPE_CRYPTO_UPDATE); 413 ntls->next_seq = seq; 414 } else { 415 nfp_ccm_mbox_post(nn, skb, NFP_CCM_TYPE_CRYPTO_UPDATE, 416 sizeof(struct nfp_crypto_reply_simple)); 417 } 418 } 419 420 static const struct tlsdev_ops nfp_net_tls_ops = { 421 .tls_dev_add = nfp_net_tls_add, 422 .tls_dev_del = nfp_net_tls_del, 423 .tls_dev_resync = nfp_net_tls_resync, 424 }; 425 426 static int nfp_net_tls_reset(struct nfp_net *nn) 427 { 428 struct nfp_crypto_req_reset *req; 429 struct sk_buff *skb; 430 431 skb = nfp_net_tls_alloc_simple(nn, sizeof(*req), GFP_KERNEL); 432 if (!skb) 433 return -ENOMEM; 434 435 req = (void *)skb->data; 436 req->ep_id = 0; 437 438 return nfp_net_tls_communicate_simple(nn, skb, "reset", 439 NFP_CCM_TYPE_CRYPTO_RESET); 440 } 441 442 int nfp_net_tls_init(struct nfp_net *nn) 443 { 444 struct net_device *netdev = nn->dp.netdev; 445 int err; 446 447 if (!(nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK)) 448 return 0; 449 450 if ((nn->tlv_caps.mbox_cmsg_types & NFP_NET_TLS_CCM_MBOX_OPS_MASK) != 451 NFP_NET_TLS_CCM_MBOX_OPS_MASK) 452 return 0; 453 454 if (!nfp_ccm_mbox_fits(nn, sizeof(struct nfp_crypto_req_add_v6))) { 455 nn_warn(nn, "disabling TLS offload - mbox too small: %d\n", 456 nn->tlv_caps.mbox_len); 457 return 0; 458 } 459 460 err = nfp_net_tls_reset(nn); 461 if (err) 462 return err; 463 464 nn_ctrl_bar_lock(nn); 465 nn_writel(nn, nn->tlv_caps.crypto_enable_off, 0); 466 err = __nfp_net_reconfig(nn, NFP_NET_CFG_UPDATE_CRYPTO); 467 nn_ctrl_bar_unlock(nn); 468 if (err) 469 return err; 470 471 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_RX) { 472 netdev->hw_features |= NETIF_F_HW_TLS_RX; 473 netdev->features |= NETIF_F_HW_TLS_RX; 474 } 475 if (nn->tlv_caps.crypto_ops & NFP_NET_TLS_OPCODE_MASK_TX) { 476 netdev->hw_features |= NETIF_F_HW_TLS_TX; 477 netdev->features |= NETIF_F_HW_TLS_TX; 478 } 479 480 netdev->tlsdev_ops = &nfp_net_tls_ops; 481 482 return 0; 483 } 484