1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (c) 2018 Chelsio Communications, Inc. 4 * 5 * Written by: Atul Gupta (atul.gupta@chelsio.com) 6 */ 7 #include <linux/kernel.h> 8 #include <linux/module.h> 9 #include <linux/skbuff.h> 10 #include <linux/socket.h> 11 #include <linux/hash.h> 12 #include <linux/in.h> 13 #include <linux/net.h> 14 #include <linux/ip.h> 15 #include <linux/tcp.h> 16 #include <net/ipv6.h> 17 #include <net/transp_v6.h> 18 #include <net/tcp.h> 19 #include <net/tls.h> 20 21 #include "chtls.h" 22 #include "chtls_cm.h" 23 24 #define DRV_NAME "chtls" 25 26 /* 27 * chtls device management 28 * maintains a list of the chtls devices 29 */ 30 static LIST_HEAD(cdev_list); 31 static DEFINE_MUTEX(cdev_mutex); 32 33 static DEFINE_MUTEX(notify_mutex); 34 static RAW_NOTIFIER_HEAD(listen_notify_list); 35 static struct proto chtls_cpl_prot, chtls_cpl_protv6; 36 struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6; 37 static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT; 38 39 static void register_listen_notifier(struct notifier_block *nb) 40 { 41 mutex_lock(¬ify_mutex); 42 raw_notifier_chain_register(&listen_notify_list, nb); 43 mutex_unlock(¬ify_mutex); 44 } 45 46 static void unregister_listen_notifier(struct notifier_block *nb) 47 { 48 mutex_lock(¬ify_mutex); 49 raw_notifier_chain_unregister(&listen_notify_list, nb); 50 mutex_unlock(¬ify_mutex); 51 } 52 53 static int listen_notify_handler(struct notifier_block *this, 54 unsigned long event, void *data) 55 { 56 struct chtls_listen *clisten; 57 int ret = NOTIFY_DONE; 58 59 clisten = (struct chtls_listen *)data; 60 61 switch (event) { 62 case CHTLS_LISTEN_START: 63 ret = chtls_listen_start(clisten->cdev, clisten->sk); 64 kfree(clisten); 65 break; 66 case CHTLS_LISTEN_STOP: 67 chtls_listen_stop(clisten->cdev, clisten->sk); 68 kfree(clisten); 69 break; 70 } 71 return ret; 72 } 73 74 static struct notifier_block listen_notifier = { 75 .notifier_call = listen_notify_handler 76 }; 77 78 static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb) 79 { 80 if (likely(skb_transport_header(skb) != skb_network_header(skb))) 81 return tcp_v4_do_rcv(sk, skb); 82 BLOG_SKB_CB(skb)->backlog_rcv(sk, skb); 83 return 0; 84 } 85 86 static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk) 87 { 88 struct chtls_listen *clisten; 89 90 if (sk->sk_protocol != IPPROTO_TCP) 91 return -EPROTONOSUPPORT; 92 93 if (sk->sk_family == PF_INET && 94 LOOPBACK(inet_sk(sk)->inet_rcv_saddr)) 95 return -EADDRNOTAVAIL; 96 97 sk->sk_backlog_rcv = listen_backlog_rcv; 98 clisten = kmalloc(sizeof(*clisten), GFP_KERNEL); 99 if (!clisten) 100 return -ENOMEM; 101 clisten->cdev = cdev; 102 clisten->sk = sk; 103 mutex_lock(¬ify_mutex); 104 raw_notifier_call_chain(&listen_notify_list, 105 CHTLS_LISTEN_START, clisten); 106 mutex_unlock(¬ify_mutex); 107 return 0; 108 } 109 110 static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk) 111 { 112 struct chtls_listen *clisten; 113 114 if (sk->sk_protocol != IPPROTO_TCP) 115 return; 116 117 clisten = kmalloc(sizeof(*clisten), GFP_KERNEL); 118 if (!clisten) 119 return; 120 clisten->cdev = cdev; 121 clisten->sk = sk; 122 mutex_lock(¬ify_mutex); 123 raw_notifier_call_chain(&listen_notify_list, 124 CHTLS_LISTEN_STOP, clisten); 125 mutex_unlock(¬ify_mutex); 126 } 127 128 static int chtls_inline_feature(struct tls_toe_device *dev) 129 { 130 struct net_device *netdev; 131 struct chtls_dev *cdev; 132 int i; 133 134 cdev = to_chtls_dev(dev); 135 136 for (i = 0; i < cdev->lldi->nports; i++) { 137 netdev = cdev->ports[i]; 138 if (netdev->features & NETIF_F_HW_TLS_RECORD) 139 return 1; 140 } 141 return 0; 142 } 143 144 static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk) 145 { 146 struct chtls_dev *cdev = to_chtls_dev(dev); 147 148 if (sk->sk_state == TCP_LISTEN) 149 return chtls_start_listen(cdev, sk); 150 return 0; 151 } 152 153 static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk) 154 { 155 struct chtls_dev *cdev = to_chtls_dev(dev); 156 157 if (sk->sk_state == TCP_LISTEN) 158 chtls_stop_listen(cdev, sk); 159 } 160 161 static void chtls_free_uld(struct chtls_dev *cdev) 162 { 163 int i; 164 165 tls_toe_unregister_device(&cdev->tlsdev); 166 kvfree(cdev->kmap.addr); 167 idr_destroy(&cdev->hwtid_idr); 168 for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) 169 kfree_skb(cdev->rspq_skb_cache[i]); 170 kfree(cdev->lldi); 171 kfree_skb(cdev->askb); 172 kfree(cdev); 173 } 174 175 static inline void chtls_dev_release(struct kref *kref) 176 { 177 struct tls_toe_device *dev; 178 struct chtls_dev *cdev; 179 struct adapter *adap; 180 181 dev = container_of(kref, struct tls_toe_device, kref); 182 cdev = to_chtls_dev(dev); 183 184 /* Reset tls rx/tx stats */ 185 adap = pci_get_drvdata(cdev->pdev); 186 atomic_set(&adap->chcr_stats.tls_pdu_tx, 0); 187 atomic_set(&adap->chcr_stats.tls_pdu_rx, 0); 188 189 chtls_free_uld(cdev); 190 } 191 192 static void chtls_register_dev(struct chtls_dev *cdev) 193 { 194 struct tls_toe_device *tlsdev = &cdev->tlsdev; 195 196 strscpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX); 197 strlcat(tlsdev->name, cdev->lldi->ports[0]->name, 198 TLS_TOE_DEVICE_NAME_MAX); 199 tlsdev->feature = chtls_inline_feature; 200 tlsdev->hash = chtls_create_hash; 201 tlsdev->unhash = chtls_destroy_hash; 202 tlsdev->release = chtls_dev_release; 203 kref_init(&tlsdev->kref); 204 tls_toe_register_device(tlsdev); 205 cdev->cdev_state = CHTLS_CDEV_STATE_UP; 206 } 207 208 static void process_deferq(struct work_struct *task_param) 209 { 210 struct chtls_dev *cdev = container_of(task_param, 211 struct chtls_dev, deferq_task); 212 struct sk_buff *skb; 213 214 spin_lock_bh(&cdev->deferq.lock); 215 while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) { 216 spin_unlock_bh(&cdev->deferq.lock); 217 DEFERRED_SKB_CB(skb)->handler(cdev, skb); 218 spin_lock_bh(&cdev->deferq.lock); 219 } 220 spin_unlock_bh(&cdev->deferq.lock); 221 } 222 223 static int chtls_get_skb(struct chtls_dev *cdev) 224 { 225 cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL); 226 if (!cdev->askb) 227 return -ENOMEM; 228 229 skb_put(cdev->askb, sizeof(struct tcphdr)); 230 skb_reset_transport_header(cdev->askb); 231 memset(cdev->askb->data, 0, cdev->askb->len); 232 return 0; 233 } 234 235 static void *chtls_uld_add(const struct cxgb4_lld_info *info) 236 { 237 struct cxgb4_lld_info *lldi; 238 struct chtls_dev *cdev; 239 int i, j; 240 241 cdev = kzalloc(sizeof(*cdev), GFP_KERNEL); 242 if (!cdev) 243 goto out; 244 245 lldi = kzalloc(sizeof(*lldi), GFP_KERNEL); 246 if (!lldi) 247 goto out_lldi; 248 249 if (chtls_get_skb(cdev)) 250 goto out_skb; 251 252 *lldi = *info; 253 cdev->lldi = lldi; 254 cdev->pdev = lldi->pdev; 255 cdev->tids = lldi->tids; 256 cdev->ports = lldi->ports; 257 cdev->mtus = lldi->mtus; 258 cdev->tids = lldi->tids; 259 cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0])) 260 << FW_VIID_PFN_S; 261 262 for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) { 263 unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8; 264 265 cdev->rspq_skb_cache[i] = __alloc_skb(size, 266 gfp_any(), 0, 267 lldi->nodeid); 268 if (unlikely(!cdev->rspq_skb_cache[i])) 269 goto out_rspq_skb; 270 } 271 272 idr_init(&cdev->hwtid_idr); 273 INIT_WORK(&cdev->deferq_task, process_deferq); 274 spin_lock_init(&cdev->listen_lock); 275 spin_lock_init(&cdev->idr_lock); 276 cdev->send_page_order = min_t(uint, get_order(32768), 277 send_page_order); 278 cdev->max_host_sndbuf = 48 * 1024; 279 280 if (lldi->vr->key.size) 281 if (chtls_init_kmap(cdev, lldi)) 282 goto out_rspq_skb; 283 284 mutex_lock(&cdev_mutex); 285 list_add_tail(&cdev->list, &cdev_list); 286 mutex_unlock(&cdev_mutex); 287 288 return cdev; 289 out_rspq_skb: 290 for (j = 0; j < i; j++) 291 kfree_skb(cdev->rspq_skb_cache[j]); 292 kfree_skb(cdev->askb); 293 out_skb: 294 kfree(lldi); 295 out_lldi: 296 kfree(cdev); 297 out: 298 return NULL; 299 } 300 301 static void chtls_free_all_uld(void) 302 { 303 struct chtls_dev *cdev, *tmp; 304 305 mutex_lock(&cdev_mutex); 306 list_for_each_entry_safe(cdev, tmp, &cdev_list, list) { 307 if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) { 308 list_del(&cdev->list); 309 kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release); 310 } 311 } 312 mutex_unlock(&cdev_mutex); 313 } 314 315 static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state) 316 { 317 struct chtls_dev *cdev = handle; 318 319 switch (new_state) { 320 case CXGB4_STATE_UP: 321 chtls_register_dev(cdev); 322 break; 323 case CXGB4_STATE_DOWN: 324 break; 325 case CXGB4_STATE_START_RECOVERY: 326 break; 327 case CXGB4_STATE_DETACH: 328 mutex_lock(&cdev_mutex); 329 list_del(&cdev->list); 330 mutex_unlock(&cdev_mutex); 331 kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release); 332 break; 333 default: 334 break; 335 } 336 return 0; 337 } 338 339 static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl, 340 const __be64 *rsp, 341 u32 pktshift) 342 { 343 struct sk_buff *skb; 344 345 /* Allocate space for cpl_pass_accept_req which will be synthesized by 346 * driver. Once driver synthesizes cpl_pass_accept_req the skb will go 347 * through the regular cpl_pass_accept_req processing in TOM. 348 */ 349 skb = alloc_skb(size_add(gl->tot_len, 350 sizeof(struct cpl_pass_accept_req)) - 351 pktshift, GFP_ATOMIC); 352 if (unlikely(!skb)) 353 return NULL; 354 __skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req) 355 - pktshift); 356 /* For now we will copy cpl_rx_pkt in the skb */ 357 skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt)); 358 skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req) 359 , gl->va + pktshift, 360 gl->tot_len - pktshift); 361 362 return skb; 363 } 364 365 static int chtls_recv_packet(struct chtls_dev *cdev, 366 const struct pkt_gl *gl, const __be64 *rsp) 367 { 368 unsigned int opcode = *(u8 *)rsp; 369 struct sk_buff *skb; 370 int ret; 371 372 skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift); 373 if (!skb) 374 return -ENOMEM; 375 376 ret = chtls_handlers[opcode](cdev, skb); 377 if (ret & CPL_RET_BUF_DONE) 378 kfree_skb(skb); 379 380 return 0; 381 } 382 383 static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp) 384 { 385 unsigned long rspq_bin; 386 unsigned int opcode; 387 struct sk_buff *skb; 388 unsigned int len; 389 int ret; 390 391 len = 64 - sizeof(struct rsp_ctrl) - 8; 392 opcode = *(u8 *)rsp; 393 394 rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS); 395 skb = cdev->rspq_skb_cache[rspq_bin]; 396 if (skb && !skb_is_nonlinear(skb) && 397 !skb_shared(skb) && !skb_cloned(skb)) { 398 refcount_inc(&skb->users); 399 if (refcount_read(&skb->users) == 2) { 400 __skb_trim(skb, 0); 401 if (skb_tailroom(skb) >= len) 402 goto copy_out; 403 } 404 refcount_dec(&skb->users); 405 } 406 skb = alloc_skb(len, GFP_ATOMIC); 407 if (unlikely(!skb)) 408 return -ENOMEM; 409 410 copy_out: 411 __skb_put(skb, len); 412 skb_copy_to_linear_data(skb, rsp, len); 413 skb_reset_network_header(skb); 414 skb_reset_transport_header(skb); 415 ret = chtls_handlers[opcode](cdev, skb); 416 417 if (ret & CPL_RET_BUF_DONE) 418 kfree_skb(skb); 419 return 0; 420 } 421 422 static void chtls_recv(struct chtls_dev *cdev, 423 struct sk_buff **skbs, const __be64 *rsp) 424 { 425 struct sk_buff *skb = *skbs; 426 unsigned int opcode; 427 int ret; 428 429 opcode = *(u8 *)rsp; 430 431 __skb_push(skb, sizeof(struct rss_header)); 432 skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header)); 433 434 ret = chtls_handlers[opcode](cdev, skb); 435 if (ret & CPL_RET_BUF_DONE) 436 kfree_skb(skb); 437 } 438 439 static int chtls_uld_rx_handler(void *handle, const __be64 *rsp, 440 const struct pkt_gl *gl) 441 { 442 struct chtls_dev *cdev = handle; 443 unsigned int opcode; 444 struct sk_buff *skb; 445 446 opcode = *(u8 *)rsp; 447 448 if (unlikely(opcode == CPL_RX_PKT)) { 449 if (chtls_recv_packet(cdev, gl, rsp) < 0) 450 goto nomem; 451 return 0; 452 } 453 454 if (!gl) 455 return chtls_recv_rsp(cdev, rsp); 456 457 #define RX_PULL_LEN 128 458 skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN); 459 if (unlikely(!skb)) 460 goto nomem; 461 chtls_recv(cdev, &skb, rsp); 462 return 0; 463 464 nomem: 465 return -ENOMEM; 466 } 467 468 static int do_chtls_getsockopt(struct sock *sk, char __user *optval, 469 int __user *optlen) 470 { 471 struct tls_crypto_info crypto_info = { 0 }; 472 473 crypto_info.version = TLS_1_2_VERSION; 474 if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info))) 475 return -EFAULT; 476 return 0; 477 } 478 479 static int chtls_getsockopt(struct sock *sk, int level, int optname, 480 char __user *optval, int __user *optlen) 481 { 482 struct tls_context *ctx = tls_get_ctx(sk); 483 484 if (level != SOL_TLS) 485 return ctx->sk_proto->getsockopt(sk, level, 486 optname, optval, optlen); 487 488 return do_chtls_getsockopt(sk, optval, optlen); 489 } 490 491 static int do_chtls_setsockopt(struct sock *sk, int optname, 492 sockptr_t optval, unsigned int optlen) 493 { 494 struct tls_crypto_info *crypto_info, tmp_crypto_info; 495 struct chtls_sock *csk; 496 int keylen; 497 int cipher_type; 498 int rc = 0; 499 500 csk = rcu_dereference_sk_user_data(sk); 501 502 if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) { 503 rc = -EINVAL; 504 goto out; 505 } 506 507 rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info)); 508 if (rc) { 509 rc = -EFAULT; 510 goto out; 511 } 512 513 /* check version */ 514 if (tmp_crypto_info.version != TLS_1_2_VERSION) { 515 rc = -ENOTSUPP; 516 goto out; 517 } 518 519 crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info; 520 521 /* GCM mode of AES supports 128 and 256 bit encryption, so 522 * copy keys from user based on GCM cipher type. 523 */ 524 switch (tmp_crypto_info.cipher_type) { 525 case TLS_CIPHER_AES_GCM_128: { 526 /* Obtain version and type from previous copy */ 527 crypto_info[0] = tmp_crypto_info; 528 /* Now copy the following data */ 529 rc = copy_from_sockptr_offset((char *)crypto_info + 530 sizeof(*crypto_info), 531 optval, sizeof(*crypto_info), 532 sizeof(struct tls12_crypto_info_aes_gcm_128) 533 - sizeof(*crypto_info)); 534 535 if (rc) { 536 rc = -EFAULT; 537 goto out; 538 } 539 540 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; 541 cipher_type = TLS_CIPHER_AES_GCM_128; 542 break; 543 } 544 case TLS_CIPHER_AES_GCM_256: { 545 crypto_info[0] = tmp_crypto_info; 546 rc = copy_from_sockptr_offset((char *)crypto_info + 547 sizeof(*crypto_info), 548 optval, sizeof(*crypto_info), 549 sizeof(struct tls12_crypto_info_aes_gcm_256) 550 - sizeof(*crypto_info)); 551 552 if (rc) { 553 rc = -EFAULT; 554 goto out; 555 } 556 557 keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE; 558 cipher_type = TLS_CIPHER_AES_GCM_256; 559 break; 560 } 561 default: 562 rc = -EINVAL; 563 goto out; 564 } 565 rc = chtls_setkey(csk, keylen, optname, cipher_type); 566 out: 567 return rc; 568 } 569 570 static int chtls_setsockopt(struct sock *sk, int level, int optname, 571 sockptr_t optval, unsigned int optlen) 572 { 573 struct tls_context *ctx = tls_get_ctx(sk); 574 575 if (level != SOL_TLS) 576 return ctx->sk_proto->setsockopt(sk, level, 577 optname, optval, optlen); 578 579 return do_chtls_setsockopt(sk, optname, optval, optlen); 580 } 581 582 static struct cxgb4_uld_info chtls_uld_info = { 583 .name = DRV_NAME, 584 .nrxq = MAX_ULD_QSETS, 585 .ntxq = MAX_ULD_QSETS, 586 .rxq_size = 1024, 587 .add = chtls_uld_add, 588 .state_change = chtls_uld_state_change, 589 .rx_handler = chtls_uld_rx_handler, 590 }; 591 592 void chtls_install_cpl_ops(struct sock *sk) 593 { 594 if (sk->sk_family == AF_INET) 595 sk->sk_prot = &chtls_cpl_prot; 596 else 597 sk->sk_prot = &chtls_cpl_protv6; 598 } 599 600 static void __init chtls_init_ulp_ops(void) 601 { 602 chtls_cpl_prot = tcp_prot; 603 chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops, 604 &tcp_prot, PF_INET); 605 chtls_cpl_prot.close = chtls_close; 606 chtls_cpl_prot.disconnect = chtls_disconnect; 607 chtls_cpl_prot.destroy = chtls_destroy_sock; 608 chtls_cpl_prot.shutdown = chtls_shutdown; 609 chtls_cpl_prot.sendmsg = chtls_sendmsg; 610 chtls_cpl_prot.splice_eof = chtls_splice_eof; 611 chtls_cpl_prot.recvmsg = chtls_recvmsg; 612 chtls_cpl_prot.setsockopt = chtls_setsockopt; 613 chtls_cpl_prot.getsockopt = chtls_getsockopt; 614 #if IS_ENABLED(CONFIG_IPV6) 615 chtls_cpl_protv6 = chtls_cpl_prot; 616 chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6, 617 &tcpv6_prot, PF_INET6); 618 #endif 619 } 620 621 static int __init chtls_register(void) 622 { 623 chtls_init_ulp_ops(); 624 register_listen_notifier(&listen_notifier); 625 cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info); 626 return 0; 627 } 628 629 static void __exit chtls_unregister(void) 630 { 631 unregister_listen_notifier(&listen_notifier); 632 chtls_free_all_uld(); 633 cxgb4_unregister_uld(CXGB4_ULD_TLS); 634 } 635 636 module_init(chtls_register); 637 module_exit(chtls_unregister); 638 639 MODULE_DESCRIPTION("Chelsio TLS Inline driver"); 640 MODULE_LICENSE("GPL"); 641 MODULE_AUTHOR("Chelsio Communications"); 642 MODULE_VERSION(CHTLS_DRV_VERSION); 643