1 /* 2 * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 #include <linux/module.h> 35 36 #include <net/tcp.h> 37 #include <net/inet_common.h> 38 #include <linux/highmem.h> 39 #include <linux/netdevice.h> 40 #include <linux/sched/signal.h> 41 #include <linux/inetdevice.h> 42 #include <linux/inet_diag.h> 43 44 #include <net/snmp.h> 45 #include <net/tls.h> 46 #include "tls.h" 47 48 MODULE_AUTHOR("Mellanox Technologies"); 49 MODULE_DESCRIPTION("Transport Layer Security Support"); 50 MODULE_LICENSE("Dual BSD/GPL"); 51 MODULE_ALIAS_TCP_ULP("tls"); 52 53 enum { 54 TLSV4, 55 TLSV6, 56 TLS_NUM_PROTS, 57 }; 58 59 #define CHECK_CIPHER_DESC(cipher,ci) \ 60 static_assert(cipher ## _IV_SIZE <= TLS_MAX_IV_SIZE); \ 61 static_assert(cipher ## _SALT_SIZE <= TLS_MAX_SALT_SIZE); \ 62 static_assert(cipher ## _REC_SEQ_SIZE <= TLS_MAX_REC_SEQ_SIZE); \ 63 static_assert(cipher ## _TAG_SIZE == TLS_TAG_SIZE); \ 64 static_assert(sizeof_field(struct ci, iv) == cipher ## _IV_SIZE); \ 65 static_assert(sizeof_field(struct ci, key) == cipher ## _KEY_SIZE); \ 66 static_assert(sizeof_field(struct ci, salt) == cipher ## _SALT_SIZE); \ 67 static_assert(sizeof_field(struct ci, rec_seq) == cipher ## _REC_SEQ_SIZE); 68 69 #define __CIPHER_DESC(ci) \ 70 .iv_offset = offsetof(struct ci, iv), \ 71 .key_offset = offsetof(struct ci, key), \ 72 .salt_offset = offsetof(struct ci, salt), \ 73 .rec_seq_offset = offsetof(struct ci, rec_seq), \ 74 .crypto_info = sizeof(struct ci) 75 76 #define CIPHER_DESC(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 77 .nonce = cipher ## _IV_SIZE, \ 78 .iv = cipher ## _IV_SIZE, \ 79 .key = cipher ## _KEY_SIZE, \ 80 .salt = cipher ## _SALT_SIZE, \ 81 .tag = cipher ## _TAG_SIZE, \ 82 .rec_seq = cipher ## _REC_SEQ_SIZE, \ 83 .cipher_name = algname, \ 84 .offloadable = _offloadable, \ 85 __CIPHER_DESC(ci), \ 86 } 87 88 #define CIPHER_DESC_NONCE0(cipher,ci,algname,_offloadable) [cipher - TLS_CIPHER_MIN] = { \ 89 .nonce = 0, \ 90 .iv = cipher ## _IV_SIZE, \ 91 .key = cipher ## _KEY_SIZE, \ 92 .salt = cipher ## _SALT_SIZE, \ 93 .tag = cipher ## _TAG_SIZE, \ 94 .rec_seq = cipher ## _REC_SEQ_SIZE, \ 95 .cipher_name = algname, \ 96 .offloadable = _offloadable, \ 97 __CIPHER_DESC(ci), \ 98 } 99 100 const struct tls_cipher_desc tls_cipher_desc[TLS_CIPHER_MAX + 1 - TLS_CIPHER_MIN] = { 101 CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128, "gcm(aes)", true), 102 CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256, "gcm(aes)", true), 103 CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128, "ccm(aes)", false), 104 CIPHER_DESC_NONCE0(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305, "rfc7539(chacha20,poly1305)", false), 105 CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm, "gcm(sm4)", false), 106 CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm, "ccm(sm4)", false), 107 CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128, "gcm(aria)", false), 108 CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256, "gcm(aria)", false), 109 }; 110 111 CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_128, tls12_crypto_info_aes_gcm_128); 112 CHECK_CIPHER_DESC(TLS_CIPHER_AES_GCM_256, tls12_crypto_info_aes_gcm_256); 113 CHECK_CIPHER_DESC(TLS_CIPHER_AES_CCM_128, tls12_crypto_info_aes_ccm_128); 114 CHECK_CIPHER_DESC(TLS_CIPHER_CHACHA20_POLY1305, tls12_crypto_info_chacha20_poly1305); 115 CHECK_CIPHER_DESC(TLS_CIPHER_SM4_GCM, tls12_crypto_info_sm4_gcm); 116 CHECK_CIPHER_DESC(TLS_CIPHER_SM4_CCM, tls12_crypto_info_sm4_ccm); 117 CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_128, tls12_crypto_info_aria_gcm_128); 118 CHECK_CIPHER_DESC(TLS_CIPHER_ARIA_GCM_256, tls12_crypto_info_aria_gcm_256); 119 120 static const struct proto *saved_tcpv6_prot; 121 static DEFINE_MUTEX(tcpv6_prot_mutex); 122 static const struct proto *saved_tcpv4_prot; 123 static DEFINE_MUTEX(tcpv4_prot_mutex); 124 static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 125 static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG]; 126 static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 127 const struct proto *base); 128 129 void update_sk_prot(struct sock *sk, struct tls_context *ctx) 130 { 131 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 132 133 WRITE_ONCE(sk->sk_prot, 134 &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]); 135 WRITE_ONCE(sk->sk_socket->ops, 136 &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]); 137 } 138 139 int wait_on_pending_writer(struct sock *sk, long *timeo) 140 { 141 DEFINE_WAIT_FUNC(wait, woken_wake_function); 142 int ret, rc = 0; 143 144 add_wait_queue(sk_sleep(sk), &wait); 145 while (1) { 146 if (!*timeo) { 147 rc = -EAGAIN; 148 break; 149 } 150 151 if (signal_pending(current)) { 152 rc = sock_intr_errno(*timeo); 153 break; 154 } 155 156 ret = sk_wait_event(sk, timeo, 157 !READ_ONCE(sk->sk_write_pending), &wait); 158 if (ret) { 159 if (ret < 0) 160 rc = ret; 161 break; 162 } 163 } 164 remove_wait_queue(sk_sleep(sk), &wait); 165 return rc; 166 } 167 168 int tls_push_sg(struct sock *sk, 169 struct tls_context *ctx, 170 struct scatterlist *sg, 171 u16 first_offset, 172 int flags) 173 { 174 struct bio_vec bvec; 175 struct msghdr msg = { 176 .msg_flags = MSG_SPLICE_PAGES | flags, 177 }; 178 int ret = 0; 179 struct page *p; 180 size_t size; 181 int offset = first_offset; 182 183 size = sg->length - offset; 184 offset += sg->offset; 185 186 ctx->splicing_pages = true; 187 while (1) { 188 p = sg_page(sg); 189 retry: 190 bvec_set_page(&bvec, p, size, offset); 191 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, size); 192 193 ret = tcp_sendmsg_locked(sk, &msg, size); 194 195 if (ret != size) { 196 if (ret > 0) { 197 offset += ret; 198 size -= ret; 199 goto retry; 200 } 201 202 offset -= sg->offset; 203 ctx->partially_sent_offset = offset; 204 ctx->partially_sent_record = (void *)sg; 205 ctx->splicing_pages = false; 206 return ret; 207 } 208 209 put_page(p); 210 sk_mem_uncharge(sk, sg->length); 211 sg = sg_next(sg); 212 if (!sg) 213 break; 214 215 offset = sg->offset; 216 size = sg->length; 217 } 218 219 ctx->splicing_pages = false; 220 221 return 0; 222 } 223 224 static int tls_handle_open_record(struct sock *sk, int flags) 225 { 226 struct tls_context *ctx = tls_get_ctx(sk); 227 228 if (tls_is_pending_open_record(ctx)) 229 return ctx->push_pending_record(sk, flags); 230 231 return 0; 232 } 233 234 int tls_process_cmsg(struct sock *sk, struct msghdr *msg, 235 unsigned char *record_type) 236 { 237 struct cmsghdr *cmsg; 238 int rc = -EINVAL; 239 240 for_each_cmsghdr(cmsg, msg) { 241 if (!CMSG_OK(msg, cmsg)) 242 return -EINVAL; 243 if (cmsg->cmsg_level != SOL_TLS) 244 continue; 245 246 switch (cmsg->cmsg_type) { 247 case TLS_SET_RECORD_TYPE: 248 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type))) 249 return -EINVAL; 250 251 if (msg->msg_flags & MSG_MORE) 252 return -EINVAL; 253 254 *record_type = *(unsigned char *)CMSG_DATA(cmsg); 255 256 rc = tls_handle_open_record(sk, msg->msg_flags); 257 break; 258 default: 259 return -EINVAL; 260 } 261 } 262 263 return rc; 264 } 265 266 int tls_push_partial_record(struct sock *sk, struct tls_context *ctx, 267 int flags) 268 { 269 struct scatterlist *sg; 270 u16 offset; 271 272 sg = ctx->partially_sent_record; 273 offset = ctx->partially_sent_offset; 274 275 ctx->partially_sent_record = NULL; 276 return tls_push_sg(sk, ctx, sg, offset, flags); 277 } 278 279 void tls_free_partial_record(struct sock *sk, struct tls_context *ctx) 280 { 281 struct scatterlist *sg; 282 283 for (sg = ctx->partially_sent_record; sg; sg = sg_next(sg)) { 284 put_page(sg_page(sg)); 285 sk_mem_uncharge(sk, sg->length); 286 } 287 ctx->partially_sent_record = NULL; 288 } 289 290 static void tls_write_space(struct sock *sk) 291 { 292 struct tls_context *ctx = tls_get_ctx(sk); 293 294 /* If splicing_pages call lower protocol write space handler 295 * to ensure we wake up any waiting operations there. For example 296 * if splicing pages where to call sk_wait_event. 297 */ 298 if (ctx->splicing_pages) { 299 ctx->sk_write_space(sk); 300 return; 301 } 302 303 #ifdef CONFIG_TLS_DEVICE 304 if (ctx->tx_conf == TLS_HW) 305 tls_device_write_space(sk, ctx); 306 else 307 #endif 308 tls_sw_write_space(sk, ctx); 309 310 ctx->sk_write_space(sk); 311 } 312 313 /** 314 * tls_ctx_free() - free TLS ULP context 315 * @sk: socket to with @ctx is attached 316 * @ctx: TLS context structure 317 * 318 * Free TLS context. If @sk is %NULL caller guarantees that the socket 319 * to which @ctx was attached has no outstanding references. 320 */ 321 void tls_ctx_free(struct sock *sk, struct tls_context *ctx) 322 { 323 if (!ctx) 324 return; 325 326 memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send)); 327 memzero_explicit(&ctx->crypto_recv, sizeof(ctx->crypto_recv)); 328 mutex_destroy(&ctx->tx_lock); 329 330 if (sk) 331 kfree_rcu(ctx, rcu); 332 else 333 kfree(ctx); 334 } 335 336 static void tls_sk_proto_cleanup(struct sock *sk, 337 struct tls_context *ctx, long timeo) 338 { 339 if (unlikely(sk->sk_write_pending) && 340 !wait_on_pending_writer(sk, &timeo)) 341 tls_handle_open_record(sk, 0); 342 343 /* We need these for tls_sw_fallback handling of other packets */ 344 if (ctx->tx_conf == TLS_SW) { 345 tls_sw_release_resources_tx(sk); 346 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 347 } else if (ctx->tx_conf == TLS_HW) { 348 tls_device_free_resources_tx(sk); 349 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 350 } 351 352 if (ctx->rx_conf == TLS_SW) { 353 tls_sw_release_resources_rx(sk); 354 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 355 } else if (ctx->rx_conf == TLS_HW) { 356 tls_device_offload_cleanup_rx(sk); 357 TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 358 } 359 } 360 361 static void tls_sk_proto_close(struct sock *sk, long timeout) 362 { 363 struct inet_connection_sock *icsk = inet_csk(sk); 364 struct tls_context *ctx = tls_get_ctx(sk); 365 long timeo = sock_sndtimeo(sk, 0); 366 bool free_ctx; 367 368 if (ctx->tx_conf == TLS_SW) 369 tls_sw_cancel_work_tx(ctx); 370 371 lock_sock(sk); 372 free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW; 373 374 if (ctx->tx_conf != TLS_BASE || ctx->rx_conf != TLS_BASE) 375 tls_sk_proto_cleanup(sk, ctx, timeo); 376 377 write_lock_bh(&sk->sk_callback_lock); 378 if (free_ctx) 379 rcu_assign_pointer(icsk->icsk_ulp_data, NULL); 380 WRITE_ONCE(sk->sk_prot, ctx->sk_proto); 381 if (sk->sk_write_space == tls_write_space) 382 sk->sk_write_space = ctx->sk_write_space; 383 write_unlock_bh(&sk->sk_callback_lock); 384 release_sock(sk); 385 if (ctx->tx_conf == TLS_SW) 386 tls_sw_free_ctx_tx(ctx); 387 if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 388 tls_sw_strparser_done(ctx); 389 if (ctx->rx_conf == TLS_SW) 390 tls_sw_free_ctx_rx(ctx); 391 ctx->sk_proto->close(sk, timeout); 392 393 if (free_ctx) 394 tls_ctx_free(sk, ctx); 395 } 396 397 static __poll_t tls_sk_poll(struct file *file, struct socket *sock, 398 struct poll_table_struct *wait) 399 { 400 struct tls_sw_context_rx *ctx; 401 struct tls_context *tls_ctx; 402 struct sock *sk = sock->sk; 403 __poll_t mask = 0; 404 u8 shutdown; 405 int state; 406 407 mask = tcp_poll(file, sock, wait); 408 409 state = inet_sk_state_load(sk); 410 shutdown = READ_ONCE(sk->sk_shutdown); 411 if (unlikely(state != TCP_ESTABLISHED || shutdown & RCV_SHUTDOWN)) 412 return mask; 413 414 tls_ctx = tls_get_ctx(sk); 415 ctx = tls_sw_ctx_rx(tls_ctx); 416 417 if ((skb_queue_empty_lockless(&ctx->rx_list) && 418 !tls_strp_msg_ready(ctx)) || 419 READ_ONCE(ctx->key_update_pending)) 420 mask &= ~(EPOLLIN | EPOLLRDNORM); 421 422 return mask; 423 } 424 425 static int do_tls_getsockopt_conf(struct sock *sk, sockopt_t *opt, int tx) 426 { 427 int rc = 0; 428 const struct tls_cipher_desc *cipher_desc; 429 struct tls_context *ctx = tls_get_ctx(sk); 430 struct tls_crypto_info *crypto_info; 431 struct cipher_context *cctx; 432 int len = opt->optlen; 433 434 if (!opt->iter_out.ubuf || len < sizeof(*crypto_info)) { 435 rc = -EINVAL; 436 goto out; 437 } 438 439 if (!ctx) { 440 rc = -EBUSY; 441 goto out; 442 } 443 444 /* get user crypto info */ 445 if (tx) { 446 crypto_info = &ctx->crypto_send.info; 447 cctx = &ctx->tx; 448 } else { 449 crypto_info = &ctx->crypto_recv.info; 450 cctx = &ctx->rx; 451 } 452 453 if (!TLS_CRYPTO_INFO_READY(crypto_info)) { 454 rc = -EBUSY; 455 goto out; 456 } 457 458 if (len == sizeof(*crypto_info)) { 459 if (copy_to_iter(crypto_info, sizeof(*crypto_info), 460 &opt->iter_out) != sizeof(*crypto_info)) 461 rc = -EFAULT; 462 goto out; 463 } 464 465 cipher_desc = get_cipher_desc(crypto_info->cipher_type); 466 if (!cipher_desc || len != cipher_desc->crypto_info) { 467 rc = -EINVAL; 468 goto out; 469 } 470 471 memcpy(crypto_info_iv(crypto_info, cipher_desc), 472 cctx->iv + cipher_desc->salt, cipher_desc->iv); 473 memcpy(crypto_info_rec_seq(crypto_info, cipher_desc), 474 cctx->rec_seq, cipher_desc->rec_seq); 475 476 if (copy_to_iter(crypto_info, cipher_desc->crypto_info, 477 &opt->iter_out) != cipher_desc->crypto_info) 478 rc = -EFAULT; 479 480 out: 481 return rc; 482 } 483 484 static int do_tls_getsockopt_tx_zc(struct sock *sk, sockopt_t *opt) 485 { 486 struct tls_context *ctx = tls_get_ctx(sk); 487 unsigned int value; 488 int len = opt->optlen; 489 490 if (len != sizeof(value)) 491 return -EINVAL; 492 493 value = ctx->zerocopy_sendfile; 494 if (copy_to_iter(&value, sizeof(value), &opt->iter_out) != sizeof(value)) 495 return -EFAULT; 496 497 return 0; 498 } 499 500 static int do_tls_getsockopt_no_pad(struct sock *sk, sockopt_t *opt) 501 { 502 struct tls_context *ctx = tls_get_ctx(sk); 503 int value, len = opt->optlen; 504 505 if (ctx->prot_info.version != TLS_1_3_VERSION) 506 return -EINVAL; 507 508 if (len < sizeof(value)) 509 return -EINVAL; 510 511 value = -EINVAL; 512 if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) 513 value = ctx->rx_no_pad; 514 if (value < 0) 515 return value; 516 517 opt->optlen = sizeof(value); 518 if (copy_to_iter(&value, sizeof(value), &opt->iter_out) != sizeof(value)) 519 return -EFAULT; 520 521 return 0; 522 } 523 524 static int do_tls_getsockopt_tx_payload_len(struct sock *sk, sockopt_t *opt) 525 { 526 struct tls_context *ctx = tls_get_ctx(sk); 527 u16 payload_len = ctx->tx_max_payload_len; 528 int len = opt->optlen; 529 530 if (len < sizeof(payload_len)) 531 return -EINVAL; 532 533 opt->optlen = sizeof(payload_len); 534 if (copy_to_iter(&payload_len, sizeof(payload_len), 535 &opt->iter_out) != sizeof(payload_len)) 536 return -EFAULT; 537 538 return 0; 539 } 540 541 static int do_tls_getsockopt(struct sock *sk, int optname, sockopt_t *opt) 542 { 543 int rc = 0; 544 545 lock_sock(sk); 546 547 switch (optname) { 548 case TLS_TX: 549 case TLS_RX: 550 rc = do_tls_getsockopt_conf(sk, opt, optname == TLS_TX); 551 break; 552 case TLS_TX_ZEROCOPY_RO: 553 rc = do_tls_getsockopt_tx_zc(sk, opt); 554 break; 555 case TLS_RX_EXPECT_NO_PAD: 556 rc = do_tls_getsockopt_no_pad(sk, opt); 557 break; 558 case TLS_TX_MAX_PAYLOAD_LEN: 559 rc = do_tls_getsockopt_tx_payload_len(sk, opt); 560 break; 561 default: 562 rc = -ENOPROTOOPT; 563 break; 564 } 565 566 release_sock(sk); 567 568 return rc; 569 } 570 571 static int tls_getsockopt(struct sock *sk, int level, int optname, 572 char __user *optval, int __user *optlen) 573 { 574 struct tls_context *ctx = tls_get_ctx(sk); 575 sockopt_t opt; 576 int err; 577 578 if (level != SOL_TLS) 579 return ctx->sk_proto->getsockopt(sk, level, 580 optname, optval, optlen); 581 582 err = sockopt_init_user(&opt, optval, optlen); 583 if (err) 584 return err; 585 586 err = do_tls_getsockopt(sk, optname, &opt); 587 if (err) 588 return err; 589 590 if (put_user(opt.optlen, optlen)) 591 return -EFAULT; 592 593 return 0; 594 } 595 596 static int validate_crypto_info(const struct tls_crypto_info *crypto_info, 597 const struct tls_crypto_info *alt_crypto_info) 598 { 599 if (crypto_info->version != TLS_1_2_VERSION && 600 crypto_info->version != TLS_1_3_VERSION) 601 return -EINVAL; 602 603 switch (crypto_info->cipher_type) { 604 case TLS_CIPHER_ARIA_GCM_128: 605 case TLS_CIPHER_ARIA_GCM_256: 606 if (crypto_info->version != TLS_1_2_VERSION) 607 return -EINVAL; 608 break; 609 } 610 611 /* Ensure that TLS version and ciphers are same in both directions */ 612 if (TLS_CRYPTO_INFO_READY(alt_crypto_info)) { 613 if (alt_crypto_info->version != crypto_info->version || 614 alt_crypto_info->cipher_type != crypto_info->cipher_type) 615 return -EINVAL; 616 } 617 618 return 0; 619 } 620 621 static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval, 622 unsigned int optlen, int tx) 623 { 624 struct tls_crypto_info *crypto_info, *alt_crypto_info; 625 struct tls_crypto_info *old_crypto_info = NULL; 626 struct tls_context *ctx = tls_get_ctx(sk); 627 const struct tls_cipher_desc *cipher_desc; 628 union tls_crypto_context *crypto_ctx; 629 union tls_crypto_context tmp = {}; 630 bool update = false; 631 int rc = 0; 632 int conf; 633 634 /* TLS and sockmap are mutually exclusive. A socket already in a 635 * sockmap (i.e. with a psock attached) cannot be upgraded to TLS. 636 * sockmap rejects TLS sockets already (see sk_psock_init()). 637 */ 638 rcu_read_lock(); 639 if (sk_psock(sk)) { 640 rcu_read_unlock(); 641 return -EINVAL; 642 } 643 rcu_read_unlock(); 644 645 if (sockptr_is_null(optval) || (optlen < sizeof(*crypto_info))) 646 return -EINVAL; 647 648 if (tx) { 649 crypto_ctx = &ctx->crypto_send; 650 alt_crypto_info = &ctx->crypto_recv.info; 651 } else { 652 crypto_ctx = &ctx->crypto_recv; 653 alt_crypto_info = &ctx->crypto_send.info; 654 } 655 656 crypto_info = &crypto_ctx->info; 657 658 if (TLS_CRYPTO_INFO_READY(crypto_info)) { 659 /* Currently we only support setting crypto info more 660 * than one time for TLS 1.3 661 */ 662 if (crypto_info->version != TLS_1_3_VERSION) { 663 TLS_INC_STATS(sock_net(sk), tx ? LINUX_MIB_TLSTXREKEYERROR 664 : LINUX_MIB_TLSRXREKEYERROR); 665 return -EBUSY; 666 } 667 668 update = true; 669 old_crypto_info = crypto_info; 670 crypto_info = &tmp.info; 671 crypto_ctx = &tmp; 672 } 673 674 rc = copy_from_sockptr(crypto_info, optval, sizeof(*crypto_info)); 675 if (rc) { 676 rc = -EFAULT; 677 goto err_crypto_info; 678 } 679 680 if (update) { 681 /* Ensure that TLS version and ciphers are not modified */ 682 if (crypto_info->version != old_crypto_info->version || 683 crypto_info->cipher_type != old_crypto_info->cipher_type) 684 rc = -EINVAL; 685 } else { 686 rc = validate_crypto_info(crypto_info, alt_crypto_info); 687 } 688 if (rc) 689 goto err_crypto_info; 690 691 cipher_desc = get_cipher_desc(crypto_info->cipher_type); 692 if (!cipher_desc) { 693 rc = -EINVAL; 694 goto err_crypto_info; 695 } 696 697 if (optlen != cipher_desc->crypto_info) { 698 rc = -EINVAL; 699 goto err_crypto_info; 700 } 701 702 rc = copy_from_sockptr_offset((u8 *)crypto_ctx + sizeof(*crypto_info), 703 optval, sizeof(*crypto_info), 704 optlen - sizeof(*crypto_info)); 705 if (rc) { 706 rc = -EFAULT; 707 goto err_crypto_info; 708 } 709 710 if (tx) { 711 rc = tls_set_device_offload(sk); 712 conf = TLS_HW; 713 if (!rc) { 714 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE); 715 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE); 716 } else { 717 rc = tls_set_sw_offload(sk, 1, 718 update ? crypto_info : NULL); 719 if (rc) 720 goto err_crypto_info; 721 722 if (update) { 723 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXREKEYOK); 724 } else { 725 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW); 726 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW); 727 } 728 conf = TLS_SW; 729 } 730 } else { 731 rc = tls_set_device_offload_rx(sk, ctx); 732 conf = TLS_HW; 733 if (!rc) { 734 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE); 735 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE); 736 } else { 737 rc = tls_set_sw_offload(sk, 0, 738 update ? crypto_info : NULL); 739 if (rc) 740 goto err_crypto_info; 741 742 if (update) { 743 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXREKEYOK); 744 } else { 745 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW); 746 TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW); 747 } 748 conf = TLS_SW; 749 } 750 if (!update) 751 tls_sw_strparser_arm(sk, ctx); 752 } 753 754 if (tx) 755 ctx->tx_conf = conf; 756 else 757 ctx->rx_conf = conf; 758 update_sk_prot(sk, ctx); 759 760 if (update) 761 return 0; 762 763 if (tx) { 764 ctx->sk_write_space = sk->sk_write_space; 765 sk->sk_write_space = tls_write_space; 766 } else { 767 struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx); 768 769 tls_strp_check_rcv(&rx_ctx->strp, true); 770 } 771 return 0; 772 773 err_crypto_info: 774 if (update) { 775 TLS_INC_STATS(sock_net(sk), tx ? LINUX_MIB_TLSTXREKEYERROR 776 : LINUX_MIB_TLSRXREKEYERROR); 777 } 778 memzero_explicit(crypto_ctx, sizeof(*crypto_ctx)); 779 return rc; 780 } 781 782 static int do_tls_setsockopt_tx_zc(struct sock *sk, sockptr_t optval, 783 unsigned int optlen) 784 { 785 struct tls_context *ctx = tls_get_ctx(sk); 786 unsigned int value; 787 788 if (sockptr_is_null(optval) || optlen != sizeof(value)) 789 return -EINVAL; 790 791 if (copy_from_sockptr(&value, optval, sizeof(value))) 792 return -EFAULT; 793 794 if (value > 1) 795 return -EINVAL; 796 797 ctx->zerocopy_sendfile = value; 798 799 return 0; 800 } 801 802 static int do_tls_setsockopt_no_pad(struct sock *sk, sockptr_t optval, 803 unsigned int optlen) 804 { 805 struct tls_context *ctx = tls_get_ctx(sk); 806 u32 val; 807 int rc; 808 809 if (ctx->prot_info.version != TLS_1_3_VERSION || 810 sockptr_is_null(optval) || optlen < sizeof(val)) 811 return -EINVAL; 812 813 rc = copy_from_sockptr(&val, optval, sizeof(val)); 814 if (rc) 815 return -EFAULT; 816 if (val > 1) 817 return -EINVAL; 818 rc = check_zeroed_sockptr(optval, sizeof(val), optlen - sizeof(val)); 819 if (rc < 1) 820 return rc == 0 ? -EINVAL : rc; 821 822 lock_sock(sk); 823 rc = -EINVAL; 824 if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) { 825 ctx->rx_no_pad = val; 826 tls_update_rx_zc_capable(ctx); 827 rc = 0; 828 } 829 release_sock(sk); 830 831 return rc; 832 } 833 834 static bool tls_tx_record_is_open(struct tls_context *ctx) 835 { 836 switch (ctx->tx_conf) { 837 case TLS_SW: 838 return !!tls_sw_ctx_tx(ctx)->open_rec; 839 case TLS_HW: 840 return !!tls_offload_ctx_tx(ctx)->open_record; 841 default: 842 return false; 843 } 844 } 845 846 static int do_tls_setsockopt_tx_payload_len(struct sock *sk, sockptr_t optval, 847 unsigned int optlen) 848 { 849 struct tls_context *ctx = tls_get_ctx(sk); 850 u16 value; 851 bool tls_13 = ctx->prot_info.version == TLS_1_3_VERSION; 852 853 if (tls_tx_record_is_open(ctx)) 854 return -EBUSY; 855 856 if (sockptr_is_null(optval) || optlen != sizeof(value)) 857 return -EINVAL; 858 859 if (copy_from_sockptr(&value, optval, sizeof(value))) 860 return -EFAULT; 861 862 if (value < TLS_MIN_RECORD_SIZE_LIM - (tls_13 ? 1 : 0) || 863 value > TLS_MAX_PAYLOAD_SIZE) 864 return -EINVAL; 865 866 ctx->tx_max_payload_len = value; 867 868 return 0; 869 } 870 871 static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval, 872 unsigned int optlen) 873 { 874 struct tls_context *ctx; 875 int rc = 0; 876 877 switch (optname) { 878 case TLS_TX: 879 case TLS_RX: 880 lock_sock(sk); 881 rc = do_tls_setsockopt_conf(sk, optval, optlen, 882 optname == TLS_TX); 883 release_sock(sk); 884 break; 885 case TLS_TX_ZEROCOPY_RO: 886 lock_sock(sk); 887 rc = do_tls_setsockopt_tx_zc(sk, optval, optlen); 888 release_sock(sk); 889 break; 890 case TLS_RX_EXPECT_NO_PAD: 891 rc = do_tls_setsockopt_no_pad(sk, optval, optlen); 892 break; 893 case TLS_TX_MAX_PAYLOAD_LEN: 894 ctx = tls_get_ctx(sk); 895 /* Senders sleep with tx_lock held, so take it interruptible */ 896 rc = mutex_lock_interruptible(&ctx->tx_lock); 897 if (rc) 898 break; 899 lock_sock(sk); 900 rc = do_tls_setsockopt_tx_payload_len(sk, optval, optlen); 901 release_sock(sk); 902 mutex_unlock(&ctx->tx_lock); 903 break; 904 default: 905 rc = -ENOPROTOOPT; 906 break; 907 } 908 return rc; 909 } 910 911 static int tls_setsockopt(struct sock *sk, int level, int optname, 912 sockptr_t optval, unsigned int optlen) 913 { 914 struct tls_context *ctx = tls_get_ctx(sk); 915 916 if (level != SOL_TLS) 917 return ctx->sk_proto->setsockopt(sk, level, optname, optval, 918 optlen); 919 920 return do_tls_setsockopt(sk, optname, optval, optlen); 921 } 922 923 static int tls_disconnect(struct sock *sk, int flags) 924 { 925 return -EOPNOTSUPP; 926 } 927 928 struct tls_context *tls_ctx_create(struct sock *sk) 929 { 930 struct inet_connection_sock *icsk = inet_csk(sk); 931 struct tls_context *ctx; 932 933 ctx = kzalloc_obj(*ctx, GFP_ATOMIC); 934 if (!ctx) 935 return NULL; 936 937 mutex_init(&ctx->tx_lock); 938 ctx->sk_proto = READ_ONCE(sk->sk_prot); 939 ctx->sk = sk; 940 /* Release semantic of rcu_assign_pointer() ensures that 941 * ctx->sk_proto is visible before changing sk->sk_prot in 942 * update_sk_prot(), and prevents reading uninitialized value in 943 * tls_{getsockopt, setsockopt}. Note that we do not need a 944 * read barrier in tls_{getsockopt,setsockopt} as there is an 945 * address dependency between sk->sk_proto->{getsockopt,setsockopt} 946 * and ctx->sk_proto. 947 */ 948 rcu_assign_pointer(icsk->icsk_ulp_data, ctx); 949 return ctx; 950 } 951 952 static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 953 const struct proto_ops *base) 954 { 955 ops[TLS_BASE][TLS_BASE] = *base; 956 957 ops[TLS_SW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 958 ops[TLS_SW ][TLS_BASE].splice_eof = tls_sw_splice_eof; 959 960 ops[TLS_BASE][TLS_SW ] = ops[TLS_BASE][TLS_BASE]; 961 ops[TLS_BASE][TLS_SW ].splice_read = tls_sw_splice_read; 962 ops[TLS_BASE][TLS_SW ].poll = tls_sk_poll; 963 ops[TLS_BASE][TLS_SW ].read_sock = tls_sw_read_sock; 964 965 ops[TLS_SW ][TLS_SW ] = ops[TLS_SW ][TLS_BASE]; 966 ops[TLS_SW ][TLS_SW ].splice_read = tls_sw_splice_read; 967 ops[TLS_SW ][TLS_SW ].poll = tls_sk_poll; 968 ops[TLS_SW ][TLS_SW ].read_sock = tls_sw_read_sock; 969 970 #ifdef CONFIG_TLS_DEVICE 971 ops[TLS_HW ][TLS_BASE] = ops[TLS_BASE][TLS_BASE]; 972 973 ops[TLS_HW ][TLS_SW ] = ops[TLS_BASE][TLS_SW ]; 974 975 ops[TLS_BASE][TLS_HW ] = ops[TLS_BASE][TLS_SW ]; 976 977 ops[TLS_SW ][TLS_HW ] = ops[TLS_SW ][TLS_SW ]; 978 979 ops[TLS_HW ][TLS_HW ] = ops[TLS_HW ][TLS_SW ]; 980 #endif 981 } 982 983 static void tls_build_proto(struct sock *sk) 984 { 985 int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4; 986 struct proto *prot = READ_ONCE(sk->sk_prot); 987 988 /* Build IPv6 TLS whenever the address of tcpv6 _prot changes */ 989 if (ip_ver == TLSV6 && 990 unlikely(prot != smp_load_acquire(&saved_tcpv6_prot))) { 991 mutex_lock(&tcpv6_prot_mutex); 992 if (likely(prot != saved_tcpv6_prot)) { 993 build_protos(tls_prots[TLSV6], prot); 994 build_proto_ops(tls_proto_ops[TLSV6], 995 sk->sk_socket->ops); 996 smp_store_release(&saved_tcpv6_prot, prot); 997 } 998 mutex_unlock(&tcpv6_prot_mutex); 999 } 1000 1001 if (ip_ver == TLSV4 && 1002 unlikely(prot != smp_load_acquire(&saved_tcpv4_prot))) { 1003 mutex_lock(&tcpv4_prot_mutex); 1004 if (likely(prot != saved_tcpv4_prot)) { 1005 build_protos(tls_prots[TLSV4], prot); 1006 build_proto_ops(tls_proto_ops[TLSV4], 1007 sk->sk_socket->ops); 1008 smp_store_release(&saved_tcpv4_prot, prot); 1009 } 1010 mutex_unlock(&tcpv4_prot_mutex); 1011 } 1012 } 1013 1014 static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG], 1015 const struct proto *base) 1016 { 1017 prot[TLS_BASE][TLS_BASE] = *base; 1018 prot[TLS_BASE][TLS_BASE].setsockopt = tls_setsockopt; 1019 prot[TLS_BASE][TLS_BASE].getsockopt = tls_getsockopt; 1020 prot[TLS_BASE][TLS_BASE].disconnect = tls_disconnect; 1021 prot[TLS_BASE][TLS_BASE].close = tls_sk_proto_close; 1022 1023 prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 1024 prot[TLS_SW][TLS_BASE].sendmsg = tls_sw_sendmsg; 1025 prot[TLS_SW][TLS_BASE].splice_eof = tls_sw_splice_eof; 1026 1027 prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE]; 1028 prot[TLS_BASE][TLS_SW].recvmsg = tls_sw_recvmsg; 1029 prot[TLS_BASE][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 1030 prot[TLS_BASE][TLS_SW].close = tls_sk_proto_close; 1031 1032 prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE]; 1033 prot[TLS_SW][TLS_SW].recvmsg = tls_sw_recvmsg; 1034 prot[TLS_SW][TLS_SW].sock_is_readable = tls_sw_sock_is_readable; 1035 prot[TLS_SW][TLS_SW].close = tls_sk_proto_close; 1036 1037 #ifdef CONFIG_TLS_DEVICE 1038 prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE]; 1039 prot[TLS_HW][TLS_BASE].sendmsg = tls_device_sendmsg; 1040 prot[TLS_HW][TLS_BASE].splice_eof = tls_device_splice_eof; 1041 1042 prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW]; 1043 prot[TLS_HW][TLS_SW].sendmsg = tls_device_sendmsg; 1044 prot[TLS_HW][TLS_SW].splice_eof = tls_device_splice_eof; 1045 1046 prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW]; 1047 1048 prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW]; 1049 1050 prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW]; 1051 #endif 1052 } 1053 1054 static int tls_init(struct sock *sk) 1055 { 1056 struct tls_context *ctx; 1057 int rc = 0; 1058 1059 tls_build_proto(sk); 1060 1061 /* The TLS ulp is currently supported only for TCP sockets 1062 * in ESTABLISHED state. 1063 * Supporting sockets in LISTEN state will require us 1064 * to modify the accept implementation to clone rather then 1065 * share the ulp context. 1066 */ 1067 if (sk->sk_state != TCP_ESTABLISHED) 1068 return -ENOTCONN; 1069 1070 /* allocate tls context */ 1071 write_lock_bh(&sk->sk_callback_lock); 1072 ctx = tls_ctx_create(sk); 1073 if (!ctx) { 1074 rc = -ENOMEM; 1075 goto out; 1076 } 1077 1078 ctx->tx_conf = TLS_BASE; 1079 ctx->rx_conf = TLS_BASE; 1080 ctx->tx_max_payload_len = TLS_MAX_PAYLOAD_SIZE; 1081 update_sk_prot(sk, ctx); 1082 out: 1083 write_unlock_bh(&sk->sk_callback_lock); 1084 return rc; 1085 } 1086 1087 static void tls_update(struct sock *sk, struct proto *p, 1088 void (*write_space)(struct sock *sk)) 1089 { 1090 struct tls_context *ctx; 1091 1092 WARN_ON_ONCE(sk->sk_prot == p); 1093 1094 ctx = tls_get_ctx(sk); 1095 if (likely(ctx)) { 1096 ctx->sk_write_space = write_space; 1097 ctx->sk_proto = p; 1098 } else { 1099 /* Pairs with lockless read in sk_clone_lock(). */ 1100 WRITE_ONCE(sk->sk_prot, p); 1101 sk->sk_write_space = write_space; 1102 } 1103 } 1104 1105 static u16 tls_user_config(struct tls_context *ctx, bool tx) 1106 { 1107 u16 config = tx ? ctx->tx_conf : ctx->rx_conf; 1108 1109 switch (config) { 1110 case TLS_BASE: 1111 return TLS_CONF_BASE; 1112 case TLS_SW: 1113 return TLS_CONF_SW; 1114 case TLS_HW: 1115 return TLS_CONF_HW; 1116 } 1117 return 0; 1118 } 1119 1120 static int tls_get_info(struct sock *sk, struct sk_buff *skb, bool net_admin) 1121 { 1122 u16 version, cipher_type; 1123 struct tls_context *ctx; 1124 struct nlattr *start; 1125 int err; 1126 1127 start = nla_nest_start_noflag(skb, INET_ULP_INFO_TLS); 1128 if (!start) 1129 return -EMSGSIZE; 1130 1131 rcu_read_lock(); 1132 ctx = rcu_dereference(inet_csk(sk)->icsk_ulp_data); 1133 if (!ctx) { 1134 err = 0; 1135 goto nla_failure; 1136 } 1137 version = ctx->prot_info.version; 1138 if (version) { 1139 err = nla_put_u16(skb, TLS_INFO_VERSION, version); 1140 if (err) 1141 goto nla_failure; 1142 } 1143 cipher_type = ctx->prot_info.cipher_type; 1144 if (cipher_type) { 1145 err = nla_put_u16(skb, TLS_INFO_CIPHER, cipher_type); 1146 if (err) 1147 goto nla_failure; 1148 } 1149 err = nla_put_u16(skb, TLS_INFO_TXCONF, tls_user_config(ctx, true)); 1150 if (err) 1151 goto nla_failure; 1152 1153 err = nla_put_u16(skb, TLS_INFO_RXCONF, tls_user_config(ctx, false)); 1154 if (err) 1155 goto nla_failure; 1156 1157 if (ctx->tx_conf == TLS_HW && ctx->zerocopy_sendfile) { 1158 err = nla_put_flag(skb, TLS_INFO_ZC_RO_TX); 1159 if (err) 1160 goto nla_failure; 1161 } 1162 if (ctx->rx_no_pad) { 1163 err = nla_put_flag(skb, TLS_INFO_RX_NO_PAD); 1164 if (err) 1165 goto nla_failure; 1166 } 1167 1168 err = nla_put_u16(skb, TLS_INFO_TX_MAX_PAYLOAD_LEN, 1169 ctx->tx_max_payload_len); 1170 1171 if (err) 1172 goto nla_failure; 1173 1174 rcu_read_unlock(); 1175 nla_nest_end(skb, start); 1176 return 0; 1177 1178 nla_failure: 1179 rcu_read_unlock(); 1180 nla_nest_cancel(skb, start); 1181 return err; 1182 } 1183 1184 static size_t tls_get_info_size(const struct sock *sk, bool net_admin) 1185 { 1186 size_t size = 0; 1187 1188 size += nla_total_size(0) + /* INET_ULP_INFO_TLS */ 1189 nla_total_size(sizeof(u16)) + /* TLS_INFO_VERSION */ 1190 nla_total_size(sizeof(u16)) + /* TLS_INFO_CIPHER */ 1191 nla_total_size(sizeof(u16)) + /* TLS_INFO_RXCONF */ 1192 nla_total_size(sizeof(u16)) + /* TLS_INFO_TXCONF */ 1193 nla_total_size(0) + /* TLS_INFO_ZC_RO_TX */ 1194 nla_total_size(0) + /* TLS_INFO_RX_NO_PAD */ 1195 nla_total_size(sizeof(u16)) + /* TLS_INFO_TX_MAX_PAYLOAD_LEN */ 1196 0; 1197 1198 return size; 1199 } 1200 1201 static int __net_init tls_init_net(struct net *net) 1202 { 1203 int err; 1204 1205 net->mib.tls_statistics = alloc_percpu(struct linux_tls_mib); 1206 if (!net->mib.tls_statistics) 1207 return -ENOMEM; 1208 1209 err = tls_proc_init(net); 1210 if (err) 1211 goto err_free_stats; 1212 1213 return 0; 1214 err_free_stats: 1215 free_percpu(net->mib.tls_statistics); 1216 return err; 1217 } 1218 1219 static void __net_exit tls_exit_net(struct net *net) 1220 { 1221 tls_proc_fini(net); 1222 free_percpu(net->mib.tls_statistics); 1223 } 1224 1225 static struct pernet_operations tls_proc_ops = { 1226 .init = tls_init_net, 1227 .exit = tls_exit_net, 1228 }; 1229 1230 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = { 1231 .name = "tls", 1232 .owner = THIS_MODULE, 1233 .init = tls_init, 1234 .update = tls_update, 1235 .get_info = tls_get_info, 1236 .get_info_size = tls_get_info_size, 1237 }; 1238 1239 static int __init tls_register(void) 1240 { 1241 int err; 1242 1243 err = register_pernet_subsys(&tls_proc_ops); 1244 if (err) 1245 return err; 1246 1247 err = tls_strp_dev_init(); 1248 if (err) 1249 goto err_pernet; 1250 1251 err = tls_device_init(); 1252 if (err) 1253 goto err_strp; 1254 1255 tcp_register_ulp(&tcp_tls_ulp_ops); 1256 1257 return 0; 1258 err_strp: 1259 tls_strp_dev_exit(); 1260 err_pernet: 1261 unregister_pernet_subsys(&tls_proc_ops); 1262 return err; 1263 } 1264 1265 static void __exit tls_unregister(void) 1266 { 1267 tcp_unregister_ulp(&tcp_tls_ulp_ops); 1268 tls_strp_dev_exit(); 1269 tls_device_cleanup(); 1270 unregister_pernet_subsys(&tls_proc_ops); 1271 } 1272 1273 module_init(tls_register); 1274 module_exit(tls_unregister); 1275