1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * af_alg: User-space algorithm interface 4 * 5 * This file provides the user-space API for algorithms. 6 * 7 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au> 8 */ 9 10 #include <linux/atomic.h> 11 #include <crypto/if_alg.h> 12 #include <linux/crypto.h> 13 #include <linux/init.h> 14 #include <linux/kernel.h> 15 #include <linux/list.h> 16 #include <linux/module.h> 17 #include <linux/net.h> 18 #include <linux/rwsem.h> 19 #include <linux/sched/signal.h> 20 #include <linux/security.h> 21 22 struct alg_type_list { 23 const struct af_alg_type *type; 24 struct list_head list; 25 }; 26 27 static atomic_long_t alg_memory_allocated; 28 29 static struct proto alg_proto = { 30 .name = "ALG", 31 .owner = THIS_MODULE, 32 .memory_allocated = &alg_memory_allocated, 33 .obj_size = sizeof(struct alg_sock), 34 }; 35 36 static LIST_HEAD(alg_types); 37 static DECLARE_RWSEM(alg_types_sem); 38 39 static const struct af_alg_type *alg_get_type(const char *name) 40 { 41 const struct af_alg_type *type = ERR_PTR(-ENOENT); 42 struct alg_type_list *node; 43 44 down_read(&alg_types_sem); 45 list_for_each_entry(node, &alg_types, list) { 46 if (strcmp(node->type->name, name)) 47 continue; 48 49 if (try_module_get(node->type->owner)) 50 type = node->type; 51 break; 52 } 53 up_read(&alg_types_sem); 54 55 return type; 56 } 57 58 int af_alg_register_type(const struct af_alg_type *type) 59 { 60 struct alg_type_list *node; 61 int err = -EEXIST; 62 63 down_write(&alg_types_sem); 64 list_for_each_entry(node, &alg_types, list) { 65 if (!strcmp(node->type->name, type->name)) 66 goto unlock; 67 } 68 69 node = kmalloc(sizeof(*node), GFP_KERNEL); 70 err = -ENOMEM; 71 if (!node) 72 goto unlock; 73 74 type->ops->owner = THIS_MODULE; 75 if (type->ops_nokey) 76 type->ops_nokey->owner = THIS_MODULE; 77 node->type = type; 78 list_add(&node->list, &alg_types); 79 err = 0; 80 81 unlock: 82 up_write(&alg_types_sem); 83 84 return err; 85 } 86 EXPORT_SYMBOL_GPL(af_alg_register_type); 87 88 int af_alg_unregister_type(const struct af_alg_type *type) 89 { 90 struct alg_type_list *node; 91 int err = -ENOENT; 92 93 down_write(&alg_types_sem); 94 list_for_each_entry(node, &alg_types, list) { 95 if (strcmp(node->type->name, type->name)) 96 continue; 97 98 list_del(&node->list); 99 kfree(node); 100 err = 0; 101 break; 102 } 103 up_write(&alg_types_sem); 104 105 return err; 106 } 107 EXPORT_SYMBOL_GPL(af_alg_unregister_type); 108 109 static void alg_do_release(const struct af_alg_type *type, void *private) 110 { 111 if (!type) 112 return; 113 114 type->release(private); 115 module_put(type->owner); 116 } 117 118 int af_alg_release(struct socket *sock) 119 { 120 if (sock->sk) { 121 sock_put(sock->sk); 122 sock->sk = NULL; 123 } 124 return 0; 125 } 126 EXPORT_SYMBOL_GPL(af_alg_release); 127 128 void af_alg_release_parent(struct sock *sk) 129 { 130 struct alg_sock *ask = alg_sk(sk); 131 unsigned int nokey = atomic_read(&ask->nokey_refcnt); 132 133 sk = ask->parent; 134 ask = alg_sk(sk); 135 136 if (nokey) 137 atomic_dec(&ask->nokey_refcnt); 138 139 if (atomic_dec_and_test(&ask->refcnt)) 140 sock_put(sk); 141 } 142 EXPORT_SYMBOL_GPL(af_alg_release_parent); 143 144 static int alg_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len) 145 { 146 const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY; 147 struct sock *sk = sock->sk; 148 struct alg_sock *ask = alg_sk(sk); 149 struct sockaddr_alg *sa = (void *)uaddr; 150 const struct af_alg_type *type; 151 void *private; 152 int err; 153 154 if (sock->state == SS_CONNECTED) 155 return -EINVAL; 156 157 if (addr_len < sizeof(*sa)) 158 return -EINVAL; 159 160 /* If caller uses non-allowed flag, return error. */ 161 if ((sa->salg_feat & ~allowed) || (sa->salg_mask & ~allowed)) 162 return -EINVAL; 163 164 sa->salg_type[sizeof(sa->salg_type) - 1] = 0; 165 sa->salg_name[sizeof(sa->salg_name) + addr_len - sizeof(*sa) - 1] = 0; 166 167 type = alg_get_type(sa->salg_type); 168 if (PTR_ERR(type) == -ENOENT) { 169 request_module("algif-%s", sa->salg_type); 170 type = alg_get_type(sa->salg_type); 171 } 172 173 if (IS_ERR(type)) 174 return PTR_ERR(type); 175 176 private = type->bind(sa->salg_name, sa->salg_feat, sa->salg_mask); 177 if (IS_ERR(private)) { 178 module_put(type->owner); 179 return PTR_ERR(private); 180 } 181 182 err = -EBUSY; 183 lock_sock(sk); 184 if (atomic_read(&ask->refcnt)) 185 goto unlock; 186 187 swap(ask->type, type); 188 swap(ask->private, private); 189 190 err = 0; 191 192 unlock: 193 release_sock(sk); 194 195 alg_do_release(type, private); 196 197 return err; 198 } 199 200 static int alg_setkey(struct sock *sk, sockptr_t ukey, unsigned int keylen) 201 { 202 struct alg_sock *ask = alg_sk(sk); 203 const struct af_alg_type *type = ask->type; 204 u8 *key; 205 int err; 206 207 key = sock_kmalloc(sk, keylen, GFP_KERNEL); 208 if (!key) 209 return -ENOMEM; 210 211 err = -EFAULT; 212 if (copy_from_sockptr(key, ukey, keylen)) 213 goto out; 214 215 err = type->setkey(ask->private, key, keylen); 216 217 out: 218 sock_kzfree_s(sk, key, keylen); 219 220 return err; 221 } 222 223 static int alg_setsockopt(struct socket *sock, int level, int optname, 224 sockptr_t optval, unsigned int optlen) 225 { 226 struct sock *sk = sock->sk; 227 struct alg_sock *ask = alg_sk(sk); 228 const struct af_alg_type *type; 229 int err = -EBUSY; 230 231 lock_sock(sk); 232 if (atomic_read(&ask->refcnt) != atomic_read(&ask->nokey_refcnt)) 233 goto unlock; 234 235 type = ask->type; 236 237 err = -ENOPROTOOPT; 238 if (level != SOL_ALG || !type) 239 goto unlock; 240 241 switch (optname) { 242 case ALG_SET_KEY: 243 if (sock->state == SS_CONNECTED) 244 goto unlock; 245 if (!type->setkey) 246 goto unlock; 247 248 err = alg_setkey(sk, optval, optlen); 249 break; 250 case ALG_SET_AEAD_AUTHSIZE: 251 if (sock->state == SS_CONNECTED) 252 goto unlock; 253 if (!type->setauthsize) 254 goto unlock; 255 err = type->setauthsize(ask->private, optlen); 256 } 257 258 unlock: 259 release_sock(sk); 260 261 return err; 262 } 263 264 int af_alg_accept(struct sock *sk, struct socket *newsock, bool kern) 265 { 266 struct alg_sock *ask = alg_sk(sk); 267 const struct af_alg_type *type; 268 struct sock *sk2; 269 unsigned int nokey; 270 int err; 271 272 lock_sock(sk); 273 type = ask->type; 274 275 err = -EINVAL; 276 if (!type) 277 goto unlock; 278 279 sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, kern); 280 err = -ENOMEM; 281 if (!sk2) 282 goto unlock; 283 284 sock_init_data(newsock, sk2); 285 security_sock_graft(sk2, newsock); 286 security_sk_clone(sk, sk2); 287 288 err = type->accept(ask->private, sk2); 289 290 nokey = err == -ENOKEY; 291 if (nokey && type->accept_nokey) 292 err = type->accept_nokey(ask->private, sk2); 293 294 if (err) 295 goto unlock; 296 297 if (atomic_inc_return_relaxed(&ask->refcnt) == 1) 298 sock_hold(sk); 299 if (nokey) { 300 atomic_inc(&ask->nokey_refcnt); 301 atomic_set(&alg_sk(sk2)->nokey_refcnt, 1); 302 } 303 alg_sk(sk2)->parent = sk; 304 alg_sk(sk2)->type = type; 305 306 newsock->ops = type->ops; 307 newsock->state = SS_CONNECTED; 308 309 if (nokey) 310 newsock->ops = type->ops_nokey; 311 312 err = 0; 313 314 unlock: 315 release_sock(sk); 316 317 return err; 318 } 319 EXPORT_SYMBOL_GPL(af_alg_accept); 320 321 static int alg_accept(struct socket *sock, struct socket *newsock, int flags, 322 bool kern) 323 { 324 return af_alg_accept(sock->sk, newsock, kern); 325 } 326 327 static const struct proto_ops alg_proto_ops = { 328 .family = PF_ALG, 329 .owner = THIS_MODULE, 330 331 .connect = sock_no_connect, 332 .socketpair = sock_no_socketpair, 333 .getname = sock_no_getname, 334 .ioctl = sock_no_ioctl, 335 .listen = sock_no_listen, 336 .shutdown = sock_no_shutdown, 337 .mmap = sock_no_mmap, 338 .sendpage = sock_no_sendpage, 339 .sendmsg = sock_no_sendmsg, 340 .recvmsg = sock_no_recvmsg, 341 342 .bind = alg_bind, 343 .release = af_alg_release, 344 .setsockopt = alg_setsockopt, 345 .accept = alg_accept, 346 }; 347 348 static void alg_sock_destruct(struct sock *sk) 349 { 350 struct alg_sock *ask = alg_sk(sk); 351 352 alg_do_release(ask->type, ask->private); 353 } 354 355 static int alg_create(struct net *net, struct socket *sock, int protocol, 356 int kern) 357 { 358 struct sock *sk; 359 int err; 360 361 if (sock->type != SOCK_SEQPACKET) 362 return -ESOCKTNOSUPPORT; 363 if (protocol != 0) 364 return -EPROTONOSUPPORT; 365 366 err = -ENOMEM; 367 sk = sk_alloc(net, PF_ALG, GFP_KERNEL, &alg_proto, kern); 368 if (!sk) 369 goto out; 370 371 sock->ops = &alg_proto_ops; 372 sock_init_data(sock, sk); 373 374 sk->sk_destruct = alg_sock_destruct; 375 376 return 0; 377 out: 378 return err; 379 } 380 381 static const struct net_proto_family alg_family = { 382 .family = PF_ALG, 383 .create = alg_create, 384 .owner = THIS_MODULE, 385 }; 386 387 int af_alg_make_sg(struct af_alg_sgl *sgl, struct iov_iter *iter, int len) 388 { 389 size_t off; 390 ssize_t n; 391 int npages, i; 392 393 n = iov_iter_get_pages(iter, sgl->pages, len, ALG_MAX_PAGES, &off); 394 if (n < 0) 395 return n; 396 397 npages = (off + n + PAGE_SIZE - 1) >> PAGE_SHIFT; 398 if (WARN_ON(npages == 0)) 399 return -EINVAL; 400 /* Add one extra for linking */ 401 sg_init_table(sgl->sg, npages + 1); 402 403 for (i = 0, len = n; i < npages; i++) { 404 int plen = min_t(int, len, PAGE_SIZE - off); 405 406 sg_set_page(sgl->sg + i, sgl->pages[i], plen, off); 407 408 off = 0; 409 len -= plen; 410 } 411 sg_mark_end(sgl->sg + npages - 1); 412 sgl->npages = npages; 413 414 return n; 415 } 416 EXPORT_SYMBOL_GPL(af_alg_make_sg); 417 418 static void af_alg_link_sg(struct af_alg_sgl *sgl_prev, 419 struct af_alg_sgl *sgl_new) 420 { 421 sg_unmark_end(sgl_prev->sg + sgl_prev->npages - 1); 422 sg_chain(sgl_prev->sg, sgl_prev->npages + 1, sgl_new->sg); 423 } 424 425 void af_alg_free_sg(struct af_alg_sgl *sgl) 426 { 427 int i; 428 429 for (i = 0; i < sgl->npages; i++) 430 put_page(sgl->pages[i]); 431 } 432 EXPORT_SYMBOL_GPL(af_alg_free_sg); 433 434 static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) 435 { 436 struct cmsghdr *cmsg; 437 438 for_each_cmsghdr(cmsg, msg) { 439 if (!CMSG_OK(msg, cmsg)) 440 return -EINVAL; 441 if (cmsg->cmsg_level != SOL_ALG) 442 continue; 443 444 switch (cmsg->cmsg_type) { 445 case ALG_SET_IV: 446 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv))) 447 return -EINVAL; 448 con->iv = (void *)CMSG_DATA(cmsg); 449 if (cmsg->cmsg_len < CMSG_LEN(con->iv->ivlen + 450 sizeof(*con->iv))) 451 return -EINVAL; 452 break; 453 454 case ALG_SET_OP: 455 if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) 456 return -EINVAL; 457 con->op = *(u32 *)CMSG_DATA(cmsg); 458 break; 459 460 case ALG_SET_AEAD_ASSOCLEN: 461 if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) 462 return -EINVAL; 463 con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg); 464 break; 465 466 default: 467 return -EINVAL; 468 } 469 } 470 471 return 0; 472 } 473 474 /** 475 * af_alg_alloc_tsgl - allocate the TX SGL 476 * 477 * @sk socket of connection to user space 478 * @return: 0 upon success, < 0 upon error 479 */ 480 static int af_alg_alloc_tsgl(struct sock *sk) 481 { 482 struct alg_sock *ask = alg_sk(sk); 483 struct af_alg_ctx *ctx = ask->private; 484 struct af_alg_tsgl *sgl; 485 struct scatterlist *sg = NULL; 486 487 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); 488 if (!list_empty(&ctx->tsgl_list)) 489 sg = sgl->sg; 490 491 if (!sg || sgl->cur >= MAX_SGL_ENTS) { 492 sgl = sock_kmalloc(sk, 493 struct_size(sgl, sg, (MAX_SGL_ENTS + 1)), 494 GFP_KERNEL); 495 if (!sgl) 496 return -ENOMEM; 497 498 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); 499 sgl->cur = 0; 500 501 if (sg) 502 sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); 503 504 list_add_tail(&sgl->list, &ctx->tsgl_list); 505 } 506 507 return 0; 508 } 509 510 /** 511 * aead_count_tsgl - Count number of TX SG entries 512 * 513 * The counting starts from the beginning of the SGL to @bytes. If 514 * an offset is provided, the counting of the SG entries starts at the offset. 515 * 516 * @sk socket of connection to user space 517 * @bytes Count the number of SG entries holding given number of bytes. 518 * @offset Start the counting of SG entries from the given offset. 519 * @return Number of TX SG entries found given the constraints 520 */ 521 unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes, size_t offset) 522 { 523 const struct alg_sock *ask = alg_sk(sk); 524 const struct af_alg_ctx *ctx = ask->private; 525 const struct af_alg_tsgl *sgl; 526 unsigned int i; 527 unsigned int sgl_count = 0; 528 529 if (!bytes) 530 return 0; 531 532 list_for_each_entry(sgl, &ctx->tsgl_list, list) { 533 const struct scatterlist *sg = sgl->sg; 534 535 for (i = 0; i < sgl->cur; i++) { 536 size_t bytes_count; 537 538 /* Skip offset */ 539 if (offset >= sg[i].length) { 540 offset -= sg[i].length; 541 bytes -= sg[i].length; 542 continue; 543 } 544 545 bytes_count = sg[i].length - offset; 546 547 offset = 0; 548 sgl_count++; 549 550 /* If we have seen requested number of bytes, stop */ 551 if (bytes_count >= bytes) 552 return sgl_count; 553 554 bytes -= bytes_count; 555 } 556 } 557 558 return sgl_count; 559 } 560 EXPORT_SYMBOL_GPL(af_alg_count_tsgl); 561 562 /** 563 * aead_pull_tsgl - Release the specified buffers from TX SGL 564 * 565 * If @dst is non-null, reassign the pages to dst. The caller must release 566 * the pages. If @dst_offset is given only reassign the pages to @dst starting 567 * at the @dst_offset (byte). The caller must ensure that @dst is large 568 * enough (e.g. by using af_alg_count_tsgl with the same offset). 569 * 570 * @sk socket of connection to user space 571 * @used Number of bytes to pull from TX SGL 572 * @dst If non-NULL, buffer is reassigned to dst SGL instead of releasing. The 573 * caller must release the buffers in dst. 574 * @dst_offset Reassign the TX SGL from given offset. All buffers before 575 * reaching the offset is released. 576 */ 577 void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst, 578 size_t dst_offset) 579 { 580 struct alg_sock *ask = alg_sk(sk); 581 struct af_alg_ctx *ctx = ask->private; 582 struct af_alg_tsgl *sgl; 583 struct scatterlist *sg; 584 unsigned int i, j = 0; 585 586 while (!list_empty(&ctx->tsgl_list)) { 587 sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, 588 list); 589 sg = sgl->sg; 590 591 for (i = 0; i < sgl->cur; i++) { 592 size_t plen = min_t(size_t, used, sg[i].length); 593 struct page *page = sg_page(sg + i); 594 595 if (!page) 596 continue; 597 598 /* 599 * Assumption: caller created af_alg_count_tsgl(len) 600 * SG entries in dst. 601 */ 602 if (dst) { 603 if (dst_offset >= plen) { 604 /* discard page before offset */ 605 dst_offset -= plen; 606 } else { 607 /* reassign page to dst after offset */ 608 get_page(page); 609 sg_set_page(dst + j, page, 610 plen - dst_offset, 611 sg[i].offset + dst_offset); 612 dst_offset = 0; 613 j++; 614 } 615 } 616 617 sg[i].length -= plen; 618 sg[i].offset += plen; 619 620 used -= plen; 621 ctx->used -= plen; 622 623 if (sg[i].length) 624 return; 625 626 put_page(page); 627 sg_assign_page(sg + i, NULL); 628 } 629 630 list_del(&sgl->list); 631 sock_kfree_s(sk, sgl, struct_size(sgl, sg, MAX_SGL_ENTS + 1)); 632 } 633 634 if (!ctx->used) 635 ctx->merge = 0; 636 ctx->init = ctx->more; 637 } 638 EXPORT_SYMBOL_GPL(af_alg_pull_tsgl); 639 640 /** 641 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request 642 * 643 * @areq Request holding the TX and RX SGL 644 */ 645 static void af_alg_free_areq_sgls(struct af_alg_async_req *areq) 646 { 647 struct sock *sk = areq->sk; 648 struct alg_sock *ask = alg_sk(sk); 649 struct af_alg_ctx *ctx = ask->private; 650 struct af_alg_rsgl *rsgl, *tmp; 651 struct scatterlist *tsgl; 652 struct scatterlist *sg; 653 unsigned int i; 654 655 list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { 656 atomic_sub(rsgl->sg_num_bytes, &ctx->rcvused); 657 af_alg_free_sg(&rsgl->sgl); 658 list_del(&rsgl->list); 659 if (rsgl != &areq->first_rsgl) 660 sock_kfree_s(sk, rsgl, sizeof(*rsgl)); 661 } 662 663 tsgl = areq->tsgl; 664 if (tsgl) { 665 for_each_sg(tsgl, sg, areq->tsgl_entries, i) { 666 if (!sg_page(sg)) 667 continue; 668 put_page(sg_page(sg)); 669 } 670 671 sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); 672 } 673 } 674 675 /** 676 * af_alg_wait_for_wmem - wait for availability of writable memory 677 * 678 * @sk socket of connection to user space 679 * @flags If MSG_DONTWAIT is set, then only report if function would sleep 680 * @return 0 when writable memory is available, < 0 upon error 681 */ 682 static int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags) 683 { 684 DEFINE_WAIT_FUNC(wait, woken_wake_function); 685 int err = -ERESTARTSYS; 686 long timeout; 687 688 if (flags & MSG_DONTWAIT) 689 return -EAGAIN; 690 691 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 692 693 add_wait_queue(sk_sleep(sk), &wait); 694 for (;;) { 695 if (signal_pending(current)) 696 break; 697 timeout = MAX_SCHEDULE_TIMEOUT; 698 if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) { 699 err = 0; 700 break; 701 } 702 } 703 remove_wait_queue(sk_sleep(sk), &wait); 704 705 return err; 706 } 707 708 /** 709 * af_alg_wmem_wakeup - wakeup caller when writable memory is available 710 * 711 * @sk socket of connection to user space 712 */ 713 void af_alg_wmem_wakeup(struct sock *sk) 714 { 715 struct socket_wq *wq; 716 717 if (!af_alg_writable(sk)) 718 return; 719 720 rcu_read_lock(); 721 wq = rcu_dereference(sk->sk_wq); 722 if (skwq_has_sleeper(wq)) 723 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | 724 EPOLLRDNORM | 725 EPOLLRDBAND); 726 sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN); 727 rcu_read_unlock(); 728 } 729 EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup); 730 731 /** 732 * af_alg_wait_for_data - wait for availability of TX data 733 * 734 * @sk socket of connection to user space 735 * @flags If MSG_DONTWAIT is set, then only report if function would sleep 736 * @min Set to minimum request size if partial requests are allowed. 737 * @return 0 when writable memory is available, < 0 upon error 738 */ 739 int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min) 740 { 741 DEFINE_WAIT_FUNC(wait, woken_wake_function); 742 struct alg_sock *ask = alg_sk(sk); 743 struct af_alg_ctx *ctx = ask->private; 744 long timeout; 745 int err = -ERESTARTSYS; 746 747 if (flags & MSG_DONTWAIT) 748 return -EAGAIN; 749 750 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 751 752 add_wait_queue(sk_sleep(sk), &wait); 753 for (;;) { 754 if (signal_pending(current)) 755 break; 756 timeout = MAX_SCHEDULE_TIMEOUT; 757 if (sk_wait_event(sk, &timeout, 758 ctx->init && (!ctx->more || 759 (min && ctx->used >= min)), 760 &wait)) { 761 err = 0; 762 break; 763 } 764 } 765 remove_wait_queue(sk_sleep(sk), &wait); 766 767 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 768 769 return err; 770 } 771 EXPORT_SYMBOL_GPL(af_alg_wait_for_data); 772 773 /** 774 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel 775 * 776 * @sk socket of connection to user space 777 */ 778 static void af_alg_data_wakeup(struct sock *sk) 779 { 780 struct alg_sock *ask = alg_sk(sk); 781 struct af_alg_ctx *ctx = ask->private; 782 struct socket_wq *wq; 783 784 if (!ctx->used) 785 return; 786 787 rcu_read_lock(); 788 wq = rcu_dereference(sk->sk_wq); 789 if (skwq_has_sleeper(wq)) 790 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 791 EPOLLRDNORM | 792 EPOLLRDBAND); 793 sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT); 794 rcu_read_unlock(); 795 } 796 797 /** 798 * af_alg_sendmsg - implementation of sendmsg system call handler 799 * 800 * The sendmsg system call handler obtains the user data and stores it 801 * in ctx->tsgl_list. This implies allocation of the required numbers of 802 * struct af_alg_tsgl. 803 * 804 * In addition, the ctx is filled with the information sent via CMSG. 805 * 806 * @sock socket of connection to user space 807 * @msg message from user space 808 * @size size of message from user space 809 * @ivsize the size of the IV for the cipher operation to verify that the 810 * user-space-provided IV has the right size 811 * @return the number of copied data upon success, < 0 upon error 812 */ 813 int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, 814 unsigned int ivsize) 815 { 816 struct sock *sk = sock->sk; 817 struct alg_sock *ask = alg_sk(sk); 818 struct af_alg_ctx *ctx = ask->private; 819 struct af_alg_tsgl *sgl; 820 struct af_alg_control con = {}; 821 long copied = 0; 822 bool enc = false; 823 bool init = false; 824 int err = 0; 825 826 if (msg->msg_controllen) { 827 err = af_alg_cmsg_send(msg, &con); 828 if (err) 829 return err; 830 831 init = true; 832 switch (con.op) { 833 case ALG_OP_ENCRYPT: 834 enc = true; 835 break; 836 case ALG_OP_DECRYPT: 837 enc = false; 838 break; 839 default: 840 return -EINVAL; 841 } 842 843 if (con.iv && con.iv->ivlen != ivsize) 844 return -EINVAL; 845 } 846 847 lock_sock(sk); 848 if (ctx->init && (init || !ctx->more)) { 849 err = -EINVAL; 850 goto unlock; 851 } 852 ctx->init = true; 853 854 if (init) { 855 ctx->enc = enc; 856 if (con.iv) 857 memcpy(ctx->iv, con.iv->iv, ivsize); 858 859 ctx->aead_assoclen = con.aead_assoclen; 860 } 861 862 while (size) { 863 struct scatterlist *sg; 864 size_t len = size; 865 size_t plen; 866 867 /* use the existing memory in an allocated page */ 868 if (ctx->merge) { 869 sgl = list_entry(ctx->tsgl_list.prev, 870 struct af_alg_tsgl, list); 871 sg = sgl->sg + sgl->cur - 1; 872 len = min_t(size_t, len, 873 PAGE_SIZE - sg->offset - sg->length); 874 875 err = memcpy_from_msg(page_address(sg_page(sg)) + 876 sg->offset + sg->length, 877 msg, len); 878 if (err) 879 goto unlock; 880 881 sg->length += len; 882 ctx->merge = (sg->offset + sg->length) & 883 (PAGE_SIZE - 1); 884 885 ctx->used += len; 886 copied += len; 887 size -= len; 888 continue; 889 } 890 891 if (!af_alg_writable(sk)) { 892 err = af_alg_wait_for_wmem(sk, msg->msg_flags); 893 if (err) 894 goto unlock; 895 } 896 897 /* allocate a new page */ 898 len = min_t(unsigned long, len, af_alg_sndbuf(sk)); 899 900 err = af_alg_alloc_tsgl(sk); 901 if (err) 902 goto unlock; 903 904 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, 905 list); 906 sg = sgl->sg; 907 if (sgl->cur) 908 sg_unmark_end(sg + sgl->cur - 1); 909 910 do { 911 unsigned int i = sgl->cur; 912 913 plen = min_t(size_t, len, PAGE_SIZE); 914 915 sg_assign_page(sg + i, alloc_page(GFP_KERNEL)); 916 if (!sg_page(sg + i)) { 917 err = -ENOMEM; 918 goto unlock; 919 } 920 921 err = memcpy_from_msg(page_address(sg_page(sg + i)), 922 msg, plen); 923 if (err) { 924 __free_page(sg_page(sg + i)); 925 sg_assign_page(sg + i, NULL); 926 goto unlock; 927 } 928 929 sg[i].length = plen; 930 len -= plen; 931 ctx->used += plen; 932 copied += plen; 933 size -= plen; 934 sgl->cur++; 935 } while (len && sgl->cur < MAX_SGL_ENTS); 936 937 if (!size) 938 sg_mark_end(sg + sgl->cur - 1); 939 940 ctx->merge = plen & (PAGE_SIZE - 1); 941 } 942 943 err = 0; 944 945 ctx->more = msg->msg_flags & MSG_MORE; 946 947 unlock: 948 af_alg_data_wakeup(sk); 949 release_sock(sk); 950 951 return copied ?: err; 952 } 953 EXPORT_SYMBOL_GPL(af_alg_sendmsg); 954 955 /** 956 * af_alg_sendpage - sendpage system call handler 957 * 958 * This is a generic implementation of sendpage to fill ctx->tsgl_list. 959 */ 960 ssize_t af_alg_sendpage(struct socket *sock, struct page *page, 961 int offset, size_t size, int flags) 962 { 963 struct sock *sk = sock->sk; 964 struct alg_sock *ask = alg_sk(sk); 965 struct af_alg_ctx *ctx = ask->private; 966 struct af_alg_tsgl *sgl; 967 int err = -EINVAL; 968 969 if (flags & MSG_SENDPAGE_NOTLAST) 970 flags |= MSG_MORE; 971 972 lock_sock(sk); 973 if (!ctx->more && ctx->used) 974 goto unlock; 975 976 if (!size) 977 goto done; 978 979 if (!af_alg_writable(sk)) { 980 err = af_alg_wait_for_wmem(sk, flags); 981 if (err) 982 goto unlock; 983 } 984 985 err = af_alg_alloc_tsgl(sk); 986 if (err) 987 goto unlock; 988 989 ctx->merge = 0; 990 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); 991 992 if (sgl->cur) 993 sg_unmark_end(sgl->sg + sgl->cur - 1); 994 995 sg_mark_end(sgl->sg + sgl->cur); 996 997 get_page(page); 998 sg_set_page(sgl->sg + sgl->cur, page, size, offset); 999 sgl->cur++; 1000 ctx->used += size; 1001 1002 done: 1003 ctx->more = flags & MSG_MORE; 1004 1005 unlock: 1006 af_alg_data_wakeup(sk); 1007 release_sock(sk); 1008 1009 return err ?: size; 1010 } 1011 EXPORT_SYMBOL_GPL(af_alg_sendpage); 1012 1013 /** 1014 * af_alg_free_resources - release resources required for crypto request 1015 */ 1016 void af_alg_free_resources(struct af_alg_async_req *areq) 1017 { 1018 struct sock *sk = areq->sk; 1019 1020 af_alg_free_areq_sgls(areq); 1021 sock_kfree_s(sk, areq, areq->areqlen); 1022 } 1023 EXPORT_SYMBOL_GPL(af_alg_free_resources); 1024 1025 /** 1026 * af_alg_async_cb - AIO callback handler 1027 * 1028 * This handler cleans up the struct af_alg_async_req upon completion of the 1029 * AIO operation. 1030 * 1031 * The number of bytes to be generated with the AIO operation must be set 1032 * in areq->outlen before the AIO callback handler is invoked. 1033 */ 1034 void af_alg_async_cb(struct crypto_async_request *_req, int err) 1035 { 1036 struct af_alg_async_req *areq = _req->data; 1037 struct sock *sk = areq->sk; 1038 struct kiocb *iocb = areq->iocb; 1039 unsigned int resultlen; 1040 1041 /* Buffer size written by crypto operation. */ 1042 resultlen = areq->outlen; 1043 1044 af_alg_free_resources(areq); 1045 sock_put(sk); 1046 1047 iocb->ki_complete(iocb, err ? err : (int)resultlen, 0); 1048 } 1049 EXPORT_SYMBOL_GPL(af_alg_async_cb); 1050 1051 /** 1052 * af_alg_poll - poll system call handler 1053 */ 1054 __poll_t af_alg_poll(struct file *file, struct socket *sock, 1055 poll_table *wait) 1056 { 1057 struct sock *sk = sock->sk; 1058 struct alg_sock *ask = alg_sk(sk); 1059 struct af_alg_ctx *ctx = ask->private; 1060 __poll_t mask; 1061 1062 sock_poll_wait(file, sock, wait); 1063 mask = 0; 1064 1065 if (!ctx->more || ctx->used) 1066 mask |= EPOLLIN | EPOLLRDNORM; 1067 1068 if (af_alg_writable(sk)) 1069 mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND; 1070 1071 return mask; 1072 } 1073 EXPORT_SYMBOL_GPL(af_alg_poll); 1074 1075 /** 1076 * af_alg_alloc_areq - allocate struct af_alg_async_req 1077 * 1078 * @sk socket of connection to user space 1079 * @areqlen size of struct af_alg_async_req + crypto_*_reqsize 1080 * @return allocated data structure or ERR_PTR upon error 1081 */ 1082 struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, 1083 unsigned int areqlen) 1084 { 1085 struct af_alg_async_req *areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); 1086 1087 if (unlikely(!areq)) 1088 return ERR_PTR(-ENOMEM); 1089 1090 areq->areqlen = areqlen; 1091 areq->sk = sk; 1092 areq->last_rsgl = NULL; 1093 INIT_LIST_HEAD(&areq->rsgl_list); 1094 areq->tsgl = NULL; 1095 areq->tsgl_entries = 0; 1096 1097 return areq; 1098 } 1099 EXPORT_SYMBOL_GPL(af_alg_alloc_areq); 1100 1101 /** 1102 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto 1103 * operation 1104 * 1105 * @sk socket of connection to user space 1106 * @msg user space message 1107 * @flags flags used to invoke recvmsg with 1108 * @areq instance of the cryptographic request that will hold the RX SGL 1109 * @maxsize maximum number of bytes to be pulled from user space 1110 * @outlen number of bytes in the RX SGL 1111 * @return 0 on success, < 0 upon error 1112 */ 1113 int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, 1114 struct af_alg_async_req *areq, size_t maxsize, 1115 size_t *outlen) 1116 { 1117 struct alg_sock *ask = alg_sk(sk); 1118 struct af_alg_ctx *ctx = ask->private; 1119 size_t len = 0; 1120 1121 while (maxsize > len && msg_data_left(msg)) { 1122 struct af_alg_rsgl *rsgl; 1123 size_t seglen; 1124 int err; 1125 1126 /* limit the amount of readable buffers */ 1127 if (!af_alg_readable(sk)) 1128 break; 1129 1130 seglen = min_t(size_t, (maxsize - len), 1131 msg_data_left(msg)); 1132 1133 if (list_empty(&areq->rsgl_list)) { 1134 rsgl = &areq->first_rsgl; 1135 } else { 1136 rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); 1137 if (unlikely(!rsgl)) 1138 return -ENOMEM; 1139 } 1140 1141 rsgl->sgl.npages = 0; 1142 list_add_tail(&rsgl->list, &areq->rsgl_list); 1143 1144 /* make one iovec available as scatterlist */ 1145 err = af_alg_make_sg(&rsgl->sgl, &msg->msg_iter, seglen); 1146 if (err < 0) { 1147 rsgl->sg_num_bytes = 0; 1148 return err; 1149 } 1150 1151 /* chain the new scatterlist with previous one */ 1152 if (areq->last_rsgl) 1153 af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl); 1154 1155 areq->last_rsgl = rsgl; 1156 len += err; 1157 atomic_add(err, &ctx->rcvused); 1158 rsgl->sg_num_bytes = err; 1159 iov_iter_advance(&msg->msg_iter, err); 1160 } 1161 1162 *outlen = len; 1163 return 0; 1164 } 1165 EXPORT_SYMBOL_GPL(af_alg_get_rsgl); 1166 1167 static int __init af_alg_init(void) 1168 { 1169 int err = proto_register(&alg_proto, 0); 1170 1171 if (err) 1172 goto out; 1173 1174 err = sock_register(&alg_family); 1175 if (err != 0) 1176 goto out_unregister_proto; 1177 1178 out: 1179 return err; 1180 1181 out_unregister_proto: 1182 proto_unregister(&alg_proto); 1183 goto out; 1184 } 1185 1186 static void __exit af_alg_exit(void) 1187 { 1188 sock_unregister(PF_ALG); 1189 proto_unregister(&alg_proto); 1190 } 1191 1192 module_init(af_alg_init); 1193 module_exit(af_alg_exit); 1194 MODULE_LICENSE("GPL"); 1195 MODULE_ALIAS_NETPROTO(AF_ALG); 1196