1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * af_alg: User-space algorithm interface 4 * 5 * This file provides the user-space API for algorithms. 6 * 7 * Copyright (c) 2010 Herbert Xu <herbert@gondor.apana.org.au> 8 */ 9 10 #include <linux/atomic.h> 11 #include <linux/capability.h> 12 #include <crypto/if_alg.h> 13 #include <linux/crypto.h> 14 #include <linux/init.h> 15 #include <linux/kernel.h> 16 #include <linux/key.h> 17 #include <linux/key-type.h> 18 #include <linux/list.h> 19 #include <linux/module.h> 20 #include <linux/net.h> 21 #include <linux/rwsem.h> 22 #include <linux/sched.h> 23 #include <linux/sched/signal.h> 24 #include <linux/security.h> 25 #include <linux/string.h> 26 #include <linux/sysctl.h> 27 #include <linux/user_namespace.h> 28 #include <keys/user-type.h> 29 #include <keys/trusted-type.h> 30 #include <keys/encrypted-type.h> 31 32 static int af_alg_restrict = 1; 33 34 static const struct ctl_table af_alg_table[] = { 35 { 36 .procname = "af_alg_restrict", 37 .data = &af_alg_restrict, 38 .maxlen = sizeof(int), 39 .mode = 0644, 40 .proc_handler = proc_dointvec_minmax, 41 .extra1 = SYSCTL_ZERO, 42 .extra2 = SYSCTL_TWO, 43 }, 44 }; 45 46 static struct ctl_table_header *af_alg_header; 47 48 struct alg_type_list { 49 const struct af_alg_type *type; 50 struct list_head list; 51 }; 52 53 static struct proto alg_proto = { 54 .name = "ALG", 55 .owner = THIS_MODULE, 56 .obj_size = sizeof(struct alg_sock), 57 }; 58 59 static LIST_HEAD(alg_types); 60 static DECLARE_RWSEM(alg_types_sem); 61 62 static const struct af_alg_type *alg_get_type(const char *name) 63 { 64 const struct af_alg_type *type = ERR_PTR(-ENOENT); 65 struct alg_type_list *node; 66 67 down_read(&alg_types_sem); 68 list_for_each_entry(node, &alg_types, list) { 69 if (strcmp(node->type->name, name)) 70 continue; 71 72 if (try_module_get(node->type->owner)) 73 type = node->type; 74 break; 75 } 76 up_read(&alg_types_sem); 77 78 return type; 79 } 80 81 int af_alg_register_type(const struct af_alg_type *type) 82 { 83 struct alg_type_list *node; 84 int err = -EEXIST; 85 86 down_write(&alg_types_sem); 87 list_for_each_entry(node, &alg_types, list) { 88 if (!strcmp(node->type->name, type->name)) 89 goto unlock; 90 } 91 92 node = kmalloc_obj(*node); 93 err = -ENOMEM; 94 if (!node) 95 goto unlock; 96 97 type->ops->owner = THIS_MODULE; 98 if (type->ops_nokey) 99 type->ops_nokey->owner = THIS_MODULE; 100 node->type = type; 101 list_add(&node->list, &alg_types); 102 err = 0; 103 104 unlock: 105 up_write(&alg_types_sem); 106 107 return err; 108 } 109 EXPORT_SYMBOL_GPL(af_alg_register_type); 110 111 int af_alg_unregister_type(const struct af_alg_type *type) 112 { 113 struct alg_type_list *node; 114 int err = -ENOENT; 115 116 down_write(&alg_types_sem); 117 list_for_each_entry(node, &alg_types, list) { 118 if (strcmp(node->type->name, type->name)) 119 continue; 120 121 list_del(&node->list); 122 kfree(node); 123 err = 0; 124 break; 125 } 126 up_write(&alg_types_sem); 127 128 return err; 129 } 130 EXPORT_SYMBOL_GPL(af_alg_unregister_type); 131 132 static bool af_alg_capable(void) 133 { 134 return ns_capable_noaudit(&init_user_ns, CAP_NET_ADMIN) || 135 capable(CAP_SYS_ADMIN); 136 } 137 138 int af_alg_check_restriction(const char *name, 139 const struct af_alg_allowlist_entry allowlist[]) 140 { 141 int level = READ_ONCE(af_alg_restrict); 142 143 if (level == 0) 144 return 0; 145 if (level == 1) { 146 for (const struct af_alg_allowlist_entry *ent = allowlist; 147 ent->name; ent++) { 148 if (strcmp(name, ent->name) == 0) { 149 if ((ent->flags & AF_ALG_UNPRIVILEGED) || 150 af_alg_capable()) 151 return 0; 152 /* List contains at most one entry per name. */ 153 break; 154 } 155 } 156 } 157 /* 158 * Use -ENOENT (the error code for "algorithm not found") instead of 159 * -EACCES or -EPERM, for the highest chance of correctly triggering 160 * fallback code paths in userspace programs. 161 * 162 * Don't log a warning, since it would be noisy. iwd tries to bind a 163 * bunch of algorithms that it never uses. 164 */ 165 return -ENOENT; 166 } 167 EXPORT_SYMBOL_GPL(af_alg_check_restriction); 168 169 static void alg_do_release(const struct af_alg_type *type, void *private) 170 { 171 if (!type) 172 return; 173 174 type->release(private); 175 module_put(type->owner); 176 } 177 178 int af_alg_release(struct socket *sock) 179 { 180 if (sock->sk) { 181 sock_put(sock->sk); 182 sock->sk = NULL; 183 } 184 return 0; 185 } 186 EXPORT_SYMBOL_GPL(af_alg_release); 187 188 void af_alg_release_parent(struct sock *sk) 189 { 190 struct alg_sock *ask = alg_sk(sk); 191 unsigned int nokey = atomic_read(&ask->nokey_refcnt); 192 193 sk = ask->parent; 194 ask = alg_sk(sk); 195 196 if (nokey) 197 atomic_dec(&ask->nokey_refcnt); 198 199 if (atomic_dec_and_test(&ask->refcnt)) 200 sock_put(sk); 201 } 202 EXPORT_SYMBOL_GPL(af_alg_release_parent); 203 204 static int alg_bind(struct socket *sock, struct sockaddr_unsized *uaddr, int addr_len) 205 { 206 const u32 allowed = CRYPTO_ALG_KERN_DRIVER_ONLY; 207 struct sock *sk = sock->sk; 208 struct alg_sock *ask = alg_sk(sk); 209 struct sockaddr_alg_new *sa = (void *)uaddr; 210 const struct af_alg_type *type; 211 void *private; 212 int err; 213 214 if (sock->state == SS_CONNECTED) 215 return -EINVAL; 216 217 BUILD_BUG_ON(offsetof(struct sockaddr_alg_new, salg_name) != 218 offsetof(struct sockaddr_alg, salg_name)); 219 BUILD_BUG_ON(offsetof(struct sockaddr_alg, salg_name) != sizeof(*sa)); 220 221 if (addr_len < sizeof(*sa) + 1) 222 return -EINVAL; 223 224 /* If caller uses non-allowed flag, return error. */ 225 if ((sa->salg_feat & ~allowed) || (sa->salg_mask & ~allowed)) 226 return -EINVAL; 227 228 sa->salg_type[sizeof(sa->salg_type) - 1] = 0; 229 sa->salg_name[addr_len - sizeof(*sa) - 1] = 0; 230 231 type = alg_get_type(sa->salg_type); 232 if (PTR_ERR(type) == -ENOENT) { 233 request_module("algif-%s", sa->salg_type); 234 type = alg_get_type(sa->salg_type); 235 } 236 237 if (IS_ERR(type)) 238 return PTR_ERR(type); 239 240 private = type->bind(sa->salg_name); 241 if (IS_ERR(private)) { 242 module_put(type->owner); 243 return PTR_ERR(private); 244 } 245 246 err = -EBUSY; 247 lock_sock(sk); 248 if (atomic_read(&ask->refcnt)) 249 goto unlock; 250 251 swap(ask->type, type); 252 swap(ask->private, private); 253 254 err = 0; 255 256 unlock: 257 release_sock(sk); 258 259 alg_do_release(type, private); 260 261 return err; 262 } 263 264 static int alg_setkey(struct sock *sk, sockptr_t ukey, unsigned int keylen) 265 { 266 struct alg_sock *ask = alg_sk(sk); 267 const struct af_alg_type *type = ask->type; 268 u8 *key; 269 int err; 270 271 key = sock_kmalloc(sk, keylen, GFP_KERNEL); 272 if (!key) 273 return -ENOMEM; 274 275 err = -EFAULT; 276 if (copy_from_sockptr(key, ukey, keylen)) 277 goto out; 278 279 err = type->setkey(ask->private, key, keylen); 280 281 out: 282 sock_kzfree_s(sk, key, keylen); 283 284 return err; 285 } 286 287 #ifdef CONFIG_KEYS 288 289 static const u8 *key_data_ptr_user(const struct key *key, 290 unsigned int *datalen) 291 { 292 const struct user_key_payload *ukp; 293 294 ukp = user_key_payload_locked(key); 295 if (IS_ERR_OR_NULL(ukp)) 296 return ERR_PTR(-EKEYREVOKED); 297 298 *datalen = key->datalen; 299 300 return ukp->data; 301 } 302 303 static const u8 *key_data_ptr_encrypted(const struct key *key, 304 unsigned int *datalen) 305 { 306 const struct encrypted_key_payload *ekp; 307 308 ekp = dereference_key_locked(key); 309 if (IS_ERR_OR_NULL(ekp)) 310 return ERR_PTR(-EKEYREVOKED); 311 312 *datalen = ekp->decrypted_datalen; 313 314 return ekp->decrypted_data; 315 } 316 317 static const u8 *key_data_ptr_trusted(const struct key *key, 318 unsigned int *datalen) 319 { 320 const struct trusted_key_payload *tkp; 321 322 tkp = dereference_key_locked(key); 323 if (IS_ERR_OR_NULL(tkp)) 324 return ERR_PTR(-EKEYREVOKED); 325 326 *datalen = tkp->key_len; 327 328 return tkp->key; 329 } 330 331 static struct key *lookup_key(key_serial_t serial) 332 { 333 key_ref_t key_ref; 334 335 key_ref = lookup_user_key(serial, 0, KEY_NEED_SEARCH); 336 if (IS_ERR(key_ref)) 337 return ERR_CAST(key_ref); 338 339 return key_ref_to_ptr(key_ref); 340 } 341 342 static int alg_setkey_by_key_serial(struct alg_sock *ask, sockptr_t optval, 343 unsigned int optlen) 344 { 345 const struct af_alg_type *type = ask->type; 346 u8 *key_data = NULL; 347 unsigned int key_datalen; 348 key_serial_t serial; 349 struct key *key; 350 const u8 *ret; 351 int err; 352 353 if (optlen != sizeof(serial)) 354 return -EINVAL; 355 356 if (copy_from_sockptr(&serial, optval, optlen)) 357 return -EFAULT; 358 359 key = lookup_key(serial); 360 if (IS_ERR(key)) 361 return PTR_ERR(key); 362 363 down_read(&key->sem); 364 365 ret = ERR_PTR(-ENOPROTOOPT); 366 if (!strcmp(key->type->name, "user") || 367 !strcmp(key->type->name, "logon")) { 368 ret = key_data_ptr_user(key, &key_datalen); 369 } else if (IS_REACHABLE(CONFIG_ENCRYPTED_KEYS) && 370 !strcmp(key->type->name, "encrypted")) { 371 ret = key_data_ptr_encrypted(key, &key_datalen); 372 } else if (IS_REACHABLE(CONFIG_TRUSTED_KEYS) && 373 !strcmp(key->type->name, "trusted")) { 374 ret = key_data_ptr_trusted(key, &key_datalen); 375 } 376 377 if (IS_ERR(ret)) { 378 up_read(&key->sem); 379 key_put(key); 380 return PTR_ERR(ret); 381 } 382 383 key_data = sock_kmemdup(&ask->sk, ret, key_datalen, GFP_KERNEL); 384 if (!key_data) { 385 up_read(&key->sem); 386 key_put(key); 387 return -ENOMEM; 388 } 389 390 up_read(&key->sem); 391 key_put(key); 392 393 err = type->setkey(ask->private, key_data, key_datalen); 394 395 sock_kzfree_s(&ask->sk, key_data, key_datalen); 396 397 return err; 398 } 399 400 #else 401 402 static inline int alg_setkey_by_key_serial(struct alg_sock *ask, 403 sockptr_t optval, 404 unsigned int optlen) 405 { 406 return -ENOPROTOOPT; 407 } 408 409 #endif 410 411 static int alg_setsockopt(struct socket *sock, int level, int optname, 412 sockptr_t optval, unsigned int optlen) 413 { 414 struct sock *sk = sock->sk; 415 struct alg_sock *ask = alg_sk(sk); 416 const struct af_alg_type *type; 417 int err = -EBUSY; 418 419 lock_sock(sk); 420 if (atomic_read(&ask->refcnt) != atomic_read(&ask->nokey_refcnt)) 421 goto unlock; 422 423 type = ask->type; 424 425 err = -ENOPROTOOPT; 426 if (level != SOL_ALG || !type) 427 goto unlock; 428 429 switch (optname) { 430 case ALG_SET_KEY: 431 case ALG_SET_KEY_BY_KEY_SERIAL: 432 if (sock->state == SS_CONNECTED) 433 goto unlock; 434 if (!type->setkey) 435 goto unlock; 436 437 if (optname == ALG_SET_KEY_BY_KEY_SERIAL) 438 err = alg_setkey_by_key_serial(ask, optval, optlen); 439 else 440 err = alg_setkey(sk, optval, optlen); 441 break; 442 case ALG_SET_AEAD_AUTHSIZE: 443 if (sock->state == SS_CONNECTED) 444 goto unlock; 445 if (!type->setauthsize) 446 goto unlock; 447 err = type->setauthsize(ask->private, optlen); 448 break; 449 case ALG_SET_DRBG_ENTROPY: 450 if (sock->state == SS_CONNECTED) 451 goto unlock; 452 if (!type->setentropy) 453 goto unlock; 454 455 err = type->setentropy(ask->private, optval, optlen); 456 } 457 458 unlock: 459 release_sock(sk); 460 461 return err; 462 } 463 464 int af_alg_accept(struct sock *sk, struct socket *newsock, 465 struct proto_accept_arg *arg) 466 { 467 struct alg_sock *ask = alg_sk(sk); 468 const struct af_alg_type *type; 469 struct sock *sk2; 470 unsigned int nokey; 471 int err; 472 473 lock_sock(sk); 474 type = ask->type; 475 476 err = -EINVAL; 477 if (!type) 478 goto unlock; 479 480 sk2 = sk_alloc(sock_net(sk), PF_ALG, GFP_KERNEL, &alg_proto, arg->kern); 481 err = -ENOMEM; 482 if (!sk2) 483 goto unlock; 484 485 sock_init_data(newsock, sk2); 486 security_sock_graft(sk2, newsock); 487 security_sk_clone(sk, sk2); 488 489 /* 490 * newsock->ops assigned here to allow type->accept call to override 491 * them when required. 492 */ 493 newsock->ops = type->ops; 494 err = type->accept(ask->private, sk2); 495 496 nokey = err == -ENOKEY; 497 if (nokey && type->accept_nokey) 498 err = type->accept_nokey(ask->private, sk2); 499 500 if (err) 501 goto unlock; 502 503 if (atomic_inc_return_relaxed(&ask->refcnt) == 1) 504 sock_hold(sk); 505 if (nokey) { 506 atomic_inc(&ask->nokey_refcnt); 507 atomic_set(&alg_sk(sk2)->nokey_refcnt, 1); 508 } 509 alg_sk(sk2)->parent = sk; 510 alg_sk(sk2)->type = type; 511 512 newsock->state = SS_CONNECTED; 513 514 if (nokey) 515 newsock->ops = type->ops_nokey; 516 517 err = 0; 518 519 unlock: 520 release_sock(sk); 521 522 return err; 523 } 524 EXPORT_SYMBOL_GPL(af_alg_accept); 525 526 static int alg_accept(struct socket *sock, struct socket *newsock, 527 struct proto_accept_arg *arg) 528 { 529 return af_alg_accept(sock->sk, newsock, arg); 530 } 531 532 static const struct proto_ops alg_proto_ops = { 533 .family = PF_ALG, 534 .owner = THIS_MODULE, 535 536 .connect = sock_no_connect, 537 .socketpair = sock_no_socketpair, 538 .getname = sock_no_getname, 539 .ioctl = sock_no_ioctl, 540 .listen = sock_no_listen, 541 .shutdown = sock_no_shutdown, 542 .mmap = sock_no_mmap, 543 .sendmsg = sock_no_sendmsg, 544 .recvmsg = sock_no_recvmsg, 545 546 .bind = alg_bind, 547 .release = af_alg_release, 548 .setsockopt = alg_setsockopt, 549 .accept = alg_accept, 550 }; 551 552 static void alg_sock_destruct(struct sock *sk) 553 { 554 struct alg_sock *ask = alg_sk(sk); 555 556 alg_do_release(ask->type, ask->private); 557 } 558 559 static int alg_create(struct net *net, struct socket *sock, int protocol, 560 int kern) 561 { 562 struct sock *sk; 563 int err; 564 565 if (READ_ONCE(af_alg_restrict) == 2) 566 return -EAFNOSUPPORT; 567 568 if (sock->type != SOCK_SEQPACKET) 569 return -ESOCKTNOSUPPORT; 570 if (protocol != 0) 571 return -EPROTONOSUPPORT; 572 573 err = -ENOMEM; 574 sk = sk_alloc(net, PF_ALG, GFP_KERNEL, &alg_proto, kern); 575 if (!sk) 576 goto out; 577 578 sock->ops = &alg_proto_ops; 579 sock_init_data(sock, sk); 580 581 sk->sk_destruct = alg_sock_destruct; 582 583 return 0; 584 out: 585 return err; 586 } 587 588 static const struct net_proto_family alg_family = { 589 .family = PF_ALG, 590 .create = alg_create, 591 .owner = THIS_MODULE, 592 }; 593 594 static void af_alg_link_sg(struct af_alg_sgl *sgl_prev, 595 struct af_alg_sgl *sgl_new) 596 { 597 sg_unmark_end(sgl_prev->sgt.sgl + sgl_prev->sgt.nents - 1); 598 sg_chain(sgl_prev->sgt.sgl, sgl_prev->sgt.nents + 1, sgl_new->sgt.sgl); 599 } 600 601 void af_alg_free_sg(struct af_alg_sgl *sgl) 602 { 603 int i; 604 605 if (sgl->sgt.sgl) { 606 if (sgl->need_unpin) 607 for (i = 0; i < sgl->sgt.nents; i++) 608 unpin_user_page(sg_page(&sgl->sgt.sgl[i])); 609 if (sgl->sgt.sgl != sgl->sgl) 610 kvfree(sgl->sgt.sgl); 611 sgl->sgt.sgl = NULL; 612 } 613 } 614 EXPORT_SYMBOL_GPL(af_alg_free_sg); 615 616 static int af_alg_cmsg_send(struct msghdr *msg, struct af_alg_control *con) 617 { 618 struct cmsghdr *cmsg; 619 620 for_each_cmsghdr(cmsg, msg) { 621 if (!CMSG_OK(msg, cmsg)) 622 return -EINVAL; 623 if (cmsg->cmsg_level != SOL_ALG) 624 continue; 625 626 switch (cmsg->cmsg_type) { 627 case ALG_SET_IV: 628 if (cmsg->cmsg_len < CMSG_LEN(sizeof(*con->iv))) 629 return -EINVAL; 630 con->iv = (void *)CMSG_DATA(cmsg); 631 if (cmsg->cmsg_len < CMSG_LEN(con->iv->ivlen + 632 sizeof(*con->iv))) 633 return -EINVAL; 634 break; 635 636 case ALG_SET_OP: 637 if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) 638 return -EINVAL; 639 con->op = *(u32 *)CMSG_DATA(cmsg); 640 break; 641 642 case ALG_SET_AEAD_ASSOCLEN: 643 if (cmsg->cmsg_len < CMSG_LEN(sizeof(u32))) 644 return -EINVAL; 645 con->aead_assoclen = *(u32 *)CMSG_DATA(cmsg); 646 if (con->aead_assoclen >= 0x80000000u) 647 return -EINVAL; 648 break; 649 650 default: 651 return -EINVAL; 652 } 653 } 654 655 return 0; 656 } 657 658 /** 659 * af_alg_alloc_tsgl - allocate the TX SGL 660 * 661 * @sk: socket of connection to user space 662 * Return: 0 upon success, < 0 upon error 663 */ 664 static int af_alg_alloc_tsgl(struct sock *sk) 665 { 666 struct alg_sock *ask = alg_sk(sk); 667 struct af_alg_ctx *ctx = ask->private; 668 struct af_alg_tsgl *sgl; 669 struct scatterlist *sg = NULL; 670 671 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, list); 672 if (!list_empty(&ctx->tsgl_list)) 673 sg = sgl->sg; 674 675 if (!sg || sgl->cur >= MAX_SGL_ENTS) { 676 sgl = sock_kmalloc(sk, 677 struct_size(sgl, sg, (MAX_SGL_ENTS + 1)), 678 GFP_KERNEL); 679 if (!sgl) 680 return -ENOMEM; 681 682 sg_init_table(sgl->sg, MAX_SGL_ENTS + 1); 683 sgl->cur = 0; 684 685 if (sg) { 686 sg_unmark_end(sg + MAX_SGL_ENTS - 1); 687 sg_chain(sg, MAX_SGL_ENTS + 1, sgl->sg); 688 } 689 690 list_add_tail(&sgl->list, &ctx->tsgl_list); 691 } 692 693 return 0; 694 } 695 696 /** 697 * af_alg_count_tsgl - Count number of TX SG entries 698 * 699 * The counting starts from the beginning of the SGL to @bytes. 700 * 701 * @sk: socket of connection to user space 702 * @bytes: Count the number of SG entries holding given number of bytes. 703 * Return: Number of TX SG entries found given the constraints 704 */ 705 unsigned int af_alg_count_tsgl(struct sock *sk, size_t bytes) 706 { 707 const struct alg_sock *ask = alg_sk(sk); 708 const struct af_alg_ctx *ctx = ask->private; 709 const struct af_alg_tsgl *sgl; 710 unsigned int i; 711 unsigned int sgl_count = 0; 712 713 if (!bytes) 714 return 0; 715 716 list_for_each_entry(sgl, &ctx->tsgl_list, list) { 717 const struct scatterlist *sg = sgl->sg; 718 719 for (i = 0; i < sgl->cur; i++) { 720 sgl_count++; 721 if (sg[i].length >= bytes) 722 return sgl_count; 723 724 bytes -= sg[i].length; 725 } 726 } 727 728 return sgl_count; 729 } 730 EXPORT_SYMBOL_GPL(af_alg_count_tsgl); 731 732 /** 733 * af_alg_pull_tsgl - Release the specified buffers from TX SGL 734 * 735 * If @dst is non-null, reassign the pages to @dst. The caller must release 736 * the pages. 737 * 738 * @sk: socket of connection to user space 739 * @used: Number of bytes to pull from TX SGL 740 * @dst: If non-NULL, buffer is reassigned to dst SGL instead of releasing. The 741 * caller must release the buffers in dst. 742 */ 743 void af_alg_pull_tsgl(struct sock *sk, size_t used, struct scatterlist *dst) 744 { 745 struct alg_sock *ask = alg_sk(sk); 746 struct af_alg_ctx *ctx = ask->private; 747 struct af_alg_tsgl *sgl; 748 struct scatterlist *sg; 749 unsigned int i, j = 0; 750 751 while (!list_empty(&ctx->tsgl_list)) { 752 sgl = list_first_entry(&ctx->tsgl_list, struct af_alg_tsgl, 753 list); 754 sg = sgl->sg; 755 756 for (i = 0; i < sgl->cur; i++) { 757 size_t plen = min_t(size_t, used, sg[i].length); 758 struct page *page = sg_page(sg + i); 759 760 if (!page) 761 continue; 762 763 /* 764 * Assumption: caller created af_alg_count_tsgl(len) 765 * SG entries in dst. 766 */ 767 if (dst && plen) { 768 /* reassign page to dst */ 769 get_page(page); 770 sg_set_page(dst + j, page, plen, sg[i].offset); 771 j++; 772 } 773 774 sg[i].length -= plen; 775 sg[i].offset += plen; 776 777 used -= plen; 778 ctx->used -= plen; 779 780 if (sg[i].length) 781 return; 782 783 put_page(page); 784 sg_assign_page(sg + i, NULL); 785 } 786 787 list_del(&sgl->list); 788 sock_kfree_s(sk, sgl, struct_size(sgl, sg, MAX_SGL_ENTS + 1)); 789 } 790 791 if (!ctx->used) 792 ctx->merge = 0; 793 ctx->init = ctx->more; 794 } 795 EXPORT_SYMBOL_GPL(af_alg_pull_tsgl); 796 797 /** 798 * af_alg_free_areq_sgls - Release TX and RX SGLs of the request 799 * 800 * @areq: Request holding the TX and RX SGL 801 */ 802 static void af_alg_free_areq_sgls(struct af_alg_async_req *areq) 803 { 804 struct sock *sk = areq->sk; 805 struct alg_sock *ask = alg_sk(sk); 806 struct af_alg_ctx *ctx = ask->private; 807 struct af_alg_rsgl *rsgl, *tmp; 808 struct scatterlist *tsgl; 809 struct scatterlist *sg; 810 unsigned int i; 811 812 list_for_each_entry_safe(rsgl, tmp, &areq->rsgl_list, list) { 813 atomic_sub(rsgl->sg_num_bytes, &ctx->rcvused); 814 af_alg_free_sg(&rsgl->sgl); 815 list_del(&rsgl->list); 816 if (rsgl != &areq->first_rsgl) 817 sock_kfree_s(sk, rsgl, sizeof(*rsgl)); 818 } 819 820 tsgl = areq->tsgl; 821 if (tsgl) { 822 for_each_sg(tsgl, sg, areq->tsgl_entries, i) { 823 if (!sg_page(sg)) 824 continue; 825 put_page(sg_page(sg)); 826 } 827 828 sock_kfree_s(sk, tsgl, areq->tsgl_entries * sizeof(*tsgl)); 829 } 830 } 831 832 /** 833 * af_alg_wait_for_wmem - wait for availability of writable memory 834 * 835 * @sk: socket of connection to user space 836 * @flags: If MSG_DONTWAIT is set, then only report if function would sleep 837 * Return: 0 when writable memory is available, < 0 upon error 838 */ 839 static int af_alg_wait_for_wmem(struct sock *sk, unsigned int flags) 840 { 841 DEFINE_WAIT_FUNC(wait, woken_wake_function); 842 int err = -ERESTARTSYS; 843 long timeout; 844 845 if (flags & MSG_DONTWAIT) 846 return -EAGAIN; 847 848 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk); 849 850 add_wait_queue(sk_sleep(sk), &wait); 851 for (;;) { 852 if (signal_pending(current)) 853 break; 854 timeout = MAX_SCHEDULE_TIMEOUT; 855 if (sk_wait_event(sk, &timeout, af_alg_writable(sk), &wait)) { 856 err = 0; 857 break; 858 } 859 } 860 remove_wait_queue(sk_sleep(sk), &wait); 861 862 return err; 863 } 864 865 /** 866 * af_alg_wmem_wakeup - wakeup caller when writable memory is available 867 * 868 * @sk: socket of connection to user space 869 */ 870 void af_alg_wmem_wakeup(struct sock *sk) 871 { 872 struct socket_wq *wq; 873 874 if (!af_alg_writable(sk)) 875 return; 876 877 rcu_read_lock(); 878 wq = rcu_dereference(sk->sk_wq); 879 if (skwq_has_sleeper(wq)) 880 wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | 881 EPOLLRDNORM | 882 EPOLLRDBAND); 883 sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN); 884 rcu_read_unlock(); 885 } 886 EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup); 887 888 /** 889 * af_alg_wait_for_data - wait for availability of TX data 890 * 891 * @sk: socket of connection to user space 892 * @flags: If MSG_DONTWAIT is set, then only report if function would sleep 893 * @min: Set to minimum request size if partial requests are allowed. 894 * Return: 0 when writable memory is available, < 0 upon error 895 */ 896 int af_alg_wait_for_data(struct sock *sk, unsigned flags, unsigned min) 897 { 898 DEFINE_WAIT_FUNC(wait, woken_wake_function); 899 struct alg_sock *ask = alg_sk(sk); 900 struct af_alg_ctx *ctx = ask->private; 901 long timeout; 902 int err = -ERESTARTSYS; 903 904 if (flags & MSG_DONTWAIT) 905 return -EAGAIN; 906 907 sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk); 908 909 add_wait_queue(sk_sleep(sk), &wait); 910 for (;;) { 911 if (signal_pending(current)) 912 break; 913 timeout = MAX_SCHEDULE_TIMEOUT; 914 if (sk_wait_event(sk, &timeout, 915 ctx->init && (!ctx->more || 916 (min && ctx->used >= min)), 917 &wait)) { 918 err = 0; 919 break; 920 } 921 } 922 remove_wait_queue(sk_sleep(sk), &wait); 923 924 sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk); 925 926 return err; 927 } 928 EXPORT_SYMBOL_GPL(af_alg_wait_for_data); 929 930 /** 931 * af_alg_data_wakeup - wakeup caller when new data can be sent to kernel 932 * 933 * @sk: socket of connection to user space 934 */ 935 static void af_alg_data_wakeup(struct sock *sk) 936 { 937 struct alg_sock *ask = alg_sk(sk); 938 struct af_alg_ctx *ctx = ask->private; 939 struct socket_wq *wq; 940 941 if (!ctx->used) 942 return; 943 944 rcu_read_lock(); 945 wq = rcu_dereference(sk->sk_wq); 946 if (skwq_has_sleeper(wq)) 947 wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT | 948 EPOLLRDNORM | 949 EPOLLRDBAND); 950 sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT); 951 rcu_read_unlock(); 952 } 953 954 /** 955 * af_alg_sendmsg - implementation of sendmsg system call handler 956 * 957 * The sendmsg system call handler obtains the user data and stores it 958 * in ctx->tsgl_list. This implies allocation of the required numbers of 959 * struct af_alg_tsgl. 960 * 961 * In addition, the ctx is filled with the information sent via CMSG. 962 * 963 * @sock: socket of connection to user space 964 * @msg: message from user space 965 * @size: size of message from user space 966 * @ivsize: the size of the IV for the cipher operation to verify that the 967 * user-space-provided IV has the right size 968 * Return: the number of copied data upon success, < 0 upon error 969 */ 970 int af_alg_sendmsg(struct socket *sock, struct msghdr *msg, size_t size, 971 unsigned int ivsize) 972 { 973 struct sock *sk = sock->sk; 974 struct alg_sock *ask = alg_sk(sk); 975 struct af_alg_ctx *ctx = ask->private; 976 struct af_alg_tsgl *sgl; 977 struct af_alg_control con = {}; 978 long copied = 0; 979 bool enc = false; 980 bool init = false; 981 int err = 0; 982 983 if (msg->msg_controllen) { 984 err = af_alg_cmsg_send(msg, &con); 985 if (err) 986 return err; 987 988 init = true; 989 switch (con.op) { 990 case ALG_OP_ENCRYPT: 991 enc = true; 992 break; 993 case ALG_OP_DECRYPT: 994 enc = false; 995 break; 996 default: 997 return -EINVAL; 998 } 999 1000 if (con.iv && con.iv->ivlen != ivsize) 1001 return -EINVAL; 1002 } 1003 1004 lock_sock(sk); 1005 if (ctx->write) { 1006 release_sock(sk); 1007 return -EBUSY; 1008 } 1009 ctx->write = true; 1010 1011 if (ctx->init && !ctx->more) { 1012 if (ctx->used) { 1013 err = -EINVAL; 1014 goto unlock; 1015 } 1016 1017 pr_info_once( 1018 "%s sent an empty control message without MSG_MORE.\n", 1019 current->comm); 1020 } 1021 ctx->init = true; 1022 1023 if (init) { 1024 ctx->enc = enc; 1025 if (con.iv) 1026 memcpy(ctx->iv, con.iv->iv, ivsize); 1027 1028 ctx->aead_assoclen = con.aead_assoclen; 1029 } 1030 1031 while (size) { 1032 struct scatterlist *sg; 1033 size_t len = size; 1034 ssize_t plen; 1035 1036 /* use the existing memory in an allocated page */ 1037 if (ctx->merge) { 1038 sgl = list_entry(ctx->tsgl_list.prev, 1039 struct af_alg_tsgl, list); 1040 sg = sgl->sg + sgl->cur - 1; 1041 len = min_t(size_t, len, 1042 PAGE_SIZE - sg->offset - sg->length); 1043 1044 err = memcpy_from_msg(page_address(sg_page(sg)) + 1045 sg->offset + sg->length, 1046 msg, len); 1047 if (err) 1048 goto unlock; 1049 1050 sg->length += len; 1051 ctx->merge = (sg->offset + sg->length) & 1052 (PAGE_SIZE - 1); 1053 1054 ctx->used += len; 1055 copied += len; 1056 size -= len; 1057 continue; 1058 } 1059 1060 ctx->merge = 0; 1061 1062 if (!af_alg_writable(sk)) { 1063 err = af_alg_wait_for_wmem(sk, msg->msg_flags); 1064 if (err) 1065 goto unlock; 1066 } 1067 1068 /* allocate a new page */ 1069 len = min_t(unsigned long, len, af_alg_sndbuf(sk)); 1070 1071 err = af_alg_alloc_tsgl(sk); 1072 if (err) 1073 goto unlock; 1074 1075 sgl = list_entry(ctx->tsgl_list.prev, struct af_alg_tsgl, 1076 list); 1077 sg = sgl->sg; 1078 if (sgl->cur) 1079 sg_unmark_end(sg + sgl->cur - 1); 1080 1081 do { 1082 struct page *pg; 1083 unsigned int i = sgl->cur; 1084 1085 plen = min_t(size_t, len, PAGE_SIZE); 1086 1087 pg = alloc_page(GFP_KERNEL); 1088 if (!pg) { 1089 err = -ENOMEM; 1090 goto unlock; 1091 } 1092 1093 sg_assign_page(sg + i, pg); 1094 1095 err = memcpy_from_msg(page_address(sg_page(sg + i)), 1096 msg, plen); 1097 if (err) { 1098 __free_page(sg_page(sg + i)); 1099 sg_assign_page(sg + i, NULL); 1100 goto unlock; 1101 } 1102 1103 sg[i].length = plen; 1104 len -= plen; 1105 ctx->used += plen; 1106 copied += plen; 1107 size -= plen; 1108 sgl->cur++; 1109 } while (len && sgl->cur < MAX_SGL_ENTS); 1110 1111 ctx->merge = plen & (PAGE_SIZE - 1); 1112 1113 if (!size) 1114 sg_mark_end(sg + sgl->cur - 1); 1115 } 1116 1117 err = 0; 1118 1119 ctx->more = msg->msg_flags & MSG_MORE; 1120 1121 unlock: 1122 af_alg_data_wakeup(sk); 1123 ctx->write = false; 1124 release_sock(sk); 1125 1126 return copied ?: err; 1127 } 1128 EXPORT_SYMBOL_GPL(af_alg_sendmsg); 1129 1130 /** 1131 * af_alg_free_resources - release resources required for crypto request 1132 * @areq: Request holding the TX and RX SGL 1133 */ 1134 void af_alg_free_resources(struct af_alg_async_req *areq) 1135 { 1136 struct sock *sk = areq->sk; 1137 struct af_alg_ctx *ctx; 1138 1139 af_alg_free_areq_sgls(areq); 1140 sock_kfree_s(sk, areq, areq->areqlen); 1141 1142 ctx = alg_sk(sk)->private; 1143 ctx->inflight = false; 1144 } 1145 EXPORT_SYMBOL_GPL(af_alg_free_resources); 1146 1147 /** 1148 * af_alg_poll - poll system call handler 1149 * @file: file pointer 1150 * @sock: socket to poll 1151 * @wait: poll_table 1152 */ 1153 __poll_t af_alg_poll(struct file *file, struct socket *sock, 1154 poll_table *wait) 1155 { 1156 struct sock *sk = sock->sk; 1157 struct alg_sock *ask = alg_sk(sk); 1158 struct af_alg_ctx *ctx = ask->private; 1159 __poll_t mask; 1160 1161 sock_poll_wait(file, sock, wait); 1162 mask = 0; 1163 1164 if (!ctx->more || ctx->used) 1165 mask |= EPOLLIN | EPOLLRDNORM; 1166 1167 if (af_alg_writable(sk)) 1168 mask |= EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND; 1169 1170 return mask; 1171 } 1172 EXPORT_SYMBOL_GPL(af_alg_poll); 1173 1174 /** 1175 * af_alg_alloc_areq - allocate struct af_alg_async_req 1176 * 1177 * @sk: socket of connection to user space 1178 * @areqlen: size of struct af_alg_async_req + crypto_*_reqsize 1179 * Return: allocated data structure or ERR_PTR upon error 1180 */ 1181 struct af_alg_async_req *af_alg_alloc_areq(struct sock *sk, 1182 unsigned int areqlen) 1183 { 1184 struct af_alg_ctx *ctx = alg_sk(sk)->private; 1185 struct af_alg_async_req *areq; 1186 1187 /* Only one request can be in flight. */ 1188 if (WARN_ON_ONCE(ctx->inflight)) 1189 return ERR_PTR(-EBUSY); 1190 1191 areq = sock_kmalloc(sk, areqlen, GFP_KERNEL); 1192 if (unlikely(!areq)) 1193 return ERR_PTR(-ENOMEM); 1194 1195 memset(areq, 0, areqlen); 1196 1197 ctx->inflight = true; 1198 1199 areq->areqlen = areqlen; 1200 areq->sk = sk; 1201 areq->first_rsgl.sgl.sgt.sgl = areq->first_rsgl.sgl.sgl; 1202 INIT_LIST_HEAD(&areq->rsgl_list); 1203 1204 return areq; 1205 } 1206 EXPORT_SYMBOL_GPL(af_alg_alloc_areq); 1207 1208 /** 1209 * af_alg_get_rsgl - create the RX SGL for the output data from the crypto 1210 * operation 1211 * 1212 * @sk: socket of connection to user space 1213 * @msg: user space message 1214 * @flags: flags used to invoke recvmsg with 1215 * @areq: instance of the cryptographic request that will hold the RX SGL 1216 * @maxsize: maximum number of bytes to be pulled from user space 1217 * @outlen: number of bytes in the RX SGL 1218 * Return: 0 on success, < 0 upon error 1219 */ 1220 int af_alg_get_rsgl(struct sock *sk, struct msghdr *msg, int flags, 1221 struct af_alg_async_req *areq, size_t maxsize, 1222 size_t *outlen) 1223 { 1224 struct alg_sock *ask = alg_sk(sk); 1225 struct af_alg_ctx *ctx = ask->private; 1226 size_t len = 0; 1227 1228 while (maxsize > len && msg_data_left(msg)) { 1229 struct af_alg_rsgl *rsgl; 1230 ssize_t err; 1231 size_t seglen; 1232 1233 /* limit the amount of readable buffers */ 1234 if (!af_alg_readable(sk)) 1235 break; 1236 1237 seglen = min_t(size_t, (maxsize - len), 1238 msg_data_left(msg)); 1239 /* Never pin more pages than the remaining RX accounting budget. */ 1240 seglen = min_t(size_t, seglen, af_alg_rcvbuf(sk)); 1241 1242 if (list_empty(&areq->rsgl_list)) { 1243 rsgl = &areq->first_rsgl; 1244 } else { 1245 rsgl = sock_kmalloc(sk, sizeof(*rsgl), GFP_KERNEL); 1246 if (unlikely(!rsgl)) 1247 return -ENOMEM; 1248 } 1249 1250 rsgl->sgl.need_unpin = 1251 iov_iter_extract_will_pin(&msg->msg_iter); 1252 rsgl->sgl.sgt.sgl = rsgl->sgl.sgl; 1253 rsgl->sgl.sgt.nents = 0; 1254 rsgl->sgl.sgt.orig_nents = 0; 1255 list_add_tail(&rsgl->list, &areq->rsgl_list); 1256 1257 sg_init_table(rsgl->sgl.sgt.sgl, ALG_MAX_PAGES); 1258 err = extract_iter_to_sg(&msg->msg_iter, seglen, &rsgl->sgl.sgt, 1259 ALG_MAX_PAGES, 0); 1260 if (err < 0) { 1261 rsgl->sg_num_bytes = 0; 1262 return err; 1263 } 1264 1265 sg_mark_end(rsgl->sgl.sgt.sgl + rsgl->sgl.sgt.nents - 1); 1266 1267 /* chain the new scatterlist with previous one */ 1268 if (areq->last_rsgl) 1269 af_alg_link_sg(&areq->last_rsgl->sgl, &rsgl->sgl); 1270 1271 areq->last_rsgl = rsgl; 1272 len += err; 1273 atomic_add(err, &ctx->rcvused); 1274 rsgl->sg_num_bytes = err; 1275 } 1276 1277 *outlen = len; 1278 return 0; 1279 } 1280 EXPORT_SYMBOL_GPL(af_alg_get_rsgl); 1281 1282 static int __init af_alg_init(void) 1283 { 1284 int err; 1285 1286 af_alg_header = register_sysctl("crypto", af_alg_table); 1287 1288 err = proto_register(&alg_proto, 0); 1289 if (err) 1290 goto out_unregister_sysctl; 1291 1292 err = sock_register(&alg_family); 1293 if (err) 1294 goto out_unregister_proto; 1295 1296 return 0; 1297 1298 out_unregister_proto: 1299 proto_unregister(&alg_proto); 1300 out_unregister_sysctl: 1301 unregister_sysctl_table(af_alg_header); 1302 return err; 1303 } 1304 1305 static void __exit af_alg_exit(void) 1306 { 1307 sock_unregister(PF_ALG); 1308 proto_unregister(&alg_proto); 1309 unregister_sysctl_table(af_alg_header); 1310 } 1311 1312 module_init(af_alg_init); 1313 module_exit(af_alg_exit); 1314 MODULE_DESCRIPTION("Crypto userspace interface"); 1315 MODULE_LICENSE("GPL"); 1316 MODULE_ALIAS_NETPROTO(AF_ALG); 1317