1 /*- 2 * Copyright (c) 2010-2022 Hans Petter Selasky 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include <sys/stdint.h> 27 #include <sys/stddef.h> 28 #include <sys/param.h> 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/conf.h> 32 #include <sys/kernel.h> 33 #include <sys/bus.h> 34 #include <sys/linker_set.h> 35 #include <sys/module.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/condvar.h> 39 #include <sys/sysctl.h> 40 #include <sys/unistd.h> 41 #include <sys/malloc.h> 42 #include <sys/priv.h> 43 #include <sys/uio.h> 44 #include <sys/poll.h> 45 #include <sys/sx.h> 46 #include <sys/rwlock.h> 47 #include <sys/queue.h> 48 #include <sys/fcntl.h> 49 #include <sys/proc.h> 50 #include <sys/vnode.h> 51 #include <sys/selinfo.h> 52 #include <sys/ptrace.h> 53 #include <sys/sysent.h> 54 55 #include <machine/bus.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 #include <vm/vm_object.h> 60 #include <vm/vm_page.h> 61 #include <vm/vm_pager.h> 62 63 #include <fs/cuse/cuse_defs.h> 64 #include <fs/cuse/cuse_ioctl.h> 65 66 /* set this define to zero to disable this feature */ 67 #define CUSE_COPY_BUFFER_MAX \ 68 CUSE_BUFFER_MAX 69 70 #define CUSE_ALLOC_PAGES_MAX \ 71 (CUSE_ALLOC_BYTES_MAX / PAGE_SIZE) 72 73 #if (CUSE_ALLOC_PAGES_MAX == 0) 74 #error "PAGE_SIZE is too big!" 75 #endif 76 77 static int 78 cuse_modevent(module_t mod, int type, void *data) 79 { 80 switch (type) { 81 case MOD_LOAD: 82 case MOD_UNLOAD: 83 return (0); 84 default: 85 return (EOPNOTSUPP); 86 } 87 } 88 89 static moduledata_t cuse_mod = { 90 .name = "cuse", 91 .evhand = &cuse_modevent, 92 }; 93 94 DECLARE_MODULE(cuse, cuse_mod, SI_SUB_DEVFS, SI_ORDER_FIRST); 95 MODULE_VERSION(cuse, 1); 96 97 /* 98 * Prevent cuse4bsd.ko and cuse.ko from loading at the same time by 99 * declaring support for the cuse4bsd interface in cuse.ko: 100 */ 101 MODULE_VERSION(cuse4bsd, 1); 102 103 #ifdef FEATURE 104 FEATURE(cuse, "Userspace character devices"); 105 #endif 106 107 struct cuse_command; 108 struct cuse_server; 109 struct cuse_client; 110 111 struct cuse_client_command { 112 TAILQ_ENTRY(cuse_client_command) entry; 113 struct cuse_command sub; 114 struct sx sx; 115 struct cv cv; 116 struct thread *entered; 117 struct cuse_client *client; 118 struct proc *proc_curr; 119 int proc_refs; 120 int got_signal; 121 int error; 122 int command; 123 }; 124 125 struct cuse_memory { 126 TAILQ_ENTRY(cuse_memory) entry; 127 vm_object_t object; 128 uint32_t page_count; 129 uint32_t alloc_nr; 130 }; 131 132 struct cuse_server_dev { 133 TAILQ_ENTRY(cuse_server_dev) entry; 134 struct cuse_server *server; 135 struct cdev *kern_dev; 136 struct cuse_dev *user_dev; 137 }; 138 139 struct cuse_server { 140 TAILQ_ENTRY(cuse_server) entry; 141 TAILQ_HEAD(, cuse_client_command) head; 142 TAILQ_HEAD(, cuse_server_dev) hdev; 143 TAILQ_HEAD(, cuse_client) hcli; 144 TAILQ_HEAD(, cuse_memory) hmem; 145 struct mtx mtx; 146 struct cv cv; 147 struct selinfo selinfo; 148 pid_t pid; 149 int is_closing; 150 int refs; 151 }; 152 153 struct cuse_client { 154 TAILQ_ENTRY(cuse_client) entry; 155 TAILQ_ENTRY(cuse_client) entry_ref; 156 struct cuse_client_command cmds[CUSE_CMD_MAX]; 157 struct cuse_server *server; 158 struct cuse_server_dev *server_dev; 159 160 uintptr_t read_base; 161 uintptr_t write_base; 162 int read_length; 163 int write_length; 164 uint8_t read_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4); 165 uint8_t write_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4); 166 uint8_t ioctl_buffer[CUSE_BUFFER_MAX] __aligned(4); 167 168 int fflags; /* file flags */ 169 int cflags; /* client flags */ 170 #define CUSE_CLI_IS_CLOSING 0x01 171 #define CUSE_CLI_KNOTE_NEED_READ 0x02 172 #define CUSE_CLI_KNOTE_NEED_WRITE 0x04 173 #define CUSE_CLI_KNOTE_HAS_READ 0x08 174 #define CUSE_CLI_KNOTE_HAS_WRITE 0x10 175 }; 176 177 #define CUSE_CLIENT_CLOSING(pcc) \ 178 ((pcc)->cflags & CUSE_CLI_IS_CLOSING) 179 180 static MALLOC_DEFINE(M_CUSE, "cuse", "CUSE memory"); 181 182 static TAILQ_HEAD(, cuse_server) cuse_server_head; 183 static struct mtx cuse_global_mtx; 184 static struct cdev *cuse_dev; 185 static struct cuse_server *cuse_alloc_unit[CUSE_DEVICES_MAX]; 186 static int cuse_alloc_unit_id[CUSE_DEVICES_MAX]; 187 188 static void cuse_server_wakeup_all_client_locked(struct cuse_server *pcs); 189 static void cuse_client_kqfilter_read_detach(struct knote *kn); 190 static void cuse_client_kqfilter_write_detach(struct knote *kn); 191 static int cuse_client_kqfilter_read_event(struct knote *kn, long hint); 192 static int cuse_client_kqfilter_write_event(struct knote *kn, long hint); 193 194 static const struct filterops cuse_client_kqfilter_read_ops = { 195 .f_isfd = 1, 196 .f_detach = cuse_client_kqfilter_read_detach, 197 .f_event = cuse_client_kqfilter_read_event, 198 .f_copy = knote_triv_copy, 199 }; 200 201 static const struct filterops cuse_client_kqfilter_write_ops = { 202 .f_isfd = 1, 203 .f_detach = cuse_client_kqfilter_write_detach, 204 .f_event = cuse_client_kqfilter_write_event, 205 .f_copy = knote_triv_copy, 206 }; 207 208 static d_open_t cuse_client_open; 209 static d_close_t cuse_client_close; 210 static d_ioctl_t cuse_client_ioctl; 211 static d_read_t cuse_client_read; 212 static d_write_t cuse_client_write; 213 static d_poll_t cuse_client_poll; 214 static d_mmap_single_t cuse_client_mmap_single; 215 static d_kqfilter_t cuse_client_kqfilter; 216 217 static struct cdevsw cuse_client_devsw = { 218 .d_version = D_VERSION, 219 .d_open = cuse_client_open, 220 .d_close = cuse_client_close, 221 .d_ioctl = cuse_client_ioctl, 222 .d_name = "cuse_client", 223 .d_flags = D_TRACKCLOSE, 224 .d_read = cuse_client_read, 225 .d_write = cuse_client_write, 226 .d_poll = cuse_client_poll, 227 .d_mmap_single = cuse_client_mmap_single, 228 .d_kqfilter = cuse_client_kqfilter, 229 }; 230 231 static d_open_t cuse_server_open; 232 static d_close_t cuse_server_close; 233 static d_ioctl_t cuse_server_ioctl; 234 static d_read_t cuse_server_read; 235 static d_write_t cuse_server_write; 236 static d_poll_t cuse_server_poll; 237 static d_mmap_single_t cuse_server_mmap_single; 238 239 static struct cdevsw cuse_server_devsw = { 240 .d_version = D_VERSION, 241 .d_open = cuse_server_open, 242 .d_close = cuse_server_close, 243 .d_ioctl = cuse_server_ioctl, 244 .d_name = "cuse_server", 245 .d_flags = D_TRACKCLOSE, 246 .d_read = cuse_server_read, 247 .d_write = cuse_server_write, 248 .d_poll = cuse_server_poll, 249 .d_mmap_single = cuse_server_mmap_single, 250 }; 251 252 static void cuse_client_is_closing(struct cuse_client *); 253 static int cuse_free_unit_by_id_locked(struct cuse_server *, int); 254 255 static void 256 cuse_global_lock(void) 257 { 258 mtx_lock(&cuse_global_mtx); 259 } 260 261 static void 262 cuse_global_unlock(void) 263 { 264 mtx_unlock(&cuse_global_mtx); 265 } 266 267 static void 268 cuse_server_lock(struct cuse_server *pcs) 269 { 270 mtx_lock(&pcs->mtx); 271 } 272 273 static void 274 cuse_server_unlock(struct cuse_server *pcs) 275 { 276 mtx_unlock(&pcs->mtx); 277 } 278 279 static bool 280 cuse_server_is_locked(struct cuse_server *pcs) 281 { 282 return (mtx_owned(&pcs->mtx)); 283 } 284 285 static void 286 cuse_cmd_lock(struct cuse_client_command *pccmd) 287 { 288 sx_xlock(&pccmd->sx); 289 } 290 291 static void 292 cuse_cmd_unlock(struct cuse_client_command *pccmd) 293 { 294 sx_xunlock(&pccmd->sx); 295 } 296 297 static void 298 cuse_kern_init(void *arg) 299 { 300 TAILQ_INIT(&cuse_server_head); 301 302 mtx_init(&cuse_global_mtx, "cuse-global-mtx", NULL, MTX_DEF); 303 304 cuse_dev = make_dev(&cuse_server_devsw, 0, 305 UID_ROOT, GID_OPERATOR, 0600, "cuse"); 306 307 printf("Cuse v%d.%d.%d @ /dev/cuse\n", 308 (CUSE_VERSION >> 16) & 0xFF, (CUSE_VERSION >> 8) & 0xFF, 309 (CUSE_VERSION >> 0) & 0xFF); 310 } 311 SYSINIT(cuse_kern_init, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_init, NULL); 312 313 static void 314 cuse_kern_uninit(void *arg) 315 { 316 void *ptr; 317 318 while (1) { 319 printf("Cuse: Please exit all /dev/cuse instances " 320 "and processes which have used this device.\n"); 321 322 pause("DRAIN", 2 * hz); 323 324 cuse_global_lock(); 325 ptr = TAILQ_FIRST(&cuse_server_head); 326 cuse_global_unlock(); 327 328 if (ptr == NULL) 329 break; 330 } 331 332 if (cuse_dev != NULL) 333 destroy_dev(cuse_dev); 334 335 mtx_destroy(&cuse_global_mtx); 336 } 337 SYSUNINIT(cuse_kern_uninit, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_uninit, NULL); 338 339 static int 340 cuse_server_get(struct cuse_server **ppcs) 341 { 342 struct cuse_server *pcs; 343 int error; 344 345 error = devfs_get_cdevpriv((void **)&pcs); 346 if (error != 0) { 347 *ppcs = NULL; 348 return (error); 349 } 350 if (pcs->is_closing) { 351 *ppcs = NULL; 352 return (EINVAL); 353 } 354 *ppcs = pcs; 355 return (0); 356 } 357 358 static void 359 cuse_server_is_closing(struct cuse_server *pcs) 360 { 361 struct cuse_client *pcc; 362 363 if (pcs->is_closing) 364 return; 365 366 pcs->is_closing = 1; 367 368 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 369 cuse_client_is_closing(pcc); 370 } 371 } 372 373 static struct cuse_client_command * 374 cuse_server_find_command(struct cuse_server *pcs, struct thread *td) 375 { 376 struct cuse_client *pcc; 377 int n; 378 379 if (pcs->is_closing) 380 goto done; 381 382 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 383 if (CUSE_CLIENT_CLOSING(pcc)) 384 continue; 385 for (n = 0; n != CUSE_CMD_MAX; n++) { 386 if (pcc->cmds[n].entered == td) 387 return (&pcc->cmds[n]); 388 } 389 } 390 done: 391 return (NULL); 392 } 393 394 static void 395 cuse_str_filter(char *ptr) 396 { 397 int c; 398 399 while (((c = *ptr) != 0)) { 400 if ((c >= 'a') && (c <= 'z')) { 401 ptr++; 402 continue; 403 } 404 if ((c >= 'A') && (c <= 'Z')) { 405 ptr++; 406 continue; 407 } 408 if ((c >= '0') && (c <= '9')) { 409 ptr++; 410 continue; 411 } 412 if ((c == '.') || (c == '_') || (c == '/')) { 413 ptr++; 414 continue; 415 } 416 *ptr = '_'; 417 418 ptr++; 419 } 420 } 421 422 static int 423 cuse_convert_error(int error) 424 { 425 ; /* indent fix */ 426 switch (error) { 427 case CUSE_ERR_NONE: 428 return (0); 429 case CUSE_ERR_BUSY: 430 return (EBUSY); 431 case CUSE_ERR_WOULDBLOCK: 432 return (EWOULDBLOCK); 433 case CUSE_ERR_INVALID: 434 return (EINVAL); 435 case CUSE_ERR_NO_MEMORY: 436 return (ENOMEM); 437 case CUSE_ERR_FAULT: 438 return (EFAULT); 439 case CUSE_ERR_SIGNAL: 440 return (EINTR); 441 case CUSE_ERR_NO_DEVICE: 442 return (ENODEV); 443 default: 444 return (ENXIO); 445 } 446 } 447 448 static void 449 cuse_vm_memory_free(struct cuse_memory *mem) 450 { 451 /* last user is gone - free */ 452 vm_object_deallocate(mem->object); 453 454 /* free CUSE memory */ 455 free(mem, M_CUSE); 456 } 457 458 static int 459 cuse_server_alloc_memory(struct cuse_server *pcs, uint32_t alloc_nr, 460 uint32_t page_count) 461 { 462 struct cuse_memory *temp; 463 struct cuse_memory *mem; 464 vm_object_t object; 465 int error; 466 467 mem = malloc(sizeof(*mem), M_CUSE, M_WAITOK | M_ZERO); 468 469 object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * page_count, 470 VM_PROT_DEFAULT, 0, curthread->td_ucred); 471 if (object == NULL) { 472 error = ENOMEM; 473 goto error_0; 474 } 475 476 cuse_server_lock(pcs); 477 /* check if allocation number already exists */ 478 TAILQ_FOREACH(temp, &pcs->hmem, entry) { 479 if (temp->alloc_nr == alloc_nr) 480 break; 481 } 482 if (temp != NULL) { 483 cuse_server_unlock(pcs); 484 error = EBUSY; 485 goto error_1; 486 } 487 mem->object = object; 488 mem->page_count = page_count; 489 mem->alloc_nr = alloc_nr; 490 TAILQ_INSERT_TAIL(&pcs->hmem, mem, entry); 491 cuse_server_unlock(pcs); 492 493 return (0); 494 495 error_1: 496 vm_object_deallocate(object); 497 error_0: 498 free(mem, M_CUSE); 499 return (error); 500 } 501 502 static int 503 cuse_server_free_memory(struct cuse_server *pcs, uint32_t alloc_nr) 504 { 505 struct cuse_memory *mem; 506 507 cuse_server_lock(pcs); 508 TAILQ_FOREACH(mem, &pcs->hmem, entry) { 509 if (mem->alloc_nr == alloc_nr) 510 break; 511 } 512 if (mem == NULL) { 513 cuse_server_unlock(pcs); 514 return (EINVAL); 515 } 516 TAILQ_REMOVE(&pcs->hmem, mem, entry); 517 cuse_server_unlock(pcs); 518 519 cuse_vm_memory_free(mem); 520 521 return (0); 522 } 523 524 static int 525 cuse_client_get(struct cuse_client **ppcc) 526 { 527 struct cuse_client *pcc; 528 int error; 529 530 /* try to get private data */ 531 error = devfs_get_cdevpriv((void **)&pcc); 532 if (error != 0) { 533 *ppcc = NULL; 534 return (error); 535 } 536 if (CUSE_CLIENT_CLOSING(pcc) || pcc->server->is_closing) { 537 *ppcc = NULL; 538 return (EINVAL); 539 } 540 *ppcc = pcc; 541 return (0); 542 } 543 544 static void 545 cuse_client_is_closing(struct cuse_client *pcc) 546 { 547 struct cuse_client_command *pccmd; 548 uint32_t n; 549 550 if (CUSE_CLIENT_CLOSING(pcc)) 551 return; 552 553 pcc->cflags |= CUSE_CLI_IS_CLOSING; 554 pcc->server_dev = NULL; 555 556 for (n = 0; n != CUSE_CMD_MAX; n++) { 557 pccmd = &pcc->cmds[n]; 558 559 if (pccmd->entry.tqe_prev != NULL) { 560 TAILQ_REMOVE(&pcc->server->head, pccmd, entry); 561 pccmd->entry.tqe_prev = NULL; 562 } 563 cv_broadcast(&pccmd->cv); 564 } 565 } 566 567 static void 568 cuse_client_send_command_locked(struct cuse_client_command *pccmd, 569 uintptr_t data_ptr, unsigned long arg, int fflags, int ioflag) 570 { 571 unsigned long cuse_fflags = 0; 572 struct cuse_server *pcs; 573 574 if (fflags & FREAD) 575 cuse_fflags |= CUSE_FFLAG_READ; 576 577 if (fflags & FWRITE) 578 cuse_fflags |= CUSE_FFLAG_WRITE; 579 580 if (ioflag & IO_NDELAY) 581 cuse_fflags |= CUSE_FFLAG_NONBLOCK; 582 #if defined(__LP64__) 583 if (SV_CURPROC_FLAG(SV_ILP32)) 584 cuse_fflags |= CUSE_FFLAG_COMPAT32; 585 #endif 586 pccmd->sub.fflags = cuse_fflags; 587 pccmd->sub.data_pointer = data_ptr; 588 pccmd->sub.argument = arg; 589 590 pcs = pccmd->client->server; 591 592 if ((pccmd->entry.tqe_prev == NULL) && 593 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) && 594 (pcs->is_closing == 0)) { 595 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry); 596 cv_signal(&pcs->cv); 597 } 598 } 599 600 static void 601 cuse_client_got_signal(struct cuse_client_command *pccmd) 602 { 603 struct cuse_server *pcs; 604 605 pccmd->got_signal = 1; 606 607 pccmd = &pccmd->client->cmds[CUSE_CMD_SIGNAL]; 608 609 pcs = pccmd->client->server; 610 611 if ((pccmd->entry.tqe_prev == NULL) && 612 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) && 613 (pcs->is_closing == 0)) { 614 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry); 615 cv_signal(&pcs->cv); 616 } 617 } 618 619 static int 620 cuse_client_receive_command_locked(struct cuse_client_command *pccmd, 621 uint8_t *arg_ptr, uint32_t arg_len) 622 { 623 struct cuse_server *pcs; 624 int error; 625 626 pcs = pccmd->client->server; 627 error = 0; 628 629 pccmd->proc_curr = curthread->td_proc; 630 631 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) { 632 error = CUSE_ERR_OTHER; 633 goto done; 634 } 635 while (pccmd->command == CUSE_CMD_NONE) { 636 if (error != 0) { 637 cv_wait(&pccmd->cv, &pcs->mtx); 638 } else { 639 error = cv_wait_sig(&pccmd->cv, &pcs->mtx); 640 641 if (error != 0) 642 cuse_client_got_signal(pccmd); 643 } 644 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) { 645 error = CUSE_ERR_OTHER; 646 goto done; 647 } 648 } 649 650 error = pccmd->error; 651 pccmd->command = CUSE_CMD_NONE; 652 cv_signal(&pccmd->cv); 653 654 done: 655 656 /* wait until all process references are gone */ 657 658 pccmd->proc_curr = NULL; 659 660 while (pccmd->proc_refs != 0) 661 cv_wait(&pccmd->cv, &pcs->mtx); 662 663 return (error); 664 } 665 666 /*------------------------------------------------------------------------* 667 * CUSE SERVER PART 668 *------------------------------------------------------------------------*/ 669 670 static void 671 cuse_server_free_dev(struct cuse_server_dev *pcsd) 672 { 673 struct cuse_server *pcs; 674 struct cuse_client *pcc; 675 676 /* get server pointer */ 677 pcs = pcsd->server; 678 679 /* prevent creation of more devices */ 680 cuse_server_lock(pcs); 681 if (pcsd->kern_dev != NULL) 682 pcsd->kern_dev->si_drv1 = NULL; 683 684 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 685 if (pcc->server_dev == pcsd) 686 cuse_client_is_closing(pcc); 687 } 688 cuse_server_unlock(pcs); 689 690 /* destroy device, if any */ 691 if (pcsd->kern_dev != NULL) { 692 /* destroy device synchronously */ 693 destroy_dev(pcsd->kern_dev); 694 } 695 free(pcsd, M_CUSE); 696 } 697 698 static void 699 cuse_server_unref(struct cuse_server *pcs) 700 { 701 struct cuse_server_dev *pcsd; 702 struct cuse_memory *mem; 703 704 cuse_server_lock(pcs); 705 if (--(pcs->refs) != 0) { 706 cuse_server_unlock(pcs); 707 return; 708 } 709 cuse_server_is_closing(pcs); 710 /* final client wakeup, if any */ 711 cuse_server_wakeup_all_client_locked(pcs); 712 713 cuse_global_lock(); 714 TAILQ_REMOVE(&cuse_server_head, pcs, entry); 715 cuse_global_unlock(); 716 717 while ((pcsd = TAILQ_FIRST(&pcs->hdev)) != NULL) { 718 TAILQ_REMOVE(&pcs->hdev, pcsd, entry); 719 cuse_server_unlock(pcs); 720 cuse_server_free_dev(pcsd); 721 cuse_server_lock(pcs); 722 } 723 724 cuse_free_unit_by_id_locked(pcs, -1); 725 726 while ((mem = TAILQ_FIRST(&pcs->hmem)) != NULL) { 727 TAILQ_REMOVE(&pcs->hmem, mem, entry); 728 cuse_server_unlock(pcs); 729 cuse_vm_memory_free(mem); 730 cuse_server_lock(pcs); 731 } 732 733 knlist_clear(&pcs->selinfo.si_note, 1); 734 knlist_destroy(&pcs->selinfo.si_note); 735 736 cuse_server_unlock(pcs); 737 738 seldrain(&pcs->selinfo); 739 740 cv_destroy(&pcs->cv); 741 742 mtx_destroy(&pcs->mtx); 743 744 free(pcs, M_CUSE); 745 } 746 747 static int 748 cuse_server_do_close(struct cuse_server *pcs) 749 { 750 int retval; 751 752 cuse_server_lock(pcs); 753 cuse_server_is_closing(pcs); 754 /* final client wakeup, if any */ 755 cuse_server_wakeup_all_client_locked(pcs); 756 757 knlist_clear(&pcs->selinfo.si_note, 1); 758 759 retval = pcs->refs; 760 cuse_server_unlock(pcs); 761 762 return (retval); 763 } 764 765 static void 766 cuse_server_free(void *arg) 767 { 768 struct cuse_server *pcs = arg; 769 770 /* 771 * The final server unref should be done by the server thread 772 * to prevent deadlock in the client cdevpriv destructor, 773 * which cannot destroy itself. 774 */ 775 while (cuse_server_do_close(pcs) != 1) 776 pause("W", hz); 777 778 /* drop final refcount */ 779 cuse_server_unref(pcs); 780 } 781 782 static int 783 cuse_server_open(struct cdev *dev, int fflags, int devtype, struct thread *td) 784 { 785 struct cuse_server *pcs; 786 787 pcs = malloc(sizeof(*pcs), M_CUSE, M_WAITOK | M_ZERO); 788 789 if (devfs_set_cdevpriv(pcs, &cuse_server_free)) { 790 printf("Cuse: Cannot set cdevpriv.\n"); 791 free(pcs, M_CUSE); 792 return (ENOMEM); 793 } 794 /* store current process ID */ 795 pcs->pid = curproc->p_pid; 796 797 TAILQ_INIT(&pcs->head); 798 TAILQ_INIT(&pcs->hdev); 799 TAILQ_INIT(&pcs->hcli); 800 TAILQ_INIT(&pcs->hmem); 801 802 cv_init(&pcs->cv, "cuse-server-cv"); 803 804 mtx_init(&pcs->mtx, "cuse-server-mtx", NULL, MTX_DEF); 805 806 knlist_init_mtx(&pcs->selinfo.si_note, &pcs->mtx); 807 808 cuse_global_lock(); 809 pcs->refs++; 810 TAILQ_INSERT_TAIL(&cuse_server_head, pcs, entry); 811 cuse_global_unlock(); 812 813 return (0); 814 } 815 816 static int 817 cuse_server_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 818 { 819 struct cuse_server *pcs; 820 821 if (cuse_server_get(&pcs) == 0) 822 cuse_server_do_close(pcs); 823 824 return (0); 825 } 826 827 static int 828 cuse_server_read(struct cdev *dev, struct uio *uio, int ioflag) 829 { 830 return (ENXIO); 831 } 832 833 static int 834 cuse_server_write(struct cdev *dev, struct uio *uio, int ioflag) 835 { 836 return (ENXIO); 837 } 838 839 static int 840 cuse_server_ioctl_copy_locked(struct cuse_server *pcs, 841 struct cuse_client_command *pccmd, 842 struct cuse_data_chunk *pchk, bool isread) 843 { 844 struct proc *p_proc; 845 uint32_t offset; 846 int error; 847 848 offset = pchk->peer_ptr - CUSE_BUF_MIN_PTR; 849 850 if (pchk->length > CUSE_BUFFER_MAX) 851 return (EFAULT); 852 853 if (offset >= CUSE_BUFFER_MAX) 854 return (EFAULT); 855 856 if ((offset + pchk->length) > CUSE_BUFFER_MAX) 857 return (EFAULT); 858 859 p_proc = pccmd->proc_curr; 860 if (p_proc == NULL) 861 return (ENXIO); 862 863 if (pccmd->proc_refs < 0) 864 return (ENOMEM); 865 866 pccmd->proc_refs++; 867 868 cuse_server_unlock(pcs); 869 870 if (!isread) { 871 error = copyin( 872 (void *)pchk->local_ptr, 873 pccmd->client->ioctl_buffer + offset, 874 pchk->length); 875 } else { 876 error = copyout( 877 pccmd->client->ioctl_buffer + offset, 878 (void *)pchk->local_ptr, 879 pchk->length); 880 } 881 882 cuse_server_lock(pcs); 883 884 pccmd->proc_refs--; 885 886 if (pccmd->proc_curr == NULL) 887 cv_signal(&pccmd->cv); 888 889 return (error); 890 } 891 892 static int 893 cuse_proc2proc_copy(struct proc *proc_s, vm_offset_t data_s, 894 struct proc *proc_d, vm_offset_t data_d, size_t len) 895 { 896 struct thread *td; 897 struct proc *proc_cur; 898 int error; 899 900 td = curthread; 901 proc_cur = td->td_proc; 902 903 if (proc_cur == proc_d) { 904 struct iovec iov = { 905 .iov_base = (caddr_t)data_d, 906 .iov_len = len, 907 }; 908 struct uio uio = { 909 .uio_iov = &iov, 910 .uio_iovcnt = 1, 911 .uio_offset = (off_t)data_s, 912 .uio_resid = len, 913 .uio_segflg = UIO_USERSPACE, 914 .uio_rw = UIO_READ, 915 .uio_td = td, 916 }; 917 918 PHOLD(proc_s); 919 error = proc_rwmem(proc_s, &uio); 920 PRELE(proc_s); 921 922 } else if (proc_cur == proc_s) { 923 struct iovec iov = { 924 .iov_base = (caddr_t)data_s, 925 .iov_len = len, 926 }; 927 struct uio uio = { 928 .uio_iov = &iov, 929 .uio_iovcnt = 1, 930 .uio_offset = (off_t)data_d, 931 .uio_resid = len, 932 .uio_segflg = UIO_USERSPACE, 933 .uio_rw = UIO_WRITE, 934 .uio_td = td, 935 }; 936 937 PHOLD(proc_d); 938 error = proc_rwmem(proc_d, &uio); 939 PRELE(proc_d); 940 } else { 941 error = EINVAL; 942 } 943 return (error); 944 } 945 946 static int 947 cuse_server_data_copy_locked(struct cuse_server *pcs, 948 struct cuse_client_command *pccmd, 949 struct cuse_data_chunk *pchk, bool isread) 950 { 951 struct proc *p_proc; 952 int error; 953 954 p_proc = pccmd->proc_curr; 955 if (p_proc == NULL) 956 return (ENXIO); 957 958 if (pccmd->proc_refs < 0) 959 return (ENOMEM); 960 961 pccmd->proc_refs++; 962 963 cuse_server_unlock(pcs); 964 965 if (!isread) { 966 error = cuse_proc2proc_copy( 967 curthread->td_proc, pchk->local_ptr, 968 p_proc, pchk->peer_ptr, 969 pchk->length); 970 } else { 971 error = cuse_proc2proc_copy( 972 p_proc, pchk->peer_ptr, 973 curthread->td_proc, pchk->local_ptr, 974 pchk->length); 975 } 976 977 cuse_server_lock(pcs); 978 979 pccmd->proc_refs--; 980 981 if (pccmd->proc_curr == NULL) 982 cv_signal(&pccmd->cv); 983 984 return (error); 985 } 986 987 static int 988 cuse_server_data_copy_optimized_locked(struct cuse_server *pcs, 989 struct cuse_client_command *pccmd, 990 struct cuse_data_chunk *pchk, bool isread) 991 { 992 uintptr_t offset; 993 int error; 994 995 /* 996 * Check if data is stored locally to avoid accessing 997 * other process's data space: 998 */ 999 if (isread) { 1000 offset = pchk->peer_ptr - pccmd->client->write_base; 1001 1002 if (offset < (uintptr_t)pccmd->client->write_length && 1003 pchk->length <= (unsigned long)pccmd->client->write_length && 1004 offset + pchk->length <= (uintptr_t)pccmd->client->write_length) { 1005 cuse_server_unlock(pcs); 1006 error = copyout(pccmd->client->write_buffer + offset, 1007 (void *)pchk->local_ptr, pchk->length); 1008 goto done; 1009 } 1010 } else { 1011 offset = pchk->peer_ptr - pccmd->client->read_base; 1012 1013 if (offset < (uintptr_t)pccmd->client->read_length && 1014 pchk->length <= (unsigned long)pccmd->client->read_length && 1015 offset + pchk->length <= (uintptr_t)pccmd->client->read_length) { 1016 cuse_server_unlock(pcs); 1017 error = copyin((void *)pchk->local_ptr, 1018 pccmd->client->read_buffer + offset, pchk->length); 1019 goto done; 1020 } 1021 } 1022 1023 /* use process to process copy function */ 1024 error = cuse_server_data_copy_locked(pcs, pccmd, pchk, isread); 1025 done: 1026 return (error); 1027 } 1028 1029 static int 1030 cuse_alloc_unit_by_id_locked(struct cuse_server *pcs, int id) 1031 { 1032 int n; 1033 int x = 0; 1034 int match; 1035 1036 do { 1037 for (match = n = 0; n != CUSE_DEVICES_MAX; n++) { 1038 if (cuse_alloc_unit[n] != NULL) { 1039 if ((cuse_alloc_unit_id[n] ^ id) & CUSE_ID_MASK) 1040 continue; 1041 if ((cuse_alloc_unit_id[n] & ~CUSE_ID_MASK) == x) { 1042 x++; 1043 match = 1; 1044 } 1045 } 1046 } 1047 } while (match); 1048 1049 if (x < 256) { 1050 for (n = 0; n != CUSE_DEVICES_MAX; n++) { 1051 if (cuse_alloc_unit[n] == NULL) { 1052 cuse_alloc_unit[n] = pcs; 1053 cuse_alloc_unit_id[n] = id | x; 1054 return (x); 1055 } 1056 } 1057 } 1058 return (-1); 1059 } 1060 1061 static void 1062 cuse_server_wakeup_locked(struct cuse_server *pcs) 1063 { 1064 selwakeup(&pcs->selinfo); 1065 KNOTE_LOCKED(&pcs->selinfo.si_note, 0); 1066 } 1067 1068 static void 1069 cuse_server_wakeup_all_client_locked(struct cuse_server *pcs) 1070 { 1071 struct cuse_client *pcc; 1072 1073 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 1074 pcc->cflags |= (CUSE_CLI_KNOTE_NEED_READ | 1075 CUSE_CLI_KNOTE_NEED_WRITE); 1076 } 1077 cuse_server_wakeup_locked(pcs); 1078 } 1079 1080 static int 1081 cuse_free_unit_by_id_locked(struct cuse_server *pcs, int id) 1082 { 1083 int n; 1084 int found = 0; 1085 1086 for (n = 0; n != CUSE_DEVICES_MAX; n++) { 1087 if (cuse_alloc_unit[n] == pcs) { 1088 if (cuse_alloc_unit_id[n] == id || id == -1) { 1089 cuse_alloc_unit[n] = NULL; 1090 cuse_alloc_unit_id[n] = 0; 1091 found = 1; 1092 } 1093 } 1094 } 1095 1096 return (found ? 0 : EINVAL); 1097 } 1098 1099 static int 1100 cuse_server_ioctl(struct cdev *dev, unsigned long cmd, 1101 caddr_t data, int fflag, struct thread *td) 1102 { 1103 struct cuse_server *pcs; 1104 int error; 1105 1106 error = cuse_server_get(&pcs); 1107 if (error != 0) 1108 return (error); 1109 1110 switch (cmd) { 1111 struct cuse_client_command *pccmd; 1112 struct cuse_client *pcc; 1113 struct cuse_command *pcmd; 1114 struct cuse_alloc_info *pai; 1115 struct cuse_create_dev *pcd; 1116 struct cuse_server_dev *pcsd; 1117 struct cuse_data_chunk *pchk; 1118 int n; 1119 1120 case CUSE_IOCTL_GET_COMMAND: 1121 pcmd = (void *)data; 1122 1123 cuse_server_lock(pcs); 1124 1125 while ((pccmd = TAILQ_FIRST(&pcs->head)) == NULL) { 1126 error = cv_wait_sig(&pcs->cv, &pcs->mtx); 1127 1128 if (pcs->is_closing) 1129 error = ENXIO; 1130 1131 if (error) { 1132 cuse_server_unlock(pcs); 1133 return (error); 1134 } 1135 } 1136 1137 TAILQ_REMOVE(&pcs->head, pccmd, entry); 1138 pccmd->entry.tqe_prev = NULL; 1139 1140 pccmd->entered = curthread; 1141 1142 *pcmd = pccmd->sub; 1143 1144 cuse_server_unlock(pcs); 1145 1146 break; 1147 1148 case CUSE_IOCTL_SYNC_COMMAND: 1149 1150 cuse_server_lock(pcs); 1151 while ((pccmd = cuse_server_find_command(pcs, curthread)) != NULL) { 1152 /* send sync command */ 1153 pccmd->entered = NULL; 1154 pccmd->error = *(int *)data; 1155 pccmd->command = CUSE_CMD_SYNC; 1156 1157 /* signal peer, if any */ 1158 cv_signal(&pccmd->cv); 1159 } 1160 cuse_server_unlock(pcs); 1161 1162 break; 1163 1164 case CUSE_IOCTL_ALLOC_UNIT: 1165 1166 cuse_server_lock(pcs); 1167 n = cuse_alloc_unit_by_id_locked(pcs, 1168 CUSE_ID_DEFAULT(0)); 1169 cuse_server_unlock(pcs); 1170 1171 if (n < 0) 1172 error = ENOMEM; 1173 else 1174 *(int *)data = n; 1175 break; 1176 1177 case CUSE_IOCTL_ALLOC_UNIT_BY_ID: 1178 1179 n = *(int *)data; 1180 1181 n = (n & CUSE_ID_MASK); 1182 1183 cuse_server_lock(pcs); 1184 n = cuse_alloc_unit_by_id_locked(pcs, n); 1185 cuse_server_unlock(pcs); 1186 1187 if (n < 0) 1188 error = ENOMEM; 1189 else 1190 *(int *)data = n; 1191 break; 1192 1193 case CUSE_IOCTL_FREE_UNIT: 1194 1195 n = *(int *)data; 1196 1197 n = CUSE_ID_DEFAULT(n); 1198 1199 cuse_server_lock(pcs); 1200 error = cuse_free_unit_by_id_locked(pcs, n); 1201 cuse_server_unlock(pcs); 1202 break; 1203 1204 case CUSE_IOCTL_FREE_UNIT_BY_ID: 1205 1206 n = *(int *)data; 1207 1208 cuse_server_lock(pcs); 1209 error = cuse_free_unit_by_id_locked(pcs, n); 1210 cuse_server_unlock(pcs); 1211 break; 1212 1213 case CUSE_IOCTL_ALLOC_MEMORY: 1214 1215 pai = (void *)data; 1216 1217 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) { 1218 error = ENOMEM; 1219 break; 1220 } 1221 if (pai->page_count > CUSE_ALLOC_PAGES_MAX) { 1222 error = ENOMEM; 1223 break; 1224 } 1225 error = cuse_server_alloc_memory(pcs, 1226 pai->alloc_nr, pai->page_count); 1227 break; 1228 1229 case CUSE_IOCTL_FREE_MEMORY: 1230 pai = (void *)data; 1231 1232 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) { 1233 error = ENOMEM; 1234 break; 1235 } 1236 error = cuse_server_free_memory(pcs, pai->alloc_nr); 1237 break; 1238 1239 case CUSE_IOCTL_GET_SIG: 1240 1241 cuse_server_lock(pcs); 1242 pccmd = cuse_server_find_command(pcs, curthread); 1243 1244 if (pccmd != NULL) { 1245 n = pccmd->got_signal; 1246 pccmd->got_signal = 0; 1247 } else { 1248 n = 0; 1249 } 1250 cuse_server_unlock(pcs); 1251 1252 *(int *)data = n; 1253 1254 break; 1255 1256 case CUSE_IOCTL_SET_PFH: 1257 1258 cuse_server_lock(pcs); 1259 pccmd = cuse_server_find_command(pcs, curthread); 1260 1261 if (pccmd != NULL) { 1262 pcc = pccmd->client; 1263 for (n = 0; n != CUSE_CMD_MAX; n++) { 1264 pcc->cmds[n].sub.per_file_handle = *(uintptr_t *)data; 1265 } 1266 } else { 1267 error = ENXIO; 1268 } 1269 cuse_server_unlock(pcs); 1270 break; 1271 1272 case CUSE_IOCTL_CREATE_DEV: 1273 1274 error = priv_check(curthread, PRIV_DRIVER); 1275 if (error) 1276 break; 1277 1278 pcd = (void *)data; 1279 1280 /* filter input */ 1281 1282 pcd->devname[sizeof(pcd->devname) - 1] = 0; 1283 1284 if (pcd->devname[0] == 0) { 1285 error = EINVAL; 1286 break; 1287 } 1288 cuse_str_filter(pcd->devname); 1289 1290 pcd->permissions &= 0777; 1291 1292 /* try to allocate a character device */ 1293 1294 pcsd = malloc(sizeof(*pcsd), M_CUSE, M_WAITOK | M_ZERO); 1295 1296 pcsd->server = pcs; 1297 1298 pcsd->user_dev = pcd->dev; 1299 1300 pcsd->kern_dev = make_dev_credf(MAKEDEV_CHECKNAME, 1301 &cuse_client_devsw, 0, NULL, pcd->user_id, pcd->group_id, 1302 pcd->permissions, "%s", pcd->devname); 1303 1304 if (pcsd->kern_dev == NULL) { 1305 free(pcsd, M_CUSE); 1306 error = ENOMEM; 1307 break; 1308 } 1309 pcsd->kern_dev->si_drv1 = pcsd; 1310 1311 cuse_server_lock(pcs); 1312 TAILQ_INSERT_TAIL(&pcs->hdev, pcsd, entry); 1313 cuse_server_unlock(pcs); 1314 1315 break; 1316 1317 case CUSE_IOCTL_DESTROY_DEV: 1318 1319 error = priv_check(curthread, PRIV_DRIVER); 1320 if (error) 1321 break; 1322 1323 cuse_server_lock(pcs); 1324 1325 error = EINVAL; 1326 1327 pcsd = TAILQ_FIRST(&pcs->hdev); 1328 while (pcsd != NULL) { 1329 if (pcsd->user_dev == *(struct cuse_dev **)data) { 1330 TAILQ_REMOVE(&pcs->hdev, pcsd, entry); 1331 cuse_server_unlock(pcs); 1332 cuse_server_free_dev(pcsd); 1333 cuse_server_lock(pcs); 1334 error = 0; 1335 pcsd = TAILQ_FIRST(&pcs->hdev); 1336 } else { 1337 pcsd = TAILQ_NEXT(pcsd, entry); 1338 } 1339 } 1340 1341 cuse_server_unlock(pcs); 1342 break; 1343 1344 case CUSE_IOCTL_WRITE_DATA: 1345 case CUSE_IOCTL_READ_DATA: 1346 1347 cuse_server_lock(pcs); 1348 pchk = (struct cuse_data_chunk *)data; 1349 1350 pccmd = cuse_server_find_command(pcs, curthread); 1351 1352 if (pccmd == NULL) { 1353 error = ENXIO; /* invalid request */ 1354 } else if (pchk->peer_ptr < CUSE_BUF_MIN_PTR) { 1355 error = EFAULT; /* NULL pointer */ 1356 } else if (pchk->length == 0) { 1357 /* NOP */ 1358 } else if (pchk->peer_ptr < CUSE_BUF_MAX_PTR) { 1359 error = cuse_server_ioctl_copy_locked(pcs, pccmd, 1360 pchk, cmd == CUSE_IOCTL_READ_DATA); 1361 } else { 1362 error = cuse_server_data_copy_optimized_locked( 1363 pcs, pccmd, pchk, cmd == CUSE_IOCTL_READ_DATA); 1364 } 1365 1366 /* 1367 * Sometimes the functions above drop the server lock 1368 * early as an optimization: 1369 */ 1370 if (cuse_server_is_locked(pcs)) 1371 cuse_server_unlock(pcs); 1372 break; 1373 1374 case CUSE_IOCTL_SELWAKEUP: 1375 cuse_server_lock(pcs); 1376 /* 1377 * We don't know which direction caused the event. 1378 * Wakeup both! 1379 */ 1380 cuse_server_wakeup_all_client_locked(pcs); 1381 cuse_server_unlock(pcs); 1382 break; 1383 1384 default: 1385 error = ENXIO; 1386 break; 1387 } 1388 return (error); 1389 } 1390 1391 static int 1392 cuse_server_poll(struct cdev *dev, int events, struct thread *td) 1393 { 1394 return (events & (POLLHUP | POLLPRI | POLLIN | 1395 POLLRDNORM | POLLOUT | POLLWRNORM)); 1396 } 1397 1398 static int 1399 cuse_common_mmap_single(struct cuse_server *pcs, 1400 vm_ooffset_t *offset, vm_size_t size, struct vm_object **object) 1401 { 1402 struct cuse_memory *mem; 1403 int error; 1404 1405 /* verify size */ 1406 if ((size % PAGE_SIZE) != 0 || (size < PAGE_SIZE)) 1407 return (EINVAL); 1408 1409 cuse_server_lock(pcs); 1410 error = ENOMEM; 1411 1412 /* lookup memory structure, if any */ 1413 TAILQ_FOREACH(mem, &pcs->hmem, entry) { 1414 vm_ooffset_t min_off; 1415 vm_ooffset_t max_off; 1416 1417 min_off = (mem->alloc_nr << CUSE_ALLOC_UNIT_SHIFT); 1418 max_off = min_off + (PAGE_SIZE * mem->page_count); 1419 1420 if (*offset >= min_off && *offset < max_off) { 1421 /* range check size */ 1422 if (size > (max_off - *offset)) { 1423 error = EINVAL; 1424 } else { 1425 /* get new VM object offset to use */ 1426 *offset -= min_off; 1427 vm_object_reference(mem->object); 1428 *object = mem->object; 1429 error = 0; 1430 } 1431 break; 1432 } 1433 } 1434 cuse_server_unlock(pcs); 1435 return (error); 1436 } 1437 1438 static int 1439 cuse_server_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 1440 vm_size_t size, struct vm_object **object, int nprot) 1441 { 1442 struct cuse_server *pcs; 1443 int error; 1444 1445 error = cuse_server_get(&pcs); 1446 if (error != 0) 1447 return (error); 1448 1449 return (cuse_common_mmap_single(pcs, offset, size, object)); 1450 } 1451 1452 /*------------------------------------------------------------------------* 1453 * CUSE CLIENT PART 1454 *------------------------------------------------------------------------*/ 1455 static void 1456 cuse_client_free(void *arg) 1457 { 1458 struct cuse_client *pcc = arg; 1459 struct cuse_client_command *pccmd; 1460 struct cuse_server *pcs; 1461 int n; 1462 1463 pcs = pcc->server; 1464 1465 cuse_server_lock(pcs); 1466 cuse_client_is_closing(pcc); 1467 TAILQ_REMOVE(&pcs->hcli, pcc, entry); 1468 cuse_server_unlock(pcs); 1469 1470 for (n = 0; n != CUSE_CMD_MAX; n++) { 1471 pccmd = &pcc->cmds[n]; 1472 1473 sx_destroy(&pccmd->sx); 1474 cv_destroy(&pccmd->cv); 1475 } 1476 1477 free(pcc, M_CUSE); 1478 1479 /* drop reference on server */ 1480 cuse_server_unref(pcs); 1481 } 1482 1483 static int 1484 cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td) 1485 { 1486 struct cuse_client_command *pccmd; 1487 struct cuse_server_dev *pcsd; 1488 struct cuse_client *pcc; 1489 struct cuse_server *pcs; 1490 struct cuse_dev *pcd; 1491 int error; 1492 int n; 1493 1494 pcsd = dev->si_drv1; 1495 if (pcsd != NULL) { 1496 pcs = pcsd->server; 1497 pcd = pcsd->user_dev; 1498 1499 cuse_server_lock(pcs); 1500 /* 1501 * Check that the refcount didn't wrap and that the 1502 * same process is not both client and server. This 1503 * can easily lead to deadlocks when destroying the 1504 * CUSE character device nodes: 1505 */ 1506 pcs->refs++; 1507 if (pcs->refs < 0 || pcs->pid == curproc->p_pid) { 1508 /* overflow or wrong PID */ 1509 pcs->refs--; 1510 cuse_server_unlock(pcs); 1511 return (EINVAL); 1512 } 1513 cuse_server_unlock(pcs); 1514 } else { 1515 return (EINVAL); 1516 } 1517 1518 pcc = malloc(sizeof(*pcc), M_CUSE, M_WAITOK | M_ZERO); 1519 pcc->fflags = fflags; 1520 pcc->server_dev = pcsd; 1521 pcc->server = pcs; 1522 1523 for (n = 0; n != CUSE_CMD_MAX; n++) { 1524 pccmd = &pcc->cmds[n]; 1525 1526 pccmd->sub.dev = pcd; 1527 pccmd->sub.command = n; 1528 pccmd->client = pcc; 1529 1530 sx_init(&pccmd->sx, "cuse-client-sx"); 1531 cv_init(&pccmd->cv, "cuse-client-cv"); 1532 } 1533 1534 cuse_server_lock(pcs); 1535 1536 /* cuse_client_free() assumes that the client is listed somewhere! */ 1537 /* always enqueue */ 1538 1539 TAILQ_INSERT_TAIL(&pcs->hcli, pcc, entry); 1540 1541 /* check if server is closing */ 1542 if ((pcs->is_closing != 0) || (dev->si_drv1 == NULL)) { 1543 error = EINVAL; 1544 } else { 1545 error = 0; 1546 } 1547 cuse_server_unlock(pcs); 1548 1549 if (error != 0) 1550 return (error); 1551 1552 if ((error = devfs_set_cdevpriv(pcc, &cuse_client_free)) != 0) 1553 return (error); 1554 1555 pccmd = &pcc->cmds[CUSE_CMD_OPEN]; 1556 1557 cuse_cmd_lock(pccmd); 1558 1559 cuse_server_lock(pcs); 1560 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0); 1561 1562 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1563 cuse_server_unlock(pcs); 1564 1565 if (error < 0) { 1566 error = cuse_convert_error(error); 1567 } else { 1568 error = 0; 1569 } 1570 1571 cuse_cmd_unlock(pccmd); 1572 1573 return (error); 1574 } 1575 1576 static int 1577 cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 1578 { 1579 struct cuse_client_command *pccmd; 1580 struct cuse_client *pcc; 1581 struct cuse_server *pcs; 1582 int error; 1583 1584 error = cuse_client_get(&pcc); 1585 if (error != 0) 1586 return (0); 1587 1588 pccmd = &pcc->cmds[CUSE_CMD_CLOSE]; 1589 pcs = pcc->server; 1590 1591 cuse_cmd_lock(pccmd); 1592 1593 cuse_server_lock(pcs); 1594 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0); 1595 1596 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1597 cuse_cmd_unlock(pccmd); 1598 1599 cuse_client_is_closing(pcc); 1600 cuse_server_unlock(pcs); 1601 1602 return (0); 1603 } 1604 1605 static void 1606 cuse_client_kqfilter_poll(struct cdev *dev, struct cuse_client *pcc) 1607 { 1608 struct cuse_server *pcs = pcc->server; 1609 int temp; 1610 1611 cuse_server_lock(pcs); 1612 temp = (pcc->cflags & (CUSE_CLI_KNOTE_HAS_READ | 1613 CUSE_CLI_KNOTE_HAS_WRITE)); 1614 pcc->cflags &= ~(CUSE_CLI_KNOTE_NEED_READ | 1615 CUSE_CLI_KNOTE_NEED_WRITE); 1616 cuse_server_unlock(pcs); 1617 1618 if (temp != 0) { 1619 /* get the latest polling state from the server */ 1620 temp = cuse_client_poll(dev, POLLIN | POLLOUT, NULL); 1621 1622 if (temp & (POLLIN | POLLOUT)) { 1623 cuse_server_lock(pcs); 1624 if (temp & POLLIN) 1625 pcc->cflags |= CUSE_CLI_KNOTE_NEED_READ; 1626 if (temp & POLLOUT) 1627 pcc->cflags |= CUSE_CLI_KNOTE_NEED_WRITE; 1628 1629 /* make sure the "knote" gets woken up */ 1630 cuse_server_wakeup_locked(pcc->server); 1631 cuse_server_unlock(pcs); 1632 } 1633 } 1634 } 1635 1636 static int 1637 cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag) 1638 { 1639 struct cuse_client_command *pccmd; 1640 struct cuse_client *pcc; 1641 struct cuse_server *pcs; 1642 int error; 1643 int temp; 1644 int len; 1645 1646 error = cuse_client_get(&pcc); 1647 if (error != 0) 1648 return (error); 1649 1650 pccmd = &pcc->cmds[CUSE_CMD_READ]; 1651 pcs = pcc->server; 1652 1653 if (uio->uio_segflg != UIO_USERSPACE) { 1654 return (EINVAL); 1655 } 1656 uio->uio_segflg = UIO_NOCOPY; 1657 1658 cuse_cmd_lock(pccmd); 1659 1660 while (uio->uio_resid != 0) { 1661 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) { 1662 error = ENOMEM; 1663 break; 1664 } 1665 len = uio->uio_iov->iov_len; 1666 1667 cuse_server_lock(pcs); 1668 if (len <= CUSE_COPY_BUFFER_MAX) { 1669 /* set read buffer region for small reads */ 1670 pcc->read_base = (uintptr_t)uio->uio_iov->iov_base; 1671 pcc->read_length = len; 1672 } 1673 cuse_client_send_command_locked(pccmd, 1674 (uintptr_t)uio->uio_iov->iov_base, 1675 (unsigned long)(unsigned int)len, pcc->fflags, ioflag); 1676 1677 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1678 /* 1679 * After finishing reading data, disable the read 1680 * region for the cuse_server_data_copy_optimized_locked() 1681 * function: 1682 */ 1683 pcc->read_base = 0; 1684 pcc->read_length = 0; 1685 cuse_server_unlock(pcs); 1686 1687 /* 1688 * The return value indicates the read length, when 1689 * not negative. Range check it just in case to avoid 1690 * passing invalid length values to uiomove(). 1691 */ 1692 if (error > len) { 1693 error = ERANGE; 1694 break; 1695 } else if (error > 0 && len <= CUSE_COPY_BUFFER_MAX) { 1696 temp = copyout(pcc->read_buffer, 1697 uio->uio_iov->iov_base, error); 1698 if (temp != 0) { 1699 error = temp; 1700 break; 1701 } 1702 } 1703 if (error < 0) { 1704 error = cuse_convert_error(error); 1705 break; 1706 } else if (error == len) { 1707 error = uiomove(NULL, error, uio); 1708 if (error) 1709 break; 1710 } else { 1711 error = uiomove(NULL, error, uio); 1712 break; 1713 } 1714 } 1715 cuse_cmd_unlock(pccmd); 1716 1717 uio->uio_segflg = UIO_USERSPACE;/* restore segment flag */ 1718 1719 if (error == EWOULDBLOCK) 1720 cuse_client_kqfilter_poll(dev, pcc); 1721 1722 return (error); 1723 } 1724 1725 static int 1726 cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag) 1727 { 1728 struct cuse_client_command *pccmd; 1729 struct cuse_client *pcc; 1730 struct cuse_server *pcs; 1731 int error; 1732 int len; 1733 1734 error = cuse_client_get(&pcc); 1735 if (error != 0) 1736 return (error); 1737 1738 pccmd = &pcc->cmds[CUSE_CMD_WRITE]; 1739 pcs = pcc->server; 1740 1741 if (uio->uio_segflg != UIO_USERSPACE) { 1742 return (EINVAL); 1743 } 1744 uio->uio_segflg = UIO_NOCOPY; 1745 1746 cuse_cmd_lock(pccmd); 1747 1748 while (uio->uio_resid != 0) { 1749 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) { 1750 error = ENOMEM; 1751 break; 1752 } 1753 len = uio->uio_iov->iov_len; 1754 1755 if (len <= CUSE_COPY_BUFFER_MAX) { 1756 error = copyin(uio->uio_iov->iov_base, 1757 pcc->write_buffer, len); 1758 if (error != 0) 1759 break; 1760 } 1761 1762 cuse_server_lock(pcs); 1763 if (len <= CUSE_COPY_BUFFER_MAX) { 1764 /* set write buffer region for small writes */ 1765 pcc->write_base = (uintptr_t)uio->uio_iov->iov_base; 1766 pcc->write_length = len; 1767 } 1768 cuse_client_send_command_locked(pccmd, 1769 (uintptr_t)uio->uio_iov->iov_base, 1770 (unsigned long)(unsigned int)len, pcc->fflags, ioflag); 1771 1772 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1773 1774 /* 1775 * After finishing writing data, disable the write 1776 * region for the cuse_server_data_copy_optimized_locked() 1777 * function: 1778 */ 1779 pcc->write_base = 0; 1780 pcc->write_length = 0; 1781 cuse_server_unlock(pcs); 1782 1783 /* 1784 * The return value indicates the write length, when 1785 * not negative. Range check it just in case to avoid 1786 * passing invalid length values to uiomove(). 1787 */ 1788 if (error > len) { 1789 error = ERANGE; 1790 break; 1791 } else if (error < 0) { 1792 error = cuse_convert_error(error); 1793 break; 1794 } else if (error == len) { 1795 error = uiomove(NULL, error, uio); 1796 if (error) 1797 break; 1798 } else { 1799 error = uiomove(NULL, error, uio); 1800 break; 1801 } 1802 } 1803 cuse_cmd_unlock(pccmd); 1804 1805 /* restore segment flag */ 1806 uio->uio_segflg = UIO_USERSPACE; 1807 1808 if (error == EWOULDBLOCK) 1809 cuse_client_kqfilter_poll(dev, pcc); 1810 1811 return (error); 1812 } 1813 1814 int 1815 cuse_client_ioctl(struct cdev *dev, unsigned long cmd, 1816 caddr_t data, int fflag, struct thread *td) 1817 { 1818 struct cuse_client_command *pccmd; 1819 struct cuse_client *pcc; 1820 struct cuse_server *pcs; 1821 int error; 1822 int len; 1823 1824 error = cuse_client_get(&pcc); 1825 if (error != 0) 1826 return (error); 1827 1828 len = IOCPARM_LEN(cmd); 1829 if (len > CUSE_BUFFER_MAX) 1830 return (ENOMEM); 1831 1832 pccmd = &pcc->cmds[CUSE_CMD_IOCTL]; 1833 pcs = pcc->server; 1834 1835 cuse_cmd_lock(pccmd); 1836 1837 if (cmd & (IOC_IN | IOC_VOID)) 1838 memcpy(pcc->ioctl_buffer, data, len); 1839 1840 /* 1841 * When the ioctl-length is zero drivers can pass information 1842 * through the data pointer of the ioctl. Make sure this information 1843 * is forwarded to the driver. 1844 */ 1845 1846 cuse_server_lock(pcs); 1847 cuse_client_send_command_locked(pccmd, 1848 (len == 0) ? *(long *)data : CUSE_BUF_MIN_PTR, 1849 (unsigned long)cmd, pcc->fflags, 1850 (fflag & O_NONBLOCK) ? IO_NDELAY : 0); 1851 1852 error = cuse_client_receive_command_locked(pccmd, data, len); 1853 cuse_server_unlock(pcs); 1854 1855 if (error < 0) { 1856 error = cuse_convert_error(error); 1857 } else { 1858 error = 0; 1859 } 1860 1861 if (cmd & IOC_OUT) 1862 memcpy(data, pcc->ioctl_buffer, len); 1863 1864 cuse_cmd_unlock(pccmd); 1865 1866 if (error == EWOULDBLOCK) 1867 cuse_client_kqfilter_poll(dev, pcc); 1868 1869 return (error); 1870 } 1871 1872 static int 1873 cuse_client_poll(struct cdev *dev, int events, struct thread *td) 1874 { 1875 struct cuse_client_command *pccmd; 1876 struct cuse_client *pcc; 1877 struct cuse_server *pcs; 1878 unsigned long temp; 1879 int error; 1880 int revents; 1881 1882 error = cuse_client_get(&pcc); 1883 if (error != 0) 1884 goto pollnval; 1885 1886 temp = 0; 1887 pcs = pcc->server; 1888 1889 if (events & (POLLPRI | POLLIN | POLLRDNORM)) 1890 temp |= CUSE_POLL_READ; 1891 1892 if (events & (POLLOUT | POLLWRNORM)) 1893 temp |= CUSE_POLL_WRITE; 1894 1895 if (events & POLLHUP) 1896 temp |= CUSE_POLL_ERROR; 1897 1898 pccmd = &pcc->cmds[CUSE_CMD_POLL]; 1899 1900 cuse_cmd_lock(pccmd); 1901 1902 /* Need to selrecord() first to not loose any events. */ 1903 if (temp != 0 && td != NULL) 1904 selrecord(td, &pcs->selinfo); 1905 1906 cuse_server_lock(pcs); 1907 cuse_client_send_command_locked(pccmd, 1908 0, temp, pcc->fflags, IO_NDELAY); 1909 1910 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1911 cuse_server_unlock(pcs); 1912 1913 cuse_cmd_unlock(pccmd); 1914 1915 if (error < 0) { 1916 goto pollnval; 1917 } else { 1918 revents = 0; 1919 if (error & CUSE_POLL_READ) 1920 revents |= (events & (POLLPRI | POLLIN | POLLRDNORM)); 1921 if (error & CUSE_POLL_WRITE) 1922 revents |= (events & (POLLOUT | POLLWRNORM)); 1923 if (error & CUSE_POLL_ERROR) 1924 revents |= (events & POLLHUP); 1925 } 1926 return (revents); 1927 1928 pollnval: 1929 /* XXX many clients don't understand POLLNVAL */ 1930 return (events & (POLLHUP | POLLPRI | POLLIN | 1931 POLLRDNORM | POLLOUT | POLLWRNORM)); 1932 } 1933 1934 static int 1935 cuse_client_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 1936 vm_size_t size, struct vm_object **object, int nprot) 1937 { 1938 struct cuse_client *pcc; 1939 int error; 1940 1941 error = cuse_client_get(&pcc); 1942 if (error != 0) 1943 return (error); 1944 1945 return (cuse_common_mmap_single(pcc->server, offset, size, object)); 1946 } 1947 1948 static void 1949 cuse_client_kqfilter_read_detach(struct knote *kn) 1950 { 1951 struct cuse_client *pcc; 1952 struct cuse_server *pcs; 1953 1954 pcc = kn->kn_hook; 1955 pcs = pcc->server; 1956 1957 cuse_server_lock(pcs); 1958 knlist_remove(&pcs->selinfo.si_note, kn, 1); 1959 cuse_server_unlock(pcs); 1960 } 1961 1962 static void 1963 cuse_client_kqfilter_write_detach(struct knote *kn) 1964 { 1965 struct cuse_client *pcc; 1966 struct cuse_server *pcs; 1967 1968 pcc = kn->kn_hook; 1969 pcs = pcc->server; 1970 1971 cuse_server_lock(pcs); 1972 knlist_remove(&pcs->selinfo.si_note, kn, 1); 1973 cuse_server_unlock(pcs); 1974 } 1975 1976 static int 1977 cuse_client_kqfilter_read_event(struct knote *kn, long hint) 1978 { 1979 struct cuse_client *pcc; 1980 1981 pcc = kn->kn_hook; 1982 1983 mtx_assert(&pcc->server->mtx, MA_OWNED); 1984 1985 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_READ) ? 1 : 0); 1986 } 1987 1988 static int 1989 cuse_client_kqfilter_write_event(struct knote *kn, long hint) 1990 { 1991 struct cuse_client *pcc; 1992 1993 pcc = kn->kn_hook; 1994 1995 mtx_assert(&pcc->server->mtx, MA_OWNED); 1996 1997 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_WRITE) ? 1 : 0); 1998 } 1999 2000 static int 2001 cuse_client_kqfilter(struct cdev *dev, struct knote *kn) 2002 { 2003 struct cuse_client *pcc; 2004 struct cuse_server *pcs; 2005 int error; 2006 2007 error = cuse_client_get(&pcc); 2008 if (error != 0) 2009 return (error); 2010 2011 pcs = pcc->server; 2012 2013 cuse_server_lock(pcs); 2014 switch (kn->kn_filter) { 2015 case EVFILT_READ: 2016 pcc->cflags |= CUSE_CLI_KNOTE_HAS_READ; 2017 kn->kn_hook = pcc; 2018 kn->kn_fop = &cuse_client_kqfilter_read_ops; 2019 knlist_add(&pcs->selinfo.si_note, kn, 1); 2020 break; 2021 case EVFILT_WRITE: 2022 pcc->cflags |= CUSE_CLI_KNOTE_HAS_WRITE; 2023 kn->kn_hook = pcc; 2024 kn->kn_fop = &cuse_client_kqfilter_write_ops; 2025 knlist_add(&pcs->selinfo.si_note, kn, 1); 2026 break; 2027 default: 2028 error = EINVAL; 2029 break; 2030 } 2031 cuse_server_unlock(pcs); 2032 2033 if (error == 0) 2034 cuse_client_kqfilter_poll(dev, pcc); 2035 return (error); 2036 } 2037