1 /*- 2 * Copyright (c) 2010-2022 Hans Petter Selasky 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include <sys/stdint.h> 27 #include <sys/stddef.h> 28 #include <sys/param.h> 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/conf.h> 32 #include <sys/kernel.h> 33 #include <sys/bus.h> 34 #include <sys/linker_set.h> 35 #include <sys/module.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/condvar.h> 39 #include <sys/sysctl.h> 40 #include <sys/unistd.h> 41 #include <sys/malloc.h> 42 #include <sys/priv.h> 43 #include <sys/uio.h> 44 #include <sys/poll.h> 45 #include <sys/sx.h> 46 #include <sys/rwlock.h> 47 #include <sys/queue.h> 48 #include <sys/fcntl.h> 49 #include <sys/proc.h> 50 #include <sys/vnode.h> 51 #include <sys/selinfo.h> 52 #include <sys/ptrace.h> 53 #include <sys/sysent.h> 54 55 #include <machine/bus.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 #include <vm/vm_object.h> 60 #include <vm/vm_page.h> 61 #include <vm/vm_pager.h> 62 63 #include <fs/cuse/cuse_defs.h> 64 #include <fs/cuse/cuse_ioctl.h> 65 66 /* set this define to zero to disable this feature */ 67 #define CUSE_COPY_BUFFER_MAX \ 68 CUSE_BUFFER_MAX 69 70 #define CUSE_ALLOC_PAGES_MAX \ 71 (CUSE_ALLOC_BYTES_MAX / PAGE_SIZE) 72 73 #if (CUSE_ALLOC_PAGES_MAX == 0) 74 #error "PAGE_SIZE is too big!" 75 #endif 76 77 static int 78 cuse_modevent(module_t mod, int type, void *data) 79 { 80 switch (type) { 81 case MOD_LOAD: 82 case MOD_UNLOAD: 83 return (0); 84 default: 85 return (EOPNOTSUPP); 86 } 87 } 88 89 static moduledata_t cuse_mod = { 90 .name = "cuse", 91 .evhand = &cuse_modevent, 92 }; 93 94 DECLARE_MODULE(cuse, cuse_mod, SI_SUB_DEVFS, SI_ORDER_FIRST); 95 MODULE_VERSION(cuse, 1); 96 97 /* 98 * Prevent cuse4bsd.ko and cuse.ko from loading at the same time by 99 * declaring support for the cuse4bsd interface in cuse.ko: 100 */ 101 MODULE_VERSION(cuse4bsd, 1); 102 103 #ifdef FEATURE 104 FEATURE(cuse, "Userspace character devices"); 105 #endif 106 107 struct cuse_command; 108 struct cuse_server; 109 struct cuse_client; 110 111 struct cuse_client_command { 112 TAILQ_ENTRY(cuse_client_command) entry; 113 struct cuse_command sub; 114 struct sx sx; 115 struct cv cv; 116 struct thread *entered; 117 struct cuse_client *client; 118 struct proc *proc_curr; 119 int proc_refs; 120 int got_signal; 121 int error; 122 int command; 123 }; 124 125 struct cuse_memory { 126 TAILQ_ENTRY(cuse_memory) entry; 127 vm_object_t object; 128 uint32_t page_count; 129 uint32_t alloc_nr; 130 }; 131 132 struct cuse_server_dev { 133 TAILQ_ENTRY(cuse_server_dev) entry; 134 struct cuse_server *server; 135 struct cdev *kern_dev; 136 struct cuse_dev *user_dev; 137 }; 138 139 struct cuse_server { 140 TAILQ_ENTRY(cuse_server) entry; 141 TAILQ_HEAD(, cuse_client_command) head; 142 TAILQ_HEAD(, cuse_server_dev) hdev; 143 TAILQ_HEAD(, cuse_client) hcli; 144 TAILQ_HEAD(, cuse_memory) hmem; 145 struct mtx mtx; 146 struct cv cv; 147 struct selinfo selinfo; 148 pid_t pid; 149 int is_closing; 150 int refs; 151 }; 152 153 struct cuse_client { 154 TAILQ_ENTRY(cuse_client) entry; 155 TAILQ_ENTRY(cuse_client) entry_ref; 156 struct cuse_client_command cmds[CUSE_CMD_MAX]; 157 struct cuse_server *server; 158 struct cuse_server_dev *server_dev; 159 160 uintptr_t read_base; 161 uintptr_t write_base; 162 int read_length; 163 int write_length; 164 uint8_t read_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4); 165 uint8_t write_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4); 166 uint8_t ioctl_buffer[CUSE_BUFFER_MAX] __aligned(4); 167 168 int fflags; /* file flags */ 169 int cflags; /* client flags */ 170 #define CUSE_CLI_IS_CLOSING 0x01 171 #define CUSE_CLI_KNOTE_NEED_READ 0x02 172 #define CUSE_CLI_KNOTE_NEED_WRITE 0x04 173 #define CUSE_CLI_KNOTE_HAS_READ 0x08 174 #define CUSE_CLI_KNOTE_HAS_WRITE 0x10 175 }; 176 177 #define CUSE_CLIENT_CLOSING(pcc) \ 178 ((pcc)->cflags & CUSE_CLI_IS_CLOSING) 179 180 static MALLOC_DEFINE(M_CUSE, "cuse", "CUSE memory"); 181 182 static TAILQ_HEAD(, cuse_server) cuse_server_head; 183 static struct mtx cuse_global_mtx; 184 static struct cdev *cuse_dev; 185 static struct cuse_server *cuse_alloc_unit[CUSE_DEVICES_MAX]; 186 static int cuse_alloc_unit_id[CUSE_DEVICES_MAX]; 187 188 static void cuse_server_wakeup_all_client_locked(struct cuse_server *pcs); 189 static void cuse_client_kqfilter_read_detach(struct knote *kn); 190 static void cuse_client_kqfilter_write_detach(struct knote *kn); 191 static int cuse_client_kqfilter_read_event(struct knote *kn, long hint); 192 static int cuse_client_kqfilter_write_event(struct knote *kn, long hint); 193 194 static const struct filterops cuse_client_kqfilter_read_ops = { 195 .f_isfd = 1, 196 .f_detach = cuse_client_kqfilter_read_detach, 197 .f_event = cuse_client_kqfilter_read_event, 198 .f_copy = knote_triv_copy, 199 }; 200 201 static const struct filterops cuse_client_kqfilter_write_ops = { 202 .f_isfd = 1, 203 .f_detach = cuse_client_kqfilter_write_detach, 204 .f_event = cuse_client_kqfilter_write_event, 205 .f_copy = knote_triv_copy, 206 }; 207 208 static d_open_t cuse_client_open; 209 static d_close_t cuse_client_close; 210 static d_ioctl_t cuse_client_ioctl; 211 static d_read_t cuse_client_read; 212 static d_write_t cuse_client_write; 213 static d_poll_t cuse_client_poll; 214 static d_mmap_single_t cuse_client_mmap_single; 215 static d_kqfilter_t cuse_client_kqfilter; 216 217 static struct cdevsw cuse_client_devsw = { 218 .d_version = D_VERSION, 219 .d_open = cuse_client_open, 220 .d_close = cuse_client_close, 221 .d_ioctl = cuse_client_ioctl, 222 .d_name = "cuse_client", 223 .d_flags = D_TRACKCLOSE, 224 .d_read = cuse_client_read, 225 .d_write = cuse_client_write, 226 .d_poll = cuse_client_poll, 227 .d_mmap_single = cuse_client_mmap_single, 228 .d_kqfilter = cuse_client_kqfilter, 229 }; 230 231 static d_open_t cuse_server_open; 232 static d_close_t cuse_server_close; 233 static d_ioctl_t cuse_server_ioctl; 234 static d_read_t cuse_server_read; 235 static d_write_t cuse_server_write; 236 static d_poll_t cuse_server_poll; 237 static d_mmap_single_t cuse_server_mmap_single; 238 239 static struct cdevsw cuse_server_devsw = { 240 .d_version = D_VERSION, 241 .d_open = cuse_server_open, 242 .d_close = cuse_server_close, 243 .d_ioctl = cuse_server_ioctl, 244 .d_name = "cuse_server", 245 .d_flags = D_TRACKCLOSE, 246 .d_read = cuse_server_read, 247 .d_write = cuse_server_write, 248 .d_poll = cuse_server_poll, 249 .d_mmap_single = cuse_server_mmap_single, 250 }; 251 252 static void cuse_client_is_closing(struct cuse_client *); 253 static int cuse_free_unit_by_id_locked(struct cuse_server *, int); 254 255 static void 256 cuse_global_lock(void) 257 { 258 mtx_lock(&cuse_global_mtx); 259 } 260 261 static void 262 cuse_global_unlock(void) 263 { 264 mtx_unlock(&cuse_global_mtx); 265 } 266 267 static void 268 cuse_server_lock(struct cuse_server *pcs) 269 { 270 mtx_lock(&pcs->mtx); 271 } 272 273 static void 274 cuse_server_unlock(struct cuse_server *pcs) 275 { 276 mtx_unlock(&pcs->mtx); 277 } 278 279 static bool 280 cuse_server_is_locked(struct cuse_server *pcs) 281 { 282 return (mtx_owned(&pcs->mtx)); 283 } 284 285 static void 286 cuse_cmd_lock(struct cuse_client_command *pccmd) 287 { 288 sx_xlock(&pccmd->sx); 289 } 290 291 static void 292 cuse_cmd_unlock(struct cuse_client_command *pccmd) 293 { 294 sx_xunlock(&pccmd->sx); 295 } 296 297 static void 298 cuse_kern_init(void *arg) 299 { 300 TAILQ_INIT(&cuse_server_head); 301 302 mtx_init(&cuse_global_mtx, "cuse-global-mtx", NULL, MTX_DEF); 303 304 cuse_dev = make_dev(&cuse_server_devsw, 0, 305 UID_ROOT, GID_OPERATOR, 0600, "cuse"); 306 307 printf("Cuse v%d.%d.%d @ /dev/cuse\n", 308 (CUSE_VERSION >> 16) & 0xFF, (CUSE_VERSION >> 8) & 0xFF, 309 (CUSE_VERSION >> 0) & 0xFF); 310 } 311 SYSINIT(cuse_kern_init, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_init, NULL); 312 313 static void 314 cuse_kern_uninit(void *arg) 315 { 316 void *ptr; 317 318 while (1) { 319 printf("Cuse: Please exit all /dev/cuse instances " 320 "and processes which have used this device.\n"); 321 322 pause("DRAIN", 2 * hz); 323 324 cuse_global_lock(); 325 ptr = TAILQ_FIRST(&cuse_server_head); 326 cuse_global_unlock(); 327 328 if (ptr == NULL) 329 break; 330 } 331 332 if (cuse_dev != NULL) 333 destroy_dev(cuse_dev); 334 335 mtx_destroy(&cuse_global_mtx); 336 } 337 SYSUNINIT(cuse_kern_uninit, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_uninit, NULL); 338 339 static int 340 cuse_server_get(struct cuse_server **ppcs) 341 { 342 struct cuse_server *pcs; 343 int error; 344 345 error = devfs_get_cdevpriv((void **)&pcs); 346 if (error != 0) { 347 *ppcs = NULL; 348 return (error); 349 } 350 if (pcs->is_closing) { 351 *ppcs = NULL; 352 return (EINVAL); 353 } 354 *ppcs = pcs; 355 return (0); 356 } 357 358 static void 359 cuse_server_is_closing(struct cuse_server *pcs) 360 { 361 struct cuse_client *pcc; 362 363 if (pcs->is_closing) 364 return; 365 366 pcs->is_closing = 1; 367 368 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 369 cuse_client_is_closing(pcc); 370 } 371 } 372 373 static struct cuse_client_command * 374 cuse_server_find_command(struct cuse_server *pcs, struct thread *td) 375 { 376 struct cuse_client *pcc; 377 int n; 378 379 if (pcs->is_closing) 380 goto done; 381 382 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 383 if (CUSE_CLIENT_CLOSING(pcc)) 384 continue; 385 for (n = 0; n != CUSE_CMD_MAX; n++) { 386 if (pcc->cmds[n].entered == td) 387 return (&pcc->cmds[n]); 388 } 389 } 390 done: 391 return (NULL); 392 } 393 394 static void 395 cuse_str_filter(char *ptr) 396 { 397 int c; 398 399 while (((c = *ptr) != 0)) { 400 if ((c >= 'a') && (c <= 'z')) { 401 ptr++; 402 continue; 403 } 404 if ((c >= 'A') && (c <= 'Z')) { 405 ptr++; 406 continue; 407 } 408 if ((c >= '0') && (c <= '9')) { 409 ptr++; 410 continue; 411 } 412 if ((c == '.') || (c == '_') || (c == '/')) { 413 ptr++; 414 continue; 415 } 416 *ptr = '_'; 417 418 ptr++; 419 } 420 } 421 422 static int 423 cuse_convert_error(int error) 424 { 425 ; /* indent fix */ 426 switch (error) { 427 case CUSE_ERR_NONE: 428 return (0); 429 case CUSE_ERR_BUSY: 430 return (EBUSY); 431 case CUSE_ERR_WOULDBLOCK: 432 return (EWOULDBLOCK); 433 case CUSE_ERR_INVALID: 434 return (EINVAL); 435 case CUSE_ERR_NO_MEMORY: 436 return (ENOMEM); 437 case CUSE_ERR_FAULT: 438 return (EFAULT); 439 case CUSE_ERR_SIGNAL: 440 return (EINTR); 441 case CUSE_ERR_NO_DEVICE: 442 return (ENODEV); 443 default: 444 return (ENXIO); 445 } 446 } 447 448 static void 449 cuse_vm_memory_free(struct cuse_memory *mem) 450 { 451 /* last user is gone - free */ 452 vm_object_deallocate(mem->object); 453 454 /* free CUSE memory */ 455 free(mem, M_CUSE); 456 } 457 458 static int 459 cuse_server_alloc_memory(struct cuse_server *pcs, uint32_t alloc_nr, 460 uint32_t page_count) 461 { 462 struct cuse_memory *temp; 463 struct cuse_memory *mem; 464 vm_object_t object; 465 int error; 466 467 mem = malloc(sizeof(*mem), M_CUSE, M_WAITOK | M_ZERO); 468 469 object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * page_count, 470 VM_PROT_DEFAULT, 0, curthread->td_ucred); 471 if (object == NULL) { 472 error = ENOMEM; 473 goto error_0; 474 } 475 476 cuse_server_lock(pcs); 477 /* check if allocation number already exists */ 478 TAILQ_FOREACH(temp, &pcs->hmem, entry) { 479 if (temp->alloc_nr == alloc_nr) 480 break; 481 } 482 if (temp != NULL) { 483 cuse_server_unlock(pcs); 484 error = EBUSY; 485 goto error_1; 486 } 487 mem->object = object; 488 mem->page_count = page_count; 489 mem->alloc_nr = alloc_nr; 490 TAILQ_INSERT_TAIL(&pcs->hmem, mem, entry); 491 cuse_server_unlock(pcs); 492 493 return (0); 494 495 error_1: 496 vm_object_deallocate(object); 497 error_0: 498 free(mem, M_CUSE); 499 return (error); 500 } 501 502 static int 503 cuse_server_free_memory(struct cuse_server *pcs, uint32_t alloc_nr) 504 { 505 struct cuse_memory *mem; 506 507 cuse_server_lock(pcs); 508 TAILQ_FOREACH(mem, &pcs->hmem, entry) { 509 if (mem->alloc_nr == alloc_nr) 510 break; 511 } 512 if (mem == NULL) { 513 cuse_server_unlock(pcs); 514 return (EINVAL); 515 } 516 TAILQ_REMOVE(&pcs->hmem, mem, entry); 517 cuse_server_unlock(pcs); 518 519 cuse_vm_memory_free(mem); 520 521 return (0); 522 } 523 524 static int 525 cuse_client_get(struct cuse_client **ppcc) 526 { 527 struct cuse_client *pcc; 528 int error; 529 530 /* try to get private data */ 531 error = devfs_get_cdevpriv((void **)&pcc); 532 if (error != 0) { 533 *ppcc = NULL; 534 return (error); 535 } 536 if (CUSE_CLIENT_CLOSING(pcc) || pcc->server->is_closing) { 537 *ppcc = NULL; 538 return (EINVAL); 539 } 540 *ppcc = pcc; 541 return (0); 542 } 543 544 static void 545 cuse_client_is_closing(struct cuse_client *pcc) 546 { 547 struct cuse_client_command *pccmd; 548 uint32_t n; 549 550 if (CUSE_CLIENT_CLOSING(pcc)) 551 return; 552 553 pcc->cflags |= CUSE_CLI_IS_CLOSING; 554 pcc->server_dev = NULL; 555 556 for (n = 0; n != CUSE_CMD_MAX; n++) { 557 pccmd = &pcc->cmds[n]; 558 559 if (pccmd->entry.tqe_prev != NULL) { 560 TAILQ_REMOVE(&pcc->server->head, pccmd, entry); 561 pccmd->entry.tqe_prev = NULL; 562 } 563 cv_broadcast(&pccmd->cv); 564 } 565 } 566 567 static void 568 cuse_client_send_command_locked(struct cuse_client_command *pccmd, 569 uintptr_t data_ptr, unsigned long arg, int fflags, int ioflag) 570 { 571 unsigned long cuse_fflags = 0; 572 struct cuse_server *pcs; 573 574 if (fflags & FREAD) 575 cuse_fflags |= CUSE_FFLAG_READ; 576 577 if (fflags & FWRITE) 578 cuse_fflags |= CUSE_FFLAG_WRITE; 579 580 if (ioflag & IO_NDELAY) 581 cuse_fflags |= CUSE_FFLAG_NONBLOCK; 582 #if defined(__LP64__) 583 if (SV_CURPROC_FLAG(SV_ILP32)) 584 cuse_fflags |= CUSE_FFLAG_COMPAT32; 585 #endif 586 pccmd->sub.fflags = cuse_fflags; 587 pccmd->sub.data_pointer = data_ptr; 588 pccmd->sub.argument = arg; 589 590 pcs = pccmd->client->server; 591 592 if ((pccmd->entry.tqe_prev == NULL) && 593 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) && 594 (pcs->is_closing == 0)) { 595 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry); 596 cv_signal(&pcs->cv); 597 } 598 } 599 600 static void 601 cuse_client_got_signal(struct cuse_client_command *pccmd) 602 { 603 struct cuse_server *pcs; 604 605 pccmd->got_signal = 1; 606 607 pccmd = &pccmd->client->cmds[CUSE_CMD_SIGNAL]; 608 609 pcs = pccmd->client->server; 610 611 if ((pccmd->entry.tqe_prev == NULL) && 612 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) && 613 (pcs->is_closing == 0)) { 614 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry); 615 cv_signal(&pcs->cv); 616 } 617 } 618 619 static int 620 cuse_client_receive_command_locked(struct cuse_client_command *pccmd, 621 uint8_t *arg_ptr, uint32_t arg_len) 622 { 623 struct cuse_server *pcs; 624 int error; 625 626 pcs = pccmd->client->server; 627 error = 0; 628 629 pccmd->proc_curr = curthread->td_proc; 630 631 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) { 632 error = CUSE_ERR_OTHER; 633 goto done; 634 } 635 while (pccmd->command == CUSE_CMD_NONE) { 636 if (error != 0) { 637 cv_wait(&pccmd->cv, &pcs->mtx); 638 } else { 639 error = cv_wait_sig(&pccmd->cv, &pcs->mtx); 640 641 if (error != 0) 642 cuse_client_got_signal(pccmd); 643 } 644 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) { 645 error = CUSE_ERR_OTHER; 646 goto done; 647 } 648 } 649 650 error = pccmd->error; 651 pccmd->command = CUSE_CMD_NONE; 652 cv_signal(&pccmd->cv); 653 654 done: 655 656 /* wait until all process references are gone */ 657 658 pccmd->proc_curr = NULL; 659 660 while (pccmd->proc_refs != 0) 661 cv_wait(&pccmd->cv, &pcs->mtx); 662 663 return (error); 664 } 665 666 /*------------------------------------------------------------------------* 667 * CUSE SERVER PART 668 *------------------------------------------------------------------------*/ 669 670 static void 671 cuse_server_free_dev(struct cuse_server_dev *pcsd) 672 { 673 struct cuse_server *pcs; 674 struct cuse_client *pcc; 675 676 /* get server pointer */ 677 pcs = pcsd->server; 678 679 /* prevent creation of more devices */ 680 cuse_server_lock(pcs); 681 if (pcsd->kern_dev != NULL) 682 pcsd->kern_dev->si_drv1 = NULL; 683 684 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 685 if (pcc->server_dev == pcsd) 686 cuse_client_is_closing(pcc); 687 } 688 cuse_server_unlock(pcs); 689 690 /* destroy device, if any */ 691 if (pcsd->kern_dev != NULL) { 692 /* destroy device synchronously */ 693 destroy_dev(pcsd->kern_dev); 694 } 695 free(pcsd, M_CUSE); 696 } 697 698 static void 699 cuse_server_unref(struct cuse_server *pcs) 700 { 701 struct cuse_server_dev *pcsd; 702 struct cuse_memory *mem; 703 704 cuse_server_lock(pcs); 705 if (--(pcs->refs) != 0) { 706 cuse_server_unlock(pcs); 707 return; 708 } 709 cuse_server_is_closing(pcs); 710 /* final client wakeup, if any */ 711 cuse_server_wakeup_all_client_locked(pcs); 712 713 cuse_global_lock(); 714 TAILQ_REMOVE(&cuse_server_head, pcs, entry); 715 cuse_global_unlock(); 716 717 while ((pcsd = TAILQ_FIRST(&pcs->hdev)) != NULL) { 718 TAILQ_REMOVE(&pcs->hdev, pcsd, entry); 719 cuse_server_unlock(pcs); 720 cuse_server_free_dev(pcsd); 721 cuse_server_lock(pcs); 722 } 723 724 cuse_free_unit_by_id_locked(pcs, -1); 725 726 while ((mem = TAILQ_FIRST(&pcs->hmem)) != NULL) { 727 TAILQ_REMOVE(&pcs->hmem, mem, entry); 728 cuse_server_unlock(pcs); 729 cuse_vm_memory_free(mem); 730 cuse_server_lock(pcs); 731 } 732 733 knlist_clear(&pcs->selinfo.si_note, 1); 734 knlist_destroy(&pcs->selinfo.si_note); 735 736 cuse_server_unlock(pcs); 737 738 seldrain(&pcs->selinfo); 739 740 cv_destroy(&pcs->cv); 741 742 mtx_destroy(&pcs->mtx); 743 744 free(pcs, M_CUSE); 745 } 746 747 static int 748 cuse_server_do_close(struct cuse_server *pcs) 749 { 750 int retval; 751 752 cuse_server_lock(pcs); 753 cuse_server_is_closing(pcs); 754 /* final client wakeup, if any */ 755 cuse_server_wakeup_all_client_locked(pcs); 756 757 knlist_clear(&pcs->selinfo.si_note, 1); 758 759 retval = pcs->refs; 760 cuse_server_unlock(pcs); 761 762 return (retval); 763 } 764 765 static void 766 cuse_server_free(void *arg) 767 { 768 struct cuse_server *pcs = arg; 769 770 /* 771 * The final server unref should be done by the server thread 772 * to prevent deadlock in the client cdevpriv destructor, 773 * which cannot destroy itself. 774 */ 775 while (cuse_server_do_close(pcs) != 1) 776 pause("W", hz); 777 778 /* drop final refcount */ 779 cuse_server_unref(pcs); 780 } 781 782 static int 783 cuse_server_open(struct cdev *dev, int fflags, int devtype, struct thread *td) 784 { 785 struct cuse_server *pcs; 786 787 pcs = malloc(sizeof(*pcs), M_CUSE, M_WAITOK | M_ZERO); 788 789 if (devfs_set_cdevpriv(pcs, &cuse_server_free)) { 790 printf("Cuse: Cannot set cdevpriv.\n"); 791 free(pcs, M_CUSE); 792 return (ENOMEM); 793 } 794 /* store current process ID */ 795 pcs->pid = curproc->p_pid; 796 797 TAILQ_INIT(&pcs->head); 798 TAILQ_INIT(&pcs->hdev); 799 TAILQ_INIT(&pcs->hcli); 800 TAILQ_INIT(&pcs->hmem); 801 802 cv_init(&pcs->cv, "cuse-server-cv"); 803 804 mtx_init(&pcs->mtx, "cuse-server-mtx", NULL, MTX_DEF); 805 806 knlist_init_mtx(&pcs->selinfo.si_note, &pcs->mtx); 807 808 cuse_global_lock(); 809 pcs->refs++; 810 TAILQ_INSERT_TAIL(&cuse_server_head, pcs, entry); 811 cuse_global_unlock(); 812 813 return (0); 814 } 815 816 static int 817 cuse_server_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 818 { 819 struct cuse_server *pcs; 820 821 if (cuse_server_get(&pcs) == 0) 822 cuse_server_do_close(pcs); 823 824 return (0); 825 } 826 827 static int 828 cuse_server_read(struct cdev *dev, struct uio *uio, int ioflag) 829 { 830 return (ENXIO); 831 } 832 833 static int 834 cuse_server_write(struct cdev *dev, struct uio *uio, int ioflag) 835 { 836 return (ENXIO); 837 } 838 839 static int 840 cuse_server_ioctl_copy_locked(struct cuse_server *pcs, 841 struct cuse_client_command *pccmd, 842 struct cuse_data_chunk *pchk, bool isread) 843 { 844 struct proc *p_proc; 845 uint32_t offset; 846 int error; 847 848 offset = pchk->peer_ptr - CUSE_BUF_MIN_PTR; 849 850 if (pchk->length > CUSE_BUFFER_MAX) 851 return (EFAULT); 852 853 if (offset >= CUSE_BUFFER_MAX) 854 return (EFAULT); 855 856 if ((offset + pchk->length) > CUSE_BUFFER_MAX) 857 return (EFAULT); 858 859 p_proc = pccmd->proc_curr; 860 if (p_proc == NULL) 861 return (ENXIO); 862 863 if (pccmd->proc_refs < 0) 864 return (ENOMEM); 865 866 pccmd->proc_refs++; 867 868 cuse_server_unlock(pcs); 869 870 if (!isread) { 871 error = copyin( 872 (void *)pchk->local_ptr, 873 pccmd->client->ioctl_buffer + offset, 874 pchk->length); 875 } else { 876 error = copyout( 877 pccmd->client->ioctl_buffer + offset, 878 (void *)pchk->local_ptr, 879 pchk->length); 880 } 881 882 cuse_server_lock(pcs); 883 884 pccmd->proc_refs--; 885 886 if (pccmd->proc_curr == NULL) 887 cv_signal(&pccmd->cv); 888 889 return (error); 890 } 891 892 static int 893 cuse_proc2proc_copy(struct proc *proc_s, vm_offset_t data_s, 894 struct proc *proc_d, vm_offset_t data_d, size_t len) 895 { 896 struct thread *td; 897 struct proc *proc_cur; 898 int error; 899 900 td = curthread; 901 proc_cur = td->td_proc; 902 903 if (proc_cur == proc_d) { 904 struct iovec iov = { 905 .iov_base = (caddr_t)data_d, 906 .iov_len = len, 907 }; 908 struct uio uio = { 909 .uio_iov = &iov, 910 .uio_iovcnt = 1, 911 .uio_offset = (off_t)data_s, 912 .uio_resid = len, 913 .uio_segflg = UIO_USERSPACE, 914 .uio_rw = UIO_READ, 915 .uio_td = td, 916 }; 917 918 PHOLD(proc_s); 919 error = proc_rwmem(proc_s, &uio); 920 PRELE(proc_s); 921 922 } else if (proc_cur == proc_s) { 923 struct iovec iov = { 924 .iov_base = (caddr_t)data_s, 925 .iov_len = len, 926 }; 927 struct uio uio = { 928 .uio_iov = &iov, 929 .uio_iovcnt = 1, 930 .uio_offset = (off_t)data_d, 931 .uio_resid = len, 932 .uio_segflg = UIO_USERSPACE, 933 .uio_rw = UIO_WRITE, 934 .uio_td = td, 935 }; 936 937 PHOLD(proc_d); 938 error = proc_rwmem(proc_d, &uio); 939 PRELE(proc_d); 940 } else { 941 error = EINVAL; 942 } 943 return (error); 944 } 945 946 static int 947 cuse_server_data_copy_locked(struct cuse_server *pcs, 948 struct cuse_client_command *pccmd, 949 struct cuse_data_chunk *pchk, bool isread) 950 { 951 struct proc *p_proc; 952 int error; 953 954 p_proc = pccmd->proc_curr; 955 if (p_proc == NULL) 956 return (ENXIO); 957 958 if (pccmd->proc_refs < 0) 959 return (ENOMEM); 960 961 pccmd->proc_refs++; 962 963 cuse_server_unlock(pcs); 964 965 if (!isread) { 966 error = cuse_proc2proc_copy( 967 curthread->td_proc, pchk->local_ptr, 968 p_proc, pchk->peer_ptr, 969 pchk->length); 970 } else { 971 error = cuse_proc2proc_copy( 972 p_proc, pchk->peer_ptr, 973 curthread->td_proc, pchk->local_ptr, 974 pchk->length); 975 } 976 977 cuse_server_lock(pcs); 978 979 pccmd->proc_refs--; 980 981 if (pccmd->proc_curr == NULL) 982 cv_signal(&pccmd->cv); 983 984 return (error); 985 } 986 987 static int 988 cuse_server_data_copy_optimized_locked(struct cuse_server *pcs, 989 struct cuse_client_command *pccmd, 990 struct cuse_data_chunk *pchk, bool isread) 991 { 992 uintptr_t offset; 993 int error; 994 995 /* 996 * Check if data is stored locally to avoid accessing 997 * other process's data space: 998 */ 999 if (isread) { 1000 offset = pchk->peer_ptr - pccmd->client->write_base; 1001 1002 if (offset < (uintptr_t)pccmd->client->write_length && 1003 pchk->length <= (unsigned long)pccmd->client->write_length && 1004 offset + pchk->length <= (uintptr_t)pccmd->client->write_length) { 1005 cuse_server_unlock(pcs); 1006 error = copyout(pccmd->client->write_buffer + offset, 1007 (void *)pchk->local_ptr, pchk->length); 1008 goto done; 1009 } 1010 } else { 1011 offset = pchk->peer_ptr - pccmd->client->read_base; 1012 1013 if (offset < (uintptr_t)pccmd->client->read_length && 1014 pchk->length <= (unsigned long)pccmd->client->read_length && 1015 offset + pchk->length <= (uintptr_t)pccmd->client->read_length) { 1016 cuse_server_unlock(pcs); 1017 error = copyin((void *)pchk->local_ptr, 1018 pccmd->client->read_buffer + offset, pchk->length); 1019 goto done; 1020 } 1021 } 1022 1023 /* use process to process copy function */ 1024 error = cuse_server_data_copy_locked(pcs, pccmd, pchk, isread); 1025 done: 1026 return (error); 1027 } 1028 1029 static int 1030 cuse_alloc_unit_by_id_locked(struct cuse_server *pcs, int id) 1031 { 1032 int n; 1033 int x = 0; 1034 int match; 1035 1036 do { 1037 for (match = n = 0; n != CUSE_DEVICES_MAX; n++) { 1038 if (cuse_alloc_unit[n] != NULL) { 1039 if ((cuse_alloc_unit_id[n] ^ id) & CUSE_ID_MASK) 1040 continue; 1041 if ((cuse_alloc_unit_id[n] & ~CUSE_ID_MASK) == x) { 1042 x++; 1043 match = 1; 1044 } 1045 } 1046 } 1047 } while (match); 1048 1049 if (x < 256) { 1050 for (n = 0; n != CUSE_DEVICES_MAX; n++) { 1051 if (cuse_alloc_unit[n] == NULL) { 1052 cuse_alloc_unit[n] = pcs; 1053 cuse_alloc_unit_id[n] = id | x; 1054 return (x); 1055 } 1056 } 1057 } 1058 return (-1); 1059 } 1060 1061 static void 1062 cuse_server_wakeup_locked(struct cuse_server *pcs) 1063 { 1064 selwakeup(&pcs->selinfo); 1065 KNOTE_LOCKED(&pcs->selinfo.si_note, 0); 1066 } 1067 1068 static void 1069 cuse_server_wakeup_all_client_locked(struct cuse_server *pcs) 1070 { 1071 struct cuse_client *pcc; 1072 1073 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 1074 pcc->cflags |= (CUSE_CLI_KNOTE_NEED_READ | 1075 CUSE_CLI_KNOTE_NEED_WRITE); 1076 } 1077 cuse_server_wakeup_locked(pcs); 1078 } 1079 1080 static int 1081 cuse_free_unit_by_id_locked(struct cuse_server *pcs, int id) 1082 { 1083 int n; 1084 int found = 0; 1085 1086 for (n = 0; n != CUSE_DEVICES_MAX; n++) { 1087 if (cuse_alloc_unit[n] == pcs) { 1088 if (cuse_alloc_unit_id[n] == id || id == -1) { 1089 cuse_alloc_unit[n] = NULL; 1090 cuse_alloc_unit_id[n] = 0; 1091 found = 1; 1092 } 1093 } 1094 } 1095 1096 return (found ? 0 : EINVAL); 1097 } 1098 1099 static int 1100 cuse_server_ioctl(struct cdev *dev, unsigned long cmd, 1101 caddr_t data, int fflag, struct thread *td) 1102 { 1103 struct cuse_server *pcs; 1104 int error; 1105 1106 error = cuse_server_get(&pcs); 1107 if (error != 0) 1108 return (error); 1109 1110 switch (cmd) { 1111 struct cuse_client_command *pccmd; 1112 struct cuse_client *pcc; 1113 struct cuse_command *pcmd; 1114 struct cuse_alloc_info *pai; 1115 struct cuse_create_dev *pcd; 1116 struct cuse_server_dev *pcsd; 1117 struct cuse_data_chunk *pchk; 1118 int n; 1119 1120 case CUSE_IOCTL_GET_COMMAND: 1121 pcmd = (void *)data; 1122 1123 cuse_server_lock(pcs); 1124 1125 while ((pccmd = TAILQ_FIRST(&pcs->head)) == NULL) { 1126 error = cv_wait_sig(&pcs->cv, &pcs->mtx); 1127 1128 if (pcs->is_closing) 1129 error = ENXIO; 1130 1131 if (error) { 1132 cuse_server_unlock(pcs); 1133 return (error); 1134 } 1135 } 1136 1137 TAILQ_REMOVE(&pcs->head, pccmd, entry); 1138 pccmd->entry.tqe_prev = NULL; 1139 1140 pccmd->entered = curthread; 1141 1142 *pcmd = pccmd->sub; 1143 1144 cuse_server_unlock(pcs); 1145 1146 break; 1147 1148 case CUSE_IOCTL_SYNC_COMMAND: 1149 1150 cuse_server_lock(pcs); 1151 while ((pccmd = cuse_server_find_command(pcs, curthread)) != NULL) { 1152 /* send sync command */ 1153 pccmd->entered = NULL; 1154 pccmd->error = *(int *)data; 1155 pccmd->command = CUSE_CMD_SYNC; 1156 1157 /* signal peer, if any */ 1158 cv_signal(&pccmd->cv); 1159 } 1160 cuse_server_unlock(pcs); 1161 1162 break; 1163 1164 case CUSE_IOCTL_ALLOC_UNIT: 1165 1166 cuse_server_lock(pcs); 1167 n = cuse_alloc_unit_by_id_locked(pcs, 1168 CUSE_ID_DEFAULT(0)); 1169 cuse_server_unlock(pcs); 1170 1171 if (n < 0) 1172 error = ENOMEM; 1173 else 1174 *(int *)data = n; 1175 break; 1176 1177 case CUSE_IOCTL_ALLOC_UNIT_BY_ID: 1178 1179 n = *(int *)data; 1180 1181 n = (n & CUSE_ID_MASK); 1182 1183 cuse_server_lock(pcs); 1184 n = cuse_alloc_unit_by_id_locked(pcs, n); 1185 cuse_server_unlock(pcs); 1186 1187 if (n < 0) 1188 error = ENOMEM; 1189 else 1190 *(int *)data = n; 1191 break; 1192 1193 case CUSE_IOCTL_FREE_UNIT: 1194 1195 n = *(int *)data; 1196 1197 n = CUSE_ID_DEFAULT(n); 1198 1199 cuse_server_lock(pcs); 1200 error = cuse_free_unit_by_id_locked(pcs, n); 1201 cuse_server_unlock(pcs); 1202 break; 1203 1204 case CUSE_IOCTL_FREE_UNIT_BY_ID: 1205 1206 n = *(int *)data; 1207 1208 cuse_server_lock(pcs); 1209 error = cuse_free_unit_by_id_locked(pcs, n); 1210 cuse_server_unlock(pcs); 1211 break; 1212 1213 case CUSE_IOCTL_ALLOC_MEMORY: 1214 1215 pai = (void *)data; 1216 1217 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) { 1218 error = ENOMEM; 1219 break; 1220 } 1221 if (pai->page_count > CUSE_ALLOC_PAGES_MAX) { 1222 error = ENOMEM; 1223 break; 1224 } 1225 error = cuse_server_alloc_memory(pcs, 1226 pai->alloc_nr, pai->page_count); 1227 break; 1228 1229 case CUSE_IOCTL_FREE_MEMORY: 1230 pai = (void *)data; 1231 1232 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) { 1233 error = ENOMEM; 1234 break; 1235 } 1236 error = cuse_server_free_memory(pcs, pai->alloc_nr); 1237 break; 1238 1239 case CUSE_IOCTL_GET_SIG: 1240 1241 cuse_server_lock(pcs); 1242 pccmd = cuse_server_find_command(pcs, curthread); 1243 1244 if (pccmd != NULL) { 1245 n = pccmd->got_signal; 1246 pccmd->got_signal = 0; 1247 } else { 1248 n = 0; 1249 } 1250 cuse_server_unlock(pcs); 1251 1252 *(int *)data = n; 1253 1254 break; 1255 1256 case CUSE_IOCTL_SET_PFH: 1257 1258 cuse_server_lock(pcs); 1259 pccmd = cuse_server_find_command(pcs, curthread); 1260 1261 if (pccmd != NULL) { 1262 pcc = pccmd->client; 1263 for (n = 0; n != CUSE_CMD_MAX; n++) { 1264 pcc->cmds[n].sub.per_file_handle = *(uintptr_t *)data; 1265 } 1266 } else { 1267 error = ENXIO; 1268 } 1269 cuse_server_unlock(pcs); 1270 break; 1271 1272 case CUSE_IOCTL_CREATE_DEV: 1273 1274 error = priv_check(curthread, PRIV_DRIVER); 1275 if (error) 1276 break; 1277 1278 pcd = (void *)data; 1279 1280 /* filter input */ 1281 1282 pcd->devname[sizeof(pcd->devname) - 1] = 0; 1283 1284 if (pcd->devname[0] == 0) { 1285 error = EINVAL; 1286 break; 1287 } 1288 cuse_str_filter(pcd->devname); 1289 1290 pcd->permissions &= 0777; 1291 1292 /* try to allocate a character device */ 1293 1294 pcsd = malloc(sizeof(*pcsd), M_CUSE, M_WAITOK | M_ZERO); 1295 1296 pcsd->server = pcs; 1297 1298 pcsd->user_dev = pcd->dev; 1299 1300 pcsd->kern_dev = make_dev_credf(MAKEDEV_CHECKNAME, 1301 &cuse_client_devsw, 0, NULL, pcd->user_id, pcd->group_id, 1302 pcd->permissions, "%s", pcd->devname); 1303 1304 if (pcsd->kern_dev == NULL) { 1305 free(pcsd, M_CUSE); 1306 error = ENOMEM; 1307 break; 1308 } 1309 pcsd->kern_dev->si_drv1 = pcsd; 1310 1311 cuse_server_lock(pcs); 1312 TAILQ_INSERT_TAIL(&pcs->hdev, pcsd, entry); 1313 cuse_server_unlock(pcs); 1314 1315 break; 1316 1317 case CUSE_IOCTL_DESTROY_DEV: 1318 1319 error = priv_check(curthread, PRIV_DRIVER); 1320 if (error) 1321 break; 1322 1323 cuse_server_lock(pcs); 1324 1325 error = EINVAL; 1326 1327 pcsd = TAILQ_FIRST(&pcs->hdev); 1328 while (pcsd != NULL) { 1329 if (pcsd->user_dev == *(struct cuse_dev **)data) { 1330 TAILQ_REMOVE(&pcs->hdev, pcsd, entry); 1331 cuse_server_unlock(pcs); 1332 cuse_server_free_dev(pcsd); 1333 cuse_server_lock(pcs); 1334 error = 0; 1335 pcsd = TAILQ_FIRST(&pcs->hdev); 1336 } else { 1337 pcsd = TAILQ_NEXT(pcsd, entry); 1338 } 1339 } 1340 1341 cuse_server_unlock(pcs); 1342 break; 1343 1344 case CUSE_IOCTL_WRITE_DATA: 1345 case CUSE_IOCTL_READ_DATA: 1346 1347 cuse_server_lock(pcs); 1348 pchk = (struct cuse_data_chunk *)data; 1349 1350 pccmd = cuse_server_find_command(pcs, curthread); 1351 1352 if (pccmd == NULL) { 1353 error = ENXIO; /* invalid request */ 1354 } else if (pchk->peer_ptr < CUSE_BUF_MIN_PTR) { 1355 error = EFAULT; /* NULL pointer */ 1356 } else if (pchk->length == 0) { 1357 /* NOP */ 1358 } else if (pchk->peer_ptr < CUSE_BUF_MAX_PTR) { 1359 error = cuse_server_ioctl_copy_locked(pcs, pccmd, 1360 pchk, cmd == CUSE_IOCTL_READ_DATA); 1361 } else { 1362 error = cuse_server_data_copy_optimized_locked( 1363 pcs, pccmd, pchk, cmd == CUSE_IOCTL_READ_DATA); 1364 } 1365 1366 /* 1367 * Sometimes the functions above drop the server lock 1368 * early as an optimization: 1369 */ 1370 if (cuse_server_is_locked(pcs)) 1371 cuse_server_unlock(pcs); 1372 break; 1373 1374 case CUSE_IOCTL_SELWAKEUP: 1375 cuse_server_lock(pcs); 1376 /* 1377 * We don't know which direction caused the event. 1378 * Wakeup both! 1379 */ 1380 cuse_server_wakeup_all_client_locked(pcs); 1381 cuse_server_unlock(pcs); 1382 break; 1383 1384 default: 1385 error = ENXIO; 1386 break; 1387 } 1388 return (error); 1389 } 1390 1391 static int 1392 cuse_server_poll(struct cdev *dev, int events, struct thread *td) 1393 { 1394 return (events & (POLLHUP | POLLPRI | POLLIN | 1395 POLLRDNORM | POLLOUT | POLLWRNORM)); 1396 } 1397 1398 static int 1399 cuse_common_mmap_single(struct cuse_server *pcs, 1400 vm_ooffset_t *offset, vm_size_t size, struct vm_object **object) 1401 { 1402 struct cuse_memory *mem; 1403 int error; 1404 1405 /* verify size */ 1406 if ((size % PAGE_SIZE) != 0 || (size < PAGE_SIZE)) 1407 return (EINVAL); 1408 1409 cuse_server_lock(pcs); 1410 error = ENOMEM; 1411 1412 /* lookup memory structure, if any */ 1413 TAILQ_FOREACH(mem, &pcs->hmem, entry) { 1414 vm_ooffset_t min_off; 1415 vm_ooffset_t max_off; 1416 1417 min_off = (mem->alloc_nr << CUSE_ALLOC_UNIT_SHIFT); 1418 max_off = min_off + (PAGE_SIZE * mem->page_count); 1419 1420 if (*offset >= min_off && *offset < max_off) { 1421 /* range check size */ 1422 if (size > (max_off - *offset)) { 1423 error = EINVAL; 1424 } else { 1425 /* get new VM object offset to use */ 1426 *offset -= min_off; 1427 vm_object_reference(mem->object); 1428 *object = mem->object; 1429 error = 0; 1430 } 1431 break; 1432 } 1433 } 1434 cuse_server_unlock(pcs); 1435 return (error); 1436 } 1437 1438 static int 1439 cuse_server_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 1440 vm_size_t size, struct vm_object **object, int nprot) 1441 { 1442 struct cuse_server *pcs; 1443 int error; 1444 1445 error = cuse_server_get(&pcs); 1446 if (error != 0) 1447 return (error); 1448 1449 return (cuse_common_mmap_single(pcs, offset, size, object)); 1450 } 1451 1452 /*------------------------------------------------------------------------* 1453 * CUSE CLIENT PART 1454 *------------------------------------------------------------------------*/ 1455 static void 1456 cuse_client_free(void *arg) 1457 { 1458 struct cuse_client *pcc = arg; 1459 struct cuse_client_command *pccmd; 1460 struct cuse_server *pcs; 1461 int n; 1462 1463 pcs = pcc->server; 1464 1465 cuse_server_lock(pcs); 1466 cuse_client_is_closing(pcc); 1467 TAILQ_REMOVE(&pcs->hcli, pcc, entry); 1468 cuse_server_unlock(pcs); 1469 1470 for (n = 0; n != CUSE_CMD_MAX; n++) { 1471 pccmd = &pcc->cmds[n]; 1472 1473 sx_destroy(&pccmd->sx); 1474 cv_destroy(&pccmd->cv); 1475 } 1476 1477 free(pcc, M_CUSE); 1478 1479 /* drop reference on server */ 1480 cuse_server_unref(pcs); 1481 } 1482 1483 static int 1484 cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td) 1485 { 1486 struct cuse_client_command *pccmd; 1487 struct cuse_server_dev *pcsd; 1488 struct cuse_client *pcc; 1489 struct cuse_server *pcs; 1490 struct cuse_dev *pcd; 1491 int error; 1492 int n; 1493 1494 pcsd = dev->si_drv1; 1495 if (pcsd != NULL) { 1496 pcs = pcsd->server; 1497 pcd = pcsd->user_dev; 1498 1499 cuse_server_lock(pcs); 1500 /* 1501 * Check that the refcount didn't wrap and that the 1502 * same process is not both client and server. This 1503 * can easily lead to deadlocks when destroying the 1504 * CUSE character device nodes: 1505 */ 1506 pcs->refs++; 1507 if (pcs->refs < 0 || pcs->pid == curproc->p_pid) { 1508 /* overflow or wrong PID */ 1509 pcs->refs--; 1510 cuse_server_unlock(pcs); 1511 return (EINVAL); 1512 } 1513 cuse_server_unlock(pcs); 1514 } else { 1515 return (EINVAL); 1516 } 1517 1518 pcc = malloc(sizeof(*pcc), M_CUSE, M_WAITOK | M_ZERO); 1519 if (devfs_set_cdevpriv(pcc, &cuse_client_free)) { 1520 printf("Cuse: Cannot set cdevpriv.\n"); 1521 /* drop reference on server */ 1522 cuse_server_unref(pcs); 1523 free(pcc, M_CUSE); 1524 return (ENOMEM); 1525 } 1526 pcc->fflags = fflags; 1527 pcc->server_dev = pcsd; 1528 pcc->server = pcs; 1529 1530 for (n = 0; n != CUSE_CMD_MAX; n++) { 1531 pccmd = &pcc->cmds[n]; 1532 1533 pccmd->sub.dev = pcd; 1534 pccmd->sub.command = n; 1535 pccmd->client = pcc; 1536 1537 sx_init(&pccmd->sx, "cuse-client-sx"); 1538 cv_init(&pccmd->cv, "cuse-client-cv"); 1539 } 1540 1541 cuse_server_lock(pcs); 1542 1543 /* cuse_client_free() assumes that the client is listed somewhere! */ 1544 /* always enqueue */ 1545 1546 TAILQ_INSERT_TAIL(&pcs->hcli, pcc, entry); 1547 1548 /* check if server is closing */ 1549 if ((pcs->is_closing != 0) || (dev->si_drv1 == NULL)) { 1550 error = EINVAL; 1551 } else { 1552 error = 0; 1553 } 1554 cuse_server_unlock(pcs); 1555 1556 if (error) { 1557 devfs_clear_cdevpriv(); /* XXX bugfix */ 1558 return (error); 1559 } 1560 pccmd = &pcc->cmds[CUSE_CMD_OPEN]; 1561 1562 cuse_cmd_lock(pccmd); 1563 1564 cuse_server_lock(pcs); 1565 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0); 1566 1567 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1568 cuse_server_unlock(pcs); 1569 1570 if (error < 0) { 1571 error = cuse_convert_error(error); 1572 } else { 1573 error = 0; 1574 } 1575 1576 cuse_cmd_unlock(pccmd); 1577 1578 if (error) 1579 devfs_clear_cdevpriv(); /* XXX bugfix */ 1580 1581 return (error); 1582 } 1583 1584 static int 1585 cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 1586 { 1587 struct cuse_client_command *pccmd; 1588 struct cuse_client *pcc; 1589 struct cuse_server *pcs; 1590 int error; 1591 1592 error = cuse_client_get(&pcc); 1593 if (error != 0) 1594 return (0); 1595 1596 pccmd = &pcc->cmds[CUSE_CMD_CLOSE]; 1597 pcs = pcc->server; 1598 1599 cuse_cmd_lock(pccmd); 1600 1601 cuse_server_lock(pcs); 1602 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0); 1603 1604 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1605 cuse_cmd_unlock(pccmd); 1606 1607 cuse_client_is_closing(pcc); 1608 cuse_server_unlock(pcs); 1609 1610 return (0); 1611 } 1612 1613 static void 1614 cuse_client_kqfilter_poll(struct cdev *dev, struct cuse_client *pcc) 1615 { 1616 struct cuse_server *pcs = pcc->server; 1617 int temp; 1618 1619 cuse_server_lock(pcs); 1620 temp = (pcc->cflags & (CUSE_CLI_KNOTE_HAS_READ | 1621 CUSE_CLI_KNOTE_HAS_WRITE)); 1622 pcc->cflags &= ~(CUSE_CLI_KNOTE_NEED_READ | 1623 CUSE_CLI_KNOTE_NEED_WRITE); 1624 cuse_server_unlock(pcs); 1625 1626 if (temp != 0) { 1627 /* get the latest polling state from the server */ 1628 temp = cuse_client_poll(dev, POLLIN | POLLOUT, NULL); 1629 1630 if (temp & (POLLIN | POLLOUT)) { 1631 cuse_server_lock(pcs); 1632 if (temp & POLLIN) 1633 pcc->cflags |= CUSE_CLI_KNOTE_NEED_READ; 1634 if (temp & POLLOUT) 1635 pcc->cflags |= CUSE_CLI_KNOTE_NEED_WRITE; 1636 1637 /* make sure the "knote" gets woken up */ 1638 cuse_server_wakeup_locked(pcc->server); 1639 cuse_server_unlock(pcs); 1640 } 1641 } 1642 } 1643 1644 static int 1645 cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag) 1646 { 1647 struct cuse_client_command *pccmd; 1648 struct cuse_client *pcc; 1649 struct cuse_server *pcs; 1650 int error; 1651 int temp; 1652 int len; 1653 1654 error = cuse_client_get(&pcc); 1655 if (error != 0) 1656 return (error); 1657 1658 pccmd = &pcc->cmds[CUSE_CMD_READ]; 1659 pcs = pcc->server; 1660 1661 if (uio->uio_segflg != UIO_USERSPACE) { 1662 return (EINVAL); 1663 } 1664 uio->uio_segflg = UIO_NOCOPY; 1665 1666 cuse_cmd_lock(pccmd); 1667 1668 while (uio->uio_resid != 0) { 1669 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) { 1670 error = ENOMEM; 1671 break; 1672 } 1673 len = uio->uio_iov->iov_len; 1674 1675 cuse_server_lock(pcs); 1676 if (len <= CUSE_COPY_BUFFER_MAX) { 1677 /* set read buffer region for small reads */ 1678 pcc->read_base = (uintptr_t)uio->uio_iov->iov_base; 1679 pcc->read_length = len; 1680 } 1681 cuse_client_send_command_locked(pccmd, 1682 (uintptr_t)uio->uio_iov->iov_base, 1683 (unsigned long)(unsigned int)len, pcc->fflags, ioflag); 1684 1685 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1686 /* 1687 * After finishing reading data, disable the read 1688 * region for the cuse_server_data_copy_optimized_locked() 1689 * function: 1690 */ 1691 pcc->read_base = 0; 1692 pcc->read_length = 0; 1693 cuse_server_unlock(pcs); 1694 1695 /* 1696 * The return value indicates the read length, when 1697 * not negative. Range check it just in case to avoid 1698 * passing invalid length values to uiomove(). 1699 */ 1700 if (error > len) { 1701 error = ERANGE; 1702 break; 1703 } else if (error > 0 && len <= CUSE_COPY_BUFFER_MAX) { 1704 temp = copyout(pcc->read_buffer, 1705 uio->uio_iov->iov_base, error); 1706 if (temp != 0) { 1707 error = temp; 1708 break; 1709 } 1710 } 1711 if (error < 0) { 1712 error = cuse_convert_error(error); 1713 break; 1714 } else if (error == len) { 1715 error = uiomove(NULL, error, uio); 1716 if (error) 1717 break; 1718 } else { 1719 error = uiomove(NULL, error, uio); 1720 break; 1721 } 1722 } 1723 cuse_cmd_unlock(pccmd); 1724 1725 uio->uio_segflg = UIO_USERSPACE;/* restore segment flag */ 1726 1727 if (error == EWOULDBLOCK) 1728 cuse_client_kqfilter_poll(dev, pcc); 1729 1730 return (error); 1731 } 1732 1733 static int 1734 cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag) 1735 { 1736 struct cuse_client_command *pccmd; 1737 struct cuse_client *pcc; 1738 struct cuse_server *pcs; 1739 int error; 1740 int len; 1741 1742 error = cuse_client_get(&pcc); 1743 if (error != 0) 1744 return (error); 1745 1746 pccmd = &pcc->cmds[CUSE_CMD_WRITE]; 1747 pcs = pcc->server; 1748 1749 if (uio->uio_segflg != UIO_USERSPACE) { 1750 return (EINVAL); 1751 } 1752 uio->uio_segflg = UIO_NOCOPY; 1753 1754 cuse_cmd_lock(pccmd); 1755 1756 while (uio->uio_resid != 0) { 1757 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) { 1758 error = ENOMEM; 1759 break; 1760 } 1761 len = uio->uio_iov->iov_len; 1762 1763 if (len <= CUSE_COPY_BUFFER_MAX) { 1764 error = copyin(uio->uio_iov->iov_base, 1765 pcc->write_buffer, len); 1766 if (error != 0) 1767 break; 1768 } 1769 1770 cuse_server_lock(pcs); 1771 if (len <= CUSE_COPY_BUFFER_MAX) { 1772 /* set write buffer region for small writes */ 1773 pcc->write_base = (uintptr_t)uio->uio_iov->iov_base; 1774 pcc->write_length = len; 1775 } 1776 cuse_client_send_command_locked(pccmd, 1777 (uintptr_t)uio->uio_iov->iov_base, 1778 (unsigned long)(unsigned int)len, pcc->fflags, ioflag); 1779 1780 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1781 1782 /* 1783 * After finishing writing data, disable the write 1784 * region for the cuse_server_data_copy_optimized_locked() 1785 * function: 1786 */ 1787 pcc->write_base = 0; 1788 pcc->write_length = 0; 1789 cuse_server_unlock(pcs); 1790 1791 /* 1792 * The return value indicates the write length, when 1793 * not negative. Range check it just in case to avoid 1794 * passing invalid length values to uiomove(). 1795 */ 1796 if (error > len) { 1797 error = ERANGE; 1798 break; 1799 } else if (error < 0) { 1800 error = cuse_convert_error(error); 1801 break; 1802 } else if (error == len) { 1803 error = uiomove(NULL, error, uio); 1804 if (error) 1805 break; 1806 } else { 1807 error = uiomove(NULL, error, uio); 1808 break; 1809 } 1810 } 1811 cuse_cmd_unlock(pccmd); 1812 1813 /* restore segment flag */ 1814 uio->uio_segflg = UIO_USERSPACE; 1815 1816 if (error == EWOULDBLOCK) 1817 cuse_client_kqfilter_poll(dev, pcc); 1818 1819 return (error); 1820 } 1821 1822 int 1823 cuse_client_ioctl(struct cdev *dev, unsigned long cmd, 1824 caddr_t data, int fflag, struct thread *td) 1825 { 1826 struct cuse_client_command *pccmd; 1827 struct cuse_client *pcc; 1828 struct cuse_server *pcs; 1829 int error; 1830 int len; 1831 1832 error = cuse_client_get(&pcc); 1833 if (error != 0) 1834 return (error); 1835 1836 len = IOCPARM_LEN(cmd); 1837 if (len > CUSE_BUFFER_MAX) 1838 return (ENOMEM); 1839 1840 pccmd = &pcc->cmds[CUSE_CMD_IOCTL]; 1841 pcs = pcc->server; 1842 1843 cuse_cmd_lock(pccmd); 1844 1845 if (cmd & (IOC_IN | IOC_VOID)) 1846 memcpy(pcc->ioctl_buffer, data, len); 1847 1848 /* 1849 * When the ioctl-length is zero drivers can pass information 1850 * through the data pointer of the ioctl. Make sure this information 1851 * is forwarded to the driver. 1852 */ 1853 1854 cuse_server_lock(pcs); 1855 cuse_client_send_command_locked(pccmd, 1856 (len == 0) ? *(long *)data : CUSE_BUF_MIN_PTR, 1857 (unsigned long)cmd, pcc->fflags, 1858 (fflag & O_NONBLOCK) ? IO_NDELAY : 0); 1859 1860 error = cuse_client_receive_command_locked(pccmd, data, len); 1861 cuse_server_unlock(pcs); 1862 1863 if (error < 0) { 1864 error = cuse_convert_error(error); 1865 } else { 1866 error = 0; 1867 } 1868 1869 if (cmd & IOC_OUT) 1870 memcpy(data, pcc->ioctl_buffer, len); 1871 1872 cuse_cmd_unlock(pccmd); 1873 1874 if (error == EWOULDBLOCK) 1875 cuse_client_kqfilter_poll(dev, pcc); 1876 1877 return (error); 1878 } 1879 1880 static int 1881 cuse_client_poll(struct cdev *dev, int events, struct thread *td) 1882 { 1883 struct cuse_client_command *pccmd; 1884 struct cuse_client *pcc; 1885 struct cuse_server *pcs; 1886 unsigned long temp; 1887 int error; 1888 int revents; 1889 1890 error = cuse_client_get(&pcc); 1891 if (error != 0) 1892 goto pollnval; 1893 1894 temp = 0; 1895 pcs = pcc->server; 1896 1897 if (events & (POLLPRI | POLLIN | POLLRDNORM)) 1898 temp |= CUSE_POLL_READ; 1899 1900 if (events & (POLLOUT | POLLWRNORM)) 1901 temp |= CUSE_POLL_WRITE; 1902 1903 if (events & POLLHUP) 1904 temp |= CUSE_POLL_ERROR; 1905 1906 pccmd = &pcc->cmds[CUSE_CMD_POLL]; 1907 1908 cuse_cmd_lock(pccmd); 1909 1910 /* Need to selrecord() first to not loose any events. */ 1911 if (temp != 0 && td != NULL) 1912 selrecord(td, &pcs->selinfo); 1913 1914 cuse_server_lock(pcs); 1915 cuse_client_send_command_locked(pccmd, 1916 0, temp, pcc->fflags, IO_NDELAY); 1917 1918 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1919 cuse_server_unlock(pcs); 1920 1921 cuse_cmd_unlock(pccmd); 1922 1923 if (error < 0) { 1924 goto pollnval; 1925 } else { 1926 revents = 0; 1927 if (error & CUSE_POLL_READ) 1928 revents |= (events & (POLLPRI | POLLIN | POLLRDNORM)); 1929 if (error & CUSE_POLL_WRITE) 1930 revents |= (events & (POLLOUT | POLLWRNORM)); 1931 if (error & CUSE_POLL_ERROR) 1932 revents |= (events & POLLHUP); 1933 } 1934 return (revents); 1935 1936 pollnval: 1937 /* XXX many clients don't understand POLLNVAL */ 1938 return (events & (POLLHUP | POLLPRI | POLLIN | 1939 POLLRDNORM | POLLOUT | POLLWRNORM)); 1940 } 1941 1942 static int 1943 cuse_client_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 1944 vm_size_t size, struct vm_object **object, int nprot) 1945 { 1946 struct cuse_client *pcc; 1947 int error; 1948 1949 error = cuse_client_get(&pcc); 1950 if (error != 0) 1951 return (error); 1952 1953 return (cuse_common_mmap_single(pcc->server, offset, size, object)); 1954 } 1955 1956 static void 1957 cuse_client_kqfilter_read_detach(struct knote *kn) 1958 { 1959 struct cuse_client *pcc; 1960 struct cuse_server *pcs; 1961 1962 pcc = kn->kn_hook; 1963 pcs = pcc->server; 1964 1965 cuse_server_lock(pcs); 1966 knlist_remove(&pcs->selinfo.si_note, kn, 1); 1967 cuse_server_unlock(pcs); 1968 } 1969 1970 static void 1971 cuse_client_kqfilter_write_detach(struct knote *kn) 1972 { 1973 struct cuse_client *pcc; 1974 struct cuse_server *pcs; 1975 1976 pcc = kn->kn_hook; 1977 pcs = pcc->server; 1978 1979 cuse_server_lock(pcs); 1980 knlist_remove(&pcs->selinfo.si_note, kn, 1); 1981 cuse_server_unlock(pcs); 1982 } 1983 1984 static int 1985 cuse_client_kqfilter_read_event(struct knote *kn, long hint) 1986 { 1987 struct cuse_client *pcc; 1988 1989 pcc = kn->kn_hook; 1990 1991 mtx_assert(&pcc->server->mtx, MA_OWNED); 1992 1993 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_READ) ? 1 : 0); 1994 } 1995 1996 static int 1997 cuse_client_kqfilter_write_event(struct knote *kn, long hint) 1998 { 1999 struct cuse_client *pcc; 2000 2001 pcc = kn->kn_hook; 2002 2003 mtx_assert(&pcc->server->mtx, MA_OWNED); 2004 2005 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_WRITE) ? 1 : 0); 2006 } 2007 2008 static int 2009 cuse_client_kqfilter(struct cdev *dev, struct knote *kn) 2010 { 2011 struct cuse_client *pcc; 2012 struct cuse_server *pcs; 2013 int error; 2014 2015 error = cuse_client_get(&pcc); 2016 if (error != 0) 2017 return (error); 2018 2019 pcs = pcc->server; 2020 2021 cuse_server_lock(pcs); 2022 switch (kn->kn_filter) { 2023 case EVFILT_READ: 2024 pcc->cflags |= CUSE_CLI_KNOTE_HAS_READ; 2025 kn->kn_hook = pcc; 2026 kn->kn_fop = &cuse_client_kqfilter_read_ops; 2027 knlist_add(&pcs->selinfo.si_note, kn, 1); 2028 break; 2029 case EVFILT_WRITE: 2030 pcc->cflags |= CUSE_CLI_KNOTE_HAS_WRITE; 2031 kn->kn_hook = pcc; 2032 kn->kn_fop = &cuse_client_kqfilter_write_ops; 2033 knlist_add(&pcs->selinfo.si_note, kn, 1); 2034 break; 2035 default: 2036 error = EINVAL; 2037 break; 2038 } 2039 cuse_server_unlock(pcs); 2040 2041 if (error == 0) 2042 cuse_client_kqfilter_poll(dev, pcc); 2043 return (error); 2044 } 2045