1 /*- 2 * Copyright (c) 2010-2022 Hans Petter Selasky 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. Redistributions in binary form must reproduce the above copyright 10 * notice, this list of conditions and the following disclaimer in the 11 * documentation and/or other materials provided with the distribution. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 */ 25 26 #include <sys/stdint.h> 27 #include <sys/stddef.h> 28 #include <sys/param.h> 29 #include <sys/types.h> 30 #include <sys/systm.h> 31 #include <sys/conf.h> 32 #include <sys/kernel.h> 33 #include <sys/bus.h> 34 #include <sys/linker_set.h> 35 #include <sys/module.h> 36 #include <sys/lock.h> 37 #include <sys/mutex.h> 38 #include <sys/condvar.h> 39 #include <sys/sysctl.h> 40 #include <sys/unistd.h> 41 #include <sys/malloc.h> 42 #include <sys/priv.h> 43 #include <sys/uio.h> 44 #include <sys/poll.h> 45 #include <sys/sx.h> 46 #include <sys/rwlock.h> 47 #include <sys/queue.h> 48 #include <sys/fcntl.h> 49 #include <sys/proc.h> 50 #include <sys/vnode.h> 51 #include <sys/selinfo.h> 52 #include <sys/ptrace.h> 53 #include <sys/sysent.h> 54 55 #include <machine/bus.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 #include <vm/vm_object.h> 60 #include <vm/vm_page.h> 61 #include <vm/vm_pager.h> 62 63 #include <fs/cuse/cuse_defs.h> 64 #include <fs/cuse/cuse_ioctl.h> 65 66 /* set this define to zero to disable this feature */ 67 #define CUSE_COPY_BUFFER_MAX \ 68 CUSE_BUFFER_MAX 69 70 #define CUSE_ALLOC_PAGES_MAX \ 71 (CUSE_ALLOC_BYTES_MAX / PAGE_SIZE) 72 73 #if (CUSE_ALLOC_PAGES_MAX == 0) 74 #error "PAGE_SIZE is too big!" 75 #endif 76 77 #ifdef FEATURE 78 FEATURE(cuse, "Userspace character devices"); 79 #endif 80 81 struct cuse_command; 82 struct cuse_server; 83 struct cuse_client; 84 85 struct cuse_client_command { 86 TAILQ_ENTRY(cuse_client_command) entry; 87 struct cuse_command sub; 88 struct sx sx; 89 struct cv cv; 90 struct thread *entered; 91 struct cuse_client *client; 92 struct proc *proc_curr; 93 int proc_refs; 94 int got_signal; 95 int error; 96 int command; 97 }; 98 99 struct cuse_memory { 100 TAILQ_ENTRY(cuse_memory) entry; 101 vm_object_t object; 102 uint32_t page_count; 103 uint32_t alloc_nr; 104 }; 105 106 struct cuse_server_dev { 107 TAILQ_ENTRY(cuse_server_dev) entry; 108 struct cuse_server *server; 109 struct cdev *kern_dev; 110 struct cuse_dev *user_dev; 111 }; 112 113 struct cuse_server { 114 TAILQ_ENTRY(cuse_server) entry; 115 TAILQ_HEAD(, cuse_client_command) head; 116 TAILQ_HEAD(, cuse_server_dev) hdev; 117 TAILQ_HEAD(, cuse_client) hcli; 118 TAILQ_HEAD(, cuse_memory) hmem; 119 struct mtx mtx; 120 struct cv cv; 121 struct selinfo selinfo; 122 pid_t pid; 123 int is_closing; 124 int refs; 125 }; 126 127 struct cuse_client { 128 TAILQ_ENTRY(cuse_client) entry; 129 TAILQ_ENTRY(cuse_client) entry_ref; 130 struct cuse_client_command cmds[CUSE_CMD_MAX]; 131 struct cuse_server *server; 132 struct cuse_server_dev *server_dev; 133 134 uintptr_t read_base; 135 uintptr_t write_base; 136 int read_length; 137 int write_length; 138 uint8_t read_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4); 139 uint8_t write_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4); 140 uint8_t ioctl_buffer[CUSE_BUFFER_MAX] __aligned(4); 141 142 int fflags; /* file flags */ 143 int cflags; /* client flags */ 144 #define CUSE_CLI_IS_CLOSING 0x01 145 #define CUSE_CLI_KNOTE_NEED_READ 0x02 146 #define CUSE_CLI_KNOTE_NEED_WRITE 0x04 147 #define CUSE_CLI_KNOTE_HAS_READ 0x08 148 #define CUSE_CLI_KNOTE_HAS_WRITE 0x10 149 }; 150 151 #define CUSE_CLIENT_CLOSING(pcc) \ 152 ((pcc)->cflags & CUSE_CLI_IS_CLOSING) 153 154 static MALLOC_DEFINE(M_CUSE, "cuse", "CUSE memory"); 155 156 static TAILQ_HEAD(, cuse_server) cuse_server_head; 157 static struct mtx cuse_global_mtx; 158 static struct cdev *cuse_dev; 159 static struct cuse_server *cuse_alloc_unit[CUSE_DEVICES_MAX]; 160 static int cuse_alloc_unit_id[CUSE_DEVICES_MAX]; 161 162 static void cuse_server_wakeup_all_client_locked(struct cuse_server *pcs); 163 static void cuse_client_kqfilter_read_detach(struct knote *kn); 164 static void cuse_client_kqfilter_write_detach(struct knote *kn); 165 static int cuse_client_kqfilter_read_event(struct knote *kn, long hint); 166 static int cuse_client_kqfilter_write_event(struct knote *kn, long hint); 167 168 static const struct filterops cuse_client_kqfilter_read_ops = { 169 .f_isfd = 1, 170 .f_detach = cuse_client_kqfilter_read_detach, 171 .f_event = cuse_client_kqfilter_read_event, 172 .f_copy = knote_triv_copy, 173 }; 174 175 static const struct filterops cuse_client_kqfilter_write_ops = { 176 .f_isfd = 1, 177 .f_detach = cuse_client_kqfilter_write_detach, 178 .f_event = cuse_client_kqfilter_write_event, 179 .f_copy = knote_triv_copy, 180 }; 181 182 static d_open_t cuse_client_open; 183 static d_close_t cuse_client_close; 184 static d_ioctl_t cuse_client_ioctl; 185 static d_read_t cuse_client_read; 186 static d_write_t cuse_client_write; 187 static d_poll_t cuse_client_poll; 188 static d_mmap_single_t cuse_client_mmap_single; 189 static d_kqfilter_t cuse_client_kqfilter; 190 191 static struct cdevsw cuse_client_devsw = { 192 .d_version = D_VERSION, 193 .d_open = cuse_client_open, 194 .d_close = cuse_client_close, 195 .d_ioctl = cuse_client_ioctl, 196 .d_name = "cuse_client", 197 .d_flags = D_TRACKCLOSE, 198 .d_read = cuse_client_read, 199 .d_write = cuse_client_write, 200 .d_poll = cuse_client_poll, 201 .d_mmap_single = cuse_client_mmap_single, 202 .d_kqfilter = cuse_client_kqfilter, 203 }; 204 205 static d_open_t cuse_server_open; 206 static d_close_t cuse_server_close; 207 static d_ioctl_t cuse_server_ioctl; 208 static d_read_t cuse_server_read; 209 static d_write_t cuse_server_write; 210 static d_poll_t cuse_server_poll; 211 static d_mmap_single_t cuse_server_mmap_single; 212 static d_purge_t cuse_server_purge; 213 214 static struct cdevsw cuse_server_devsw = { 215 .d_version = D_VERSION, 216 .d_open = cuse_server_open, 217 .d_close = cuse_server_close, 218 .d_ioctl = cuse_server_ioctl, 219 .d_name = "cuse_server", 220 .d_flags = D_TRACKCLOSE, 221 .d_read = cuse_server_read, 222 .d_write = cuse_server_write, 223 .d_poll = cuse_server_poll, 224 .d_mmap_single = cuse_server_mmap_single, 225 .d_purge = cuse_server_purge, 226 }; 227 228 static void cuse_client_is_closing(struct cuse_client *); 229 static int cuse_free_unit_by_id_locked(struct cuse_server *, int); 230 231 static void 232 cuse_global_lock(void) 233 { 234 mtx_lock(&cuse_global_mtx); 235 } 236 237 static void 238 cuse_global_unlock(void) 239 { 240 mtx_unlock(&cuse_global_mtx); 241 } 242 243 static void 244 cuse_server_lock(struct cuse_server *pcs) 245 { 246 mtx_lock(&pcs->mtx); 247 } 248 249 static void 250 cuse_server_unlock(struct cuse_server *pcs) 251 { 252 mtx_unlock(&pcs->mtx); 253 } 254 255 static bool 256 cuse_server_is_locked(struct cuse_server *pcs) 257 { 258 return (mtx_owned(&pcs->mtx)); 259 } 260 261 static void 262 cuse_cmd_lock(struct cuse_client_command *pccmd) 263 { 264 sx_xlock(&pccmd->sx); 265 } 266 267 static void 268 cuse_cmd_unlock(struct cuse_client_command *pccmd) 269 { 270 sx_xunlock(&pccmd->sx); 271 } 272 273 static int 274 cuse_server_get(struct cuse_server **ppcs) 275 { 276 struct cuse_server *pcs; 277 int error; 278 279 error = devfs_get_cdevpriv((void **)&pcs); 280 if (error != 0) { 281 *ppcs = NULL; 282 return (error); 283 } 284 if (pcs->is_closing) { 285 *ppcs = NULL; 286 return (EINVAL); 287 } 288 *ppcs = pcs; 289 return (0); 290 } 291 292 static void 293 cuse_server_is_closing(struct cuse_server *pcs) 294 { 295 struct cuse_client *pcc; 296 297 if (pcs->is_closing) 298 return; 299 300 pcs->is_closing = 1; 301 302 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 303 cuse_client_is_closing(pcc); 304 } 305 } 306 307 static struct cuse_client_command * 308 cuse_server_find_command(struct cuse_server *pcs, struct thread *td) 309 { 310 struct cuse_client *pcc; 311 int n; 312 313 if (pcs->is_closing) 314 goto done; 315 316 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 317 if (CUSE_CLIENT_CLOSING(pcc)) 318 continue; 319 for (n = 0; n != CUSE_CMD_MAX; n++) { 320 if (pcc->cmds[n].entered == td) 321 return (&pcc->cmds[n]); 322 } 323 } 324 done: 325 return (NULL); 326 } 327 328 static void 329 cuse_str_filter(char *ptr) 330 { 331 int c; 332 333 while (((c = *ptr) != 0)) { 334 if ((c >= 'a') && (c <= 'z')) { 335 ptr++; 336 continue; 337 } 338 if ((c >= 'A') && (c <= 'Z')) { 339 ptr++; 340 continue; 341 } 342 if ((c >= '0') && (c <= '9')) { 343 ptr++; 344 continue; 345 } 346 if ((c == '.') || (c == '_') || (c == '/')) { 347 ptr++; 348 continue; 349 } 350 *ptr = '_'; 351 352 ptr++; 353 } 354 } 355 356 static int 357 cuse_convert_error(int error) 358 { 359 switch (error) { 360 case CUSE_ERR_NONE: 361 return (0); 362 case CUSE_ERR_BUSY: 363 return (EBUSY); 364 case CUSE_ERR_WOULDBLOCK: 365 return (EWOULDBLOCK); 366 case CUSE_ERR_INVALID: 367 return (EINVAL); 368 case CUSE_ERR_NO_MEMORY: 369 return (ENOMEM); 370 case CUSE_ERR_FAULT: 371 return (EFAULT); 372 case CUSE_ERR_SIGNAL: 373 return (EINTR); 374 case CUSE_ERR_NO_DEVICE: 375 return (ENODEV); 376 default: 377 return (ENXIO); 378 } 379 } 380 381 static void 382 cuse_vm_memory_free(struct cuse_memory *mem) 383 { 384 /* last user is gone - free */ 385 vm_object_deallocate(mem->object); 386 387 /* free CUSE memory */ 388 free(mem, M_CUSE); 389 } 390 391 static int 392 cuse_server_alloc_memory(struct cuse_server *pcs, uint32_t alloc_nr, 393 uint32_t page_count) 394 { 395 struct cuse_memory *temp; 396 struct cuse_memory *mem; 397 vm_object_t object; 398 int error; 399 400 mem = malloc(sizeof(*mem), M_CUSE, M_WAITOK | M_ZERO); 401 402 object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * page_count, 403 VM_PROT_DEFAULT, 0, curthread->td_ucred); 404 if (object == NULL) { 405 error = ENOMEM; 406 goto error_0; 407 } 408 409 cuse_server_lock(pcs); 410 /* check if allocation number already exists */ 411 TAILQ_FOREACH(temp, &pcs->hmem, entry) { 412 if (temp->alloc_nr == alloc_nr) 413 break; 414 } 415 if (temp != NULL) { 416 cuse_server_unlock(pcs); 417 error = EBUSY; 418 goto error_1; 419 } 420 mem->object = object; 421 mem->page_count = page_count; 422 mem->alloc_nr = alloc_nr; 423 TAILQ_INSERT_TAIL(&pcs->hmem, mem, entry); 424 cuse_server_unlock(pcs); 425 426 return (0); 427 428 error_1: 429 vm_object_deallocate(object); 430 error_0: 431 free(mem, M_CUSE); 432 return (error); 433 } 434 435 static int 436 cuse_server_free_memory(struct cuse_server *pcs, uint32_t alloc_nr) 437 { 438 struct cuse_memory *mem; 439 440 cuse_server_lock(pcs); 441 TAILQ_FOREACH(mem, &pcs->hmem, entry) { 442 if (mem->alloc_nr == alloc_nr) 443 break; 444 } 445 if (mem == NULL) { 446 cuse_server_unlock(pcs); 447 return (EINVAL); 448 } 449 TAILQ_REMOVE(&pcs->hmem, mem, entry); 450 cuse_server_unlock(pcs); 451 452 cuse_vm_memory_free(mem); 453 454 return (0); 455 } 456 457 static int 458 cuse_client_get(struct cuse_client **ppcc) 459 { 460 struct cuse_client *pcc; 461 int error; 462 463 /* try to get private data */ 464 error = devfs_get_cdevpriv((void **)&pcc); 465 if (error != 0) { 466 *ppcc = NULL; 467 return (error); 468 } 469 if (CUSE_CLIENT_CLOSING(pcc) || pcc->server->is_closing) { 470 *ppcc = NULL; 471 return (EINVAL); 472 } 473 *ppcc = pcc; 474 return (0); 475 } 476 477 static void 478 cuse_client_is_closing(struct cuse_client *pcc) 479 { 480 struct cuse_client_command *pccmd; 481 uint32_t n; 482 483 if (CUSE_CLIENT_CLOSING(pcc)) 484 return; 485 486 pcc->cflags |= CUSE_CLI_IS_CLOSING; 487 pcc->server_dev = NULL; 488 489 for (n = 0; n != CUSE_CMD_MAX; n++) { 490 pccmd = &pcc->cmds[n]; 491 492 if (pccmd->entry.tqe_prev != NULL) { 493 TAILQ_REMOVE(&pcc->server->head, pccmd, entry); 494 pccmd->entry.tqe_prev = NULL; 495 } 496 cv_broadcast(&pccmd->cv); 497 } 498 } 499 500 static void 501 cuse_client_send_command_locked(struct cuse_client_command *pccmd, 502 uintptr_t data_ptr, unsigned long arg, int fflags, int ioflag) 503 { 504 unsigned long cuse_fflags = 0; 505 struct cuse_server *pcs; 506 507 if (fflags & FREAD) 508 cuse_fflags |= CUSE_FFLAG_READ; 509 510 if (fflags & FWRITE) 511 cuse_fflags |= CUSE_FFLAG_WRITE; 512 513 if (ioflag & IO_NDELAY) 514 cuse_fflags |= CUSE_FFLAG_NONBLOCK; 515 #if defined(__LP64__) 516 if (SV_CURPROC_FLAG(SV_ILP32)) 517 cuse_fflags |= CUSE_FFLAG_COMPAT32; 518 #endif 519 pccmd->sub.fflags = cuse_fflags; 520 pccmd->sub.data_pointer = data_ptr; 521 pccmd->sub.argument = arg; 522 523 pcs = pccmd->client->server; 524 525 if ((pccmd->entry.tqe_prev == NULL) && 526 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) && 527 (pcs->is_closing == 0)) { 528 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry); 529 cv_signal(&pcs->cv); 530 } 531 } 532 533 static void 534 cuse_client_got_signal(struct cuse_client_command *pccmd) 535 { 536 struct cuse_server *pcs; 537 538 pccmd->got_signal = 1; 539 540 pccmd = &pccmd->client->cmds[CUSE_CMD_SIGNAL]; 541 542 pcs = pccmd->client->server; 543 544 if ((pccmd->entry.tqe_prev == NULL) && 545 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) && 546 (pcs->is_closing == 0)) { 547 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry); 548 cv_signal(&pcs->cv); 549 } 550 } 551 552 static int 553 cuse_client_receive_command_locked(struct cuse_client_command *pccmd, 554 uint8_t *arg_ptr, uint32_t arg_len) 555 { 556 struct cuse_server *pcs; 557 int error; 558 559 pcs = pccmd->client->server; 560 error = 0; 561 562 pccmd->proc_curr = curthread->td_proc; 563 564 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) { 565 error = CUSE_ERR_OTHER; 566 goto done; 567 } 568 while (pccmd->command == CUSE_CMD_NONE) { 569 if (error != 0) { 570 cv_wait(&pccmd->cv, &pcs->mtx); 571 } else { 572 error = cv_wait_sig(&pccmd->cv, &pcs->mtx); 573 574 if (error != 0) 575 cuse_client_got_signal(pccmd); 576 } 577 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) { 578 error = CUSE_ERR_OTHER; 579 goto done; 580 } 581 } 582 583 error = pccmd->error; 584 pccmd->command = CUSE_CMD_NONE; 585 cv_signal(&pccmd->cv); 586 587 done: 588 589 /* wait until all process references are gone */ 590 591 pccmd->proc_curr = NULL; 592 593 while (pccmd->proc_refs != 0) 594 cv_wait(&pccmd->cv, &pcs->mtx); 595 596 return (error); 597 } 598 599 /*------------------------------------------------------------------------* 600 * CUSE SERVER PART 601 *------------------------------------------------------------------------*/ 602 603 static void 604 cuse_server_free_dev(struct cuse_server_dev *pcsd) 605 { 606 struct cuse_server *pcs; 607 struct cuse_client *pcc; 608 609 /* get server pointer */ 610 pcs = pcsd->server; 611 612 /* prevent creation of more devices */ 613 cuse_server_lock(pcs); 614 pcsd->kern_dev->si_drv1 = NULL; 615 616 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 617 if (pcc->server_dev == pcsd) 618 cuse_client_is_closing(pcc); 619 } 620 cuse_server_unlock(pcs); 621 622 /* destroy device synchronously */ 623 destroy_dev(pcsd->kern_dev); 624 free(pcsd, M_CUSE); 625 } 626 627 static void 628 cuse_server_free_devs_locked(struct cuse_server *pcs) 629 { 630 struct cuse_server_dev *pcsd; 631 632 while ((pcsd = TAILQ_FIRST(&pcs->hdev)) != NULL) { 633 TAILQ_REMOVE(&pcs->hdev, pcsd, entry); 634 cuse_server_unlock(pcs); 635 cuse_server_free_dev(pcsd); 636 cuse_server_lock(pcs); 637 } 638 } 639 640 static void 641 cuse_server_unref(struct cuse_server *pcs) 642 { 643 struct cuse_memory *mem; 644 645 /* 646 * Take the global lock before the server lock, to avoid a lock order 647 * reversal in cuse_kern_uninit(). 648 */ 649 cuse_global_lock(); 650 cuse_server_lock(pcs); 651 MPASS(pcs->refs > 0); 652 if (--(pcs->refs) != 0) { 653 cuse_server_unlock(pcs); 654 cuse_global_unlock(); 655 return; 656 } 657 TAILQ_REMOVE(&cuse_server_head, pcs, entry); 658 cuse_global_unlock(); 659 660 cuse_server_is_closing(pcs); 661 /* final client wakeup, if any */ 662 cuse_server_wakeup_all_client_locked(pcs); 663 664 /* The cdevpriv destructor destroys the devices before unreffing. */ 665 MPASS(TAILQ_EMPTY(&pcs->hdev)); 666 667 cuse_free_unit_by_id_locked(pcs, -1); 668 669 while ((mem = TAILQ_FIRST(&pcs->hmem)) != NULL) { 670 TAILQ_REMOVE(&pcs->hmem, mem, entry); 671 cuse_server_unlock(pcs); 672 cuse_vm_memory_free(mem); 673 cuse_server_lock(pcs); 674 } 675 676 knlist_clear(&pcs->selinfo.si_note, 1); 677 knlist_destroy(&pcs->selinfo.si_note); 678 679 cuse_server_unlock(pcs); 680 681 seldrain(&pcs->selinfo); 682 683 cv_destroy(&pcs->cv); 684 685 mtx_destroy(&pcs->mtx); 686 687 MPASS(pcs->refs == 0); 688 free(pcs, M_CUSE); 689 } 690 691 static int 692 cuse_server_do_close(struct cuse_server *pcs) 693 { 694 int retval; 695 696 cuse_server_lock(pcs); 697 cuse_server_is_closing(pcs); 698 /* final client wakeup, if any */ 699 cuse_server_wakeup_all_client_locked(pcs); 700 701 knlist_clear(&pcs->selinfo.si_note, 1); 702 703 retval = pcs->refs; 704 cuse_server_unlock(pcs); 705 706 return (retval); 707 } 708 709 static void 710 cuse_server_dtor(void *arg) 711 { 712 struct cuse_server *pcs = arg; 713 714 cuse_server_lock(pcs); 715 cuse_server_is_closing(pcs); 716 cuse_server_free_devs_locked(pcs); 717 cuse_server_unlock(pcs); 718 719 /* drop final refcount */ 720 cuse_server_unref(pcs); 721 } 722 723 static int 724 cuse_server_open(struct cdev *dev, int fflags, int devtype, struct thread *td) 725 { 726 struct cuse_server *pcs; 727 728 pcs = malloc(sizeof(*pcs), M_CUSE, M_WAITOK | M_ZERO); 729 730 if (devfs_set_cdevpriv(pcs, &cuse_server_dtor)) { 731 printf("Cuse: Cannot set cdevpriv.\n"); 732 free(pcs, M_CUSE); 733 return (ENOMEM); 734 } 735 /* store current process ID */ 736 pcs->pid = curproc->p_pid; 737 738 TAILQ_INIT(&pcs->head); 739 TAILQ_INIT(&pcs->hdev); 740 TAILQ_INIT(&pcs->hcli); 741 TAILQ_INIT(&pcs->hmem); 742 743 cv_init(&pcs->cv, "cuse-server-cv"); 744 745 mtx_init(&pcs->mtx, "cuse-server-mtx", NULL, MTX_DEF); 746 747 knlist_init_mtx(&pcs->selinfo.si_note, &pcs->mtx); 748 749 cuse_global_lock(); 750 pcs->refs++; 751 TAILQ_INSERT_TAIL(&cuse_server_head, pcs, entry); 752 cuse_global_unlock(); 753 754 return (0); 755 } 756 757 static int 758 cuse_server_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 759 { 760 struct cuse_server *pcs; 761 762 if (cuse_server_get(&pcs) == 0) 763 cuse_server_do_close(pcs); 764 765 return (0); 766 } 767 768 static int 769 cuse_server_read(struct cdev *dev, struct uio *uio, int ioflag) 770 { 771 return (ENXIO); 772 } 773 774 static int 775 cuse_server_write(struct cdev *dev, struct uio *uio, int ioflag) 776 { 777 return (ENXIO); 778 } 779 780 static int 781 cuse_server_ioctl_copy_locked(struct cuse_server *pcs, 782 struct cuse_client_command *pccmd, 783 struct cuse_data_chunk *pchk, bool isread) 784 { 785 struct proc *p_proc; 786 uint32_t offset; 787 int error; 788 789 offset = pchk->peer_ptr - CUSE_BUF_MIN_PTR; 790 791 if (pchk->length > CUSE_BUFFER_MAX) 792 return (EFAULT); 793 794 if (offset >= CUSE_BUFFER_MAX) 795 return (EFAULT); 796 797 if ((offset + pchk->length) > CUSE_BUFFER_MAX) 798 return (EFAULT); 799 800 p_proc = pccmd->proc_curr; 801 if (p_proc == NULL) 802 return (ENXIO); 803 804 if (pccmd->proc_refs < 0) 805 return (ENOMEM); 806 807 pccmd->proc_refs++; 808 809 cuse_server_unlock(pcs); 810 811 if (!isread) { 812 error = copyin( 813 (void *)pchk->local_ptr, 814 pccmd->client->ioctl_buffer + offset, 815 pchk->length); 816 } else { 817 error = copyout( 818 pccmd->client->ioctl_buffer + offset, 819 (void *)pchk->local_ptr, 820 pchk->length); 821 } 822 823 cuse_server_lock(pcs); 824 825 pccmd->proc_refs--; 826 827 if (pccmd->proc_curr == NULL) 828 cv_signal(&pccmd->cv); 829 830 return (error); 831 } 832 833 static int 834 cuse_proc2proc_copy(struct proc *proc_s, vm_offset_t data_s, 835 struct proc *proc_d, vm_offset_t data_d, size_t len) 836 { 837 struct thread *td; 838 struct proc *proc_cur; 839 int error; 840 841 td = curthread; 842 proc_cur = td->td_proc; 843 844 if (proc_cur == proc_d) { 845 struct iovec iov = { 846 .iov_base = (caddr_t)data_d, 847 .iov_len = len, 848 }; 849 struct uio uio = { 850 .uio_iov = &iov, 851 .uio_iovcnt = 1, 852 .uio_offset = (off_t)data_s, 853 .uio_resid = len, 854 .uio_segflg = UIO_USERSPACE, 855 .uio_rw = UIO_READ, 856 .uio_td = td, 857 }; 858 859 PHOLD(proc_s); 860 error = proc_rwmem(proc_s, &uio, 0); 861 PRELE(proc_s); 862 863 } else if (proc_cur == proc_s) { 864 struct iovec iov = { 865 .iov_base = (caddr_t)data_s, 866 .iov_len = len, 867 }; 868 struct uio uio = { 869 .uio_iov = &iov, 870 .uio_iovcnt = 1, 871 .uio_offset = (off_t)data_d, 872 .uio_resid = len, 873 .uio_segflg = UIO_USERSPACE, 874 .uio_rw = UIO_WRITE, 875 .uio_td = td, 876 }; 877 878 PHOLD(proc_d); 879 error = proc_rwmem(proc_d, &uio, 0); 880 PRELE(proc_d); 881 } else { 882 error = EINVAL; 883 } 884 return (error); 885 } 886 887 static int 888 cuse_server_data_copy_locked(struct cuse_server *pcs, 889 struct cuse_client_command *pccmd, 890 struct cuse_data_chunk *pchk, bool isread) 891 { 892 struct proc *p_proc; 893 int error; 894 895 p_proc = pccmd->proc_curr; 896 if (p_proc == NULL) 897 return (ENXIO); 898 899 if (pccmd->proc_refs < 0) 900 return (ENOMEM); 901 902 pccmd->proc_refs++; 903 904 cuse_server_unlock(pcs); 905 906 if (!isread) { 907 error = cuse_proc2proc_copy( 908 curthread->td_proc, pchk->local_ptr, 909 p_proc, pchk->peer_ptr, 910 pchk->length); 911 } else { 912 error = cuse_proc2proc_copy( 913 p_proc, pchk->peer_ptr, 914 curthread->td_proc, pchk->local_ptr, 915 pchk->length); 916 } 917 918 cuse_server_lock(pcs); 919 920 pccmd->proc_refs--; 921 922 if (pccmd->proc_curr == NULL) 923 cv_signal(&pccmd->cv); 924 925 return (error); 926 } 927 928 static int 929 cuse_server_data_copy_optimized_locked(struct cuse_server *pcs, 930 struct cuse_client_command *pccmd, 931 struct cuse_data_chunk *pchk, bool isread) 932 { 933 uintptr_t offset; 934 int error; 935 936 /* 937 * Check if data is stored locally to avoid accessing 938 * other process's data space: 939 */ 940 if (isread) { 941 offset = pchk->peer_ptr - pccmd->client->write_base; 942 943 if (offset < (uintptr_t)pccmd->client->write_length && 944 pchk->length <= (unsigned long)pccmd->client->write_length && 945 offset + pchk->length <= (uintptr_t)pccmd->client->write_length) { 946 cuse_server_unlock(pcs); 947 error = copyout(pccmd->client->write_buffer + offset, 948 (void *)pchk->local_ptr, pchk->length); 949 goto done; 950 } 951 } else { 952 offset = pchk->peer_ptr - pccmd->client->read_base; 953 954 if (offset < (uintptr_t)pccmd->client->read_length && 955 pchk->length <= (unsigned long)pccmd->client->read_length && 956 offset + pchk->length <= (uintptr_t)pccmd->client->read_length) { 957 cuse_server_unlock(pcs); 958 error = copyin((void *)pchk->local_ptr, 959 pccmd->client->read_buffer + offset, pchk->length); 960 goto done; 961 } 962 } 963 964 /* use process to process copy function */ 965 error = cuse_server_data_copy_locked(pcs, pccmd, pchk, isread); 966 done: 967 return (error); 968 } 969 970 static int 971 cuse_alloc_unit_by_id_locked(struct cuse_server *pcs, int id) 972 { 973 int n; 974 int x = 0; 975 int match; 976 977 do { 978 for (match = n = 0; n != CUSE_DEVICES_MAX; n++) { 979 if (cuse_alloc_unit[n] != NULL) { 980 if ((cuse_alloc_unit_id[n] ^ id) & CUSE_ID_MASK) 981 continue; 982 if ((cuse_alloc_unit_id[n] & ~CUSE_ID_MASK) == x) { 983 x++; 984 match = 1; 985 } 986 } 987 } 988 } while (match); 989 990 if (x < 256) { 991 for (n = 0; n != CUSE_DEVICES_MAX; n++) { 992 if (cuse_alloc_unit[n] == NULL) { 993 cuse_alloc_unit[n] = pcs; 994 cuse_alloc_unit_id[n] = id | x; 995 return (x); 996 } 997 } 998 } 999 return (-1); 1000 } 1001 1002 static void 1003 cuse_server_wakeup_locked(struct cuse_server *pcs) 1004 { 1005 selwakeup(&pcs->selinfo); 1006 KNOTE_LOCKED(&pcs->selinfo.si_note, 0); 1007 } 1008 1009 static void 1010 cuse_server_wakeup_all_client_locked(struct cuse_server *pcs) 1011 { 1012 struct cuse_client *pcc; 1013 1014 TAILQ_FOREACH(pcc, &pcs->hcli, entry) { 1015 pcc->cflags |= (CUSE_CLI_KNOTE_NEED_READ | 1016 CUSE_CLI_KNOTE_NEED_WRITE); 1017 } 1018 cuse_server_wakeup_locked(pcs); 1019 } 1020 1021 static int 1022 cuse_free_unit_by_id_locked(struct cuse_server *pcs, int id) 1023 { 1024 int n; 1025 int found = 0; 1026 1027 for (n = 0; n != CUSE_DEVICES_MAX; n++) { 1028 if (cuse_alloc_unit[n] == pcs) { 1029 if (cuse_alloc_unit_id[n] == id || id == -1) { 1030 cuse_alloc_unit[n] = NULL; 1031 cuse_alloc_unit_id[n] = 0; 1032 found = 1; 1033 } 1034 } 1035 } 1036 1037 return (found ? 0 : EINVAL); 1038 } 1039 1040 static int 1041 cuse_server_ioctl(struct cdev *dev, unsigned long cmd, 1042 caddr_t data, int fflag, struct thread *td) 1043 { 1044 struct cuse_server *pcs; 1045 int error; 1046 1047 error = cuse_server_get(&pcs); 1048 if (error != 0) 1049 return (error); 1050 1051 switch (cmd) { 1052 struct cuse_client_command *pccmd; 1053 struct cuse_client *pcc; 1054 struct cuse_command *pcmd; 1055 struct cuse_alloc_info *pai; 1056 struct cuse_create_dev *pcd; 1057 struct cuse_server_dev *pcsd; 1058 struct cuse_data_chunk *pchk; 1059 struct make_dev_args args; 1060 int n; 1061 1062 case CUSE_IOCTL_GET_COMMAND: 1063 pcmd = (void *)data; 1064 1065 cuse_server_lock(pcs); 1066 1067 while ((pccmd = TAILQ_FIRST(&pcs->head)) == NULL) { 1068 if (pcs->is_closing) 1069 error = ENXIO; 1070 else 1071 error = cv_wait_sig(&pcs->cv, &pcs->mtx); 1072 1073 if (error) { 1074 cuse_server_unlock(pcs); 1075 return (error); 1076 } 1077 } 1078 1079 TAILQ_REMOVE(&pcs->head, pccmd, entry); 1080 pccmd->entry.tqe_prev = NULL; 1081 1082 pccmd->entered = curthread; 1083 1084 *pcmd = pccmd->sub; 1085 1086 cuse_server_unlock(pcs); 1087 1088 break; 1089 1090 case CUSE_IOCTL_SYNC_COMMAND: 1091 1092 cuse_server_lock(pcs); 1093 while ((pccmd = cuse_server_find_command(pcs, curthread)) != NULL) { 1094 /* send sync command */ 1095 pccmd->entered = NULL; 1096 pccmd->error = *(int *)data; 1097 pccmd->command = CUSE_CMD_SYNC; 1098 1099 /* signal peer, if any */ 1100 cv_signal(&pccmd->cv); 1101 } 1102 cuse_server_unlock(pcs); 1103 1104 break; 1105 1106 case CUSE_IOCTL_ALLOC_UNIT: 1107 1108 cuse_server_lock(pcs); 1109 n = cuse_alloc_unit_by_id_locked(pcs, 1110 CUSE_ID_DEFAULT(0)); 1111 cuse_server_unlock(pcs); 1112 1113 if (n < 0) 1114 error = ENOMEM; 1115 else 1116 *(int *)data = n; 1117 break; 1118 1119 case CUSE_IOCTL_ALLOC_UNIT_BY_ID: 1120 1121 n = *(int *)data; 1122 1123 n = (n & CUSE_ID_MASK); 1124 1125 cuse_server_lock(pcs); 1126 n = cuse_alloc_unit_by_id_locked(pcs, n); 1127 cuse_server_unlock(pcs); 1128 1129 if (n < 0) 1130 error = ENOMEM; 1131 else 1132 *(int *)data = n; 1133 break; 1134 1135 case CUSE_IOCTL_FREE_UNIT: 1136 1137 n = *(int *)data; 1138 1139 n = CUSE_ID_DEFAULT(n); 1140 1141 cuse_server_lock(pcs); 1142 error = cuse_free_unit_by_id_locked(pcs, n); 1143 cuse_server_unlock(pcs); 1144 break; 1145 1146 case CUSE_IOCTL_FREE_UNIT_BY_ID: 1147 1148 n = *(int *)data; 1149 1150 cuse_server_lock(pcs); 1151 error = cuse_free_unit_by_id_locked(pcs, n); 1152 cuse_server_unlock(pcs); 1153 break; 1154 1155 case CUSE_IOCTL_ALLOC_MEMORY: 1156 1157 pai = (void *)data; 1158 1159 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) { 1160 error = ENOMEM; 1161 break; 1162 } 1163 if (pai->page_count > CUSE_ALLOC_PAGES_MAX) { 1164 error = ENOMEM; 1165 break; 1166 } 1167 error = cuse_server_alloc_memory(pcs, 1168 pai->alloc_nr, pai->page_count); 1169 break; 1170 1171 case CUSE_IOCTL_FREE_MEMORY: 1172 pai = (void *)data; 1173 1174 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) { 1175 error = ENOMEM; 1176 break; 1177 } 1178 error = cuse_server_free_memory(pcs, pai->alloc_nr); 1179 break; 1180 1181 case CUSE_IOCTL_GET_SIG: 1182 1183 cuse_server_lock(pcs); 1184 pccmd = cuse_server_find_command(pcs, curthread); 1185 1186 if (pccmd != NULL) { 1187 n = pccmd->got_signal; 1188 pccmd->got_signal = 0; 1189 } else { 1190 n = 0; 1191 } 1192 cuse_server_unlock(pcs); 1193 1194 *(int *)data = n; 1195 1196 break; 1197 1198 case CUSE_IOCTL_SET_PFH: 1199 1200 cuse_server_lock(pcs); 1201 pccmd = cuse_server_find_command(pcs, curthread); 1202 1203 if (pccmd != NULL) { 1204 pcc = pccmd->client; 1205 for (n = 0; n != CUSE_CMD_MAX; n++) { 1206 pcc->cmds[n].sub.per_file_handle = *(uintptr_t *)data; 1207 } 1208 } else { 1209 error = ENXIO; 1210 } 1211 cuse_server_unlock(pcs); 1212 break; 1213 1214 case CUSE_IOCTL_CREATE_DEV: 1215 1216 error = priv_check(curthread, PRIV_DRIVER); 1217 if (error) 1218 break; 1219 1220 pcd = (void *)data; 1221 1222 /* filter input */ 1223 1224 pcd->devname[sizeof(pcd->devname) - 1] = 0; 1225 1226 if (pcd->devname[0] == 0) { 1227 error = EINVAL; 1228 break; 1229 } 1230 cuse_str_filter(pcd->devname); 1231 1232 pcd->permissions &= 0777; 1233 1234 /* try to allocate a character device */ 1235 1236 pcsd = malloc(sizeof(*pcsd), M_CUSE, M_WAITOK | M_ZERO); 1237 1238 pcsd->server = pcs; 1239 1240 pcsd->user_dev = pcd->dev; 1241 1242 make_dev_args_init(&args); 1243 args.mda_flags = MAKEDEV_CHECKNAME; 1244 args.mda_devsw = &cuse_client_devsw; 1245 args.mda_uid = pcd->user_id; 1246 args.mda_gid = pcd->group_id; 1247 args.mda_mode = pcd->permissions; 1248 args.mda_si_drv1 = pcsd; 1249 1250 error = make_dev_s(&args, &pcsd->kern_dev, "%s", pcd->devname); 1251 if (error != 0) { 1252 free(pcsd, M_CUSE); 1253 break; 1254 } 1255 1256 cuse_server_lock(pcs); 1257 TAILQ_INSERT_TAIL(&pcs->hdev, pcsd, entry); 1258 cuse_server_unlock(pcs); 1259 1260 break; 1261 1262 case CUSE_IOCTL_DESTROY_DEV: 1263 1264 error = priv_check(curthread, PRIV_DRIVER); 1265 if (error) 1266 break; 1267 1268 cuse_server_lock(pcs); 1269 1270 error = EINVAL; 1271 1272 pcsd = TAILQ_FIRST(&pcs->hdev); 1273 while (pcsd != NULL) { 1274 if (pcsd->user_dev == *(struct cuse_dev **)data) { 1275 TAILQ_REMOVE(&pcs->hdev, pcsd, entry); 1276 cuse_server_unlock(pcs); 1277 cuse_server_free_dev(pcsd); 1278 cuse_server_lock(pcs); 1279 error = 0; 1280 pcsd = TAILQ_FIRST(&pcs->hdev); 1281 } else { 1282 pcsd = TAILQ_NEXT(pcsd, entry); 1283 } 1284 } 1285 1286 cuse_server_unlock(pcs); 1287 break; 1288 1289 case CUSE_IOCTL_WRITE_DATA: 1290 case CUSE_IOCTL_READ_DATA: 1291 1292 cuse_server_lock(pcs); 1293 pchk = (struct cuse_data_chunk *)data; 1294 1295 pccmd = cuse_server_find_command(pcs, curthread); 1296 1297 if (pccmd == NULL) { 1298 error = ENXIO; /* invalid request */ 1299 } else if (pchk->peer_ptr < CUSE_BUF_MIN_PTR) { 1300 error = EFAULT; /* NULL pointer */ 1301 } else if (pchk->length == 0) { 1302 /* NOP */ 1303 } else if (pchk->peer_ptr < CUSE_BUF_MAX_PTR) { 1304 error = cuse_server_ioctl_copy_locked(pcs, pccmd, 1305 pchk, cmd == CUSE_IOCTL_READ_DATA); 1306 } else { 1307 error = cuse_server_data_copy_optimized_locked( 1308 pcs, pccmd, pchk, cmd == CUSE_IOCTL_READ_DATA); 1309 } 1310 1311 /* 1312 * Sometimes the functions above drop the server lock 1313 * early as an optimization: 1314 */ 1315 if (cuse_server_is_locked(pcs)) 1316 cuse_server_unlock(pcs); 1317 break; 1318 1319 case CUSE_IOCTL_SELWAKEUP: 1320 cuse_server_lock(pcs); 1321 /* 1322 * We don't know which direction caused the event. 1323 * Wakeup both! 1324 */ 1325 cuse_server_wakeup_all_client_locked(pcs); 1326 cuse_server_unlock(pcs); 1327 break; 1328 1329 default: 1330 error = ENXIO; 1331 break; 1332 } 1333 return (error); 1334 } 1335 1336 static int 1337 cuse_server_poll(struct cdev *dev, int events, struct thread *td) 1338 { 1339 return (events & (POLLHUP | POLLPRI | POLLIN | 1340 POLLRDNORM | POLLOUT | POLLWRNORM)); 1341 } 1342 1343 static int 1344 cuse_common_mmap_single(struct cuse_server *pcs, 1345 vm_ooffset_t *offset, vm_size_t size, struct vm_object **object) 1346 { 1347 struct cuse_memory *mem; 1348 int error; 1349 1350 /* verify size */ 1351 if ((size % PAGE_SIZE) != 0 || (size < PAGE_SIZE)) 1352 return (EINVAL); 1353 1354 cuse_server_lock(pcs); 1355 error = ENOMEM; 1356 1357 /* lookup memory structure, if any */ 1358 TAILQ_FOREACH(mem, &pcs->hmem, entry) { 1359 vm_ooffset_t min_off; 1360 vm_ooffset_t max_off; 1361 1362 min_off = (mem->alloc_nr << CUSE_ALLOC_UNIT_SHIFT); 1363 max_off = min_off + (PAGE_SIZE * mem->page_count); 1364 1365 if (*offset >= min_off && *offset < max_off) { 1366 /* range check size */ 1367 if (size > (max_off - *offset)) { 1368 error = EINVAL; 1369 } else { 1370 /* get new VM object offset to use */ 1371 *offset -= min_off; 1372 vm_object_reference(mem->object); 1373 *object = mem->object; 1374 error = 0; 1375 } 1376 break; 1377 } 1378 } 1379 cuse_server_unlock(pcs); 1380 return (error); 1381 } 1382 1383 static int 1384 cuse_server_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 1385 vm_size_t size, struct vm_object **object, int nprot) 1386 { 1387 struct cuse_server *pcs; 1388 int error; 1389 1390 error = cuse_server_get(&pcs); 1391 if (error != 0) 1392 return (error); 1393 1394 return (cuse_common_mmap_single(pcs, offset, size, object)); 1395 } 1396 1397 static void 1398 cuse_server_purge(struct cdev *dev __unused) 1399 { 1400 struct cuse_server *pcs; 1401 1402 /* 1403 * Wake up the servers sleeping in CUSE_IOCTL_GET_COMMAND, so that 1404 * destroy_dev() can return. 1405 */ 1406 cuse_global_lock(); 1407 TAILQ_FOREACH(pcs, &cuse_server_head, entry) { 1408 cuse_server_lock(pcs); 1409 cuse_server_is_closing(pcs); 1410 cv_broadcast(&pcs->cv); 1411 cuse_server_unlock(pcs); 1412 } 1413 cuse_global_unlock(); 1414 } 1415 1416 /*------------------------------------------------------------------------* 1417 * CUSE CLIENT PART 1418 *------------------------------------------------------------------------*/ 1419 static void 1420 cuse_client_free(void *arg) 1421 { 1422 struct cuse_client *pcc = arg; 1423 struct cuse_client_command *pccmd; 1424 struct cuse_server *pcs; 1425 int n; 1426 1427 pcs = pcc->server; 1428 1429 cuse_server_lock(pcs); 1430 cuse_client_is_closing(pcc); 1431 TAILQ_REMOVE(&pcs->hcli, pcc, entry); 1432 cuse_server_unlock(pcs); 1433 1434 for (n = 0; n != CUSE_CMD_MAX; n++) { 1435 pccmd = &pcc->cmds[n]; 1436 1437 sx_destroy(&pccmd->sx); 1438 cv_destroy(&pccmd->cv); 1439 } 1440 1441 free(pcc, M_CUSE); 1442 1443 /* drop reference on server */ 1444 cuse_server_unref(pcs); 1445 } 1446 1447 static int 1448 cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td) 1449 { 1450 struct cuse_client_command *pccmd; 1451 struct cuse_server_dev *pcsd; 1452 struct cuse_client *pcc; 1453 struct cuse_server *pcs; 1454 struct cuse_dev *pcd; 1455 int error; 1456 int n; 1457 1458 pcsd = dev->si_drv1; 1459 if (pcsd != NULL) { 1460 pcs = pcsd->server; 1461 pcd = pcsd->user_dev; 1462 1463 cuse_server_lock(pcs); 1464 /* 1465 * Check that the refcount didn't wrap and that the 1466 * same process is not both client and server. This 1467 * can easily lead to deadlocks when destroying the 1468 * CUSE character device nodes: 1469 */ 1470 pcs->refs++; 1471 if (pcs->refs < 0 || pcs->pid == curproc->p_pid) { 1472 /* overflow or wrong PID */ 1473 pcs->refs--; 1474 cuse_server_unlock(pcs); 1475 return (EINVAL); 1476 } 1477 cuse_server_unlock(pcs); 1478 } else { 1479 return (EINVAL); 1480 } 1481 1482 pcc = malloc(sizeof(*pcc), M_CUSE, M_WAITOK | M_ZERO); 1483 pcc->fflags = fflags; 1484 pcc->server_dev = pcsd; 1485 pcc->server = pcs; 1486 1487 for (n = 0; n != CUSE_CMD_MAX; n++) { 1488 pccmd = &pcc->cmds[n]; 1489 1490 pccmd->sub.dev = pcd; 1491 pccmd->sub.command = n; 1492 pccmd->client = pcc; 1493 1494 sx_init(&pccmd->sx, "cuse-client-sx"); 1495 cv_init(&pccmd->cv, "cuse-client-cv"); 1496 } 1497 1498 cuse_server_lock(pcs); 1499 1500 /* cuse_client_free() assumes that the client is listed somewhere! */ 1501 /* always enqueue */ 1502 1503 TAILQ_INSERT_TAIL(&pcs->hcli, pcc, entry); 1504 1505 /* check if server is closing */ 1506 if ((pcs->is_closing != 0) || (dev->si_drv1 == NULL)) { 1507 error = EINVAL; 1508 } else { 1509 error = 0; 1510 } 1511 cuse_server_unlock(pcs); 1512 1513 /* 1514 * On error, free the client and unref the server, so that the 1515 * exiting server process does not become unkillable. 1516 */ 1517 if (error != 0) { 1518 cuse_client_free(pcc); 1519 return (error); 1520 } 1521 1522 if ((error = devfs_set_cdevpriv(pcc, &cuse_client_free)) != 0) { 1523 cuse_client_free(pcc); 1524 return (error); 1525 } 1526 1527 pccmd = &pcc->cmds[CUSE_CMD_OPEN]; 1528 1529 cuse_cmd_lock(pccmd); 1530 1531 cuse_server_lock(pcs); 1532 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0); 1533 1534 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1535 cuse_server_unlock(pcs); 1536 1537 if (error < 0) { 1538 error = cuse_convert_error(error); 1539 } else { 1540 error = 0; 1541 } 1542 1543 cuse_cmd_unlock(pccmd); 1544 1545 return (error); 1546 } 1547 1548 static int 1549 cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td) 1550 { 1551 struct cuse_client_command *pccmd; 1552 struct cuse_client *pcc; 1553 struct cuse_server *pcs; 1554 int error; 1555 1556 error = cuse_client_get(&pcc); 1557 if (error != 0) 1558 return (0); 1559 1560 pccmd = &pcc->cmds[CUSE_CMD_CLOSE]; 1561 pcs = pcc->server; 1562 1563 cuse_cmd_lock(pccmd); 1564 1565 cuse_server_lock(pcs); 1566 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0); 1567 1568 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1569 cuse_cmd_unlock(pccmd); 1570 1571 cuse_client_is_closing(pcc); 1572 cuse_server_unlock(pcs); 1573 1574 return (0); 1575 } 1576 1577 static void 1578 cuse_client_kqfilter_poll(struct cdev *dev, struct cuse_client *pcc) 1579 { 1580 struct cuse_server *pcs = pcc->server; 1581 int temp; 1582 1583 cuse_server_lock(pcs); 1584 temp = (pcc->cflags & (CUSE_CLI_KNOTE_HAS_READ | 1585 CUSE_CLI_KNOTE_HAS_WRITE)); 1586 pcc->cflags &= ~(CUSE_CLI_KNOTE_NEED_READ | 1587 CUSE_CLI_KNOTE_NEED_WRITE); 1588 cuse_server_unlock(pcs); 1589 1590 if (temp != 0) { 1591 /* get the latest polling state from the server */ 1592 temp = cuse_client_poll(dev, POLLIN | POLLOUT, NULL); 1593 1594 if (temp & (POLLIN | POLLOUT)) { 1595 cuse_server_lock(pcs); 1596 if (temp & POLLIN) 1597 pcc->cflags |= CUSE_CLI_KNOTE_NEED_READ; 1598 if (temp & POLLOUT) 1599 pcc->cflags |= CUSE_CLI_KNOTE_NEED_WRITE; 1600 1601 /* make sure the "knote" gets woken up */ 1602 cuse_server_wakeup_locked(pcc->server); 1603 cuse_server_unlock(pcs); 1604 } 1605 } 1606 } 1607 1608 static int 1609 cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag) 1610 { 1611 struct cuse_client_command *pccmd; 1612 struct cuse_client *pcc; 1613 struct cuse_server *pcs; 1614 int error; 1615 int temp; 1616 int len; 1617 1618 error = cuse_client_get(&pcc); 1619 if (error != 0) 1620 return (error); 1621 1622 pccmd = &pcc->cmds[CUSE_CMD_READ]; 1623 pcs = pcc->server; 1624 1625 if (uio->uio_segflg != UIO_USERSPACE) { 1626 return (EINVAL); 1627 } 1628 uio->uio_segflg = UIO_NOCOPY; 1629 1630 cuse_cmd_lock(pccmd); 1631 1632 while (uio->uio_resid != 0) { 1633 len = uio->uio_iov->iov_len; 1634 /* 1635 * The uiomove() below does not step past an iovec it has 1636 * just emptied, so do it here, to avoid an infinite loop 1637 * where we are requesting zero-byte transfers. 1638 */ 1639 if (len == 0) { 1640 uio->uio_iov++; 1641 uio->uio_iovcnt--; 1642 continue; 1643 } 1644 if (len > CUSE_LENGTH_MAX) { 1645 error = ENOMEM; 1646 break; 1647 } 1648 1649 cuse_server_lock(pcs); 1650 if (len <= CUSE_COPY_BUFFER_MAX) { 1651 /* set read buffer region for small reads */ 1652 pcc->read_base = (uintptr_t)uio->uio_iov->iov_base; 1653 pcc->read_length = len; 1654 } 1655 cuse_client_send_command_locked(pccmd, 1656 (uintptr_t)uio->uio_iov->iov_base, 1657 (unsigned long)(unsigned int)len, pcc->fflags, ioflag); 1658 1659 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1660 /* 1661 * After finishing reading data, disable the read 1662 * region for the cuse_server_data_copy_optimized_locked() 1663 * function: 1664 */ 1665 pcc->read_base = 0; 1666 pcc->read_length = 0; 1667 cuse_server_unlock(pcs); 1668 1669 /* 1670 * The return value indicates the read length, when 1671 * not negative. Range check it just in case to avoid 1672 * passing invalid length values to uiomove(). 1673 */ 1674 if (error > len) { 1675 error = ERANGE; 1676 break; 1677 } else if (error > 0 && len <= CUSE_COPY_BUFFER_MAX) { 1678 temp = copyout(pcc->read_buffer, 1679 uio->uio_iov->iov_base, error); 1680 if (temp != 0) { 1681 error = temp; 1682 break; 1683 } 1684 } 1685 if (error < 0) { 1686 error = cuse_convert_error(error); 1687 break; 1688 } else if (error == len) { 1689 error = uiomove(NULL, error, uio); 1690 if (error) 1691 break; 1692 } else { 1693 error = uiomove(NULL, error, uio); 1694 break; 1695 } 1696 } 1697 cuse_cmd_unlock(pccmd); 1698 1699 uio->uio_segflg = UIO_USERSPACE;/* restore segment flag */ 1700 1701 if (error == EWOULDBLOCK) 1702 cuse_client_kqfilter_poll(dev, pcc); 1703 1704 return (error); 1705 } 1706 1707 static int 1708 cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag) 1709 { 1710 struct cuse_client_command *pccmd; 1711 struct cuse_client *pcc; 1712 struct cuse_server *pcs; 1713 int error; 1714 int len; 1715 1716 error = cuse_client_get(&pcc); 1717 if (error != 0) 1718 return (error); 1719 1720 pccmd = &pcc->cmds[CUSE_CMD_WRITE]; 1721 pcs = pcc->server; 1722 1723 if (uio->uio_segflg != UIO_USERSPACE) { 1724 return (EINVAL); 1725 } 1726 uio->uio_segflg = UIO_NOCOPY; 1727 1728 cuse_cmd_lock(pccmd); 1729 1730 while (uio->uio_resid != 0) { 1731 len = uio->uio_iov->iov_len; 1732 /* 1733 * The uiomove() below does not step past an iovec it has 1734 * just emptied, so do it here, to avoid an infinite loop 1735 * where we are requesting zero-byte transfers. 1736 */ 1737 if (len == 0) { 1738 uio->uio_iov++; 1739 uio->uio_iovcnt--; 1740 continue; 1741 } 1742 if (len > CUSE_LENGTH_MAX) { 1743 error = ENOMEM; 1744 break; 1745 } 1746 1747 if (len <= CUSE_COPY_BUFFER_MAX) { 1748 error = copyin(uio->uio_iov->iov_base, 1749 pcc->write_buffer, len); 1750 if (error != 0) 1751 break; 1752 } 1753 1754 cuse_server_lock(pcs); 1755 if (len <= CUSE_COPY_BUFFER_MAX) { 1756 /* set write buffer region for small writes */ 1757 pcc->write_base = (uintptr_t)uio->uio_iov->iov_base; 1758 pcc->write_length = len; 1759 } 1760 cuse_client_send_command_locked(pccmd, 1761 (uintptr_t)uio->uio_iov->iov_base, 1762 (unsigned long)(unsigned int)len, pcc->fflags, ioflag); 1763 1764 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1765 1766 /* 1767 * After finishing writing data, disable the write 1768 * region for the cuse_server_data_copy_optimized_locked() 1769 * function: 1770 */ 1771 pcc->write_base = 0; 1772 pcc->write_length = 0; 1773 cuse_server_unlock(pcs); 1774 1775 /* 1776 * The return value indicates the write length, when 1777 * not negative. Range check it just in case to avoid 1778 * passing invalid length values to uiomove(). 1779 */ 1780 if (error > len) { 1781 error = ERANGE; 1782 break; 1783 } else if (error < 0) { 1784 error = cuse_convert_error(error); 1785 break; 1786 } else if (error == len) { 1787 error = uiomove(NULL, error, uio); 1788 if (error) 1789 break; 1790 } else { 1791 error = uiomove(NULL, error, uio); 1792 break; 1793 } 1794 } 1795 cuse_cmd_unlock(pccmd); 1796 1797 /* restore segment flag */ 1798 uio->uio_segflg = UIO_USERSPACE; 1799 1800 if (error == EWOULDBLOCK) 1801 cuse_client_kqfilter_poll(dev, pcc); 1802 1803 return (error); 1804 } 1805 1806 int 1807 cuse_client_ioctl(struct cdev *dev, unsigned long cmd, 1808 caddr_t data, int fflag, struct thread *td) 1809 { 1810 struct cuse_client_command *pccmd; 1811 struct cuse_client *pcc; 1812 struct cuse_server *pcs; 1813 int error; 1814 int len; 1815 1816 error = cuse_client_get(&pcc); 1817 if (error != 0) 1818 return (error); 1819 1820 len = IOCPARM_LEN(cmd); 1821 if (len > CUSE_BUFFER_MAX) 1822 return (ENOMEM); 1823 1824 pccmd = &pcc->cmds[CUSE_CMD_IOCTL]; 1825 pcs = pcc->server; 1826 1827 cuse_cmd_lock(pccmd); 1828 1829 if (cmd & (IOC_IN | IOC_VOID)) 1830 memcpy(pcc->ioctl_buffer, data, len); 1831 1832 /* 1833 * When the ioctl-length is zero drivers can pass information 1834 * through the data pointer of the ioctl. Make sure this information 1835 * is forwarded to the driver. 1836 */ 1837 1838 cuse_server_lock(pcs); 1839 cuse_client_send_command_locked(pccmd, 1840 (len == 0) ? *(long *)data : CUSE_BUF_MIN_PTR, 1841 (unsigned long)cmd, pcc->fflags, 1842 (fflag & O_NONBLOCK) ? IO_NDELAY : 0); 1843 1844 error = cuse_client_receive_command_locked(pccmd, data, len); 1845 cuse_server_unlock(pcs); 1846 1847 if (error < 0) { 1848 error = cuse_convert_error(error); 1849 } else { 1850 error = 0; 1851 } 1852 1853 if (cmd & IOC_OUT) 1854 memcpy(data, pcc->ioctl_buffer, len); 1855 1856 cuse_cmd_unlock(pccmd); 1857 1858 if (error == EWOULDBLOCK) 1859 cuse_client_kqfilter_poll(dev, pcc); 1860 1861 return (error); 1862 } 1863 1864 static int 1865 cuse_client_poll(struct cdev *dev, int events, struct thread *td) 1866 { 1867 struct cuse_client_command *pccmd; 1868 struct cuse_client *pcc; 1869 struct cuse_server *pcs; 1870 unsigned long temp; 1871 int error; 1872 int revents; 1873 1874 error = cuse_client_get(&pcc); 1875 if (error != 0) 1876 goto pollnval; 1877 1878 temp = 0; 1879 pcs = pcc->server; 1880 1881 if (events & (POLLPRI | POLLIN | POLLRDNORM)) 1882 temp |= CUSE_POLL_READ; 1883 1884 if (events & (POLLOUT | POLLWRNORM)) 1885 temp |= CUSE_POLL_WRITE; 1886 1887 if (events & POLLHUP) 1888 temp |= CUSE_POLL_ERROR; 1889 1890 pccmd = &pcc->cmds[CUSE_CMD_POLL]; 1891 1892 cuse_cmd_lock(pccmd); 1893 1894 /* Need to selrecord() first to not loose any events. */ 1895 if (temp != 0 && td != NULL) 1896 selrecord(td, &pcs->selinfo); 1897 1898 cuse_server_lock(pcs); 1899 cuse_client_send_command_locked(pccmd, 1900 0, temp, pcc->fflags, IO_NDELAY); 1901 1902 error = cuse_client_receive_command_locked(pccmd, 0, 0); 1903 cuse_server_unlock(pcs); 1904 1905 cuse_cmd_unlock(pccmd); 1906 1907 if (error < 0) { 1908 goto pollnval; 1909 } else { 1910 revents = 0; 1911 if (error & CUSE_POLL_READ) 1912 revents |= (events & (POLLPRI | POLLIN | POLLRDNORM)); 1913 if (error & CUSE_POLL_WRITE) 1914 revents |= (events & (POLLOUT | POLLWRNORM)); 1915 if (error & CUSE_POLL_ERROR) 1916 revents |= (events & POLLHUP); 1917 } 1918 return (revents); 1919 1920 pollnval: 1921 /* XXX many clients don't understand POLLNVAL */ 1922 return (events & (POLLHUP | POLLPRI | POLLIN | 1923 POLLRDNORM | POLLOUT | POLLWRNORM)); 1924 } 1925 1926 static int 1927 cuse_client_mmap_single(struct cdev *dev, vm_ooffset_t *offset, 1928 vm_size_t size, struct vm_object **object, int nprot) 1929 { 1930 struct cuse_client *pcc; 1931 int error; 1932 1933 error = cuse_client_get(&pcc); 1934 if (error != 0) 1935 return (error); 1936 1937 return (cuse_common_mmap_single(pcc->server, offset, size, object)); 1938 } 1939 1940 static void 1941 cuse_client_kqfilter_read_detach(struct knote *kn) 1942 { 1943 struct cuse_client *pcc; 1944 struct cuse_server *pcs; 1945 1946 pcc = kn->kn_hook; 1947 pcs = pcc->server; 1948 1949 cuse_server_lock(pcs); 1950 knlist_remove(&pcs->selinfo.si_note, kn, 1); 1951 cuse_server_unlock(pcs); 1952 } 1953 1954 static void 1955 cuse_client_kqfilter_write_detach(struct knote *kn) 1956 { 1957 struct cuse_client *pcc; 1958 struct cuse_server *pcs; 1959 1960 pcc = kn->kn_hook; 1961 pcs = pcc->server; 1962 1963 cuse_server_lock(pcs); 1964 knlist_remove(&pcs->selinfo.si_note, kn, 1); 1965 cuse_server_unlock(pcs); 1966 } 1967 1968 static int 1969 cuse_client_kqfilter_read_event(struct knote *kn, long hint) 1970 { 1971 struct cuse_client *pcc; 1972 1973 pcc = kn->kn_hook; 1974 1975 mtx_assert(&pcc->server->mtx, MA_OWNED); 1976 1977 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_READ) ? 1 : 0); 1978 } 1979 1980 static int 1981 cuse_client_kqfilter_write_event(struct knote *kn, long hint) 1982 { 1983 struct cuse_client *pcc; 1984 1985 pcc = kn->kn_hook; 1986 1987 mtx_assert(&pcc->server->mtx, MA_OWNED); 1988 1989 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_WRITE) ? 1 : 0); 1990 } 1991 1992 static int 1993 cuse_client_kqfilter(struct cdev *dev, struct knote *kn) 1994 { 1995 struct cuse_client *pcc; 1996 struct cuse_server *pcs; 1997 int error; 1998 1999 error = cuse_client_get(&pcc); 2000 if (error != 0) 2001 return (error); 2002 2003 pcs = pcc->server; 2004 2005 cuse_server_lock(pcs); 2006 switch (kn->kn_filter) { 2007 case EVFILT_READ: 2008 pcc->cflags |= CUSE_CLI_KNOTE_HAS_READ; 2009 kn->kn_hook = pcc; 2010 kn->kn_fop = &cuse_client_kqfilter_read_ops; 2011 knlist_add(&pcs->selinfo.si_note, kn, 1); 2012 break; 2013 case EVFILT_WRITE: 2014 pcc->cflags |= CUSE_CLI_KNOTE_HAS_WRITE; 2015 kn->kn_hook = pcc; 2016 kn->kn_fop = &cuse_client_kqfilter_write_ops; 2017 knlist_add(&pcs->selinfo.si_note, kn, 1); 2018 break; 2019 default: 2020 error = EINVAL; 2021 break; 2022 } 2023 cuse_server_unlock(pcs); 2024 2025 if (error == 0) 2026 cuse_client_kqfilter_poll(dev, pcc); 2027 return (error); 2028 } 2029 2030 static int 2031 cuse_kern_init(void) 2032 { 2033 TAILQ_INIT(&cuse_server_head); 2034 2035 mtx_init(&cuse_global_mtx, "cuse-global-mtx", NULL, MTX_DEF); 2036 2037 cuse_dev = make_dev_credf(MAKEDEV_CHECKNAME, 2038 &cuse_server_devsw, 0, NULL, UID_ROOT, GID_OPERATOR, 0600, "cuse"); 2039 if (cuse_dev == NULL) 2040 return (ENODEV); 2041 2042 return (0); 2043 } 2044 2045 static void 2046 cuse_kern_uninit(void) 2047 { 2048 /* destroy_dev() runs the cdevpriv destructor of every open instance. */ 2049 if (cuse_dev != NULL) 2050 destroy_dev(cuse_dev); 2051 2052 MPASS(TAILQ_EMPTY(&cuse_server_head)); 2053 2054 mtx_destroy(&cuse_global_mtx); 2055 } 2056 2057 static int 2058 cuse_modevent(module_t mod, int type, void *data) 2059 { 2060 int err = 0; 2061 2062 switch (type) { 2063 case MOD_LOAD: 2064 err = cuse_kern_init(); 2065 break; 2066 case MOD_UNLOAD: 2067 cuse_kern_uninit(); 2068 break; 2069 default: 2070 err = EOPNOTSUPP; 2071 } 2072 2073 return (err); 2074 } 2075 2076 static moduledata_t cuse_mod = { 2077 .name = "cuse", 2078 .evhand = &cuse_modevent, 2079 }; 2080 2081 DECLARE_MODULE(cuse, cuse_mod, SI_SUB_DEVFS, SI_ORDER_FIRST); 2082 MODULE_VERSION(cuse, 1); 2083 2084 /* 2085 * Prevent cuse4bsd.ko and cuse.ko from loading at the same time by 2086 * declaring support for the cuse4bsd interface in cuse.ko: 2087 */ 2088 MODULE_VERSION(cuse4bsd, 1); 2089