1 /*-
2 * Copyright (c) 2010-2022 Hans Petter Selasky
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include <sys/stdint.h>
27 #include <sys/stddef.h>
28 #include <sys/param.h>
29 #include <sys/types.h>
30 #include <sys/systm.h>
31 #include <sys/conf.h>
32 #include <sys/kernel.h>
33 #include <sys/bus.h>
34 #include <sys/linker_set.h>
35 #include <sys/module.h>
36 #include <sys/lock.h>
37 #include <sys/mutex.h>
38 #include <sys/condvar.h>
39 #include <sys/sysctl.h>
40 #include <sys/unistd.h>
41 #include <sys/malloc.h>
42 #include <sys/priv.h>
43 #include <sys/uio.h>
44 #include <sys/poll.h>
45 #include <sys/sx.h>
46 #include <sys/rwlock.h>
47 #include <sys/queue.h>
48 #include <sys/fcntl.h>
49 #include <sys/proc.h>
50 #include <sys/vnode.h>
51 #include <sys/selinfo.h>
52 #include <sys/ptrace.h>
53 #include <sys/sysent.h>
54
55 #include <machine/bus.h>
56
57 #include <vm/vm.h>
58 #include <vm/pmap.h>
59 #include <vm/vm_object.h>
60 #include <vm/vm_page.h>
61 #include <vm/vm_pager.h>
62
63 #include <fs/cuse/cuse_defs.h>
64 #include <fs/cuse/cuse_ioctl.h>
65
66 /* set this define to zero to disable this feature */
67 #define CUSE_COPY_BUFFER_MAX \
68 CUSE_BUFFER_MAX
69
70 #define CUSE_ALLOC_PAGES_MAX \
71 (CUSE_ALLOC_BYTES_MAX / PAGE_SIZE)
72
73 #if (CUSE_ALLOC_PAGES_MAX == 0)
74 #error "PAGE_SIZE is too big!"
75 #endif
76
77 static int
cuse_modevent(module_t mod,int type,void * data)78 cuse_modevent(module_t mod, int type, void *data)
79 {
80 switch (type) {
81 case MOD_LOAD:
82 case MOD_UNLOAD:
83 return (0);
84 default:
85 return (EOPNOTSUPP);
86 }
87 }
88
89 static moduledata_t cuse_mod = {
90 .name = "cuse",
91 .evhand = &cuse_modevent,
92 };
93
94 DECLARE_MODULE(cuse, cuse_mod, SI_SUB_DEVFS, SI_ORDER_FIRST);
95 MODULE_VERSION(cuse, 1);
96
97 /*
98 * Prevent cuse4bsd.ko and cuse.ko from loading at the same time by
99 * declaring support for the cuse4bsd interface in cuse.ko:
100 */
101 MODULE_VERSION(cuse4bsd, 1);
102
103 #ifdef FEATURE
104 FEATURE(cuse, "Userspace character devices");
105 #endif
106
107 struct cuse_command;
108 struct cuse_server;
109 struct cuse_client;
110
111 struct cuse_client_command {
112 TAILQ_ENTRY(cuse_client_command) entry;
113 struct cuse_command sub;
114 struct sx sx;
115 struct cv cv;
116 struct thread *entered;
117 struct cuse_client *client;
118 struct proc *proc_curr;
119 int proc_refs;
120 int got_signal;
121 int error;
122 int command;
123 };
124
125 struct cuse_memory {
126 TAILQ_ENTRY(cuse_memory) entry;
127 vm_object_t object;
128 uint32_t page_count;
129 uint32_t alloc_nr;
130 };
131
132 struct cuse_server_dev {
133 TAILQ_ENTRY(cuse_server_dev) entry;
134 struct cuse_server *server;
135 struct cdev *kern_dev;
136 struct cuse_dev *user_dev;
137 };
138
139 struct cuse_server {
140 TAILQ_ENTRY(cuse_server) entry;
141 TAILQ_HEAD(, cuse_client_command) head;
142 TAILQ_HEAD(, cuse_server_dev) hdev;
143 TAILQ_HEAD(, cuse_client) hcli;
144 TAILQ_HEAD(, cuse_memory) hmem;
145 struct mtx mtx;
146 struct cv cv;
147 struct selinfo selinfo;
148 pid_t pid;
149 int is_closing;
150 int refs;
151 };
152
153 struct cuse_client {
154 TAILQ_ENTRY(cuse_client) entry;
155 TAILQ_ENTRY(cuse_client) entry_ref;
156 struct cuse_client_command cmds[CUSE_CMD_MAX];
157 struct cuse_server *server;
158 struct cuse_server_dev *server_dev;
159
160 uintptr_t read_base;
161 uintptr_t write_base;
162 int read_length;
163 int write_length;
164 uint8_t read_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4);
165 uint8_t write_buffer[CUSE_COPY_BUFFER_MAX] __aligned(4);
166 uint8_t ioctl_buffer[CUSE_BUFFER_MAX] __aligned(4);
167
168 int fflags; /* file flags */
169 int cflags; /* client flags */
170 #define CUSE_CLI_IS_CLOSING 0x01
171 #define CUSE_CLI_KNOTE_NEED_READ 0x02
172 #define CUSE_CLI_KNOTE_NEED_WRITE 0x04
173 #define CUSE_CLI_KNOTE_HAS_READ 0x08
174 #define CUSE_CLI_KNOTE_HAS_WRITE 0x10
175 };
176
177 #define CUSE_CLIENT_CLOSING(pcc) \
178 ((pcc)->cflags & CUSE_CLI_IS_CLOSING)
179
180 static MALLOC_DEFINE(M_CUSE, "cuse", "CUSE memory");
181
182 static TAILQ_HEAD(, cuse_server) cuse_server_head;
183 static struct mtx cuse_global_mtx;
184 static struct cdev *cuse_dev;
185 static struct cuse_server *cuse_alloc_unit[CUSE_DEVICES_MAX];
186 static int cuse_alloc_unit_id[CUSE_DEVICES_MAX];
187
188 static void cuse_server_wakeup_all_client_locked(struct cuse_server *pcs);
189 static void cuse_client_kqfilter_read_detach(struct knote *kn);
190 static void cuse_client_kqfilter_write_detach(struct knote *kn);
191 static int cuse_client_kqfilter_read_event(struct knote *kn, long hint);
192 static int cuse_client_kqfilter_write_event(struct knote *kn, long hint);
193
194 static const struct filterops cuse_client_kqfilter_read_ops = {
195 .f_isfd = 1,
196 .f_detach = cuse_client_kqfilter_read_detach,
197 .f_event = cuse_client_kqfilter_read_event,
198 .f_copy = knote_triv_copy,
199 };
200
201 static const struct filterops cuse_client_kqfilter_write_ops = {
202 .f_isfd = 1,
203 .f_detach = cuse_client_kqfilter_write_detach,
204 .f_event = cuse_client_kqfilter_write_event,
205 .f_copy = knote_triv_copy,
206 };
207
208 static d_open_t cuse_client_open;
209 static d_close_t cuse_client_close;
210 static d_ioctl_t cuse_client_ioctl;
211 static d_read_t cuse_client_read;
212 static d_write_t cuse_client_write;
213 static d_poll_t cuse_client_poll;
214 static d_mmap_single_t cuse_client_mmap_single;
215 static d_kqfilter_t cuse_client_kqfilter;
216
217 static struct cdevsw cuse_client_devsw = {
218 .d_version = D_VERSION,
219 .d_open = cuse_client_open,
220 .d_close = cuse_client_close,
221 .d_ioctl = cuse_client_ioctl,
222 .d_name = "cuse_client",
223 .d_flags = D_TRACKCLOSE,
224 .d_read = cuse_client_read,
225 .d_write = cuse_client_write,
226 .d_poll = cuse_client_poll,
227 .d_mmap_single = cuse_client_mmap_single,
228 .d_kqfilter = cuse_client_kqfilter,
229 };
230
231 static d_open_t cuse_server_open;
232 static d_close_t cuse_server_close;
233 static d_ioctl_t cuse_server_ioctl;
234 static d_read_t cuse_server_read;
235 static d_write_t cuse_server_write;
236 static d_poll_t cuse_server_poll;
237 static d_mmap_single_t cuse_server_mmap_single;
238
239 static struct cdevsw cuse_server_devsw = {
240 .d_version = D_VERSION,
241 .d_open = cuse_server_open,
242 .d_close = cuse_server_close,
243 .d_ioctl = cuse_server_ioctl,
244 .d_name = "cuse_server",
245 .d_flags = D_TRACKCLOSE,
246 .d_read = cuse_server_read,
247 .d_write = cuse_server_write,
248 .d_poll = cuse_server_poll,
249 .d_mmap_single = cuse_server_mmap_single,
250 };
251
252 static void cuse_client_is_closing(struct cuse_client *);
253 static int cuse_free_unit_by_id_locked(struct cuse_server *, int);
254
255 static void
cuse_global_lock(void)256 cuse_global_lock(void)
257 {
258 mtx_lock(&cuse_global_mtx);
259 }
260
261 static void
cuse_global_unlock(void)262 cuse_global_unlock(void)
263 {
264 mtx_unlock(&cuse_global_mtx);
265 }
266
267 static void
cuse_server_lock(struct cuse_server * pcs)268 cuse_server_lock(struct cuse_server *pcs)
269 {
270 mtx_lock(&pcs->mtx);
271 }
272
273 static void
cuse_server_unlock(struct cuse_server * pcs)274 cuse_server_unlock(struct cuse_server *pcs)
275 {
276 mtx_unlock(&pcs->mtx);
277 }
278
279 static bool
cuse_server_is_locked(struct cuse_server * pcs)280 cuse_server_is_locked(struct cuse_server *pcs)
281 {
282 return (mtx_owned(&pcs->mtx));
283 }
284
285 static void
cuse_cmd_lock(struct cuse_client_command * pccmd)286 cuse_cmd_lock(struct cuse_client_command *pccmd)
287 {
288 sx_xlock(&pccmd->sx);
289 }
290
291 static void
cuse_cmd_unlock(struct cuse_client_command * pccmd)292 cuse_cmd_unlock(struct cuse_client_command *pccmd)
293 {
294 sx_xunlock(&pccmd->sx);
295 }
296
297 static void
cuse_kern_init(void * arg)298 cuse_kern_init(void *arg)
299 {
300 TAILQ_INIT(&cuse_server_head);
301
302 mtx_init(&cuse_global_mtx, "cuse-global-mtx", NULL, MTX_DEF);
303
304 cuse_dev = make_dev(&cuse_server_devsw, 0,
305 UID_ROOT, GID_OPERATOR, 0600, "cuse");
306
307 printf("Cuse v%d.%d.%d @ /dev/cuse\n",
308 (CUSE_VERSION >> 16) & 0xFF, (CUSE_VERSION >> 8) & 0xFF,
309 (CUSE_VERSION >> 0) & 0xFF);
310 }
311 SYSINIT(cuse_kern_init, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_init, NULL);
312
313 static void
cuse_kern_uninit(void * arg)314 cuse_kern_uninit(void *arg)
315 {
316 void *ptr;
317
318 while (1) {
319 printf("Cuse: Please exit all /dev/cuse instances "
320 "and processes which have used this device.\n");
321
322 pause("DRAIN", 2 * hz);
323
324 cuse_global_lock();
325 ptr = TAILQ_FIRST(&cuse_server_head);
326 cuse_global_unlock();
327
328 if (ptr == NULL)
329 break;
330 }
331
332 if (cuse_dev != NULL)
333 destroy_dev(cuse_dev);
334
335 mtx_destroy(&cuse_global_mtx);
336 }
337 SYSUNINIT(cuse_kern_uninit, SI_SUB_DEVFS, SI_ORDER_ANY, cuse_kern_uninit, NULL);
338
339 static int
cuse_server_get(struct cuse_server ** ppcs)340 cuse_server_get(struct cuse_server **ppcs)
341 {
342 struct cuse_server *pcs;
343 int error;
344
345 error = devfs_get_cdevpriv((void **)&pcs);
346 if (error != 0) {
347 *ppcs = NULL;
348 return (error);
349 }
350 if (pcs->is_closing) {
351 *ppcs = NULL;
352 return (EINVAL);
353 }
354 *ppcs = pcs;
355 return (0);
356 }
357
358 static void
cuse_server_is_closing(struct cuse_server * pcs)359 cuse_server_is_closing(struct cuse_server *pcs)
360 {
361 struct cuse_client *pcc;
362
363 if (pcs->is_closing)
364 return;
365
366 pcs->is_closing = 1;
367
368 TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
369 cuse_client_is_closing(pcc);
370 }
371 }
372
373 static struct cuse_client_command *
cuse_server_find_command(struct cuse_server * pcs,struct thread * td)374 cuse_server_find_command(struct cuse_server *pcs, struct thread *td)
375 {
376 struct cuse_client *pcc;
377 int n;
378
379 if (pcs->is_closing)
380 goto done;
381
382 TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
383 if (CUSE_CLIENT_CLOSING(pcc))
384 continue;
385 for (n = 0; n != CUSE_CMD_MAX; n++) {
386 if (pcc->cmds[n].entered == td)
387 return (&pcc->cmds[n]);
388 }
389 }
390 done:
391 return (NULL);
392 }
393
394 static void
cuse_str_filter(char * ptr)395 cuse_str_filter(char *ptr)
396 {
397 int c;
398
399 while (((c = *ptr) != 0)) {
400 if ((c >= 'a') && (c <= 'z')) {
401 ptr++;
402 continue;
403 }
404 if ((c >= 'A') && (c <= 'Z')) {
405 ptr++;
406 continue;
407 }
408 if ((c >= '0') && (c <= '9')) {
409 ptr++;
410 continue;
411 }
412 if ((c == '.') || (c == '_') || (c == '/')) {
413 ptr++;
414 continue;
415 }
416 *ptr = '_';
417
418 ptr++;
419 }
420 }
421
422 static int
cuse_convert_error(int error)423 cuse_convert_error(int error)
424 {
425 ; /* indent fix */
426 switch (error) {
427 case CUSE_ERR_NONE:
428 return (0);
429 case CUSE_ERR_BUSY:
430 return (EBUSY);
431 case CUSE_ERR_WOULDBLOCK:
432 return (EWOULDBLOCK);
433 case CUSE_ERR_INVALID:
434 return (EINVAL);
435 case CUSE_ERR_NO_MEMORY:
436 return (ENOMEM);
437 case CUSE_ERR_FAULT:
438 return (EFAULT);
439 case CUSE_ERR_SIGNAL:
440 return (EINTR);
441 case CUSE_ERR_NO_DEVICE:
442 return (ENODEV);
443 default:
444 return (ENXIO);
445 }
446 }
447
448 static void
cuse_vm_memory_free(struct cuse_memory * mem)449 cuse_vm_memory_free(struct cuse_memory *mem)
450 {
451 /* last user is gone - free */
452 vm_object_deallocate(mem->object);
453
454 /* free CUSE memory */
455 free(mem, M_CUSE);
456 }
457
458 static int
cuse_server_alloc_memory(struct cuse_server * pcs,uint32_t alloc_nr,uint32_t page_count)459 cuse_server_alloc_memory(struct cuse_server *pcs, uint32_t alloc_nr,
460 uint32_t page_count)
461 {
462 struct cuse_memory *temp;
463 struct cuse_memory *mem;
464 vm_object_t object;
465 int error;
466
467 mem = malloc(sizeof(*mem), M_CUSE, M_WAITOK | M_ZERO);
468
469 object = vm_pager_allocate(OBJT_SWAP, NULL, PAGE_SIZE * page_count,
470 VM_PROT_DEFAULT, 0, curthread->td_ucred);
471 if (object == NULL) {
472 error = ENOMEM;
473 goto error_0;
474 }
475
476 cuse_server_lock(pcs);
477 /* check if allocation number already exists */
478 TAILQ_FOREACH(temp, &pcs->hmem, entry) {
479 if (temp->alloc_nr == alloc_nr)
480 break;
481 }
482 if (temp != NULL) {
483 cuse_server_unlock(pcs);
484 error = EBUSY;
485 goto error_1;
486 }
487 mem->object = object;
488 mem->page_count = page_count;
489 mem->alloc_nr = alloc_nr;
490 TAILQ_INSERT_TAIL(&pcs->hmem, mem, entry);
491 cuse_server_unlock(pcs);
492
493 return (0);
494
495 error_1:
496 vm_object_deallocate(object);
497 error_0:
498 free(mem, M_CUSE);
499 return (error);
500 }
501
502 static int
cuse_server_free_memory(struct cuse_server * pcs,uint32_t alloc_nr)503 cuse_server_free_memory(struct cuse_server *pcs, uint32_t alloc_nr)
504 {
505 struct cuse_memory *mem;
506
507 cuse_server_lock(pcs);
508 TAILQ_FOREACH(mem, &pcs->hmem, entry) {
509 if (mem->alloc_nr == alloc_nr)
510 break;
511 }
512 if (mem == NULL) {
513 cuse_server_unlock(pcs);
514 return (EINVAL);
515 }
516 TAILQ_REMOVE(&pcs->hmem, mem, entry);
517 cuse_server_unlock(pcs);
518
519 cuse_vm_memory_free(mem);
520
521 return (0);
522 }
523
524 static int
cuse_client_get(struct cuse_client ** ppcc)525 cuse_client_get(struct cuse_client **ppcc)
526 {
527 struct cuse_client *pcc;
528 int error;
529
530 /* try to get private data */
531 error = devfs_get_cdevpriv((void **)&pcc);
532 if (error != 0) {
533 *ppcc = NULL;
534 return (error);
535 }
536 if (CUSE_CLIENT_CLOSING(pcc) || pcc->server->is_closing) {
537 *ppcc = NULL;
538 return (EINVAL);
539 }
540 *ppcc = pcc;
541 return (0);
542 }
543
544 static void
cuse_client_is_closing(struct cuse_client * pcc)545 cuse_client_is_closing(struct cuse_client *pcc)
546 {
547 struct cuse_client_command *pccmd;
548 uint32_t n;
549
550 if (CUSE_CLIENT_CLOSING(pcc))
551 return;
552
553 pcc->cflags |= CUSE_CLI_IS_CLOSING;
554 pcc->server_dev = NULL;
555
556 for (n = 0; n != CUSE_CMD_MAX; n++) {
557 pccmd = &pcc->cmds[n];
558
559 if (pccmd->entry.tqe_prev != NULL) {
560 TAILQ_REMOVE(&pcc->server->head, pccmd, entry);
561 pccmd->entry.tqe_prev = NULL;
562 }
563 cv_broadcast(&pccmd->cv);
564 }
565 }
566
567 static void
cuse_client_send_command_locked(struct cuse_client_command * pccmd,uintptr_t data_ptr,unsigned long arg,int fflags,int ioflag)568 cuse_client_send_command_locked(struct cuse_client_command *pccmd,
569 uintptr_t data_ptr, unsigned long arg, int fflags, int ioflag)
570 {
571 unsigned long cuse_fflags = 0;
572 struct cuse_server *pcs;
573
574 if (fflags & FREAD)
575 cuse_fflags |= CUSE_FFLAG_READ;
576
577 if (fflags & FWRITE)
578 cuse_fflags |= CUSE_FFLAG_WRITE;
579
580 if (ioflag & IO_NDELAY)
581 cuse_fflags |= CUSE_FFLAG_NONBLOCK;
582 #if defined(__LP64__)
583 if (SV_CURPROC_FLAG(SV_ILP32))
584 cuse_fflags |= CUSE_FFLAG_COMPAT32;
585 #endif
586 pccmd->sub.fflags = cuse_fflags;
587 pccmd->sub.data_pointer = data_ptr;
588 pccmd->sub.argument = arg;
589
590 pcs = pccmd->client->server;
591
592 if ((pccmd->entry.tqe_prev == NULL) &&
593 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) &&
594 (pcs->is_closing == 0)) {
595 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry);
596 cv_signal(&pcs->cv);
597 }
598 }
599
600 static void
cuse_client_got_signal(struct cuse_client_command * pccmd)601 cuse_client_got_signal(struct cuse_client_command *pccmd)
602 {
603 struct cuse_server *pcs;
604
605 pccmd->got_signal = 1;
606
607 pccmd = &pccmd->client->cmds[CUSE_CMD_SIGNAL];
608
609 pcs = pccmd->client->server;
610
611 if ((pccmd->entry.tqe_prev == NULL) &&
612 (CUSE_CLIENT_CLOSING(pccmd->client) == 0) &&
613 (pcs->is_closing == 0)) {
614 TAILQ_INSERT_TAIL(&pcs->head, pccmd, entry);
615 cv_signal(&pcs->cv);
616 }
617 }
618
619 static int
cuse_client_receive_command_locked(struct cuse_client_command * pccmd,uint8_t * arg_ptr,uint32_t arg_len)620 cuse_client_receive_command_locked(struct cuse_client_command *pccmd,
621 uint8_t *arg_ptr, uint32_t arg_len)
622 {
623 struct cuse_server *pcs;
624 int error;
625
626 pcs = pccmd->client->server;
627 error = 0;
628
629 pccmd->proc_curr = curthread->td_proc;
630
631 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) {
632 error = CUSE_ERR_OTHER;
633 goto done;
634 }
635 while (pccmd->command == CUSE_CMD_NONE) {
636 if (error != 0) {
637 cv_wait(&pccmd->cv, &pcs->mtx);
638 } else {
639 error = cv_wait_sig(&pccmd->cv, &pcs->mtx);
640
641 if (error != 0)
642 cuse_client_got_signal(pccmd);
643 }
644 if (CUSE_CLIENT_CLOSING(pccmd->client) || pcs->is_closing) {
645 error = CUSE_ERR_OTHER;
646 goto done;
647 }
648 }
649
650 error = pccmd->error;
651 pccmd->command = CUSE_CMD_NONE;
652 cv_signal(&pccmd->cv);
653
654 done:
655
656 /* wait until all process references are gone */
657
658 pccmd->proc_curr = NULL;
659
660 while (pccmd->proc_refs != 0)
661 cv_wait(&pccmd->cv, &pcs->mtx);
662
663 return (error);
664 }
665
666 /*------------------------------------------------------------------------*
667 * CUSE SERVER PART
668 *------------------------------------------------------------------------*/
669
670 static void
cuse_server_free_dev(struct cuse_server_dev * pcsd)671 cuse_server_free_dev(struct cuse_server_dev *pcsd)
672 {
673 struct cuse_server *pcs;
674 struct cuse_client *pcc;
675
676 /* get server pointer */
677 pcs = pcsd->server;
678
679 /* prevent creation of more devices */
680 cuse_server_lock(pcs);
681 if (pcsd->kern_dev != NULL)
682 pcsd->kern_dev->si_drv1 = NULL;
683
684 TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
685 if (pcc->server_dev == pcsd)
686 cuse_client_is_closing(pcc);
687 }
688 cuse_server_unlock(pcs);
689
690 /* destroy device, if any */
691 if (pcsd->kern_dev != NULL) {
692 /* destroy device synchronously */
693 destroy_dev(pcsd->kern_dev);
694 }
695 free(pcsd, M_CUSE);
696 }
697
698 static void
cuse_server_unref(struct cuse_server * pcs)699 cuse_server_unref(struct cuse_server *pcs)
700 {
701 struct cuse_server_dev *pcsd;
702 struct cuse_memory *mem;
703
704 cuse_server_lock(pcs);
705 if (--(pcs->refs) != 0) {
706 cuse_server_unlock(pcs);
707 return;
708 }
709 cuse_server_is_closing(pcs);
710 /* final client wakeup, if any */
711 cuse_server_wakeup_all_client_locked(pcs);
712
713 cuse_global_lock();
714 TAILQ_REMOVE(&cuse_server_head, pcs, entry);
715 cuse_global_unlock();
716
717 while ((pcsd = TAILQ_FIRST(&pcs->hdev)) != NULL) {
718 TAILQ_REMOVE(&pcs->hdev, pcsd, entry);
719 cuse_server_unlock(pcs);
720 cuse_server_free_dev(pcsd);
721 cuse_server_lock(pcs);
722 }
723
724 cuse_free_unit_by_id_locked(pcs, -1);
725
726 while ((mem = TAILQ_FIRST(&pcs->hmem)) != NULL) {
727 TAILQ_REMOVE(&pcs->hmem, mem, entry);
728 cuse_server_unlock(pcs);
729 cuse_vm_memory_free(mem);
730 cuse_server_lock(pcs);
731 }
732
733 knlist_clear(&pcs->selinfo.si_note, 1);
734 knlist_destroy(&pcs->selinfo.si_note);
735
736 cuse_server_unlock(pcs);
737
738 seldrain(&pcs->selinfo);
739
740 cv_destroy(&pcs->cv);
741
742 mtx_destroy(&pcs->mtx);
743
744 free(pcs, M_CUSE);
745 }
746
747 static int
cuse_server_do_close(struct cuse_server * pcs)748 cuse_server_do_close(struct cuse_server *pcs)
749 {
750 int retval;
751
752 cuse_server_lock(pcs);
753 cuse_server_is_closing(pcs);
754 /* final client wakeup, if any */
755 cuse_server_wakeup_all_client_locked(pcs);
756
757 knlist_clear(&pcs->selinfo.si_note, 1);
758
759 retval = pcs->refs;
760 cuse_server_unlock(pcs);
761
762 return (retval);
763 }
764
765 static void
cuse_server_free(void * arg)766 cuse_server_free(void *arg)
767 {
768 struct cuse_server *pcs = arg;
769
770 /*
771 * The final server unref should be done by the server thread
772 * to prevent deadlock in the client cdevpriv destructor,
773 * which cannot destroy itself.
774 */
775 while (cuse_server_do_close(pcs) != 1)
776 pause("W", hz);
777
778 /* drop final refcount */
779 cuse_server_unref(pcs);
780 }
781
782 static int
cuse_server_open(struct cdev * dev,int fflags,int devtype,struct thread * td)783 cuse_server_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
784 {
785 struct cuse_server *pcs;
786
787 pcs = malloc(sizeof(*pcs), M_CUSE, M_WAITOK | M_ZERO);
788
789 if (devfs_set_cdevpriv(pcs, &cuse_server_free)) {
790 printf("Cuse: Cannot set cdevpriv.\n");
791 free(pcs, M_CUSE);
792 return (ENOMEM);
793 }
794 /* store current process ID */
795 pcs->pid = curproc->p_pid;
796
797 TAILQ_INIT(&pcs->head);
798 TAILQ_INIT(&pcs->hdev);
799 TAILQ_INIT(&pcs->hcli);
800 TAILQ_INIT(&pcs->hmem);
801
802 cv_init(&pcs->cv, "cuse-server-cv");
803
804 mtx_init(&pcs->mtx, "cuse-server-mtx", NULL, MTX_DEF);
805
806 knlist_init_mtx(&pcs->selinfo.si_note, &pcs->mtx);
807
808 cuse_global_lock();
809 pcs->refs++;
810 TAILQ_INSERT_TAIL(&cuse_server_head, pcs, entry);
811 cuse_global_unlock();
812
813 return (0);
814 }
815
816 static int
cuse_server_close(struct cdev * dev,int fflag,int devtype,struct thread * td)817 cuse_server_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
818 {
819 struct cuse_server *pcs;
820
821 if (cuse_server_get(&pcs) == 0)
822 cuse_server_do_close(pcs);
823
824 return (0);
825 }
826
827 static int
cuse_server_read(struct cdev * dev,struct uio * uio,int ioflag)828 cuse_server_read(struct cdev *dev, struct uio *uio, int ioflag)
829 {
830 return (ENXIO);
831 }
832
833 static int
cuse_server_write(struct cdev * dev,struct uio * uio,int ioflag)834 cuse_server_write(struct cdev *dev, struct uio *uio, int ioflag)
835 {
836 return (ENXIO);
837 }
838
839 static int
cuse_server_ioctl_copy_locked(struct cuse_server * pcs,struct cuse_client_command * pccmd,struct cuse_data_chunk * pchk,bool isread)840 cuse_server_ioctl_copy_locked(struct cuse_server *pcs,
841 struct cuse_client_command *pccmd,
842 struct cuse_data_chunk *pchk, bool isread)
843 {
844 struct proc *p_proc;
845 uint32_t offset;
846 int error;
847
848 offset = pchk->peer_ptr - CUSE_BUF_MIN_PTR;
849
850 if (pchk->length > CUSE_BUFFER_MAX)
851 return (EFAULT);
852
853 if (offset >= CUSE_BUFFER_MAX)
854 return (EFAULT);
855
856 if ((offset + pchk->length) > CUSE_BUFFER_MAX)
857 return (EFAULT);
858
859 p_proc = pccmd->proc_curr;
860 if (p_proc == NULL)
861 return (ENXIO);
862
863 if (pccmd->proc_refs < 0)
864 return (ENOMEM);
865
866 pccmd->proc_refs++;
867
868 cuse_server_unlock(pcs);
869
870 if (!isread) {
871 error = copyin(
872 (void *)pchk->local_ptr,
873 pccmd->client->ioctl_buffer + offset,
874 pchk->length);
875 } else {
876 error = copyout(
877 pccmd->client->ioctl_buffer + offset,
878 (void *)pchk->local_ptr,
879 pchk->length);
880 }
881
882 cuse_server_lock(pcs);
883
884 pccmd->proc_refs--;
885
886 if (pccmd->proc_curr == NULL)
887 cv_signal(&pccmd->cv);
888
889 return (error);
890 }
891
892 static int
cuse_proc2proc_copy(struct proc * proc_s,vm_offset_t data_s,struct proc * proc_d,vm_offset_t data_d,size_t len)893 cuse_proc2proc_copy(struct proc *proc_s, vm_offset_t data_s,
894 struct proc *proc_d, vm_offset_t data_d, size_t len)
895 {
896 struct thread *td;
897 struct proc *proc_cur;
898 int error;
899
900 td = curthread;
901 proc_cur = td->td_proc;
902
903 if (proc_cur == proc_d) {
904 struct iovec iov = {
905 .iov_base = (caddr_t)data_d,
906 .iov_len = len,
907 };
908 struct uio uio = {
909 .uio_iov = &iov,
910 .uio_iovcnt = 1,
911 .uio_offset = (off_t)data_s,
912 .uio_resid = len,
913 .uio_segflg = UIO_USERSPACE,
914 .uio_rw = UIO_READ,
915 .uio_td = td,
916 };
917
918 PHOLD(proc_s);
919 error = proc_rwmem(proc_s, &uio);
920 PRELE(proc_s);
921
922 } else if (proc_cur == proc_s) {
923 struct iovec iov = {
924 .iov_base = (caddr_t)data_s,
925 .iov_len = len,
926 };
927 struct uio uio = {
928 .uio_iov = &iov,
929 .uio_iovcnt = 1,
930 .uio_offset = (off_t)data_d,
931 .uio_resid = len,
932 .uio_segflg = UIO_USERSPACE,
933 .uio_rw = UIO_WRITE,
934 .uio_td = td,
935 };
936
937 PHOLD(proc_d);
938 error = proc_rwmem(proc_d, &uio);
939 PRELE(proc_d);
940 } else {
941 error = EINVAL;
942 }
943 return (error);
944 }
945
946 static int
cuse_server_data_copy_locked(struct cuse_server * pcs,struct cuse_client_command * pccmd,struct cuse_data_chunk * pchk,bool isread)947 cuse_server_data_copy_locked(struct cuse_server *pcs,
948 struct cuse_client_command *pccmd,
949 struct cuse_data_chunk *pchk, bool isread)
950 {
951 struct proc *p_proc;
952 int error;
953
954 p_proc = pccmd->proc_curr;
955 if (p_proc == NULL)
956 return (ENXIO);
957
958 if (pccmd->proc_refs < 0)
959 return (ENOMEM);
960
961 pccmd->proc_refs++;
962
963 cuse_server_unlock(pcs);
964
965 if (!isread) {
966 error = cuse_proc2proc_copy(
967 curthread->td_proc, pchk->local_ptr,
968 p_proc, pchk->peer_ptr,
969 pchk->length);
970 } else {
971 error = cuse_proc2proc_copy(
972 p_proc, pchk->peer_ptr,
973 curthread->td_proc, pchk->local_ptr,
974 pchk->length);
975 }
976
977 cuse_server_lock(pcs);
978
979 pccmd->proc_refs--;
980
981 if (pccmd->proc_curr == NULL)
982 cv_signal(&pccmd->cv);
983
984 return (error);
985 }
986
987 static int
cuse_server_data_copy_optimized_locked(struct cuse_server * pcs,struct cuse_client_command * pccmd,struct cuse_data_chunk * pchk,bool isread)988 cuse_server_data_copy_optimized_locked(struct cuse_server *pcs,
989 struct cuse_client_command *pccmd,
990 struct cuse_data_chunk *pchk, bool isread)
991 {
992 uintptr_t offset;
993 int error;
994
995 /*
996 * Check if data is stored locally to avoid accessing
997 * other process's data space:
998 */
999 if (isread) {
1000 offset = pchk->peer_ptr - pccmd->client->write_base;
1001
1002 if (offset < (uintptr_t)pccmd->client->write_length &&
1003 pchk->length <= (unsigned long)pccmd->client->write_length &&
1004 offset + pchk->length <= (uintptr_t)pccmd->client->write_length) {
1005 cuse_server_unlock(pcs);
1006 error = copyout(pccmd->client->write_buffer + offset,
1007 (void *)pchk->local_ptr, pchk->length);
1008 goto done;
1009 }
1010 } else {
1011 offset = pchk->peer_ptr - pccmd->client->read_base;
1012
1013 if (offset < (uintptr_t)pccmd->client->read_length &&
1014 pchk->length <= (unsigned long)pccmd->client->read_length &&
1015 offset + pchk->length <= (uintptr_t)pccmd->client->read_length) {
1016 cuse_server_unlock(pcs);
1017 error = copyin((void *)pchk->local_ptr,
1018 pccmd->client->read_buffer + offset, pchk->length);
1019 goto done;
1020 }
1021 }
1022
1023 /* use process to process copy function */
1024 error = cuse_server_data_copy_locked(pcs, pccmd, pchk, isread);
1025 done:
1026 return (error);
1027 }
1028
1029 static int
cuse_alloc_unit_by_id_locked(struct cuse_server * pcs,int id)1030 cuse_alloc_unit_by_id_locked(struct cuse_server *pcs, int id)
1031 {
1032 int n;
1033 int x = 0;
1034 int match;
1035
1036 do {
1037 for (match = n = 0; n != CUSE_DEVICES_MAX; n++) {
1038 if (cuse_alloc_unit[n] != NULL) {
1039 if ((cuse_alloc_unit_id[n] ^ id) & CUSE_ID_MASK)
1040 continue;
1041 if ((cuse_alloc_unit_id[n] & ~CUSE_ID_MASK) == x) {
1042 x++;
1043 match = 1;
1044 }
1045 }
1046 }
1047 } while (match);
1048
1049 if (x < 256) {
1050 for (n = 0; n != CUSE_DEVICES_MAX; n++) {
1051 if (cuse_alloc_unit[n] == NULL) {
1052 cuse_alloc_unit[n] = pcs;
1053 cuse_alloc_unit_id[n] = id | x;
1054 return (x);
1055 }
1056 }
1057 }
1058 return (-1);
1059 }
1060
1061 static void
cuse_server_wakeup_locked(struct cuse_server * pcs)1062 cuse_server_wakeup_locked(struct cuse_server *pcs)
1063 {
1064 selwakeup(&pcs->selinfo);
1065 KNOTE_LOCKED(&pcs->selinfo.si_note, 0);
1066 }
1067
1068 static void
cuse_server_wakeup_all_client_locked(struct cuse_server * pcs)1069 cuse_server_wakeup_all_client_locked(struct cuse_server *pcs)
1070 {
1071 struct cuse_client *pcc;
1072
1073 TAILQ_FOREACH(pcc, &pcs->hcli, entry) {
1074 pcc->cflags |= (CUSE_CLI_KNOTE_NEED_READ |
1075 CUSE_CLI_KNOTE_NEED_WRITE);
1076 }
1077 cuse_server_wakeup_locked(pcs);
1078 }
1079
1080 static int
cuse_free_unit_by_id_locked(struct cuse_server * pcs,int id)1081 cuse_free_unit_by_id_locked(struct cuse_server *pcs, int id)
1082 {
1083 int n;
1084 int found = 0;
1085
1086 for (n = 0; n != CUSE_DEVICES_MAX; n++) {
1087 if (cuse_alloc_unit[n] == pcs) {
1088 if (cuse_alloc_unit_id[n] == id || id == -1) {
1089 cuse_alloc_unit[n] = NULL;
1090 cuse_alloc_unit_id[n] = 0;
1091 found = 1;
1092 }
1093 }
1094 }
1095
1096 return (found ? 0 : EINVAL);
1097 }
1098
1099 static int
cuse_server_ioctl(struct cdev * dev,unsigned long cmd,caddr_t data,int fflag,struct thread * td)1100 cuse_server_ioctl(struct cdev *dev, unsigned long cmd,
1101 caddr_t data, int fflag, struct thread *td)
1102 {
1103 struct cuse_server *pcs;
1104 int error;
1105
1106 error = cuse_server_get(&pcs);
1107 if (error != 0)
1108 return (error);
1109
1110 switch (cmd) {
1111 struct cuse_client_command *pccmd;
1112 struct cuse_client *pcc;
1113 struct cuse_command *pcmd;
1114 struct cuse_alloc_info *pai;
1115 struct cuse_create_dev *pcd;
1116 struct cuse_server_dev *pcsd;
1117 struct cuse_data_chunk *pchk;
1118 int n;
1119
1120 case CUSE_IOCTL_GET_COMMAND:
1121 pcmd = (void *)data;
1122
1123 cuse_server_lock(pcs);
1124
1125 while ((pccmd = TAILQ_FIRST(&pcs->head)) == NULL) {
1126 error = cv_wait_sig(&pcs->cv, &pcs->mtx);
1127
1128 if (pcs->is_closing)
1129 error = ENXIO;
1130
1131 if (error) {
1132 cuse_server_unlock(pcs);
1133 return (error);
1134 }
1135 }
1136
1137 TAILQ_REMOVE(&pcs->head, pccmd, entry);
1138 pccmd->entry.tqe_prev = NULL;
1139
1140 pccmd->entered = curthread;
1141
1142 *pcmd = pccmd->sub;
1143
1144 cuse_server_unlock(pcs);
1145
1146 break;
1147
1148 case CUSE_IOCTL_SYNC_COMMAND:
1149
1150 cuse_server_lock(pcs);
1151 while ((pccmd = cuse_server_find_command(pcs, curthread)) != NULL) {
1152 /* send sync command */
1153 pccmd->entered = NULL;
1154 pccmd->error = *(int *)data;
1155 pccmd->command = CUSE_CMD_SYNC;
1156
1157 /* signal peer, if any */
1158 cv_signal(&pccmd->cv);
1159 }
1160 cuse_server_unlock(pcs);
1161
1162 break;
1163
1164 case CUSE_IOCTL_ALLOC_UNIT:
1165
1166 cuse_server_lock(pcs);
1167 n = cuse_alloc_unit_by_id_locked(pcs,
1168 CUSE_ID_DEFAULT(0));
1169 cuse_server_unlock(pcs);
1170
1171 if (n < 0)
1172 error = ENOMEM;
1173 else
1174 *(int *)data = n;
1175 break;
1176
1177 case CUSE_IOCTL_ALLOC_UNIT_BY_ID:
1178
1179 n = *(int *)data;
1180
1181 n = (n & CUSE_ID_MASK);
1182
1183 cuse_server_lock(pcs);
1184 n = cuse_alloc_unit_by_id_locked(pcs, n);
1185 cuse_server_unlock(pcs);
1186
1187 if (n < 0)
1188 error = ENOMEM;
1189 else
1190 *(int *)data = n;
1191 break;
1192
1193 case CUSE_IOCTL_FREE_UNIT:
1194
1195 n = *(int *)data;
1196
1197 n = CUSE_ID_DEFAULT(n);
1198
1199 cuse_server_lock(pcs);
1200 error = cuse_free_unit_by_id_locked(pcs, n);
1201 cuse_server_unlock(pcs);
1202 break;
1203
1204 case CUSE_IOCTL_FREE_UNIT_BY_ID:
1205
1206 n = *(int *)data;
1207
1208 cuse_server_lock(pcs);
1209 error = cuse_free_unit_by_id_locked(pcs, n);
1210 cuse_server_unlock(pcs);
1211 break;
1212
1213 case CUSE_IOCTL_ALLOC_MEMORY:
1214
1215 pai = (void *)data;
1216
1217 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) {
1218 error = ENOMEM;
1219 break;
1220 }
1221 if (pai->page_count > CUSE_ALLOC_PAGES_MAX) {
1222 error = ENOMEM;
1223 break;
1224 }
1225 error = cuse_server_alloc_memory(pcs,
1226 pai->alloc_nr, pai->page_count);
1227 break;
1228
1229 case CUSE_IOCTL_FREE_MEMORY:
1230 pai = (void *)data;
1231
1232 if (pai->alloc_nr >= CUSE_ALLOC_UNIT_MAX) {
1233 error = ENOMEM;
1234 break;
1235 }
1236 error = cuse_server_free_memory(pcs, pai->alloc_nr);
1237 break;
1238
1239 case CUSE_IOCTL_GET_SIG:
1240
1241 cuse_server_lock(pcs);
1242 pccmd = cuse_server_find_command(pcs, curthread);
1243
1244 if (pccmd != NULL) {
1245 n = pccmd->got_signal;
1246 pccmd->got_signal = 0;
1247 } else {
1248 n = 0;
1249 }
1250 cuse_server_unlock(pcs);
1251
1252 *(int *)data = n;
1253
1254 break;
1255
1256 case CUSE_IOCTL_SET_PFH:
1257
1258 cuse_server_lock(pcs);
1259 pccmd = cuse_server_find_command(pcs, curthread);
1260
1261 if (pccmd != NULL) {
1262 pcc = pccmd->client;
1263 for (n = 0; n != CUSE_CMD_MAX; n++) {
1264 pcc->cmds[n].sub.per_file_handle = *(uintptr_t *)data;
1265 }
1266 } else {
1267 error = ENXIO;
1268 }
1269 cuse_server_unlock(pcs);
1270 break;
1271
1272 case CUSE_IOCTL_CREATE_DEV:
1273
1274 error = priv_check(curthread, PRIV_DRIVER);
1275 if (error)
1276 break;
1277
1278 pcd = (void *)data;
1279
1280 /* filter input */
1281
1282 pcd->devname[sizeof(pcd->devname) - 1] = 0;
1283
1284 if (pcd->devname[0] == 0) {
1285 error = EINVAL;
1286 break;
1287 }
1288 cuse_str_filter(pcd->devname);
1289
1290 pcd->permissions &= 0777;
1291
1292 /* try to allocate a character device */
1293
1294 pcsd = malloc(sizeof(*pcsd), M_CUSE, M_WAITOK | M_ZERO);
1295
1296 pcsd->server = pcs;
1297
1298 pcsd->user_dev = pcd->dev;
1299
1300 pcsd->kern_dev = make_dev_credf(MAKEDEV_CHECKNAME,
1301 &cuse_client_devsw, 0, NULL, pcd->user_id, pcd->group_id,
1302 pcd->permissions, "%s", pcd->devname);
1303
1304 if (pcsd->kern_dev == NULL) {
1305 free(pcsd, M_CUSE);
1306 error = ENOMEM;
1307 break;
1308 }
1309 pcsd->kern_dev->si_drv1 = pcsd;
1310
1311 cuse_server_lock(pcs);
1312 TAILQ_INSERT_TAIL(&pcs->hdev, pcsd, entry);
1313 cuse_server_unlock(pcs);
1314
1315 break;
1316
1317 case CUSE_IOCTL_DESTROY_DEV:
1318
1319 error = priv_check(curthread, PRIV_DRIVER);
1320 if (error)
1321 break;
1322
1323 cuse_server_lock(pcs);
1324
1325 error = EINVAL;
1326
1327 pcsd = TAILQ_FIRST(&pcs->hdev);
1328 while (pcsd != NULL) {
1329 if (pcsd->user_dev == *(struct cuse_dev **)data) {
1330 TAILQ_REMOVE(&pcs->hdev, pcsd, entry);
1331 cuse_server_unlock(pcs);
1332 cuse_server_free_dev(pcsd);
1333 cuse_server_lock(pcs);
1334 error = 0;
1335 pcsd = TAILQ_FIRST(&pcs->hdev);
1336 } else {
1337 pcsd = TAILQ_NEXT(pcsd, entry);
1338 }
1339 }
1340
1341 cuse_server_unlock(pcs);
1342 break;
1343
1344 case CUSE_IOCTL_WRITE_DATA:
1345 case CUSE_IOCTL_READ_DATA:
1346
1347 cuse_server_lock(pcs);
1348 pchk = (struct cuse_data_chunk *)data;
1349
1350 pccmd = cuse_server_find_command(pcs, curthread);
1351
1352 if (pccmd == NULL) {
1353 error = ENXIO; /* invalid request */
1354 } else if (pchk->peer_ptr < CUSE_BUF_MIN_PTR) {
1355 error = EFAULT; /* NULL pointer */
1356 } else if (pchk->length == 0) {
1357 /* NOP */
1358 } else if (pchk->peer_ptr < CUSE_BUF_MAX_PTR) {
1359 error = cuse_server_ioctl_copy_locked(pcs, pccmd,
1360 pchk, cmd == CUSE_IOCTL_READ_DATA);
1361 } else {
1362 error = cuse_server_data_copy_optimized_locked(
1363 pcs, pccmd, pchk, cmd == CUSE_IOCTL_READ_DATA);
1364 }
1365
1366 /*
1367 * Sometimes the functions above drop the server lock
1368 * early as an optimization:
1369 */
1370 if (cuse_server_is_locked(pcs))
1371 cuse_server_unlock(pcs);
1372 break;
1373
1374 case CUSE_IOCTL_SELWAKEUP:
1375 cuse_server_lock(pcs);
1376 /*
1377 * We don't know which direction caused the event.
1378 * Wakeup both!
1379 */
1380 cuse_server_wakeup_all_client_locked(pcs);
1381 cuse_server_unlock(pcs);
1382 break;
1383
1384 default:
1385 error = ENXIO;
1386 break;
1387 }
1388 return (error);
1389 }
1390
1391 static int
cuse_server_poll(struct cdev * dev,int events,struct thread * td)1392 cuse_server_poll(struct cdev *dev, int events, struct thread *td)
1393 {
1394 return (events & (POLLHUP | POLLPRI | POLLIN |
1395 POLLRDNORM | POLLOUT | POLLWRNORM));
1396 }
1397
1398 static int
cuse_common_mmap_single(struct cuse_server * pcs,vm_ooffset_t * offset,vm_size_t size,struct vm_object ** object)1399 cuse_common_mmap_single(struct cuse_server *pcs,
1400 vm_ooffset_t *offset, vm_size_t size, struct vm_object **object)
1401 {
1402 struct cuse_memory *mem;
1403 int error;
1404
1405 /* verify size */
1406 if ((size % PAGE_SIZE) != 0 || (size < PAGE_SIZE))
1407 return (EINVAL);
1408
1409 cuse_server_lock(pcs);
1410 error = ENOMEM;
1411
1412 /* lookup memory structure, if any */
1413 TAILQ_FOREACH(mem, &pcs->hmem, entry) {
1414 vm_ooffset_t min_off;
1415 vm_ooffset_t max_off;
1416
1417 min_off = (mem->alloc_nr << CUSE_ALLOC_UNIT_SHIFT);
1418 max_off = min_off + (PAGE_SIZE * mem->page_count);
1419
1420 if (*offset >= min_off && *offset < max_off) {
1421 /* range check size */
1422 if (size > (max_off - *offset)) {
1423 error = EINVAL;
1424 } else {
1425 /* get new VM object offset to use */
1426 *offset -= min_off;
1427 vm_object_reference(mem->object);
1428 *object = mem->object;
1429 error = 0;
1430 }
1431 break;
1432 }
1433 }
1434 cuse_server_unlock(pcs);
1435 return (error);
1436 }
1437
1438 static int
cuse_server_mmap_single(struct cdev * dev,vm_ooffset_t * offset,vm_size_t size,struct vm_object ** object,int nprot)1439 cuse_server_mmap_single(struct cdev *dev, vm_ooffset_t *offset,
1440 vm_size_t size, struct vm_object **object, int nprot)
1441 {
1442 struct cuse_server *pcs;
1443 int error;
1444
1445 error = cuse_server_get(&pcs);
1446 if (error != 0)
1447 return (error);
1448
1449 return (cuse_common_mmap_single(pcs, offset, size, object));
1450 }
1451
1452 /*------------------------------------------------------------------------*
1453 * CUSE CLIENT PART
1454 *------------------------------------------------------------------------*/
1455 static void
cuse_client_free(void * arg)1456 cuse_client_free(void *arg)
1457 {
1458 struct cuse_client *pcc = arg;
1459 struct cuse_client_command *pccmd;
1460 struct cuse_server *pcs;
1461 int n;
1462
1463 pcs = pcc->server;
1464
1465 cuse_server_lock(pcs);
1466 cuse_client_is_closing(pcc);
1467 TAILQ_REMOVE(&pcs->hcli, pcc, entry);
1468 cuse_server_unlock(pcs);
1469
1470 for (n = 0; n != CUSE_CMD_MAX; n++) {
1471 pccmd = &pcc->cmds[n];
1472
1473 sx_destroy(&pccmd->sx);
1474 cv_destroy(&pccmd->cv);
1475 }
1476
1477 free(pcc, M_CUSE);
1478
1479 /* drop reference on server */
1480 cuse_server_unref(pcs);
1481 }
1482
1483 static int
cuse_client_open(struct cdev * dev,int fflags,int devtype,struct thread * td)1484 cuse_client_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
1485 {
1486 struct cuse_client_command *pccmd;
1487 struct cuse_server_dev *pcsd;
1488 struct cuse_client *pcc;
1489 struct cuse_server *pcs;
1490 struct cuse_dev *pcd;
1491 int error;
1492 int n;
1493
1494 pcsd = dev->si_drv1;
1495 if (pcsd != NULL) {
1496 pcs = pcsd->server;
1497 pcd = pcsd->user_dev;
1498
1499 cuse_server_lock(pcs);
1500 /*
1501 * Check that the refcount didn't wrap and that the
1502 * same process is not both client and server. This
1503 * can easily lead to deadlocks when destroying the
1504 * CUSE character device nodes:
1505 */
1506 pcs->refs++;
1507 if (pcs->refs < 0 || pcs->pid == curproc->p_pid) {
1508 /* overflow or wrong PID */
1509 pcs->refs--;
1510 cuse_server_unlock(pcs);
1511 return (EINVAL);
1512 }
1513 cuse_server_unlock(pcs);
1514 } else {
1515 return (EINVAL);
1516 }
1517
1518 pcc = malloc(sizeof(*pcc), M_CUSE, M_WAITOK | M_ZERO);
1519 if (devfs_set_cdevpriv(pcc, &cuse_client_free)) {
1520 printf("Cuse: Cannot set cdevpriv.\n");
1521 /* drop reference on server */
1522 cuse_server_unref(pcs);
1523 free(pcc, M_CUSE);
1524 return (ENOMEM);
1525 }
1526 pcc->fflags = fflags;
1527 pcc->server_dev = pcsd;
1528 pcc->server = pcs;
1529
1530 for (n = 0; n != CUSE_CMD_MAX; n++) {
1531 pccmd = &pcc->cmds[n];
1532
1533 pccmd->sub.dev = pcd;
1534 pccmd->sub.command = n;
1535 pccmd->client = pcc;
1536
1537 sx_init(&pccmd->sx, "cuse-client-sx");
1538 cv_init(&pccmd->cv, "cuse-client-cv");
1539 }
1540
1541 cuse_server_lock(pcs);
1542
1543 /* cuse_client_free() assumes that the client is listed somewhere! */
1544 /* always enqueue */
1545
1546 TAILQ_INSERT_TAIL(&pcs->hcli, pcc, entry);
1547
1548 /* check if server is closing */
1549 if ((pcs->is_closing != 0) || (dev->si_drv1 == NULL)) {
1550 error = EINVAL;
1551 } else {
1552 error = 0;
1553 }
1554 cuse_server_unlock(pcs);
1555
1556 if (error) {
1557 devfs_clear_cdevpriv(); /* XXX bugfix */
1558 return (error);
1559 }
1560 pccmd = &pcc->cmds[CUSE_CMD_OPEN];
1561
1562 cuse_cmd_lock(pccmd);
1563
1564 cuse_server_lock(pcs);
1565 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0);
1566
1567 error = cuse_client_receive_command_locked(pccmd, 0, 0);
1568 cuse_server_unlock(pcs);
1569
1570 if (error < 0) {
1571 error = cuse_convert_error(error);
1572 } else {
1573 error = 0;
1574 }
1575
1576 cuse_cmd_unlock(pccmd);
1577
1578 if (error)
1579 devfs_clear_cdevpriv(); /* XXX bugfix */
1580
1581 return (error);
1582 }
1583
1584 static int
cuse_client_close(struct cdev * dev,int fflag,int devtype,struct thread * td)1585 cuse_client_close(struct cdev *dev, int fflag, int devtype, struct thread *td)
1586 {
1587 struct cuse_client_command *pccmd;
1588 struct cuse_client *pcc;
1589 struct cuse_server *pcs;
1590 int error;
1591
1592 error = cuse_client_get(&pcc);
1593 if (error != 0)
1594 return (0);
1595
1596 pccmd = &pcc->cmds[CUSE_CMD_CLOSE];
1597 pcs = pcc->server;
1598
1599 cuse_cmd_lock(pccmd);
1600
1601 cuse_server_lock(pcs);
1602 cuse_client_send_command_locked(pccmd, 0, 0, pcc->fflags, 0);
1603
1604 error = cuse_client_receive_command_locked(pccmd, 0, 0);
1605 cuse_cmd_unlock(pccmd);
1606
1607 cuse_client_is_closing(pcc);
1608 cuse_server_unlock(pcs);
1609
1610 return (0);
1611 }
1612
1613 static void
cuse_client_kqfilter_poll(struct cdev * dev,struct cuse_client * pcc)1614 cuse_client_kqfilter_poll(struct cdev *dev, struct cuse_client *pcc)
1615 {
1616 struct cuse_server *pcs = pcc->server;
1617 int temp;
1618
1619 cuse_server_lock(pcs);
1620 temp = (pcc->cflags & (CUSE_CLI_KNOTE_HAS_READ |
1621 CUSE_CLI_KNOTE_HAS_WRITE));
1622 pcc->cflags &= ~(CUSE_CLI_KNOTE_NEED_READ |
1623 CUSE_CLI_KNOTE_NEED_WRITE);
1624 cuse_server_unlock(pcs);
1625
1626 if (temp != 0) {
1627 /* get the latest polling state from the server */
1628 temp = cuse_client_poll(dev, POLLIN | POLLOUT, NULL);
1629
1630 if (temp & (POLLIN | POLLOUT)) {
1631 cuse_server_lock(pcs);
1632 if (temp & POLLIN)
1633 pcc->cflags |= CUSE_CLI_KNOTE_NEED_READ;
1634 if (temp & POLLOUT)
1635 pcc->cflags |= CUSE_CLI_KNOTE_NEED_WRITE;
1636
1637 /* make sure the "knote" gets woken up */
1638 cuse_server_wakeup_locked(pcc->server);
1639 cuse_server_unlock(pcs);
1640 }
1641 }
1642 }
1643
1644 static int
cuse_client_read(struct cdev * dev,struct uio * uio,int ioflag)1645 cuse_client_read(struct cdev *dev, struct uio *uio, int ioflag)
1646 {
1647 struct cuse_client_command *pccmd;
1648 struct cuse_client *pcc;
1649 struct cuse_server *pcs;
1650 int error;
1651 int temp;
1652 int len;
1653
1654 error = cuse_client_get(&pcc);
1655 if (error != 0)
1656 return (error);
1657
1658 pccmd = &pcc->cmds[CUSE_CMD_READ];
1659 pcs = pcc->server;
1660
1661 if (uio->uio_segflg != UIO_USERSPACE) {
1662 return (EINVAL);
1663 }
1664 uio->uio_segflg = UIO_NOCOPY;
1665
1666 cuse_cmd_lock(pccmd);
1667
1668 while (uio->uio_resid != 0) {
1669 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) {
1670 error = ENOMEM;
1671 break;
1672 }
1673 len = uio->uio_iov->iov_len;
1674
1675 cuse_server_lock(pcs);
1676 if (len <= CUSE_COPY_BUFFER_MAX) {
1677 /* set read buffer region for small reads */
1678 pcc->read_base = (uintptr_t)uio->uio_iov->iov_base;
1679 pcc->read_length = len;
1680 }
1681 cuse_client_send_command_locked(pccmd,
1682 (uintptr_t)uio->uio_iov->iov_base,
1683 (unsigned long)(unsigned int)len, pcc->fflags, ioflag);
1684
1685 error = cuse_client_receive_command_locked(pccmd, 0, 0);
1686 /*
1687 * After finishing reading data, disable the read
1688 * region for the cuse_server_data_copy_optimized_locked()
1689 * function:
1690 */
1691 pcc->read_base = 0;
1692 pcc->read_length = 0;
1693 cuse_server_unlock(pcs);
1694
1695 /*
1696 * The return value indicates the read length, when
1697 * not negative. Range check it just in case to avoid
1698 * passing invalid length values to uiomove().
1699 */
1700 if (error > len) {
1701 error = ERANGE;
1702 break;
1703 } else if (error > 0 && len <= CUSE_COPY_BUFFER_MAX) {
1704 temp = copyout(pcc->read_buffer,
1705 uio->uio_iov->iov_base, error);
1706 if (temp != 0) {
1707 error = temp;
1708 break;
1709 }
1710 }
1711 if (error < 0) {
1712 error = cuse_convert_error(error);
1713 break;
1714 } else if (error == len) {
1715 error = uiomove(NULL, error, uio);
1716 if (error)
1717 break;
1718 } else {
1719 error = uiomove(NULL, error, uio);
1720 break;
1721 }
1722 }
1723 cuse_cmd_unlock(pccmd);
1724
1725 uio->uio_segflg = UIO_USERSPACE;/* restore segment flag */
1726
1727 if (error == EWOULDBLOCK)
1728 cuse_client_kqfilter_poll(dev, pcc);
1729
1730 return (error);
1731 }
1732
1733 static int
cuse_client_write(struct cdev * dev,struct uio * uio,int ioflag)1734 cuse_client_write(struct cdev *dev, struct uio *uio, int ioflag)
1735 {
1736 struct cuse_client_command *pccmd;
1737 struct cuse_client *pcc;
1738 struct cuse_server *pcs;
1739 int error;
1740 int len;
1741
1742 error = cuse_client_get(&pcc);
1743 if (error != 0)
1744 return (error);
1745
1746 pccmd = &pcc->cmds[CUSE_CMD_WRITE];
1747 pcs = pcc->server;
1748
1749 if (uio->uio_segflg != UIO_USERSPACE) {
1750 return (EINVAL);
1751 }
1752 uio->uio_segflg = UIO_NOCOPY;
1753
1754 cuse_cmd_lock(pccmd);
1755
1756 while (uio->uio_resid != 0) {
1757 if (uio->uio_iov->iov_len > CUSE_LENGTH_MAX) {
1758 error = ENOMEM;
1759 break;
1760 }
1761 len = uio->uio_iov->iov_len;
1762
1763 if (len <= CUSE_COPY_BUFFER_MAX) {
1764 error = copyin(uio->uio_iov->iov_base,
1765 pcc->write_buffer, len);
1766 if (error != 0)
1767 break;
1768 }
1769
1770 cuse_server_lock(pcs);
1771 if (len <= CUSE_COPY_BUFFER_MAX) {
1772 /* set write buffer region for small writes */
1773 pcc->write_base = (uintptr_t)uio->uio_iov->iov_base;
1774 pcc->write_length = len;
1775 }
1776 cuse_client_send_command_locked(pccmd,
1777 (uintptr_t)uio->uio_iov->iov_base,
1778 (unsigned long)(unsigned int)len, pcc->fflags, ioflag);
1779
1780 error = cuse_client_receive_command_locked(pccmd, 0, 0);
1781
1782 /*
1783 * After finishing writing data, disable the write
1784 * region for the cuse_server_data_copy_optimized_locked()
1785 * function:
1786 */
1787 pcc->write_base = 0;
1788 pcc->write_length = 0;
1789 cuse_server_unlock(pcs);
1790
1791 /*
1792 * The return value indicates the write length, when
1793 * not negative. Range check it just in case to avoid
1794 * passing invalid length values to uiomove().
1795 */
1796 if (error > len) {
1797 error = ERANGE;
1798 break;
1799 } else if (error < 0) {
1800 error = cuse_convert_error(error);
1801 break;
1802 } else if (error == len) {
1803 error = uiomove(NULL, error, uio);
1804 if (error)
1805 break;
1806 } else {
1807 error = uiomove(NULL, error, uio);
1808 break;
1809 }
1810 }
1811 cuse_cmd_unlock(pccmd);
1812
1813 /* restore segment flag */
1814 uio->uio_segflg = UIO_USERSPACE;
1815
1816 if (error == EWOULDBLOCK)
1817 cuse_client_kqfilter_poll(dev, pcc);
1818
1819 return (error);
1820 }
1821
1822 int
cuse_client_ioctl(struct cdev * dev,unsigned long cmd,caddr_t data,int fflag,struct thread * td)1823 cuse_client_ioctl(struct cdev *dev, unsigned long cmd,
1824 caddr_t data, int fflag, struct thread *td)
1825 {
1826 struct cuse_client_command *pccmd;
1827 struct cuse_client *pcc;
1828 struct cuse_server *pcs;
1829 int error;
1830 int len;
1831
1832 error = cuse_client_get(&pcc);
1833 if (error != 0)
1834 return (error);
1835
1836 len = IOCPARM_LEN(cmd);
1837 if (len > CUSE_BUFFER_MAX)
1838 return (ENOMEM);
1839
1840 pccmd = &pcc->cmds[CUSE_CMD_IOCTL];
1841 pcs = pcc->server;
1842
1843 cuse_cmd_lock(pccmd);
1844
1845 if (cmd & (IOC_IN | IOC_VOID))
1846 memcpy(pcc->ioctl_buffer, data, len);
1847
1848 /*
1849 * When the ioctl-length is zero drivers can pass information
1850 * through the data pointer of the ioctl. Make sure this information
1851 * is forwarded to the driver.
1852 */
1853
1854 cuse_server_lock(pcs);
1855 cuse_client_send_command_locked(pccmd,
1856 (len == 0) ? *(long *)data : CUSE_BUF_MIN_PTR,
1857 (unsigned long)cmd, pcc->fflags,
1858 (fflag & O_NONBLOCK) ? IO_NDELAY : 0);
1859
1860 error = cuse_client_receive_command_locked(pccmd, data, len);
1861 cuse_server_unlock(pcs);
1862
1863 if (error < 0) {
1864 error = cuse_convert_error(error);
1865 } else {
1866 error = 0;
1867 }
1868
1869 if (cmd & IOC_OUT)
1870 memcpy(data, pcc->ioctl_buffer, len);
1871
1872 cuse_cmd_unlock(pccmd);
1873
1874 if (error == EWOULDBLOCK)
1875 cuse_client_kqfilter_poll(dev, pcc);
1876
1877 return (error);
1878 }
1879
1880 static int
cuse_client_poll(struct cdev * dev,int events,struct thread * td)1881 cuse_client_poll(struct cdev *dev, int events, struct thread *td)
1882 {
1883 struct cuse_client_command *pccmd;
1884 struct cuse_client *pcc;
1885 struct cuse_server *pcs;
1886 unsigned long temp;
1887 int error;
1888 int revents;
1889
1890 error = cuse_client_get(&pcc);
1891 if (error != 0)
1892 goto pollnval;
1893
1894 temp = 0;
1895 pcs = pcc->server;
1896
1897 if (events & (POLLPRI | POLLIN | POLLRDNORM))
1898 temp |= CUSE_POLL_READ;
1899
1900 if (events & (POLLOUT | POLLWRNORM))
1901 temp |= CUSE_POLL_WRITE;
1902
1903 if (events & POLLHUP)
1904 temp |= CUSE_POLL_ERROR;
1905
1906 pccmd = &pcc->cmds[CUSE_CMD_POLL];
1907
1908 cuse_cmd_lock(pccmd);
1909
1910 /* Need to selrecord() first to not loose any events. */
1911 if (temp != 0 && td != NULL)
1912 selrecord(td, &pcs->selinfo);
1913
1914 cuse_server_lock(pcs);
1915 cuse_client_send_command_locked(pccmd,
1916 0, temp, pcc->fflags, IO_NDELAY);
1917
1918 error = cuse_client_receive_command_locked(pccmd, 0, 0);
1919 cuse_server_unlock(pcs);
1920
1921 cuse_cmd_unlock(pccmd);
1922
1923 if (error < 0) {
1924 goto pollnval;
1925 } else {
1926 revents = 0;
1927 if (error & CUSE_POLL_READ)
1928 revents |= (events & (POLLPRI | POLLIN | POLLRDNORM));
1929 if (error & CUSE_POLL_WRITE)
1930 revents |= (events & (POLLOUT | POLLWRNORM));
1931 if (error & CUSE_POLL_ERROR)
1932 revents |= (events & POLLHUP);
1933 }
1934 return (revents);
1935
1936 pollnval:
1937 /* XXX many clients don't understand POLLNVAL */
1938 return (events & (POLLHUP | POLLPRI | POLLIN |
1939 POLLRDNORM | POLLOUT | POLLWRNORM));
1940 }
1941
1942 static int
cuse_client_mmap_single(struct cdev * dev,vm_ooffset_t * offset,vm_size_t size,struct vm_object ** object,int nprot)1943 cuse_client_mmap_single(struct cdev *dev, vm_ooffset_t *offset,
1944 vm_size_t size, struct vm_object **object, int nprot)
1945 {
1946 struct cuse_client *pcc;
1947 int error;
1948
1949 error = cuse_client_get(&pcc);
1950 if (error != 0)
1951 return (error);
1952
1953 return (cuse_common_mmap_single(pcc->server, offset, size, object));
1954 }
1955
1956 static void
cuse_client_kqfilter_read_detach(struct knote * kn)1957 cuse_client_kqfilter_read_detach(struct knote *kn)
1958 {
1959 struct cuse_client *pcc;
1960 struct cuse_server *pcs;
1961
1962 pcc = kn->kn_hook;
1963 pcs = pcc->server;
1964
1965 cuse_server_lock(pcs);
1966 knlist_remove(&pcs->selinfo.si_note, kn, 1);
1967 cuse_server_unlock(pcs);
1968 }
1969
1970 static void
cuse_client_kqfilter_write_detach(struct knote * kn)1971 cuse_client_kqfilter_write_detach(struct knote *kn)
1972 {
1973 struct cuse_client *pcc;
1974 struct cuse_server *pcs;
1975
1976 pcc = kn->kn_hook;
1977 pcs = pcc->server;
1978
1979 cuse_server_lock(pcs);
1980 knlist_remove(&pcs->selinfo.si_note, kn, 1);
1981 cuse_server_unlock(pcs);
1982 }
1983
1984 static int
cuse_client_kqfilter_read_event(struct knote * kn,long hint)1985 cuse_client_kqfilter_read_event(struct knote *kn, long hint)
1986 {
1987 struct cuse_client *pcc;
1988
1989 pcc = kn->kn_hook;
1990
1991 mtx_assert(&pcc->server->mtx, MA_OWNED);
1992
1993 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_READ) ? 1 : 0);
1994 }
1995
1996 static int
cuse_client_kqfilter_write_event(struct knote * kn,long hint)1997 cuse_client_kqfilter_write_event(struct knote *kn, long hint)
1998 {
1999 struct cuse_client *pcc;
2000
2001 pcc = kn->kn_hook;
2002
2003 mtx_assert(&pcc->server->mtx, MA_OWNED);
2004
2005 return ((pcc->cflags & CUSE_CLI_KNOTE_NEED_WRITE) ? 1 : 0);
2006 }
2007
2008 static int
cuse_client_kqfilter(struct cdev * dev,struct knote * kn)2009 cuse_client_kqfilter(struct cdev *dev, struct knote *kn)
2010 {
2011 struct cuse_client *pcc;
2012 struct cuse_server *pcs;
2013 int error;
2014
2015 error = cuse_client_get(&pcc);
2016 if (error != 0)
2017 return (error);
2018
2019 pcs = pcc->server;
2020
2021 cuse_server_lock(pcs);
2022 switch (kn->kn_filter) {
2023 case EVFILT_READ:
2024 pcc->cflags |= CUSE_CLI_KNOTE_HAS_READ;
2025 kn->kn_hook = pcc;
2026 kn->kn_fop = &cuse_client_kqfilter_read_ops;
2027 knlist_add(&pcs->selinfo.si_note, kn, 1);
2028 break;
2029 case EVFILT_WRITE:
2030 pcc->cflags |= CUSE_CLI_KNOTE_HAS_WRITE;
2031 kn->kn_hook = pcc;
2032 kn->kn_fop = &cuse_client_kqfilter_write_ops;
2033 knlist_add(&pcs->selinfo.si_note, kn, 1);
2034 break;
2035 default:
2036 error = EINVAL;
2037 break;
2038 }
2039 cuse_server_unlock(pcs);
2040
2041 if (error == 0)
2042 cuse_client_kqfilter_poll(dev, pcc);
2043 return (error);
2044 }
2045