1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
5 * Copyright 2004 John-Mark Gurney <jmg@FreeBSD.org>
6 * Copyright (c) 2009 Apple, Inc.
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include "opt_ktrace.h"
32 #include "opt_kqueue.h"
33
34 #ifdef COMPAT_FREEBSD11
35 #define _WANT_FREEBSD11_KEVENT
36 #endif
37
38 #include <sys/param.h>
39 #include <sys/systm.h>
40 #include <sys/capsicum.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/mutex.h>
45 #include <sys/proc.h>
46 #include <sys/malloc.h>
47 #include <sys/unistd.h>
48 #include <sys/file.h>
49 #include <sys/filedesc.h>
50 #include <sys/filio.h>
51 #include <sys/fcntl.h>
52 #include <sys/imgact.h>
53 #include <sys/jail.h>
54 #include <sys/jaildesc.h>
55 #include <sys/kthread.h>
56 #include <sys/selinfo.h>
57 #include <sys/queue.h>
58 #include <sys/event.h>
59 #include <sys/eventvar.h>
60 #include <sys/poll.h>
61 #include <sys/protosw.h>
62 #include <sys/resourcevar.h>
63 #include <sys/sbuf.h>
64 #include <sys/sigio.h>
65 #include <sys/signalvar.h>
66 #include <sys/socket.h>
67 #include <sys/socketvar.h>
68 #include <sys/stat.h>
69 #include <sys/sysctl.h>
70 #include <sys/sysent.h>
71 #include <sys/sysproto.h>
72 #include <sys/syscallsubr.h>
73 #include <sys/taskqueue.h>
74 #include <sys/uio.h>
75 #include <sys/user.h>
76 #ifdef KTRACE
77 #include <sys/ktrace.h>
78 #endif
79 #include <machine/atomic.h>
80 #ifdef COMPAT_FREEBSD32
81 #include <compat/freebsd32/freebsd32.h>
82 #include <compat/freebsd32/freebsd32_util.h>
83 #endif
84
85 #include <vm/uma.h>
86
87 static MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
88
89 /*
90 * This lock is used if multiple kq locks are required. This possibly
91 * should be made into a per proc lock.
92 */
93 static struct mtx kq_global;
94 MTX_SYSINIT(kq_global, &kq_global, "kqueue order", MTX_DEF);
95 #define KQ_GLOBAL_LOCK(lck, haslck) do { \
96 if (!haslck) \
97 mtx_lock(lck); \
98 haslck = 1; \
99 } while (0)
100 #define KQ_GLOBAL_UNLOCK(lck, haslck) do { \
101 if (haslck) \
102 mtx_unlock(lck); \
103 haslck = 0; \
104 } while (0)
105
106 TASKQUEUE_DEFINE_THREAD(kqueue_ctx);
107
108 static int kevent_copyout(void *arg, struct kevent *kevp, int count);
109 static int kevent_copyin(void *arg, struct kevent *kevp, int count);
110 static int kqueue_register(struct kqueue *kq, struct kevent *kev,
111 struct thread *td, int mflag);
112 static int kqueue_acquire(struct file *fp, struct kqueue **kqp);
113 static void kqueue_release(struct kqueue *kq, int locked);
114 static void kqueue_destroy(struct kqueue *kq);
115 static void kqueue_drain(struct kqueue *kq, struct thread *td);
116 static int kqueue_expand(struct kqueue *kq, const struct filterops *fops,
117 uintptr_t ident, int mflag);
118 static void kqueue_task(void *arg, int pending);
119 static int kqueue_scan(struct kqueue *kq, int maxevents,
120 struct kevent_copyops *k_ops,
121 const struct timespec *timeout,
122 struct kevent *keva, struct thread *td);
123 static void kqueue_wakeup(struct kqueue *kq);
124 static const struct filterops *kqueue_fo_find(int filt);
125 static void kqueue_fo_release(int filt);
126 struct g_kevent_args;
127 static int kern_kevent_generic(struct thread *td,
128 struct g_kevent_args *uap,
129 struct kevent_copyops *k_ops, const char *struct_name);
130
131 static fo_ioctl_t kqueue_ioctl;
132 static fo_poll_t kqueue_poll;
133 static fo_kqfilter_t kqueue_kqfilter;
134 static fo_stat_t kqueue_stat;
135 static fo_close_t kqueue_close;
136 static fo_fill_kinfo_t kqueue_fill_kinfo;
137 static fo_fork_t kqueue_fork;
138
139 static const struct fileops kqueueops = {
140 .fo_read = invfo_rdwr,
141 .fo_write = invfo_rdwr,
142 .fo_truncate = invfo_truncate,
143 .fo_ioctl = kqueue_ioctl,
144 .fo_poll = kqueue_poll,
145 .fo_kqfilter = kqueue_kqfilter,
146 .fo_stat = kqueue_stat,
147 .fo_close = kqueue_close,
148 .fo_chmod = invfo_chmod,
149 .fo_chown = invfo_chown,
150 .fo_sendfile = invfo_sendfile,
151 .fo_cmp = file_kcmp_generic,
152 .fo_fork = kqueue_fork,
153 .fo_fill_kinfo = kqueue_fill_kinfo,
154 .fo_flags = DFLAG_FORK,
155 };
156
157 static int knote_attach(struct knote *kn, struct kqueue *kq);
158 static void knote_drop(struct knote *kn, struct thread *td);
159 static void knote_drop_detached(struct knote *kn, struct thread *td);
160 static void knote_enqueue(struct knote *kn);
161 static void knote_dequeue(struct knote *kn);
162 static void knote_init(void *);
163 static struct knote *knote_alloc(int mflag);
164 static void knote_free(struct knote *kn);
165
166 static void filt_kqdetach(struct knote *kn);
167 static int filt_kqueue(struct knote *kn, long hint);
168 static int filt_procattach(struct knote *kn);
169 static void filt_procdetach(struct knote *kn);
170 static int filt_proc(struct knote *kn, long hint);
171 static int filt_jailattach(struct knote *kn);
172 static void filt_jaildetach(struct knote *kn);
173 static int filt_jail(struct knote *kn, long hint);
174 static int filt_fileattach(struct knote *kn);
175 static void filt_timerexpire(void *knx);
176 static void filt_timerexpire_l(struct knote *kn, bool proc_locked);
177 static int filt_timerattach(struct knote *kn);
178 static void filt_timerdetach(struct knote *kn);
179 static void filt_timerstart(struct knote *kn, sbintime_t to);
180 static void filt_timertouch(struct knote *kn, struct kevent *kev,
181 u_long type);
182 static int filt_timercopy(struct knote *kn, struct proc *p1);
183 static int filt_timervalidate(struct knote *kn, sbintime_t *to);
184 static int filt_timer(struct knote *kn, long hint);
185 static int filt_userattach(struct knote *kn);
186 static void filt_userdetach(struct knote *kn);
187 static int filt_user(struct knote *kn, long hint);
188 static void filt_usertouch(struct knote *kn, struct kevent *kev,
189 u_long type);
190
191 static const struct filterops file_filtops = {
192 .f_isfd = 1,
193 .f_attach = filt_fileattach,
194 .f_copy = knote_triv_copy,
195 };
196 static const struct filterops kqread_filtops = {
197 .f_isfd = 1,
198 .f_detach = filt_kqdetach,
199 .f_event = filt_kqueue,
200 .f_copy = knote_triv_copy,
201 };
202 /* XXX - move to kern_proc.c? */
203 static const struct filterops proc_filtops = {
204 .f_isfd = 0,
205 .f_attach = filt_procattach,
206 .f_detach = filt_procdetach,
207 .f_event = filt_proc,
208 .f_copy = knote_triv_copy,
209 };
210 static const struct filterops jail_filtops = {
211 .f_isfd = 0,
212 .f_attach = filt_jailattach,
213 .f_detach = filt_jaildetach,
214 .f_event = filt_jail,
215 .f_copy = knote_triv_copy,
216 };
217 static const struct filterops timer_filtops = {
218 .f_isfd = 0,
219 .f_attach = filt_timerattach,
220 .f_detach = filt_timerdetach,
221 .f_event = filt_timer,
222 .f_touch = filt_timertouch,
223 .f_copy = filt_timercopy,
224 };
225 static const struct filterops user_filtops = {
226 .f_attach = filt_userattach,
227 .f_detach = filt_userdetach,
228 .f_event = filt_user,
229 .f_touch = filt_usertouch,
230 .f_copy = knote_triv_copy,
231 };
232
233 static uma_zone_t knote_zone;
234 static unsigned int __exclusive_cache_line kq_ncallouts;
235 static unsigned int kq_calloutmax = 4 * 1024;
236 SYSCTL_UINT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
237 &kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue");
238
239 /* XXX - ensure not influx ? */
240 #define KNOTE_ACTIVATE(kn, islock) do { \
241 if ((islock)) \
242 mtx_assert(&(kn)->kn_kq->kq_lock, MA_OWNED); \
243 else \
244 KQ_LOCK((kn)->kn_kq); \
245 (kn)->kn_status |= KN_ACTIVE; \
246 if (((kn)->kn_status & (KN_QUEUED | KN_DISABLED)) == 0) \
247 knote_enqueue((kn)); \
248 if (!(islock)) \
249 KQ_UNLOCK((kn)->kn_kq); \
250 } while (0)
251 #define KQ_LOCK(kq) do { \
252 mtx_lock(&(kq)->kq_lock); \
253 } while (0)
254 #define KQ_FLUX_WAKEUP(kq) do { \
255 if (((kq)->kq_state & KQ_FLUXWAIT) == KQ_FLUXWAIT) { \
256 (kq)->kq_state &= ~KQ_FLUXWAIT; \
257 wakeup((kq)); \
258 } \
259 } while (0)
260 #define KQ_FLUX_SLEEP_WMESG(kq, kn, flags, wmesg) do { \
261 KASSERT((kn)->kn_kq == (kq), \
262 ("%s: knote %p not on kqueue %p", __func__, kn, kq)); \
263 (kq)->kq_state |= KQ_FLUXWAIT; \
264 msleep((kq), &(kq)->kq_lock, PSOCK | (flags), (wmesg), 0); \
265 } while (0)
266 #define KQ_FLUX_SLEEP(kq, kn, flags) \
267 KQ_FLUX_SLEEP_WMESG(kq, kn, flags, "kqfluxwt")
268 #define KQ_UNLOCK_FLUX(kq) do { \
269 KQ_FLUX_WAKEUP(kq); \
270 mtx_unlock(&(kq)->kq_lock); \
271 } while (0)
272 #define KQ_UNLOCK(kq) do { \
273 mtx_unlock(&(kq)->kq_lock); \
274 } while (0)
275 #define KQ_OWNED(kq) do { \
276 mtx_assert(&(kq)->kq_lock, MA_OWNED); \
277 } while (0)
278 #define KQ_NOTOWNED(kq) do { \
279 mtx_assert(&(kq)->kq_lock, MA_NOTOWNED); \
280 } while (0)
281
282 static struct knlist *
kn_list_lock(struct knote * kn)283 kn_list_lock(struct knote *kn)
284 {
285 struct knlist *knl;
286
287 knl = kn->kn_knlist;
288 if (knl != NULL)
289 knl->kl_lock(knl->kl_lockarg);
290 return (knl);
291 }
292
293 static void
kn_list_unlock(struct knlist * knl)294 kn_list_unlock(struct knlist *knl)
295 {
296 bool do_free;
297
298 if (knl == NULL)
299 return;
300 do_free = knl->kl_autodestroy && knlist_empty(knl);
301 knl->kl_unlock(knl->kl_lockarg);
302 if (do_free) {
303 knlist_destroy(knl);
304 free(knl, M_KQUEUE);
305 }
306 }
307
308 static bool
kn_in_flux(struct knote * kn)309 kn_in_flux(struct knote *kn)
310 {
311
312 return (kn->kn_influx > 0);
313 }
314
315 static void
kn_enter_flux(struct knote * kn)316 kn_enter_flux(struct knote *kn)
317 {
318
319 KQ_OWNED(kn->kn_kq);
320 MPASS(kn->kn_influx < INT_MAX);
321 kn->kn_influx++;
322 }
323
324 static bool
kn_leave_flux(struct knote * kn)325 kn_leave_flux(struct knote *kn)
326 {
327
328 KQ_OWNED(kn->kn_kq);
329 MPASS(kn->kn_influx > 0);
330 kn->kn_influx--;
331 return (kn->kn_influx == 0);
332 }
333
334 #define KNL_ASSERT_LOCK(knl, islocked) do { \
335 if (islocked) \
336 KNL_ASSERT_LOCKED(knl); \
337 else \
338 KNL_ASSERT_UNLOCKED(knl); \
339 } while (0)
340 #ifdef INVARIANTS
341 #define KNL_ASSERT_LOCKED(knl) do { \
342 knl->kl_assert_lock((knl)->kl_lockarg, LA_LOCKED); \
343 } while (0)
344 #define KNL_ASSERT_UNLOCKED(knl) do { \
345 knl->kl_assert_lock((knl)->kl_lockarg, LA_UNLOCKED); \
346 } while (0)
347 #else /* !INVARIANTS */
348 #define KNL_ASSERT_LOCKED(knl) do {} while (0)
349 #define KNL_ASSERT_UNLOCKED(knl) do {} while (0)
350 #endif /* INVARIANTS */
351
352 #ifndef KN_HASHSIZE
353 #define KN_HASHSIZE 64 /* XXX should be tunable */
354 #endif
355
356 #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
357
358 static int
filt_nullattach(struct knote * kn)359 filt_nullattach(struct knote *kn)
360 {
361
362 return (ENXIO);
363 };
364
365 static const struct filterops null_filtops = {
366 .f_isfd = 0,
367 .f_attach = filt_nullattach,
368 .f_copy = knote_triv_copy,
369 };
370
371 /* XXX - make SYSINIT to add these, and move into respective modules. */
372 extern const struct filterops sig_filtops;
373 extern const struct filterops fs_filtops;
374
375 /*
376 * Table for all system-defined filters.
377 */
378 static struct mtx filterops_lock;
379 MTX_SYSINIT(kqueue_filterops, &filterops_lock, "protect sysfilt_ops", MTX_DEF);
380 static struct {
381 const struct filterops *for_fop;
382 int for_nolock;
383 int for_refcnt;
384 } sysfilt_ops[EVFILT_SYSCOUNT] = {
385 [~EVFILT_READ] = { &file_filtops, 1 },
386 [~EVFILT_WRITE] = { &file_filtops, 1 },
387 [~EVFILT_AIO] = { &null_filtops },
388 [~EVFILT_VNODE] = { &file_filtops, 1 },
389 [~EVFILT_PROC] = { &proc_filtops, 1 },
390 [~EVFILT_SIGNAL] = { &sig_filtops, 1 },
391 [~EVFILT_TIMER] = { &timer_filtops, 1 },
392 [~EVFILT_PROCDESC] = { &file_filtops, 1 },
393 [~EVFILT_FS] = { &fs_filtops, 1 },
394 [~EVFILT_LIO] = { &null_filtops },
395 [~EVFILT_USER] = { &user_filtops, 1 },
396 [~EVFILT_SENDFILE] = { &null_filtops },
397 [~EVFILT_EMPTY] = { &file_filtops, 1 },
398 [~EVFILT_JAIL] = { &jail_filtops, 1 },
399 [~EVFILT_JAILDESC] = { &file_filtops, 1 },
400 };
401
402 /*
403 * Simple redirection for all cdevsw style objects to call their fo_kqfilter
404 * method.
405 */
406 static int
filt_fileattach(struct knote * kn)407 filt_fileattach(struct knote *kn)
408 {
409
410 return (fo_kqfilter(kn->kn_fp, kn));
411 }
412
413 /*ARGSUSED*/
414 static int
kqueue_kqfilter(struct file * fp,struct knote * kn)415 kqueue_kqfilter(struct file *fp, struct knote *kn)
416 {
417 struct kqueue *kq = kn->kn_fp->f_data;
418
419 if (kn->kn_filter != EVFILT_READ)
420 return (EINVAL);
421
422 kn->kn_status |= KN_KQUEUE;
423 kn->kn_fop = &kqread_filtops;
424 knlist_add(&kq->kq_sel.si_note, kn, 0);
425
426 return (0);
427 }
428
429 static void
filt_kqdetach(struct knote * kn)430 filt_kqdetach(struct knote *kn)
431 {
432 struct kqueue *kq = kn->kn_fp->f_data;
433
434 knlist_remove(&kq->kq_sel.si_note, kn, 0);
435 }
436
437 /*ARGSUSED*/
438 static int
filt_kqueue(struct knote * kn,long hint)439 filt_kqueue(struct knote *kn, long hint)
440 {
441 struct kqueue *kq = kn->kn_fp->f_data;
442
443 kn->kn_data = kq->kq_count;
444 return (kn->kn_data > 0);
445 }
446
447 /* XXX - move to kern_proc.c? */
448 static int
filt_procattach(struct knote * kn)449 filt_procattach(struct knote *kn)
450 {
451 struct proc *p;
452 int error;
453 bool exiting, immediate;
454
455 exiting = immediate = false;
456 if (kn->kn_sfflags & (NOTE_EXIT | NOTE_REAP))
457 p = pfind_any(kn->kn_id);
458 else
459 p = pfind(kn->kn_id);
460 if (p == NULL)
461 return (ESRCH);
462 if (p->p_flag & P_WEXIT)
463 exiting = true;
464
465 if ((error = p_cansee(curthread, p))) {
466 PROC_UNLOCK(p);
467 return (error);
468 }
469
470 kn->kn_ptr.p_proc = p;
471 kn->kn_flags |= EV_CLEAR; /* automatically set */
472
473 /*
474 * Internal flag indicating registration done by kernel for the
475 * purposes of getting a NOTE_CHILD notification.
476 */
477 if (kn->kn_flags & EV_FLAG2) {
478 kn->kn_flags &= ~EV_FLAG2;
479 kn->kn_data = kn->kn_sdata; /* ppid */
480 kn->kn_fflags = NOTE_CHILD;
481 kn->kn_sfflags &= ~NOTE_PCTRLMASK;
482 immediate = true; /* Force immediate activation of child note. */
483 }
484 /*
485 * Internal flag indicating registration done by kernel (for other than
486 * NOTE_CHILD).
487 */
488 if (kn->kn_flags & EV_FLAG1) {
489 kn->kn_flags &= ~EV_FLAG1;
490 }
491
492 knlist_add(p->p_klist, kn, 1);
493
494 /*
495 * Immediately activate any child notes or, in the case of a zombie
496 * target process, exit notes. The latter is necessary to handle the
497 * case where the target process, e.g. a child, dies before the kevent
498 * is registered.
499 */
500 if (immediate || (exiting && filt_proc(kn, NOTE_EXIT)))
501 KNOTE_ACTIVATE(kn, 0);
502
503 PROC_UNLOCK(p);
504
505 return (0);
506 }
507
508 /*
509 * The knote may be attached to a different process, which may exit,
510 * leaving nothing for the knote to be attached to. So when the process
511 * exits, the knote is marked as DETACHED and also flagged as ONESHOT so
512 * it will be deleted when read out. However, as part of the knote deletion,
513 * this routine is called, so a check is needed to avoid actually performing
514 * a detach, because the original process does not exist any more.
515 */
516 /* XXX - move to kern_proc.c? */
517 static void
filt_procdetach(struct knote * kn)518 filt_procdetach(struct knote *kn)
519 {
520
521 knlist_remove(kn->kn_knlist, kn, 0);
522 kn->kn_ptr.p_proc = NULL;
523 }
524
525 /* XXX - move to kern_proc.c? */
526 static int
filt_proc(struct knote * kn,long hint)527 filt_proc(struct knote *kn, long hint)
528 {
529 struct proc *p;
530 u_int event;
531
532 p = kn->kn_ptr.p_proc;
533 if (p == NULL) /* already activated, from attach filter */
534 return (0);
535
536 /* Mask off extra data. */
537 event = (u_int)hint & NOTE_PCTRLMASK;
538
539 /* If the user is interested in this event, record it. */
540 if ((kn->kn_sfflags & event) != 0)
541 kn->kn_fflags |= kn->kn_sfflags & event;
542
543 /* Report exit status */
544 if ((kn->kn_fflags & NOTE_EXIT) != 0)
545 kn->kn_data = KW_EXITCODE(p->p_xexit, p->p_xsig);
546
547 /* Process is gone, so flag the event as finished. */
548 if ((event & NOTE_REAP) != 0 ||
549 ((event & NOTE_EXIT) != 0 && (kn->kn_sfflags & NOTE_REAP) == 0)) {
550 kn->kn_flags |= EV_EOF | EV_ONESHOT;
551 kn->kn_ptr.p_proc = NULL;
552 if (kn->kn_fflags == 0)
553 kn->kn_flags |= EV_DROP;
554 return (1);
555 }
556
557 return (kn->kn_fflags != 0);
558 }
559
560 /*
561 * Called when the process forked. It mostly does the same as the
562 * knote(), activating all knotes registered to be activated when the
563 * process forked. Additionally, for each knote attached to the
564 * parent, check whether user wants to track the new process. If so
565 * attach a new knote to it, and immediately report an event with the
566 * child's pid.
567 */
568 void
knote_fork(struct knlist * list,int pid)569 knote_fork(struct knlist *list, int pid)
570 {
571 struct kqueue *kq;
572 struct knote *kn;
573 struct kevent kev;
574 int error;
575
576 MPASS(list != NULL);
577 KNL_ASSERT_LOCKED(list);
578 if (SLIST_EMPTY(&list->kl_list))
579 return;
580
581 memset(&kev, 0, sizeof(kev));
582 SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
583 kq = kn->kn_kq;
584 KQ_LOCK(kq);
585 if (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0) {
586 KQ_UNLOCK(kq);
587 continue;
588 }
589
590 /*
591 * The same as knote(), activate the event.
592 */
593 if ((kn->kn_sfflags & NOTE_TRACK) == 0) {
594 if (kn->kn_fop->f_event(kn, NOTE_FORK))
595 KNOTE_ACTIVATE(kn, 1);
596 KQ_UNLOCK(kq);
597 continue;
598 }
599
600 /*
601 * The NOTE_TRACK case. In addition to the activation
602 * of the event, we need to register new events to
603 * track the child. Drop the locks in preparation for
604 * the call to kqueue_register().
605 */
606 kn_enter_flux(kn);
607 KQ_UNLOCK(kq);
608 list->kl_unlock(list->kl_lockarg);
609
610 /*
611 * Activate existing knote and register tracking knotes with
612 * new process.
613 *
614 * First register a knote to get just the child notice. This
615 * must be a separate note from a potential NOTE_EXIT
616 * notification since both NOTE_CHILD and NOTE_EXIT are defined
617 * to use the data field (in conflicting ways).
618 */
619 kev.ident = pid;
620 kev.filter = kn->kn_filter;
621 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_ONESHOT |
622 EV_FLAG2;
623 kev.fflags = kn->kn_sfflags;
624 kev.data = kn->kn_id; /* parent */
625 kev.udata = kn->kn_kevent.udata;/* preserve udata */
626 error = kqueue_register(kq, &kev, NULL, M_NOWAIT);
627 if (error)
628 kn->kn_fflags |= NOTE_TRACKERR;
629
630 /*
631 * Then register another knote to track other potential events
632 * from the new process.
633 */
634 kev.ident = pid;
635 kev.filter = kn->kn_filter;
636 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1;
637 kev.fflags = kn->kn_sfflags;
638 kev.data = kn->kn_id; /* parent */
639 kev.udata = kn->kn_kevent.udata;/* preserve udata */
640 error = kqueue_register(kq, &kev, NULL, M_NOWAIT);
641
642 /*
643 * Serialize updates to the kn_kevent fields with threads
644 * scanning the queue.
645 */
646 list->kl_lock(list->kl_lockarg);
647 if (error)
648 kn->kn_fflags |= NOTE_TRACKERR;
649 if (kn->kn_fop->f_event(kn, NOTE_FORK)) {
650 KQ_LOCK(kq);
651 KNOTE_ACTIVATE(kn, 1);
652 } else {
653 KQ_LOCK(kq);
654 }
655 kn_leave_flux(kn);
656 KQ_UNLOCK_FLUX(kq);
657 }
658 }
659
660 int
filt_jailattach(struct knote * kn)661 filt_jailattach(struct knote *kn)
662 {
663 struct prison *pr;
664
665 if (kn->kn_id == 0) {
666 /* Let jid=0 watch the current prison (including prison0). */
667 pr = curthread->td_ucred->cr_prison;
668 mtx_lock(&pr->pr_mtx);
669 } else {
670 sx_slock(&allprison_lock);
671 pr = prison_find_child(curthread->td_ucred->cr_prison,
672 kn->kn_id);
673 sx_sunlock(&allprison_lock);
674 if (pr == NULL)
675 return (ENOENT);
676 if (!prison_isalive(pr)) {
677 mtx_unlock(&pr->pr_mtx);
678 return (ENOENT);
679 }
680 }
681 kn->kn_ptr.p_prison = pr;
682 kn->kn_flags |= EV_CLEAR;
683 knlist_add(pr->pr_klist, kn, 1);
684 mtx_unlock(&pr->pr_mtx);
685 return (0);
686 }
687
688 void
filt_jaildetach(struct knote * kn)689 filt_jaildetach(struct knote *kn)
690 {
691 if (kn->kn_ptr.p_prison != NULL) {
692 knlist_remove(kn->kn_knlist, kn, 0);
693 kn->kn_ptr.p_prison = NULL;
694 } else
695 kn->kn_status |= KN_DETACHED;
696 }
697
698 int
filt_jail(struct knote * kn,long hint)699 filt_jail(struct knote *kn, long hint)
700 {
701 struct prison *pr;
702 u_int event;
703
704 pr = kn->kn_ptr.p_prison;
705 if (pr == NULL) /* already activated, from attach filter */
706 return (0);
707
708 /*
709 * Mask off extra data. In the NOTE_JAIL_CHILD case, that's
710 * everything except the NOTE_JAIL_CHILD bit itself, since a
711 * JID is any positive integer.
712 */
713 event = ((u_int)hint & NOTE_JAIL_CHILD) ? NOTE_JAIL_CHILD :
714 (u_int)hint & NOTE_JAIL_CTRLMASK;
715
716 /* If the user is interested in this event, record it. */
717 if (kn->kn_sfflags & event) {
718 kn->kn_fflags |= event;
719 /* Report the created jail id or attached process id. */
720 if (event == NOTE_JAIL_CHILD || event == NOTE_JAIL_ATTACH) {
721 if (kn->kn_data != 0)
722 kn->kn_fflags |= NOTE_JAIL_MULTI;
723 kn->kn_data = (kn->kn_fflags & NOTE_JAIL_MULTI) ? 0U :
724 (u_int)hint & ~event;
725 }
726 }
727
728 /* Prison is gone, so flag the event as finished. */
729 if (event == NOTE_JAIL_REMOVE) {
730 kn->kn_flags |= EV_EOF | EV_ONESHOT;
731 kn->kn_ptr.p_prison = NULL;
732 if (kn->kn_fflags == 0)
733 kn->kn_flags |= EV_DROP;
734 return (1);
735 }
736
737 return (kn->kn_fflags != 0);
738 }
739
740 /*
741 * XXX: EVFILT_TIMER should perhaps live in kern_time.c beside the
742 * interval timer support code.
743 */
744
745 #define NOTE_TIMER_PRECMASK \
746 (NOTE_SECONDS | NOTE_MSECONDS | NOTE_USECONDS | NOTE_NSECONDS)
747
748 static sbintime_t
timer2sbintime(int64_t data,unsigned int flags)749 timer2sbintime(int64_t data, unsigned int flags)
750 {
751 int64_t secs;
752
753 /*
754 * Macros for converting to the fractional second portion of an
755 * sbintime_t using 64bit multiplication to improve precision.
756 */
757 #define NS_TO_SBT(ns) (((ns) * (((uint64_t)1 << 63) / 500000000)) >> 32)
758 #define US_TO_SBT(us) (((us) * (((uint64_t)1 << 63) / 500000)) >> 32)
759 #define MS_TO_SBT(ms) (((ms) * (((uint64_t)1 << 63) / 500)) >> 32)
760 switch (flags & NOTE_TIMER_PRECMASK) {
761 case NOTE_SECONDS:
762 #ifdef __LP64__
763 if (data > (SBT_MAX / SBT_1S))
764 return (SBT_MAX);
765 #endif
766 return ((sbintime_t)data << 32);
767 case NOTE_MSECONDS: /* FALLTHROUGH */
768 case 0:
769 if (data >= 1000) {
770 secs = data / 1000;
771 #ifdef __LP64__
772 if (secs > (SBT_MAX / SBT_1S))
773 return (SBT_MAX);
774 #endif
775 return (secs << 32 | MS_TO_SBT(data % 1000));
776 }
777 return (MS_TO_SBT(data));
778 case NOTE_USECONDS:
779 if (data >= 1000000) {
780 secs = data / 1000000;
781 #ifdef __LP64__
782 if (secs > (SBT_MAX / SBT_1S))
783 return (SBT_MAX);
784 #endif
785 return (secs << 32 | US_TO_SBT(data % 1000000));
786 }
787 return (US_TO_SBT(data));
788 case NOTE_NSECONDS:
789 if (data >= 1000000000) {
790 secs = data / 1000000000;
791 #ifdef __LP64__
792 if (secs > (SBT_MAX / SBT_1S))
793 return (SBT_MAX);
794 #endif
795 return (secs << 32 | NS_TO_SBT(data % 1000000000));
796 }
797 return (NS_TO_SBT(data));
798 default:
799 break;
800 }
801 return (-1);
802 }
803
804 struct kq_timer_cb_data {
805 struct callout c;
806 struct proc *p;
807 struct knote *kn;
808 int cpuid;
809 int flags;
810 TAILQ_ENTRY(kq_timer_cb_data) link;
811 sbintime_t next; /* next timer event fires at */
812 sbintime_t to; /* precalculated timer period, 0 for abs */
813 };
814
815 #define KQ_TIMER_CB_ENQUEUED 0x01
816
817 static void
kqtimer_sched_callout(struct kq_timer_cb_data * kc)818 kqtimer_sched_callout(struct kq_timer_cb_data *kc)
819 {
820 callout_reset_sbt_on(&kc->c, kc->next, 0, filt_timerexpire, kc->kn,
821 kc->cpuid, C_ABSOLUTE);
822 }
823
824 void
kqtimer_proc_continue(struct proc * p)825 kqtimer_proc_continue(struct proc *p)
826 {
827 struct kq_timer_cb_data *kc, *kc1;
828 sbintime_t now;
829
830 PROC_LOCK_ASSERT(p, MA_OWNED);
831
832 now = sbinuptime();
833 TAILQ_FOREACH_SAFE(kc, &p->p_kqtim_stop, link, kc1) {
834 TAILQ_REMOVE(&p->p_kqtim_stop, kc, link);
835 kc->flags &= ~KQ_TIMER_CB_ENQUEUED;
836 if (kc->next <= now)
837 filt_timerexpire_l(kc->kn, true);
838 else
839 kqtimer_sched_callout(kc);
840 }
841 }
842
843 static void
filt_timerexpire_l(struct knote * kn,bool proc_locked)844 filt_timerexpire_l(struct knote *kn, bool proc_locked)
845 {
846 struct kq_timer_cb_data *kc;
847 struct proc *p;
848 uint64_t delta;
849 sbintime_t now;
850
851 kc = kn->kn_ptr.p_v;
852
853 if ((kn->kn_flags & EV_ONESHOT) != 0 || kc->to == 0) {
854 kn->kn_data++;
855 KNOTE_ACTIVATE(kn, 0);
856 return;
857 }
858
859 now = sbinuptime();
860 if (now >= kc->next) {
861 delta = (now - kc->next) / kc->to;
862 if (delta == 0)
863 delta = 1;
864 kn->kn_data += delta;
865 kc->next += delta * kc->to;
866 if (now >= kc->next) /* overflow */
867 kc->next = now + kc->to;
868 KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */
869 }
870
871 /*
872 * Initial check for stopped kc->p is racy. It is fine to
873 * miss the set of the stop flags, at worst we would schedule
874 * one more callout. On the other hand, it is not fine to not
875 * schedule when we we missed clearing of the flags, we
876 * recheck them under the lock and observe consistent state.
877 */
878 p = kc->p;
879 if (P_SHOULDSTOP(p) || P_KILLED(p)) {
880 if (!proc_locked)
881 PROC_LOCK(p);
882 if (P_SHOULDSTOP(p) || P_KILLED(p)) {
883 if ((kc->flags & KQ_TIMER_CB_ENQUEUED) == 0) {
884 /*
885 * Insert into head so that
886 * kqtimer_proc_continue() does not
887 * iterate into us again.
888 */
889 kc->flags |= KQ_TIMER_CB_ENQUEUED;
890 TAILQ_INSERT_HEAD(&p->p_kqtim_stop, kc, link);
891 }
892 if (!proc_locked)
893 PROC_UNLOCK(p);
894 return;
895 }
896 if (!proc_locked)
897 PROC_UNLOCK(p);
898 }
899 kqtimer_sched_callout(kc);
900 }
901
902 static void
filt_timerexpire(void * knx)903 filt_timerexpire(void *knx)
904 {
905 filt_timerexpire_l(knx, false);
906 }
907
908 /*
909 * data contains amount of time to sleep
910 */
911 static int
filt_timervalidate(struct knote * kn,sbintime_t * to)912 filt_timervalidate(struct knote *kn, sbintime_t *to)
913 {
914 struct bintime bt;
915 sbintime_t sbt;
916
917 if (kn->kn_sdata < 0)
918 return (EINVAL);
919 if (kn->kn_sdata == 0 && (kn->kn_flags & EV_ONESHOT) == 0)
920 kn->kn_sdata = 1;
921 /*
922 * The only fflags values supported are the timer unit
923 * (precision) and the absolute time indicator.
924 */
925 if ((kn->kn_sfflags & ~(NOTE_TIMER_PRECMASK | NOTE_ABSTIME)) != 0)
926 return (EINVAL);
927
928 *to = timer2sbintime(kn->kn_sdata, kn->kn_sfflags);
929 if (*to < 0)
930 return (EINVAL);
931 if ((kn->kn_sfflags & NOTE_ABSTIME) != 0) {
932 getboottimebin(&bt);
933 sbt = bttosbt(bt);
934 *to = MAX(0, *to - sbt);
935 }
936 return (0);
937 }
938
939 static int
filt_timerattach(struct knote * kn)940 filt_timerattach(struct knote *kn)
941 {
942 struct kq_timer_cb_data *kc;
943 sbintime_t to;
944 int error;
945
946 to = -1;
947 error = filt_timervalidate(kn, &to);
948 if (error != 0)
949 return (error);
950 KASSERT(to > 0 || (kn->kn_flags & EV_ONESHOT) != 0 ||
951 (kn->kn_sfflags & NOTE_ABSTIME) != 0,
952 ("%s: periodic timer has a calculated zero timeout", __func__));
953 KASSERT(to >= 0,
954 ("%s: timer has a calculated negative timeout", __func__));
955
956 if (atomic_fetchadd_int(&kq_ncallouts, 1) + 1 > kq_calloutmax) {
957 atomic_subtract_int(&kq_ncallouts, 1);
958 return (ENOMEM);
959 }
960
961 if ((kn->kn_sfflags & NOTE_ABSTIME) == 0)
962 kn->kn_flags |= EV_CLEAR; /* automatically set */
963 kn->kn_status &= ~KN_DETACHED; /* knlist_add clears it */
964 kn->kn_ptr.p_v = kc = malloc(sizeof(*kc), M_KQUEUE, M_WAITOK);
965 kc->kn = kn;
966 kc->p = curproc;
967 kc->cpuid = PCPU_GET(cpuid);
968 kc->flags = 0;
969 callout_init(&kc->c, 1);
970 filt_timerstart(kn, to);
971
972 return (0);
973 }
974
975 static int
filt_timercopy(struct knote * kn,struct proc * p)976 filt_timercopy(struct knote *kn, struct proc *p)
977 {
978 struct kq_timer_cb_data *kc_src, *kc;
979
980 if (atomic_fetchadd_int(&kq_ncallouts, 1) + 1 > kq_calloutmax) {
981 atomic_subtract_int(&kq_ncallouts, 1);
982 return (ENOMEM);
983 }
984
985 kn->kn_status &= ~KN_DETACHED;
986 kc_src = kn->kn_ptr.p_v;
987 kn->kn_ptr.p_v = kc = malloc(sizeof(*kc), M_KQUEUE, M_WAITOK);
988 kc->kn = kn;
989 kc->p = p;
990 kc->flags = kc_src->flags & ~KQ_TIMER_CB_ENQUEUED;
991 kc->next = kc_src->next;
992 kc->to = kc_src->to;
993 kc->cpuid = PCPU_GET(cpuid);
994 callout_init(&kc->c, 1);
995 kqtimer_sched_callout(kc);
996 return (0);
997 }
998
999 static void
filt_timerstart(struct knote * kn,sbintime_t to)1000 filt_timerstart(struct knote *kn, sbintime_t to)
1001 {
1002 struct kq_timer_cb_data *kc;
1003
1004 kc = kn->kn_ptr.p_v;
1005 if ((kn->kn_sfflags & NOTE_ABSTIME) != 0) {
1006 kc->next = to;
1007 kc->to = 0;
1008 } else {
1009 kc->next = to + sbinuptime();
1010 kc->to = to;
1011 }
1012 kqtimer_sched_callout(kc);
1013 }
1014
1015 static void
filt_timerdetach(struct knote * kn)1016 filt_timerdetach(struct knote *kn)
1017 {
1018 struct kq_timer_cb_data *kc;
1019 unsigned int old __unused;
1020 bool pending;
1021
1022 kc = kn->kn_ptr.p_v;
1023 do {
1024 callout_drain(&kc->c);
1025
1026 /*
1027 * kqtimer_proc_continue() might have rescheduled this callout.
1028 * Double-check, using the process mutex as an interlock.
1029 */
1030 PROC_LOCK(kc->p);
1031 if ((kc->flags & KQ_TIMER_CB_ENQUEUED) != 0) {
1032 kc->flags &= ~KQ_TIMER_CB_ENQUEUED;
1033 TAILQ_REMOVE(&kc->p->p_kqtim_stop, kc, link);
1034 }
1035 pending = callout_pending(&kc->c);
1036 PROC_UNLOCK(kc->p);
1037 } while (pending);
1038 free(kc, M_KQUEUE);
1039 old = atomic_fetchadd_int(&kq_ncallouts, -1);
1040 KASSERT(old > 0, ("Number of callouts cannot become negative"));
1041 kn->kn_status |= KN_DETACHED; /* knlist_remove sets it */
1042 }
1043
1044 static void
filt_timertouch(struct knote * kn,struct kevent * kev,u_long type)1045 filt_timertouch(struct knote *kn, struct kevent *kev, u_long type)
1046 {
1047 struct kq_timer_cb_data *kc;
1048 struct kqueue *kq;
1049 sbintime_t to;
1050 int error;
1051
1052 switch (type) {
1053 case EVENT_REGISTER:
1054 /* Handle re-added timers that update data/fflags */
1055 if (kev->flags & EV_ADD) {
1056 kc = kn->kn_ptr.p_v;
1057
1058 /* Drain any existing callout. */
1059 callout_drain(&kc->c);
1060
1061 /* Throw away any existing undelivered record
1062 * of the timer expiration. This is done under
1063 * the presumption that if a process is
1064 * re-adding this timer with new parameters,
1065 * it is no longer interested in what may have
1066 * happened under the old parameters. If it is
1067 * interested, it can wait for the expiration,
1068 * delete the old timer definition, and then
1069 * add the new one.
1070 *
1071 * This has to be done while the kq is locked:
1072 * - if enqueued, dequeue
1073 * - make it no longer active
1074 * - clear the count of expiration events
1075 */
1076 kq = kn->kn_kq;
1077 KQ_LOCK(kq);
1078 if (kn->kn_status & KN_QUEUED)
1079 knote_dequeue(kn);
1080
1081 kn->kn_status &= ~KN_ACTIVE;
1082 kn->kn_data = 0;
1083 KQ_UNLOCK(kq);
1084
1085 /* Reschedule timer based on new data/fflags */
1086 kn->kn_sfflags = kev->fflags;
1087 kn->kn_sdata = kev->data;
1088 error = filt_timervalidate(kn, &to);
1089 if (error != 0) {
1090 kn->kn_flags |= EV_ERROR;
1091 kn->kn_data = error;
1092 } else
1093 filt_timerstart(kn, to);
1094 }
1095 break;
1096
1097 case EVENT_PROCESS:
1098 *kev = kn->kn_kevent;
1099 if (kn->kn_flags & EV_CLEAR) {
1100 kn->kn_data = 0;
1101 kn->kn_fflags = 0;
1102 }
1103 break;
1104
1105 default:
1106 panic("filt_timertouch() - invalid type (%ld)", type);
1107 break;
1108 }
1109 }
1110
1111 static int
filt_timer(struct knote * kn,long hint)1112 filt_timer(struct knote *kn, long hint)
1113 {
1114
1115 return (kn->kn_data != 0);
1116 }
1117
1118 static int
filt_userattach(struct knote * kn)1119 filt_userattach(struct knote *kn)
1120 {
1121
1122 /*
1123 * EVFILT_USER knotes are not attached to anything in the kernel.
1124 */
1125 kn->kn_hook = NULL;
1126 if (kn->kn_fflags & NOTE_TRIGGER)
1127 kn->kn_hookid = 1;
1128 else
1129 kn->kn_hookid = 0;
1130 return (0);
1131 }
1132
1133 static void
filt_userdetach(__unused struct knote * kn)1134 filt_userdetach(__unused struct knote *kn)
1135 {
1136
1137 /*
1138 * EVFILT_USER knotes are not attached to anything in the kernel.
1139 */
1140 }
1141
1142 static int
filt_user(struct knote * kn,__unused long hint)1143 filt_user(struct knote *kn, __unused long hint)
1144 {
1145
1146 return (kn->kn_hookid);
1147 }
1148
1149 static void
filt_usertouch(struct knote * kn,struct kevent * kev,u_long type)1150 filt_usertouch(struct knote *kn, struct kevent *kev, u_long type)
1151 {
1152 u_int ffctrl;
1153
1154 switch (type) {
1155 case EVENT_REGISTER:
1156 if (kev->fflags & NOTE_TRIGGER)
1157 kn->kn_hookid = 1;
1158
1159 ffctrl = kev->fflags & NOTE_FFCTRLMASK;
1160 kev->fflags &= NOTE_FFLAGSMASK;
1161 switch (ffctrl) {
1162 case NOTE_FFNOP:
1163 break;
1164
1165 case NOTE_FFAND:
1166 kn->kn_sfflags &= kev->fflags;
1167 break;
1168
1169 case NOTE_FFOR:
1170 kn->kn_sfflags |= kev->fflags;
1171 break;
1172
1173 case NOTE_FFCOPY:
1174 kn->kn_sfflags = kev->fflags;
1175 break;
1176
1177 default:
1178 /* XXX Return error? */
1179 break;
1180 }
1181 kn->kn_sdata = kev->data;
1182 if (kev->flags & EV_CLEAR) {
1183 kn->kn_hookid = 0;
1184 kn->kn_data = 0;
1185 kn->kn_fflags = 0;
1186 }
1187 break;
1188
1189 case EVENT_PROCESS:
1190 *kev = kn->kn_kevent;
1191 kev->fflags = kn->kn_sfflags;
1192 kev->data = kn->kn_sdata;
1193 if (kn->kn_flags & EV_CLEAR) {
1194 kn->kn_hookid = 0;
1195 kn->kn_data = 0;
1196 kn->kn_fflags = 0;
1197 }
1198 break;
1199
1200 default:
1201 panic("filt_usertouch() - invalid type (%ld)", type);
1202 break;
1203 }
1204 }
1205
1206 int
sys_kqueue(struct thread * td,struct kqueue_args * uap)1207 sys_kqueue(struct thread *td, struct kqueue_args *uap)
1208 {
1209
1210 return (kern_kqueue(td, 0, false, NULL));
1211 }
1212
1213 int
sys_kqueuex(struct thread * td,struct kqueuex_args * uap)1214 sys_kqueuex(struct thread *td, struct kqueuex_args *uap)
1215 {
1216 int flags;
1217
1218 if ((uap->flags & ~(KQUEUE_CLOEXEC | KQUEUE_CPONFORK)) != 0)
1219 return (EINVAL);
1220 flags = 0;
1221 if ((uap->flags & KQUEUE_CLOEXEC) != 0)
1222 flags |= O_CLOEXEC;
1223 return (kern_kqueue(td, flags, (uap->flags & KQUEUE_CPONFORK) != 0,
1224 NULL));
1225 }
1226
1227 static void
kqueue_init(struct kqueue * kq,bool cponfork)1228 kqueue_init(struct kqueue *kq, bool cponfork)
1229 {
1230
1231 mtx_init(&kq->kq_lock, "kqueue", NULL, MTX_DEF | MTX_DUPOK);
1232 TAILQ_INIT(&kq->kq_head);
1233 knlist_init_mtx(&kq->kq_sel.si_note, &kq->kq_lock);
1234 TASK_INIT(&kq->kq_task, 0, kqueue_task, kq);
1235 if (cponfork)
1236 kq->kq_state |= KQ_CPONFORK;
1237 }
1238
1239 static int
kern_kqueue_alloc(struct thread * td,struct filedesc * fdp,int * fdip,struct file ** fpp,int flags,struct filecaps * fcaps,bool cponfork,struct kqueue ** kqp)1240 kern_kqueue_alloc(struct thread *td, struct filedesc *fdp, int *fdip,
1241 struct file **fpp, int flags, struct filecaps *fcaps, bool cponfork,
1242 struct kqueue **kqp)
1243 {
1244 struct ucred *cred;
1245 struct kqueue *kq;
1246 int error;
1247
1248 cred = td->td_ucred;
1249 if (!chgkqcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_KQUEUES)))
1250 return (ENOMEM);
1251
1252 error = fdip != NULL ? falloc_caps(td, fpp, fdip, flags, fcaps) :
1253 _falloc_noinstall(td, fpp, 1);
1254 if (error != 0) {
1255 chgkqcnt(cred->cr_ruidinfo, -1, 0);
1256 return (error);
1257 }
1258
1259 /* An extra reference on `fp' has been held for us by falloc(). */
1260 kq = malloc(sizeof(*kq), M_KQUEUE, M_WAITOK | M_ZERO);
1261 kqueue_init(kq, cponfork);
1262 kq->kq_fdp = fdp;
1263 kq->kq_cred = crhold(cred);
1264
1265 if (fdip != NULL)
1266 FILEDESC_XLOCK(fdp);
1267 TAILQ_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list);
1268 if (fdip != NULL)
1269 FILEDESC_XUNLOCK(fdp);
1270
1271 finit(*fpp, FREAD | FWRITE, DTYPE_KQUEUE, kq, &kqueueops);
1272 *kqp = kq;
1273 return (0);
1274 }
1275
1276 int
kern_kqueue(struct thread * td,int flags,bool cponfork,struct filecaps * fcaps)1277 kern_kqueue(struct thread *td, int flags, bool cponfork, struct filecaps *fcaps)
1278 {
1279 struct kqueue *kq;
1280 struct file *fp;
1281 int fd, error;
1282
1283 error = kern_kqueue_alloc(td, td->td_proc->p_fd, &fd, &fp, flags,
1284 fcaps, cponfork, &kq);
1285 if (error != 0)
1286 return (error);
1287
1288 fdrop(fp, td);
1289
1290 td->td_retval[0] = fd;
1291 return (0);
1292 }
1293
1294 struct g_kevent_args {
1295 int fd;
1296 const void *changelist;
1297 int nchanges;
1298 void *eventlist;
1299 int nevents;
1300 const struct timespec *timeout;
1301 };
1302
1303 int
sys_kevent(struct thread * td,struct kevent_args * uap)1304 sys_kevent(struct thread *td, struct kevent_args *uap)
1305 {
1306 struct kevent_copyops k_ops = {
1307 .arg = uap,
1308 .k_copyout = kevent_copyout,
1309 .k_copyin = kevent_copyin,
1310 .kevent_size = sizeof(struct kevent),
1311 };
1312 struct g_kevent_args gk_args = {
1313 .fd = uap->fd,
1314 .changelist = uap->changelist,
1315 .nchanges = uap->nchanges,
1316 .eventlist = uap->eventlist,
1317 .nevents = uap->nevents,
1318 .timeout = uap->timeout,
1319 };
1320
1321 return (kern_kevent_generic(td, &gk_args, &k_ops, "kevent"));
1322 }
1323
1324 static int
kern_kevent_generic(struct thread * td,struct g_kevent_args * uap,struct kevent_copyops * k_ops,const char * struct_name)1325 kern_kevent_generic(struct thread *td, struct g_kevent_args *uap,
1326 struct kevent_copyops *k_ops, const char *struct_name)
1327 {
1328 struct timespec ts, *tsp;
1329 #ifdef KTRACE
1330 struct kevent *eventlist = uap->eventlist;
1331 #endif
1332 int error;
1333
1334 if (uap->timeout != NULL) {
1335 error = copyin(uap->timeout, &ts, sizeof(ts));
1336 if (error)
1337 return (error);
1338 tsp = &ts;
1339 } else
1340 tsp = NULL;
1341
1342 #ifdef KTRACE
1343 if (KTRPOINT(td, KTR_STRUCT_ARRAY))
1344 ktrstructarray(struct_name, UIO_USERSPACE, uap->changelist,
1345 uap->nchanges, k_ops->kevent_size);
1346 #endif
1347
1348 error = kern_kevent(td, uap->fd, uap->nchanges, uap->nevents,
1349 k_ops, tsp);
1350
1351 #ifdef KTRACE
1352 if (error == 0 && KTRPOINT(td, KTR_STRUCT_ARRAY))
1353 ktrstructarray(struct_name, UIO_USERSPACE, eventlist,
1354 td->td_retval[0], k_ops->kevent_size);
1355 #endif
1356
1357 return (error);
1358 }
1359
1360 /*
1361 * Copy 'count' items into the destination list pointed to by uap->eventlist.
1362 */
1363 static int
kevent_copyout(void * arg,struct kevent * kevp,int count)1364 kevent_copyout(void *arg, struct kevent *kevp, int count)
1365 {
1366 struct kevent_args *uap;
1367 int error;
1368
1369 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1370 uap = (struct kevent_args *)arg;
1371
1372 error = copyout(kevp, uap->eventlist, count * sizeof *kevp);
1373 if (error == 0)
1374 uap->eventlist += count;
1375 return (error);
1376 }
1377
1378 /*
1379 * Copy 'count' items from the list pointed to by uap->changelist.
1380 */
1381 static int
kevent_copyin(void * arg,struct kevent * kevp,int count)1382 kevent_copyin(void *arg, struct kevent *kevp, int count)
1383 {
1384 struct kevent_args *uap;
1385 int error;
1386
1387 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1388 uap = (struct kevent_args *)arg;
1389
1390 error = copyin(uap->changelist, kevp, count * sizeof *kevp);
1391 if (error == 0)
1392 uap->changelist += count;
1393 return (error);
1394 }
1395
1396 #ifdef COMPAT_FREEBSD11
1397 static int
kevent11_copyout(void * arg,struct kevent * kevp,int count)1398 kevent11_copyout(void *arg, struct kevent *kevp, int count)
1399 {
1400 struct freebsd11_kevent_args *uap;
1401 struct freebsd11_kevent kev11;
1402 int error, i;
1403
1404 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1405 uap = (struct freebsd11_kevent_args *)arg;
1406
1407 for (i = 0; i < count; i++) {
1408 kev11.ident = kevp->ident;
1409 kev11.filter = kevp->filter;
1410 kev11.flags = kevp->flags;
1411 kev11.fflags = kevp->fflags;
1412 kev11.data = kevp->data;
1413 kev11.udata = kevp->udata;
1414 error = copyout(&kev11, uap->eventlist, sizeof(kev11));
1415 if (error != 0)
1416 break;
1417 uap->eventlist++;
1418 kevp++;
1419 }
1420 return (error);
1421 }
1422
1423 /*
1424 * Copy 'count' items from the list pointed to by uap->changelist.
1425 */
1426 static int
kevent11_copyin(void * arg,struct kevent * kevp,int count)1427 kevent11_copyin(void *arg, struct kevent *kevp, int count)
1428 {
1429 struct freebsd11_kevent_args *uap;
1430 struct freebsd11_kevent kev11;
1431 int error, i;
1432
1433 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1434 uap = (struct freebsd11_kevent_args *)arg;
1435
1436 for (i = 0; i < count; i++) {
1437 error = copyin(uap->changelist, &kev11, sizeof(kev11));
1438 if (error != 0)
1439 break;
1440 kevp->ident = kev11.ident;
1441 kevp->filter = kev11.filter;
1442 kevp->flags = kev11.flags;
1443 kevp->fflags = kev11.fflags;
1444 kevp->data = (uintptr_t)kev11.data;
1445 kevp->udata = kev11.udata;
1446 bzero(&kevp->ext, sizeof(kevp->ext));
1447 uap->changelist++;
1448 kevp++;
1449 }
1450 return (error);
1451 }
1452
1453 int
freebsd11_kevent(struct thread * td,struct freebsd11_kevent_args * uap)1454 freebsd11_kevent(struct thread *td, struct freebsd11_kevent_args *uap)
1455 {
1456 struct kevent_copyops k_ops = {
1457 .arg = uap,
1458 .k_copyout = kevent11_copyout,
1459 .k_copyin = kevent11_copyin,
1460 .kevent_size = sizeof(struct freebsd11_kevent),
1461 };
1462 struct g_kevent_args gk_args = {
1463 .fd = uap->fd,
1464 .changelist = uap->changelist,
1465 .nchanges = uap->nchanges,
1466 .eventlist = uap->eventlist,
1467 .nevents = uap->nevents,
1468 .timeout = uap->timeout,
1469 };
1470
1471 return (kern_kevent_generic(td, &gk_args, &k_ops, "freebsd11_kevent"));
1472 }
1473 #endif
1474
1475 int
kern_kevent(struct thread * td,int fd,int nchanges,int nevents,struct kevent_copyops * k_ops,const struct timespec * timeout)1476 kern_kevent(struct thread *td, int fd, int nchanges, int nevents,
1477 struct kevent_copyops *k_ops, const struct timespec *timeout)
1478 {
1479 cap_rights_t rights;
1480 struct file *fp;
1481 int error;
1482
1483 cap_rights_init_zero(&rights);
1484 if (nchanges > 0)
1485 cap_rights_set_one(&rights, CAP_KQUEUE_CHANGE);
1486 if (nevents > 0)
1487 cap_rights_set_one(&rights, CAP_KQUEUE_EVENT);
1488 error = fget(td, fd, &rights, &fp);
1489 if (error != 0)
1490 return (error);
1491
1492 error = kern_kevent_fp(td, fp, nchanges, nevents, k_ops, timeout);
1493 fdrop(fp, td);
1494
1495 return (error);
1496 }
1497
1498 static int
kqueue_kevent(struct kqueue * kq,struct thread * td,int nchanges,int nevents,struct kevent_copyops * k_ops,const struct timespec * timeout)1499 kqueue_kevent(struct kqueue *kq, struct thread *td, int nchanges, int nevents,
1500 struct kevent_copyops *k_ops, const struct timespec *timeout)
1501 {
1502 struct kevent keva[KQ_NEVENTS];
1503 struct kevent *kevp, *changes;
1504 int i, n, nerrors, error;
1505
1506 if (nchanges < 0)
1507 return (EINVAL);
1508
1509 nerrors = 0;
1510 while (nchanges > 0) {
1511 n = nchanges > KQ_NEVENTS ? KQ_NEVENTS : nchanges;
1512 error = k_ops->k_copyin(k_ops->arg, keva, n);
1513 if (error)
1514 return (error);
1515 changes = keva;
1516 for (i = 0; i < n; i++) {
1517 kevp = &changes[i];
1518 if (!kevp->filter)
1519 continue;
1520 kevp->flags &= ~EV_SYSFLAGS;
1521 error = kqueue_register(kq, kevp, td, M_WAITOK);
1522 if (error || (kevp->flags & EV_RECEIPT)) {
1523 if (nevents == 0)
1524 return (error);
1525 kevp->flags = EV_ERROR;
1526 kevp->data = error;
1527 (void)k_ops->k_copyout(k_ops->arg, kevp, 1);
1528 nevents--;
1529 nerrors++;
1530 }
1531 }
1532 nchanges -= n;
1533 }
1534 if (nerrors) {
1535 td->td_retval[0] = nerrors;
1536 return (0);
1537 }
1538
1539 return (kqueue_scan(kq, nevents, k_ops, timeout, keva, td));
1540 }
1541
1542 int
kern_kevent_fp(struct thread * td,struct file * fp,int nchanges,int nevents,struct kevent_copyops * k_ops,const struct timespec * timeout)1543 kern_kevent_fp(struct thread *td, struct file *fp, int nchanges, int nevents,
1544 struct kevent_copyops *k_ops, const struct timespec *timeout)
1545 {
1546 struct kqueue *kq;
1547 int error;
1548
1549 error = kqueue_acquire(fp, &kq);
1550 if (error != 0)
1551 return (error);
1552 error = kqueue_kevent(kq, td, nchanges, nevents, k_ops, timeout);
1553 kqueue_release(kq, 0);
1554 return (error);
1555 }
1556
1557 /*
1558 * Performs a kevent() call on a temporarily created kqueue. This can be
1559 * used to perform one-shot polling, similar to poll() and select().
1560 */
1561 int
kern_kevent_anonymous(struct thread * td,int nevents,struct kevent_copyops * k_ops)1562 kern_kevent_anonymous(struct thread *td, int nevents,
1563 struct kevent_copyops *k_ops)
1564 {
1565 struct kqueue kq = {};
1566 int error;
1567
1568 kqueue_init(&kq, false);
1569 kq.kq_refcnt = 1;
1570 error = kqueue_kevent(&kq, td, nevents, nevents, k_ops, NULL);
1571 kqueue_drain(&kq, td);
1572 kqueue_destroy(&kq);
1573 return (error);
1574 }
1575
1576 int
kqueue_add_filteropts(int filt,const struct filterops * filtops)1577 kqueue_add_filteropts(int filt, const struct filterops *filtops)
1578 {
1579 int error;
1580
1581 error = 0;
1582 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) {
1583 printf(
1584 "trying to add a filterop that is out of range: %d is beyond %d\n",
1585 ~filt, EVFILT_SYSCOUNT);
1586 return EINVAL;
1587 }
1588 mtx_lock(&filterops_lock);
1589 if (sysfilt_ops[~filt].for_fop != &null_filtops &&
1590 sysfilt_ops[~filt].for_fop != NULL)
1591 error = EEXIST;
1592 else {
1593 sysfilt_ops[~filt].for_fop = filtops;
1594 sysfilt_ops[~filt].for_refcnt = 0;
1595 }
1596 mtx_unlock(&filterops_lock);
1597
1598 return (error);
1599 }
1600
1601 int
kqueue_del_filteropts(int filt)1602 kqueue_del_filteropts(int filt)
1603 {
1604 int error;
1605
1606 error = 0;
1607 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
1608 return EINVAL;
1609
1610 mtx_lock(&filterops_lock);
1611 if (sysfilt_ops[~filt].for_fop == &null_filtops ||
1612 sysfilt_ops[~filt].for_fop == NULL)
1613 error = EINVAL;
1614 else if (sysfilt_ops[~filt].for_refcnt != 0)
1615 error = EBUSY;
1616 else {
1617 sysfilt_ops[~filt].for_fop = &null_filtops;
1618 sysfilt_ops[~filt].for_refcnt = 0;
1619 }
1620 mtx_unlock(&filterops_lock);
1621
1622 return error;
1623 }
1624
1625 static const struct filterops *
kqueue_fo_find(int filt)1626 kqueue_fo_find(int filt)
1627 {
1628
1629 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
1630 return NULL;
1631
1632 if (sysfilt_ops[~filt].for_nolock)
1633 return sysfilt_ops[~filt].for_fop;
1634
1635 mtx_lock(&filterops_lock);
1636 sysfilt_ops[~filt].for_refcnt++;
1637 if (sysfilt_ops[~filt].for_fop == NULL)
1638 sysfilt_ops[~filt].for_fop = &null_filtops;
1639 mtx_unlock(&filterops_lock);
1640
1641 return sysfilt_ops[~filt].for_fop;
1642 }
1643
1644 static void
kqueue_fo_release(int filt)1645 kqueue_fo_release(int filt)
1646 {
1647
1648 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
1649 return;
1650
1651 if (sysfilt_ops[~filt].for_nolock)
1652 return;
1653
1654 mtx_lock(&filterops_lock);
1655 KASSERT(sysfilt_ops[~filt].for_refcnt > 0,
1656 ("filter object %d refcount not valid on release", filt));
1657 sysfilt_ops[~filt].for_refcnt--;
1658 mtx_unlock(&filterops_lock);
1659 }
1660
1661 /*
1662 * A ref to kq (obtained via kqueue_acquire) must be held.
1663 */
1664 static int
kqueue_register(struct kqueue * kq,struct kevent * kev,struct thread * td,int mflag)1665 kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td,
1666 int mflag)
1667 {
1668 const struct filterops *fops;
1669 struct file *fp;
1670 struct knote *kn, *tkn;
1671 struct knlist *knl;
1672 int error, filt, event;
1673 int haskqglobal, filedesc_unlock;
1674
1675 if ((kev->flags & (EV_ENABLE | EV_DISABLE)) == (EV_ENABLE | EV_DISABLE))
1676 return (EINVAL);
1677
1678 fp = NULL;
1679 kn = NULL;
1680 knl = NULL;
1681 error = 0;
1682 haskqglobal = 0;
1683 filedesc_unlock = 0;
1684
1685 filt = kev->filter;
1686 fops = kqueue_fo_find(filt);
1687 if (fops == NULL)
1688 return EINVAL;
1689
1690 if (kev->flags & EV_ADD) {
1691 /* Reject an invalid flag pair early */
1692 if (kev->flags & EV_KEEPUDATA) {
1693 tkn = NULL;
1694 error = EINVAL;
1695 goto done;
1696 }
1697
1698 /*
1699 * Prevent waiting with locks. Non-sleepable
1700 * allocation failures are handled in the loop, only
1701 * if the spare knote appears to be actually required.
1702 */
1703 tkn = knote_alloc(mflag);
1704 } else {
1705 tkn = NULL;
1706 }
1707
1708 findkn:
1709 if (fops->f_isfd) {
1710 KASSERT(td != NULL, ("td is NULL"));
1711 if (kev->ident > INT_MAX)
1712 error = EBADF;
1713 else
1714 error = fget(td, kev->ident, &cap_event_rights, &fp);
1715 if (error)
1716 goto done;
1717
1718 if ((kev->flags & EV_ADD) == EV_ADD && kqueue_expand(kq, fops,
1719 kev->ident, M_NOWAIT) != 0) {
1720 /* try again */
1721 fdrop(fp, td);
1722 fp = NULL;
1723 error = kqueue_expand(kq, fops, kev->ident, mflag);
1724 if (error)
1725 goto done;
1726 goto findkn;
1727 }
1728
1729 if (fp->f_type == DTYPE_KQUEUE) {
1730 /*
1731 * If we add some intelligence about what we are doing,
1732 * we should be able to support events on ourselves.
1733 * We need to know when we are doing this to prevent
1734 * getting both the knlist lock and the kq lock since
1735 * they are the same thing.
1736 */
1737 if (fp->f_data == kq) {
1738 error = EINVAL;
1739 goto done;
1740 }
1741
1742 /*
1743 * Pre-lock the filedesc before the global
1744 * lock mutex, see the comment in
1745 * kqueue_close().
1746 */
1747 FILEDESC_XLOCK(td->td_proc->p_fd);
1748 filedesc_unlock = 1;
1749 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
1750 }
1751
1752 KQ_LOCK(kq);
1753 if (kev->ident < kq->kq_knlistsize) {
1754 SLIST_FOREACH(kn, &kq->kq_knlist[kev->ident], kn_link) {
1755 MPASS(kn->kn_kq == kq);
1756 if (kev->filter == kn->kn_filter)
1757 break;
1758 }
1759 }
1760 } else {
1761 if ((kev->flags & EV_ADD) == EV_ADD) {
1762 error = kqueue_expand(kq, fops, kev->ident, mflag);
1763 if (error != 0)
1764 goto done;
1765 }
1766
1767 KQ_LOCK(kq);
1768
1769 /*
1770 * If possible, find an existing knote to use for this kevent.
1771 */
1772 if (kev->filter == EVFILT_PROC &&
1773 (kev->flags & (EV_FLAG1 | EV_FLAG2)) != 0) {
1774 /* This is an internal creation of a process tracking
1775 * note. Don't attempt to coalesce this with an
1776 * existing note.
1777 */
1778 ;
1779 } else if (kq->kq_knhashmask != 0) {
1780 struct klist *list;
1781
1782 list = &kq->kq_knhash[
1783 KN_HASH((u_long)kev->ident, kq->kq_knhashmask)];
1784 SLIST_FOREACH(kn, list, kn_link) {
1785 MPASS(kn->kn_kq == kq);
1786 if (kev->ident == kn->kn_id &&
1787 kev->filter == kn->kn_filter)
1788 break;
1789 }
1790 }
1791 }
1792
1793 /* knote is in the process of changing, wait for it to stabilize. */
1794 if (kn != NULL && kn_in_flux(kn)) {
1795 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1796 if (filedesc_unlock) {
1797 FILEDESC_XUNLOCK(td->td_proc->p_fd);
1798 filedesc_unlock = 0;
1799 }
1800 KQ_FLUX_SLEEP(kq, kn, PDROP);
1801 if (fp != NULL) {
1802 fdrop(fp, td);
1803 fp = NULL;
1804 }
1805 goto findkn;
1806 }
1807
1808 /*
1809 * kn now contains the matching knote, or NULL if no match
1810 */
1811 if (kn == NULL) {
1812 if (kev->flags & EV_ADD) {
1813 kn = tkn;
1814 tkn = NULL;
1815 if (kn == NULL) {
1816 KQ_UNLOCK(kq);
1817 error = ENOMEM;
1818 goto done;
1819 }
1820
1821 /*
1822 * Now that the kqueue is locked, make sure the fd
1823 * didn't change out from under us.
1824 */
1825 if (fops->f_isfd &&
1826 fget_noref_unlocked(td->td_proc->p_fd,
1827 kev->ident) != fp) {
1828 KQ_UNLOCK(kq);
1829 tkn = kn;
1830 error = EBADF;
1831 goto done;
1832 }
1833 kn->kn_fp = fp;
1834 kn->kn_kq = kq;
1835 kn->kn_fop = fops;
1836
1837 kn->kn_sfflags = kev->fflags;
1838 kn->kn_sdata = kev->data;
1839 kev->fflags = 0;
1840 kev->data = 0;
1841 kn->kn_kevent = *kev;
1842 kn->kn_kevent.flags &= ~(EV_ADD | EV_DELETE |
1843 EV_ENABLE | EV_DISABLE | EV_FORCEONESHOT);
1844 kn->kn_status = KN_DETACHED;
1845 if ((kev->flags & EV_DISABLE) != 0)
1846 kn->kn_status |= KN_DISABLED;
1847 kn_enter_flux(kn);
1848
1849 error = knote_attach(kn, kq);
1850 KQ_UNLOCK(kq);
1851 if (error != 0) {
1852 tkn = kn;
1853 goto done;
1854 }
1855
1856 /*
1857 * We transfer ownership of fops/fp to the knote
1858 * structure and avoid releasing them at the end of
1859 * this routine, now that all of the remaining exit
1860 * paths will knote_drop() to release the reference
1861 * counts we held on them above.
1862 */
1863 fops = NULL;
1864 fp = NULL;
1865
1866 if ((error = kn->kn_fop->f_attach(kn)) != 0) {
1867 knote_drop_detached(kn, td);
1868 goto done;
1869 }
1870 knl = kn_list_lock(kn);
1871 goto done_ev_add;
1872 } else {
1873 /* No matching knote and the EV_ADD flag is not set. */
1874 KQ_UNLOCK(kq);
1875 error = ENOENT;
1876 goto done;
1877 }
1878 }
1879
1880 if (kev->flags & EV_DELETE) {
1881 kn_enter_flux(kn);
1882 KQ_UNLOCK(kq);
1883 knote_drop(kn, td);
1884 goto done;
1885 }
1886
1887 if (kev->flags & EV_FORCEONESHOT) {
1888 kn->kn_flags |= EV_ONESHOT;
1889 KNOTE_ACTIVATE(kn, 1);
1890 }
1891
1892 if ((kev->flags & EV_ENABLE) != 0)
1893 kn->kn_status &= ~KN_DISABLED;
1894 else if ((kev->flags & EV_DISABLE) != 0)
1895 kn->kn_status |= KN_DISABLED;
1896
1897 /*
1898 * The user may change some filter values after the initial EV_ADD,
1899 * but doing so will not reset any filter which has already been
1900 * triggered.
1901 */
1902 kn->kn_status |= KN_SCAN;
1903 kn_enter_flux(kn);
1904 KQ_UNLOCK(kq);
1905 knl = kn_list_lock(kn);
1906 if ((kev->flags & EV_KEEPUDATA) == 0)
1907 kn->kn_kevent.udata = kev->udata;
1908 if (!fops->f_isfd && fops->f_touch != NULL) {
1909 fops->f_touch(kn, kev, EVENT_REGISTER);
1910 } else {
1911 kn->kn_sfflags = kev->fflags;
1912 kn->kn_sdata = kev->data;
1913 }
1914
1915 done_ev_add:
1916 /*
1917 * We can get here with kn->kn_knlist == NULL. This can happen when
1918 * the initial attach event decides that the event is "completed"
1919 * already, e.g., filt_procattach() is called on a zombie process. It
1920 * will call filt_proc() which will remove it from the list, and NULL
1921 * kn_knlist.
1922 *
1923 * KN_DISABLED will be stable while the knote is in flux, so the
1924 * unlocked read will not race with an update.
1925 */
1926 if ((kn->kn_status & KN_DISABLED) == 0)
1927 event = kn->kn_fop->f_event(kn, 0);
1928 else
1929 event = 0;
1930
1931 KQ_LOCK(kq);
1932 if (event)
1933 kn->kn_status |= KN_ACTIVE;
1934 if ((kn->kn_status & (KN_ACTIVE | KN_DISABLED | KN_QUEUED)) ==
1935 KN_ACTIVE)
1936 knote_enqueue(kn);
1937 kn->kn_status &= ~KN_SCAN;
1938 kn_leave_flux(kn);
1939 kn_list_unlock(knl);
1940 KQ_UNLOCK_FLUX(kq);
1941
1942 done:
1943 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1944 if (filedesc_unlock)
1945 FILEDESC_XUNLOCK(td->td_proc->p_fd);
1946 if (fp != NULL)
1947 fdrop(fp, td);
1948 knote_free(tkn);
1949 if (fops != NULL)
1950 kqueue_fo_release(filt);
1951 return (error);
1952 }
1953
1954 static int
kqueue_acquire_ref(struct kqueue * kq)1955 kqueue_acquire_ref(struct kqueue *kq)
1956 {
1957 KQ_LOCK(kq);
1958 if ((kq->kq_state & KQ_CLOSING) == KQ_CLOSING) {
1959 KQ_UNLOCK(kq);
1960 return (EBADF);
1961 }
1962 kq->kq_refcnt++;
1963 KQ_UNLOCK(kq);
1964 return (0);
1965 }
1966
1967 static int
kqueue_acquire(struct file * fp,struct kqueue ** kqp)1968 kqueue_acquire(struct file *fp, struct kqueue **kqp)
1969 {
1970 struct kqueue *kq;
1971 int error;
1972
1973 kq = fp->f_data;
1974 if (fp->f_type != DTYPE_KQUEUE || kq == NULL)
1975 return (EINVAL);
1976 error = kqueue_acquire_ref(kq);
1977 if (error == 0)
1978 *kqp = kq;
1979 return (error);
1980 }
1981
1982 static void
kqueue_release(struct kqueue * kq,int locked)1983 kqueue_release(struct kqueue *kq, int locked)
1984 {
1985 if (locked)
1986 KQ_OWNED(kq);
1987 else
1988 KQ_LOCK(kq);
1989 kq->kq_refcnt--;
1990 if (kq->kq_refcnt == 1)
1991 wakeup(&kq->kq_refcnt);
1992 if (!locked)
1993 KQ_UNLOCK(kq);
1994 }
1995
1996 static void
ast_kqueue(struct thread * td,int tda __unused)1997 ast_kqueue(struct thread *td, int tda __unused)
1998 {
1999 taskqueue_quiesce(taskqueue_kqueue_ctx);
2000 }
2001
2002 static void
kqueue_schedtask(struct kqueue * kq)2003 kqueue_schedtask(struct kqueue *kq)
2004 {
2005 KQ_OWNED(kq);
2006 KASSERT(((kq->kq_state & KQ_TASKDRAIN) != KQ_TASKDRAIN),
2007 ("scheduling kqueue task while draining"));
2008
2009 if ((kq->kq_state & KQ_TASKSCHED) != KQ_TASKSCHED) {
2010 taskqueue_enqueue(taskqueue_kqueue_ctx, &kq->kq_task);
2011 kq->kq_state |= KQ_TASKSCHED;
2012 ast_sched(curthread, TDA_KQUEUE);
2013 }
2014 }
2015
2016 /*
2017 * Expand the kq to make sure we have storage for fops/ident pair.
2018 *
2019 * Return 0 on success (or no work necessary), return errno on failure.
2020 */
2021 static int
kqueue_expand(struct kqueue * kq,const struct filterops * fops,uintptr_t ident,int mflag)2022 kqueue_expand(struct kqueue *kq, const struct filterops *fops, uintptr_t ident,
2023 int mflag)
2024 {
2025 struct klist *list, *tmp_knhash, *to_free;
2026 u_long tmp_knhashmask;
2027 int error, fd, size;
2028
2029 KQ_NOTOWNED(kq);
2030
2031 error = 0;
2032 to_free = NULL;
2033 if (fops->f_isfd) {
2034 fd = ident;
2035 size = atomic_load_int(&kq->kq_knlistsize);
2036 if (size <= fd) {
2037 do {
2038 size += KQEXTENT;
2039 } while (size <= fd);
2040 list = malloc(size * sizeof(*list), M_KQUEUE, mflag);
2041 if (list == NULL)
2042 return ENOMEM;
2043 KQ_LOCK(kq);
2044 if ((kq->kq_state & KQ_CLOSING) != 0) {
2045 to_free = list;
2046 error = EBADF;
2047 } else if (kq->kq_knlistsize >= size) {
2048 to_free = list;
2049 } else {
2050 if (kq->kq_knlist != NULL) {
2051 bcopy(kq->kq_knlist, list,
2052 kq->kq_knlistsize * sizeof(*list));
2053 to_free = kq->kq_knlist;
2054 kq->kq_knlist = NULL;
2055 }
2056 bzero((caddr_t)list +
2057 kq->kq_knlistsize * sizeof(*list),
2058 (size - kq->kq_knlistsize) * sizeof(*list));
2059 kq->kq_knlistsize = size;
2060 kq->kq_knlist = list;
2061 }
2062 MPASS(error != 0 || kq->kq_knlistsize > fd);
2063 KQ_UNLOCK(kq);
2064 }
2065 } else {
2066 if (kq->kq_knhashmask == 0) {
2067 tmp_knhash = hashinit_flags(KN_HASHSIZE, M_KQUEUE,
2068 &tmp_knhashmask, (mflag & M_WAITOK) != 0 ?
2069 HASH_WAITOK : HASH_NOWAIT);
2070 if (tmp_knhash == NULL)
2071 return (ENOMEM);
2072 KQ_LOCK(kq);
2073 if ((kq->kq_state & KQ_CLOSING) != 0) {
2074 to_free = tmp_knhash;
2075 error = EBADF;
2076 } else if (kq->kq_knhashmask == 0) {
2077 kq->kq_knhash = tmp_knhash;
2078 kq->kq_knhashmask = tmp_knhashmask;
2079 } else {
2080 to_free = tmp_knhash;
2081 }
2082 KQ_UNLOCK(kq);
2083 }
2084 }
2085 free(to_free, M_KQUEUE);
2086
2087 KQ_NOTOWNED(kq);
2088 return (error);
2089 }
2090
2091 static void
kqueue_task(void * arg,int pending)2092 kqueue_task(void *arg, int pending)
2093 {
2094 struct kqueue *kq;
2095 int haskqglobal;
2096
2097 haskqglobal = 0;
2098 kq = arg;
2099
2100 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
2101 KQ_LOCK(kq);
2102
2103 KNOTE_LOCKED(&kq->kq_sel.si_note, 0);
2104
2105 kq->kq_state &= ~KQ_TASKSCHED;
2106 if ((kq->kq_state & KQ_TASKDRAIN) == KQ_TASKDRAIN) {
2107 wakeup(&kq->kq_state);
2108 }
2109 KQ_UNLOCK(kq);
2110 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
2111 }
2112
2113 /*
2114 * Scan, update kn_data (if not ONESHOT), and copyout triggered events.
2115 * We treat KN_MARKER knotes as if they are in flux.
2116 */
2117 static int
kqueue_scan(struct kqueue * kq,int maxevents,struct kevent_copyops * k_ops,const struct timespec * tsp,struct kevent * keva,struct thread * td)2118 kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops,
2119 const struct timespec *tsp, struct kevent *keva, struct thread *td)
2120 {
2121 struct kevent *kevp;
2122 struct knote *kn, marker;
2123 struct knlist *knl;
2124 sbintime_t asbt, rsbt;
2125 int count, error, haskqglobal, influx, nkev, touch;
2126
2127 count = maxevents;
2128 nkev = 0;
2129 error = 0;
2130 haskqglobal = 0;
2131
2132 if (maxevents == 0)
2133 goto done_nl;
2134 if (maxevents < 0) {
2135 error = EINVAL;
2136 goto done_nl;
2137 }
2138
2139 rsbt = 0;
2140 if (tsp != NULL) {
2141 if (!timespecvalid_interval(tsp)) {
2142 error = EINVAL;
2143 goto done_nl;
2144 }
2145 if (timespecisset(tsp)) {
2146 if (tsp->tv_sec <= INT32_MAX) {
2147 rsbt = tstosbt(*tsp);
2148 if (TIMESEL(&asbt, rsbt))
2149 asbt += tc_tick_sbt;
2150 if (asbt <= SBT_MAX - rsbt)
2151 asbt += rsbt;
2152 else
2153 asbt = 0;
2154 rsbt >>= tc_precexp;
2155 } else
2156 asbt = 0;
2157 } else
2158 asbt = -1;
2159 } else
2160 asbt = 0;
2161 memset(&marker, 0, sizeof(marker));
2162 marker.kn_status = KN_MARKER;
2163 marker.kn_kq = kq;
2164 KQ_LOCK(kq);
2165
2166 retry:
2167 kevp = keva;
2168 if (kq->kq_count == 0) {
2169 if (asbt == -1) {
2170 error = EWOULDBLOCK;
2171 } else {
2172 kq->kq_state |= KQ_SLEEP;
2173 error = msleep_sbt(kq, &kq->kq_lock, PSOCK | PCATCH,
2174 "kqread", asbt, rsbt, C_ABSOLUTE);
2175 }
2176 if (error == 0)
2177 goto retry;
2178 /* don't restart after signals... */
2179 if (error == ERESTART)
2180 error = EINTR;
2181 else if (error == EWOULDBLOCK)
2182 error = 0;
2183 goto done;
2184 }
2185
2186 TAILQ_INSERT_TAIL(&kq->kq_head, &marker, kn_tqe);
2187 influx = 0;
2188 while (count) {
2189 KQ_OWNED(kq);
2190 kn = TAILQ_FIRST(&kq->kq_head);
2191
2192 if ((kn->kn_status == KN_MARKER && kn != &marker) ||
2193 kn_in_flux(kn)) {
2194 if (influx) {
2195 influx = 0;
2196 KQ_FLUX_WAKEUP(kq);
2197 }
2198 KQ_FLUX_SLEEP(kq, kn, 0);
2199 continue;
2200 }
2201
2202 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
2203 if ((kn->kn_status & KN_DISABLED) == KN_DISABLED) {
2204 kn->kn_status &= ~KN_QUEUED;
2205 kq->kq_count--;
2206 continue;
2207 }
2208 if (kn == &marker) {
2209 KQ_FLUX_WAKEUP(kq);
2210 if (count == maxevents)
2211 goto retry;
2212 goto done;
2213 }
2214 KASSERT(!kn_in_flux(kn),
2215 ("knote %p is unexpectedly in flux", kn));
2216
2217 if ((kn->kn_flags & EV_DROP) == EV_DROP) {
2218 kn->kn_status &= ~KN_QUEUED;
2219 kn_enter_flux(kn);
2220 kq->kq_count--;
2221 KQ_UNLOCK(kq);
2222 /*
2223 * We don't need to lock the list since we've
2224 * marked it as in flux.
2225 */
2226 knote_drop(kn, td);
2227 KQ_LOCK(kq);
2228 continue;
2229 } else if ((kn->kn_flags & EV_ONESHOT) == EV_ONESHOT) {
2230 kn->kn_status &= ~KN_QUEUED;
2231 kn_enter_flux(kn);
2232 kq->kq_count--;
2233 KQ_UNLOCK(kq);
2234 /*
2235 * We don't need to lock the list since we've
2236 * marked the knote as being in flux.
2237 */
2238 *kevp = kn->kn_kevent;
2239 knote_drop(kn, td);
2240 KQ_LOCK(kq);
2241 kn = NULL;
2242 } else {
2243 kn->kn_status |= KN_SCAN;
2244 kn_enter_flux(kn);
2245 KQ_UNLOCK(kq);
2246 if ((kn->kn_status & KN_KQUEUE) == KN_KQUEUE)
2247 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
2248 knl = kn_list_lock(kn);
2249 if (kn->kn_fop->f_event(kn, 0) == 0) {
2250 KQ_LOCK(kq);
2251 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
2252 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE |
2253 KN_SCAN);
2254 kn_leave_flux(kn);
2255 kq->kq_count--;
2256 kn_list_unlock(knl);
2257 influx = 1;
2258 continue;
2259 }
2260 touch = (!kn->kn_fop->f_isfd &&
2261 kn->kn_fop->f_touch != NULL);
2262 if (touch)
2263 kn->kn_fop->f_touch(kn, kevp, EVENT_PROCESS);
2264 else
2265 *kevp = kn->kn_kevent;
2266 KQ_LOCK(kq);
2267 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
2268 if (kn->kn_flags & (EV_CLEAR | EV_DISPATCH)) {
2269 /*
2270 * Manually clear knotes who weren't
2271 * 'touch'ed.
2272 */
2273 if (touch == 0 && kn->kn_flags & EV_CLEAR) {
2274 kn->kn_data = 0;
2275 kn->kn_fflags = 0;
2276 }
2277 if (kn->kn_flags & EV_DISPATCH)
2278 kn->kn_status |= KN_DISABLED;
2279 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE);
2280 kq->kq_count--;
2281 } else
2282 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
2283
2284 kn->kn_status &= ~KN_SCAN;
2285 kn_leave_flux(kn);
2286 kn_list_unlock(knl);
2287 influx = 1;
2288 }
2289
2290 /* we are returning a copy to the user */
2291 kevp++;
2292 nkev++;
2293 count--;
2294
2295 if (nkev == KQ_NEVENTS) {
2296 influx = 0;
2297 KQ_UNLOCK_FLUX(kq);
2298 error = k_ops->k_copyout(k_ops->arg, keva, nkev);
2299 nkev = 0;
2300 kevp = keva;
2301 KQ_LOCK(kq);
2302 if (error)
2303 break;
2304 }
2305 }
2306 TAILQ_REMOVE(&kq->kq_head, &marker, kn_tqe);
2307 done:
2308 KQ_OWNED(kq);
2309 KQ_UNLOCK_FLUX(kq);
2310 done_nl:
2311 KQ_NOTOWNED(kq);
2312 if (nkev != 0)
2313 error = k_ops->k_copyout(k_ops->arg, keva, nkev);
2314 td->td_retval[0] = maxevents - count;
2315 return (error);
2316 }
2317
2318 /*ARGSUSED*/
2319 static int
kqueue_ioctl(struct file * fp,u_long cmd,void * data,struct ucred * active_cred,struct thread * td)2320 kqueue_ioctl(struct file *fp, u_long cmd, void *data,
2321 struct ucred *active_cred, struct thread *td)
2322 {
2323 /*
2324 * Enabling sigio causes two major problems:
2325 * 1) infinite recursion:
2326 * Synopsys: kevent is being used to track signals and have FIOASYNC
2327 * set. On receipt of a signal this will cause a kqueue to recurse
2328 * into itself over and over. Sending the sigio causes the kqueue
2329 * to become ready, which in turn posts sigio again, forever.
2330 * Solution: this can be solved by setting a flag in the kqueue that
2331 * we have a SIGIO in progress.
2332 * 2) locking problems:
2333 * Synopsys: Kqueue is a leaf subsystem, but adding signalling puts
2334 * us above the proc and pgrp locks.
2335 * Solution: Post a signal using an async mechanism, being sure to
2336 * record a generation count in the delivery so that we do not deliver
2337 * a signal to the wrong process.
2338 *
2339 * Note, these two mechanisms are somewhat mutually exclusive!
2340 */
2341 #if 0
2342 struct kqueue *kq;
2343
2344 kq = fp->f_data;
2345 switch (cmd) {
2346 case FIOASYNC:
2347 if (*(int *)data) {
2348 kq->kq_state |= KQ_ASYNC;
2349 } else {
2350 kq->kq_state &= ~KQ_ASYNC;
2351 }
2352 return (0);
2353
2354 case FIOSETOWN:
2355 return (fsetown(*(int *)data, &kq->kq_sigio));
2356
2357 case FIOGETOWN:
2358 *(int *)data = fgetown(&kq->kq_sigio);
2359 return (0);
2360 }
2361 #endif
2362
2363 return (ENOTTY);
2364 }
2365
2366 /*ARGSUSED*/
2367 static int
kqueue_poll(struct file * fp,int events,struct ucred * active_cred,struct thread * td)2368 kqueue_poll(struct file *fp, int events, struct ucred *active_cred,
2369 struct thread *td)
2370 {
2371 struct kqueue *kq;
2372 int revents = 0;
2373 int error;
2374
2375 if ((error = kqueue_acquire(fp, &kq)))
2376 return POLLERR;
2377
2378 KQ_LOCK(kq);
2379 if (events & (POLLIN | POLLRDNORM)) {
2380 if (kq->kq_count) {
2381 revents |= events & (POLLIN | POLLRDNORM);
2382 } else {
2383 selrecord(td, &kq->kq_sel);
2384 if (SEL_WAITING(&kq->kq_sel))
2385 kq->kq_state |= KQ_SEL;
2386 }
2387 }
2388 kqueue_release(kq, 1);
2389 KQ_UNLOCK(kq);
2390 return (revents);
2391 }
2392
2393 /*ARGSUSED*/
2394 static int
kqueue_stat(struct file * fp,struct stat * st,struct ucred * active_cred)2395 kqueue_stat(struct file *fp, struct stat *st, struct ucred *active_cred)
2396 {
2397
2398 bzero((void *)st, sizeof *st);
2399 /*
2400 * We no longer return kq_count because the unlocked value is useless.
2401 * If you spent all this time getting the count, why not spend your
2402 * syscall better by calling kevent?
2403 *
2404 * XXX - This is needed for libc_r.
2405 */
2406 st->st_mode = S_IFIFO;
2407 return (0);
2408 }
2409
2410 static void
kqueue_drain(struct kqueue * kq,struct thread * td)2411 kqueue_drain(struct kqueue *kq, struct thread *td)
2412 {
2413 struct knote *kn;
2414 int i;
2415
2416 KQ_LOCK(kq);
2417
2418 KASSERT((kq->kq_state & KQ_CLOSING) != KQ_CLOSING,
2419 ("kqueue already closing"));
2420 kq->kq_state |= KQ_CLOSING;
2421 if (kq->kq_refcnt > 1)
2422 msleep(&kq->kq_refcnt, &kq->kq_lock, PSOCK, "kqclose", 0);
2423
2424 KASSERT(kq->kq_refcnt == 1, ("other refs are out there!"));
2425
2426 KASSERT(knlist_empty(&kq->kq_sel.si_note),
2427 ("kqueue's knlist not empty"));
2428
2429 for (i = 0; i < kq->kq_knlistsize; i++) {
2430 while ((kn = SLIST_FIRST(&kq->kq_knlist[i])) != NULL) {
2431 if (kn_in_flux(kn)) {
2432 KQ_FLUX_SLEEP_WMESG(kq, kn, 0, "kqclo1");
2433 continue;
2434 }
2435 kn_enter_flux(kn);
2436 KQ_UNLOCK(kq);
2437 knote_drop(kn, td);
2438 KQ_LOCK(kq);
2439 }
2440 }
2441 if (kq->kq_knhashmask != 0) {
2442 for (i = 0; i <= kq->kq_knhashmask; i++) {
2443 while ((kn = SLIST_FIRST(&kq->kq_knhash[i])) != NULL) {
2444 if (kn_in_flux(kn)) {
2445 KQ_FLUX_SLEEP_WMESG(kq, kn, 0,
2446 "kqclo2");
2447 continue;
2448 }
2449 kn_enter_flux(kn);
2450 KQ_UNLOCK(kq);
2451 knote_drop(kn, td);
2452 KQ_LOCK(kq);
2453 }
2454 }
2455 }
2456
2457 if ((kq->kq_state & KQ_TASKSCHED) == KQ_TASKSCHED) {
2458 kq->kq_state |= KQ_TASKDRAIN;
2459 msleep(&kq->kq_state, &kq->kq_lock, PSOCK, "kqtqdr", 0);
2460 }
2461
2462 if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
2463 selwakeuppri(&kq->kq_sel, PSOCK);
2464 if (!SEL_WAITING(&kq->kq_sel))
2465 kq->kq_state &= ~KQ_SEL;
2466 }
2467
2468 KQ_UNLOCK(kq);
2469 }
2470
2471 static void
kqueue_destroy(struct kqueue * kq)2472 kqueue_destroy(struct kqueue *kq)
2473 {
2474
2475 KASSERT(kq->kq_fdp == NULL,
2476 ("kqueue still attached to a file descriptor"));
2477 seldrain(&kq->kq_sel);
2478 knlist_destroy(&kq->kq_sel.si_note);
2479 mtx_destroy(&kq->kq_lock);
2480
2481 if (kq->kq_knhash != NULL)
2482 free(kq->kq_knhash, M_KQUEUE);
2483 if (kq->kq_knlist != NULL)
2484 free(kq->kq_knlist, M_KQUEUE);
2485
2486 funsetown(&kq->kq_sigio);
2487 }
2488
2489 /*ARGSUSED*/
2490 static int
kqueue_close(struct file * fp,struct thread * td)2491 kqueue_close(struct file *fp, struct thread *td)
2492 {
2493 struct kqueue *kq = fp->f_data;
2494 struct filedesc *fdp;
2495 int error;
2496 int filedesc_unlock;
2497
2498 if ((error = kqueue_acquire(fp, &kq)))
2499 return error;
2500 kqueue_drain(kq, td);
2501
2502 /*
2503 * We could be called due to the knote_drop() doing fdrop(),
2504 * called from kqueue_register(). In this case the global
2505 * lock is owned, and filedesc sx is locked before, to not
2506 * take the sleepable lock after non-sleepable.
2507 */
2508 fdp = kq->kq_fdp;
2509 kq->kq_fdp = NULL;
2510 if (!sx_xlocked(FILEDESC_LOCK(fdp))) {
2511 FILEDESC_XLOCK(fdp);
2512 filedesc_unlock = 1;
2513 } else
2514 filedesc_unlock = 0;
2515 TAILQ_REMOVE(&fdp->fd_kqlist, kq, kq_list);
2516 if (filedesc_unlock)
2517 FILEDESC_XUNLOCK(fdp);
2518
2519 kqueue_destroy(kq);
2520 chgkqcnt(kq->kq_cred->cr_ruidinfo, -1, 0);
2521 crfree(kq->kq_cred);
2522 free(kq, M_KQUEUE);
2523 fp->f_data = NULL;
2524
2525 return (0);
2526 }
2527
2528 static int
kqueue_fill_kinfo(struct file * fp,struct kinfo_file * kif,struct filedesc * fdp)2529 kqueue_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
2530 {
2531 struct kqueue *kq = fp->f_data;
2532
2533 kif->kf_type = KF_TYPE_KQUEUE;
2534 kif->kf_un.kf_kqueue.kf_kqueue_addr = (uintptr_t)kq;
2535 kif->kf_un.kf_kqueue.kf_kqueue_count = kq->kq_count;
2536 kif->kf_un.kf_kqueue.kf_kqueue_state = kq->kq_state;
2537 return (0);
2538 }
2539
2540 static void
kqueue_wakeup(struct kqueue * kq)2541 kqueue_wakeup(struct kqueue *kq)
2542 {
2543 KQ_OWNED(kq);
2544
2545 if ((kq->kq_state & KQ_SLEEP) == KQ_SLEEP) {
2546 kq->kq_state &= ~KQ_SLEEP;
2547 wakeup(kq);
2548 }
2549 if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
2550 selwakeuppri(&kq->kq_sel, PSOCK);
2551 if (!SEL_WAITING(&kq->kq_sel))
2552 kq->kq_state &= ~KQ_SEL;
2553 }
2554 if (!knlist_empty(&kq->kq_sel.si_note))
2555 kqueue_schedtask(kq);
2556 if ((kq->kq_state & KQ_ASYNC) == KQ_ASYNC) {
2557 pgsigio(&kq->kq_sigio, SIGIO, 0);
2558 }
2559 }
2560
2561 /*
2562 * Walk down a list of knotes, activating them if their event has triggered.
2563 *
2564 * There is a possibility to optimize in the case of one kq watching another.
2565 * Instead of scheduling a task to wake it up, you could pass enough state
2566 * down the chain to make up the parent kqueue. Make this code functional
2567 * first.
2568 */
2569 void
knote(struct knlist * list,long hint,int lockflags)2570 knote(struct knlist *list, long hint, int lockflags)
2571 {
2572 struct kqueue *kq;
2573 struct knote *kn, *tkn;
2574 int error;
2575
2576 if (list == NULL)
2577 return;
2578
2579 KNL_ASSERT_LOCK(list, lockflags & KNF_LISTLOCKED);
2580
2581 if ((lockflags & KNF_LISTLOCKED) == 0)
2582 list->kl_lock(list->kl_lockarg);
2583
2584 /*
2585 * If we unlock the list lock (and enter influx), we can
2586 * eliminate the kqueue scheduling, but this will introduce
2587 * four lock/unlock's for each knote to test. Also, marker
2588 * would be needed to keep iteration position, since filters
2589 * or other threads could remove events.
2590 */
2591 SLIST_FOREACH_SAFE(kn, &list->kl_list, kn_selnext, tkn) {
2592 kq = kn->kn_kq;
2593 KQ_LOCK(kq);
2594 if (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0) {
2595 /*
2596 * Do not process the influx notes, except for
2597 * the influx coming from the kq unlock in the
2598 * kqueue_scan(). In the later case, we do
2599 * not interfere with the scan, since the code
2600 * fragment in kqueue_scan() locks the knlist,
2601 * and cannot proceed until we finished.
2602 */
2603 KQ_UNLOCK(kq);
2604 } else if ((lockflags & KNF_NOKQLOCK) != 0) {
2605 kn_enter_flux(kn);
2606 KQ_UNLOCK(kq);
2607 error = kn->kn_fop->f_event(kn, hint);
2608 KQ_LOCK(kq);
2609 kn_leave_flux(kn);
2610 if (error)
2611 KNOTE_ACTIVATE(kn, 1);
2612 KQ_UNLOCK_FLUX(kq);
2613 } else {
2614 if (kn->kn_fop->f_event(kn, hint))
2615 KNOTE_ACTIVATE(kn, 1);
2616 KQ_UNLOCK(kq);
2617 }
2618 }
2619 if ((lockflags & KNF_LISTLOCKED) == 0)
2620 list->kl_unlock(list->kl_lockarg);
2621 }
2622
2623 /*
2624 * add a knote to a knlist
2625 */
2626 void
knlist_add(struct knlist * knl,struct knote * kn,int islocked)2627 knlist_add(struct knlist *knl, struct knote *kn, int islocked)
2628 {
2629
2630 KNL_ASSERT_LOCK(knl, islocked);
2631 KQ_NOTOWNED(kn->kn_kq);
2632 KASSERT(kn_in_flux(kn), ("knote %p not in flux", kn));
2633 KASSERT((kn->kn_status & KN_DETACHED) != 0,
2634 ("knote %p was not detached", kn));
2635 KASSERT(kn->kn_knlist == NULL,
2636 ("knote %p was already on knlist %p", kn, kn->kn_knlist));
2637 if (!islocked)
2638 knl->kl_lock(knl->kl_lockarg);
2639 SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext);
2640 if (!islocked)
2641 knl->kl_unlock(knl->kl_lockarg);
2642 KQ_LOCK(kn->kn_kq);
2643 kn->kn_knlist = knl;
2644 kn->kn_status &= ~KN_DETACHED;
2645 KQ_UNLOCK(kn->kn_kq);
2646 }
2647
2648 static void
knlist_remove_kq(struct knlist * knl,struct knote * kn,int knlislocked,int kqislocked)2649 knlist_remove_kq(struct knlist *knl, struct knote *kn, int knlislocked,
2650 int kqislocked)
2651 {
2652
2653 KASSERT(!kqislocked || knlislocked, ("kq locked w/o knl locked"));
2654 KNL_ASSERT_LOCK(knl, knlislocked);
2655 mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED);
2656 KASSERT(kqislocked || kn_in_flux(kn), ("knote %p not in flux", kn));
2657 KASSERT((kn->kn_status & KN_DETACHED) == 0,
2658 ("knote %p was already detached", kn));
2659 KASSERT(kn->kn_knlist == knl,
2660 ("knote %p was not on knlist %p", kn, knl));
2661 if (!knlislocked)
2662 knl->kl_lock(knl->kl_lockarg);
2663 SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext);
2664 kn->kn_knlist = NULL;
2665 if (!knlislocked)
2666 kn_list_unlock(knl);
2667 if (!kqislocked)
2668 KQ_LOCK(kn->kn_kq);
2669 kn->kn_status |= KN_DETACHED;
2670 if (!kqislocked)
2671 KQ_UNLOCK(kn->kn_kq);
2672 }
2673
2674 /*
2675 * remove knote from the specified knlist
2676 */
2677 void
knlist_remove(struct knlist * knl,struct knote * kn,int islocked)2678 knlist_remove(struct knlist *knl, struct knote *kn, int islocked)
2679 {
2680
2681 knlist_remove_kq(knl, kn, islocked, 0);
2682 }
2683
2684 int
knlist_empty(struct knlist * knl)2685 knlist_empty(struct knlist *knl)
2686 {
2687
2688 KNL_ASSERT_LOCKED(knl);
2689 return (SLIST_EMPTY(&knl->kl_list));
2690 }
2691
2692 static struct mtx knlist_lock;
2693 MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects",
2694 MTX_DEF);
2695 static void knlist_mtx_lock(void *arg);
2696 static void knlist_mtx_unlock(void *arg);
2697
2698 static void
knlist_mtx_lock(void * arg)2699 knlist_mtx_lock(void *arg)
2700 {
2701
2702 mtx_lock((struct mtx *)arg);
2703 }
2704
2705 static void
knlist_mtx_unlock(void * arg)2706 knlist_mtx_unlock(void *arg)
2707 {
2708
2709 mtx_unlock((struct mtx *)arg);
2710 }
2711
2712 static void
knlist_mtx_assert_lock(void * arg,int what)2713 knlist_mtx_assert_lock(void *arg, int what)
2714 {
2715
2716 if (what == LA_LOCKED)
2717 mtx_assert((struct mtx *)arg, MA_OWNED);
2718 else
2719 mtx_assert((struct mtx *)arg, MA_NOTOWNED);
2720 }
2721
2722 void
knlist_init(struct knlist * knl,void * lock,void (* kl_lock)(void *),void (* kl_unlock)(void *),void (* kl_assert_lock)(void *,int))2723 knlist_init(struct knlist *knl, void *lock, void (*kl_lock)(void *),
2724 void (*kl_unlock)(void *),
2725 void (*kl_assert_lock)(void *, int))
2726 {
2727
2728 if (lock == NULL)
2729 knl->kl_lockarg = &knlist_lock;
2730 else
2731 knl->kl_lockarg = lock;
2732
2733 if (kl_lock == NULL)
2734 knl->kl_lock = knlist_mtx_lock;
2735 else
2736 knl->kl_lock = kl_lock;
2737 if (kl_unlock == NULL)
2738 knl->kl_unlock = knlist_mtx_unlock;
2739 else
2740 knl->kl_unlock = kl_unlock;
2741 if (kl_assert_lock == NULL)
2742 knl->kl_assert_lock = knlist_mtx_assert_lock;
2743 else
2744 knl->kl_assert_lock = kl_assert_lock;
2745
2746 knl->kl_autodestroy = 0;
2747 SLIST_INIT(&knl->kl_list);
2748 }
2749
2750 void
knlist_init_mtx(struct knlist * knl,struct mtx * lock)2751 knlist_init_mtx(struct knlist *knl, struct mtx *lock)
2752 {
2753
2754 knlist_init(knl, lock, NULL, NULL, NULL);
2755 }
2756
2757 struct knlist *
knlist_alloc(struct mtx * lock)2758 knlist_alloc(struct mtx *lock)
2759 {
2760 struct knlist *knl;
2761
2762 knl = malloc(sizeof(struct knlist), M_KQUEUE, M_WAITOK);
2763 knlist_init_mtx(knl, lock);
2764 return (knl);
2765 }
2766
2767 void
knlist_destroy(struct knlist * knl)2768 knlist_destroy(struct knlist *knl)
2769 {
2770
2771 KASSERT(KNLIST_EMPTY(knl),
2772 ("destroying knlist %p with knotes on it", knl));
2773 }
2774
2775 void
knlist_detach(struct knlist * knl)2776 knlist_detach(struct knlist *knl)
2777 {
2778
2779 KNL_ASSERT_LOCKED(knl);
2780 knl->kl_autodestroy = 1;
2781 if (knlist_empty(knl)) {
2782 knlist_destroy(knl);
2783 free(knl, M_KQUEUE);
2784 }
2785 }
2786
2787 /*
2788 * Even if we are locked, we may need to drop the lock to allow any influx
2789 * knotes time to "settle".
2790 */
2791 void
knlist_cleardel(struct knlist * knl,struct thread * td,int islocked,int killkn)2792 knlist_cleardel(struct knlist *knl, struct thread *td, int islocked, int killkn)
2793 {
2794 struct knote *kn, *kn2;
2795 struct kqueue *kq;
2796 bool dropped;
2797
2798 KASSERT(!knl->kl_autodestroy, ("cleardel for autodestroy %p", knl));
2799 if (islocked)
2800 KNL_ASSERT_LOCKED(knl);
2801 else {
2802 KNL_ASSERT_UNLOCKED(knl);
2803 knl->kl_lock(knl->kl_lockarg);
2804 }
2805
2806 for (;;) {
2807 /*
2808 * Each pass removes as many knotes as we can before dropping
2809 * into FLUXWAIT. Active knotes are simply detached and either
2810 * freed or converted to one-shot, as the attached subject is
2811 * essentially disappearing.
2812 */
2813 dropped = false;
2814 SLIST_FOREACH_SAFE(kn, &knl->kl_list, kn_selnext, kn2) {
2815 kq = kn->kn_kq;
2816 KQ_LOCK(kq);
2817 if (kn_in_flux(kn)) {
2818 KQ_UNLOCK(kq);
2819 continue;
2820 }
2821 knlist_remove_kq(knl, kn, 1, 1);
2822 if (killkn) {
2823 kn_enter_flux(kn);
2824 KQ_UNLOCK(kq);
2825 knl->kl_unlock(knl->kl_lockarg);
2826 knote_drop_detached(kn, td);
2827 knl->kl_lock(knl->kl_lockarg);
2828 dropped = true;
2829 break;
2830 } else {
2831 /* Make sure cleared knotes disappear soon */
2832 kn->kn_flags |= EV_EOF | EV_ONESHOT;
2833 KQ_UNLOCK(kq);
2834 }
2835 kq = NULL;
2836 }
2837 if (dropped)
2838 continue;
2839
2840 if (SLIST_EMPTY(&knl->kl_list))
2841 break;
2842
2843 /* there are still in flux knotes remaining */
2844 kn = SLIST_FIRST(&knl->kl_list);
2845 kq = kn->kn_kq;
2846 KQ_LOCK(kq);
2847 KASSERT(kn_in_flux(kn), ("knote removed w/o list lock"));
2848 knl->kl_unlock(knl->kl_lockarg);
2849 KQ_FLUX_SLEEP_WMESG(kq, kn, PDROP, "kqkclr");
2850 kq = NULL;
2851 knl->kl_lock(knl->kl_lockarg);
2852 }
2853
2854 if (islocked)
2855 KNL_ASSERT_LOCKED(knl);
2856 else {
2857 knl->kl_unlock(knl->kl_lockarg);
2858 KNL_ASSERT_UNLOCKED(knl);
2859 }
2860 }
2861
2862 /*
2863 * Remove all knotes referencing a specified fd must be called with FILEDESC
2864 * lock. This prevents a race where a new fd comes along and occupies the
2865 * entry and we attach a knote to the fd.
2866 */
2867 void
knote_fdclose(struct thread * td,int fd)2868 knote_fdclose(struct thread *td, int fd)
2869 {
2870 struct filedesc *fdp = td->td_proc->p_fd;
2871 struct kqueue *kq;
2872 struct knote *kn;
2873
2874 FILEDESC_XLOCK_ASSERT(fdp);
2875
2876 /*
2877 * We shouldn't have to worry about new kevents appearing on fd
2878 * since filedesc is locked.
2879 */
2880 TAILQ_FOREACH(kq, &fdp->fd_kqlist, kq_list) {
2881 KQ_LOCK(kq);
2882 if (kq->kq_knlistsize <= fd ||
2883 SLIST_EMPTY(&kq->kq_knlist[fd])) {
2884 KQ_UNLOCK(kq);
2885 continue;
2886 }
2887
2888 while ((kn = SLIST_FIRST(&kq->kq_knlist[fd])) != NULL) {
2889 if (kn_in_flux(kn)) {
2890 /*
2891 * Wait for this knote to stabilize, it could be
2892 * the case that it's in the process of being
2893 * dropped anyways.
2894 */
2895 KQ_FLUX_SLEEP(kq, kn, 0);
2896 continue;
2897 }
2898 kn_enter_flux(kn);
2899 KQ_UNLOCK(kq);
2900 knote_drop(kn, td);
2901 KQ_LOCK(kq);
2902 }
2903 KQ_UNLOCK_FLUX(kq);
2904 }
2905 }
2906
2907 static int
knote_attach(struct knote * kn,struct kqueue * kq)2908 knote_attach(struct knote *kn, struct kqueue *kq)
2909 {
2910 struct klist *list;
2911
2912 KASSERT(kn_in_flux(kn), ("knote %p not marked influx", kn));
2913 KQ_OWNED(kq);
2914 MPASS(kn->kn_kq == kq);
2915
2916 if ((kq->kq_state & KQ_CLOSING) != 0)
2917 return (EBADF);
2918 if (kn->kn_fop->f_isfd) {
2919 if (kn->kn_id >= kq->kq_knlistsize)
2920 return (ENOMEM);
2921 list = &kq->kq_knlist[kn->kn_id];
2922 } else {
2923 if (kq->kq_knhash == NULL)
2924 return (ENOMEM);
2925 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
2926 }
2927 SLIST_INSERT_HEAD(list, kn, kn_link);
2928 return (0);
2929 }
2930
2931 static void
knote_drop(struct knote * kn,struct thread * td)2932 knote_drop(struct knote *kn, struct thread *td)
2933 {
2934
2935 if ((kn->kn_status & KN_DETACHED) == 0)
2936 kn->kn_fop->f_detach(kn);
2937 knote_drop_detached(kn, td);
2938 }
2939
2940 static void
knote_drop_detached(struct knote * kn,struct thread * td)2941 knote_drop_detached(struct knote *kn, struct thread *td)
2942 {
2943 struct kqueue *kq;
2944 struct klist *list;
2945
2946 kq = kn->kn_kq;
2947
2948 KASSERT((kn->kn_status & KN_DETACHED) != 0,
2949 ("knote %p still attached", kn));
2950 KQ_NOTOWNED(kq);
2951
2952 KQ_LOCK(kq);
2953 for (;;) {
2954 KASSERT(kn->kn_influx >= 1,
2955 ("knote_drop called on %p with influx %d",
2956 kn, kn->kn_influx));
2957 if (kn->kn_influx == 1)
2958 break;
2959 KQ_FLUX_SLEEP(kq, kn, 0);
2960 }
2961
2962 MPASS(kn->kn_kq == kq);
2963 if (kn->kn_fop->f_isfd)
2964 list = &kq->kq_knlist[kn->kn_id];
2965 else
2966 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
2967
2968 SLIST_REMOVE(list, kn, knote, kn_link);
2969 if (kn->kn_status & KN_QUEUED)
2970 knote_dequeue(kn);
2971 KQ_UNLOCK_FLUX(kq);
2972
2973 if (kn->kn_fop->f_isfd) {
2974 fdrop(kn->kn_fp, td);
2975 kn->kn_fp = NULL;
2976 }
2977 kqueue_fo_release(kn->kn_kevent.filter);
2978 kn->kn_fop = NULL;
2979 knote_free(kn);
2980 }
2981
2982 static void
knote_enqueue(struct knote * kn)2983 knote_enqueue(struct knote *kn)
2984 {
2985 struct kqueue *kq = kn->kn_kq;
2986
2987 KQ_OWNED(kn->kn_kq);
2988 KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued"));
2989
2990 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
2991 kn->kn_status |= KN_QUEUED;
2992 kq->kq_count++;
2993 kqueue_wakeup(kq);
2994 }
2995
2996 static void
knote_dequeue(struct knote * kn)2997 knote_dequeue(struct knote *kn)
2998 {
2999 struct kqueue *kq = kn->kn_kq;
3000
3001 KQ_OWNED(kn->kn_kq);
3002 KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued"));
3003
3004 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
3005 kn->kn_status &= ~KN_QUEUED;
3006 kq->kq_count--;
3007 }
3008
3009 static void
knote_init(void * dummy __unused)3010 knote_init(void *dummy __unused)
3011 {
3012
3013 knote_zone = uma_zcreate("KNOTE", sizeof(struct knote), NULL, NULL,
3014 NULL, NULL, UMA_ALIGN_PTR, 0);
3015 ast_register(TDA_KQUEUE, ASTR_ASTF_REQUIRED, 0, ast_kqueue);
3016 prison0.pr_klist = knlist_alloc(&prison0.pr_mtx);
3017 }
3018 SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL);
3019
3020 static struct knote *
knote_alloc(int mflag)3021 knote_alloc(int mflag)
3022 {
3023
3024 return (uma_zalloc(knote_zone, mflag | M_ZERO));
3025 }
3026
3027 static void
knote_free(struct knote * kn)3028 knote_free(struct knote *kn)
3029 {
3030
3031 uma_zfree(knote_zone, kn);
3032 }
3033
3034 /*
3035 * Register the kev w/ the kq specified by fd.
3036 */
3037 int
kqfd_register(int fd,struct kevent * kev,struct thread * td,int mflag)3038 kqfd_register(int fd, struct kevent *kev, struct thread *td, int mflag)
3039 {
3040 struct kqueue *kq;
3041 struct file *fp;
3042 cap_rights_t rights;
3043 int error;
3044
3045 error = fget(td, fd, cap_rights_init_one(&rights, CAP_KQUEUE_CHANGE),
3046 &fp);
3047 if (error != 0)
3048 return (error);
3049 if ((error = kqueue_acquire(fp, &kq)) != 0)
3050 goto noacquire;
3051
3052 error = kqueue_register(kq, kev, td, mflag);
3053 kqueue_release(kq, 0);
3054
3055 noacquire:
3056 fdrop(fp, td);
3057 return (error);
3058 }
3059
3060 static int
kqueue_fork_alloc(struct filedesc * fdp,struct file * fp,struct file ** fp1,struct thread * td)3061 kqueue_fork_alloc(struct filedesc *fdp, struct file *fp, struct file **fp1,
3062 struct thread *td)
3063 {
3064 struct kqueue *kq, *kq1;
3065 int error;
3066
3067 MPASS(fp->f_type == DTYPE_KQUEUE);
3068 kq = fp->f_data;
3069 if ((kq->kq_state & KQ_CPONFORK) == 0)
3070 return (EOPNOTSUPP);
3071 error = kqueue_acquire_ref(kq);
3072 if (error != 0)
3073 return (error);
3074 error = kern_kqueue_alloc(td, fdp, NULL, fp1, 0, NULL, true, &kq1);
3075 if (error == 0) {
3076 kq1->kq_forksrc = kq;
3077 (*fp1)->f_flag = fp->f_flag & (FREAD | FWRITE | FEXEC |
3078 O_CLOEXEC | O_CLOFORK);
3079 } else {
3080 kqueue_release(kq, 0);
3081 }
3082 return (error);
3083 }
3084
3085 static void
kqueue_fork_copy_knote(struct kqueue * kq,struct kqueue * kq1,struct knote * kn,struct proc * p1,struct filedesc * fdp)3086 kqueue_fork_copy_knote(struct kqueue *kq, struct kqueue *kq1, struct knote *kn,
3087 struct proc *p1, struct filedesc *fdp)
3088 {
3089 struct knote *kn1;
3090 struct knlist *knl;
3091 const struct filterops *fop;
3092 int error;
3093 bool enqueue;
3094
3095 KASSERT(kn->kn_influx != 0,
3096 ("%s: knote %p not in flux", __func__, kn));
3097 KASSERT((kn->kn_status & KN_DETACHED) == 0,
3098 ("%s: knote %p not detached", __func__, kn));
3099
3100 if ((kn->kn_status & KN_MARKER) != 0)
3101 return;
3102 if ((kn->kn_status & KN_KQUEUE) != 0) {
3103 /*
3104 * We cannot hold references to a kqueue outside of the process
3105 * itself, kqueue_close() does not handle this possibility.
3106 */
3107 return;
3108 }
3109 fop = kn->kn_fop;
3110 if (fop->f_copy == NULL || (fop->f_isfd &&
3111 fdp->fd_files->fdt_ofiles[kn->kn_kevent.ident].fde_file == NULL))
3112 return;
3113 error = kqueue_expand(kq1, fop, kn->kn_kevent.ident, M_WAITOK);
3114 if (error != 0)
3115 return;
3116
3117 kn1 = knote_alloc(M_WAITOK);
3118
3119 knl = kn_list_lock(kn);
3120 KQ_LOCK(kq);
3121 *kn1 = *kn;
3122 KQ_UNLOCK(kq);
3123 kn_list_unlock(knl);
3124 kn1->kn_status = KN_DETACHED | (kn1->kn_status & KN_CPONFORK);
3125 kn1->kn_kq = kq1;
3126 kn1->kn_knlist = NULL;
3127 error = fop->f_copy(kn1, p1);
3128 if (error != 0) {
3129 knote_free(kn1);
3130 return;
3131 }
3132 (void)kqueue_fo_find(kn->kn_kevent.filter);
3133 if (fop->f_isfd && !fhold(kn1->kn_fp)) {
3134 fop->f_detach(kn1);
3135 kqueue_fo_release(kn->kn_kevent.filter);
3136 knote_free(kn1);
3137 return;
3138 }
3139 if (kn->kn_knlist != NULL) {
3140 knl = kn_list_lock(kn);
3141 knlist_add(kn->kn_knlist, kn1, 1);
3142 } else {
3143 knl = NULL;
3144 }
3145 enqueue = kn->kn_fop->f_event(kn1, 0) != 0;
3146 kn_list_unlock(knl);
3147
3148 KQ_LOCK(kq1);
3149 knote_attach(kn1, kq1);
3150 kn1->kn_influx = 0;
3151 if (enqueue && (kn1->kn_status & KN_QUEUED) == 0)
3152 knote_enqueue(kn1);
3153 KQ_UNLOCK(kq1);
3154 }
3155
3156 static void
kqueue_fork_copy_list(struct klist * knlist,struct knote * marker,struct kqueue * kq,struct kqueue * kq1,struct proc * p1,struct filedesc * fdp)3157 kqueue_fork_copy_list(struct klist *knlist, struct knote *marker,
3158 struct kqueue *kq, struct kqueue *kq1, struct proc *p1,
3159 struct filedesc *fdp)
3160 {
3161 struct knote *kn;
3162
3163 KQ_OWNED(kq);
3164 kn = SLIST_FIRST(knlist);
3165 while (kn != NULL) {
3166 MPASS(kn->kn_kq == kq);
3167 if ((kn->kn_status & KN_DETACHED) != 0 ||
3168 (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0)) {
3169 kn = SLIST_NEXT(kn, kn_link);
3170 continue;
3171 }
3172 kn_enter_flux(kn);
3173 SLIST_INSERT_AFTER(kn, marker, kn_link);
3174 KQ_UNLOCK(kq);
3175 kqueue_fork_copy_knote(kq, kq1, kn, p1, fdp);
3176 KQ_LOCK(kq);
3177 kn_leave_flux(kn);
3178 kn = SLIST_NEXT(marker, kn_link);
3179 /* XXXKIB switch kn_link to LIST? */
3180 SLIST_REMOVE(knlist, marker, knote, kn_link);
3181 }
3182 }
3183
3184 static int
kqueue_fork_copy(struct filedesc * fdp,struct file * fp,struct file * fp1,struct proc * p1,struct thread * td)3185 kqueue_fork_copy(struct filedesc *fdp, struct file *fp, struct file *fp1,
3186 struct proc *p1, struct thread *td)
3187 {
3188 struct kqueue *kq, *kq1;
3189 struct knote marker;
3190 int error, i;
3191
3192 error = 0;
3193 MPASS(fp == NULL);
3194 MPASS(fp1->f_type == DTYPE_KQUEUE);
3195
3196 kq1 = fp1->f_data;
3197 kq = kq1->kq_forksrc;
3198 memset(&marker, 0, sizeof(marker));
3199 marker.kn_status = KN_MARKER;
3200 marker.kn_kq = kq;
3201
3202 KQ_LOCK(kq);
3203 for (i = 0; i < kq->kq_knlistsize; i++) {
3204 kqueue_fork_copy_list(&kq->kq_knlist[i], &marker, kq, kq1,
3205 p1, fdp);
3206 }
3207 if (kq->kq_knhashmask != 0) {
3208 for (i = 0; i <= kq->kq_knhashmask; i++) {
3209 kqueue_fork_copy_list(&kq->kq_knhash[i], &marker, kq,
3210 kq1, p1, fdp);
3211 }
3212 }
3213 kqueue_release(kq, 1);
3214 kq1->kq_forksrc = NULL;
3215 KQ_UNLOCK_FLUX(kq);
3216 return (error);
3217 }
3218
3219 static int
kqueue_fork(struct filedesc * fdp,struct file * fp,struct file ** fp1,struct proc * p1,struct thread * td)3220 kqueue_fork(struct filedesc *fdp, struct file *fp, struct file **fp1,
3221 struct proc *p1, struct thread *td)
3222 {
3223 if (*fp1 == NULL)
3224 return (kqueue_fork_alloc(fdp, fp, fp1, td));
3225 return (kqueue_fork_copy(fdp, fp, *fp1, p1, td));
3226 }
3227
3228 int
knote_triv_copy(struct knote * kn __unused,struct proc * p1 __unused)3229 knote_triv_copy(struct knote *kn __unused, struct proc *p1 __unused)
3230 {
3231 return (0);
3232 }
3233
3234 struct knote_status_export_bit {
3235 int kn_status_bit;
3236 int knt_status_bit;
3237 };
3238
3239 #define ST(name) \
3240 { .kn_status_bit = KN_##name, .knt_status_bit = KNOTE_STATUS_##name }
3241 static const struct knote_status_export_bit knote_status_export_bits[] = {
3242 ST(ACTIVE),
3243 ST(QUEUED),
3244 ST(DISABLED),
3245 ST(DETACHED),
3246 ST(KQUEUE),
3247 };
3248 #undef ST
3249
3250 static int
knote_status_export(int kn_status)3251 knote_status_export(int kn_status)
3252 {
3253 const struct knote_status_export_bit *b;
3254 unsigned i;
3255 int res;
3256
3257 res = 0;
3258 for (i = 0; i < nitems(knote_status_export_bits); i++) {
3259 b = &knote_status_export_bits[i];
3260 if ((kn_status & b->kn_status_bit) != 0)
3261 res |= b->knt_status_bit;
3262 }
3263 return (res);
3264 }
3265
3266 static int
kern_proc_kqueue_report_one(struct sbuf * s,struct proc * p,int kq_fd,struct kqueue * kq,struct knote * kn,bool compat32 __unused)3267 kern_proc_kqueue_report_one(struct sbuf *s, struct proc *p,
3268 int kq_fd, struct kqueue *kq, struct knote *kn, bool compat32 __unused)
3269 {
3270 struct kinfo_knote kin;
3271 #ifdef COMPAT_FREEBSD32
3272 struct kinfo_knote32 kin32;
3273 #endif
3274 int error;
3275
3276 if (kn->kn_status == KN_MARKER)
3277 return (0);
3278
3279 memset(&kin, 0, sizeof(kin));
3280 kin.knt_kq_fd = kq_fd;
3281 memcpy(&kin.knt_event, &kn->kn_kevent, sizeof(struct kevent));
3282 kin.knt_status = knote_status_export(kn->kn_status);
3283 kn_enter_flux(kn);
3284 KQ_UNLOCK_FLUX(kq);
3285 if (kn->kn_fop->f_userdump != NULL)
3286 (void)kn->kn_fop->f_userdump(p, kn, &kin);
3287 #ifdef COMPAT_FREEBSD32
3288 if (compat32) {
3289 freebsd32_kinfo_knote_to_32(&kin, &kin32);
3290 error = sbuf_bcat(s, &kin32, sizeof(kin32));
3291 } else
3292 #endif
3293 error = sbuf_bcat(s, &kin, sizeof(kin));
3294 KQ_LOCK(kq);
3295 kn_leave_flux(kn);
3296 return (error);
3297 }
3298
3299 static int
kern_proc_kqueue_report(struct sbuf * s,struct proc * p,int kq_fd,struct kqueue * kq,bool compat32)3300 kern_proc_kqueue_report(struct sbuf *s, struct proc *p, int kq_fd,
3301 struct kqueue *kq, bool compat32)
3302 {
3303 struct knote *kn;
3304 int error, i;
3305
3306 error = 0;
3307 KQ_LOCK(kq);
3308 for (i = 0; i < kq->kq_knlistsize; i++) {
3309 SLIST_FOREACH(kn, &kq->kq_knlist[i], kn_link) {
3310 MPASS(kn->kn_kq == kq);
3311 error = kern_proc_kqueue_report_one(s, p, kq_fd,
3312 kq, kn, compat32);
3313 if (error != 0)
3314 goto out;
3315 }
3316 }
3317 if (kq->kq_knhashmask == 0)
3318 goto out;
3319 for (i = 0; i <= kq->kq_knhashmask; i++) {
3320 SLIST_FOREACH(kn, &kq->kq_knhash[i], kn_link) {
3321 MPASS(kn->kn_kq == kq);
3322 error = kern_proc_kqueue_report_one(s, p, kq_fd,
3323 kq, kn, compat32);
3324 if (error != 0)
3325 goto out;
3326 }
3327 }
3328 out:
3329 KQ_UNLOCK_FLUX(kq);
3330 return (error);
3331 }
3332
3333 struct kern_proc_kqueues_out1_cb_args {
3334 struct sbuf *s;
3335 bool compat32;
3336 };
3337
3338 static int
kern_proc_kqueues_out1_cb(struct proc * p,int fd,struct file * fp,void * arg)3339 kern_proc_kqueues_out1_cb(struct proc *p, int fd, struct file *fp, void *arg)
3340 {
3341 struct kqueue *kq;
3342 struct kern_proc_kqueues_out1_cb_args *a;
3343
3344 if (fp->f_type != DTYPE_KQUEUE)
3345 return (0);
3346 a = arg;
3347 kq = fp->f_data;
3348 return (kern_proc_kqueue_report(a->s, p, fd, kq, a->compat32));
3349 }
3350
3351 static int
kern_proc_kqueues_out1(struct thread * td,struct proc * p,struct sbuf * s,bool compat32)3352 kern_proc_kqueues_out1(struct thread *td, struct proc *p, struct sbuf *s,
3353 bool compat32)
3354 {
3355 struct kern_proc_kqueues_out1_cb_args a;
3356
3357 a.s = s;
3358 a.compat32 = compat32;
3359 return (fget_remote_foreach(td, p, kern_proc_kqueues_out1_cb, &a));
3360 }
3361
3362 struct kern_proc_kqueues_drain_ctx {
3363 struct sbuf *sb;
3364 size_t remaining;
3365 bool full;
3366 };
3367
3368 static int
kern_proc_kqueues_drain(void * arg,const char * data,int len)3369 kern_proc_kqueues_drain(void *arg, const char *data, int len)
3370 {
3371 struct kern_proc_kqueues_drain_ctx *c;
3372 size_t n;
3373
3374 c = arg;
3375 n = MIN((size_t)len, c->remaining);
3376 if (n != 0) {
3377 if (sbuf_bcat(c->sb, data, n) != 0)
3378 return (-ENOMEM);
3379 c->remaining -= n;
3380 }
3381 if (c->remaining == 0) {
3382 c->full = true;
3383 return (-ENOSPC);
3384 }
3385 return (len);
3386 }
3387
3388 int
kern_proc_kqueues_out(struct proc * p,struct sbuf * sb,size_t maxlen,bool compat32)3389 kern_proc_kqueues_out(struct proc *p, struct sbuf *sb, size_t maxlen,
3390 bool compat32)
3391 {
3392 struct kern_proc_kqueues_drain_ctx c;
3393 struct sbuf *s, sm;
3394 int error;
3395
3396 if (maxlen == -1)
3397 return (kern_proc_kqueues_out1(curthread, p, sb, compat32));
3398
3399 c.sb = sb;
3400 c.remaining = maxlen;
3401 c.full = false;
3402 s = sbuf_new(&sm, NULL, PAGE_SIZE, SBUF_FIXEDLEN);
3403 sbuf_set_drain(s, kern_proc_kqueues_drain, &c);
3404 error = kern_proc_kqueues_out1(curthread, p, s, compat32);
3405 sbuf_finish(s);
3406 sbuf_delete(s);
3407 if (c.full)
3408 error = 0;
3409 return (error);
3410 }
3411
3412 static int
sysctl_kern_proc_kqueue_one(struct thread * td,struct sbuf * s,struct proc * p,int kq_fd,bool compat32)3413 sysctl_kern_proc_kqueue_one(struct thread *td, struct sbuf *s, struct proc *p,
3414 int kq_fd, bool compat32)
3415 {
3416 struct file *fp;
3417 struct kqueue *kq;
3418 int error;
3419
3420 error = fget_remote(td, p, kq_fd, NULL, NULL, &fp);
3421 if (error == 0) {
3422 if (fp->f_type != DTYPE_KQUEUE) {
3423 error = EINVAL;
3424 } else {
3425 kq = fp->f_data;
3426 error = kern_proc_kqueue_report(s, p, kq_fd, kq,
3427 compat32);
3428 }
3429 fdrop(fp, td);
3430 }
3431 return (error);
3432 }
3433
3434 static int
sysctl_kern_proc_kqueue(SYSCTL_HANDLER_ARGS)3435 sysctl_kern_proc_kqueue(SYSCTL_HANDLER_ARGS)
3436 {
3437 struct thread *td;
3438 struct proc *p;
3439 struct sbuf *s, sm;
3440 int error, error1, *name;
3441 bool compat32;
3442
3443 name = (int *)arg1;
3444 if ((u_int)arg2 > 2 || (u_int)arg2 == 0)
3445 return (EINVAL);
3446
3447 td = curthread;
3448 #ifdef COMPAT_FREEBSD32
3449 compat32 = SV_CURPROC_FLAG(SV_ILP32);
3450 #else
3451 compat32 = false;
3452 #endif
3453
3454 error = pget((pid_t)name[0], PGET_NOTWEXIT, &p);
3455 if (error != 0)
3456 return (error);
3457
3458 _PHOLD(p);
3459 execve_block_wait(td, p);
3460 error = p_candebug(td, p);
3461 if (error != 0)
3462 goto out1;
3463 PROC_UNLOCK(p);
3464
3465 s = sbuf_new_for_sysctl(&sm, NULL, 0, req);
3466 if (s == NULL) {
3467 error = ENOMEM;
3468 goto out;
3469 }
3470 sbuf_clear_flags(s, SBUF_INCLUDENUL);
3471
3472 if ((u_int)arg2 == 1) {
3473 error = kern_proc_kqueues_out1(td, p, s, compat32);
3474 } else {
3475 error = sysctl_kern_proc_kqueue_one(td, s, p,
3476 name[1] /* kq_fd */, compat32);
3477 }
3478
3479 error1 = sbuf_finish(s);
3480 if (error == 0)
3481 error = error1;
3482 sbuf_delete(s);
3483
3484 out:
3485 PROC_LOCK(p);
3486 out1:
3487 execve_unblock(td, p);
3488 _PRELE(p);
3489 PROC_UNLOCK(p);
3490 return (error);
3491 }
3492
3493 static SYSCTL_NODE(_kern_proc, KERN_PROC_KQUEUE, kq,
3494 CTLFLAG_RD | CTLFLAG_MPSAFE,
3495 sysctl_kern_proc_kqueue, "KQueue events");
3496