1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org>
5 * Copyright 2004 John-Mark Gurney <jmg@FreeBSD.org>
6 * Copyright (c) 2009 Apple, Inc.
7 * All rights reserved.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31 #include <sys/cdefs.h>
32 #include "opt_ktrace.h"
33 #include "opt_kqueue.h"
34
35 #ifdef COMPAT_FREEBSD11
36 #define _WANT_FREEBSD11_KEVENT
37 #endif
38
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/capsicum.h>
42 #include <sys/kernel.h>
43 #include <sys/limits.h>
44 #include <sys/lock.h>
45 #include <sys/mutex.h>
46 #include <sys/proc.h>
47 #include <sys/malloc.h>
48 #include <sys/unistd.h>
49 #include <sys/file.h>
50 #include <sys/filedesc.h>
51 #include <sys/filio.h>
52 #include <sys/fcntl.h>
53 #include <sys/jail.h>
54 #include <sys/jaildesc.h>
55 #include <sys/kthread.h>
56 #include <sys/selinfo.h>
57 #include <sys/queue.h>
58 #include <sys/event.h>
59 #include <sys/eventvar.h>
60 #include <sys/poll.h>
61 #include <sys/protosw.h>
62 #include <sys/resourcevar.h>
63 #include <sys/sbuf.h>
64 #include <sys/sigio.h>
65 #include <sys/signalvar.h>
66 #include <sys/socket.h>
67 #include <sys/socketvar.h>
68 #include <sys/stat.h>
69 #include <sys/sysctl.h>
70 #include <sys/sysent.h>
71 #include <sys/sysproto.h>
72 #include <sys/syscallsubr.h>
73 #include <sys/taskqueue.h>
74 #include <sys/uio.h>
75 #include <sys/user.h>
76 #ifdef KTRACE
77 #include <sys/ktrace.h>
78 #endif
79 #include <machine/atomic.h>
80 #ifdef COMPAT_FREEBSD32
81 #include <compat/freebsd32/freebsd32.h>
82 #include <compat/freebsd32/freebsd32_util.h>
83 #endif
84
85 #include <vm/uma.h>
86
87 static MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system");
88
89 /*
90 * This lock is used if multiple kq locks are required. This possibly
91 * should be made into a per proc lock.
92 */
93 static struct mtx kq_global;
94 MTX_SYSINIT(kq_global, &kq_global, "kqueue order", MTX_DEF);
95 #define KQ_GLOBAL_LOCK(lck, haslck) do { \
96 if (!haslck) \
97 mtx_lock(lck); \
98 haslck = 1; \
99 } while (0)
100 #define KQ_GLOBAL_UNLOCK(lck, haslck) do { \
101 if (haslck) \
102 mtx_unlock(lck); \
103 haslck = 0; \
104 } while (0)
105
106 TASKQUEUE_DEFINE_THREAD(kqueue_ctx);
107
108 static int kevent_copyout(void *arg, struct kevent *kevp, int count);
109 static int kevent_copyin(void *arg, struct kevent *kevp, int count);
110 static int kqueue_register(struct kqueue *kq, struct kevent *kev,
111 struct thread *td, int mflag);
112 static int kqueue_acquire(struct file *fp, struct kqueue **kqp);
113 static void kqueue_release(struct kqueue *kq, int locked);
114 static void kqueue_destroy(struct kqueue *kq);
115 static void kqueue_drain(struct kqueue *kq, struct thread *td);
116 static int kqueue_expand(struct kqueue *kq, const struct filterops *fops,
117 uintptr_t ident, int mflag);
118 static void kqueue_task(void *arg, int pending);
119 static int kqueue_scan(struct kqueue *kq, int maxevents,
120 struct kevent_copyops *k_ops,
121 const struct timespec *timeout,
122 struct kevent *keva, struct thread *td);
123 static void kqueue_wakeup(struct kqueue *kq);
124 static const struct filterops *kqueue_fo_find(int filt);
125 static void kqueue_fo_release(int filt);
126 struct g_kevent_args;
127 static int kern_kevent_generic(struct thread *td,
128 struct g_kevent_args *uap,
129 struct kevent_copyops *k_ops, const char *struct_name);
130
131 static fo_ioctl_t kqueue_ioctl;
132 static fo_poll_t kqueue_poll;
133 static fo_kqfilter_t kqueue_kqfilter;
134 static fo_stat_t kqueue_stat;
135 static fo_close_t kqueue_close;
136 static fo_fill_kinfo_t kqueue_fill_kinfo;
137 static fo_fork_t kqueue_fork;
138
139 static const struct fileops kqueueops = {
140 .fo_read = invfo_rdwr,
141 .fo_write = invfo_rdwr,
142 .fo_truncate = invfo_truncate,
143 .fo_ioctl = kqueue_ioctl,
144 .fo_poll = kqueue_poll,
145 .fo_kqfilter = kqueue_kqfilter,
146 .fo_stat = kqueue_stat,
147 .fo_close = kqueue_close,
148 .fo_chmod = invfo_chmod,
149 .fo_chown = invfo_chown,
150 .fo_sendfile = invfo_sendfile,
151 .fo_cmp = file_kcmp_generic,
152 .fo_fork = kqueue_fork,
153 .fo_fill_kinfo = kqueue_fill_kinfo,
154 .fo_flags = DFLAG_FORK,
155 };
156
157 static int knote_attach(struct knote *kn, struct kqueue *kq);
158 static void knote_drop(struct knote *kn, struct thread *td);
159 static void knote_drop_detached(struct knote *kn, struct thread *td);
160 static void knote_enqueue(struct knote *kn);
161 static void knote_dequeue(struct knote *kn);
162 static void knote_init(void *);
163 static struct knote *knote_alloc(int mflag);
164 static void knote_free(struct knote *kn);
165
166 static void filt_kqdetach(struct knote *kn);
167 static int filt_kqueue(struct knote *kn, long hint);
168 static int filt_procattach(struct knote *kn);
169 static void filt_procdetach(struct knote *kn);
170 static int filt_proc(struct knote *kn, long hint);
171 static int filt_jailattach(struct knote *kn);
172 static void filt_jaildetach(struct knote *kn);
173 static int filt_jail(struct knote *kn, long hint);
174 static int filt_fileattach(struct knote *kn);
175 static void filt_timerexpire(void *knx);
176 static void filt_timerexpire_l(struct knote *kn, bool proc_locked);
177 static int filt_timerattach(struct knote *kn);
178 static void filt_timerdetach(struct knote *kn);
179 static void filt_timerstart(struct knote *kn, sbintime_t to);
180 static void filt_timertouch(struct knote *kn, struct kevent *kev,
181 u_long type);
182 static int filt_timercopy(struct knote *kn, struct proc *p1);
183 static int filt_timervalidate(struct knote *kn, sbintime_t *to);
184 static int filt_timer(struct knote *kn, long hint);
185 static int filt_userattach(struct knote *kn);
186 static void filt_userdetach(struct knote *kn);
187 static int filt_user(struct knote *kn, long hint);
188 static void filt_usertouch(struct knote *kn, struct kevent *kev,
189 u_long type);
190
191 static const struct filterops file_filtops = {
192 .f_isfd = 1,
193 .f_attach = filt_fileattach,
194 .f_copy = knote_triv_copy,
195 };
196 static const struct filterops kqread_filtops = {
197 .f_isfd = 1,
198 .f_detach = filt_kqdetach,
199 .f_event = filt_kqueue,
200 .f_copy = knote_triv_copy,
201 };
202 /* XXX - move to kern_proc.c? */
203 static const struct filterops proc_filtops = {
204 .f_isfd = 0,
205 .f_attach = filt_procattach,
206 .f_detach = filt_procdetach,
207 .f_event = filt_proc,
208 .f_copy = knote_triv_copy,
209 };
210 static const struct filterops jail_filtops = {
211 .f_isfd = 0,
212 .f_attach = filt_jailattach,
213 .f_detach = filt_jaildetach,
214 .f_event = filt_jail,
215 .f_copy = knote_triv_copy,
216 };
217 static const struct filterops timer_filtops = {
218 .f_isfd = 0,
219 .f_attach = filt_timerattach,
220 .f_detach = filt_timerdetach,
221 .f_event = filt_timer,
222 .f_touch = filt_timertouch,
223 .f_copy = filt_timercopy,
224 };
225 static const struct filterops user_filtops = {
226 .f_attach = filt_userattach,
227 .f_detach = filt_userdetach,
228 .f_event = filt_user,
229 .f_touch = filt_usertouch,
230 .f_copy = knote_triv_copy,
231 };
232
233 static uma_zone_t knote_zone;
234 static unsigned int __exclusive_cache_line kq_ncallouts;
235 static unsigned int kq_calloutmax = 4 * 1024;
236 SYSCTL_UINT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW,
237 &kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue");
238
239 /* XXX - ensure not influx ? */
240 #define KNOTE_ACTIVATE(kn, islock) do { \
241 if ((islock)) \
242 mtx_assert(&(kn)->kn_kq->kq_lock, MA_OWNED); \
243 else \
244 KQ_LOCK((kn)->kn_kq); \
245 (kn)->kn_status |= KN_ACTIVE; \
246 if (((kn)->kn_status & (KN_QUEUED | KN_DISABLED)) == 0) \
247 knote_enqueue((kn)); \
248 if (!(islock)) \
249 KQ_UNLOCK((kn)->kn_kq); \
250 } while (0)
251 #define KQ_LOCK(kq) do { \
252 mtx_lock(&(kq)->kq_lock); \
253 } while (0)
254 #define KQ_FLUX_WAKEUP(kq) do { \
255 if (((kq)->kq_state & KQ_FLUXWAIT) == KQ_FLUXWAIT) { \
256 (kq)->kq_state &= ~KQ_FLUXWAIT; \
257 wakeup((kq)); \
258 } \
259 } while (0)
260 #define KQ_UNLOCK_FLUX(kq) do { \
261 KQ_FLUX_WAKEUP(kq); \
262 mtx_unlock(&(kq)->kq_lock); \
263 } while (0)
264 #define KQ_UNLOCK(kq) do { \
265 mtx_unlock(&(kq)->kq_lock); \
266 } while (0)
267 #define KQ_OWNED(kq) do { \
268 mtx_assert(&(kq)->kq_lock, MA_OWNED); \
269 } while (0)
270 #define KQ_NOTOWNED(kq) do { \
271 mtx_assert(&(kq)->kq_lock, MA_NOTOWNED); \
272 } while (0)
273
274 static struct knlist *
kn_list_lock(struct knote * kn)275 kn_list_lock(struct knote *kn)
276 {
277 struct knlist *knl;
278
279 knl = kn->kn_knlist;
280 if (knl != NULL)
281 knl->kl_lock(knl->kl_lockarg);
282 return (knl);
283 }
284
285 static void
kn_list_unlock(struct knlist * knl)286 kn_list_unlock(struct knlist *knl)
287 {
288 bool do_free;
289
290 if (knl == NULL)
291 return;
292 do_free = knl->kl_autodestroy && knlist_empty(knl);
293 knl->kl_unlock(knl->kl_lockarg);
294 if (do_free) {
295 knlist_destroy(knl);
296 free(knl, M_KQUEUE);
297 }
298 }
299
300 static bool
kn_in_flux(struct knote * kn)301 kn_in_flux(struct knote *kn)
302 {
303
304 return (kn->kn_influx > 0);
305 }
306
307 static void
kn_enter_flux(struct knote * kn)308 kn_enter_flux(struct knote *kn)
309 {
310
311 KQ_OWNED(kn->kn_kq);
312 MPASS(kn->kn_influx < INT_MAX);
313 kn->kn_influx++;
314 }
315
316 static bool
kn_leave_flux(struct knote * kn)317 kn_leave_flux(struct knote *kn)
318 {
319
320 KQ_OWNED(kn->kn_kq);
321 MPASS(kn->kn_influx > 0);
322 kn->kn_influx--;
323 return (kn->kn_influx == 0);
324 }
325
326 #define KNL_ASSERT_LOCK(knl, islocked) do { \
327 if (islocked) \
328 KNL_ASSERT_LOCKED(knl); \
329 else \
330 KNL_ASSERT_UNLOCKED(knl); \
331 } while (0)
332 #ifdef INVARIANTS
333 #define KNL_ASSERT_LOCKED(knl) do { \
334 knl->kl_assert_lock((knl)->kl_lockarg, LA_LOCKED); \
335 } while (0)
336 #define KNL_ASSERT_UNLOCKED(knl) do { \
337 knl->kl_assert_lock((knl)->kl_lockarg, LA_UNLOCKED); \
338 } while (0)
339 #else /* !INVARIANTS */
340 #define KNL_ASSERT_LOCKED(knl) do {} while (0)
341 #define KNL_ASSERT_UNLOCKED(knl) do {} while (0)
342 #endif /* INVARIANTS */
343
344 #ifndef KN_HASHSIZE
345 #define KN_HASHSIZE 64 /* XXX should be tunable */
346 #endif
347
348 #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask))
349
350 static int
filt_nullattach(struct knote * kn)351 filt_nullattach(struct knote *kn)
352 {
353
354 return (ENXIO);
355 };
356
357 static const struct filterops null_filtops = {
358 .f_isfd = 0,
359 .f_attach = filt_nullattach,
360 .f_copy = knote_triv_copy,
361 };
362
363 /* XXX - make SYSINIT to add these, and move into respective modules. */
364 extern const struct filterops sig_filtops;
365 extern const struct filterops fs_filtops;
366
367 /*
368 * Table for all system-defined filters.
369 */
370 static struct mtx filterops_lock;
371 MTX_SYSINIT(kqueue_filterops, &filterops_lock, "protect sysfilt_ops", MTX_DEF);
372 static struct {
373 const struct filterops *for_fop;
374 int for_nolock;
375 int for_refcnt;
376 } sysfilt_ops[EVFILT_SYSCOUNT] = {
377 [~EVFILT_READ] = { &file_filtops, 1 },
378 [~EVFILT_WRITE] = { &file_filtops, 1 },
379 [~EVFILT_AIO] = { &null_filtops },
380 [~EVFILT_VNODE] = { &file_filtops, 1 },
381 [~EVFILT_PROC] = { &proc_filtops, 1 },
382 [~EVFILT_SIGNAL] = { &sig_filtops, 1 },
383 [~EVFILT_TIMER] = { &timer_filtops, 1 },
384 [~EVFILT_PROCDESC] = { &file_filtops, 1 },
385 [~EVFILT_FS] = { &fs_filtops, 1 },
386 [~EVFILT_LIO] = { &null_filtops },
387 [~EVFILT_USER] = { &user_filtops, 1 },
388 [~EVFILT_SENDFILE] = { &null_filtops },
389 [~EVFILT_EMPTY] = { &file_filtops, 1 },
390 [~EVFILT_JAIL] = { &jail_filtops, 1 },
391 [~EVFILT_JAILDESC] = { &file_filtops, 1 },
392 };
393
394 /*
395 * Simple redirection for all cdevsw style objects to call their fo_kqfilter
396 * method.
397 */
398 static int
filt_fileattach(struct knote * kn)399 filt_fileattach(struct knote *kn)
400 {
401
402 return (fo_kqfilter(kn->kn_fp, kn));
403 }
404
405 /*ARGSUSED*/
406 static int
kqueue_kqfilter(struct file * fp,struct knote * kn)407 kqueue_kqfilter(struct file *fp, struct knote *kn)
408 {
409 struct kqueue *kq = kn->kn_fp->f_data;
410
411 if (kn->kn_filter != EVFILT_READ)
412 return (EINVAL);
413
414 kn->kn_status |= KN_KQUEUE;
415 kn->kn_fop = &kqread_filtops;
416 knlist_add(&kq->kq_sel.si_note, kn, 0);
417
418 return (0);
419 }
420
421 static void
filt_kqdetach(struct knote * kn)422 filt_kqdetach(struct knote *kn)
423 {
424 struct kqueue *kq = kn->kn_fp->f_data;
425
426 knlist_remove(&kq->kq_sel.si_note, kn, 0);
427 }
428
429 /*ARGSUSED*/
430 static int
filt_kqueue(struct knote * kn,long hint)431 filt_kqueue(struct knote *kn, long hint)
432 {
433 struct kqueue *kq = kn->kn_fp->f_data;
434
435 kn->kn_data = kq->kq_count;
436 return (kn->kn_data > 0);
437 }
438
439 /* XXX - move to kern_proc.c? */
440 static int
filt_procattach(struct knote * kn)441 filt_procattach(struct knote *kn)
442 {
443 struct proc *p;
444 int error;
445 bool exiting, immediate;
446
447 exiting = immediate = false;
448 if (kn->kn_sfflags & NOTE_EXIT)
449 p = pfind_any(kn->kn_id);
450 else
451 p = pfind(kn->kn_id);
452 if (p == NULL)
453 return (ESRCH);
454 if (p->p_flag & P_WEXIT)
455 exiting = true;
456
457 if ((error = p_cansee(curthread, p))) {
458 PROC_UNLOCK(p);
459 return (error);
460 }
461
462 kn->kn_ptr.p_proc = p;
463 kn->kn_flags |= EV_CLEAR; /* automatically set */
464
465 /*
466 * Internal flag indicating registration done by kernel for the
467 * purposes of getting a NOTE_CHILD notification.
468 */
469 if (kn->kn_flags & EV_FLAG2) {
470 kn->kn_flags &= ~EV_FLAG2;
471 kn->kn_data = kn->kn_sdata; /* ppid */
472 kn->kn_fflags = NOTE_CHILD;
473 kn->kn_sfflags &= ~(NOTE_EXIT | NOTE_EXEC | NOTE_FORK);
474 immediate = true; /* Force immediate activation of child note. */
475 }
476 /*
477 * Internal flag indicating registration done by kernel (for other than
478 * NOTE_CHILD).
479 */
480 if (kn->kn_flags & EV_FLAG1) {
481 kn->kn_flags &= ~EV_FLAG1;
482 }
483
484 knlist_add(p->p_klist, kn, 1);
485
486 /*
487 * Immediately activate any child notes or, in the case of a zombie
488 * target process, exit notes. The latter is necessary to handle the
489 * case where the target process, e.g. a child, dies before the kevent
490 * is registered.
491 */
492 if (immediate || (exiting && filt_proc(kn, NOTE_EXIT)))
493 KNOTE_ACTIVATE(kn, 0);
494
495 PROC_UNLOCK(p);
496
497 return (0);
498 }
499
500 /*
501 * The knote may be attached to a different process, which may exit,
502 * leaving nothing for the knote to be attached to. So when the process
503 * exits, the knote is marked as DETACHED and also flagged as ONESHOT so
504 * it will be deleted when read out. However, as part of the knote deletion,
505 * this routine is called, so a check is needed to avoid actually performing
506 * a detach, because the original process does not exist any more.
507 */
508 /* XXX - move to kern_proc.c? */
509 static void
filt_procdetach(struct knote * kn)510 filt_procdetach(struct knote *kn)
511 {
512
513 knlist_remove(kn->kn_knlist, kn, 0);
514 kn->kn_ptr.p_proc = NULL;
515 }
516
517 /* XXX - move to kern_proc.c? */
518 static int
filt_proc(struct knote * kn,long hint)519 filt_proc(struct knote *kn, long hint)
520 {
521 struct proc *p;
522 u_int event;
523
524 p = kn->kn_ptr.p_proc;
525 if (p == NULL) /* already activated, from attach filter */
526 return (0);
527
528 /* Mask off extra data. */
529 event = (u_int)hint & NOTE_PCTRLMASK;
530
531 /* If the user is interested in this event, record it. */
532 if (kn->kn_sfflags & event)
533 kn->kn_fflags |= event;
534
535 /* Process is gone, so flag the event as finished. */
536 if (event == NOTE_EXIT) {
537 kn->kn_flags |= EV_EOF | EV_ONESHOT;
538 kn->kn_ptr.p_proc = NULL;
539 if (kn->kn_fflags & NOTE_EXIT)
540 kn->kn_data = KW_EXITCODE(p->p_xexit, p->p_xsig);
541 if (kn->kn_fflags == 0)
542 kn->kn_flags |= EV_DROP;
543 return (1);
544 }
545
546 return (kn->kn_fflags != 0);
547 }
548
549 /*
550 * Called when the process forked. It mostly does the same as the
551 * knote(), activating all knotes registered to be activated when the
552 * process forked. Additionally, for each knote attached to the
553 * parent, check whether user wants to track the new process. If so
554 * attach a new knote to it, and immediately report an event with the
555 * child's pid.
556 */
557 void
knote_fork(struct knlist * list,int pid)558 knote_fork(struct knlist *list, int pid)
559 {
560 struct kqueue *kq;
561 struct knote *kn;
562 struct kevent kev;
563 int error;
564
565 MPASS(list != NULL);
566 KNL_ASSERT_LOCKED(list);
567 if (SLIST_EMPTY(&list->kl_list))
568 return;
569
570 memset(&kev, 0, sizeof(kev));
571 SLIST_FOREACH(kn, &list->kl_list, kn_selnext) {
572 kq = kn->kn_kq;
573 KQ_LOCK(kq);
574 if (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0) {
575 KQ_UNLOCK(kq);
576 continue;
577 }
578
579 /*
580 * The same as knote(), activate the event.
581 */
582 if ((kn->kn_sfflags & NOTE_TRACK) == 0) {
583 if (kn->kn_fop->f_event(kn, NOTE_FORK))
584 KNOTE_ACTIVATE(kn, 1);
585 KQ_UNLOCK(kq);
586 continue;
587 }
588
589 /*
590 * The NOTE_TRACK case. In addition to the activation
591 * of the event, we need to register new events to
592 * track the child. Drop the locks in preparation for
593 * the call to kqueue_register().
594 */
595 kn_enter_flux(kn);
596 KQ_UNLOCK(kq);
597 list->kl_unlock(list->kl_lockarg);
598
599 /*
600 * Activate existing knote and register tracking knotes with
601 * new process.
602 *
603 * First register a knote to get just the child notice. This
604 * must be a separate note from a potential NOTE_EXIT
605 * notification since both NOTE_CHILD and NOTE_EXIT are defined
606 * to use the data field (in conflicting ways).
607 */
608 kev.ident = pid;
609 kev.filter = kn->kn_filter;
610 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_ONESHOT |
611 EV_FLAG2;
612 kev.fflags = kn->kn_sfflags;
613 kev.data = kn->kn_id; /* parent */
614 kev.udata = kn->kn_kevent.udata;/* preserve udata */
615 error = kqueue_register(kq, &kev, NULL, M_NOWAIT);
616 if (error)
617 kn->kn_fflags |= NOTE_TRACKERR;
618
619 /*
620 * Then register another knote to track other potential events
621 * from the new process.
622 */
623 kev.ident = pid;
624 kev.filter = kn->kn_filter;
625 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1;
626 kev.fflags = kn->kn_sfflags;
627 kev.data = kn->kn_id; /* parent */
628 kev.udata = kn->kn_kevent.udata;/* preserve udata */
629 error = kqueue_register(kq, &kev, NULL, M_NOWAIT);
630
631 /*
632 * Serialize updates to the kn_kevent fields with threads
633 * scanning the queue.
634 */
635 list->kl_lock(list->kl_lockarg);
636 if (error)
637 kn->kn_fflags |= NOTE_TRACKERR;
638 if (kn->kn_fop->f_event(kn, NOTE_FORK)) {
639 KQ_LOCK(kq);
640 KNOTE_ACTIVATE(kn, 1);
641 } else {
642 KQ_LOCK(kq);
643 }
644 kn_leave_flux(kn);
645 KQ_UNLOCK_FLUX(kq);
646 }
647 }
648
649 int
filt_jailattach(struct knote * kn)650 filt_jailattach(struct knote *kn)
651 {
652 struct prison *pr;
653
654 if (kn->kn_id == 0) {
655 /* Let jid=0 watch the current prison (including prison0). */
656 pr = curthread->td_ucred->cr_prison;
657 mtx_lock(&pr->pr_mtx);
658 } else {
659 sx_slock(&allprison_lock);
660 pr = prison_find_child(curthread->td_ucred->cr_prison,
661 kn->kn_id);
662 sx_sunlock(&allprison_lock);
663 if (pr == NULL)
664 return (ENOENT);
665 if (!prison_isalive(pr)) {
666 mtx_unlock(&pr->pr_mtx);
667 return (ENOENT);
668 }
669 }
670 kn->kn_ptr.p_prison = pr;
671 kn->kn_flags |= EV_CLEAR;
672 knlist_add(pr->pr_klist, kn, 1);
673 mtx_unlock(&pr->pr_mtx);
674 return (0);
675 }
676
677 void
filt_jaildetach(struct knote * kn)678 filt_jaildetach(struct knote *kn)
679 {
680 if (kn->kn_ptr.p_prison != NULL) {
681 knlist_remove(kn->kn_knlist, kn, 0);
682 kn->kn_ptr.p_prison = NULL;
683 } else
684 kn->kn_status |= KN_DETACHED;
685 }
686
687 int
filt_jail(struct knote * kn,long hint)688 filt_jail(struct knote *kn, long hint)
689 {
690 struct prison *pr;
691 u_int event;
692
693 pr = kn->kn_ptr.p_prison;
694 if (pr == NULL) /* already activated, from attach filter */
695 return (0);
696
697 /*
698 * Mask off extra data. In the NOTE_JAIL_CHILD case, that's
699 * everything except the NOTE_JAIL_CHILD bit itself, since a
700 * JID is any positive integer.
701 */
702 event = ((u_int)hint & NOTE_JAIL_CHILD) ? NOTE_JAIL_CHILD :
703 (u_int)hint & NOTE_JAIL_CTRLMASK;
704
705 /* If the user is interested in this event, record it. */
706 if (kn->kn_sfflags & event) {
707 kn->kn_fflags |= event;
708 /* Report the created jail id or attached process id. */
709 if (event == NOTE_JAIL_CHILD || event == NOTE_JAIL_ATTACH) {
710 if (kn->kn_data != 0)
711 kn->kn_fflags |= NOTE_JAIL_MULTI;
712 kn->kn_data = (kn->kn_fflags & NOTE_JAIL_MULTI) ? 0U :
713 (u_int)hint & ~event;
714 }
715 }
716
717 /* Prison is gone, so flag the event as finished. */
718 if (event == NOTE_JAIL_REMOVE) {
719 kn->kn_flags |= EV_EOF | EV_ONESHOT;
720 kn->kn_ptr.p_prison = NULL;
721 if (kn->kn_fflags == 0)
722 kn->kn_flags |= EV_DROP;
723 return (1);
724 }
725
726 return (kn->kn_fflags != 0);
727 }
728
729 /*
730 * XXX: EVFILT_TIMER should perhaps live in kern_time.c beside the
731 * interval timer support code.
732 */
733
734 #define NOTE_TIMER_PRECMASK \
735 (NOTE_SECONDS | NOTE_MSECONDS | NOTE_USECONDS | NOTE_NSECONDS)
736
737 static sbintime_t
timer2sbintime(int64_t data,int flags)738 timer2sbintime(int64_t data, int flags)
739 {
740 int64_t secs;
741
742 /*
743 * Macros for converting to the fractional second portion of an
744 * sbintime_t using 64bit multiplication to improve precision.
745 */
746 #define NS_TO_SBT(ns) (((ns) * (((uint64_t)1 << 63) / 500000000)) >> 32)
747 #define US_TO_SBT(us) (((us) * (((uint64_t)1 << 63) / 500000)) >> 32)
748 #define MS_TO_SBT(ms) (((ms) * (((uint64_t)1 << 63) / 500)) >> 32)
749 switch (flags & NOTE_TIMER_PRECMASK) {
750 case NOTE_SECONDS:
751 #ifdef __LP64__
752 if (data > (SBT_MAX / SBT_1S))
753 return (SBT_MAX);
754 #endif
755 return ((sbintime_t)data << 32);
756 case NOTE_MSECONDS: /* FALLTHROUGH */
757 case 0:
758 if (data >= 1000) {
759 secs = data / 1000;
760 #ifdef __LP64__
761 if (secs > (SBT_MAX / SBT_1S))
762 return (SBT_MAX);
763 #endif
764 return (secs << 32 | MS_TO_SBT(data % 1000));
765 }
766 return (MS_TO_SBT(data));
767 case NOTE_USECONDS:
768 if (data >= 1000000) {
769 secs = data / 1000000;
770 #ifdef __LP64__
771 if (secs > (SBT_MAX / SBT_1S))
772 return (SBT_MAX);
773 #endif
774 return (secs << 32 | US_TO_SBT(data % 1000000));
775 }
776 return (US_TO_SBT(data));
777 case NOTE_NSECONDS:
778 if (data >= 1000000000) {
779 secs = data / 1000000000;
780 #ifdef __LP64__
781 if (secs > (SBT_MAX / SBT_1S))
782 return (SBT_MAX);
783 #endif
784 return (secs << 32 | NS_TO_SBT(data % 1000000000));
785 }
786 return (NS_TO_SBT(data));
787 default:
788 break;
789 }
790 return (-1);
791 }
792
793 struct kq_timer_cb_data {
794 struct callout c;
795 struct proc *p;
796 struct knote *kn;
797 int cpuid;
798 int flags;
799 TAILQ_ENTRY(kq_timer_cb_data) link;
800 sbintime_t next; /* next timer event fires at */
801 sbintime_t to; /* precalculated timer period, 0 for abs */
802 };
803
804 #define KQ_TIMER_CB_ENQUEUED 0x01
805
806 static void
kqtimer_sched_callout(struct kq_timer_cb_data * kc)807 kqtimer_sched_callout(struct kq_timer_cb_data *kc)
808 {
809 callout_reset_sbt_on(&kc->c, kc->next, 0, filt_timerexpire, kc->kn,
810 kc->cpuid, C_ABSOLUTE);
811 }
812
813 void
kqtimer_proc_continue(struct proc * p)814 kqtimer_proc_continue(struct proc *p)
815 {
816 struct kq_timer_cb_data *kc, *kc1;
817 struct bintime bt;
818 sbintime_t now;
819
820 PROC_LOCK_ASSERT(p, MA_OWNED);
821
822 getboottimebin(&bt);
823 now = bttosbt(bt);
824
825 TAILQ_FOREACH_SAFE(kc, &p->p_kqtim_stop, link, kc1) {
826 TAILQ_REMOVE(&p->p_kqtim_stop, kc, link);
827 kc->flags &= ~KQ_TIMER_CB_ENQUEUED;
828 if (kc->next <= now)
829 filt_timerexpire_l(kc->kn, true);
830 else
831 kqtimer_sched_callout(kc);
832 }
833 }
834
835 static void
filt_timerexpire_l(struct knote * kn,bool proc_locked)836 filt_timerexpire_l(struct knote *kn, bool proc_locked)
837 {
838 struct kq_timer_cb_data *kc;
839 struct proc *p;
840 uint64_t delta;
841 sbintime_t now;
842
843 kc = kn->kn_ptr.p_v;
844
845 if ((kn->kn_flags & EV_ONESHOT) != 0 || kc->to == 0) {
846 kn->kn_data++;
847 KNOTE_ACTIVATE(kn, 0);
848 return;
849 }
850
851 now = sbinuptime();
852 if (now >= kc->next) {
853 delta = (now - kc->next) / kc->to;
854 if (delta == 0)
855 delta = 1;
856 kn->kn_data += delta;
857 kc->next += delta * kc->to;
858 if (now >= kc->next) /* overflow */
859 kc->next = now + kc->to;
860 KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */
861 }
862
863 /*
864 * Initial check for stopped kc->p is racy. It is fine to
865 * miss the set of the stop flags, at worst we would schedule
866 * one more callout. On the other hand, it is not fine to not
867 * schedule when we we missed clearing of the flags, we
868 * recheck them under the lock and observe consistent state.
869 */
870 p = kc->p;
871 if (P_SHOULDSTOP(p) || P_KILLED(p)) {
872 if (!proc_locked)
873 PROC_LOCK(p);
874 if (P_SHOULDSTOP(p) || P_KILLED(p)) {
875 if ((kc->flags & KQ_TIMER_CB_ENQUEUED) == 0) {
876 kc->flags |= KQ_TIMER_CB_ENQUEUED;
877 TAILQ_INSERT_TAIL(&p->p_kqtim_stop, kc, link);
878 }
879 if (!proc_locked)
880 PROC_UNLOCK(p);
881 return;
882 }
883 if (!proc_locked)
884 PROC_UNLOCK(p);
885 }
886 kqtimer_sched_callout(kc);
887 }
888
889 static void
filt_timerexpire(void * knx)890 filt_timerexpire(void *knx)
891 {
892 filt_timerexpire_l(knx, false);
893 }
894
895 /*
896 * data contains amount of time to sleep
897 */
898 static int
filt_timervalidate(struct knote * kn,sbintime_t * to)899 filt_timervalidate(struct knote *kn, sbintime_t *to)
900 {
901 struct bintime bt;
902 sbintime_t sbt;
903
904 if (kn->kn_sdata < 0)
905 return (EINVAL);
906 if (kn->kn_sdata == 0 && (kn->kn_flags & EV_ONESHOT) == 0)
907 kn->kn_sdata = 1;
908 /*
909 * The only fflags values supported are the timer unit
910 * (precision) and the absolute time indicator.
911 */
912 if ((kn->kn_sfflags & ~(NOTE_TIMER_PRECMASK | NOTE_ABSTIME)) != 0)
913 return (EINVAL);
914
915 *to = timer2sbintime(kn->kn_sdata, kn->kn_sfflags);
916 if (*to < 0)
917 return (EINVAL);
918 if ((kn->kn_sfflags & NOTE_ABSTIME) != 0) {
919 getboottimebin(&bt);
920 sbt = bttosbt(bt);
921 *to = MAX(0, *to - sbt);
922 }
923 return (0);
924 }
925
926 static int
filt_timerattach(struct knote * kn)927 filt_timerattach(struct knote *kn)
928 {
929 struct kq_timer_cb_data *kc;
930 sbintime_t to;
931 int error;
932
933 to = -1;
934 error = filt_timervalidate(kn, &to);
935 if (error != 0)
936 return (error);
937 KASSERT(to > 0 || (kn->kn_flags & EV_ONESHOT) != 0 ||
938 (kn->kn_sfflags & NOTE_ABSTIME) != 0,
939 ("%s: periodic timer has a calculated zero timeout", __func__));
940 KASSERT(to >= 0,
941 ("%s: timer has a calculated negative timeout", __func__));
942
943 if (atomic_fetchadd_int(&kq_ncallouts, 1) + 1 > kq_calloutmax) {
944 atomic_subtract_int(&kq_ncallouts, 1);
945 return (ENOMEM);
946 }
947
948 if ((kn->kn_sfflags & NOTE_ABSTIME) == 0)
949 kn->kn_flags |= EV_CLEAR; /* automatically set */
950 kn->kn_status &= ~KN_DETACHED; /* knlist_add clears it */
951 kn->kn_ptr.p_v = kc = malloc(sizeof(*kc), M_KQUEUE, M_WAITOK);
952 kc->kn = kn;
953 kc->p = curproc;
954 kc->cpuid = PCPU_GET(cpuid);
955 kc->flags = 0;
956 callout_init(&kc->c, 1);
957 filt_timerstart(kn, to);
958
959 return (0);
960 }
961
962 static int
filt_timercopy(struct knote * kn,struct proc * p)963 filt_timercopy(struct knote *kn, struct proc *p)
964 {
965 struct kq_timer_cb_data *kc_src, *kc;
966
967 if (atomic_fetchadd_int(&kq_ncallouts, 1) + 1 > kq_calloutmax) {
968 atomic_subtract_int(&kq_ncallouts, 1);
969 return (ENOMEM);
970 }
971
972 kn->kn_status &= ~KN_DETACHED;
973 kc_src = kn->kn_ptr.p_v;
974 kn->kn_ptr.p_v = kc = malloc(sizeof(*kc), M_KQUEUE, M_WAITOK);
975 kc->kn = kn;
976 kc->p = p;
977 kc->flags = kc_src->flags & ~KQ_TIMER_CB_ENQUEUED;
978 kc->next = kc_src->next;
979 kc->to = kc_src->to;
980 kc->cpuid = PCPU_GET(cpuid);
981 callout_init(&kc->c, 1);
982 kqtimer_sched_callout(kc);
983 return (0);
984 }
985
986 static void
filt_timerstart(struct knote * kn,sbintime_t to)987 filt_timerstart(struct knote *kn, sbintime_t to)
988 {
989 struct kq_timer_cb_data *kc;
990
991 kc = kn->kn_ptr.p_v;
992 if ((kn->kn_sfflags & NOTE_ABSTIME) != 0) {
993 kc->next = to;
994 kc->to = 0;
995 } else {
996 kc->next = to + sbinuptime();
997 kc->to = to;
998 }
999 kqtimer_sched_callout(kc);
1000 }
1001
1002 static void
filt_timerdetach(struct knote * kn)1003 filt_timerdetach(struct knote *kn)
1004 {
1005 struct kq_timer_cb_data *kc;
1006 unsigned int old __unused;
1007 bool pending;
1008
1009 kc = kn->kn_ptr.p_v;
1010 do {
1011 callout_drain(&kc->c);
1012
1013 /*
1014 * kqtimer_proc_continue() might have rescheduled this callout.
1015 * Double-check, using the process mutex as an interlock.
1016 */
1017 PROC_LOCK(kc->p);
1018 if ((kc->flags & KQ_TIMER_CB_ENQUEUED) != 0) {
1019 kc->flags &= ~KQ_TIMER_CB_ENQUEUED;
1020 TAILQ_REMOVE(&kc->p->p_kqtim_stop, kc, link);
1021 }
1022 pending = callout_pending(&kc->c);
1023 PROC_UNLOCK(kc->p);
1024 } while (pending);
1025 free(kc, M_KQUEUE);
1026 old = atomic_fetchadd_int(&kq_ncallouts, -1);
1027 KASSERT(old > 0, ("Number of callouts cannot become negative"));
1028 kn->kn_status |= KN_DETACHED; /* knlist_remove sets it */
1029 }
1030
1031 static void
filt_timertouch(struct knote * kn,struct kevent * kev,u_long type)1032 filt_timertouch(struct knote *kn, struct kevent *kev, u_long type)
1033 {
1034 struct kq_timer_cb_data *kc;
1035 struct kqueue *kq;
1036 sbintime_t to;
1037 int error;
1038
1039 switch (type) {
1040 case EVENT_REGISTER:
1041 /* Handle re-added timers that update data/fflags */
1042 if (kev->flags & EV_ADD) {
1043 kc = kn->kn_ptr.p_v;
1044
1045 /* Drain any existing callout. */
1046 callout_drain(&kc->c);
1047
1048 /* Throw away any existing undelivered record
1049 * of the timer expiration. This is done under
1050 * the presumption that if a process is
1051 * re-adding this timer with new parameters,
1052 * it is no longer interested in what may have
1053 * happened under the old parameters. If it is
1054 * interested, it can wait for the expiration,
1055 * delete the old timer definition, and then
1056 * add the new one.
1057 *
1058 * This has to be done while the kq is locked:
1059 * - if enqueued, dequeue
1060 * - make it no longer active
1061 * - clear the count of expiration events
1062 */
1063 kq = kn->kn_kq;
1064 KQ_LOCK(kq);
1065 if (kn->kn_status & KN_QUEUED)
1066 knote_dequeue(kn);
1067
1068 kn->kn_status &= ~KN_ACTIVE;
1069 kn->kn_data = 0;
1070 KQ_UNLOCK(kq);
1071
1072 /* Reschedule timer based on new data/fflags */
1073 kn->kn_sfflags = kev->fflags;
1074 kn->kn_sdata = kev->data;
1075 error = filt_timervalidate(kn, &to);
1076 if (error != 0) {
1077 kn->kn_flags |= EV_ERROR;
1078 kn->kn_data = error;
1079 } else
1080 filt_timerstart(kn, to);
1081 }
1082 break;
1083
1084 case EVENT_PROCESS:
1085 *kev = kn->kn_kevent;
1086 if (kn->kn_flags & EV_CLEAR) {
1087 kn->kn_data = 0;
1088 kn->kn_fflags = 0;
1089 }
1090 break;
1091
1092 default:
1093 panic("filt_timertouch() - invalid type (%ld)", type);
1094 break;
1095 }
1096 }
1097
1098 static int
filt_timer(struct knote * kn,long hint)1099 filt_timer(struct knote *kn, long hint)
1100 {
1101
1102 return (kn->kn_data != 0);
1103 }
1104
1105 static int
filt_userattach(struct knote * kn)1106 filt_userattach(struct knote *kn)
1107 {
1108
1109 /*
1110 * EVFILT_USER knotes are not attached to anything in the kernel.
1111 */
1112 kn->kn_hook = NULL;
1113 if (kn->kn_fflags & NOTE_TRIGGER)
1114 kn->kn_hookid = 1;
1115 else
1116 kn->kn_hookid = 0;
1117 return (0);
1118 }
1119
1120 static void
filt_userdetach(__unused struct knote * kn)1121 filt_userdetach(__unused struct knote *kn)
1122 {
1123
1124 /*
1125 * EVFILT_USER knotes are not attached to anything in the kernel.
1126 */
1127 }
1128
1129 static int
filt_user(struct knote * kn,__unused long hint)1130 filt_user(struct knote *kn, __unused long hint)
1131 {
1132
1133 return (kn->kn_hookid);
1134 }
1135
1136 static void
filt_usertouch(struct knote * kn,struct kevent * kev,u_long type)1137 filt_usertouch(struct knote *kn, struct kevent *kev, u_long type)
1138 {
1139 u_int ffctrl;
1140
1141 switch (type) {
1142 case EVENT_REGISTER:
1143 if (kev->fflags & NOTE_TRIGGER)
1144 kn->kn_hookid = 1;
1145
1146 ffctrl = kev->fflags & NOTE_FFCTRLMASK;
1147 kev->fflags &= NOTE_FFLAGSMASK;
1148 switch (ffctrl) {
1149 case NOTE_FFNOP:
1150 break;
1151
1152 case NOTE_FFAND:
1153 kn->kn_sfflags &= kev->fflags;
1154 break;
1155
1156 case NOTE_FFOR:
1157 kn->kn_sfflags |= kev->fflags;
1158 break;
1159
1160 case NOTE_FFCOPY:
1161 kn->kn_sfflags = kev->fflags;
1162 break;
1163
1164 default:
1165 /* XXX Return error? */
1166 break;
1167 }
1168 kn->kn_sdata = kev->data;
1169 if (kev->flags & EV_CLEAR) {
1170 kn->kn_hookid = 0;
1171 kn->kn_data = 0;
1172 kn->kn_fflags = 0;
1173 }
1174 break;
1175
1176 case EVENT_PROCESS:
1177 *kev = kn->kn_kevent;
1178 kev->fflags = kn->kn_sfflags;
1179 kev->data = kn->kn_sdata;
1180 if (kn->kn_flags & EV_CLEAR) {
1181 kn->kn_hookid = 0;
1182 kn->kn_data = 0;
1183 kn->kn_fflags = 0;
1184 }
1185 break;
1186
1187 default:
1188 panic("filt_usertouch() - invalid type (%ld)", type);
1189 break;
1190 }
1191 }
1192
1193 int
sys_kqueue(struct thread * td,struct kqueue_args * uap)1194 sys_kqueue(struct thread *td, struct kqueue_args *uap)
1195 {
1196
1197 return (kern_kqueue(td, 0, false, NULL));
1198 }
1199
1200 int
sys_kqueuex(struct thread * td,struct kqueuex_args * uap)1201 sys_kqueuex(struct thread *td, struct kqueuex_args *uap)
1202 {
1203 int flags;
1204
1205 if ((uap->flags & ~(KQUEUE_CLOEXEC | KQUEUE_CPONFORK)) != 0)
1206 return (EINVAL);
1207 flags = 0;
1208 if ((uap->flags & KQUEUE_CLOEXEC) != 0)
1209 flags |= O_CLOEXEC;
1210 return (kern_kqueue(td, flags, (uap->flags & KQUEUE_CPONFORK) != 0,
1211 NULL));
1212 }
1213
1214 static void
kqueue_init(struct kqueue * kq,bool cponfork)1215 kqueue_init(struct kqueue *kq, bool cponfork)
1216 {
1217
1218 mtx_init(&kq->kq_lock, "kqueue", NULL, MTX_DEF | MTX_DUPOK);
1219 TAILQ_INIT(&kq->kq_head);
1220 knlist_init_mtx(&kq->kq_sel.si_note, &kq->kq_lock);
1221 TASK_INIT(&kq->kq_task, 0, kqueue_task, kq);
1222 if (cponfork)
1223 kq->kq_state |= KQ_CPONFORK;
1224 }
1225
1226 static int
kern_kqueue_alloc(struct thread * td,struct filedesc * fdp,int * fdip,struct file ** fpp,int flags,struct filecaps * fcaps,bool cponfork,struct kqueue ** kqp)1227 kern_kqueue_alloc(struct thread *td, struct filedesc *fdp, int *fdip,
1228 struct file **fpp, int flags, struct filecaps *fcaps, bool cponfork,
1229 struct kqueue **kqp)
1230 {
1231 struct ucred *cred;
1232 struct kqueue *kq;
1233 int error;
1234
1235 cred = td->td_ucred;
1236 if (!chgkqcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_KQUEUES)))
1237 return (ENOMEM);
1238
1239 error = fdip != NULL ? falloc_caps(td, fpp, fdip, flags, fcaps) :
1240 _falloc_noinstall(td, fpp, 1);
1241 if (error != 0) {
1242 chgkqcnt(cred->cr_ruidinfo, -1, 0);
1243 return (error);
1244 }
1245
1246 /* An extra reference on `fp' has been held for us by falloc(). */
1247 kq = malloc(sizeof(*kq), M_KQUEUE, M_WAITOK | M_ZERO);
1248 kqueue_init(kq, cponfork);
1249 kq->kq_fdp = fdp;
1250 kq->kq_cred = crhold(cred);
1251
1252 if (fdip != NULL)
1253 FILEDESC_XLOCK(fdp);
1254 TAILQ_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list);
1255 if (fdip != NULL)
1256 FILEDESC_XUNLOCK(fdp);
1257
1258 finit(*fpp, FREAD | FWRITE, DTYPE_KQUEUE, kq, &kqueueops);
1259 *kqp = kq;
1260 return (0);
1261 }
1262
1263 int
kern_kqueue(struct thread * td,int flags,bool cponfork,struct filecaps * fcaps)1264 kern_kqueue(struct thread *td, int flags, bool cponfork, struct filecaps *fcaps)
1265 {
1266 struct kqueue *kq;
1267 struct file *fp;
1268 int fd, error;
1269
1270 error = kern_kqueue_alloc(td, td->td_proc->p_fd, &fd, &fp, flags,
1271 fcaps, cponfork, &kq);
1272 if (error != 0)
1273 return (error);
1274
1275 fdrop(fp, td);
1276
1277 td->td_retval[0] = fd;
1278 return (0);
1279 }
1280
1281 struct g_kevent_args {
1282 int fd;
1283 const void *changelist;
1284 int nchanges;
1285 void *eventlist;
1286 int nevents;
1287 const struct timespec *timeout;
1288 };
1289
1290 int
sys_kevent(struct thread * td,struct kevent_args * uap)1291 sys_kevent(struct thread *td, struct kevent_args *uap)
1292 {
1293 struct kevent_copyops k_ops = {
1294 .arg = uap,
1295 .k_copyout = kevent_copyout,
1296 .k_copyin = kevent_copyin,
1297 .kevent_size = sizeof(struct kevent),
1298 };
1299 struct g_kevent_args gk_args = {
1300 .fd = uap->fd,
1301 .changelist = uap->changelist,
1302 .nchanges = uap->nchanges,
1303 .eventlist = uap->eventlist,
1304 .nevents = uap->nevents,
1305 .timeout = uap->timeout,
1306 };
1307
1308 return (kern_kevent_generic(td, &gk_args, &k_ops, "kevent"));
1309 }
1310
1311 static int
kern_kevent_generic(struct thread * td,struct g_kevent_args * uap,struct kevent_copyops * k_ops,const char * struct_name)1312 kern_kevent_generic(struct thread *td, struct g_kevent_args *uap,
1313 struct kevent_copyops *k_ops, const char *struct_name)
1314 {
1315 struct timespec ts, *tsp;
1316 #ifdef KTRACE
1317 struct kevent *eventlist = uap->eventlist;
1318 #endif
1319 int error;
1320
1321 if (uap->timeout != NULL) {
1322 error = copyin(uap->timeout, &ts, sizeof(ts));
1323 if (error)
1324 return (error);
1325 tsp = &ts;
1326 } else
1327 tsp = NULL;
1328
1329 #ifdef KTRACE
1330 if (KTRPOINT(td, KTR_STRUCT_ARRAY))
1331 ktrstructarray(struct_name, UIO_USERSPACE, uap->changelist,
1332 uap->nchanges, k_ops->kevent_size);
1333 #endif
1334
1335 error = kern_kevent(td, uap->fd, uap->nchanges, uap->nevents,
1336 k_ops, tsp);
1337
1338 #ifdef KTRACE
1339 if (error == 0 && KTRPOINT(td, KTR_STRUCT_ARRAY))
1340 ktrstructarray(struct_name, UIO_USERSPACE, eventlist,
1341 td->td_retval[0], k_ops->kevent_size);
1342 #endif
1343
1344 return (error);
1345 }
1346
1347 /*
1348 * Copy 'count' items into the destination list pointed to by uap->eventlist.
1349 */
1350 static int
kevent_copyout(void * arg,struct kevent * kevp,int count)1351 kevent_copyout(void *arg, struct kevent *kevp, int count)
1352 {
1353 struct kevent_args *uap;
1354 int error;
1355
1356 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1357 uap = (struct kevent_args *)arg;
1358
1359 error = copyout(kevp, uap->eventlist, count * sizeof *kevp);
1360 if (error == 0)
1361 uap->eventlist += count;
1362 return (error);
1363 }
1364
1365 /*
1366 * Copy 'count' items from the list pointed to by uap->changelist.
1367 */
1368 static int
kevent_copyin(void * arg,struct kevent * kevp,int count)1369 kevent_copyin(void *arg, struct kevent *kevp, int count)
1370 {
1371 struct kevent_args *uap;
1372 int error;
1373
1374 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1375 uap = (struct kevent_args *)arg;
1376
1377 error = copyin(uap->changelist, kevp, count * sizeof *kevp);
1378 if (error == 0)
1379 uap->changelist += count;
1380 return (error);
1381 }
1382
1383 #ifdef COMPAT_FREEBSD11
1384 static int
kevent11_copyout(void * arg,struct kevent * kevp,int count)1385 kevent11_copyout(void *arg, struct kevent *kevp, int count)
1386 {
1387 struct freebsd11_kevent_args *uap;
1388 struct freebsd11_kevent kev11;
1389 int error, i;
1390
1391 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1392 uap = (struct freebsd11_kevent_args *)arg;
1393
1394 for (i = 0; i < count; i++) {
1395 kev11.ident = kevp->ident;
1396 kev11.filter = kevp->filter;
1397 kev11.flags = kevp->flags;
1398 kev11.fflags = kevp->fflags;
1399 kev11.data = kevp->data;
1400 kev11.udata = kevp->udata;
1401 error = copyout(&kev11, uap->eventlist, sizeof(kev11));
1402 if (error != 0)
1403 break;
1404 uap->eventlist++;
1405 kevp++;
1406 }
1407 return (error);
1408 }
1409
1410 /*
1411 * Copy 'count' items from the list pointed to by uap->changelist.
1412 */
1413 static int
kevent11_copyin(void * arg,struct kevent * kevp,int count)1414 kevent11_copyin(void *arg, struct kevent *kevp, int count)
1415 {
1416 struct freebsd11_kevent_args *uap;
1417 struct freebsd11_kevent kev11;
1418 int error, i;
1419
1420 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count));
1421 uap = (struct freebsd11_kevent_args *)arg;
1422
1423 for (i = 0; i < count; i++) {
1424 error = copyin(uap->changelist, &kev11, sizeof(kev11));
1425 if (error != 0)
1426 break;
1427 kevp->ident = kev11.ident;
1428 kevp->filter = kev11.filter;
1429 kevp->flags = kev11.flags;
1430 kevp->fflags = kev11.fflags;
1431 kevp->data = (uintptr_t)kev11.data;
1432 kevp->udata = kev11.udata;
1433 bzero(&kevp->ext, sizeof(kevp->ext));
1434 uap->changelist++;
1435 kevp++;
1436 }
1437 return (error);
1438 }
1439
1440 int
freebsd11_kevent(struct thread * td,struct freebsd11_kevent_args * uap)1441 freebsd11_kevent(struct thread *td, struct freebsd11_kevent_args *uap)
1442 {
1443 struct kevent_copyops k_ops = {
1444 .arg = uap,
1445 .k_copyout = kevent11_copyout,
1446 .k_copyin = kevent11_copyin,
1447 .kevent_size = sizeof(struct freebsd11_kevent),
1448 };
1449 struct g_kevent_args gk_args = {
1450 .fd = uap->fd,
1451 .changelist = uap->changelist,
1452 .nchanges = uap->nchanges,
1453 .eventlist = uap->eventlist,
1454 .nevents = uap->nevents,
1455 .timeout = uap->timeout,
1456 };
1457
1458 return (kern_kevent_generic(td, &gk_args, &k_ops, "freebsd11_kevent"));
1459 }
1460 #endif
1461
1462 int
kern_kevent(struct thread * td,int fd,int nchanges,int nevents,struct kevent_copyops * k_ops,const struct timespec * timeout)1463 kern_kevent(struct thread *td, int fd, int nchanges, int nevents,
1464 struct kevent_copyops *k_ops, const struct timespec *timeout)
1465 {
1466 cap_rights_t rights;
1467 struct file *fp;
1468 int error;
1469
1470 cap_rights_init_zero(&rights);
1471 if (nchanges > 0)
1472 cap_rights_set_one(&rights, CAP_KQUEUE_CHANGE);
1473 if (nevents > 0)
1474 cap_rights_set_one(&rights, CAP_KQUEUE_EVENT);
1475 error = fget(td, fd, &rights, &fp);
1476 if (error != 0)
1477 return (error);
1478
1479 error = kern_kevent_fp(td, fp, nchanges, nevents, k_ops, timeout);
1480 fdrop(fp, td);
1481
1482 return (error);
1483 }
1484
1485 static int
kqueue_kevent(struct kqueue * kq,struct thread * td,int nchanges,int nevents,struct kevent_copyops * k_ops,const struct timespec * timeout)1486 kqueue_kevent(struct kqueue *kq, struct thread *td, int nchanges, int nevents,
1487 struct kevent_copyops *k_ops, const struct timespec *timeout)
1488 {
1489 struct kevent keva[KQ_NEVENTS];
1490 struct kevent *kevp, *changes;
1491 int i, n, nerrors, error;
1492
1493 if (nchanges < 0)
1494 return (EINVAL);
1495
1496 nerrors = 0;
1497 while (nchanges > 0) {
1498 n = nchanges > KQ_NEVENTS ? KQ_NEVENTS : nchanges;
1499 error = k_ops->k_copyin(k_ops->arg, keva, n);
1500 if (error)
1501 return (error);
1502 changes = keva;
1503 for (i = 0; i < n; i++) {
1504 kevp = &changes[i];
1505 if (!kevp->filter)
1506 continue;
1507 kevp->flags &= ~EV_SYSFLAGS;
1508 error = kqueue_register(kq, kevp, td, M_WAITOK);
1509 if (error || (kevp->flags & EV_RECEIPT)) {
1510 if (nevents == 0)
1511 return (error);
1512 kevp->flags = EV_ERROR;
1513 kevp->data = error;
1514 (void)k_ops->k_copyout(k_ops->arg, kevp, 1);
1515 nevents--;
1516 nerrors++;
1517 }
1518 }
1519 nchanges -= n;
1520 }
1521 if (nerrors) {
1522 td->td_retval[0] = nerrors;
1523 return (0);
1524 }
1525
1526 return (kqueue_scan(kq, nevents, k_ops, timeout, keva, td));
1527 }
1528
1529 int
kern_kevent_fp(struct thread * td,struct file * fp,int nchanges,int nevents,struct kevent_copyops * k_ops,const struct timespec * timeout)1530 kern_kevent_fp(struct thread *td, struct file *fp, int nchanges, int nevents,
1531 struct kevent_copyops *k_ops, const struct timespec *timeout)
1532 {
1533 struct kqueue *kq;
1534 int error;
1535
1536 error = kqueue_acquire(fp, &kq);
1537 if (error != 0)
1538 return (error);
1539 error = kqueue_kevent(kq, td, nchanges, nevents, k_ops, timeout);
1540 kqueue_release(kq, 0);
1541 return (error);
1542 }
1543
1544 /*
1545 * Performs a kevent() call on a temporarily created kqueue. This can be
1546 * used to perform one-shot polling, similar to poll() and select().
1547 */
1548 int
kern_kevent_anonymous(struct thread * td,int nevents,struct kevent_copyops * k_ops)1549 kern_kevent_anonymous(struct thread *td, int nevents,
1550 struct kevent_copyops *k_ops)
1551 {
1552 struct kqueue kq = {};
1553 int error;
1554
1555 kqueue_init(&kq, false);
1556 kq.kq_refcnt = 1;
1557 error = kqueue_kevent(&kq, td, nevents, nevents, k_ops, NULL);
1558 kqueue_drain(&kq, td);
1559 kqueue_destroy(&kq);
1560 return (error);
1561 }
1562
1563 int
kqueue_add_filteropts(int filt,const struct filterops * filtops)1564 kqueue_add_filteropts(int filt, const struct filterops *filtops)
1565 {
1566 int error;
1567
1568 error = 0;
1569 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) {
1570 printf(
1571 "trying to add a filterop that is out of range: %d is beyond %d\n",
1572 ~filt, EVFILT_SYSCOUNT);
1573 return EINVAL;
1574 }
1575 mtx_lock(&filterops_lock);
1576 if (sysfilt_ops[~filt].for_fop != &null_filtops &&
1577 sysfilt_ops[~filt].for_fop != NULL)
1578 error = EEXIST;
1579 else {
1580 sysfilt_ops[~filt].for_fop = filtops;
1581 sysfilt_ops[~filt].for_refcnt = 0;
1582 }
1583 mtx_unlock(&filterops_lock);
1584
1585 return (error);
1586 }
1587
1588 int
kqueue_del_filteropts(int filt)1589 kqueue_del_filteropts(int filt)
1590 {
1591 int error;
1592
1593 error = 0;
1594 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
1595 return EINVAL;
1596
1597 mtx_lock(&filterops_lock);
1598 if (sysfilt_ops[~filt].for_fop == &null_filtops ||
1599 sysfilt_ops[~filt].for_fop == NULL)
1600 error = EINVAL;
1601 else if (sysfilt_ops[~filt].for_refcnt != 0)
1602 error = EBUSY;
1603 else {
1604 sysfilt_ops[~filt].for_fop = &null_filtops;
1605 sysfilt_ops[~filt].for_refcnt = 0;
1606 }
1607 mtx_unlock(&filterops_lock);
1608
1609 return error;
1610 }
1611
1612 static const struct filterops *
kqueue_fo_find(int filt)1613 kqueue_fo_find(int filt)
1614 {
1615
1616 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
1617 return NULL;
1618
1619 if (sysfilt_ops[~filt].for_nolock)
1620 return sysfilt_ops[~filt].for_fop;
1621
1622 mtx_lock(&filterops_lock);
1623 sysfilt_ops[~filt].for_refcnt++;
1624 if (sysfilt_ops[~filt].for_fop == NULL)
1625 sysfilt_ops[~filt].for_fop = &null_filtops;
1626 mtx_unlock(&filterops_lock);
1627
1628 return sysfilt_ops[~filt].for_fop;
1629 }
1630
1631 static void
kqueue_fo_release(int filt)1632 kqueue_fo_release(int filt)
1633 {
1634
1635 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0)
1636 return;
1637
1638 if (sysfilt_ops[~filt].for_nolock)
1639 return;
1640
1641 mtx_lock(&filterops_lock);
1642 KASSERT(sysfilt_ops[~filt].for_refcnt > 0,
1643 ("filter object %d refcount not valid on release", filt));
1644 sysfilt_ops[~filt].for_refcnt--;
1645 mtx_unlock(&filterops_lock);
1646 }
1647
1648 /*
1649 * A ref to kq (obtained via kqueue_acquire) must be held.
1650 */
1651 static int
kqueue_register(struct kqueue * kq,struct kevent * kev,struct thread * td,int mflag)1652 kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td,
1653 int mflag)
1654 {
1655 const struct filterops *fops;
1656 struct file *fp;
1657 struct knote *kn, *tkn;
1658 struct knlist *knl;
1659 int error, filt, event;
1660 int haskqglobal, filedesc_unlock;
1661
1662 if ((kev->flags & (EV_ENABLE | EV_DISABLE)) == (EV_ENABLE | EV_DISABLE))
1663 return (EINVAL);
1664
1665 fp = NULL;
1666 kn = NULL;
1667 knl = NULL;
1668 error = 0;
1669 haskqglobal = 0;
1670 filedesc_unlock = 0;
1671
1672 filt = kev->filter;
1673 fops = kqueue_fo_find(filt);
1674 if (fops == NULL)
1675 return EINVAL;
1676
1677 if (kev->flags & EV_ADD) {
1678 /* Reject an invalid flag pair early */
1679 if (kev->flags & EV_KEEPUDATA) {
1680 tkn = NULL;
1681 error = EINVAL;
1682 goto done;
1683 }
1684
1685 /*
1686 * Prevent waiting with locks. Non-sleepable
1687 * allocation failures are handled in the loop, only
1688 * if the spare knote appears to be actually required.
1689 */
1690 tkn = knote_alloc(mflag);
1691 } else {
1692 tkn = NULL;
1693 }
1694
1695 findkn:
1696 if (fops->f_isfd) {
1697 KASSERT(td != NULL, ("td is NULL"));
1698 if (kev->ident > INT_MAX)
1699 error = EBADF;
1700 else
1701 error = fget(td, kev->ident, &cap_event_rights, &fp);
1702 if (error)
1703 goto done;
1704
1705 if ((kev->flags & EV_ADD) == EV_ADD && kqueue_expand(kq, fops,
1706 kev->ident, M_NOWAIT) != 0) {
1707 /* try again */
1708 fdrop(fp, td);
1709 fp = NULL;
1710 error = kqueue_expand(kq, fops, kev->ident, mflag);
1711 if (error)
1712 goto done;
1713 goto findkn;
1714 }
1715
1716 if (fp->f_type == DTYPE_KQUEUE) {
1717 /*
1718 * If we add some intelligence about what we are doing,
1719 * we should be able to support events on ourselves.
1720 * We need to know when we are doing this to prevent
1721 * getting both the knlist lock and the kq lock since
1722 * they are the same thing.
1723 */
1724 if (fp->f_data == kq) {
1725 error = EINVAL;
1726 goto done;
1727 }
1728
1729 /*
1730 * Pre-lock the filedesc before the global
1731 * lock mutex, see the comment in
1732 * kqueue_close().
1733 */
1734 FILEDESC_XLOCK(td->td_proc->p_fd);
1735 filedesc_unlock = 1;
1736 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
1737 }
1738
1739 KQ_LOCK(kq);
1740 if (kev->ident < kq->kq_knlistsize) {
1741 SLIST_FOREACH(kn, &kq->kq_knlist[kev->ident], kn_link)
1742 if (kev->filter == kn->kn_filter)
1743 break;
1744 }
1745 } else {
1746 if ((kev->flags & EV_ADD) == EV_ADD) {
1747 error = kqueue_expand(kq, fops, kev->ident, mflag);
1748 if (error != 0)
1749 goto done;
1750 }
1751
1752 KQ_LOCK(kq);
1753
1754 /*
1755 * If possible, find an existing knote to use for this kevent.
1756 */
1757 if (kev->filter == EVFILT_PROC &&
1758 (kev->flags & (EV_FLAG1 | EV_FLAG2)) != 0) {
1759 /* This is an internal creation of a process tracking
1760 * note. Don't attempt to coalesce this with an
1761 * existing note.
1762 */
1763 ;
1764 } else if (kq->kq_knhashmask != 0) {
1765 struct klist *list;
1766
1767 list = &kq->kq_knhash[
1768 KN_HASH((u_long)kev->ident, kq->kq_knhashmask)];
1769 SLIST_FOREACH(kn, list, kn_link)
1770 if (kev->ident == kn->kn_id &&
1771 kev->filter == kn->kn_filter)
1772 break;
1773 }
1774 }
1775
1776 /* knote is in the process of changing, wait for it to stabilize. */
1777 if (kn != NULL && kn_in_flux(kn)) {
1778 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1779 if (filedesc_unlock) {
1780 FILEDESC_XUNLOCK(td->td_proc->p_fd);
1781 filedesc_unlock = 0;
1782 }
1783 kq->kq_state |= KQ_FLUXWAIT;
1784 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqflxwt", 0);
1785 if (fp != NULL) {
1786 fdrop(fp, td);
1787 fp = NULL;
1788 }
1789 goto findkn;
1790 }
1791
1792 /*
1793 * kn now contains the matching knote, or NULL if no match
1794 */
1795 if (kn == NULL) {
1796 if (kev->flags & EV_ADD) {
1797 kn = tkn;
1798 tkn = NULL;
1799 if (kn == NULL) {
1800 KQ_UNLOCK(kq);
1801 error = ENOMEM;
1802 goto done;
1803 }
1804 kn->kn_fp = fp;
1805 kn->kn_kq = kq;
1806 kn->kn_fop = fops;
1807 /*
1808 * apply reference counts to knote structure, and
1809 * do not release it at the end of this routine.
1810 */
1811 fops = NULL;
1812 fp = NULL;
1813
1814 kn->kn_sfflags = kev->fflags;
1815 kn->kn_sdata = kev->data;
1816 kev->fflags = 0;
1817 kev->data = 0;
1818 kn->kn_kevent = *kev;
1819 kn->kn_kevent.flags &= ~(EV_ADD | EV_DELETE |
1820 EV_ENABLE | EV_DISABLE | EV_FORCEONESHOT);
1821 kn->kn_status = KN_DETACHED;
1822 if ((kev->flags & EV_DISABLE) != 0)
1823 kn->kn_status |= KN_DISABLED;
1824 kn_enter_flux(kn);
1825
1826 error = knote_attach(kn, kq);
1827 KQ_UNLOCK(kq);
1828 if (error != 0) {
1829 tkn = kn;
1830 goto done;
1831 }
1832
1833 if ((error = kn->kn_fop->f_attach(kn)) != 0) {
1834 knote_drop_detached(kn, td);
1835 goto done;
1836 }
1837 knl = kn_list_lock(kn);
1838 goto done_ev_add;
1839 } else {
1840 /* No matching knote and the EV_ADD flag is not set. */
1841 KQ_UNLOCK(kq);
1842 error = ENOENT;
1843 goto done;
1844 }
1845 }
1846
1847 if (kev->flags & EV_DELETE) {
1848 kn_enter_flux(kn);
1849 KQ_UNLOCK(kq);
1850 knote_drop(kn, td);
1851 goto done;
1852 }
1853
1854 if (kev->flags & EV_FORCEONESHOT) {
1855 kn->kn_flags |= EV_ONESHOT;
1856 KNOTE_ACTIVATE(kn, 1);
1857 }
1858
1859 if ((kev->flags & EV_ENABLE) != 0)
1860 kn->kn_status &= ~KN_DISABLED;
1861 else if ((kev->flags & EV_DISABLE) != 0)
1862 kn->kn_status |= KN_DISABLED;
1863
1864 /*
1865 * The user may change some filter values after the initial EV_ADD,
1866 * but doing so will not reset any filter which has already been
1867 * triggered.
1868 */
1869 kn->kn_status |= KN_SCAN;
1870 kn_enter_flux(kn);
1871 KQ_UNLOCK(kq);
1872 knl = kn_list_lock(kn);
1873 if ((kev->flags & EV_KEEPUDATA) == 0)
1874 kn->kn_kevent.udata = kev->udata;
1875 if (!fops->f_isfd && fops->f_touch != NULL) {
1876 fops->f_touch(kn, kev, EVENT_REGISTER);
1877 } else {
1878 kn->kn_sfflags = kev->fflags;
1879 kn->kn_sdata = kev->data;
1880 }
1881
1882 done_ev_add:
1883 /*
1884 * We can get here with kn->kn_knlist == NULL. This can happen when
1885 * the initial attach event decides that the event is "completed"
1886 * already, e.g., filt_procattach() is called on a zombie process. It
1887 * will call filt_proc() which will remove it from the list, and NULL
1888 * kn_knlist.
1889 *
1890 * KN_DISABLED will be stable while the knote is in flux, so the
1891 * unlocked read will not race with an update.
1892 */
1893 if ((kn->kn_status & KN_DISABLED) == 0)
1894 event = kn->kn_fop->f_event(kn, 0);
1895 else
1896 event = 0;
1897
1898 KQ_LOCK(kq);
1899 if (event)
1900 kn->kn_status |= KN_ACTIVE;
1901 if ((kn->kn_status & (KN_ACTIVE | KN_DISABLED | KN_QUEUED)) ==
1902 KN_ACTIVE)
1903 knote_enqueue(kn);
1904 kn->kn_status &= ~KN_SCAN;
1905 kn_leave_flux(kn);
1906 kn_list_unlock(knl);
1907 KQ_UNLOCK_FLUX(kq);
1908
1909 done:
1910 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
1911 if (filedesc_unlock)
1912 FILEDESC_XUNLOCK(td->td_proc->p_fd);
1913 if (fp != NULL)
1914 fdrop(fp, td);
1915 knote_free(tkn);
1916 if (fops != NULL)
1917 kqueue_fo_release(filt);
1918 return (error);
1919 }
1920
1921 static int
kqueue_acquire_ref(struct kqueue * kq)1922 kqueue_acquire_ref(struct kqueue *kq)
1923 {
1924 KQ_LOCK(kq);
1925 if ((kq->kq_state & KQ_CLOSING) == KQ_CLOSING) {
1926 KQ_UNLOCK(kq);
1927 return (EBADF);
1928 }
1929 kq->kq_refcnt++;
1930 KQ_UNLOCK(kq);
1931 return (0);
1932 }
1933
1934 static int
kqueue_acquire(struct file * fp,struct kqueue ** kqp)1935 kqueue_acquire(struct file *fp, struct kqueue **kqp)
1936 {
1937 struct kqueue *kq;
1938 int error;
1939
1940 kq = fp->f_data;
1941 if (fp->f_type != DTYPE_KQUEUE || kq == NULL)
1942 return (EINVAL);
1943 error = kqueue_acquire_ref(kq);
1944 if (error == 0)
1945 *kqp = kq;
1946 return (error);
1947 }
1948
1949 static void
kqueue_release(struct kqueue * kq,int locked)1950 kqueue_release(struct kqueue *kq, int locked)
1951 {
1952 if (locked)
1953 KQ_OWNED(kq);
1954 else
1955 KQ_LOCK(kq);
1956 kq->kq_refcnt--;
1957 if (kq->kq_refcnt == 1)
1958 wakeup(&kq->kq_refcnt);
1959 if (!locked)
1960 KQ_UNLOCK(kq);
1961 }
1962
1963 static void
ast_kqueue(struct thread * td,int tda __unused)1964 ast_kqueue(struct thread *td, int tda __unused)
1965 {
1966 taskqueue_quiesce(taskqueue_kqueue_ctx);
1967 }
1968
1969 static void
kqueue_schedtask(struct kqueue * kq)1970 kqueue_schedtask(struct kqueue *kq)
1971 {
1972 KQ_OWNED(kq);
1973 KASSERT(((kq->kq_state & KQ_TASKDRAIN) != KQ_TASKDRAIN),
1974 ("scheduling kqueue task while draining"));
1975
1976 if ((kq->kq_state & KQ_TASKSCHED) != KQ_TASKSCHED) {
1977 taskqueue_enqueue(taskqueue_kqueue_ctx, &kq->kq_task);
1978 kq->kq_state |= KQ_TASKSCHED;
1979 ast_sched(curthread, TDA_KQUEUE);
1980 }
1981 }
1982
1983 /*
1984 * Expand the kq to make sure we have storage for fops/ident pair.
1985 *
1986 * Return 0 on success (or no work necessary), return errno on failure.
1987 */
1988 static int
kqueue_expand(struct kqueue * kq,const struct filterops * fops,uintptr_t ident,int mflag)1989 kqueue_expand(struct kqueue *kq, const struct filterops *fops, uintptr_t ident,
1990 int mflag)
1991 {
1992 struct klist *list, *tmp_knhash, *to_free;
1993 u_long tmp_knhashmask;
1994 int error, fd, size;
1995
1996 KQ_NOTOWNED(kq);
1997
1998 error = 0;
1999 to_free = NULL;
2000 if (fops->f_isfd) {
2001 fd = ident;
2002 if (kq->kq_knlistsize <= fd) {
2003 size = kq->kq_knlistsize;
2004 while (size <= fd)
2005 size += KQEXTENT;
2006 list = malloc(size * sizeof(*list), M_KQUEUE, mflag);
2007 if (list == NULL)
2008 return ENOMEM;
2009 KQ_LOCK(kq);
2010 if ((kq->kq_state & KQ_CLOSING) != 0) {
2011 to_free = list;
2012 error = EBADF;
2013 } else if (kq->kq_knlistsize > fd) {
2014 to_free = list;
2015 } else {
2016 if (kq->kq_knlist != NULL) {
2017 bcopy(kq->kq_knlist, list,
2018 kq->kq_knlistsize * sizeof(*list));
2019 to_free = kq->kq_knlist;
2020 kq->kq_knlist = NULL;
2021 }
2022 bzero((caddr_t)list +
2023 kq->kq_knlistsize * sizeof(*list),
2024 (size - kq->kq_knlistsize) * sizeof(*list));
2025 kq->kq_knlistsize = size;
2026 kq->kq_knlist = list;
2027 }
2028 KQ_UNLOCK(kq);
2029 }
2030 } else {
2031 if (kq->kq_knhashmask == 0) {
2032 tmp_knhash = hashinit_flags(KN_HASHSIZE, M_KQUEUE,
2033 &tmp_knhashmask, (mflag & M_WAITOK) != 0 ?
2034 HASH_WAITOK : HASH_NOWAIT);
2035 if (tmp_knhash == NULL)
2036 return (ENOMEM);
2037 KQ_LOCK(kq);
2038 if ((kq->kq_state & KQ_CLOSING) != 0) {
2039 to_free = tmp_knhash;
2040 error = EBADF;
2041 } else if (kq->kq_knhashmask == 0) {
2042 kq->kq_knhash = tmp_knhash;
2043 kq->kq_knhashmask = tmp_knhashmask;
2044 } else {
2045 to_free = tmp_knhash;
2046 }
2047 KQ_UNLOCK(kq);
2048 }
2049 }
2050 free(to_free, M_KQUEUE);
2051
2052 KQ_NOTOWNED(kq);
2053 return (error);
2054 }
2055
2056 static void
kqueue_task(void * arg,int pending)2057 kqueue_task(void *arg, int pending)
2058 {
2059 struct kqueue *kq;
2060 int haskqglobal;
2061
2062 haskqglobal = 0;
2063 kq = arg;
2064
2065 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
2066 KQ_LOCK(kq);
2067
2068 KNOTE_LOCKED(&kq->kq_sel.si_note, 0);
2069
2070 kq->kq_state &= ~KQ_TASKSCHED;
2071 if ((kq->kq_state & KQ_TASKDRAIN) == KQ_TASKDRAIN) {
2072 wakeup(&kq->kq_state);
2073 }
2074 KQ_UNLOCK(kq);
2075 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
2076 }
2077
2078 /*
2079 * Scan, update kn_data (if not ONESHOT), and copyout triggered events.
2080 * We treat KN_MARKER knotes as if they are in flux.
2081 */
2082 static int
kqueue_scan(struct kqueue * kq,int maxevents,struct kevent_copyops * k_ops,const struct timespec * tsp,struct kevent * keva,struct thread * td)2083 kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops,
2084 const struct timespec *tsp, struct kevent *keva, struct thread *td)
2085 {
2086 struct kevent *kevp;
2087 struct knote *kn, *marker;
2088 struct knlist *knl;
2089 sbintime_t asbt, rsbt;
2090 int count, error, haskqglobal, influx, nkev, touch;
2091
2092 count = maxevents;
2093 nkev = 0;
2094 error = 0;
2095 haskqglobal = 0;
2096
2097 if (maxevents == 0)
2098 goto done_nl;
2099 if (maxevents < 0) {
2100 error = EINVAL;
2101 goto done_nl;
2102 }
2103
2104 rsbt = 0;
2105 if (tsp != NULL) {
2106 if (!timespecvalid_interval(tsp)) {
2107 error = EINVAL;
2108 goto done_nl;
2109 }
2110 if (timespecisset(tsp)) {
2111 if (tsp->tv_sec <= INT32_MAX) {
2112 rsbt = tstosbt(*tsp);
2113 if (TIMESEL(&asbt, rsbt))
2114 asbt += tc_tick_sbt;
2115 if (asbt <= SBT_MAX - rsbt)
2116 asbt += rsbt;
2117 else
2118 asbt = 0;
2119 rsbt >>= tc_precexp;
2120 } else
2121 asbt = 0;
2122 } else
2123 asbt = -1;
2124 } else
2125 asbt = 0;
2126 marker = knote_alloc(M_WAITOK);
2127 marker->kn_status = KN_MARKER;
2128 KQ_LOCK(kq);
2129
2130 retry:
2131 kevp = keva;
2132 if (kq->kq_count == 0) {
2133 if (asbt == -1) {
2134 error = EWOULDBLOCK;
2135 } else {
2136 kq->kq_state |= KQ_SLEEP;
2137 error = msleep_sbt(kq, &kq->kq_lock, PSOCK | PCATCH,
2138 "kqread", asbt, rsbt, C_ABSOLUTE);
2139 }
2140 if (error == 0)
2141 goto retry;
2142 /* don't restart after signals... */
2143 if (error == ERESTART)
2144 error = EINTR;
2145 else if (error == EWOULDBLOCK)
2146 error = 0;
2147 goto done;
2148 }
2149
2150 TAILQ_INSERT_TAIL(&kq->kq_head, marker, kn_tqe);
2151 influx = 0;
2152 while (count) {
2153 KQ_OWNED(kq);
2154 kn = TAILQ_FIRST(&kq->kq_head);
2155
2156 if ((kn->kn_status == KN_MARKER && kn != marker) ||
2157 kn_in_flux(kn)) {
2158 if (influx) {
2159 influx = 0;
2160 KQ_FLUX_WAKEUP(kq);
2161 }
2162 kq->kq_state |= KQ_FLUXWAIT;
2163 error = msleep(kq, &kq->kq_lock, PSOCK,
2164 "kqflxwt", 0);
2165 continue;
2166 }
2167
2168 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
2169 if ((kn->kn_status & KN_DISABLED) == KN_DISABLED) {
2170 kn->kn_status &= ~KN_QUEUED;
2171 kq->kq_count--;
2172 continue;
2173 }
2174 if (kn == marker) {
2175 KQ_FLUX_WAKEUP(kq);
2176 if (count == maxevents)
2177 goto retry;
2178 goto done;
2179 }
2180 KASSERT(!kn_in_flux(kn),
2181 ("knote %p is unexpectedly in flux", kn));
2182
2183 if ((kn->kn_flags & EV_DROP) == EV_DROP) {
2184 kn->kn_status &= ~KN_QUEUED;
2185 kn_enter_flux(kn);
2186 kq->kq_count--;
2187 KQ_UNLOCK(kq);
2188 /*
2189 * We don't need to lock the list since we've
2190 * marked it as in flux.
2191 */
2192 knote_drop(kn, td);
2193 KQ_LOCK(kq);
2194 continue;
2195 } else if ((kn->kn_flags & EV_ONESHOT) == EV_ONESHOT) {
2196 kn->kn_status &= ~KN_QUEUED;
2197 kn_enter_flux(kn);
2198 kq->kq_count--;
2199 KQ_UNLOCK(kq);
2200 /*
2201 * We don't need to lock the list since we've
2202 * marked the knote as being in flux.
2203 */
2204 *kevp = kn->kn_kevent;
2205 knote_drop(kn, td);
2206 KQ_LOCK(kq);
2207 kn = NULL;
2208 } else {
2209 kn->kn_status |= KN_SCAN;
2210 kn_enter_flux(kn);
2211 KQ_UNLOCK(kq);
2212 if ((kn->kn_status & KN_KQUEUE) == KN_KQUEUE)
2213 KQ_GLOBAL_LOCK(&kq_global, haskqglobal);
2214 knl = kn_list_lock(kn);
2215 if (kn->kn_fop->f_event(kn, 0) == 0) {
2216 KQ_LOCK(kq);
2217 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
2218 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE |
2219 KN_SCAN);
2220 kn_leave_flux(kn);
2221 kq->kq_count--;
2222 kn_list_unlock(knl);
2223 influx = 1;
2224 continue;
2225 }
2226 touch = (!kn->kn_fop->f_isfd &&
2227 kn->kn_fop->f_touch != NULL);
2228 if (touch)
2229 kn->kn_fop->f_touch(kn, kevp, EVENT_PROCESS);
2230 else
2231 *kevp = kn->kn_kevent;
2232 KQ_LOCK(kq);
2233 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal);
2234 if (kn->kn_flags & (EV_CLEAR | EV_DISPATCH)) {
2235 /*
2236 * Manually clear knotes who weren't
2237 * 'touch'ed.
2238 */
2239 if (touch == 0 && kn->kn_flags & EV_CLEAR) {
2240 kn->kn_data = 0;
2241 kn->kn_fflags = 0;
2242 }
2243 if (kn->kn_flags & EV_DISPATCH)
2244 kn->kn_status |= KN_DISABLED;
2245 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE);
2246 kq->kq_count--;
2247 } else
2248 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
2249
2250 kn->kn_status &= ~KN_SCAN;
2251 kn_leave_flux(kn);
2252 kn_list_unlock(knl);
2253 influx = 1;
2254 }
2255
2256 /* we are returning a copy to the user */
2257 kevp++;
2258 nkev++;
2259 count--;
2260
2261 if (nkev == KQ_NEVENTS) {
2262 influx = 0;
2263 KQ_UNLOCK_FLUX(kq);
2264 error = k_ops->k_copyout(k_ops->arg, keva, nkev);
2265 nkev = 0;
2266 kevp = keva;
2267 KQ_LOCK(kq);
2268 if (error)
2269 break;
2270 }
2271 }
2272 TAILQ_REMOVE(&kq->kq_head, marker, kn_tqe);
2273 done:
2274 KQ_OWNED(kq);
2275 KQ_UNLOCK_FLUX(kq);
2276 knote_free(marker);
2277 done_nl:
2278 KQ_NOTOWNED(kq);
2279 if (nkev != 0)
2280 error = k_ops->k_copyout(k_ops->arg, keva, nkev);
2281 td->td_retval[0] = maxevents - count;
2282 return (error);
2283 }
2284
2285 /*ARGSUSED*/
2286 static int
kqueue_ioctl(struct file * fp,u_long cmd,void * data,struct ucred * active_cred,struct thread * td)2287 kqueue_ioctl(struct file *fp, u_long cmd, void *data,
2288 struct ucred *active_cred, struct thread *td)
2289 {
2290 /*
2291 * Enabling sigio causes two major problems:
2292 * 1) infinite recursion:
2293 * Synopsys: kevent is being used to track signals and have FIOASYNC
2294 * set. On receipt of a signal this will cause a kqueue to recurse
2295 * into itself over and over. Sending the sigio causes the kqueue
2296 * to become ready, which in turn posts sigio again, forever.
2297 * Solution: this can be solved by setting a flag in the kqueue that
2298 * we have a SIGIO in progress.
2299 * 2) locking problems:
2300 * Synopsys: Kqueue is a leaf subsystem, but adding signalling puts
2301 * us above the proc and pgrp locks.
2302 * Solution: Post a signal using an async mechanism, being sure to
2303 * record a generation count in the delivery so that we do not deliver
2304 * a signal to the wrong process.
2305 *
2306 * Note, these two mechanisms are somewhat mutually exclusive!
2307 */
2308 #if 0
2309 struct kqueue *kq;
2310
2311 kq = fp->f_data;
2312 switch (cmd) {
2313 case FIOASYNC:
2314 if (*(int *)data) {
2315 kq->kq_state |= KQ_ASYNC;
2316 } else {
2317 kq->kq_state &= ~KQ_ASYNC;
2318 }
2319 return (0);
2320
2321 case FIOSETOWN:
2322 return (fsetown(*(int *)data, &kq->kq_sigio));
2323
2324 case FIOGETOWN:
2325 *(int *)data = fgetown(&kq->kq_sigio);
2326 return (0);
2327 }
2328 #endif
2329
2330 return (ENOTTY);
2331 }
2332
2333 /*ARGSUSED*/
2334 static int
kqueue_poll(struct file * fp,int events,struct ucred * active_cred,struct thread * td)2335 kqueue_poll(struct file *fp, int events, struct ucred *active_cred,
2336 struct thread *td)
2337 {
2338 struct kqueue *kq;
2339 int revents = 0;
2340 int error;
2341
2342 if ((error = kqueue_acquire(fp, &kq)))
2343 return POLLERR;
2344
2345 KQ_LOCK(kq);
2346 if (events & (POLLIN | POLLRDNORM)) {
2347 if (kq->kq_count) {
2348 revents |= events & (POLLIN | POLLRDNORM);
2349 } else {
2350 selrecord(td, &kq->kq_sel);
2351 if (SEL_WAITING(&kq->kq_sel))
2352 kq->kq_state |= KQ_SEL;
2353 }
2354 }
2355 kqueue_release(kq, 1);
2356 KQ_UNLOCK(kq);
2357 return (revents);
2358 }
2359
2360 /*ARGSUSED*/
2361 static int
kqueue_stat(struct file * fp,struct stat * st,struct ucred * active_cred)2362 kqueue_stat(struct file *fp, struct stat *st, struct ucred *active_cred)
2363 {
2364
2365 bzero((void *)st, sizeof *st);
2366 /*
2367 * We no longer return kq_count because the unlocked value is useless.
2368 * If you spent all this time getting the count, why not spend your
2369 * syscall better by calling kevent?
2370 *
2371 * XXX - This is needed for libc_r.
2372 */
2373 st->st_mode = S_IFIFO;
2374 return (0);
2375 }
2376
2377 static void
kqueue_drain(struct kqueue * kq,struct thread * td)2378 kqueue_drain(struct kqueue *kq, struct thread *td)
2379 {
2380 struct knote *kn;
2381 int i;
2382
2383 KQ_LOCK(kq);
2384
2385 KASSERT((kq->kq_state & KQ_CLOSING) != KQ_CLOSING,
2386 ("kqueue already closing"));
2387 kq->kq_state |= KQ_CLOSING;
2388 if (kq->kq_refcnt > 1)
2389 msleep(&kq->kq_refcnt, &kq->kq_lock, PSOCK, "kqclose", 0);
2390
2391 KASSERT(kq->kq_refcnt == 1, ("other refs are out there!"));
2392
2393 KASSERT(knlist_empty(&kq->kq_sel.si_note),
2394 ("kqueue's knlist not empty"));
2395
2396 for (i = 0; i < kq->kq_knlistsize; i++) {
2397 while ((kn = SLIST_FIRST(&kq->kq_knlist[i])) != NULL) {
2398 if (kn_in_flux(kn)) {
2399 kq->kq_state |= KQ_FLUXWAIT;
2400 msleep(kq, &kq->kq_lock, PSOCK, "kqclo1", 0);
2401 continue;
2402 }
2403 kn_enter_flux(kn);
2404 KQ_UNLOCK(kq);
2405 knote_drop(kn, td);
2406 KQ_LOCK(kq);
2407 }
2408 }
2409 if (kq->kq_knhashmask != 0) {
2410 for (i = 0; i <= kq->kq_knhashmask; i++) {
2411 while ((kn = SLIST_FIRST(&kq->kq_knhash[i])) != NULL) {
2412 if (kn_in_flux(kn)) {
2413 kq->kq_state |= KQ_FLUXWAIT;
2414 msleep(kq, &kq->kq_lock, PSOCK,
2415 "kqclo2", 0);
2416 continue;
2417 }
2418 kn_enter_flux(kn);
2419 KQ_UNLOCK(kq);
2420 knote_drop(kn, td);
2421 KQ_LOCK(kq);
2422 }
2423 }
2424 }
2425
2426 if ((kq->kq_state & KQ_TASKSCHED) == KQ_TASKSCHED) {
2427 kq->kq_state |= KQ_TASKDRAIN;
2428 msleep(&kq->kq_state, &kq->kq_lock, PSOCK, "kqtqdr", 0);
2429 }
2430
2431 if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
2432 selwakeuppri(&kq->kq_sel, PSOCK);
2433 if (!SEL_WAITING(&kq->kq_sel))
2434 kq->kq_state &= ~KQ_SEL;
2435 }
2436
2437 KQ_UNLOCK(kq);
2438 }
2439
2440 static void
kqueue_destroy(struct kqueue * kq)2441 kqueue_destroy(struct kqueue *kq)
2442 {
2443
2444 KASSERT(kq->kq_fdp == NULL,
2445 ("kqueue still attached to a file descriptor"));
2446 seldrain(&kq->kq_sel);
2447 knlist_destroy(&kq->kq_sel.si_note);
2448 mtx_destroy(&kq->kq_lock);
2449
2450 if (kq->kq_knhash != NULL)
2451 free(kq->kq_knhash, M_KQUEUE);
2452 if (kq->kq_knlist != NULL)
2453 free(kq->kq_knlist, M_KQUEUE);
2454
2455 funsetown(&kq->kq_sigio);
2456 }
2457
2458 /*ARGSUSED*/
2459 static int
kqueue_close(struct file * fp,struct thread * td)2460 kqueue_close(struct file *fp, struct thread *td)
2461 {
2462 struct kqueue *kq = fp->f_data;
2463 struct filedesc *fdp;
2464 int error;
2465 int filedesc_unlock;
2466
2467 if ((error = kqueue_acquire(fp, &kq)))
2468 return error;
2469 kqueue_drain(kq, td);
2470
2471 /*
2472 * We could be called due to the knote_drop() doing fdrop(),
2473 * called from kqueue_register(). In this case the global
2474 * lock is owned, and filedesc sx is locked before, to not
2475 * take the sleepable lock after non-sleepable.
2476 */
2477 fdp = kq->kq_fdp;
2478 kq->kq_fdp = NULL;
2479 if (!sx_xlocked(FILEDESC_LOCK(fdp))) {
2480 FILEDESC_XLOCK(fdp);
2481 filedesc_unlock = 1;
2482 } else
2483 filedesc_unlock = 0;
2484 TAILQ_REMOVE(&fdp->fd_kqlist, kq, kq_list);
2485 if (filedesc_unlock)
2486 FILEDESC_XUNLOCK(fdp);
2487
2488 kqueue_destroy(kq);
2489 chgkqcnt(kq->kq_cred->cr_ruidinfo, -1, 0);
2490 crfree(kq->kq_cred);
2491 free(kq, M_KQUEUE);
2492 fp->f_data = NULL;
2493
2494 return (0);
2495 }
2496
2497 static int
kqueue_fill_kinfo(struct file * fp,struct kinfo_file * kif,struct filedesc * fdp)2498 kqueue_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp)
2499 {
2500 struct kqueue *kq = fp->f_data;
2501
2502 kif->kf_type = KF_TYPE_KQUEUE;
2503 kif->kf_un.kf_kqueue.kf_kqueue_addr = (uintptr_t)kq;
2504 kif->kf_un.kf_kqueue.kf_kqueue_count = kq->kq_count;
2505 kif->kf_un.kf_kqueue.kf_kqueue_state = kq->kq_state;
2506 return (0);
2507 }
2508
2509 static void
kqueue_wakeup(struct kqueue * kq)2510 kqueue_wakeup(struct kqueue *kq)
2511 {
2512 KQ_OWNED(kq);
2513
2514 if ((kq->kq_state & KQ_SLEEP) == KQ_SLEEP) {
2515 kq->kq_state &= ~KQ_SLEEP;
2516 wakeup(kq);
2517 }
2518 if ((kq->kq_state & KQ_SEL) == KQ_SEL) {
2519 selwakeuppri(&kq->kq_sel, PSOCK);
2520 if (!SEL_WAITING(&kq->kq_sel))
2521 kq->kq_state &= ~KQ_SEL;
2522 }
2523 if (!knlist_empty(&kq->kq_sel.si_note))
2524 kqueue_schedtask(kq);
2525 if ((kq->kq_state & KQ_ASYNC) == KQ_ASYNC) {
2526 pgsigio(&kq->kq_sigio, SIGIO, 0);
2527 }
2528 }
2529
2530 /*
2531 * Walk down a list of knotes, activating them if their event has triggered.
2532 *
2533 * There is a possibility to optimize in the case of one kq watching another.
2534 * Instead of scheduling a task to wake it up, you could pass enough state
2535 * down the chain to make up the parent kqueue. Make this code functional
2536 * first.
2537 */
2538 void
knote(struct knlist * list,long hint,int lockflags)2539 knote(struct knlist *list, long hint, int lockflags)
2540 {
2541 struct kqueue *kq;
2542 struct knote *kn, *tkn;
2543 int error;
2544
2545 if (list == NULL)
2546 return;
2547
2548 KNL_ASSERT_LOCK(list, lockflags & KNF_LISTLOCKED);
2549
2550 if ((lockflags & KNF_LISTLOCKED) == 0)
2551 list->kl_lock(list->kl_lockarg);
2552
2553 /*
2554 * If we unlock the list lock (and enter influx), we can
2555 * eliminate the kqueue scheduling, but this will introduce
2556 * four lock/unlock's for each knote to test. Also, marker
2557 * would be needed to keep iteration position, since filters
2558 * or other threads could remove events.
2559 */
2560 SLIST_FOREACH_SAFE(kn, &list->kl_list, kn_selnext, tkn) {
2561 kq = kn->kn_kq;
2562 KQ_LOCK(kq);
2563 if (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0) {
2564 /*
2565 * Do not process the influx notes, except for
2566 * the influx coming from the kq unlock in the
2567 * kqueue_scan(). In the later case, we do
2568 * not interfere with the scan, since the code
2569 * fragment in kqueue_scan() locks the knlist,
2570 * and cannot proceed until we finished.
2571 */
2572 KQ_UNLOCK(kq);
2573 } else if ((lockflags & KNF_NOKQLOCK) != 0) {
2574 kn_enter_flux(kn);
2575 KQ_UNLOCK(kq);
2576 error = kn->kn_fop->f_event(kn, hint);
2577 KQ_LOCK(kq);
2578 kn_leave_flux(kn);
2579 if (error)
2580 KNOTE_ACTIVATE(kn, 1);
2581 KQ_UNLOCK_FLUX(kq);
2582 } else {
2583 if (kn->kn_fop->f_event(kn, hint))
2584 KNOTE_ACTIVATE(kn, 1);
2585 KQ_UNLOCK(kq);
2586 }
2587 }
2588 if ((lockflags & KNF_LISTLOCKED) == 0)
2589 list->kl_unlock(list->kl_lockarg);
2590 }
2591
2592 /*
2593 * add a knote to a knlist
2594 */
2595 void
knlist_add(struct knlist * knl,struct knote * kn,int islocked)2596 knlist_add(struct knlist *knl, struct knote *kn, int islocked)
2597 {
2598
2599 KNL_ASSERT_LOCK(knl, islocked);
2600 KQ_NOTOWNED(kn->kn_kq);
2601 KASSERT(kn_in_flux(kn), ("knote %p not in flux", kn));
2602 KASSERT((kn->kn_status & KN_DETACHED) != 0,
2603 ("knote %p was not detached", kn));
2604 if (!islocked)
2605 knl->kl_lock(knl->kl_lockarg);
2606 SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext);
2607 if (!islocked)
2608 knl->kl_unlock(knl->kl_lockarg);
2609 KQ_LOCK(kn->kn_kq);
2610 kn->kn_knlist = knl;
2611 kn->kn_status &= ~KN_DETACHED;
2612 KQ_UNLOCK(kn->kn_kq);
2613 }
2614
2615 static void
knlist_remove_kq(struct knlist * knl,struct knote * kn,int knlislocked,int kqislocked)2616 knlist_remove_kq(struct knlist *knl, struct knote *kn, int knlislocked,
2617 int kqislocked)
2618 {
2619
2620 KASSERT(!kqislocked || knlislocked, ("kq locked w/o knl locked"));
2621 KNL_ASSERT_LOCK(knl, knlislocked);
2622 mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED);
2623 KASSERT(kqislocked || kn_in_flux(kn), ("knote %p not in flux", kn));
2624 KASSERT((kn->kn_status & KN_DETACHED) == 0,
2625 ("knote %p was already detached", kn));
2626 if (!knlislocked)
2627 knl->kl_lock(knl->kl_lockarg);
2628 SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext);
2629 kn->kn_knlist = NULL;
2630 if (!knlislocked)
2631 kn_list_unlock(knl);
2632 if (!kqislocked)
2633 KQ_LOCK(kn->kn_kq);
2634 kn->kn_status |= KN_DETACHED;
2635 if (!kqislocked)
2636 KQ_UNLOCK(kn->kn_kq);
2637 }
2638
2639 /*
2640 * remove knote from the specified knlist
2641 */
2642 void
knlist_remove(struct knlist * knl,struct knote * kn,int islocked)2643 knlist_remove(struct knlist *knl, struct knote *kn, int islocked)
2644 {
2645
2646 knlist_remove_kq(knl, kn, islocked, 0);
2647 }
2648
2649 int
knlist_empty(struct knlist * knl)2650 knlist_empty(struct knlist *knl)
2651 {
2652
2653 KNL_ASSERT_LOCKED(knl);
2654 return (SLIST_EMPTY(&knl->kl_list));
2655 }
2656
2657 static struct mtx knlist_lock;
2658 MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects",
2659 MTX_DEF);
2660 static void knlist_mtx_lock(void *arg);
2661 static void knlist_mtx_unlock(void *arg);
2662
2663 static void
knlist_mtx_lock(void * arg)2664 knlist_mtx_lock(void *arg)
2665 {
2666
2667 mtx_lock((struct mtx *)arg);
2668 }
2669
2670 static void
knlist_mtx_unlock(void * arg)2671 knlist_mtx_unlock(void *arg)
2672 {
2673
2674 mtx_unlock((struct mtx *)arg);
2675 }
2676
2677 static void
knlist_mtx_assert_lock(void * arg,int what)2678 knlist_mtx_assert_lock(void *arg, int what)
2679 {
2680
2681 if (what == LA_LOCKED)
2682 mtx_assert((struct mtx *)arg, MA_OWNED);
2683 else
2684 mtx_assert((struct mtx *)arg, MA_NOTOWNED);
2685 }
2686
2687 void
knlist_init(struct knlist * knl,void * lock,void (* kl_lock)(void *),void (* kl_unlock)(void *),void (* kl_assert_lock)(void *,int))2688 knlist_init(struct knlist *knl, void *lock, void (*kl_lock)(void *),
2689 void (*kl_unlock)(void *),
2690 void (*kl_assert_lock)(void *, int))
2691 {
2692
2693 if (lock == NULL)
2694 knl->kl_lockarg = &knlist_lock;
2695 else
2696 knl->kl_lockarg = lock;
2697
2698 if (kl_lock == NULL)
2699 knl->kl_lock = knlist_mtx_lock;
2700 else
2701 knl->kl_lock = kl_lock;
2702 if (kl_unlock == NULL)
2703 knl->kl_unlock = knlist_mtx_unlock;
2704 else
2705 knl->kl_unlock = kl_unlock;
2706 if (kl_assert_lock == NULL)
2707 knl->kl_assert_lock = knlist_mtx_assert_lock;
2708 else
2709 knl->kl_assert_lock = kl_assert_lock;
2710
2711 knl->kl_autodestroy = 0;
2712 SLIST_INIT(&knl->kl_list);
2713 }
2714
2715 void
knlist_init_mtx(struct knlist * knl,struct mtx * lock)2716 knlist_init_mtx(struct knlist *knl, struct mtx *lock)
2717 {
2718
2719 knlist_init(knl, lock, NULL, NULL, NULL);
2720 }
2721
2722 struct knlist *
knlist_alloc(struct mtx * lock)2723 knlist_alloc(struct mtx *lock)
2724 {
2725 struct knlist *knl;
2726
2727 knl = malloc(sizeof(struct knlist), M_KQUEUE, M_WAITOK);
2728 knlist_init_mtx(knl, lock);
2729 return (knl);
2730 }
2731
2732 void
knlist_destroy(struct knlist * knl)2733 knlist_destroy(struct knlist *knl)
2734 {
2735
2736 KASSERT(KNLIST_EMPTY(knl),
2737 ("destroying knlist %p with knotes on it", knl));
2738 }
2739
2740 void
knlist_detach(struct knlist * knl)2741 knlist_detach(struct knlist *knl)
2742 {
2743
2744 KNL_ASSERT_LOCKED(knl);
2745 knl->kl_autodestroy = 1;
2746 if (knlist_empty(knl)) {
2747 knlist_destroy(knl);
2748 free(knl, M_KQUEUE);
2749 }
2750 }
2751
2752 /*
2753 * Even if we are locked, we may need to drop the lock to allow any influx
2754 * knotes time to "settle".
2755 */
2756 void
knlist_cleardel(struct knlist * knl,struct thread * td,int islocked,int killkn)2757 knlist_cleardel(struct knlist *knl, struct thread *td, int islocked, int killkn)
2758 {
2759 struct knote *kn, *kn2;
2760 struct kqueue *kq;
2761
2762 KASSERT(!knl->kl_autodestroy, ("cleardel for autodestroy %p", knl));
2763 if (islocked)
2764 KNL_ASSERT_LOCKED(knl);
2765 else {
2766 KNL_ASSERT_UNLOCKED(knl);
2767 again: /* need to reacquire lock since we have dropped it */
2768 knl->kl_lock(knl->kl_lockarg);
2769 }
2770
2771 SLIST_FOREACH_SAFE(kn, &knl->kl_list, kn_selnext, kn2) {
2772 kq = kn->kn_kq;
2773 KQ_LOCK(kq);
2774 if (kn_in_flux(kn)) {
2775 KQ_UNLOCK(kq);
2776 continue;
2777 }
2778 knlist_remove_kq(knl, kn, 1, 1);
2779 if (killkn) {
2780 kn_enter_flux(kn);
2781 KQ_UNLOCK(kq);
2782 knote_drop_detached(kn, td);
2783 } else {
2784 /* Make sure cleared knotes disappear soon */
2785 kn->kn_flags |= EV_EOF | EV_ONESHOT;
2786 KQ_UNLOCK(kq);
2787 }
2788 kq = NULL;
2789 }
2790
2791 if (!SLIST_EMPTY(&knl->kl_list)) {
2792 /* there are still in flux knotes remaining */
2793 kn = SLIST_FIRST(&knl->kl_list);
2794 kq = kn->kn_kq;
2795 KQ_LOCK(kq);
2796 KASSERT(kn_in_flux(kn), ("knote removed w/o list lock"));
2797 knl->kl_unlock(knl->kl_lockarg);
2798 kq->kq_state |= KQ_FLUXWAIT;
2799 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqkclr", 0);
2800 kq = NULL;
2801 goto again;
2802 }
2803
2804 if (islocked)
2805 KNL_ASSERT_LOCKED(knl);
2806 else {
2807 knl->kl_unlock(knl->kl_lockarg);
2808 KNL_ASSERT_UNLOCKED(knl);
2809 }
2810 }
2811
2812 /*
2813 * Remove all knotes referencing a specified fd must be called with FILEDESC
2814 * lock. This prevents a race where a new fd comes along and occupies the
2815 * entry and we attach a knote to the fd.
2816 */
2817 void
knote_fdclose(struct thread * td,int fd)2818 knote_fdclose(struct thread *td, int fd)
2819 {
2820 struct filedesc *fdp = td->td_proc->p_fd;
2821 struct kqueue *kq;
2822 struct knote *kn;
2823 int influx;
2824
2825 FILEDESC_XLOCK_ASSERT(fdp);
2826
2827 /*
2828 * We shouldn't have to worry about new kevents appearing on fd
2829 * since filedesc is locked.
2830 */
2831 TAILQ_FOREACH(kq, &fdp->fd_kqlist, kq_list) {
2832 KQ_LOCK(kq);
2833
2834 again:
2835 influx = 0;
2836 while (kq->kq_knlistsize > fd &&
2837 (kn = SLIST_FIRST(&kq->kq_knlist[fd])) != NULL) {
2838 if (kn_in_flux(kn)) {
2839 /* someone else might be waiting on our knote */
2840 if (influx)
2841 wakeup(kq);
2842 kq->kq_state |= KQ_FLUXWAIT;
2843 msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0);
2844 goto again;
2845 }
2846 kn_enter_flux(kn);
2847 KQ_UNLOCK(kq);
2848 influx = 1;
2849 knote_drop(kn, td);
2850 KQ_LOCK(kq);
2851 }
2852 KQ_UNLOCK_FLUX(kq);
2853 }
2854 }
2855
2856 static int
knote_attach(struct knote * kn,struct kqueue * kq)2857 knote_attach(struct knote *kn, struct kqueue *kq)
2858 {
2859 struct klist *list;
2860
2861 KASSERT(kn_in_flux(kn), ("knote %p not marked influx", kn));
2862 KQ_OWNED(kq);
2863
2864 if ((kq->kq_state & KQ_CLOSING) != 0)
2865 return (EBADF);
2866 if (kn->kn_fop->f_isfd) {
2867 if (kn->kn_id >= kq->kq_knlistsize)
2868 return (ENOMEM);
2869 list = &kq->kq_knlist[kn->kn_id];
2870 } else {
2871 if (kq->kq_knhash == NULL)
2872 return (ENOMEM);
2873 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
2874 }
2875 SLIST_INSERT_HEAD(list, kn, kn_link);
2876 return (0);
2877 }
2878
2879 static void
knote_drop(struct knote * kn,struct thread * td)2880 knote_drop(struct knote *kn, struct thread *td)
2881 {
2882
2883 if ((kn->kn_status & KN_DETACHED) == 0)
2884 kn->kn_fop->f_detach(kn);
2885 knote_drop_detached(kn, td);
2886 }
2887
2888 static void
knote_drop_detached(struct knote * kn,struct thread * td)2889 knote_drop_detached(struct knote *kn, struct thread *td)
2890 {
2891 struct kqueue *kq;
2892 struct klist *list;
2893
2894 kq = kn->kn_kq;
2895
2896 KASSERT((kn->kn_status & KN_DETACHED) != 0,
2897 ("knote %p still attached", kn));
2898 KQ_NOTOWNED(kq);
2899
2900 KQ_LOCK(kq);
2901 for (;;) {
2902 KASSERT(kn->kn_influx >= 1,
2903 ("knote_drop called on %p with influx %d",
2904 kn, kn->kn_influx));
2905 if (kn->kn_influx == 1)
2906 break;
2907 kq->kq_state |= KQ_FLUXWAIT;
2908 msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0);
2909 }
2910
2911 if (kn->kn_fop->f_isfd)
2912 list = &kq->kq_knlist[kn->kn_id];
2913 else
2914 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)];
2915
2916 if (!SLIST_EMPTY(list))
2917 SLIST_REMOVE(list, kn, knote, kn_link);
2918 if (kn->kn_status & KN_QUEUED)
2919 knote_dequeue(kn);
2920 KQ_UNLOCK_FLUX(kq);
2921
2922 if (kn->kn_fop->f_isfd) {
2923 fdrop(kn->kn_fp, td);
2924 kn->kn_fp = NULL;
2925 }
2926 kqueue_fo_release(kn->kn_kevent.filter);
2927 kn->kn_fop = NULL;
2928 knote_free(kn);
2929 }
2930
2931 static void
knote_enqueue(struct knote * kn)2932 knote_enqueue(struct knote *kn)
2933 {
2934 struct kqueue *kq = kn->kn_kq;
2935
2936 KQ_OWNED(kn->kn_kq);
2937 KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued"));
2938
2939 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe);
2940 kn->kn_status |= KN_QUEUED;
2941 kq->kq_count++;
2942 kqueue_wakeup(kq);
2943 }
2944
2945 static void
knote_dequeue(struct knote * kn)2946 knote_dequeue(struct knote *kn)
2947 {
2948 struct kqueue *kq = kn->kn_kq;
2949
2950 KQ_OWNED(kn->kn_kq);
2951 KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued"));
2952
2953 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe);
2954 kn->kn_status &= ~KN_QUEUED;
2955 kq->kq_count--;
2956 }
2957
2958 static void
knote_init(void * dummy __unused)2959 knote_init(void *dummy __unused)
2960 {
2961
2962 knote_zone = uma_zcreate("KNOTE", sizeof(struct knote), NULL, NULL,
2963 NULL, NULL, UMA_ALIGN_PTR, 0);
2964 ast_register(TDA_KQUEUE, ASTR_ASTF_REQUIRED, 0, ast_kqueue);
2965 prison0.pr_klist = knlist_alloc(&prison0.pr_mtx);
2966 }
2967 SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL);
2968
2969 static struct knote *
knote_alloc(int mflag)2970 knote_alloc(int mflag)
2971 {
2972
2973 return (uma_zalloc(knote_zone, mflag | M_ZERO));
2974 }
2975
2976 static void
knote_free(struct knote * kn)2977 knote_free(struct knote *kn)
2978 {
2979
2980 uma_zfree(knote_zone, kn);
2981 }
2982
2983 /*
2984 * Register the kev w/ the kq specified by fd.
2985 */
2986 int
kqfd_register(int fd,struct kevent * kev,struct thread * td,int mflag)2987 kqfd_register(int fd, struct kevent *kev, struct thread *td, int mflag)
2988 {
2989 struct kqueue *kq;
2990 struct file *fp;
2991 cap_rights_t rights;
2992 int error;
2993
2994 error = fget(td, fd, cap_rights_init_one(&rights, CAP_KQUEUE_CHANGE),
2995 &fp);
2996 if (error != 0)
2997 return (error);
2998 if ((error = kqueue_acquire(fp, &kq)) != 0)
2999 goto noacquire;
3000
3001 error = kqueue_register(kq, kev, td, mflag);
3002 kqueue_release(kq, 0);
3003
3004 noacquire:
3005 fdrop(fp, td);
3006 return (error);
3007 }
3008
3009 static int
kqueue_fork_alloc(struct filedesc * fdp,struct file * fp,struct file ** fp1,struct thread * td)3010 kqueue_fork_alloc(struct filedesc *fdp, struct file *fp, struct file **fp1,
3011 struct thread *td)
3012 {
3013 struct kqueue *kq, *kq1;
3014 int error;
3015
3016 MPASS(fp->f_type == DTYPE_KQUEUE);
3017 kq = fp->f_data;
3018 if ((kq->kq_state & KQ_CPONFORK) == 0)
3019 return (EOPNOTSUPP);
3020 error = kqueue_acquire_ref(kq);
3021 if (error != 0)
3022 return (error);
3023 error = kern_kqueue_alloc(td, fdp, NULL, fp1, 0, NULL, true, &kq1);
3024 if (error == 0) {
3025 kq1->kq_forksrc = kq;
3026 (*fp1)->f_flag = fp->f_flag & (FREAD | FWRITE | FEXEC |
3027 O_CLOEXEC | O_CLOFORK);
3028 } else {
3029 kqueue_release(kq, 0);
3030 }
3031 return (error);
3032 }
3033
3034 static void
kqueue_fork_copy_knote(struct kqueue * kq1,struct knote * kn,struct proc * p1,struct filedesc * fdp)3035 kqueue_fork_copy_knote(struct kqueue *kq1, struct knote *kn, struct proc *p1,
3036 struct filedesc *fdp)
3037 {
3038 struct knote *kn1;
3039 const struct filterops *fop;
3040 int error;
3041
3042 fop = kn->kn_fop;
3043 if (fop->f_copy == NULL || (fop->f_isfd &&
3044 fdp->fd_files->fdt_ofiles[kn->kn_kevent.ident].fde_file == NULL))
3045 return;
3046 error = kqueue_expand(kq1, fop, kn->kn_kevent.ident, M_WAITOK);
3047 if (error != 0)
3048 return;
3049
3050 kn1 = knote_alloc(M_WAITOK);
3051 *kn1 = *kn;
3052 kn1->kn_status |= KN_DETACHED;
3053 kn1->kn_status &= ~KN_QUEUED;
3054 kn1->kn_kq = kq1;
3055 error = fop->f_copy(kn1, p1);
3056 if (error != 0) {
3057 knote_free(kn1);
3058 return;
3059 }
3060 (void)kqueue_fo_find(kn->kn_kevent.filter);
3061 if (fop->f_isfd && !fhold(kn1->kn_fp)) {
3062 fop->f_detach(kn1);
3063 kqueue_fo_release(kn->kn_kevent.filter);
3064 knote_free(kn1);
3065 return;
3066 }
3067 if (kn->kn_knlist != NULL)
3068 knlist_add(kn->kn_knlist, kn1, 0);
3069 KQ_LOCK(kq1);
3070 knote_attach(kn1, kq1);
3071 kn1->kn_influx = 0;
3072 if ((kn->kn_status & KN_QUEUED) != 0)
3073 knote_enqueue(kn1);
3074 KQ_UNLOCK(kq1);
3075 }
3076
3077 static void
kqueue_fork_copy_list(struct klist * knlist,struct knote * marker,struct kqueue * kq,struct kqueue * kq1,struct proc * p1,struct filedesc * fdp)3078 kqueue_fork_copy_list(struct klist *knlist, struct knote *marker,
3079 struct kqueue *kq, struct kqueue *kq1, struct proc *p1,
3080 struct filedesc *fdp)
3081 {
3082 struct knote *kn;
3083
3084 KQ_OWNED(kq);
3085 kn = SLIST_FIRST(knlist);
3086 while (kn != NULL) {
3087 if ((kn->kn_status & KN_DETACHED) != 0 ||
3088 (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0)) {
3089 kn = SLIST_NEXT(kn, kn_link);
3090 continue;
3091 }
3092 kn_enter_flux(kn);
3093 SLIST_INSERT_AFTER(kn, marker, kn_link);
3094 KQ_UNLOCK(kq);
3095 kqueue_fork_copy_knote(kq1, kn, p1, fdp);
3096 KQ_LOCK(kq);
3097 kn_leave_flux(kn);
3098 kn = SLIST_NEXT(marker, kn_link);
3099 /* XXXKIB switch kn_link to LIST? */
3100 SLIST_REMOVE(knlist, marker, knote, kn_link);
3101 }
3102 }
3103
3104 static int
kqueue_fork_copy(struct filedesc * fdp,struct file * fp,struct file * fp1,struct proc * p1,struct thread * td)3105 kqueue_fork_copy(struct filedesc *fdp, struct file *fp, struct file *fp1,
3106 struct proc *p1, struct thread *td)
3107 {
3108 struct kqueue *kq, *kq1;
3109 struct knote *marker;
3110 int error, i;
3111
3112 error = 0;
3113 MPASS(fp == NULL);
3114 MPASS(fp1->f_type == DTYPE_KQUEUE);
3115
3116 kq1 = fp1->f_data;
3117 kq = kq1->kq_forksrc;
3118 marker = knote_alloc(M_WAITOK);
3119 marker->kn_status = KN_MARKER;
3120
3121 KQ_LOCK(kq);
3122 for (i = 0; i < kq->kq_knlistsize; i++) {
3123 kqueue_fork_copy_list(&kq->kq_knlist[i], marker, kq, kq1,
3124 p1, fdp);
3125 }
3126 if (kq->kq_knhashmask != 0) {
3127 for (i = 0; i <= kq->kq_knhashmask; i++) {
3128 kqueue_fork_copy_list(&kq->kq_knhash[i], marker, kq,
3129 kq1, p1, fdp);
3130 }
3131 }
3132 kqueue_release(kq, 1);
3133 kq1->kq_forksrc = NULL;
3134 KQ_UNLOCK(kq);
3135
3136 knote_free(marker);
3137 return (error);
3138 }
3139
3140 static int
kqueue_fork(struct filedesc * fdp,struct file * fp,struct file ** fp1,struct proc * p1,struct thread * td)3141 kqueue_fork(struct filedesc *fdp, struct file *fp, struct file **fp1,
3142 struct proc *p1, struct thread *td)
3143 {
3144 if (*fp1 == NULL)
3145 return (kqueue_fork_alloc(fdp, fp, fp1, td));
3146 return (kqueue_fork_copy(fdp, fp, *fp1, p1, td));
3147 }
3148
3149 int
knote_triv_copy(struct knote * kn __unused,struct proc * p1 __unused)3150 knote_triv_copy(struct knote *kn __unused, struct proc *p1 __unused)
3151 {
3152 return (0);
3153 }
3154
3155 struct knote_status_export_bit {
3156 int kn_status_bit;
3157 int knt_status_bit;
3158 };
3159
3160 #define ST(name) \
3161 { .kn_status_bit = KN_##name, .knt_status_bit = KNOTE_STATUS_##name }
3162 static const struct knote_status_export_bit knote_status_export_bits[] = {
3163 ST(ACTIVE),
3164 ST(QUEUED),
3165 ST(DISABLED),
3166 ST(DETACHED),
3167 ST(KQUEUE),
3168 };
3169 #undef ST
3170
3171 static int
knote_status_export(int kn_status)3172 knote_status_export(int kn_status)
3173 {
3174 const struct knote_status_export_bit *b;
3175 unsigned i;
3176 int res;
3177
3178 res = 0;
3179 for (i = 0; i < nitems(knote_status_export_bits); i++) {
3180 b = &knote_status_export_bits[i];
3181 if ((kn_status & b->kn_status_bit) != 0)
3182 res |= b->knt_status_bit;
3183 }
3184 return (res);
3185 }
3186
3187 static int
kern_proc_kqueue_report_one(struct sbuf * s,struct proc * p,int kq_fd,struct kqueue * kq,struct knote * kn,bool compat32 __unused)3188 kern_proc_kqueue_report_one(struct sbuf *s, struct proc *p,
3189 int kq_fd, struct kqueue *kq, struct knote *kn, bool compat32 __unused)
3190 {
3191 struct kinfo_knote kin;
3192 #ifdef COMPAT_FREEBSD32
3193 struct kinfo_knote32 kin32;
3194 #endif
3195 int error;
3196
3197 if (kn->kn_status == KN_MARKER)
3198 return (0);
3199
3200 memset(&kin, 0, sizeof(kin));
3201 kin.knt_kq_fd = kq_fd;
3202 memcpy(&kin.knt_event, &kn->kn_kevent, sizeof(struct kevent));
3203 kin.knt_status = knote_status_export(kn->kn_status);
3204 kn_enter_flux(kn);
3205 KQ_UNLOCK_FLUX(kq);
3206 if (kn->kn_fop->f_userdump != NULL)
3207 (void)kn->kn_fop->f_userdump(p, kn, &kin);
3208 #ifdef COMPAT_FREEBSD32
3209 if (compat32) {
3210 freebsd32_kinfo_knote_to_32(&kin, &kin32);
3211 error = sbuf_bcat(s, &kin32, sizeof(kin32));
3212 } else
3213 #endif
3214 error = sbuf_bcat(s, &kin, sizeof(kin));
3215 KQ_LOCK(kq);
3216 kn_leave_flux(kn);
3217 return (error);
3218 }
3219
3220 static int
kern_proc_kqueue_report(struct sbuf * s,struct proc * p,int kq_fd,struct kqueue * kq,bool compat32)3221 kern_proc_kqueue_report(struct sbuf *s, struct proc *p, int kq_fd,
3222 struct kqueue *kq, bool compat32)
3223 {
3224 struct knote *kn;
3225 int error, i;
3226
3227 error = 0;
3228 KQ_LOCK(kq);
3229 for (i = 0; i < kq->kq_knlistsize; i++) {
3230 SLIST_FOREACH(kn, &kq->kq_knlist[i], kn_link) {
3231 error = kern_proc_kqueue_report_one(s, p, kq_fd,
3232 kq, kn, compat32);
3233 if (error != 0)
3234 goto out;
3235 }
3236 }
3237 if (kq->kq_knhashmask == 0)
3238 goto out;
3239 for (i = 0; i <= kq->kq_knhashmask; i++) {
3240 SLIST_FOREACH(kn, &kq->kq_knhash[i], kn_link) {
3241 error = kern_proc_kqueue_report_one(s, p, kq_fd,
3242 kq, kn, compat32);
3243 if (error != 0)
3244 goto out;
3245 }
3246 }
3247 out:
3248 KQ_UNLOCK_FLUX(kq);
3249 return (error);
3250 }
3251
3252 struct kern_proc_kqueues_out1_cb_args {
3253 struct sbuf *s;
3254 bool compat32;
3255 };
3256
3257 static int
kern_proc_kqueues_out1_cb(struct proc * p,int fd,struct file * fp,void * arg)3258 kern_proc_kqueues_out1_cb(struct proc *p, int fd, struct file *fp, void *arg)
3259 {
3260 struct kqueue *kq;
3261 struct kern_proc_kqueues_out1_cb_args *a;
3262
3263 if (fp->f_type != DTYPE_KQUEUE)
3264 return (0);
3265 a = arg;
3266 kq = fp->f_data;
3267 return (kern_proc_kqueue_report(a->s, p, fd, kq, a->compat32));
3268 }
3269
3270 static int
kern_proc_kqueues_out1(struct thread * td,struct proc * p,struct sbuf * s,bool compat32)3271 kern_proc_kqueues_out1(struct thread *td, struct proc *p, struct sbuf *s,
3272 bool compat32)
3273 {
3274 struct kern_proc_kqueues_out1_cb_args a;
3275
3276 a.s = s;
3277 a.compat32 = compat32;
3278 return (fget_remote_foreach(td, p, kern_proc_kqueues_out1_cb, &a));
3279 }
3280
3281 int
kern_proc_kqueues_out(struct proc * p,struct sbuf * sb,size_t maxlen,bool compat32)3282 kern_proc_kqueues_out(struct proc *p, struct sbuf *sb, size_t maxlen,
3283 bool compat32)
3284 {
3285 struct sbuf *s, sm;
3286 size_t sb_len;
3287 int error;
3288
3289 if (maxlen == -1 || maxlen == 0)
3290 sb_len = 128;
3291 else
3292 sb_len = maxlen;
3293 s = sbuf_new(&sm, NULL, sb_len, maxlen == -1 ? SBUF_AUTOEXTEND :
3294 SBUF_FIXEDLEN);
3295 error = kern_proc_kqueues_out1(curthread, p, s, compat32);
3296 sbuf_finish(s);
3297 if (error == 0) {
3298 sbuf_bcat(sb, sbuf_data(s), MIN(sbuf_len(s), maxlen == -1 ?
3299 SIZE_T_MAX : maxlen));
3300 }
3301 sbuf_delete(s);
3302 return (error);
3303 }
3304
3305 static int
sysctl_kern_proc_kqueue_one(struct thread * td,struct sbuf * s,struct proc * p,int kq_fd,bool compat32)3306 sysctl_kern_proc_kqueue_one(struct thread *td, struct sbuf *s, struct proc *p,
3307 int kq_fd, bool compat32)
3308 {
3309 struct file *fp;
3310 struct kqueue *kq;
3311 int error;
3312
3313 error = fget_remote(td, p, kq_fd, &fp);
3314 if (error == 0) {
3315 if (fp->f_type != DTYPE_KQUEUE) {
3316 error = EINVAL;
3317 } else {
3318 kq = fp->f_data;
3319 error = kern_proc_kqueue_report(s, p, kq_fd, kq,
3320 compat32);
3321 }
3322 fdrop(fp, td);
3323 }
3324 return (error);
3325 }
3326
3327 static int
sysctl_kern_proc_kqueue(SYSCTL_HANDLER_ARGS)3328 sysctl_kern_proc_kqueue(SYSCTL_HANDLER_ARGS)
3329 {
3330 struct thread *td;
3331 struct proc *p;
3332 struct sbuf *s, sm;
3333 int error, error1, *name;
3334 bool compat32;
3335
3336 name = (int *)arg1;
3337 if ((u_int)arg2 > 2 || (u_int)arg2 == 0)
3338 return (EINVAL);
3339
3340 error = pget((pid_t)name[0], PGET_HOLD | PGET_CANDEBUG, &p);
3341 if (error != 0)
3342 return (error);
3343
3344 td = curthread;
3345 #ifdef COMPAT_FREEBSD32
3346 compat32 = SV_CURPROC_FLAG(SV_ILP32);
3347 #else
3348 compat32 = false;
3349 #endif
3350
3351 s = sbuf_new_for_sysctl(&sm, NULL, 0, req);
3352 if (s == NULL) {
3353 error = ENOMEM;
3354 goto out;
3355 }
3356 sbuf_clear_flags(s, SBUF_INCLUDENUL);
3357
3358 if ((u_int)arg2 == 1) {
3359 error = kern_proc_kqueues_out1(td, p, s, compat32);
3360 } else {
3361 error = sysctl_kern_proc_kqueue_one(td, s, p,
3362 name[1] /* kq_fd */, compat32);
3363 }
3364
3365 error1 = sbuf_finish(s);
3366 if (error == 0)
3367 error = error1;
3368 sbuf_delete(s);
3369
3370 out:
3371 PRELE(p);
3372 return (error);
3373 }
3374
3375 static SYSCTL_NODE(_kern_proc, KERN_PROC_KQUEUE, kq,
3376 CTLFLAG_RD | CTLFLAG_MPSAFE,
3377 sysctl_kern_proc_kqueue, "KQueue events");
3378