1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 1999,2000,2001 Jonathan Lemon <jlemon@FreeBSD.org> 5 * Copyright 2004 John-Mark Gurney <jmg@FreeBSD.org> 6 * Copyright (c) 2009 Apple, Inc. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 #include <sys/cdefs.h> 32 #include "opt_ktrace.h" 33 #include "opt_kqueue.h" 34 35 #ifdef COMPAT_FREEBSD11 36 #define _WANT_FREEBSD11_KEVENT 37 #endif 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/capsicum.h> 42 #include <sys/kernel.h> 43 #include <sys/limits.h> 44 #include <sys/lock.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/malloc.h> 48 #include <sys/unistd.h> 49 #include <sys/file.h> 50 #include <sys/filedesc.h> 51 #include <sys/filio.h> 52 #include <sys/fcntl.h> 53 #include <sys/jail.h> 54 #include <sys/jaildesc.h> 55 #include <sys/kthread.h> 56 #include <sys/selinfo.h> 57 #include <sys/queue.h> 58 #include <sys/event.h> 59 #include <sys/eventvar.h> 60 #include <sys/poll.h> 61 #include <sys/protosw.h> 62 #include <sys/resourcevar.h> 63 #include <sys/sbuf.h> 64 #include <sys/sigio.h> 65 #include <sys/signalvar.h> 66 #include <sys/socket.h> 67 #include <sys/socketvar.h> 68 #include <sys/stat.h> 69 #include <sys/sysctl.h> 70 #include <sys/sysent.h> 71 #include <sys/sysproto.h> 72 #include <sys/syscallsubr.h> 73 #include <sys/taskqueue.h> 74 #include <sys/uio.h> 75 #include <sys/user.h> 76 #ifdef KTRACE 77 #include <sys/ktrace.h> 78 #endif 79 #include <machine/atomic.h> 80 #ifdef COMPAT_FREEBSD32 81 #include <compat/freebsd32/freebsd32.h> 82 #include <compat/freebsd32/freebsd32_util.h> 83 #endif 84 85 #include <vm/uma.h> 86 87 static MALLOC_DEFINE(M_KQUEUE, "kqueue", "memory for kqueue system"); 88 89 /* 90 * This lock is used if multiple kq locks are required. This possibly 91 * should be made into a per proc lock. 92 */ 93 static struct mtx kq_global; 94 MTX_SYSINIT(kq_global, &kq_global, "kqueue order", MTX_DEF); 95 #define KQ_GLOBAL_LOCK(lck, haslck) do { \ 96 if (!haslck) \ 97 mtx_lock(lck); \ 98 haslck = 1; \ 99 } while (0) 100 #define KQ_GLOBAL_UNLOCK(lck, haslck) do { \ 101 if (haslck) \ 102 mtx_unlock(lck); \ 103 haslck = 0; \ 104 } while (0) 105 106 TASKQUEUE_DEFINE_THREAD(kqueue_ctx); 107 108 static int kevent_copyout(void *arg, struct kevent *kevp, int count); 109 static int kevent_copyin(void *arg, struct kevent *kevp, int count); 110 static int kqueue_register(struct kqueue *kq, struct kevent *kev, 111 struct thread *td, int mflag); 112 static int kqueue_acquire(struct file *fp, struct kqueue **kqp); 113 static void kqueue_release(struct kqueue *kq, int locked); 114 static void kqueue_destroy(struct kqueue *kq); 115 static void kqueue_drain(struct kqueue *kq, struct thread *td); 116 static int kqueue_expand(struct kqueue *kq, const struct filterops *fops, 117 uintptr_t ident, int mflag); 118 static void kqueue_task(void *arg, int pending); 119 static int kqueue_scan(struct kqueue *kq, int maxevents, 120 struct kevent_copyops *k_ops, 121 const struct timespec *timeout, 122 struct kevent *keva, struct thread *td); 123 static void kqueue_wakeup(struct kqueue *kq); 124 static const struct filterops *kqueue_fo_find(int filt); 125 static void kqueue_fo_release(int filt); 126 struct g_kevent_args; 127 static int kern_kevent_generic(struct thread *td, 128 struct g_kevent_args *uap, 129 struct kevent_copyops *k_ops, const char *struct_name); 130 131 static fo_ioctl_t kqueue_ioctl; 132 static fo_poll_t kqueue_poll; 133 static fo_kqfilter_t kqueue_kqfilter; 134 static fo_stat_t kqueue_stat; 135 static fo_close_t kqueue_close; 136 static fo_fill_kinfo_t kqueue_fill_kinfo; 137 static fo_fork_t kqueue_fork; 138 139 static const struct fileops kqueueops = { 140 .fo_read = invfo_rdwr, 141 .fo_write = invfo_rdwr, 142 .fo_truncate = invfo_truncate, 143 .fo_ioctl = kqueue_ioctl, 144 .fo_poll = kqueue_poll, 145 .fo_kqfilter = kqueue_kqfilter, 146 .fo_stat = kqueue_stat, 147 .fo_close = kqueue_close, 148 .fo_chmod = invfo_chmod, 149 .fo_chown = invfo_chown, 150 .fo_sendfile = invfo_sendfile, 151 .fo_cmp = file_kcmp_generic, 152 .fo_fork = kqueue_fork, 153 .fo_fill_kinfo = kqueue_fill_kinfo, 154 .fo_flags = DFLAG_FORK, 155 }; 156 157 static int knote_attach(struct knote *kn, struct kqueue *kq); 158 static void knote_drop(struct knote *kn, struct thread *td); 159 static void knote_drop_detached(struct knote *kn, struct thread *td); 160 static void knote_enqueue(struct knote *kn); 161 static void knote_dequeue(struct knote *kn); 162 static void knote_init(void *); 163 static struct knote *knote_alloc(int mflag); 164 static void knote_free(struct knote *kn); 165 166 static void filt_kqdetach(struct knote *kn); 167 static int filt_kqueue(struct knote *kn, long hint); 168 static int filt_procattach(struct knote *kn); 169 static void filt_procdetach(struct knote *kn); 170 static int filt_proc(struct knote *kn, long hint); 171 static int filt_jailattach(struct knote *kn); 172 static void filt_jaildetach(struct knote *kn); 173 static int filt_jail(struct knote *kn, long hint); 174 static int filt_fileattach(struct knote *kn); 175 static void filt_timerexpire(void *knx); 176 static void filt_timerexpire_l(struct knote *kn, bool proc_locked); 177 static int filt_timerattach(struct knote *kn); 178 static void filt_timerdetach(struct knote *kn); 179 static void filt_timerstart(struct knote *kn, sbintime_t to); 180 static void filt_timertouch(struct knote *kn, struct kevent *kev, 181 u_long type); 182 static int filt_timercopy(struct knote *kn, struct proc *p1); 183 static int filt_timervalidate(struct knote *kn, sbintime_t *to); 184 static int filt_timer(struct knote *kn, long hint); 185 static int filt_userattach(struct knote *kn); 186 static void filt_userdetach(struct knote *kn); 187 static int filt_user(struct knote *kn, long hint); 188 static void filt_usertouch(struct knote *kn, struct kevent *kev, 189 u_long type); 190 191 static const struct filterops file_filtops = { 192 .f_isfd = 1, 193 .f_attach = filt_fileattach, 194 .f_copy = knote_triv_copy, 195 }; 196 static const struct filterops kqread_filtops = { 197 .f_isfd = 1, 198 .f_detach = filt_kqdetach, 199 .f_event = filt_kqueue, 200 .f_copy = knote_triv_copy, 201 }; 202 /* XXX - move to kern_proc.c? */ 203 static const struct filterops proc_filtops = { 204 .f_isfd = 0, 205 .f_attach = filt_procattach, 206 .f_detach = filt_procdetach, 207 .f_event = filt_proc, 208 .f_copy = knote_triv_copy, 209 }; 210 static const struct filterops jail_filtops = { 211 .f_isfd = 0, 212 .f_attach = filt_jailattach, 213 .f_detach = filt_jaildetach, 214 .f_event = filt_jail, 215 .f_copy = knote_triv_copy, 216 }; 217 static const struct filterops timer_filtops = { 218 .f_isfd = 0, 219 .f_attach = filt_timerattach, 220 .f_detach = filt_timerdetach, 221 .f_event = filt_timer, 222 .f_touch = filt_timertouch, 223 .f_copy = filt_timercopy, 224 }; 225 static const struct filterops user_filtops = { 226 .f_attach = filt_userattach, 227 .f_detach = filt_userdetach, 228 .f_event = filt_user, 229 .f_touch = filt_usertouch, 230 .f_copy = knote_triv_copy, 231 }; 232 233 static uma_zone_t knote_zone; 234 static unsigned int __exclusive_cache_line kq_ncallouts; 235 static unsigned int kq_calloutmax = 4 * 1024; 236 SYSCTL_UINT(_kern, OID_AUTO, kq_calloutmax, CTLFLAG_RW, 237 &kq_calloutmax, 0, "Maximum number of callouts allocated for kqueue"); 238 239 /* XXX - ensure not influx ? */ 240 #define KNOTE_ACTIVATE(kn, islock) do { \ 241 if ((islock)) \ 242 mtx_assert(&(kn)->kn_kq->kq_lock, MA_OWNED); \ 243 else \ 244 KQ_LOCK((kn)->kn_kq); \ 245 (kn)->kn_status |= KN_ACTIVE; \ 246 if (((kn)->kn_status & (KN_QUEUED | KN_DISABLED)) == 0) \ 247 knote_enqueue((kn)); \ 248 if (!(islock)) \ 249 KQ_UNLOCK((kn)->kn_kq); \ 250 } while (0) 251 #define KQ_LOCK(kq) do { \ 252 mtx_lock(&(kq)->kq_lock); \ 253 } while (0) 254 #define KQ_FLUX_WAKEUP(kq) do { \ 255 if (((kq)->kq_state & KQ_FLUXWAIT) == KQ_FLUXWAIT) { \ 256 (kq)->kq_state &= ~KQ_FLUXWAIT; \ 257 wakeup((kq)); \ 258 } \ 259 } while (0) 260 #define KQ_UNLOCK_FLUX(kq) do { \ 261 KQ_FLUX_WAKEUP(kq); \ 262 mtx_unlock(&(kq)->kq_lock); \ 263 } while (0) 264 #define KQ_UNLOCK(kq) do { \ 265 mtx_unlock(&(kq)->kq_lock); \ 266 } while (0) 267 #define KQ_OWNED(kq) do { \ 268 mtx_assert(&(kq)->kq_lock, MA_OWNED); \ 269 } while (0) 270 #define KQ_NOTOWNED(kq) do { \ 271 mtx_assert(&(kq)->kq_lock, MA_NOTOWNED); \ 272 } while (0) 273 274 static struct knlist * 275 kn_list_lock(struct knote *kn) 276 { 277 struct knlist *knl; 278 279 knl = kn->kn_knlist; 280 if (knl != NULL) 281 knl->kl_lock(knl->kl_lockarg); 282 return (knl); 283 } 284 285 static void 286 kn_list_unlock(struct knlist *knl) 287 { 288 bool do_free; 289 290 if (knl == NULL) 291 return; 292 do_free = knl->kl_autodestroy && knlist_empty(knl); 293 knl->kl_unlock(knl->kl_lockarg); 294 if (do_free) { 295 knlist_destroy(knl); 296 free(knl, M_KQUEUE); 297 } 298 } 299 300 static bool 301 kn_in_flux(struct knote *kn) 302 { 303 304 return (kn->kn_influx > 0); 305 } 306 307 static void 308 kn_enter_flux(struct knote *kn) 309 { 310 311 KQ_OWNED(kn->kn_kq); 312 MPASS(kn->kn_influx < INT_MAX); 313 kn->kn_influx++; 314 } 315 316 static bool 317 kn_leave_flux(struct knote *kn) 318 { 319 320 KQ_OWNED(kn->kn_kq); 321 MPASS(kn->kn_influx > 0); 322 kn->kn_influx--; 323 return (kn->kn_influx == 0); 324 } 325 326 #define KNL_ASSERT_LOCK(knl, islocked) do { \ 327 if (islocked) \ 328 KNL_ASSERT_LOCKED(knl); \ 329 else \ 330 KNL_ASSERT_UNLOCKED(knl); \ 331 } while (0) 332 #ifdef INVARIANTS 333 #define KNL_ASSERT_LOCKED(knl) do { \ 334 knl->kl_assert_lock((knl)->kl_lockarg, LA_LOCKED); \ 335 } while (0) 336 #define KNL_ASSERT_UNLOCKED(knl) do { \ 337 knl->kl_assert_lock((knl)->kl_lockarg, LA_UNLOCKED); \ 338 } while (0) 339 #else /* !INVARIANTS */ 340 #define KNL_ASSERT_LOCKED(knl) do {} while (0) 341 #define KNL_ASSERT_UNLOCKED(knl) do {} while (0) 342 #endif /* INVARIANTS */ 343 344 #ifndef KN_HASHSIZE 345 #define KN_HASHSIZE 64 /* XXX should be tunable */ 346 #endif 347 348 #define KN_HASH(val, mask) (((val) ^ (val >> 8)) & (mask)) 349 350 static int 351 filt_nullattach(struct knote *kn) 352 { 353 354 return (ENXIO); 355 }; 356 357 static const struct filterops null_filtops = { 358 .f_isfd = 0, 359 .f_attach = filt_nullattach, 360 .f_copy = knote_triv_copy, 361 }; 362 363 /* XXX - make SYSINIT to add these, and move into respective modules. */ 364 extern const struct filterops sig_filtops; 365 extern const struct filterops fs_filtops; 366 367 /* 368 * Table for all system-defined filters. 369 */ 370 static struct mtx filterops_lock; 371 MTX_SYSINIT(kqueue_filterops, &filterops_lock, "protect sysfilt_ops", MTX_DEF); 372 static struct { 373 const struct filterops *for_fop; 374 int for_nolock; 375 int for_refcnt; 376 } sysfilt_ops[EVFILT_SYSCOUNT] = { 377 [~EVFILT_READ] = { &file_filtops, 1 }, 378 [~EVFILT_WRITE] = { &file_filtops, 1 }, 379 [~EVFILT_AIO] = { &null_filtops }, 380 [~EVFILT_VNODE] = { &file_filtops, 1 }, 381 [~EVFILT_PROC] = { &proc_filtops, 1 }, 382 [~EVFILT_SIGNAL] = { &sig_filtops, 1 }, 383 [~EVFILT_TIMER] = { &timer_filtops, 1 }, 384 [~EVFILT_PROCDESC] = { &file_filtops, 1 }, 385 [~EVFILT_FS] = { &fs_filtops, 1 }, 386 [~EVFILT_LIO] = { &null_filtops }, 387 [~EVFILT_USER] = { &user_filtops, 1 }, 388 [~EVFILT_SENDFILE] = { &null_filtops }, 389 [~EVFILT_EMPTY] = { &file_filtops, 1 }, 390 [~EVFILT_JAIL] = { &jail_filtops, 1 }, 391 [~EVFILT_JAILDESC] = { &file_filtops, 1 }, 392 }; 393 394 /* 395 * Simple redirection for all cdevsw style objects to call their fo_kqfilter 396 * method. 397 */ 398 static int 399 filt_fileattach(struct knote *kn) 400 { 401 402 return (fo_kqfilter(kn->kn_fp, kn)); 403 } 404 405 /*ARGSUSED*/ 406 static int 407 kqueue_kqfilter(struct file *fp, struct knote *kn) 408 { 409 struct kqueue *kq = kn->kn_fp->f_data; 410 411 if (kn->kn_filter != EVFILT_READ) 412 return (EINVAL); 413 414 kn->kn_status |= KN_KQUEUE; 415 kn->kn_fop = &kqread_filtops; 416 knlist_add(&kq->kq_sel.si_note, kn, 0); 417 418 return (0); 419 } 420 421 static void 422 filt_kqdetach(struct knote *kn) 423 { 424 struct kqueue *kq = kn->kn_fp->f_data; 425 426 knlist_remove(&kq->kq_sel.si_note, kn, 0); 427 } 428 429 /*ARGSUSED*/ 430 static int 431 filt_kqueue(struct knote *kn, long hint) 432 { 433 struct kqueue *kq = kn->kn_fp->f_data; 434 435 kn->kn_data = kq->kq_count; 436 return (kn->kn_data > 0); 437 } 438 439 /* XXX - move to kern_proc.c? */ 440 static int 441 filt_procattach(struct knote *kn) 442 { 443 struct proc *p; 444 int error; 445 bool exiting, immediate; 446 447 exiting = immediate = false; 448 if (kn->kn_sfflags & NOTE_EXIT) 449 p = pfind_any(kn->kn_id); 450 else 451 p = pfind(kn->kn_id); 452 if (p == NULL) 453 return (ESRCH); 454 if (p->p_flag & P_WEXIT) 455 exiting = true; 456 457 if ((error = p_cansee(curthread, p))) { 458 PROC_UNLOCK(p); 459 return (error); 460 } 461 462 kn->kn_ptr.p_proc = p; 463 kn->kn_flags |= EV_CLEAR; /* automatically set */ 464 465 /* 466 * Internal flag indicating registration done by kernel for the 467 * purposes of getting a NOTE_CHILD notification. 468 */ 469 if (kn->kn_flags & EV_FLAG2) { 470 kn->kn_flags &= ~EV_FLAG2; 471 kn->kn_data = kn->kn_sdata; /* ppid */ 472 kn->kn_fflags = NOTE_CHILD; 473 kn->kn_sfflags &= ~(NOTE_EXIT | NOTE_EXEC | NOTE_FORK); 474 immediate = true; /* Force immediate activation of child note. */ 475 } 476 /* 477 * Internal flag indicating registration done by kernel (for other than 478 * NOTE_CHILD). 479 */ 480 if (kn->kn_flags & EV_FLAG1) { 481 kn->kn_flags &= ~EV_FLAG1; 482 } 483 484 knlist_add(p->p_klist, kn, 1); 485 486 /* 487 * Immediately activate any child notes or, in the case of a zombie 488 * target process, exit notes. The latter is necessary to handle the 489 * case where the target process, e.g. a child, dies before the kevent 490 * is registered. 491 */ 492 if (immediate || (exiting && filt_proc(kn, NOTE_EXIT))) 493 KNOTE_ACTIVATE(kn, 0); 494 495 PROC_UNLOCK(p); 496 497 return (0); 498 } 499 500 /* 501 * The knote may be attached to a different process, which may exit, 502 * leaving nothing for the knote to be attached to. So when the process 503 * exits, the knote is marked as DETACHED and also flagged as ONESHOT so 504 * it will be deleted when read out. However, as part of the knote deletion, 505 * this routine is called, so a check is needed to avoid actually performing 506 * a detach, because the original process does not exist any more. 507 */ 508 /* XXX - move to kern_proc.c? */ 509 static void 510 filt_procdetach(struct knote *kn) 511 { 512 513 knlist_remove(kn->kn_knlist, kn, 0); 514 kn->kn_ptr.p_proc = NULL; 515 } 516 517 /* XXX - move to kern_proc.c? */ 518 static int 519 filt_proc(struct knote *kn, long hint) 520 { 521 struct proc *p; 522 u_int event; 523 524 p = kn->kn_ptr.p_proc; 525 if (p == NULL) /* already activated, from attach filter */ 526 return (0); 527 528 /* Mask off extra data. */ 529 event = (u_int)hint & NOTE_PCTRLMASK; 530 531 /* If the user is interested in this event, record it. */ 532 if (kn->kn_sfflags & event) 533 kn->kn_fflags |= event; 534 535 /* Process is gone, so flag the event as finished. */ 536 if (event == NOTE_EXIT) { 537 kn->kn_flags |= EV_EOF | EV_ONESHOT; 538 kn->kn_ptr.p_proc = NULL; 539 if (kn->kn_fflags & NOTE_EXIT) 540 kn->kn_data = KW_EXITCODE(p->p_xexit, p->p_xsig); 541 if (kn->kn_fflags == 0) 542 kn->kn_flags |= EV_DROP; 543 return (1); 544 } 545 546 return (kn->kn_fflags != 0); 547 } 548 549 /* 550 * Called when the process forked. It mostly does the same as the 551 * knote(), activating all knotes registered to be activated when the 552 * process forked. Additionally, for each knote attached to the 553 * parent, check whether user wants to track the new process. If so 554 * attach a new knote to it, and immediately report an event with the 555 * child's pid. 556 */ 557 void 558 knote_fork(struct knlist *list, int pid) 559 { 560 struct kqueue *kq; 561 struct knote *kn; 562 struct kevent kev; 563 int error; 564 565 MPASS(list != NULL); 566 KNL_ASSERT_LOCKED(list); 567 if (SLIST_EMPTY(&list->kl_list)) 568 return; 569 570 memset(&kev, 0, sizeof(kev)); 571 SLIST_FOREACH(kn, &list->kl_list, kn_selnext) { 572 kq = kn->kn_kq; 573 KQ_LOCK(kq); 574 if (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0) { 575 KQ_UNLOCK(kq); 576 continue; 577 } 578 579 /* 580 * The same as knote(), activate the event. 581 */ 582 if ((kn->kn_sfflags & NOTE_TRACK) == 0) { 583 if (kn->kn_fop->f_event(kn, NOTE_FORK)) 584 KNOTE_ACTIVATE(kn, 1); 585 KQ_UNLOCK(kq); 586 continue; 587 } 588 589 /* 590 * The NOTE_TRACK case. In addition to the activation 591 * of the event, we need to register new events to 592 * track the child. Drop the locks in preparation for 593 * the call to kqueue_register(). 594 */ 595 kn_enter_flux(kn); 596 KQ_UNLOCK(kq); 597 list->kl_unlock(list->kl_lockarg); 598 599 /* 600 * Activate existing knote and register tracking knotes with 601 * new process. 602 * 603 * First register a knote to get just the child notice. This 604 * must be a separate note from a potential NOTE_EXIT 605 * notification since both NOTE_CHILD and NOTE_EXIT are defined 606 * to use the data field (in conflicting ways). 607 */ 608 kev.ident = pid; 609 kev.filter = kn->kn_filter; 610 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_ONESHOT | 611 EV_FLAG2; 612 kev.fflags = kn->kn_sfflags; 613 kev.data = kn->kn_id; /* parent */ 614 kev.udata = kn->kn_kevent.udata;/* preserve udata */ 615 error = kqueue_register(kq, &kev, NULL, M_NOWAIT); 616 if (error) 617 kn->kn_fflags |= NOTE_TRACKERR; 618 619 /* 620 * Then register another knote to track other potential events 621 * from the new process. 622 */ 623 kev.ident = pid; 624 kev.filter = kn->kn_filter; 625 kev.flags = kn->kn_flags | EV_ADD | EV_ENABLE | EV_FLAG1; 626 kev.fflags = kn->kn_sfflags; 627 kev.data = kn->kn_id; /* parent */ 628 kev.udata = kn->kn_kevent.udata;/* preserve udata */ 629 error = kqueue_register(kq, &kev, NULL, M_NOWAIT); 630 631 /* 632 * Serialize updates to the kn_kevent fields with threads 633 * scanning the queue. 634 */ 635 list->kl_lock(list->kl_lockarg); 636 if (error) 637 kn->kn_fflags |= NOTE_TRACKERR; 638 if (kn->kn_fop->f_event(kn, NOTE_FORK)) { 639 KQ_LOCK(kq); 640 KNOTE_ACTIVATE(kn, 1); 641 } else { 642 KQ_LOCK(kq); 643 } 644 kn_leave_flux(kn); 645 KQ_UNLOCK_FLUX(kq); 646 } 647 } 648 649 int 650 filt_jailattach(struct knote *kn) 651 { 652 struct prison *pr; 653 654 if (kn->kn_id == 0) { 655 /* Let jid=0 watch the current prison (including prison0). */ 656 pr = curthread->td_ucred->cr_prison; 657 mtx_lock(&pr->pr_mtx); 658 } else { 659 sx_slock(&allprison_lock); 660 pr = prison_find_child(curthread->td_ucred->cr_prison, 661 kn->kn_id); 662 sx_sunlock(&allprison_lock); 663 if (pr == NULL) 664 return (ENOENT); 665 if (!prison_isalive(pr)) { 666 mtx_unlock(&pr->pr_mtx); 667 return (ENOENT); 668 } 669 } 670 kn->kn_ptr.p_prison = pr; 671 kn->kn_flags |= EV_CLEAR; 672 knlist_add(pr->pr_klist, kn, 1); 673 mtx_unlock(&pr->pr_mtx); 674 return (0); 675 } 676 677 void 678 filt_jaildetach(struct knote *kn) 679 { 680 if (kn->kn_ptr.p_prison != NULL) { 681 knlist_remove(kn->kn_knlist, kn, 0); 682 kn->kn_ptr.p_prison = NULL; 683 } else 684 kn->kn_status |= KN_DETACHED; 685 } 686 687 int 688 filt_jail(struct knote *kn, long hint) 689 { 690 struct prison *pr; 691 u_int event; 692 693 pr = kn->kn_ptr.p_prison; 694 if (pr == NULL) /* already activated, from attach filter */ 695 return (0); 696 697 /* 698 * Mask off extra data. In the NOTE_JAIL_CHILD case, that's 699 * everything except the NOTE_JAIL_CHILD bit itself, since a 700 * JID is any positive integer. 701 */ 702 event = ((u_int)hint & NOTE_JAIL_CHILD) ? NOTE_JAIL_CHILD : 703 (u_int)hint & NOTE_JAIL_CTRLMASK; 704 705 /* If the user is interested in this event, record it. */ 706 if (kn->kn_sfflags & event) { 707 kn->kn_fflags |= event; 708 /* Report the created jail id or attached process id. */ 709 if (event == NOTE_JAIL_CHILD || event == NOTE_JAIL_ATTACH) { 710 if (kn->kn_data != 0) 711 kn->kn_fflags |= NOTE_JAIL_MULTI; 712 kn->kn_data = (kn->kn_fflags & NOTE_JAIL_MULTI) ? 0U : 713 (u_int)hint & ~event; 714 } 715 } 716 717 /* Prison is gone, so flag the event as finished. */ 718 if (event == NOTE_JAIL_REMOVE) { 719 kn->kn_flags |= EV_EOF | EV_ONESHOT; 720 kn->kn_ptr.p_prison = NULL; 721 if (kn->kn_fflags == 0) 722 kn->kn_flags |= EV_DROP; 723 return (1); 724 } 725 726 return (kn->kn_fflags != 0); 727 } 728 729 /* 730 * XXX: EVFILT_TIMER should perhaps live in kern_time.c beside the 731 * interval timer support code. 732 */ 733 734 #define NOTE_TIMER_PRECMASK \ 735 (NOTE_SECONDS | NOTE_MSECONDS | NOTE_USECONDS | NOTE_NSECONDS) 736 737 static sbintime_t 738 timer2sbintime(int64_t data, int flags) 739 { 740 int64_t secs; 741 742 /* 743 * Macros for converting to the fractional second portion of an 744 * sbintime_t using 64bit multiplication to improve precision. 745 */ 746 #define NS_TO_SBT(ns) (((ns) * (((uint64_t)1 << 63) / 500000000)) >> 32) 747 #define US_TO_SBT(us) (((us) * (((uint64_t)1 << 63) / 500000)) >> 32) 748 #define MS_TO_SBT(ms) (((ms) * (((uint64_t)1 << 63) / 500)) >> 32) 749 switch (flags & NOTE_TIMER_PRECMASK) { 750 case NOTE_SECONDS: 751 #ifdef __LP64__ 752 if (data > (SBT_MAX / SBT_1S)) 753 return (SBT_MAX); 754 #endif 755 return ((sbintime_t)data << 32); 756 case NOTE_MSECONDS: /* FALLTHROUGH */ 757 case 0: 758 if (data >= 1000) { 759 secs = data / 1000; 760 #ifdef __LP64__ 761 if (secs > (SBT_MAX / SBT_1S)) 762 return (SBT_MAX); 763 #endif 764 return (secs << 32 | MS_TO_SBT(data % 1000)); 765 } 766 return (MS_TO_SBT(data)); 767 case NOTE_USECONDS: 768 if (data >= 1000000) { 769 secs = data / 1000000; 770 #ifdef __LP64__ 771 if (secs > (SBT_MAX / SBT_1S)) 772 return (SBT_MAX); 773 #endif 774 return (secs << 32 | US_TO_SBT(data % 1000000)); 775 } 776 return (US_TO_SBT(data)); 777 case NOTE_NSECONDS: 778 if (data >= 1000000000) { 779 secs = data / 1000000000; 780 #ifdef __LP64__ 781 if (secs > (SBT_MAX / SBT_1S)) 782 return (SBT_MAX); 783 #endif 784 return (secs << 32 | NS_TO_SBT(data % 1000000000)); 785 } 786 return (NS_TO_SBT(data)); 787 default: 788 break; 789 } 790 return (-1); 791 } 792 793 struct kq_timer_cb_data { 794 struct callout c; 795 struct proc *p; 796 struct knote *kn; 797 int cpuid; 798 int flags; 799 TAILQ_ENTRY(kq_timer_cb_data) link; 800 sbintime_t next; /* next timer event fires at */ 801 sbintime_t to; /* precalculated timer period, 0 for abs */ 802 }; 803 804 #define KQ_TIMER_CB_ENQUEUED 0x01 805 806 static void 807 kqtimer_sched_callout(struct kq_timer_cb_data *kc) 808 { 809 callout_reset_sbt_on(&kc->c, kc->next, 0, filt_timerexpire, kc->kn, 810 kc->cpuid, C_ABSOLUTE); 811 } 812 813 void 814 kqtimer_proc_continue(struct proc *p) 815 { 816 struct kq_timer_cb_data *kc, *kc1; 817 struct bintime bt; 818 sbintime_t now; 819 820 PROC_LOCK_ASSERT(p, MA_OWNED); 821 822 getboottimebin(&bt); 823 now = bttosbt(bt); 824 825 TAILQ_FOREACH_SAFE(kc, &p->p_kqtim_stop, link, kc1) { 826 TAILQ_REMOVE(&p->p_kqtim_stop, kc, link); 827 kc->flags &= ~KQ_TIMER_CB_ENQUEUED; 828 if (kc->next <= now) 829 filt_timerexpire_l(kc->kn, true); 830 else 831 kqtimer_sched_callout(kc); 832 } 833 } 834 835 static void 836 filt_timerexpire_l(struct knote *kn, bool proc_locked) 837 { 838 struct kq_timer_cb_data *kc; 839 struct proc *p; 840 uint64_t delta; 841 sbintime_t now; 842 843 kc = kn->kn_ptr.p_v; 844 845 if ((kn->kn_flags & EV_ONESHOT) != 0 || kc->to == 0) { 846 kn->kn_data++; 847 KNOTE_ACTIVATE(kn, 0); 848 return; 849 } 850 851 now = sbinuptime(); 852 if (now >= kc->next) { 853 delta = (now - kc->next) / kc->to; 854 if (delta == 0) 855 delta = 1; 856 kn->kn_data += delta; 857 kc->next += delta * kc->to; 858 if (now >= kc->next) /* overflow */ 859 kc->next = now + kc->to; 860 KNOTE_ACTIVATE(kn, 0); /* XXX - handle locking */ 861 } 862 863 /* 864 * Initial check for stopped kc->p is racy. It is fine to 865 * miss the set of the stop flags, at worst we would schedule 866 * one more callout. On the other hand, it is not fine to not 867 * schedule when we we missed clearing of the flags, we 868 * recheck them under the lock and observe consistent state. 869 */ 870 p = kc->p; 871 if (P_SHOULDSTOP(p) || P_KILLED(p)) { 872 if (!proc_locked) 873 PROC_LOCK(p); 874 if (P_SHOULDSTOP(p) || P_KILLED(p)) { 875 if ((kc->flags & KQ_TIMER_CB_ENQUEUED) == 0) { 876 kc->flags |= KQ_TIMER_CB_ENQUEUED; 877 TAILQ_INSERT_TAIL(&p->p_kqtim_stop, kc, link); 878 } 879 if (!proc_locked) 880 PROC_UNLOCK(p); 881 return; 882 } 883 if (!proc_locked) 884 PROC_UNLOCK(p); 885 } 886 kqtimer_sched_callout(kc); 887 } 888 889 static void 890 filt_timerexpire(void *knx) 891 { 892 filt_timerexpire_l(knx, false); 893 } 894 895 /* 896 * data contains amount of time to sleep 897 */ 898 static int 899 filt_timervalidate(struct knote *kn, sbintime_t *to) 900 { 901 struct bintime bt; 902 sbintime_t sbt; 903 904 if (kn->kn_sdata < 0) 905 return (EINVAL); 906 if (kn->kn_sdata == 0 && (kn->kn_flags & EV_ONESHOT) == 0) 907 kn->kn_sdata = 1; 908 /* 909 * The only fflags values supported are the timer unit 910 * (precision) and the absolute time indicator. 911 */ 912 if ((kn->kn_sfflags & ~(NOTE_TIMER_PRECMASK | NOTE_ABSTIME)) != 0) 913 return (EINVAL); 914 915 *to = timer2sbintime(kn->kn_sdata, kn->kn_sfflags); 916 if (*to < 0) 917 return (EINVAL); 918 if ((kn->kn_sfflags & NOTE_ABSTIME) != 0) { 919 getboottimebin(&bt); 920 sbt = bttosbt(bt); 921 *to = MAX(0, *to - sbt); 922 } 923 return (0); 924 } 925 926 static int 927 filt_timerattach(struct knote *kn) 928 { 929 struct kq_timer_cb_data *kc; 930 sbintime_t to; 931 int error; 932 933 to = -1; 934 error = filt_timervalidate(kn, &to); 935 if (error != 0) 936 return (error); 937 KASSERT(to > 0 || (kn->kn_flags & EV_ONESHOT) != 0 || 938 (kn->kn_sfflags & NOTE_ABSTIME) != 0, 939 ("%s: periodic timer has a calculated zero timeout", __func__)); 940 KASSERT(to >= 0, 941 ("%s: timer has a calculated negative timeout", __func__)); 942 943 if (atomic_fetchadd_int(&kq_ncallouts, 1) + 1 > kq_calloutmax) { 944 atomic_subtract_int(&kq_ncallouts, 1); 945 return (ENOMEM); 946 } 947 948 if ((kn->kn_sfflags & NOTE_ABSTIME) == 0) 949 kn->kn_flags |= EV_CLEAR; /* automatically set */ 950 kn->kn_status &= ~KN_DETACHED; /* knlist_add clears it */ 951 kn->kn_ptr.p_v = kc = malloc(sizeof(*kc), M_KQUEUE, M_WAITOK); 952 kc->kn = kn; 953 kc->p = curproc; 954 kc->cpuid = PCPU_GET(cpuid); 955 kc->flags = 0; 956 callout_init(&kc->c, 1); 957 filt_timerstart(kn, to); 958 959 return (0); 960 } 961 962 static int 963 filt_timercopy(struct knote *kn, struct proc *p) 964 { 965 struct kq_timer_cb_data *kc_src, *kc; 966 967 if (atomic_fetchadd_int(&kq_ncallouts, 1) + 1 > kq_calloutmax) { 968 atomic_subtract_int(&kq_ncallouts, 1); 969 return (ENOMEM); 970 } 971 972 kn->kn_status &= ~KN_DETACHED; 973 kc_src = kn->kn_ptr.p_v; 974 kn->kn_ptr.p_v = kc = malloc(sizeof(*kc), M_KQUEUE, M_WAITOK); 975 kc->kn = kn; 976 kc->p = p; 977 kc->flags = kc_src->flags & ~KQ_TIMER_CB_ENQUEUED; 978 kc->next = kc_src->next; 979 kc->to = kc_src->to; 980 kc->cpuid = PCPU_GET(cpuid); 981 callout_init(&kc->c, 1); 982 kqtimer_sched_callout(kc); 983 return (0); 984 } 985 986 static void 987 filt_timerstart(struct knote *kn, sbintime_t to) 988 { 989 struct kq_timer_cb_data *kc; 990 991 kc = kn->kn_ptr.p_v; 992 if ((kn->kn_sfflags & NOTE_ABSTIME) != 0) { 993 kc->next = to; 994 kc->to = 0; 995 } else { 996 kc->next = to + sbinuptime(); 997 kc->to = to; 998 } 999 kqtimer_sched_callout(kc); 1000 } 1001 1002 static void 1003 filt_timerdetach(struct knote *kn) 1004 { 1005 struct kq_timer_cb_data *kc; 1006 unsigned int old __unused; 1007 bool pending; 1008 1009 kc = kn->kn_ptr.p_v; 1010 do { 1011 callout_drain(&kc->c); 1012 1013 /* 1014 * kqtimer_proc_continue() might have rescheduled this callout. 1015 * Double-check, using the process mutex as an interlock. 1016 */ 1017 PROC_LOCK(kc->p); 1018 if ((kc->flags & KQ_TIMER_CB_ENQUEUED) != 0) { 1019 kc->flags &= ~KQ_TIMER_CB_ENQUEUED; 1020 TAILQ_REMOVE(&kc->p->p_kqtim_stop, kc, link); 1021 } 1022 pending = callout_pending(&kc->c); 1023 PROC_UNLOCK(kc->p); 1024 } while (pending); 1025 free(kc, M_KQUEUE); 1026 old = atomic_fetchadd_int(&kq_ncallouts, -1); 1027 KASSERT(old > 0, ("Number of callouts cannot become negative")); 1028 kn->kn_status |= KN_DETACHED; /* knlist_remove sets it */ 1029 } 1030 1031 static void 1032 filt_timertouch(struct knote *kn, struct kevent *kev, u_long type) 1033 { 1034 struct kq_timer_cb_data *kc; 1035 struct kqueue *kq; 1036 sbintime_t to; 1037 int error; 1038 1039 switch (type) { 1040 case EVENT_REGISTER: 1041 /* Handle re-added timers that update data/fflags */ 1042 if (kev->flags & EV_ADD) { 1043 kc = kn->kn_ptr.p_v; 1044 1045 /* Drain any existing callout. */ 1046 callout_drain(&kc->c); 1047 1048 /* Throw away any existing undelivered record 1049 * of the timer expiration. This is done under 1050 * the presumption that if a process is 1051 * re-adding this timer with new parameters, 1052 * it is no longer interested in what may have 1053 * happened under the old parameters. If it is 1054 * interested, it can wait for the expiration, 1055 * delete the old timer definition, and then 1056 * add the new one. 1057 * 1058 * This has to be done while the kq is locked: 1059 * - if enqueued, dequeue 1060 * - make it no longer active 1061 * - clear the count of expiration events 1062 */ 1063 kq = kn->kn_kq; 1064 KQ_LOCK(kq); 1065 if (kn->kn_status & KN_QUEUED) 1066 knote_dequeue(kn); 1067 1068 kn->kn_status &= ~KN_ACTIVE; 1069 kn->kn_data = 0; 1070 KQ_UNLOCK(kq); 1071 1072 /* Reschedule timer based on new data/fflags */ 1073 kn->kn_sfflags = kev->fflags; 1074 kn->kn_sdata = kev->data; 1075 error = filt_timervalidate(kn, &to); 1076 if (error != 0) { 1077 kn->kn_flags |= EV_ERROR; 1078 kn->kn_data = error; 1079 } else 1080 filt_timerstart(kn, to); 1081 } 1082 break; 1083 1084 case EVENT_PROCESS: 1085 *kev = kn->kn_kevent; 1086 if (kn->kn_flags & EV_CLEAR) { 1087 kn->kn_data = 0; 1088 kn->kn_fflags = 0; 1089 } 1090 break; 1091 1092 default: 1093 panic("filt_timertouch() - invalid type (%ld)", type); 1094 break; 1095 } 1096 } 1097 1098 static int 1099 filt_timer(struct knote *kn, long hint) 1100 { 1101 1102 return (kn->kn_data != 0); 1103 } 1104 1105 static int 1106 filt_userattach(struct knote *kn) 1107 { 1108 1109 /* 1110 * EVFILT_USER knotes are not attached to anything in the kernel. 1111 */ 1112 kn->kn_hook = NULL; 1113 if (kn->kn_fflags & NOTE_TRIGGER) 1114 kn->kn_hookid = 1; 1115 else 1116 kn->kn_hookid = 0; 1117 return (0); 1118 } 1119 1120 static void 1121 filt_userdetach(__unused struct knote *kn) 1122 { 1123 1124 /* 1125 * EVFILT_USER knotes are not attached to anything in the kernel. 1126 */ 1127 } 1128 1129 static int 1130 filt_user(struct knote *kn, __unused long hint) 1131 { 1132 1133 return (kn->kn_hookid); 1134 } 1135 1136 static void 1137 filt_usertouch(struct knote *kn, struct kevent *kev, u_long type) 1138 { 1139 u_int ffctrl; 1140 1141 switch (type) { 1142 case EVENT_REGISTER: 1143 if (kev->fflags & NOTE_TRIGGER) 1144 kn->kn_hookid = 1; 1145 1146 ffctrl = kev->fflags & NOTE_FFCTRLMASK; 1147 kev->fflags &= NOTE_FFLAGSMASK; 1148 switch (ffctrl) { 1149 case NOTE_FFNOP: 1150 break; 1151 1152 case NOTE_FFAND: 1153 kn->kn_sfflags &= kev->fflags; 1154 break; 1155 1156 case NOTE_FFOR: 1157 kn->kn_sfflags |= kev->fflags; 1158 break; 1159 1160 case NOTE_FFCOPY: 1161 kn->kn_sfflags = kev->fflags; 1162 break; 1163 1164 default: 1165 /* XXX Return error? */ 1166 break; 1167 } 1168 kn->kn_sdata = kev->data; 1169 if (kev->flags & EV_CLEAR) { 1170 kn->kn_hookid = 0; 1171 kn->kn_data = 0; 1172 kn->kn_fflags = 0; 1173 } 1174 break; 1175 1176 case EVENT_PROCESS: 1177 *kev = kn->kn_kevent; 1178 kev->fflags = kn->kn_sfflags; 1179 kev->data = kn->kn_sdata; 1180 if (kn->kn_flags & EV_CLEAR) { 1181 kn->kn_hookid = 0; 1182 kn->kn_data = 0; 1183 kn->kn_fflags = 0; 1184 } 1185 break; 1186 1187 default: 1188 panic("filt_usertouch() - invalid type (%ld)", type); 1189 break; 1190 } 1191 } 1192 1193 int 1194 sys_kqueue(struct thread *td, struct kqueue_args *uap) 1195 { 1196 1197 return (kern_kqueue(td, 0, false, NULL)); 1198 } 1199 1200 int 1201 sys_kqueuex(struct thread *td, struct kqueuex_args *uap) 1202 { 1203 int flags; 1204 1205 if ((uap->flags & ~(KQUEUE_CLOEXEC | KQUEUE_CPONFORK)) != 0) 1206 return (EINVAL); 1207 flags = 0; 1208 if ((uap->flags & KQUEUE_CLOEXEC) != 0) 1209 flags |= O_CLOEXEC; 1210 return (kern_kqueue(td, flags, (uap->flags & KQUEUE_CPONFORK) != 0, 1211 NULL)); 1212 } 1213 1214 static void 1215 kqueue_init(struct kqueue *kq, bool cponfork) 1216 { 1217 1218 mtx_init(&kq->kq_lock, "kqueue", NULL, MTX_DEF | MTX_DUPOK); 1219 TAILQ_INIT(&kq->kq_head); 1220 knlist_init_mtx(&kq->kq_sel.si_note, &kq->kq_lock); 1221 TASK_INIT(&kq->kq_task, 0, kqueue_task, kq); 1222 if (cponfork) 1223 kq->kq_state |= KQ_CPONFORK; 1224 } 1225 1226 static int 1227 kern_kqueue_alloc(struct thread *td, struct filedesc *fdp, int *fdip, 1228 struct file **fpp, int flags, struct filecaps *fcaps, bool cponfork, 1229 struct kqueue **kqp) 1230 { 1231 struct ucred *cred; 1232 struct kqueue *kq; 1233 int error; 1234 1235 cred = td->td_ucred; 1236 if (!chgkqcnt(cred->cr_ruidinfo, 1, lim_cur(td, RLIMIT_KQUEUES))) 1237 return (ENOMEM); 1238 1239 error = fdip != NULL ? falloc_caps(td, fpp, fdip, flags, fcaps) : 1240 _falloc_noinstall(td, fpp, 1); 1241 if (error != 0) { 1242 chgkqcnt(cred->cr_ruidinfo, -1, 0); 1243 return (error); 1244 } 1245 1246 /* An extra reference on `fp' has been held for us by falloc(). */ 1247 kq = malloc(sizeof(*kq), M_KQUEUE, M_WAITOK | M_ZERO); 1248 kqueue_init(kq, cponfork); 1249 kq->kq_fdp = fdp; 1250 kq->kq_cred = crhold(cred); 1251 1252 if (fdip != NULL) 1253 FILEDESC_XLOCK(fdp); 1254 TAILQ_INSERT_HEAD(&fdp->fd_kqlist, kq, kq_list); 1255 if (fdip != NULL) 1256 FILEDESC_XUNLOCK(fdp); 1257 1258 finit(*fpp, FREAD | FWRITE, DTYPE_KQUEUE, kq, &kqueueops); 1259 *kqp = kq; 1260 return (0); 1261 } 1262 1263 int 1264 kern_kqueue(struct thread *td, int flags, bool cponfork, struct filecaps *fcaps) 1265 { 1266 struct kqueue *kq; 1267 struct file *fp; 1268 int fd, error; 1269 1270 error = kern_kqueue_alloc(td, td->td_proc->p_fd, &fd, &fp, flags, 1271 fcaps, cponfork, &kq); 1272 if (error != 0) 1273 return (error); 1274 1275 fdrop(fp, td); 1276 1277 td->td_retval[0] = fd; 1278 return (0); 1279 } 1280 1281 struct g_kevent_args { 1282 int fd; 1283 const void *changelist; 1284 int nchanges; 1285 void *eventlist; 1286 int nevents; 1287 const struct timespec *timeout; 1288 }; 1289 1290 int 1291 sys_kevent(struct thread *td, struct kevent_args *uap) 1292 { 1293 struct kevent_copyops k_ops = { 1294 .arg = uap, 1295 .k_copyout = kevent_copyout, 1296 .k_copyin = kevent_copyin, 1297 .kevent_size = sizeof(struct kevent), 1298 }; 1299 struct g_kevent_args gk_args = { 1300 .fd = uap->fd, 1301 .changelist = uap->changelist, 1302 .nchanges = uap->nchanges, 1303 .eventlist = uap->eventlist, 1304 .nevents = uap->nevents, 1305 .timeout = uap->timeout, 1306 }; 1307 1308 return (kern_kevent_generic(td, &gk_args, &k_ops, "kevent")); 1309 } 1310 1311 static int 1312 kern_kevent_generic(struct thread *td, struct g_kevent_args *uap, 1313 struct kevent_copyops *k_ops, const char *struct_name) 1314 { 1315 struct timespec ts, *tsp; 1316 #ifdef KTRACE 1317 struct kevent *eventlist = uap->eventlist; 1318 #endif 1319 int error; 1320 1321 if (uap->timeout != NULL) { 1322 error = copyin(uap->timeout, &ts, sizeof(ts)); 1323 if (error) 1324 return (error); 1325 tsp = &ts; 1326 } else 1327 tsp = NULL; 1328 1329 #ifdef KTRACE 1330 if (KTRPOINT(td, KTR_STRUCT_ARRAY)) 1331 ktrstructarray(struct_name, UIO_USERSPACE, uap->changelist, 1332 uap->nchanges, k_ops->kevent_size); 1333 #endif 1334 1335 error = kern_kevent(td, uap->fd, uap->nchanges, uap->nevents, 1336 k_ops, tsp); 1337 1338 #ifdef KTRACE 1339 if (error == 0 && KTRPOINT(td, KTR_STRUCT_ARRAY)) 1340 ktrstructarray(struct_name, UIO_USERSPACE, eventlist, 1341 td->td_retval[0], k_ops->kevent_size); 1342 #endif 1343 1344 return (error); 1345 } 1346 1347 /* 1348 * Copy 'count' items into the destination list pointed to by uap->eventlist. 1349 */ 1350 static int 1351 kevent_copyout(void *arg, struct kevent *kevp, int count) 1352 { 1353 struct kevent_args *uap; 1354 int error; 1355 1356 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count)); 1357 uap = (struct kevent_args *)arg; 1358 1359 error = copyout(kevp, uap->eventlist, count * sizeof *kevp); 1360 if (error == 0) 1361 uap->eventlist += count; 1362 return (error); 1363 } 1364 1365 /* 1366 * Copy 'count' items from the list pointed to by uap->changelist. 1367 */ 1368 static int 1369 kevent_copyin(void *arg, struct kevent *kevp, int count) 1370 { 1371 struct kevent_args *uap; 1372 int error; 1373 1374 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count)); 1375 uap = (struct kevent_args *)arg; 1376 1377 error = copyin(uap->changelist, kevp, count * sizeof *kevp); 1378 if (error == 0) 1379 uap->changelist += count; 1380 return (error); 1381 } 1382 1383 #ifdef COMPAT_FREEBSD11 1384 static int 1385 kevent11_copyout(void *arg, struct kevent *kevp, int count) 1386 { 1387 struct freebsd11_kevent_args *uap; 1388 struct freebsd11_kevent kev11; 1389 int error, i; 1390 1391 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count)); 1392 uap = (struct freebsd11_kevent_args *)arg; 1393 1394 for (i = 0; i < count; i++) { 1395 kev11.ident = kevp->ident; 1396 kev11.filter = kevp->filter; 1397 kev11.flags = kevp->flags; 1398 kev11.fflags = kevp->fflags; 1399 kev11.data = kevp->data; 1400 kev11.udata = kevp->udata; 1401 error = copyout(&kev11, uap->eventlist, sizeof(kev11)); 1402 if (error != 0) 1403 break; 1404 uap->eventlist++; 1405 kevp++; 1406 } 1407 return (error); 1408 } 1409 1410 /* 1411 * Copy 'count' items from the list pointed to by uap->changelist. 1412 */ 1413 static int 1414 kevent11_copyin(void *arg, struct kevent *kevp, int count) 1415 { 1416 struct freebsd11_kevent_args *uap; 1417 struct freebsd11_kevent kev11; 1418 int error, i; 1419 1420 KASSERT(count <= KQ_NEVENTS, ("count (%d) > KQ_NEVENTS", count)); 1421 uap = (struct freebsd11_kevent_args *)arg; 1422 1423 for (i = 0; i < count; i++) { 1424 error = copyin(uap->changelist, &kev11, sizeof(kev11)); 1425 if (error != 0) 1426 break; 1427 kevp->ident = kev11.ident; 1428 kevp->filter = kev11.filter; 1429 kevp->flags = kev11.flags; 1430 kevp->fflags = kev11.fflags; 1431 kevp->data = (uintptr_t)kev11.data; 1432 kevp->udata = kev11.udata; 1433 bzero(&kevp->ext, sizeof(kevp->ext)); 1434 uap->changelist++; 1435 kevp++; 1436 } 1437 return (error); 1438 } 1439 1440 int 1441 freebsd11_kevent(struct thread *td, struct freebsd11_kevent_args *uap) 1442 { 1443 struct kevent_copyops k_ops = { 1444 .arg = uap, 1445 .k_copyout = kevent11_copyout, 1446 .k_copyin = kevent11_copyin, 1447 .kevent_size = sizeof(struct freebsd11_kevent), 1448 }; 1449 struct g_kevent_args gk_args = { 1450 .fd = uap->fd, 1451 .changelist = uap->changelist, 1452 .nchanges = uap->nchanges, 1453 .eventlist = uap->eventlist, 1454 .nevents = uap->nevents, 1455 .timeout = uap->timeout, 1456 }; 1457 1458 return (kern_kevent_generic(td, &gk_args, &k_ops, "freebsd11_kevent")); 1459 } 1460 #endif 1461 1462 int 1463 kern_kevent(struct thread *td, int fd, int nchanges, int nevents, 1464 struct kevent_copyops *k_ops, const struct timespec *timeout) 1465 { 1466 cap_rights_t rights; 1467 struct file *fp; 1468 int error; 1469 1470 cap_rights_init_zero(&rights); 1471 if (nchanges > 0) 1472 cap_rights_set_one(&rights, CAP_KQUEUE_CHANGE); 1473 if (nevents > 0) 1474 cap_rights_set_one(&rights, CAP_KQUEUE_EVENT); 1475 error = fget(td, fd, &rights, &fp); 1476 if (error != 0) 1477 return (error); 1478 1479 error = kern_kevent_fp(td, fp, nchanges, nevents, k_ops, timeout); 1480 fdrop(fp, td); 1481 1482 return (error); 1483 } 1484 1485 static int 1486 kqueue_kevent(struct kqueue *kq, struct thread *td, int nchanges, int nevents, 1487 struct kevent_copyops *k_ops, const struct timespec *timeout) 1488 { 1489 struct kevent keva[KQ_NEVENTS]; 1490 struct kevent *kevp, *changes; 1491 int i, n, nerrors, error; 1492 1493 if (nchanges < 0) 1494 return (EINVAL); 1495 1496 nerrors = 0; 1497 while (nchanges > 0) { 1498 n = nchanges > KQ_NEVENTS ? KQ_NEVENTS : nchanges; 1499 error = k_ops->k_copyin(k_ops->arg, keva, n); 1500 if (error) 1501 return (error); 1502 changes = keva; 1503 for (i = 0; i < n; i++) { 1504 kevp = &changes[i]; 1505 if (!kevp->filter) 1506 continue; 1507 kevp->flags &= ~EV_SYSFLAGS; 1508 error = kqueue_register(kq, kevp, td, M_WAITOK); 1509 if (error || (kevp->flags & EV_RECEIPT)) { 1510 if (nevents == 0) 1511 return (error); 1512 kevp->flags = EV_ERROR; 1513 kevp->data = error; 1514 (void)k_ops->k_copyout(k_ops->arg, kevp, 1); 1515 nevents--; 1516 nerrors++; 1517 } 1518 } 1519 nchanges -= n; 1520 } 1521 if (nerrors) { 1522 td->td_retval[0] = nerrors; 1523 return (0); 1524 } 1525 1526 return (kqueue_scan(kq, nevents, k_ops, timeout, keva, td)); 1527 } 1528 1529 int 1530 kern_kevent_fp(struct thread *td, struct file *fp, int nchanges, int nevents, 1531 struct kevent_copyops *k_ops, const struct timespec *timeout) 1532 { 1533 struct kqueue *kq; 1534 int error; 1535 1536 error = kqueue_acquire(fp, &kq); 1537 if (error != 0) 1538 return (error); 1539 error = kqueue_kevent(kq, td, nchanges, nevents, k_ops, timeout); 1540 kqueue_release(kq, 0); 1541 return (error); 1542 } 1543 1544 /* 1545 * Performs a kevent() call on a temporarily created kqueue. This can be 1546 * used to perform one-shot polling, similar to poll() and select(). 1547 */ 1548 int 1549 kern_kevent_anonymous(struct thread *td, int nevents, 1550 struct kevent_copyops *k_ops) 1551 { 1552 struct kqueue kq = {}; 1553 int error; 1554 1555 kqueue_init(&kq, false); 1556 kq.kq_refcnt = 1; 1557 error = kqueue_kevent(&kq, td, nevents, nevents, k_ops, NULL); 1558 kqueue_drain(&kq, td); 1559 kqueue_destroy(&kq); 1560 return (error); 1561 } 1562 1563 int 1564 kqueue_add_filteropts(int filt, const struct filterops *filtops) 1565 { 1566 int error; 1567 1568 error = 0; 1569 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) { 1570 printf( 1571 "trying to add a filterop that is out of range: %d is beyond %d\n", 1572 ~filt, EVFILT_SYSCOUNT); 1573 return EINVAL; 1574 } 1575 mtx_lock(&filterops_lock); 1576 if (sysfilt_ops[~filt].for_fop != &null_filtops && 1577 sysfilt_ops[~filt].for_fop != NULL) 1578 error = EEXIST; 1579 else { 1580 sysfilt_ops[~filt].for_fop = filtops; 1581 sysfilt_ops[~filt].for_refcnt = 0; 1582 } 1583 mtx_unlock(&filterops_lock); 1584 1585 return (error); 1586 } 1587 1588 int 1589 kqueue_del_filteropts(int filt) 1590 { 1591 int error; 1592 1593 error = 0; 1594 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) 1595 return EINVAL; 1596 1597 mtx_lock(&filterops_lock); 1598 if (sysfilt_ops[~filt].for_fop == &null_filtops || 1599 sysfilt_ops[~filt].for_fop == NULL) 1600 error = EINVAL; 1601 else if (sysfilt_ops[~filt].for_refcnt != 0) 1602 error = EBUSY; 1603 else { 1604 sysfilt_ops[~filt].for_fop = &null_filtops; 1605 sysfilt_ops[~filt].for_refcnt = 0; 1606 } 1607 mtx_unlock(&filterops_lock); 1608 1609 return error; 1610 } 1611 1612 static const struct filterops * 1613 kqueue_fo_find(int filt) 1614 { 1615 1616 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) 1617 return NULL; 1618 1619 if (sysfilt_ops[~filt].for_nolock) 1620 return sysfilt_ops[~filt].for_fop; 1621 1622 mtx_lock(&filterops_lock); 1623 sysfilt_ops[~filt].for_refcnt++; 1624 if (sysfilt_ops[~filt].for_fop == NULL) 1625 sysfilt_ops[~filt].for_fop = &null_filtops; 1626 mtx_unlock(&filterops_lock); 1627 1628 return sysfilt_ops[~filt].for_fop; 1629 } 1630 1631 static void 1632 kqueue_fo_release(int filt) 1633 { 1634 1635 if (filt > 0 || filt + EVFILT_SYSCOUNT < 0) 1636 return; 1637 1638 if (sysfilt_ops[~filt].for_nolock) 1639 return; 1640 1641 mtx_lock(&filterops_lock); 1642 KASSERT(sysfilt_ops[~filt].for_refcnt > 0, 1643 ("filter object %d refcount not valid on release", filt)); 1644 sysfilt_ops[~filt].for_refcnt--; 1645 mtx_unlock(&filterops_lock); 1646 } 1647 1648 /* 1649 * A ref to kq (obtained via kqueue_acquire) must be held. 1650 */ 1651 static int 1652 kqueue_register(struct kqueue *kq, struct kevent *kev, struct thread *td, 1653 int mflag) 1654 { 1655 const struct filterops *fops; 1656 struct file *fp; 1657 struct knote *kn, *tkn; 1658 struct knlist *knl; 1659 int error, filt, event; 1660 int haskqglobal, filedesc_unlock; 1661 1662 if ((kev->flags & (EV_ENABLE | EV_DISABLE)) == (EV_ENABLE | EV_DISABLE)) 1663 return (EINVAL); 1664 1665 fp = NULL; 1666 kn = NULL; 1667 knl = NULL; 1668 error = 0; 1669 haskqglobal = 0; 1670 filedesc_unlock = 0; 1671 1672 filt = kev->filter; 1673 fops = kqueue_fo_find(filt); 1674 if (fops == NULL) 1675 return EINVAL; 1676 1677 if (kev->flags & EV_ADD) { 1678 /* Reject an invalid flag pair early */ 1679 if (kev->flags & EV_KEEPUDATA) { 1680 tkn = NULL; 1681 error = EINVAL; 1682 goto done; 1683 } 1684 1685 /* 1686 * Prevent waiting with locks. Non-sleepable 1687 * allocation failures are handled in the loop, only 1688 * if the spare knote appears to be actually required. 1689 */ 1690 tkn = knote_alloc(mflag); 1691 } else { 1692 tkn = NULL; 1693 } 1694 1695 findkn: 1696 if (fops->f_isfd) { 1697 KASSERT(td != NULL, ("td is NULL")); 1698 if (kev->ident > INT_MAX) 1699 error = EBADF; 1700 else 1701 error = fget(td, kev->ident, &cap_event_rights, &fp); 1702 if (error) 1703 goto done; 1704 1705 if ((kev->flags & EV_ADD) == EV_ADD && kqueue_expand(kq, fops, 1706 kev->ident, M_NOWAIT) != 0) { 1707 /* try again */ 1708 fdrop(fp, td); 1709 fp = NULL; 1710 error = kqueue_expand(kq, fops, kev->ident, mflag); 1711 if (error) 1712 goto done; 1713 goto findkn; 1714 } 1715 1716 if (fp->f_type == DTYPE_KQUEUE) { 1717 /* 1718 * If we add some intelligence about what we are doing, 1719 * we should be able to support events on ourselves. 1720 * We need to know when we are doing this to prevent 1721 * getting both the knlist lock and the kq lock since 1722 * they are the same thing. 1723 */ 1724 if (fp->f_data == kq) { 1725 error = EINVAL; 1726 goto done; 1727 } 1728 1729 /* 1730 * Pre-lock the filedesc before the global 1731 * lock mutex, see the comment in 1732 * kqueue_close(). 1733 */ 1734 FILEDESC_XLOCK(td->td_proc->p_fd); 1735 filedesc_unlock = 1; 1736 KQ_GLOBAL_LOCK(&kq_global, haskqglobal); 1737 } 1738 1739 KQ_LOCK(kq); 1740 if (kev->ident < kq->kq_knlistsize) { 1741 SLIST_FOREACH(kn, &kq->kq_knlist[kev->ident], kn_link) 1742 if (kev->filter == kn->kn_filter) 1743 break; 1744 } 1745 } else { 1746 if ((kev->flags & EV_ADD) == EV_ADD) { 1747 error = kqueue_expand(kq, fops, kev->ident, mflag); 1748 if (error != 0) 1749 goto done; 1750 } 1751 1752 KQ_LOCK(kq); 1753 1754 /* 1755 * If possible, find an existing knote to use for this kevent. 1756 */ 1757 if (kev->filter == EVFILT_PROC && 1758 (kev->flags & (EV_FLAG1 | EV_FLAG2)) != 0) { 1759 /* This is an internal creation of a process tracking 1760 * note. Don't attempt to coalesce this with an 1761 * existing note. 1762 */ 1763 ; 1764 } else if (kq->kq_knhashmask != 0) { 1765 struct klist *list; 1766 1767 list = &kq->kq_knhash[ 1768 KN_HASH((u_long)kev->ident, kq->kq_knhashmask)]; 1769 SLIST_FOREACH(kn, list, kn_link) 1770 if (kev->ident == kn->kn_id && 1771 kev->filter == kn->kn_filter) 1772 break; 1773 } 1774 } 1775 1776 /* knote is in the process of changing, wait for it to stabilize. */ 1777 if (kn != NULL && kn_in_flux(kn)) { 1778 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal); 1779 if (filedesc_unlock) { 1780 FILEDESC_XUNLOCK(td->td_proc->p_fd); 1781 filedesc_unlock = 0; 1782 } 1783 kq->kq_state |= KQ_FLUXWAIT; 1784 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqflxwt", 0); 1785 if (fp != NULL) { 1786 fdrop(fp, td); 1787 fp = NULL; 1788 } 1789 goto findkn; 1790 } 1791 1792 /* 1793 * kn now contains the matching knote, or NULL if no match 1794 */ 1795 if (kn == NULL) { 1796 if (kev->flags & EV_ADD) { 1797 kn = tkn; 1798 tkn = NULL; 1799 if (kn == NULL) { 1800 KQ_UNLOCK(kq); 1801 error = ENOMEM; 1802 goto done; 1803 } 1804 kn->kn_fp = fp; 1805 kn->kn_kq = kq; 1806 kn->kn_fop = fops; 1807 /* 1808 * apply reference counts to knote structure, and 1809 * do not release it at the end of this routine. 1810 */ 1811 fops = NULL; 1812 fp = NULL; 1813 1814 kn->kn_sfflags = kev->fflags; 1815 kn->kn_sdata = kev->data; 1816 kev->fflags = 0; 1817 kev->data = 0; 1818 kn->kn_kevent = *kev; 1819 kn->kn_kevent.flags &= ~(EV_ADD | EV_DELETE | 1820 EV_ENABLE | EV_DISABLE | EV_FORCEONESHOT); 1821 kn->kn_status = KN_DETACHED; 1822 if ((kev->flags & EV_DISABLE) != 0) 1823 kn->kn_status |= KN_DISABLED; 1824 kn_enter_flux(kn); 1825 1826 error = knote_attach(kn, kq); 1827 KQ_UNLOCK(kq); 1828 if (error != 0) { 1829 tkn = kn; 1830 goto done; 1831 } 1832 1833 if ((error = kn->kn_fop->f_attach(kn)) != 0) { 1834 knote_drop_detached(kn, td); 1835 goto done; 1836 } 1837 knl = kn_list_lock(kn); 1838 goto done_ev_add; 1839 } else { 1840 /* No matching knote and the EV_ADD flag is not set. */ 1841 KQ_UNLOCK(kq); 1842 error = ENOENT; 1843 goto done; 1844 } 1845 } 1846 1847 if (kev->flags & EV_DELETE) { 1848 kn_enter_flux(kn); 1849 KQ_UNLOCK(kq); 1850 knote_drop(kn, td); 1851 goto done; 1852 } 1853 1854 if (kev->flags & EV_FORCEONESHOT) { 1855 kn->kn_flags |= EV_ONESHOT; 1856 KNOTE_ACTIVATE(kn, 1); 1857 } 1858 1859 if ((kev->flags & EV_ENABLE) != 0) 1860 kn->kn_status &= ~KN_DISABLED; 1861 else if ((kev->flags & EV_DISABLE) != 0) 1862 kn->kn_status |= KN_DISABLED; 1863 1864 /* 1865 * The user may change some filter values after the initial EV_ADD, 1866 * but doing so will not reset any filter which has already been 1867 * triggered. 1868 */ 1869 kn->kn_status |= KN_SCAN; 1870 kn_enter_flux(kn); 1871 KQ_UNLOCK(kq); 1872 knl = kn_list_lock(kn); 1873 if ((kev->flags & EV_KEEPUDATA) == 0) 1874 kn->kn_kevent.udata = kev->udata; 1875 if (!fops->f_isfd && fops->f_touch != NULL) { 1876 fops->f_touch(kn, kev, EVENT_REGISTER); 1877 } else { 1878 kn->kn_sfflags = kev->fflags; 1879 kn->kn_sdata = kev->data; 1880 } 1881 1882 done_ev_add: 1883 /* 1884 * We can get here with kn->kn_knlist == NULL. This can happen when 1885 * the initial attach event decides that the event is "completed" 1886 * already, e.g., filt_procattach() is called on a zombie process. It 1887 * will call filt_proc() which will remove it from the list, and NULL 1888 * kn_knlist. 1889 * 1890 * KN_DISABLED will be stable while the knote is in flux, so the 1891 * unlocked read will not race with an update. 1892 */ 1893 if ((kn->kn_status & KN_DISABLED) == 0) 1894 event = kn->kn_fop->f_event(kn, 0); 1895 else 1896 event = 0; 1897 1898 KQ_LOCK(kq); 1899 if (event) 1900 kn->kn_status |= KN_ACTIVE; 1901 if ((kn->kn_status & (KN_ACTIVE | KN_DISABLED | KN_QUEUED)) == 1902 KN_ACTIVE) 1903 knote_enqueue(kn); 1904 kn->kn_status &= ~KN_SCAN; 1905 kn_leave_flux(kn); 1906 kn_list_unlock(knl); 1907 KQ_UNLOCK_FLUX(kq); 1908 1909 done: 1910 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal); 1911 if (filedesc_unlock) 1912 FILEDESC_XUNLOCK(td->td_proc->p_fd); 1913 if (fp != NULL) 1914 fdrop(fp, td); 1915 knote_free(tkn); 1916 if (fops != NULL) 1917 kqueue_fo_release(filt); 1918 return (error); 1919 } 1920 1921 static int 1922 kqueue_acquire_ref(struct kqueue *kq) 1923 { 1924 KQ_LOCK(kq); 1925 if ((kq->kq_state & KQ_CLOSING) == KQ_CLOSING) { 1926 KQ_UNLOCK(kq); 1927 return (EBADF); 1928 } 1929 kq->kq_refcnt++; 1930 KQ_UNLOCK(kq); 1931 return (0); 1932 } 1933 1934 static int 1935 kqueue_acquire(struct file *fp, struct kqueue **kqp) 1936 { 1937 struct kqueue *kq; 1938 int error; 1939 1940 kq = fp->f_data; 1941 if (fp->f_type != DTYPE_KQUEUE || kq == NULL) 1942 return (EINVAL); 1943 error = kqueue_acquire_ref(kq); 1944 if (error == 0) 1945 *kqp = kq; 1946 return (error); 1947 } 1948 1949 static void 1950 kqueue_release(struct kqueue *kq, int locked) 1951 { 1952 if (locked) 1953 KQ_OWNED(kq); 1954 else 1955 KQ_LOCK(kq); 1956 kq->kq_refcnt--; 1957 if (kq->kq_refcnt == 1) 1958 wakeup(&kq->kq_refcnt); 1959 if (!locked) 1960 KQ_UNLOCK(kq); 1961 } 1962 1963 static void 1964 ast_kqueue(struct thread *td, int tda __unused) 1965 { 1966 taskqueue_quiesce(taskqueue_kqueue_ctx); 1967 } 1968 1969 static void 1970 kqueue_schedtask(struct kqueue *kq) 1971 { 1972 KQ_OWNED(kq); 1973 KASSERT(((kq->kq_state & KQ_TASKDRAIN) != KQ_TASKDRAIN), 1974 ("scheduling kqueue task while draining")); 1975 1976 if ((kq->kq_state & KQ_TASKSCHED) != KQ_TASKSCHED) { 1977 taskqueue_enqueue(taskqueue_kqueue_ctx, &kq->kq_task); 1978 kq->kq_state |= KQ_TASKSCHED; 1979 ast_sched(curthread, TDA_KQUEUE); 1980 } 1981 } 1982 1983 /* 1984 * Expand the kq to make sure we have storage for fops/ident pair. 1985 * 1986 * Return 0 on success (or no work necessary), return errno on failure. 1987 */ 1988 static int 1989 kqueue_expand(struct kqueue *kq, const struct filterops *fops, uintptr_t ident, 1990 int mflag) 1991 { 1992 struct klist *list, *tmp_knhash, *to_free; 1993 u_long tmp_knhashmask; 1994 int error, fd, size; 1995 1996 KQ_NOTOWNED(kq); 1997 1998 error = 0; 1999 to_free = NULL; 2000 if (fops->f_isfd) { 2001 fd = ident; 2002 if (kq->kq_knlistsize <= fd) { 2003 size = kq->kq_knlistsize; 2004 while (size <= fd) 2005 size += KQEXTENT; 2006 list = malloc(size * sizeof(*list), M_KQUEUE, mflag); 2007 if (list == NULL) 2008 return ENOMEM; 2009 KQ_LOCK(kq); 2010 if ((kq->kq_state & KQ_CLOSING) != 0) { 2011 to_free = list; 2012 error = EBADF; 2013 } else if (kq->kq_knlistsize > fd) { 2014 to_free = list; 2015 } else { 2016 if (kq->kq_knlist != NULL) { 2017 bcopy(kq->kq_knlist, list, 2018 kq->kq_knlistsize * sizeof(*list)); 2019 to_free = kq->kq_knlist; 2020 kq->kq_knlist = NULL; 2021 } 2022 bzero((caddr_t)list + 2023 kq->kq_knlistsize * sizeof(*list), 2024 (size - kq->kq_knlistsize) * sizeof(*list)); 2025 kq->kq_knlistsize = size; 2026 kq->kq_knlist = list; 2027 } 2028 KQ_UNLOCK(kq); 2029 } 2030 } else { 2031 if (kq->kq_knhashmask == 0) { 2032 tmp_knhash = hashinit_flags(KN_HASHSIZE, M_KQUEUE, 2033 &tmp_knhashmask, (mflag & M_WAITOK) != 0 ? 2034 HASH_WAITOK : HASH_NOWAIT); 2035 if (tmp_knhash == NULL) 2036 return (ENOMEM); 2037 KQ_LOCK(kq); 2038 if ((kq->kq_state & KQ_CLOSING) != 0) { 2039 to_free = tmp_knhash; 2040 error = EBADF; 2041 } else if (kq->kq_knhashmask == 0) { 2042 kq->kq_knhash = tmp_knhash; 2043 kq->kq_knhashmask = tmp_knhashmask; 2044 } else { 2045 to_free = tmp_knhash; 2046 } 2047 KQ_UNLOCK(kq); 2048 } 2049 } 2050 free(to_free, M_KQUEUE); 2051 2052 KQ_NOTOWNED(kq); 2053 return (error); 2054 } 2055 2056 static void 2057 kqueue_task(void *arg, int pending) 2058 { 2059 struct kqueue *kq; 2060 int haskqglobal; 2061 2062 haskqglobal = 0; 2063 kq = arg; 2064 2065 KQ_GLOBAL_LOCK(&kq_global, haskqglobal); 2066 KQ_LOCK(kq); 2067 2068 KNOTE_LOCKED(&kq->kq_sel.si_note, 0); 2069 2070 kq->kq_state &= ~KQ_TASKSCHED; 2071 if ((kq->kq_state & KQ_TASKDRAIN) == KQ_TASKDRAIN) { 2072 wakeup(&kq->kq_state); 2073 } 2074 KQ_UNLOCK(kq); 2075 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal); 2076 } 2077 2078 /* 2079 * Scan, update kn_data (if not ONESHOT), and copyout triggered events. 2080 * We treat KN_MARKER knotes as if they are in flux. 2081 */ 2082 static int 2083 kqueue_scan(struct kqueue *kq, int maxevents, struct kevent_copyops *k_ops, 2084 const struct timespec *tsp, struct kevent *keva, struct thread *td) 2085 { 2086 struct kevent *kevp; 2087 struct knote *kn, *marker; 2088 struct knlist *knl; 2089 sbintime_t asbt, rsbt; 2090 int count, error, haskqglobal, influx, nkev, touch; 2091 2092 count = maxevents; 2093 nkev = 0; 2094 error = 0; 2095 haskqglobal = 0; 2096 2097 if (maxevents == 0) 2098 goto done_nl; 2099 if (maxevents < 0) { 2100 error = EINVAL; 2101 goto done_nl; 2102 } 2103 2104 rsbt = 0; 2105 if (tsp != NULL) { 2106 if (!timespecvalid_interval(tsp)) { 2107 error = EINVAL; 2108 goto done_nl; 2109 } 2110 if (timespecisset(tsp)) { 2111 if (tsp->tv_sec <= INT32_MAX) { 2112 rsbt = tstosbt(*tsp); 2113 if (TIMESEL(&asbt, rsbt)) 2114 asbt += tc_tick_sbt; 2115 if (asbt <= SBT_MAX - rsbt) 2116 asbt += rsbt; 2117 else 2118 asbt = 0; 2119 rsbt >>= tc_precexp; 2120 } else 2121 asbt = 0; 2122 } else 2123 asbt = -1; 2124 } else 2125 asbt = 0; 2126 marker = knote_alloc(M_WAITOK); 2127 marker->kn_status = KN_MARKER; 2128 KQ_LOCK(kq); 2129 2130 retry: 2131 kevp = keva; 2132 if (kq->kq_count == 0) { 2133 if (asbt == -1) { 2134 error = EWOULDBLOCK; 2135 } else { 2136 kq->kq_state |= KQ_SLEEP; 2137 error = msleep_sbt(kq, &kq->kq_lock, PSOCK | PCATCH, 2138 "kqread", asbt, rsbt, C_ABSOLUTE); 2139 } 2140 if (error == 0) 2141 goto retry; 2142 /* don't restart after signals... */ 2143 if (error == ERESTART) 2144 error = EINTR; 2145 else if (error == EWOULDBLOCK) 2146 error = 0; 2147 goto done; 2148 } 2149 2150 TAILQ_INSERT_TAIL(&kq->kq_head, marker, kn_tqe); 2151 influx = 0; 2152 while (count) { 2153 KQ_OWNED(kq); 2154 kn = TAILQ_FIRST(&kq->kq_head); 2155 2156 if ((kn->kn_status == KN_MARKER && kn != marker) || 2157 kn_in_flux(kn)) { 2158 if (influx) { 2159 influx = 0; 2160 KQ_FLUX_WAKEUP(kq); 2161 } 2162 kq->kq_state |= KQ_FLUXWAIT; 2163 error = msleep(kq, &kq->kq_lock, PSOCK, 2164 "kqflxwt", 0); 2165 continue; 2166 } 2167 2168 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe); 2169 if ((kn->kn_status & KN_DISABLED) == KN_DISABLED) { 2170 kn->kn_status &= ~KN_QUEUED; 2171 kq->kq_count--; 2172 continue; 2173 } 2174 if (kn == marker) { 2175 KQ_FLUX_WAKEUP(kq); 2176 if (count == maxevents) 2177 goto retry; 2178 goto done; 2179 } 2180 KASSERT(!kn_in_flux(kn), 2181 ("knote %p is unexpectedly in flux", kn)); 2182 2183 if ((kn->kn_flags & EV_DROP) == EV_DROP) { 2184 kn->kn_status &= ~KN_QUEUED; 2185 kn_enter_flux(kn); 2186 kq->kq_count--; 2187 KQ_UNLOCK(kq); 2188 /* 2189 * We don't need to lock the list since we've 2190 * marked it as in flux. 2191 */ 2192 knote_drop(kn, td); 2193 KQ_LOCK(kq); 2194 continue; 2195 } else if ((kn->kn_flags & EV_ONESHOT) == EV_ONESHOT) { 2196 kn->kn_status &= ~KN_QUEUED; 2197 kn_enter_flux(kn); 2198 kq->kq_count--; 2199 KQ_UNLOCK(kq); 2200 /* 2201 * We don't need to lock the list since we've 2202 * marked the knote as being in flux. 2203 */ 2204 *kevp = kn->kn_kevent; 2205 knote_drop(kn, td); 2206 KQ_LOCK(kq); 2207 kn = NULL; 2208 } else { 2209 kn->kn_status |= KN_SCAN; 2210 kn_enter_flux(kn); 2211 KQ_UNLOCK(kq); 2212 if ((kn->kn_status & KN_KQUEUE) == KN_KQUEUE) 2213 KQ_GLOBAL_LOCK(&kq_global, haskqglobal); 2214 knl = kn_list_lock(kn); 2215 if (kn->kn_fop->f_event(kn, 0) == 0) { 2216 KQ_LOCK(kq); 2217 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal); 2218 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE | 2219 KN_SCAN); 2220 kn_leave_flux(kn); 2221 kq->kq_count--; 2222 kn_list_unlock(knl); 2223 influx = 1; 2224 continue; 2225 } 2226 touch = (!kn->kn_fop->f_isfd && 2227 kn->kn_fop->f_touch != NULL); 2228 if (touch) 2229 kn->kn_fop->f_touch(kn, kevp, EVENT_PROCESS); 2230 else 2231 *kevp = kn->kn_kevent; 2232 KQ_LOCK(kq); 2233 KQ_GLOBAL_UNLOCK(&kq_global, haskqglobal); 2234 if (kn->kn_flags & (EV_CLEAR | EV_DISPATCH)) { 2235 /* 2236 * Manually clear knotes who weren't 2237 * 'touch'ed. 2238 */ 2239 if (touch == 0 && kn->kn_flags & EV_CLEAR) { 2240 kn->kn_data = 0; 2241 kn->kn_fflags = 0; 2242 } 2243 if (kn->kn_flags & EV_DISPATCH) 2244 kn->kn_status |= KN_DISABLED; 2245 kn->kn_status &= ~(KN_QUEUED | KN_ACTIVE); 2246 kq->kq_count--; 2247 } else 2248 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe); 2249 2250 kn->kn_status &= ~KN_SCAN; 2251 kn_leave_flux(kn); 2252 kn_list_unlock(knl); 2253 influx = 1; 2254 } 2255 2256 /* we are returning a copy to the user */ 2257 kevp++; 2258 nkev++; 2259 count--; 2260 2261 if (nkev == KQ_NEVENTS) { 2262 influx = 0; 2263 KQ_UNLOCK_FLUX(kq); 2264 error = k_ops->k_copyout(k_ops->arg, keva, nkev); 2265 nkev = 0; 2266 kevp = keva; 2267 KQ_LOCK(kq); 2268 if (error) 2269 break; 2270 } 2271 } 2272 TAILQ_REMOVE(&kq->kq_head, marker, kn_tqe); 2273 done: 2274 KQ_OWNED(kq); 2275 KQ_UNLOCK_FLUX(kq); 2276 knote_free(marker); 2277 done_nl: 2278 KQ_NOTOWNED(kq); 2279 if (nkev != 0) 2280 error = k_ops->k_copyout(k_ops->arg, keva, nkev); 2281 td->td_retval[0] = maxevents - count; 2282 return (error); 2283 } 2284 2285 /*ARGSUSED*/ 2286 static int 2287 kqueue_ioctl(struct file *fp, u_long cmd, void *data, 2288 struct ucred *active_cred, struct thread *td) 2289 { 2290 /* 2291 * Enabling sigio causes two major problems: 2292 * 1) infinite recursion: 2293 * Synopsys: kevent is being used to track signals and have FIOASYNC 2294 * set. On receipt of a signal this will cause a kqueue to recurse 2295 * into itself over and over. Sending the sigio causes the kqueue 2296 * to become ready, which in turn posts sigio again, forever. 2297 * Solution: this can be solved by setting a flag in the kqueue that 2298 * we have a SIGIO in progress. 2299 * 2) locking problems: 2300 * Synopsys: Kqueue is a leaf subsystem, but adding signalling puts 2301 * us above the proc and pgrp locks. 2302 * Solution: Post a signal using an async mechanism, being sure to 2303 * record a generation count in the delivery so that we do not deliver 2304 * a signal to the wrong process. 2305 * 2306 * Note, these two mechanisms are somewhat mutually exclusive! 2307 */ 2308 #if 0 2309 struct kqueue *kq; 2310 2311 kq = fp->f_data; 2312 switch (cmd) { 2313 case FIOASYNC: 2314 if (*(int *)data) { 2315 kq->kq_state |= KQ_ASYNC; 2316 } else { 2317 kq->kq_state &= ~KQ_ASYNC; 2318 } 2319 return (0); 2320 2321 case FIOSETOWN: 2322 return (fsetown(*(int *)data, &kq->kq_sigio)); 2323 2324 case FIOGETOWN: 2325 *(int *)data = fgetown(&kq->kq_sigio); 2326 return (0); 2327 } 2328 #endif 2329 2330 return (ENOTTY); 2331 } 2332 2333 /*ARGSUSED*/ 2334 static int 2335 kqueue_poll(struct file *fp, int events, struct ucred *active_cred, 2336 struct thread *td) 2337 { 2338 struct kqueue *kq; 2339 int revents = 0; 2340 int error; 2341 2342 if ((error = kqueue_acquire(fp, &kq))) 2343 return POLLERR; 2344 2345 KQ_LOCK(kq); 2346 if (events & (POLLIN | POLLRDNORM)) { 2347 if (kq->kq_count) { 2348 revents |= events & (POLLIN | POLLRDNORM); 2349 } else { 2350 selrecord(td, &kq->kq_sel); 2351 if (SEL_WAITING(&kq->kq_sel)) 2352 kq->kq_state |= KQ_SEL; 2353 } 2354 } 2355 kqueue_release(kq, 1); 2356 KQ_UNLOCK(kq); 2357 return (revents); 2358 } 2359 2360 /*ARGSUSED*/ 2361 static int 2362 kqueue_stat(struct file *fp, struct stat *st, struct ucred *active_cred) 2363 { 2364 2365 bzero((void *)st, sizeof *st); 2366 /* 2367 * We no longer return kq_count because the unlocked value is useless. 2368 * If you spent all this time getting the count, why not spend your 2369 * syscall better by calling kevent? 2370 * 2371 * XXX - This is needed for libc_r. 2372 */ 2373 st->st_mode = S_IFIFO; 2374 return (0); 2375 } 2376 2377 static void 2378 kqueue_drain(struct kqueue *kq, struct thread *td) 2379 { 2380 struct knote *kn; 2381 int i; 2382 2383 KQ_LOCK(kq); 2384 2385 KASSERT((kq->kq_state & KQ_CLOSING) != KQ_CLOSING, 2386 ("kqueue already closing")); 2387 kq->kq_state |= KQ_CLOSING; 2388 if (kq->kq_refcnt > 1) 2389 msleep(&kq->kq_refcnt, &kq->kq_lock, PSOCK, "kqclose", 0); 2390 2391 KASSERT(kq->kq_refcnt == 1, ("other refs are out there!")); 2392 2393 KASSERT(knlist_empty(&kq->kq_sel.si_note), 2394 ("kqueue's knlist not empty")); 2395 2396 for (i = 0; i < kq->kq_knlistsize; i++) { 2397 while ((kn = SLIST_FIRST(&kq->kq_knlist[i])) != NULL) { 2398 if (kn_in_flux(kn)) { 2399 kq->kq_state |= KQ_FLUXWAIT; 2400 msleep(kq, &kq->kq_lock, PSOCK, "kqclo1", 0); 2401 continue; 2402 } 2403 kn_enter_flux(kn); 2404 KQ_UNLOCK(kq); 2405 knote_drop(kn, td); 2406 KQ_LOCK(kq); 2407 } 2408 } 2409 if (kq->kq_knhashmask != 0) { 2410 for (i = 0; i <= kq->kq_knhashmask; i++) { 2411 while ((kn = SLIST_FIRST(&kq->kq_knhash[i])) != NULL) { 2412 if (kn_in_flux(kn)) { 2413 kq->kq_state |= KQ_FLUXWAIT; 2414 msleep(kq, &kq->kq_lock, PSOCK, 2415 "kqclo2", 0); 2416 continue; 2417 } 2418 kn_enter_flux(kn); 2419 KQ_UNLOCK(kq); 2420 knote_drop(kn, td); 2421 KQ_LOCK(kq); 2422 } 2423 } 2424 } 2425 2426 if ((kq->kq_state & KQ_TASKSCHED) == KQ_TASKSCHED) { 2427 kq->kq_state |= KQ_TASKDRAIN; 2428 msleep(&kq->kq_state, &kq->kq_lock, PSOCK, "kqtqdr", 0); 2429 } 2430 2431 if ((kq->kq_state & KQ_SEL) == KQ_SEL) { 2432 selwakeuppri(&kq->kq_sel, PSOCK); 2433 if (!SEL_WAITING(&kq->kq_sel)) 2434 kq->kq_state &= ~KQ_SEL; 2435 } 2436 2437 KQ_UNLOCK(kq); 2438 } 2439 2440 static void 2441 kqueue_destroy(struct kqueue *kq) 2442 { 2443 2444 KASSERT(kq->kq_fdp == NULL, 2445 ("kqueue still attached to a file descriptor")); 2446 seldrain(&kq->kq_sel); 2447 knlist_destroy(&kq->kq_sel.si_note); 2448 mtx_destroy(&kq->kq_lock); 2449 2450 if (kq->kq_knhash != NULL) 2451 free(kq->kq_knhash, M_KQUEUE); 2452 if (kq->kq_knlist != NULL) 2453 free(kq->kq_knlist, M_KQUEUE); 2454 2455 funsetown(&kq->kq_sigio); 2456 } 2457 2458 /*ARGSUSED*/ 2459 static int 2460 kqueue_close(struct file *fp, struct thread *td) 2461 { 2462 struct kqueue *kq = fp->f_data; 2463 struct filedesc *fdp; 2464 int error; 2465 int filedesc_unlock; 2466 2467 if ((error = kqueue_acquire(fp, &kq))) 2468 return error; 2469 kqueue_drain(kq, td); 2470 2471 /* 2472 * We could be called due to the knote_drop() doing fdrop(), 2473 * called from kqueue_register(). In this case the global 2474 * lock is owned, and filedesc sx is locked before, to not 2475 * take the sleepable lock after non-sleepable. 2476 */ 2477 fdp = kq->kq_fdp; 2478 kq->kq_fdp = NULL; 2479 if (!sx_xlocked(FILEDESC_LOCK(fdp))) { 2480 FILEDESC_XLOCK(fdp); 2481 filedesc_unlock = 1; 2482 } else 2483 filedesc_unlock = 0; 2484 TAILQ_REMOVE(&fdp->fd_kqlist, kq, kq_list); 2485 if (filedesc_unlock) 2486 FILEDESC_XUNLOCK(fdp); 2487 2488 kqueue_destroy(kq); 2489 chgkqcnt(kq->kq_cred->cr_ruidinfo, -1, 0); 2490 crfree(kq->kq_cred); 2491 free(kq, M_KQUEUE); 2492 fp->f_data = NULL; 2493 2494 return (0); 2495 } 2496 2497 static int 2498 kqueue_fill_kinfo(struct file *fp, struct kinfo_file *kif, struct filedesc *fdp) 2499 { 2500 struct kqueue *kq = fp->f_data; 2501 2502 kif->kf_type = KF_TYPE_KQUEUE; 2503 kif->kf_un.kf_kqueue.kf_kqueue_addr = (uintptr_t)kq; 2504 kif->kf_un.kf_kqueue.kf_kqueue_count = kq->kq_count; 2505 kif->kf_un.kf_kqueue.kf_kqueue_state = kq->kq_state; 2506 return (0); 2507 } 2508 2509 static void 2510 kqueue_wakeup(struct kqueue *kq) 2511 { 2512 KQ_OWNED(kq); 2513 2514 if ((kq->kq_state & KQ_SLEEP) == KQ_SLEEP) { 2515 kq->kq_state &= ~KQ_SLEEP; 2516 wakeup(kq); 2517 } 2518 if ((kq->kq_state & KQ_SEL) == KQ_SEL) { 2519 selwakeuppri(&kq->kq_sel, PSOCK); 2520 if (!SEL_WAITING(&kq->kq_sel)) 2521 kq->kq_state &= ~KQ_SEL; 2522 } 2523 if (!knlist_empty(&kq->kq_sel.si_note)) 2524 kqueue_schedtask(kq); 2525 if ((kq->kq_state & KQ_ASYNC) == KQ_ASYNC) { 2526 pgsigio(&kq->kq_sigio, SIGIO, 0); 2527 } 2528 } 2529 2530 /* 2531 * Walk down a list of knotes, activating them if their event has triggered. 2532 * 2533 * There is a possibility to optimize in the case of one kq watching another. 2534 * Instead of scheduling a task to wake it up, you could pass enough state 2535 * down the chain to make up the parent kqueue. Make this code functional 2536 * first. 2537 */ 2538 void 2539 knote(struct knlist *list, long hint, int lockflags) 2540 { 2541 struct kqueue *kq; 2542 struct knote *kn, *tkn; 2543 int error; 2544 2545 if (list == NULL) 2546 return; 2547 2548 KNL_ASSERT_LOCK(list, lockflags & KNF_LISTLOCKED); 2549 2550 if ((lockflags & KNF_LISTLOCKED) == 0) 2551 list->kl_lock(list->kl_lockarg); 2552 2553 /* 2554 * If we unlock the list lock (and enter influx), we can 2555 * eliminate the kqueue scheduling, but this will introduce 2556 * four lock/unlock's for each knote to test. Also, marker 2557 * would be needed to keep iteration position, since filters 2558 * or other threads could remove events. 2559 */ 2560 SLIST_FOREACH_SAFE(kn, &list->kl_list, kn_selnext, tkn) { 2561 kq = kn->kn_kq; 2562 KQ_LOCK(kq); 2563 if (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0) { 2564 /* 2565 * Do not process the influx notes, except for 2566 * the influx coming from the kq unlock in the 2567 * kqueue_scan(). In the later case, we do 2568 * not interfere with the scan, since the code 2569 * fragment in kqueue_scan() locks the knlist, 2570 * and cannot proceed until we finished. 2571 */ 2572 KQ_UNLOCK(kq); 2573 } else if ((lockflags & KNF_NOKQLOCK) != 0) { 2574 kn_enter_flux(kn); 2575 KQ_UNLOCK(kq); 2576 error = kn->kn_fop->f_event(kn, hint); 2577 KQ_LOCK(kq); 2578 kn_leave_flux(kn); 2579 if (error) 2580 KNOTE_ACTIVATE(kn, 1); 2581 KQ_UNLOCK_FLUX(kq); 2582 } else { 2583 if (kn->kn_fop->f_event(kn, hint)) 2584 KNOTE_ACTIVATE(kn, 1); 2585 KQ_UNLOCK(kq); 2586 } 2587 } 2588 if ((lockflags & KNF_LISTLOCKED) == 0) 2589 list->kl_unlock(list->kl_lockarg); 2590 } 2591 2592 /* 2593 * add a knote to a knlist 2594 */ 2595 void 2596 knlist_add(struct knlist *knl, struct knote *kn, int islocked) 2597 { 2598 2599 KNL_ASSERT_LOCK(knl, islocked); 2600 KQ_NOTOWNED(kn->kn_kq); 2601 KASSERT(kn_in_flux(kn), ("knote %p not in flux", kn)); 2602 KASSERT((kn->kn_status & KN_DETACHED) != 0, 2603 ("knote %p was not detached", kn)); 2604 if (!islocked) 2605 knl->kl_lock(knl->kl_lockarg); 2606 SLIST_INSERT_HEAD(&knl->kl_list, kn, kn_selnext); 2607 if (!islocked) 2608 knl->kl_unlock(knl->kl_lockarg); 2609 KQ_LOCK(kn->kn_kq); 2610 kn->kn_knlist = knl; 2611 kn->kn_status &= ~KN_DETACHED; 2612 KQ_UNLOCK(kn->kn_kq); 2613 } 2614 2615 static void 2616 knlist_remove_kq(struct knlist *knl, struct knote *kn, int knlislocked, 2617 int kqislocked) 2618 { 2619 2620 KASSERT(!kqislocked || knlislocked, ("kq locked w/o knl locked")); 2621 KNL_ASSERT_LOCK(knl, knlislocked); 2622 mtx_assert(&kn->kn_kq->kq_lock, kqislocked ? MA_OWNED : MA_NOTOWNED); 2623 KASSERT(kqislocked || kn_in_flux(kn), ("knote %p not in flux", kn)); 2624 KASSERT((kn->kn_status & KN_DETACHED) == 0, 2625 ("knote %p was already detached", kn)); 2626 if (!knlislocked) 2627 knl->kl_lock(knl->kl_lockarg); 2628 SLIST_REMOVE(&knl->kl_list, kn, knote, kn_selnext); 2629 kn->kn_knlist = NULL; 2630 if (!knlislocked) 2631 kn_list_unlock(knl); 2632 if (!kqislocked) 2633 KQ_LOCK(kn->kn_kq); 2634 kn->kn_status |= KN_DETACHED; 2635 if (!kqislocked) 2636 KQ_UNLOCK(kn->kn_kq); 2637 } 2638 2639 /* 2640 * remove knote from the specified knlist 2641 */ 2642 void 2643 knlist_remove(struct knlist *knl, struct knote *kn, int islocked) 2644 { 2645 2646 knlist_remove_kq(knl, kn, islocked, 0); 2647 } 2648 2649 int 2650 knlist_empty(struct knlist *knl) 2651 { 2652 2653 KNL_ASSERT_LOCKED(knl); 2654 return (SLIST_EMPTY(&knl->kl_list)); 2655 } 2656 2657 static struct mtx knlist_lock; 2658 MTX_SYSINIT(knlist_lock, &knlist_lock, "knlist lock for lockless objects", 2659 MTX_DEF); 2660 static void knlist_mtx_lock(void *arg); 2661 static void knlist_mtx_unlock(void *arg); 2662 2663 static void 2664 knlist_mtx_lock(void *arg) 2665 { 2666 2667 mtx_lock((struct mtx *)arg); 2668 } 2669 2670 static void 2671 knlist_mtx_unlock(void *arg) 2672 { 2673 2674 mtx_unlock((struct mtx *)arg); 2675 } 2676 2677 static void 2678 knlist_mtx_assert_lock(void *arg, int what) 2679 { 2680 2681 if (what == LA_LOCKED) 2682 mtx_assert((struct mtx *)arg, MA_OWNED); 2683 else 2684 mtx_assert((struct mtx *)arg, MA_NOTOWNED); 2685 } 2686 2687 void 2688 knlist_init(struct knlist *knl, void *lock, void (*kl_lock)(void *), 2689 void (*kl_unlock)(void *), 2690 void (*kl_assert_lock)(void *, int)) 2691 { 2692 2693 if (lock == NULL) 2694 knl->kl_lockarg = &knlist_lock; 2695 else 2696 knl->kl_lockarg = lock; 2697 2698 if (kl_lock == NULL) 2699 knl->kl_lock = knlist_mtx_lock; 2700 else 2701 knl->kl_lock = kl_lock; 2702 if (kl_unlock == NULL) 2703 knl->kl_unlock = knlist_mtx_unlock; 2704 else 2705 knl->kl_unlock = kl_unlock; 2706 if (kl_assert_lock == NULL) 2707 knl->kl_assert_lock = knlist_mtx_assert_lock; 2708 else 2709 knl->kl_assert_lock = kl_assert_lock; 2710 2711 knl->kl_autodestroy = 0; 2712 SLIST_INIT(&knl->kl_list); 2713 } 2714 2715 void 2716 knlist_init_mtx(struct knlist *knl, struct mtx *lock) 2717 { 2718 2719 knlist_init(knl, lock, NULL, NULL, NULL); 2720 } 2721 2722 struct knlist * 2723 knlist_alloc(struct mtx *lock) 2724 { 2725 struct knlist *knl; 2726 2727 knl = malloc(sizeof(struct knlist), M_KQUEUE, M_WAITOK); 2728 knlist_init_mtx(knl, lock); 2729 return (knl); 2730 } 2731 2732 void 2733 knlist_destroy(struct knlist *knl) 2734 { 2735 2736 KASSERT(KNLIST_EMPTY(knl), 2737 ("destroying knlist %p with knotes on it", knl)); 2738 } 2739 2740 void 2741 knlist_detach(struct knlist *knl) 2742 { 2743 2744 KNL_ASSERT_LOCKED(knl); 2745 knl->kl_autodestroy = 1; 2746 if (knlist_empty(knl)) { 2747 knlist_destroy(knl); 2748 free(knl, M_KQUEUE); 2749 } 2750 } 2751 2752 /* 2753 * Even if we are locked, we may need to drop the lock to allow any influx 2754 * knotes time to "settle". 2755 */ 2756 void 2757 knlist_cleardel(struct knlist *knl, struct thread *td, int islocked, int killkn) 2758 { 2759 struct knote *kn, *kn2; 2760 struct kqueue *kq; 2761 2762 KASSERT(!knl->kl_autodestroy, ("cleardel for autodestroy %p", knl)); 2763 if (islocked) 2764 KNL_ASSERT_LOCKED(knl); 2765 else { 2766 KNL_ASSERT_UNLOCKED(knl); 2767 again: /* need to reacquire lock since we have dropped it */ 2768 knl->kl_lock(knl->kl_lockarg); 2769 } 2770 2771 SLIST_FOREACH_SAFE(kn, &knl->kl_list, kn_selnext, kn2) { 2772 kq = kn->kn_kq; 2773 KQ_LOCK(kq); 2774 if (kn_in_flux(kn)) { 2775 KQ_UNLOCK(kq); 2776 continue; 2777 } 2778 knlist_remove_kq(knl, kn, 1, 1); 2779 if (killkn) { 2780 kn_enter_flux(kn); 2781 KQ_UNLOCK(kq); 2782 knote_drop_detached(kn, td); 2783 } else { 2784 /* Make sure cleared knotes disappear soon */ 2785 kn->kn_flags |= EV_EOF | EV_ONESHOT; 2786 KQ_UNLOCK(kq); 2787 } 2788 kq = NULL; 2789 } 2790 2791 if (!SLIST_EMPTY(&knl->kl_list)) { 2792 /* there are still in flux knotes remaining */ 2793 kn = SLIST_FIRST(&knl->kl_list); 2794 kq = kn->kn_kq; 2795 KQ_LOCK(kq); 2796 KASSERT(kn_in_flux(kn), ("knote removed w/o list lock")); 2797 knl->kl_unlock(knl->kl_lockarg); 2798 kq->kq_state |= KQ_FLUXWAIT; 2799 msleep(kq, &kq->kq_lock, PSOCK | PDROP, "kqkclr", 0); 2800 kq = NULL; 2801 goto again; 2802 } 2803 2804 if (islocked) 2805 KNL_ASSERT_LOCKED(knl); 2806 else { 2807 knl->kl_unlock(knl->kl_lockarg); 2808 KNL_ASSERT_UNLOCKED(knl); 2809 } 2810 } 2811 2812 /* 2813 * Remove all knotes referencing a specified fd must be called with FILEDESC 2814 * lock. This prevents a race where a new fd comes along and occupies the 2815 * entry and we attach a knote to the fd. 2816 */ 2817 void 2818 knote_fdclose(struct thread *td, int fd) 2819 { 2820 struct filedesc *fdp = td->td_proc->p_fd; 2821 struct kqueue *kq; 2822 struct knote *kn; 2823 int influx; 2824 2825 FILEDESC_XLOCK_ASSERT(fdp); 2826 2827 /* 2828 * We shouldn't have to worry about new kevents appearing on fd 2829 * since filedesc is locked. 2830 */ 2831 TAILQ_FOREACH(kq, &fdp->fd_kqlist, kq_list) { 2832 KQ_LOCK(kq); 2833 2834 again: 2835 influx = 0; 2836 while (kq->kq_knlistsize > fd && 2837 (kn = SLIST_FIRST(&kq->kq_knlist[fd])) != NULL) { 2838 if (kn_in_flux(kn)) { 2839 /* someone else might be waiting on our knote */ 2840 if (influx) 2841 wakeup(kq); 2842 kq->kq_state |= KQ_FLUXWAIT; 2843 msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0); 2844 goto again; 2845 } 2846 kn_enter_flux(kn); 2847 KQ_UNLOCK(kq); 2848 influx = 1; 2849 knote_drop(kn, td); 2850 KQ_LOCK(kq); 2851 } 2852 KQ_UNLOCK_FLUX(kq); 2853 } 2854 } 2855 2856 static int 2857 knote_attach(struct knote *kn, struct kqueue *kq) 2858 { 2859 struct klist *list; 2860 2861 KASSERT(kn_in_flux(kn), ("knote %p not marked influx", kn)); 2862 KQ_OWNED(kq); 2863 2864 if ((kq->kq_state & KQ_CLOSING) != 0) 2865 return (EBADF); 2866 if (kn->kn_fop->f_isfd) { 2867 if (kn->kn_id >= kq->kq_knlistsize) 2868 return (ENOMEM); 2869 list = &kq->kq_knlist[kn->kn_id]; 2870 } else { 2871 if (kq->kq_knhash == NULL) 2872 return (ENOMEM); 2873 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)]; 2874 } 2875 SLIST_INSERT_HEAD(list, kn, kn_link); 2876 return (0); 2877 } 2878 2879 static void 2880 knote_drop(struct knote *kn, struct thread *td) 2881 { 2882 2883 if ((kn->kn_status & KN_DETACHED) == 0) 2884 kn->kn_fop->f_detach(kn); 2885 knote_drop_detached(kn, td); 2886 } 2887 2888 static void 2889 knote_drop_detached(struct knote *kn, struct thread *td) 2890 { 2891 struct kqueue *kq; 2892 struct klist *list; 2893 2894 kq = kn->kn_kq; 2895 2896 KASSERT((kn->kn_status & KN_DETACHED) != 0, 2897 ("knote %p still attached", kn)); 2898 KQ_NOTOWNED(kq); 2899 2900 KQ_LOCK(kq); 2901 for (;;) { 2902 KASSERT(kn->kn_influx >= 1, 2903 ("knote_drop called on %p with influx %d", 2904 kn, kn->kn_influx)); 2905 if (kn->kn_influx == 1) 2906 break; 2907 kq->kq_state |= KQ_FLUXWAIT; 2908 msleep(kq, &kq->kq_lock, PSOCK, "kqflxwt", 0); 2909 } 2910 2911 if (kn->kn_fop->f_isfd) 2912 list = &kq->kq_knlist[kn->kn_id]; 2913 else 2914 list = &kq->kq_knhash[KN_HASH(kn->kn_id, kq->kq_knhashmask)]; 2915 2916 if (!SLIST_EMPTY(list)) 2917 SLIST_REMOVE(list, kn, knote, kn_link); 2918 if (kn->kn_status & KN_QUEUED) 2919 knote_dequeue(kn); 2920 KQ_UNLOCK_FLUX(kq); 2921 2922 if (kn->kn_fop->f_isfd) { 2923 fdrop(kn->kn_fp, td); 2924 kn->kn_fp = NULL; 2925 } 2926 kqueue_fo_release(kn->kn_kevent.filter); 2927 kn->kn_fop = NULL; 2928 knote_free(kn); 2929 } 2930 2931 static void 2932 knote_enqueue(struct knote *kn) 2933 { 2934 struct kqueue *kq = kn->kn_kq; 2935 2936 KQ_OWNED(kn->kn_kq); 2937 KASSERT((kn->kn_status & KN_QUEUED) == 0, ("knote already queued")); 2938 2939 TAILQ_INSERT_TAIL(&kq->kq_head, kn, kn_tqe); 2940 kn->kn_status |= KN_QUEUED; 2941 kq->kq_count++; 2942 kqueue_wakeup(kq); 2943 } 2944 2945 static void 2946 knote_dequeue(struct knote *kn) 2947 { 2948 struct kqueue *kq = kn->kn_kq; 2949 2950 KQ_OWNED(kn->kn_kq); 2951 KASSERT(kn->kn_status & KN_QUEUED, ("knote not queued")); 2952 2953 TAILQ_REMOVE(&kq->kq_head, kn, kn_tqe); 2954 kn->kn_status &= ~KN_QUEUED; 2955 kq->kq_count--; 2956 } 2957 2958 static void 2959 knote_init(void *dummy __unused) 2960 { 2961 2962 knote_zone = uma_zcreate("KNOTE", sizeof(struct knote), NULL, NULL, 2963 NULL, NULL, UMA_ALIGN_PTR, 0); 2964 ast_register(TDA_KQUEUE, ASTR_ASTF_REQUIRED, 0, ast_kqueue); 2965 prison0.pr_klist = knlist_alloc(&prison0.pr_mtx); 2966 } 2967 SYSINIT(knote, SI_SUB_PSEUDO, SI_ORDER_ANY, knote_init, NULL); 2968 2969 static struct knote * 2970 knote_alloc(int mflag) 2971 { 2972 2973 return (uma_zalloc(knote_zone, mflag | M_ZERO)); 2974 } 2975 2976 static void 2977 knote_free(struct knote *kn) 2978 { 2979 2980 uma_zfree(knote_zone, kn); 2981 } 2982 2983 /* 2984 * Register the kev w/ the kq specified by fd. 2985 */ 2986 int 2987 kqfd_register(int fd, struct kevent *kev, struct thread *td, int mflag) 2988 { 2989 struct kqueue *kq; 2990 struct file *fp; 2991 cap_rights_t rights; 2992 int error; 2993 2994 error = fget(td, fd, cap_rights_init_one(&rights, CAP_KQUEUE_CHANGE), 2995 &fp); 2996 if (error != 0) 2997 return (error); 2998 if ((error = kqueue_acquire(fp, &kq)) != 0) 2999 goto noacquire; 3000 3001 error = kqueue_register(kq, kev, td, mflag); 3002 kqueue_release(kq, 0); 3003 3004 noacquire: 3005 fdrop(fp, td); 3006 return (error); 3007 } 3008 3009 static int 3010 kqueue_fork_alloc(struct filedesc *fdp, struct file *fp, struct file **fp1, 3011 struct thread *td) 3012 { 3013 struct kqueue *kq, *kq1; 3014 int error; 3015 3016 MPASS(fp->f_type == DTYPE_KQUEUE); 3017 kq = fp->f_data; 3018 if ((kq->kq_state & KQ_CPONFORK) == 0) 3019 return (EOPNOTSUPP); 3020 error = kqueue_acquire_ref(kq); 3021 if (error != 0) 3022 return (error); 3023 error = kern_kqueue_alloc(td, fdp, NULL, fp1, 0, NULL, true, &kq1); 3024 if (error == 0) { 3025 kq1->kq_forksrc = kq; 3026 (*fp1)->f_flag = fp->f_flag & (FREAD | FWRITE | FEXEC | 3027 O_CLOEXEC | O_CLOFORK); 3028 } else { 3029 kqueue_release(kq, 0); 3030 } 3031 return (error); 3032 } 3033 3034 static void 3035 kqueue_fork_copy_knote(struct kqueue *kq1, struct knote *kn, struct proc *p1, 3036 struct filedesc *fdp) 3037 { 3038 struct knote *kn1; 3039 const struct filterops *fop; 3040 int error; 3041 3042 fop = kn->kn_fop; 3043 if (fop->f_copy == NULL || (fop->f_isfd && 3044 fdp->fd_files->fdt_ofiles[kn->kn_kevent.ident].fde_file == NULL)) 3045 return; 3046 error = kqueue_expand(kq1, fop, kn->kn_kevent.ident, M_WAITOK); 3047 if (error != 0) 3048 return; 3049 3050 kn1 = knote_alloc(M_WAITOK); 3051 *kn1 = *kn; 3052 kn1->kn_status |= KN_DETACHED; 3053 kn1->kn_status &= ~KN_QUEUED; 3054 kn1->kn_kq = kq1; 3055 error = fop->f_copy(kn1, p1); 3056 if (error != 0) { 3057 knote_free(kn1); 3058 return; 3059 } 3060 (void)kqueue_fo_find(kn->kn_kevent.filter); 3061 if (fop->f_isfd && !fhold(kn1->kn_fp)) { 3062 fop->f_detach(kn1); 3063 kqueue_fo_release(kn->kn_kevent.filter); 3064 knote_free(kn1); 3065 return; 3066 } 3067 if (kn->kn_knlist != NULL) 3068 knlist_add(kn->kn_knlist, kn1, 0); 3069 KQ_LOCK(kq1); 3070 knote_attach(kn1, kq1); 3071 kn1->kn_influx = 0; 3072 if ((kn->kn_status & KN_QUEUED) != 0) 3073 knote_enqueue(kn1); 3074 KQ_UNLOCK(kq1); 3075 } 3076 3077 static void 3078 kqueue_fork_copy_list(struct klist *knlist, struct knote *marker, 3079 struct kqueue *kq, struct kqueue *kq1, struct proc *p1, 3080 struct filedesc *fdp) 3081 { 3082 struct knote *kn; 3083 3084 KQ_OWNED(kq); 3085 kn = SLIST_FIRST(knlist); 3086 while (kn != NULL) { 3087 if ((kn->kn_status & KN_DETACHED) != 0 || 3088 (kn_in_flux(kn) && (kn->kn_status & KN_SCAN) == 0)) { 3089 kn = SLIST_NEXT(kn, kn_link); 3090 continue; 3091 } 3092 kn_enter_flux(kn); 3093 SLIST_INSERT_AFTER(kn, marker, kn_link); 3094 KQ_UNLOCK(kq); 3095 kqueue_fork_copy_knote(kq1, kn, p1, fdp); 3096 KQ_LOCK(kq); 3097 kn_leave_flux(kn); 3098 kn = SLIST_NEXT(marker, kn_link); 3099 /* XXXKIB switch kn_link to LIST? */ 3100 SLIST_REMOVE(knlist, marker, knote, kn_link); 3101 } 3102 } 3103 3104 static int 3105 kqueue_fork_copy(struct filedesc *fdp, struct file *fp, struct file *fp1, 3106 struct proc *p1, struct thread *td) 3107 { 3108 struct kqueue *kq, *kq1; 3109 struct knote *marker; 3110 int error, i; 3111 3112 error = 0; 3113 MPASS(fp == NULL); 3114 MPASS(fp1->f_type == DTYPE_KQUEUE); 3115 3116 kq1 = fp1->f_data; 3117 kq = kq1->kq_forksrc; 3118 marker = knote_alloc(M_WAITOK); 3119 marker->kn_status = KN_MARKER; 3120 3121 KQ_LOCK(kq); 3122 for (i = 0; i < kq->kq_knlistsize; i++) { 3123 kqueue_fork_copy_list(&kq->kq_knlist[i], marker, kq, kq1, 3124 p1, fdp); 3125 } 3126 if (kq->kq_knhashmask != 0) { 3127 for (i = 0; i <= kq->kq_knhashmask; i++) { 3128 kqueue_fork_copy_list(&kq->kq_knhash[i], marker, kq, 3129 kq1, p1, fdp); 3130 } 3131 } 3132 kqueue_release(kq, 1); 3133 kq1->kq_forksrc = NULL; 3134 KQ_UNLOCK(kq); 3135 3136 knote_free(marker); 3137 return (error); 3138 } 3139 3140 static int 3141 kqueue_fork(struct filedesc *fdp, struct file *fp, struct file **fp1, 3142 struct proc *p1, struct thread *td) 3143 { 3144 if (*fp1 == NULL) 3145 return (kqueue_fork_alloc(fdp, fp, fp1, td)); 3146 return (kqueue_fork_copy(fdp, fp, *fp1, p1, td)); 3147 } 3148 3149 int 3150 knote_triv_copy(struct knote *kn __unused, struct proc *p1 __unused) 3151 { 3152 return (0); 3153 } 3154 3155 struct knote_status_export_bit { 3156 int kn_status_bit; 3157 int knt_status_bit; 3158 }; 3159 3160 #define ST(name) \ 3161 { .kn_status_bit = KN_##name, .knt_status_bit = KNOTE_STATUS_##name } 3162 static const struct knote_status_export_bit knote_status_export_bits[] = { 3163 ST(ACTIVE), 3164 ST(QUEUED), 3165 ST(DISABLED), 3166 ST(DETACHED), 3167 ST(KQUEUE), 3168 }; 3169 #undef ST 3170 3171 static int 3172 knote_status_export(int kn_status) 3173 { 3174 const struct knote_status_export_bit *b; 3175 unsigned i; 3176 int res; 3177 3178 res = 0; 3179 for (i = 0; i < nitems(knote_status_export_bits); i++) { 3180 b = &knote_status_export_bits[i]; 3181 if ((kn_status & b->kn_status_bit) != 0) 3182 res |= b->knt_status_bit; 3183 } 3184 return (res); 3185 } 3186 3187 static int 3188 kern_proc_kqueue_report_one(struct sbuf *s, struct proc *p, 3189 int kq_fd, struct kqueue *kq, struct knote *kn, bool compat32 __unused) 3190 { 3191 struct kinfo_knote kin; 3192 #ifdef COMPAT_FREEBSD32 3193 struct kinfo_knote32 kin32; 3194 #endif 3195 int error; 3196 3197 if (kn->kn_status == KN_MARKER) 3198 return (0); 3199 3200 memset(&kin, 0, sizeof(kin)); 3201 kin.knt_kq_fd = kq_fd; 3202 memcpy(&kin.knt_event, &kn->kn_kevent, sizeof(struct kevent)); 3203 kin.knt_status = knote_status_export(kn->kn_status); 3204 kn_enter_flux(kn); 3205 KQ_UNLOCK_FLUX(kq); 3206 if (kn->kn_fop->f_userdump != NULL) 3207 (void)kn->kn_fop->f_userdump(p, kn, &kin); 3208 #ifdef COMPAT_FREEBSD32 3209 if (compat32) { 3210 freebsd32_kinfo_knote_to_32(&kin, &kin32); 3211 error = sbuf_bcat(s, &kin32, sizeof(kin32)); 3212 } else 3213 #endif 3214 error = sbuf_bcat(s, &kin, sizeof(kin)); 3215 KQ_LOCK(kq); 3216 kn_leave_flux(kn); 3217 return (error); 3218 } 3219 3220 static int 3221 kern_proc_kqueue_report(struct sbuf *s, struct proc *p, int kq_fd, 3222 struct kqueue *kq, bool compat32) 3223 { 3224 struct knote *kn; 3225 int error, i; 3226 3227 error = 0; 3228 KQ_LOCK(kq); 3229 for (i = 0; i < kq->kq_knlistsize; i++) { 3230 SLIST_FOREACH(kn, &kq->kq_knlist[i], kn_link) { 3231 error = kern_proc_kqueue_report_one(s, p, kq_fd, 3232 kq, kn, compat32); 3233 if (error != 0) 3234 goto out; 3235 } 3236 } 3237 if (kq->kq_knhashmask == 0) 3238 goto out; 3239 for (i = 0; i <= kq->kq_knhashmask; i++) { 3240 SLIST_FOREACH(kn, &kq->kq_knhash[i], kn_link) { 3241 error = kern_proc_kqueue_report_one(s, p, kq_fd, 3242 kq, kn, compat32); 3243 if (error != 0) 3244 goto out; 3245 } 3246 } 3247 out: 3248 KQ_UNLOCK_FLUX(kq); 3249 return (error); 3250 } 3251 3252 struct kern_proc_kqueues_out1_cb_args { 3253 struct sbuf *s; 3254 bool compat32; 3255 }; 3256 3257 static int 3258 kern_proc_kqueues_out1_cb(struct proc *p, int fd, struct file *fp, void *arg) 3259 { 3260 struct kqueue *kq; 3261 struct kern_proc_kqueues_out1_cb_args *a; 3262 3263 if (fp->f_type != DTYPE_KQUEUE) 3264 return (0); 3265 a = arg; 3266 kq = fp->f_data; 3267 return (kern_proc_kqueue_report(a->s, p, fd, kq, a->compat32)); 3268 } 3269 3270 static int 3271 kern_proc_kqueues_out1(struct thread *td, struct proc *p, struct sbuf *s, 3272 bool compat32) 3273 { 3274 struct kern_proc_kqueues_out1_cb_args a; 3275 3276 a.s = s; 3277 a.compat32 = compat32; 3278 return (fget_remote_foreach(td, p, kern_proc_kqueues_out1_cb, &a)); 3279 } 3280 3281 int 3282 kern_proc_kqueues_out(struct proc *p, struct sbuf *sb, size_t maxlen, 3283 bool compat32) 3284 { 3285 struct sbuf *s, sm; 3286 size_t sb_len; 3287 int error; 3288 3289 if (maxlen == -1 || maxlen == 0) 3290 sb_len = 128; 3291 else 3292 sb_len = maxlen; 3293 s = sbuf_new(&sm, NULL, sb_len, maxlen == -1 ? SBUF_AUTOEXTEND : 3294 SBUF_FIXEDLEN); 3295 error = kern_proc_kqueues_out1(curthread, p, s, compat32); 3296 sbuf_finish(s); 3297 if (error == 0) { 3298 sbuf_bcat(sb, sbuf_data(s), MIN(sbuf_len(s), maxlen == -1 ? 3299 SIZE_T_MAX : maxlen)); 3300 } 3301 sbuf_delete(s); 3302 return (error); 3303 } 3304 3305 static int 3306 sysctl_kern_proc_kqueue_one(struct thread *td, struct sbuf *s, struct proc *p, 3307 int kq_fd, bool compat32) 3308 { 3309 struct file *fp; 3310 struct kqueue *kq; 3311 int error; 3312 3313 error = fget_remote(td, p, kq_fd, &fp); 3314 if (error == 0) { 3315 if (fp->f_type != DTYPE_KQUEUE) { 3316 error = EINVAL; 3317 } else { 3318 kq = fp->f_data; 3319 error = kern_proc_kqueue_report(s, p, kq_fd, kq, 3320 compat32); 3321 } 3322 fdrop(fp, td); 3323 } 3324 return (error); 3325 } 3326 3327 static int 3328 sysctl_kern_proc_kqueue(SYSCTL_HANDLER_ARGS) 3329 { 3330 struct thread *td; 3331 struct proc *p; 3332 struct sbuf *s, sm; 3333 int error, error1, *name; 3334 bool compat32; 3335 3336 name = (int *)arg1; 3337 if ((u_int)arg2 > 2 || (u_int)arg2 == 0) 3338 return (EINVAL); 3339 3340 error = pget((pid_t)name[0], PGET_HOLD | PGET_CANDEBUG, &p); 3341 if (error != 0) 3342 return (error); 3343 3344 td = curthread; 3345 #ifdef COMPAT_FREEBSD32 3346 compat32 = SV_CURPROC_FLAG(SV_ILP32); 3347 #else 3348 compat32 = false; 3349 #endif 3350 3351 s = sbuf_new_for_sysctl(&sm, NULL, 0, req); 3352 if (s == NULL) { 3353 error = ENOMEM; 3354 goto out; 3355 } 3356 sbuf_clear_flags(s, SBUF_INCLUDENUL); 3357 3358 if ((u_int)arg2 == 1) { 3359 error = kern_proc_kqueues_out1(td, p, s, compat32); 3360 } else { 3361 error = sysctl_kern_proc_kqueue_one(td, s, p, 3362 name[1] /* kq_fd */, compat32); 3363 } 3364 3365 error1 = sbuf_finish(s); 3366 if (error == 0) 3367 error = error1; 3368 sbuf_delete(s); 3369 3370 out: 3371 PRELE(p); 3372 return (error); 3373 } 3374 3375 static SYSCTL_NODE(_kern_proc, KERN_PROC_KQUEUE, kq, 3376 CTLFLAG_RD | CTLFLAG_MPSAFE, 3377 sysctl_kern_proc_kqueue, "KQueue events"); 3378