Lines Matching full:job
270 TAILQ_HEAD(,kaiocb) kaio_jobqueue; /* (a) job queue for process */
301 static TAILQ_HEAD(,kaiocb) aio_jobs; /* (c) Async job list */
307 static int aio_free_entry(struct kaiocb *job);
308 static void aio_process_rw(struct kaiocb *job);
309 static void aio_process_sync(struct kaiocb *job);
310 static void aio_process_mlock(struct kaiocb *job);
315 static int aio_queue_file(struct file *fp, struct kaiocb *job);
320 static int aio_qbio(struct proc *p, struct kaiocb *job);
322 static void aio_bio_done_notify(struct proc *userp, struct kaiocb *job);
323 static bool aio_clear_cancel_function_locked(struct kaiocb *job);
494 * Free a job entry. Wait for completion if it is currently active, but don't
499 aio_free_entry(struct kaiocb *job) in aio_free_entry() argument
505 p = job->userproc; in aio_free_entry()
511 MPASS(job->jobflags & KAIOCB_FINISHED); in aio_free_entry()
518 TAILQ_REMOVE(&ki->kaio_done, job, plist); in aio_free_entry()
519 TAILQ_REMOVE(&ki->kaio_all, job, allist); in aio_free_entry()
521 lj = job->lio; in aio_free_entry()
537 /* job is going away, we need to destroy any knotes */ in aio_free_entry()
538 knlist_delete(&job->klist, curthread, 1); in aio_free_entry()
540 sigqueue_take(&job->ksi); in aio_free_entry()
549 * need a thread pointer from the process owning the job that is in aio_free_entry()
554 * a kaiocb from the current process' job list either via a in aio_free_entry()
565 if (job->fd_file) in aio_free_entry()
566 fdrop(job->fd_file, curthread); in aio_free_entry()
567 crfree(job->cred); in aio_free_entry()
568 if (job->uiop != &job->uio) in aio_free_entry()
569 freeuio(job->uiop); in aio_free_entry()
570 uma_zfree(aiocb_zone, job); in aio_free_entry()
584 aio_cancel_job(struct proc *p, struct kaioinfo *ki, struct kaiocb *job) in aio_cancel_job() argument
595 MPASS((job->jobflags & KAIOCB_MARKER) == 0); in aio_cancel_job()
596 if (job->jobflags & (KAIOCB_CANCELLED | KAIOCB_FINISHED)) in aio_cancel_job()
598 MPASS((job->jobflags & KAIOCB_CANCELLING) == 0); in aio_cancel_job()
599 job->jobflags |= KAIOCB_CANCELLED; in aio_cancel_job()
601 func = job->cancel_fn; in aio_cancel_job()
604 * If there is no cancel routine, just leave the job marked as in aio_cancel_job()
605 * cancelled. The job should be in active use by a caller who in aio_cancel_job()
614 * completions of this job. This prevents the job from being in aio_cancel_job()
619 job->jobflags |= KAIOCB_CANCELLING; in aio_cancel_job()
621 func(job); in aio_cancel_job()
623 job->jobflags &= ~KAIOCB_CANCELLING; in aio_cancel_job()
624 if (job->jobflags & KAIOCB_FINISHED) { in aio_cancel_job()
625 cancelled = job->uaiocb._aiocb_private.error == ECANCELED; in aio_cancel_job()
626 TAILQ_REMOVE(&ki->kaio_jobqueue, job, plist); in aio_cancel_job()
627 aio_bio_done_notify(p, job); in aio_cancel_job()
648 struct kaiocb *job, *jobn; in aio_proc_rundown() local
665 TAILQ_FOREACH_SAFE(job, &ki->kaio_jobqueue, plist, jobn) { in aio_proc_rundown()
666 aio_cancel_job(p, ki, job); in aio_proc_rundown()
677 while ((job = TAILQ_FIRST(&ki->kaio_done)) != NULL) in aio_proc_rundown()
678 aio_free_entry(job); in aio_proc_rundown()
689 panic("LIO job not cleaned up: C:%d, FC:%d\n", in aio_proc_rundown()
702 * Select a job to run (called by an AIO daemon).
707 struct kaiocb *job; in aio_selectjob() local
713 TAILQ_FOREACH(job, &aio_jobs, list) { in aio_selectjob()
714 userp = job->userproc; in aio_selectjob()
718 TAILQ_REMOVE(&aio_jobs, job, list); in aio_selectjob()
719 if (!aio_clear_cancel_function(job)) in aio_selectjob()
727 return (job); in aio_selectjob()
768 aio_process_rw(struct kaiocb *job) in aio_process_rw() argument
780 opcode = job->uaiocb.aio_lio_opcode & ~LIO_FOFFSET; in aio_process_rw()
783 ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode)); in aio_process_rw()
785 aio_switch_vmspace(job); in aio_process_rw()
788 td->td_ucred = job->cred; in aio_process_rw()
789 job->uiop->uio_td = td; in aio_process_rw()
790 fp = job->fd_file; in aio_process_rw()
792 cnt = job->uiop->uio_resid; in aio_process_rw()
804 if (job->uiop->uio_resid == 0) in aio_process_rw()
807 error = fo_read(fp, job->uiop, fp->f_cred, in aio_process_rw()
808 (job->ioflags & KAIOCB_IO_FOFFSET) != 0 ? 0 : in aio_process_rw()
813 error = fo_write(fp, job->uiop, fp->f_cred, (job->ioflags & in aio_process_rw()
821 job->msgrcv = msgrcv_end - msgrcv_st; in aio_process_rw()
822 job->msgsnd = msgsnd_end - msgsnd_st; in aio_process_rw()
823 job->inblock = inblock_end - inblock_st; in aio_process_rw()
824 job->outblock = oublock_end - oublock_st; in aio_process_rw()
826 if (error != 0 && job->uiop->uio_resid != cnt) { in aio_process_rw()
830 PROC_LOCK(job->userproc); in aio_process_rw()
831 kern_psignal(job->userproc, SIGPIPE); in aio_process_rw()
832 PROC_UNLOCK(job->userproc); in aio_process_rw()
836 cnt -= job->uiop->uio_resid; in aio_process_rw()
839 aio_complete(job, -1, error); in aio_process_rw()
841 aio_complete(job, cnt, 0); in aio_process_rw()
845 aio_process_sync(struct kaiocb *job) in aio_process_sync() argument
849 struct file *fp = job->fd_file; in aio_process_sync()
852 KASSERT(job->uaiocb.aio_lio_opcode & LIO_SYNC, in aio_process_sync()
853 ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode)); in aio_process_sync()
855 td->td_ucred = job->cred; in aio_process_sync()
858 job->uaiocb.aio_lio_opcode); in aio_process_sync()
862 aio_complete(job, -1, error); in aio_process_sync()
864 aio_complete(job, 0, 0); in aio_process_sync()
868 aio_process_mlock(struct kaiocb *job) in aio_process_mlock() argument
870 struct aiocb *cb = &job->uaiocb; in aio_process_mlock()
873 KASSERT(job->uaiocb.aio_lio_opcode == LIO_MLOCK, in aio_process_mlock()
874 ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode)); in aio_process_mlock()
876 aio_switch_vmspace(job); in aio_process_mlock()
877 error = kern_mlock(job->userproc, job->cred, in aio_process_mlock()
879 aio_complete(job, error != 0 ? -1 : 0, error); in aio_process_mlock()
883 aio_bio_done_notify(struct proc *userp, struct kaiocb *job) in aio_bio_done_notify() argument
893 lj = job->lio; in aio_bio_done_notify()
900 TAILQ_INSERT_TAIL(&ki->kaio_done, job, plist); in aio_bio_done_notify()
901 MPASS(job->jobflags & KAIOCB_FINISHED); in aio_bio_done_notify()
906 if (job->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL || in aio_bio_done_notify()
907 job->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) in aio_bio_done_notify()
908 aio_sendsig(userp, &job->uaiocb.aio_sigevent, &job->ksi, true); in aio_bio_done_notify()
910 KNOTE_LOCKED(&job->klist, 1); in aio_bio_done_notify()
928 if (job->jobflags & KAIOCB_CHECKSYNC) { in aio_bio_done_notify()
931 if (job->fd_file != sjob->fd_file || in aio_bio_done_notify()
932 job->seqno >= sjob->seqno) in aio_bio_done_notify()
956 struct kaiocb *job; in aio_schedule_fsync() local
961 job = TAILQ_FIRST(&ki->kaio_syncready); in aio_schedule_fsync()
962 TAILQ_REMOVE(&ki->kaio_syncready, job, list); in aio_schedule_fsync()
964 aio_schedule(job, aio_process_sync); in aio_schedule_fsync()
971 aio_cancel_cleared(struct kaiocb *job) in aio_cancel_cleared() argument
980 return ((job->jobflags & KAIOCB_CLEARED) != 0); in aio_cancel_cleared()
984 aio_clear_cancel_function_locked(struct kaiocb *job) in aio_clear_cancel_function_locked() argument
987 AIO_LOCK_ASSERT(job->userproc->p_aioinfo, MA_OWNED); in aio_clear_cancel_function_locked()
988 MPASS(job->cancel_fn != NULL); in aio_clear_cancel_function_locked()
989 if (job->jobflags & KAIOCB_CANCELLING) { in aio_clear_cancel_function_locked()
990 job->jobflags |= KAIOCB_CLEARED; in aio_clear_cancel_function_locked()
993 job->cancel_fn = NULL; in aio_clear_cancel_function_locked()
998 aio_clear_cancel_function(struct kaiocb *job) in aio_clear_cancel_function() argument
1003 ki = job->userproc->p_aioinfo; in aio_clear_cancel_function()
1005 ret = aio_clear_cancel_function_locked(job); in aio_clear_cancel_function()
1011 aio_set_cancel_function_locked(struct kaiocb *job, aio_cancel_fn_t *func) in aio_set_cancel_function_locked() argument
1014 AIO_LOCK_ASSERT(job->userproc->p_aioinfo, MA_OWNED); in aio_set_cancel_function_locked()
1015 if (job->jobflags & KAIOCB_CANCELLED) in aio_set_cancel_function_locked()
1017 job->cancel_fn = func; in aio_set_cancel_function_locked()
1022 aio_set_cancel_function(struct kaiocb *job, aio_cancel_fn_t *func) in aio_set_cancel_function() argument
1027 ki = job->userproc->p_aioinfo; in aio_set_cancel_function()
1029 ret = aio_set_cancel_function_locked(job, func); in aio_set_cancel_function()
1035 aio_complete(struct kaiocb *job, long status, int error) in aio_complete() argument
1040 job->uaiocb._aiocb_private.error = error; in aio_complete()
1041 job->uaiocb._aiocb_private.status = status; in aio_complete()
1043 userp = job->userproc; in aio_complete()
1047 KASSERT(!(job->jobflags & KAIOCB_FINISHED), in aio_complete()
1049 job->jobflags |= KAIOCB_FINISHED; in aio_complete()
1050 if ((job->jobflags & (KAIOCB_QUEUEING | KAIOCB_CANCELLING)) == 0) { in aio_complete()
1051 TAILQ_REMOVE(&ki->kaio_jobqueue, job, plist); in aio_complete()
1052 aio_bio_done_notify(userp, job); in aio_complete()
1058 aio_cancel(struct kaiocb *job) in aio_cancel() argument
1061 aio_complete(job, -1, ECANCELED); in aio_cancel()
1065 aio_switch_vmspace(struct kaiocb *job) in aio_switch_vmspace() argument
1068 vmspace_switch_aio(job->userproc->p_vmspace); in aio_switch_vmspace()
1078 struct kaiocb *job; in aio_daemon() local
1123 while ((job = aio_selectjob(aiop)) != NULL) { in aio_daemon()
1126 ki = job->userproc->p_aioinfo; in aio_daemon()
1127 job->handle_fn(job); in aio_daemon()
1130 /* Decrement the active job count. */ in aio_daemon()
1143 * no job can be selected. in aio_daemon()
1218 aio_qbio(struct proc *p, struct kaiocb *job) in aio_qbio() argument
1233 cb = &job->uaiocb; in aio_qbio()
1234 fp = job->fd_file; in aio_qbio()
1250 iovcnt = job->uiop->uio_iovcnt; in aio_qbio()
1254 if (job->uiop->uio_iov[i].iov_len % vp->v_bufobj.bo_bsize != 0) in aio_qbio()
1256 if (job->uiop->uio_iov[i].iov_len > maxphys) { in aio_qbio()
1272 if (job->uiop->uio_resid > dev->si_iosize_max) { in aio_qbio()
1278 job->error = 0; in aio_qbio()
1293 refcount_init(&job->nbio, iovcnt); in aio_qbio()
1301 buf = job->uiop->uio_iov[i].iov_base; in aio_qbio()
1302 nbytes = job->uiop->uio_iov[i].iov_len; in aio_qbio()
1324 bp->bio_caller1 = job; in aio_qbio()
1507 struct kaiocb *job; in aio_aqueue() local
1532 job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO); in aio_aqueue()
1533 knlist_init_mtx(&job->klist, AIO_MTX(ki)); in aio_aqueue()
1535 error = ops->aio_copyin(ujob, job, type); in aio_aqueue()
1539 if (job->uaiocb.aio_nbytes > IOSIZE_MAX) { in aio_aqueue()
1544 if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT && in aio_aqueue()
1545 job->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL && in aio_aqueue()
1546 job->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID && in aio_aqueue()
1547 job->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) { in aio_aqueue()
1552 if ((job->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL || in aio_aqueue()
1553 job->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) && in aio_aqueue()
1554 !_SIG_VALID(job->uaiocb.aio_sigevent.sigev_signo)) { in aio_aqueue()
1561 switch (job->uaiocb.aio_lio_opcode & ~LIO_FOFFSET) { in aio_aqueue()
1567 opcode = job->uaiocb.aio_lio_opcode & ~LIO_FOFFSET; in aio_aqueue()
1568 if ((job->uaiocb.aio_lio_opcode & LIO_FOFFSET) != 0) in aio_aqueue()
1569 job->ioflags |= KAIOCB_IO_FOFFSET; in aio_aqueue()
1576 opcode = job->uaiocb.aio_lio_opcode = type; in aio_aqueue()
1578 ksiginfo_init(&job->ksi); in aio_aqueue()
1580 /* Save userspace address of the job info. */ in aio_aqueue()
1581 job->ujob = ujob; in aio_aqueue()
1591 fd = job->uaiocb.aio_fildes; in aio_aqueue()
1623 job->uaiocb.aio_offset < 0 && in aio_aqueue()
1634 job->fd_file = fp; in aio_aqueue()
1637 job->seqno = jobseqno++; in aio_aqueue()
1641 MPASS(job->uiop == &job->uio || job->uiop == NULL); in aio_aqueue()
1642 uma_zfree(aiocb_zone, job); in aio_aqueue()
1646 if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT) in aio_aqueue()
1648 evflags = job->uaiocb.aio_sigevent.sigev_notify_kevent_flags; in aio_aqueue()
1653 kqfd = job->uaiocb.aio_sigevent.sigev_notify_kqueue; in aio_aqueue()
1655 kev.ident = (uintptr_t)job->ujob; in aio_aqueue()
1658 kev.data = (intptr_t)job; in aio_aqueue()
1659 kev.udata = job->uaiocb.aio_sigevent.sigev_value.sival_ptr; in aio_aqueue()
1667 job->uaiocb._aiocb_private.error = EINPROGRESS; in aio_aqueue()
1668 job->userproc = p; in aio_aqueue()
1669 job->cred = crhold(td->td_ucred); in aio_aqueue()
1670 job->jobflags = KAIOCB_QUEUEING; in aio_aqueue()
1671 job->lio = lj; in aio_aqueue()
1675 MPASS(job->uiop != &job->uio && job->uiop != NULL); in aio_aqueue()
1678 job->iov[0].iov_base = (void *)(uintptr_t)job->uaiocb.aio_buf; in aio_aqueue()
1679 job->iov[0].iov_len = job->uaiocb.aio_nbytes; in aio_aqueue()
1680 job->uio.uio_iov = job->iov; in aio_aqueue()
1681 job->uio.uio_iovcnt = 1; in aio_aqueue()
1682 job->uio.uio_resid = job->uaiocb.aio_nbytes; in aio_aqueue()
1683 job->uio.uio_segflg = UIO_USERSPACE; in aio_aqueue()
1684 job->uiop = &job->uio; in aio_aqueue()
1688 job->uiop->uio_rw = UIO_READ; in aio_aqueue()
1691 job->uiop->uio_rw = UIO_WRITE; in aio_aqueue()
1694 job->uiop->uio_offset = job->uaiocb.aio_offset; in aio_aqueue()
1695 job->uiop->uio_td = td; in aio_aqueue()
1698 aio_schedule(job, aio_process_mlock); in aio_aqueue()
1701 error = aio_queue_file(fp, job); in aio_aqueue()
1703 error = fo_aio_queue(fp, job); in aio_aqueue()
1708 job->jobflags &= ~KAIOCB_QUEUEING; in aio_aqueue()
1709 TAILQ_INSERT_TAIL(&ki->kaio_all, job, allist); in aio_aqueue()
1714 if (job->jobflags & KAIOCB_FINISHED) { in aio_aqueue()
1720 aio_bio_done_notify(p, job); in aio_aqueue()
1722 TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, job, plist); in aio_aqueue()
1727 crfree(job->cred); in aio_aqueue()
1731 knlist_delete(&job->klist, curthread, 0); in aio_aqueue()
1733 if (job->uiop != &job->uio) in aio_aqueue()
1734 freeuio(job->uiop); in aio_aqueue()
1735 uma_zfree(aiocb_zone, job); in aio_aqueue()
1742 aio_cancel_daemon_job(struct kaiocb *job) in aio_cancel_daemon_job() argument
1746 if (!aio_cancel_cleared(job)) in aio_cancel_daemon_job()
1747 TAILQ_REMOVE(&aio_jobs, job, list); in aio_cancel_daemon_job()
1749 aio_cancel(job); in aio_cancel_daemon_job()
1753 aio_schedule(struct kaiocb *job, aio_handle_fn_t *func) in aio_schedule() argument
1757 if (!aio_set_cancel_function(job, aio_cancel_daemon_job)) { in aio_schedule()
1759 aio_cancel(job); in aio_schedule()
1762 job->handle_fn = func; in aio_schedule()
1763 TAILQ_INSERT_TAIL(&aio_jobs, job, list); in aio_schedule()
1764 aio_kick_nowait(job->userproc); in aio_schedule()
1769 aio_cancel_sync(struct kaiocb *job) in aio_cancel_sync() argument
1773 ki = job->userproc->p_aioinfo; in aio_cancel_sync()
1775 if (!aio_cancel_cleared(job)) in aio_cancel_sync()
1776 TAILQ_REMOVE(&ki->kaio_syncqueue, job, list); in aio_cancel_sync()
1778 aio_cancel(job); in aio_cancel_sync()
1782 aio_queue_file(struct file *fp, struct kaiocb *job) in aio_queue_file() argument
1791 ki = job->userproc->p_aioinfo; in aio_queue_file()
1792 error = aio_qbio(job->userproc, job); in aio_queue_file()
1810 if (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) { in aio_queue_file()
1811 aio_schedule(job, aio_process_rw); in aio_queue_file()
1813 } else if (job->uaiocb.aio_lio_opcode & LIO_SYNC) { in aio_queue_file()
1818 if (job2->fd_file == job->fd_file && in aio_queue_file()
1820 job2->seqno < job->seqno) { in aio_queue_file()
1822 job->pending++; in aio_queue_file()
1825 if (job->pending != 0) { in aio_queue_file()
1826 if (!aio_set_cancel_function_locked(job, in aio_queue_file()
1829 aio_cancel(job); in aio_queue_file()
1832 TAILQ_INSERT_TAIL(&ki->kaio_syncqueue, job, list); in aio_queue_file()
1837 aio_schedule(job, aio_process_sync); in aio_queue_file()
1912 struct kaiocb *job; in kern_aio_return() local
1920 TAILQ_FOREACH(job, &ki->kaio_done, plist) { in kern_aio_return()
1921 if (job->ujob == ujob) in kern_aio_return()
1924 if (job != NULL) { in kern_aio_return()
1925 MPASS(job->jobflags & KAIOCB_FINISHED); in kern_aio_return()
1926 status = job->uaiocb._aiocb_private.status; in kern_aio_return()
1927 error = job->uaiocb._aiocb_private.error; in kern_aio_return()
1929 td->td_ru.ru_oublock += job->outblock; in kern_aio_return()
1930 td->td_ru.ru_inblock += job->inblock; in kern_aio_return()
1931 td->td_ru.ru_msgsnd += job->msgsnd; in kern_aio_return()
1932 td->td_ru.ru_msgrcv += job->msgrcv; in kern_aio_return()
1933 aio_free_entry(job); in kern_aio_return()
1961 struct kaiocb *firstjob, *job; in kern_aio_suspend() local
1986 TAILQ_FOREACH(job, &ki->kaio_all, allist) { in kern_aio_suspend()
1988 if (job->ujob == ujoblist[i]) { in kern_aio_suspend()
1990 firstjob = job; in kern_aio_suspend()
1991 if (job->jobflags & KAIOCB_FINISHED) in kern_aio_suspend()
2047 struct kaiocb *job, *jobn, marker; in sys_aio_cancel() local
2073 * We may have to drop the list mutex in order to cancel a job. After in sys_aio_cancel()
2075 * could restart the search from the beginning after canceling a job, in sys_aio_cancel()
2076 * but this may inefficient. Instead, use a marker job to keep our in sys_aio_cancel()
2083 TAILQ_FOREACH_SAFE(job, &ki->kaio_jobqueue, plist, jobn) { in sys_aio_cancel()
2084 if (uap->fd == job->uaiocb.aio_fildes && in sys_aio_cancel()
2085 (uap->aiocbp == NULL || uap->aiocbp == job->ujob) && in sys_aio_cancel()
2086 (job->jobflags & KAIOCB_MARKER) == 0) { in sys_aio_cancel()
2087 TAILQ_INSERT_AFTER(&ki->kaio_jobqueue, job, &marker, in sys_aio_cancel()
2089 if (aio_cancel_job(p, ki, job)) { in sys_aio_cancel()
2136 struct kaiocb *job; in kern_aio_error() local
2147 TAILQ_FOREACH(job, &ki->kaio_all, allist) { in kern_aio_error()
2148 if (job->ujob == ujob) { in kern_aio_error()
2149 if (job->jobflags & KAIOCB_FINISHED) in kern_aio_error()
2151 job->uaiocb._aiocb_private.error; in kern_aio_error()
2243 struct aiocb *job; in kern_lio_listio() local
2326 job = acb_list[i]; in kern_lio_listio()
2327 if (job != NULL) { in kern_lio_listio()
2328 error = aio_aqueue(td, job, lj, LIO_NOP, ops); in kern_lio_listio()
2458 struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; in aio_biocleanup() local
2469 ki = job->userproc->p_aioinfo; in aio_biocleanup()
2485 struct kaiocb *job = (struct kaiocb *)bp->bio_caller1; in aio_biowakeup() local
2493 opcode = job->uaiocb.aio_lio_opcode; in aio_biowakeup()
2498 atomic_add_acq_long(&job->nbytes, nbytes); in aio_biowakeup()
2502 * If multiple bios experienced an error, the job will reflect the in aio_biowakeup()
2506 atomic_store_int(&job->error, abio_error); in aio_biowakeup()
2508 atomic_add_int(&job->outblock, nblks); in aio_biowakeup()
2510 atomic_add_int(&job->inblock, nblks); in aio_biowakeup()
2512 if (refcount_release(&job->nbio)) { in aio_biowakeup()
2513 abio_error = atomic_load_int(&job->error); in aio_biowakeup()
2515 aio_complete(job, -1, abio_error); in aio_biowakeup()
2517 aio_complete(job, atomic_load_long(&job->nbytes), 0); in aio_biowakeup()
2529 struct kaiocb *job; in kern_aio_waitcomplete() local
2558 job = NULL; in kern_aio_waitcomplete()
2560 while ((job = TAILQ_FIRST(&ki->kaio_done)) == NULL) { in kern_aio_waitcomplete()
2574 if (job != NULL) { in kern_aio_waitcomplete()
2575 MPASS(job->jobflags & KAIOCB_FINISHED); in kern_aio_waitcomplete()
2576 ujob = job->ujob; in kern_aio_waitcomplete()
2577 status = job->uaiocb._aiocb_private.status; in kern_aio_waitcomplete()
2578 error = job->uaiocb._aiocb_private.error; in kern_aio_waitcomplete()
2580 td->td_ru.ru_oublock += job->outblock; in kern_aio_waitcomplete()
2581 td->td_ru.ru_inblock += job->inblock; in kern_aio_waitcomplete()
2582 td->td_ru.ru_msgsnd += job->msgsnd; in kern_aio_waitcomplete()
2583 td->td_ru.ru_msgrcv += job->msgrcv; in kern_aio_waitcomplete()
2584 aio_free_entry(job); in kern_aio_waitcomplete()
2644 struct kaiocb *job; in filt_aioattach() local
2646 job = (struct kaiocb *)(uintptr_t)kn->kn_sdata; in filt_aioattach()
2649 * The job pointer must be validated before using it, so in filt_aioattach()
2655 kn->kn_ptr.p_aio = job; in filt_aioattach()
2658 knlist_add(&job->klist, kn, 0); in filt_aioattach()
2681 struct kaiocb *job = kn->kn_ptr.p_aio; in filt_aio() local
2683 kn->kn_data = job->uaiocb._aiocb_private.error; in filt_aio()
2684 if (!(job->jobflags & KAIOCB_FINISHED)) in filt_aio()