Lines Matching refs:ki
276 #define AIO_LOCK(ki) mtx_lock(&(ki)->kaio_mtx) argument
277 #define AIO_UNLOCK(ki) mtx_unlock(&(ki)->kaio_mtx) argument
278 #define AIO_LOCK_ASSERT(ki, f) mtx_assert(&(ki)->kaio_mtx, (f)) argument
279 #define AIO_MTX(ki) (&(ki)->kaio_mtx) argument
428 struct kaioinfo *ki; in aio_init_aioinfo() local
431 ki = uma_zalloc(kaio_zone, M_WAITOK); in aio_init_aioinfo()
432 mtx_init(&ki->kaio_mtx, "aiomtx", NULL, MTX_DEF | MTX_NEW); in aio_init_aioinfo()
433 ki->kaio_flags = 0; in aio_init_aioinfo()
434 ki->kaio_active_count = 0; in aio_init_aioinfo()
435 ki->kaio_count = 0; in aio_init_aioinfo()
436 ki->kaio_buffer_count = 0; in aio_init_aioinfo()
437 TAILQ_INIT(&ki->kaio_all); in aio_init_aioinfo()
438 TAILQ_INIT(&ki->kaio_done); in aio_init_aioinfo()
439 TAILQ_INIT(&ki->kaio_jobqueue); in aio_init_aioinfo()
440 TAILQ_INIT(&ki->kaio_liojoblist); in aio_init_aioinfo()
441 TAILQ_INIT(&ki->kaio_syncqueue); in aio_init_aioinfo()
442 TAILQ_INIT(&ki->kaio_syncready); in aio_init_aioinfo()
443 TASK_INIT(&ki->kaio_task, 0, aio_kick_helper, p); in aio_init_aioinfo()
444 TASK_INIT(&ki->kaio_sync_task, 0, aio_schedule_fsync, ki); in aio_init_aioinfo()
447 p->p_aioinfo = ki; in aio_init_aioinfo()
451 mtx_destroy(&ki->kaio_mtx); in aio_init_aioinfo()
452 uma_zfree(kaio_zone, ki); in aio_init_aioinfo()
498 struct kaioinfo *ki; in aio_free_entry() local
504 ki = p->p_aioinfo; in aio_free_entry()
505 MPASS(ki != NULL); in aio_free_entry()
507 AIO_LOCK_ASSERT(ki, MA_OWNED); in aio_free_entry()
512 ki->kaio_count--; in aio_free_entry()
513 MPASS(ki->kaio_count >= 0); in aio_free_entry()
515 TAILQ_REMOVE(&ki->kaio_done, job, plist); in aio_free_entry()
516 TAILQ_REMOVE(&ki->kaio_all, job, allist); in aio_free_entry()
524 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list); in aio_free_entry()
540 AIO_UNLOCK(ki); in aio_free_entry()
568 AIO_LOCK(ki); in aio_free_entry()
581 aio_cancel_job(struct proc *p, struct kaioinfo *ki, struct kaiocb *job) in aio_cancel_job() argument
586 AIO_LOCK_ASSERT(ki, MA_OWNED); in aio_cancel_job()
611 AIO_UNLOCK(ki); in aio_cancel_job()
613 AIO_LOCK(ki); in aio_cancel_job()
617 TAILQ_REMOVE(&ki->kaio_jobqueue, job, plist); in aio_cancel_job()
637 struct kaioinfo *ki; in aio_proc_rundown() local
643 ki = p->p_aioinfo; in aio_proc_rundown()
644 if (ki == NULL) in aio_proc_rundown()
647 AIO_LOCK(ki); in aio_proc_rundown()
648 ki->kaio_flags |= KAIO_RUNDOWN; in aio_proc_rundown()
656 TAILQ_FOREACH_SAFE(job, &ki->kaio_jobqueue, plist, jobn) { in aio_proc_rundown()
657 aio_cancel_job(p, ki, job); in aio_proc_rundown()
661 if (TAILQ_FIRST(&ki->kaio_jobqueue) || ki->kaio_active_count != 0) { in aio_proc_rundown()
662 ki->kaio_flags |= KAIO_WAKEUP; in aio_proc_rundown()
663 msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO, "aioprn", hz); in aio_proc_rundown()
668 while ((job = TAILQ_FIRST(&ki->kaio_done)) != NULL) in aio_proc_rundown()
671 while ((lj = TAILQ_FIRST(&ki->kaio_liojoblist)) != NULL) { in aio_proc_rundown()
673 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list); in aio_proc_rundown()
684 AIO_UNLOCK(ki); in aio_proc_rundown()
685 taskqueue_drain(taskqueue_aiod_kick, &ki->kaio_task); in aio_proc_rundown()
686 taskqueue_drain(taskqueue_aiod_kick, &ki->kaio_sync_task); in aio_proc_rundown()
687 mtx_destroy(&ki->kaio_mtx); in aio_proc_rundown()
688 uma_zfree(kaio_zone, ki); in aio_proc_rundown()
699 struct kaioinfo *ki; in aio_selectjob() local
706 ki = userp->p_aioinfo; in aio_selectjob()
708 if (ki->kaio_active_count < max_aio_per_proc) { in aio_selectjob()
714 ki->kaio_active_count++; in aio_selectjob()
877 struct kaioinfo *ki; in aio_bio_done_notify() local
882 ki = userp->p_aioinfo; in aio_bio_done_notify()
883 AIO_LOCK_ASSERT(ki, MA_OWNED); in aio_bio_done_notify()
891 TAILQ_INSERT_TAIL(&ki->kaio_done, job, plist); in aio_bio_done_notify()
894 if (ki->kaio_flags & KAIO_RUNDOWN) in aio_bio_done_notify()
921 TAILQ_FOREACH_SAFE(sjob, &ki->kaio_syncqueue, list, sjobn) { in aio_bio_done_notify()
927 TAILQ_REMOVE(&ki->kaio_syncqueue, sjob, list); in aio_bio_done_notify()
930 TAILQ_INSERT_TAIL(&ki->kaio_syncready, sjob, list); in aio_bio_done_notify()
935 &ki->kaio_sync_task); in aio_bio_done_notify()
937 if (ki->kaio_flags & KAIO_WAKEUP) { in aio_bio_done_notify()
938 ki->kaio_flags &= ~KAIO_WAKEUP; in aio_bio_done_notify()
946 struct kaioinfo *ki; in aio_schedule_fsync() local
949 ki = context; in aio_schedule_fsync()
950 AIO_LOCK(ki); in aio_schedule_fsync()
951 while (!TAILQ_EMPTY(&ki->kaio_syncready)) { in aio_schedule_fsync()
952 job = TAILQ_FIRST(&ki->kaio_syncready); in aio_schedule_fsync()
953 TAILQ_REMOVE(&ki->kaio_syncready, job, list); in aio_schedule_fsync()
954 AIO_UNLOCK(ki); in aio_schedule_fsync()
956 AIO_LOCK(ki); in aio_schedule_fsync()
958 AIO_UNLOCK(ki); in aio_schedule_fsync()
991 struct kaioinfo *ki; in aio_clear_cancel_function() local
994 ki = job->userproc->p_aioinfo; in aio_clear_cancel_function()
995 AIO_LOCK(ki); in aio_clear_cancel_function()
997 AIO_UNLOCK(ki); in aio_clear_cancel_function()
1015 struct kaioinfo *ki; in aio_set_cancel_function() local
1018 ki = job->userproc->p_aioinfo; in aio_set_cancel_function()
1019 AIO_LOCK(ki); in aio_set_cancel_function()
1021 AIO_UNLOCK(ki); in aio_set_cancel_function()
1028 struct kaioinfo *ki; in aio_complete() local
1035 ki = userp->p_aioinfo; in aio_complete()
1037 AIO_LOCK(ki); in aio_complete()
1042 TAILQ_REMOVE(&ki->kaio_jobqueue, job, plist); in aio_complete()
1045 AIO_UNLOCK(ki); in aio_complete()
1071 struct kaioinfo *ki; in aio_daemon() local
1117 ki = job->userproc->p_aioinfo; in aio_daemon()
1122 ki->kaio_active_count--; in aio_daemon()
1217 struct kaioinfo *ki; in aio_qbio() local
1268 ki = p->p_aioinfo; in aio_qbio()
1273 AIO_LOCK(ki); in aio_qbio()
1274 if (ki->kaio_buffer_count + iovcnt > max_buf_aio) { in aio_qbio()
1275 AIO_UNLOCK(ki); in aio_qbio()
1279 ki->kaio_buffer_count += iovcnt; in aio_qbio()
1280 AIO_UNLOCK(ki); in aio_qbio()
1499 struct kaioinfo *ki; in aio_aqueue() local
1512 ki = p->p_aioinfo; in aio_aqueue()
1518 ki->kaio_count >= max_aio_queue_per_proc) { in aio_aqueue()
1524 knlist_init_mtx(&job->klist, AIO_MTX(ki)); in aio_aqueue()
1698 AIO_LOCK(ki); in aio_aqueue()
1700 TAILQ_INSERT_TAIL(&ki->kaio_all, job, allist); in aio_aqueue()
1701 ki->kaio_count++; in aio_aqueue()
1713 TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, job, plist); in aio_aqueue()
1714 AIO_UNLOCK(ki); in aio_aqueue()
1762 struct kaioinfo *ki; in aio_cancel_sync() local
1764 ki = job->userproc->p_aioinfo; in aio_cancel_sync()
1765 AIO_LOCK(ki); in aio_cancel_sync()
1767 TAILQ_REMOVE(&ki->kaio_syncqueue, job, list); in aio_cancel_sync()
1768 AIO_UNLOCK(ki); in aio_cancel_sync()
1775 struct kaioinfo *ki; in aio_queue_file() local
1782 ki = job->userproc->p_aioinfo; in aio_queue_file()
1805 AIO_LOCK(ki); in aio_queue_file()
1806 TAILQ_FOREACH(job2, &ki->kaio_jobqueue, plist) { in aio_queue_file()
1817 AIO_UNLOCK(ki); in aio_queue_file()
1821 TAILQ_INSERT_TAIL(&ki->kaio_syncqueue, job, list); in aio_queue_file()
1822 AIO_UNLOCK(ki); in aio_queue_file()
1825 AIO_UNLOCK(ki); in aio_queue_file()
1837 struct kaioinfo *ki = userp->p_aioinfo; in aio_kick_nowait() local
1846 ki->kaio_active_count + num_aio_resv_start < max_aio_per_proc) { in aio_kick_nowait()
1847 taskqueue_enqueue(taskqueue_aiod_kick, &ki->kaio_task); in aio_kick_nowait()
1854 struct kaioinfo *ki = userp->p_aioinfo; in aio_kick() local
1865 ki->kaio_active_count + num_aio_resv_start < max_aio_per_proc) { in aio_kick()
1902 struct kaioinfo *ki; in kern_aio_return() local
1905 ki = p->p_aioinfo; in kern_aio_return()
1906 if (ki == NULL) in kern_aio_return()
1908 AIO_LOCK(ki); in kern_aio_return()
1909 TAILQ_FOREACH(job, &ki->kaio_done, plist) { in kern_aio_return()
1923 AIO_UNLOCK(ki); in kern_aio_return()
1928 AIO_UNLOCK(ki); in kern_aio_return()
1949 struct kaioinfo *ki; in kern_aio_suspend() local
1964 ki = p->p_aioinfo; in kern_aio_suspend()
1965 if (ki == NULL) in kern_aio_suspend()
1971 AIO_LOCK(ki); in kern_aio_suspend()
1975 TAILQ_FOREACH(job, &ki->kaio_all, allist) { in kern_aio_suspend()
1989 ki->kaio_flags |= KAIO_WAKEUP; in kern_aio_suspend()
1990 error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH, in kern_aio_suspend()
1998 AIO_UNLOCK(ki); in kern_aio_suspend()
2035 struct kaioinfo *ki; in sys_aio_cancel() local
2048 ki = p->p_aioinfo; in sys_aio_cancel()
2049 if (ki == NULL) in sys_aio_cancel()
2061 AIO_LOCK(ki); in sys_aio_cancel()
2062 TAILQ_FOREACH_SAFE(job, &ki->kaio_jobqueue, plist, jobn) { in sys_aio_cancel()
2066 if (aio_cancel_job(p, ki, job)) { in sys_aio_cancel()
2075 AIO_UNLOCK(ki); in sys_aio_cancel()
2112 struct kaioinfo *ki; in kern_aio_error() local
2115 ki = p->p_aioinfo; in kern_aio_error()
2116 if (ki == NULL) { in kern_aio_error()
2121 AIO_LOCK(ki); in kern_aio_error()
2122 TAILQ_FOREACH(job, &ki->kaio_all, allist) { in kern_aio_error()
2129 AIO_UNLOCK(ki); in kern_aio_error()
2133 AIO_UNLOCK(ki); in kern_aio_error()
2219 struct kaioinfo *ki; in kern_lio_listio() local
2238 ki = p->p_aioinfo; in kern_lio_listio()
2245 knlist_init_mtx(&lj->klist, AIO_MTX(ki)); in kern_lio_listio()
2284 AIO_LOCK(ki); in kern_lio_listio()
2285 TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list); in kern_lio_listio()
2293 AIO_UNLOCK(ki); in kern_lio_listio()
2312 AIO_LOCK(ki); in kern_lio_listio()
2315 ki->kaio_flags |= KAIO_WAKEUP; in kern_lio_listio()
2316 error = msleep(&p->p_aioinfo, AIO_MTX(ki), in kern_lio_listio()
2341 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list); in kern_lio_listio()
2346 AIO_UNLOCK(ki); in kern_lio_listio()
2349 AIO_UNLOCK(ki); in kern_lio_listio()
2434 struct kaioinfo *ki; in aio_biocleanup() local
2444 ki = job->userproc->p_aioinfo; in aio_biocleanup()
2445 AIO_LOCK(ki); in aio_biocleanup()
2446 ki->kaio_buffer_count--; in aio_biocleanup()
2447 AIO_UNLOCK(ki); in aio_biocleanup()
2503 struct kaioinfo *ki; in kern_aio_waitcomplete() local
2530 ki = p->p_aioinfo; in kern_aio_waitcomplete()
2534 AIO_LOCK(ki); in kern_aio_waitcomplete()
2535 while ((job = TAILQ_FIRST(&ki->kaio_done)) == NULL) { in kern_aio_waitcomplete()
2540 ki->kaio_flags |= KAIO_WAKEUP; in kern_aio_waitcomplete()
2541 error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH, in kern_aio_waitcomplete()
2560 AIO_UNLOCK(ki); in kern_aio_waitcomplete()
2565 AIO_UNLOCK(ki); in kern_aio_waitcomplete()