1ee877a35SJohn Dyson /* 2ee877a35SJohn Dyson * Copyright (c) 1997 John S. Dyson. All rights reserved. 3ee877a35SJohn Dyson * 4ee877a35SJohn Dyson * Redistribution and use in source and binary forms, with or without 5ee877a35SJohn Dyson * modification, are permitted provided that the following conditions 6ee877a35SJohn Dyson * are met: 7ee877a35SJohn Dyson * 1. Redistributions of source code must retain the above copyright 8ee877a35SJohn Dyson * notice, this list of conditions and the following disclaimer. 9ee877a35SJohn Dyson * 2. John S. Dyson's name may not be used to endorse or promote products 10ee877a35SJohn Dyson * derived from this software without specific prior written permission. 11ee877a35SJohn Dyson * 12ee877a35SJohn Dyson * DISCLAIMER: This code isn't warranted to do anything useful. Anything 13ee877a35SJohn Dyson * bad that happens because of using this software isn't the responsibility 14ee877a35SJohn Dyson * of the author. This software is distributed AS-IS. 15ee877a35SJohn Dyson * 16c3aac50fSPeter Wemm * $FreeBSD$ 17ee877a35SJohn Dyson */ 18ee877a35SJohn Dyson 19ee877a35SJohn Dyson /* 208a6472b7SPeter Dufault * This file contains support for the POSIX 1003.1B AIO/LIO facility. 21ee877a35SJohn Dyson */ 22ee877a35SJohn Dyson 23ee877a35SJohn Dyson #include <sys/param.h> 24ee877a35SJohn Dyson #include <sys/systm.h> 259626b608SPoul-Henning Kamp #include <sys/bio.h> 26a5c9bce7SBruce Evans #include <sys/buf.h> 27ee877a35SJohn Dyson #include <sys/sysproto.h> 28ee877a35SJohn Dyson #include <sys/filedesc.h> 29ee877a35SJohn Dyson #include <sys/kernel.h> 30c9a970a7SAlan Cox #include <sys/kthread.h> 31ee877a35SJohn Dyson #include <sys/fcntl.h> 32ee877a35SJohn Dyson #include <sys/file.h> 33fdebd4f0SBruce Evans #include <sys/lock.h> 3435e0e5b3SJohn Baldwin #include <sys/mutex.h> 35ee877a35SJohn Dyson #include <sys/unistd.h> 36ee877a35SJohn Dyson #include <sys/proc.h> 372d2f8ae7SBruce Evans #include <sys/resourcevar.h> 38ee877a35SJohn Dyson #include <sys/signalvar.h> 39bfbbc4aaSJason Evans #include <sys/protosw.h> 40bfbbc4aaSJason Evans #include <sys/socketvar.h> 41a624e84fSJohn Dyson #include <sys/sysctl.h> 42fd3bf775SJohn Dyson #include <sys/vnode.h> 43fd3bf775SJohn Dyson #include <sys/conf.h> 44cb679c38SJonathan Lemon #include <sys/event.h> 45ee877a35SJohn Dyson 46ee877a35SJohn Dyson #include <vm/vm.h> 47ee877a35SJohn Dyson #include <vm/vm_extern.h> 482244ea07SJohn Dyson #include <vm/pmap.h> 492244ea07SJohn Dyson #include <vm/vm_map.h> 50fd3bf775SJohn Dyson #include <vm/vm_zone.h> 51ee877a35SJohn Dyson #include <sys/aio.h> 525aaef07cSJohn Dyson 5308637435SBruce Evans #include <machine/limits.h> 54621dbe43SBruce Evans 55dd85920aSJason Evans #include "opt_vfs_aio.h" 56ee877a35SJohn Dyson 575dec52baSPeter Wemm #ifdef VFS_AIO 585dec52baSPeter Wemm 598c12612cSDoug Rabson static long jobrefid; 602244ea07SJohn Dyson 612244ea07SJohn Dyson #define JOBST_NULL 0x0 622244ea07SJohn Dyson #define JOBST_JOBQPROC 0x1 632244ea07SJohn Dyson #define JOBST_JOBQGLOBAL 0x2 642244ea07SJohn Dyson #define JOBST_JOBRUNNING 0x3 652244ea07SJohn Dyson #define JOBST_JOBFINISHED 0x4 66fd3bf775SJohn Dyson #define JOBST_JOBQBUF 0x5 67fd3bf775SJohn Dyson #define JOBST_JOBBFINISHED 0x6 682244ea07SJohn Dyson 6984af4da6SJohn Dyson #ifndef MAX_AIO_PER_PROC 702244ea07SJohn Dyson #define MAX_AIO_PER_PROC 32 7184af4da6SJohn Dyson #endif 7284af4da6SJohn Dyson 7384af4da6SJohn Dyson #ifndef MAX_AIO_QUEUE_PER_PROC 742244ea07SJohn Dyson #define MAX_AIO_QUEUE_PER_PROC 256 /* Bigger than AIO_LISTIO_MAX */ 7584af4da6SJohn Dyson #endif 7684af4da6SJohn Dyson 7784af4da6SJohn Dyson #ifndef MAX_AIO_PROCS 78fd3bf775SJohn Dyson #define MAX_AIO_PROCS 32 7984af4da6SJohn Dyson #endif 8084af4da6SJohn Dyson 8184af4da6SJohn Dyson #ifndef MAX_AIO_QUEUE 822244ea07SJohn Dyson #define MAX_AIO_QUEUE 1024 /* Bigger than AIO_LISTIO_MAX */ 8384af4da6SJohn Dyson #endif 8484af4da6SJohn Dyson 8584af4da6SJohn Dyson #ifndef TARGET_AIO_PROCS 86bfbbc4aaSJason Evans #define TARGET_AIO_PROCS 4 8784af4da6SJohn Dyson #endif 8884af4da6SJohn Dyson 8984af4da6SJohn Dyson #ifndef MAX_BUF_AIO 9084af4da6SJohn Dyson #define MAX_BUF_AIO 16 9184af4da6SJohn Dyson #endif 9284af4da6SJohn Dyson 9384af4da6SJohn Dyson #ifndef AIOD_TIMEOUT_DEFAULT 9484af4da6SJohn Dyson #define AIOD_TIMEOUT_DEFAULT (10 * hz) 9584af4da6SJohn Dyson #endif 9684af4da6SJohn Dyson 9784af4da6SJohn Dyson #ifndef AIOD_LIFETIME_DEFAULT 9884af4da6SJohn Dyson #define AIOD_LIFETIME_DEFAULT (30 * hz) 9984af4da6SJohn Dyson #endif 1002244ea07SJohn Dyson 101303b270bSEivind Eklund static int max_aio_procs = MAX_AIO_PROCS; 102303b270bSEivind Eklund static int num_aio_procs = 0; 103303b270bSEivind Eklund static int target_aio_procs = TARGET_AIO_PROCS; 104303b270bSEivind Eklund static int max_queue_count = MAX_AIO_QUEUE; 105303b270bSEivind Eklund static int num_queue_count = 0; 106303b270bSEivind Eklund static int num_buf_aio = 0; 107303b270bSEivind Eklund static int num_aio_resv_start = 0; 108303b270bSEivind Eklund static int aiod_timeout; 109303b270bSEivind Eklund static int aiod_lifetime; 110a624e84fSJohn Dyson 111bfbbc4aaSJason Evans static int max_aio_per_proc = MAX_AIO_PER_PROC; 112bfbbc4aaSJason Evans static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC; 113303b270bSEivind Eklund static int max_buf_aio = MAX_BUF_AIO; 114a624e84fSJohn Dyson 115a624e84fSJohn Dyson SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "AIO mgmt"); 116a624e84fSJohn Dyson 117a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, 118a624e84fSJohn Dyson CTLFLAG_RW, &max_aio_per_proc, 0, ""); 119a624e84fSJohn Dyson 120a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, 121a624e84fSJohn Dyson CTLFLAG_RW, &max_aio_queue_per_proc, 0, ""); 122a624e84fSJohn Dyson 123a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs, 124a624e84fSJohn Dyson CTLFLAG_RW, &max_aio_procs, 0, ""); 125a624e84fSJohn Dyson 126a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs, 127a624e84fSJohn Dyson CTLFLAG_RD, &num_aio_procs, 0, ""); 128a624e84fSJohn Dyson 129a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, 130a624e84fSJohn Dyson CTLFLAG_RD, &num_queue_count, 0, ""); 131a624e84fSJohn Dyson 132a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, 133a624e84fSJohn Dyson CTLFLAG_RW, &max_queue_count, 0, ""); 134a624e84fSJohn Dyson 135a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, 136a624e84fSJohn Dyson CTLFLAG_RW, &target_aio_procs, 0, ""); 137a624e84fSJohn Dyson 13884af4da6SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, 13984af4da6SJohn Dyson CTLFLAG_RW, &max_buf_aio, 0, ""); 140fd3bf775SJohn Dyson 141fd3bf775SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, 142fd3bf775SJohn Dyson CTLFLAG_RD, &num_buf_aio, 0, ""); 143fd3bf775SJohn Dyson 14484af4da6SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, 14584af4da6SJohn Dyson CTLFLAG_RW, &aiod_lifetime, 0, ""); 14684af4da6SJohn Dyson 14784af4da6SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout, 14884af4da6SJohn Dyson CTLFLAG_RW, &aiod_timeout, 0, ""); 14984af4da6SJohn Dyson 1502244ea07SJohn Dyson /* 1512244ea07SJohn Dyson * AIO process info 1522244ea07SJohn Dyson */ 15384af4da6SJohn Dyson #define AIOP_FREE 0x1 /* proc on free queue */ 15484af4da6SJohn Dyson #define AIOP_SCHED 0x2 /* proc explicitly scheduled */ 15584af4da6SJohn Dyson 156b40ce416SJulian Elischer struct aiothreadlist { 157b40ce416SJulian Elischer int aiothreadflags; /* AIO proc flags */ 158b40ce416SJulian Elischer TAILQ_ENTRY(aiothreadlist) list; /* List of processes */ 159b40ce416SJulian Elischer struct thread *aiothread; /* The AIO thread */ 160e3975643SJake Burkholder TAILQ_HEAD (,aiocblist) jobtorun; /* suggested job to run */ 1612244ea07SJohn Dyson }; 1622244ea07SJohn Dyson 16384af4da6SJohn Dyson /* 16484af4da6SJohn Dyson * data-structure for lio signal management 16584af4da6SJohn Dyson */ 16684af4da6SJohn Dyson struct aio_liojob { 16784af4da6SJohn Dyson int lioj_flags; 16884af4da6SJohn Dyson int lioj_buffer_count; 16984af4da6SJohn Dyson int lioj_buffer_finished_count; 17084af4da6SJohn Dyson int lioj_queue_count; 17184af4da6SJohn Dyson int lioj_queue_finished_count; 17284af4da6SJohn Dyson struct sigevent lioj_signal; /* signal on all I/O done */ 173e3975643SJake Burkholder TAILQ_ENTRY (aio_liojob) lioj_list; 17484af4da6SJohn Dyson struct kaioinfo *lioj_ki; 17584af4da6SJohn Dyson }; 17684af4da6SJohn Dyson #define LIOJ_SIGNAL 0x1 /* signal on all done (lio) */ 17784af4da6SJohn Dyson #define LIOJ_SIGNAL_POSTED 0x2 /* signal has been posted */ 17884af4da6SJohn Dyson 17984af4da6SJohn Dyson /* 18084af4da6SJohn Dyson * per process aio data structure 18184af4da6SJohn Dyson */ 1822244ea07SJohn Dyson struct kaioinfo { 183fd3bf775SJohn Dyson int kaio_flags; /* per process kaio flags */ 1842244ea07SJohn Dyson int kaio_maxactive_count; /* maximum number of AIOs */ 1852244ea07SJohn Dyson int kaio_active_count; /* number of currently used AIOs */ 1862244ea07SJohn Dyson int kaio_qallowed_count; /* maxiumu size of AIO queue */ 1872244ea07SJohn Dyson int kaio_queue_count; /* size of AIO queue */ 188fd3bf775SJohn Dyson int kaio_ballowed_count; /* maximum number of buffers */ 18984af4da6SJohn Dyson int kaio_queue_finished_count; /* number of daemon jobs finished */ 190fd3bf775SJohn Dyson int kaio_buffer_count; /* number of physio buffers */ 19184af4da6SJohn Dyson int kaio_buffer_finished_count; /* count of I/O done */ 19284af4da6SJohn Dyson struct proc *kaio_p; /* process that uses this kaio block */ 193e3975643SJake Burkholder TAILQ_HEAD (,aio_liojob) kaio_liojoblist; /* list of lio jobs */ 194e3975643SJake Burkholder TAILQ_HEAD (,aiocblist) kaio_jobqueue; /* job queue for process */ 195e3975643SJake Burkholder TAILQ_HEAD (,aiocblist) kaio_jobdone; /* done queue for process */ 196e3975643SJake Burkholder TAILQ_HEAD (,aiocblist) kaio_bufqueue; /* buffer job queue for process */ 197e3975643SJake Burkholder TAILQ_HEAD (,aiocblist) kaio_bufdone; /* buffer done queue for process */ 198e3975643SJake Burkholder TAILQ_HEAD (,aiocblist) kaio_sockqueue; /* queue for aios waiting on sockets */ 1992244ea07SJohn Dyson }; 2002244ea07SJohn Dyson 20184af4da6SJohn Dyson #define KAIO_RUNDOWN 0x1 /* process is being run down */ 202bfbbc4aaSJason Evans #define KAIO_WAKEUP 0x2 /* wakeup process when there is a significant event */ 203fd3bf775SJohn Dyson 204b40ce416SJulian Elischer static TAILQ_HEAD(,aiothreadlist) aio_freeproc, aio_activeproc; 205e3975643SJake Burkholder static TAILQ_HEAD(,aiocblist) aio_jobs; /* Async job list */ 206e3975643SJake Burkholder static TAILQ_HEAD(,aiocblist) aio_bufjobs; /* Phys I/O job list */ 2072244ea07SJohn Dyson 208fd3bf775SJohn Dyson static void aio_init_aioinfo(struct proc *p); 209fd3bf775SJohn Dyson static void aio_onceonly(void *); 210fd3bf775SJohn Dyson static int aio_free_entry(struct aiocblist *aiocbe); 211fd3bf775SJohn Dyson static void aio_process(struct aiocblist *aiocbe); 2122244ea07SJohn Dyson static int aio_newproc(void); 213b40ce416SJulian Elischer static int aio_aqueue(struct thread *td, struct aiocb *job, int type); 214fd3bf775SJohn Dyson static void aio_physwakeup(struct buf *bp); 2159c8a2647SAlan Cox static int aio_fphysio(struct proc *p, struct aiocblist *aiocbe); 216fd3bf775SJohn Dyson static int aio_qphysio(struct proc *p, struct aiocblist *iocb); 2179c8b8baaSPeter Wemm static void aio_daemon(void *uproc); 21813644654SAlan Cox static void process_signal(void *aioj); 2192244ea07SJohn Dyson 2202244ea07SJohn Dyson SYSINIT(aio, SI_SUB_VFS, SI_ORDER_ANY, aio_onceonly, NULL); 2212244ea07SJohn Dyson 222bfbbc4aaSJason Evans static vm_zone_t kaio_zone = 0, aiop_zone = 0, aiocb_zone = 0, aiol_zone = 0; 223bfbbc4aaSJason Evans static vm_zone_t aiolio_zone = 0; 224fd3bf775SJohn Dyson 225fd3bf775SJohn Dyson /* 2262244ea07SJohn Dyson * Startup initialization 2272244ea07SJohn Dyson */ 22888ed460eSAlan Cox static void 229fd3bf775SJohn Dyson aio_onceonly(void *na) 230fd3bf775SJohn Dyson { 2312244ea07SJohn Dyson TAILQ_INIT(&aio_freeproc); 2322244ea07SJohn Dyson TAILQ_INIT(&aio_activeproc); 2332244ea07SJohn Dyson TAILQ_INIT(&aio_jobs); 234fd3bf775SJohn Dyson TAILQ_INIT(&aio_bufjobs); 235fd3bf775SJohn Dyson kaio_zone = zinit("AIO", sizeof (struct kaioinfo), 0, 0, 1); 236b40ce416SJulian Elischer aiop_zone = zinit("AIOP", sizeof (struct aiothreadlist), 0, 0, 1); 237fd3bf775SJohn Dyson aiocb_zone = zinit("AIOCB", sizeof (struct aiocblist), 0, 0, 1); 238fd3bf775SJohn Dyson aiol_zone = zinit("AIOL", AIO_LISTIO_MAX * sizeof (int), 0, 0, 1); 239bfbbc4aaSJason Evans aiolio_zone = zinit("AIOLIO", AIO_LISTIO_MAX * sizeof (struct 240bfbbc4aaSJason Evans aio_liojob), 0, 0, 1); 24184af4da6SJohn Dyson aiod_timeout = AIOD_TIMEOUT_DEFAULT; 24284af4da6SJohn Dyson aiod_lifetime = AIOD_LIFETIME_DEFAULT; 243fd3bf775SJohn Dyson jobrefid = 1; 2442244ea07SJohn Dyson } 2452244ea07SJohn Dyson 2462244ea07SJohn Dyson /* 247bfbbc4aaSJason Evans * Init the per-process aioinfo structure. The aioinfo limits are set 248bfbbc4aaSJason Evans * per-process for user limit (resource) management. 2492244ea07SJohn Dyson */ 25088ed460eSAlan Cox static void 251fd3bf775SJohn Dyson aio_init_aioinfo(struct proc *p) 252fd3bf775SJohn Dyson { 2532244ea07SJohn Dyson struct kaioinfo *ki; 2542244ea07SJohn Dyson if (p->p_aioinfo == NULL) { 255fd3bf775SJohn Dyson ki = zalloc(kaio_zone); 2562244ea07SJohn Dyson p->p_aioinfo = ki; 25784af4da6SJohn Dyson ki->kaio_flags = 0; 258a624e84fSJohn Dyson ki->kaio_maxactive_count = max_aio_per_proc; 2592244ea07SJohn Dyson ki->kaio_active_count = 0; 260a624e84fSJohn Dyson ki->kaio_qallowed_count = max_aio_queue_per_proc; 2612244ea07SJohn Dyson ki->kaio_queue_count = 0; 26284af4da6SJohn Dyson ki->kaio_ballowed_count = max_buf_aio; 263fd3bf775SJohn Dyson ki->kaio_buffer_count = 0; 26484af4da6SJohn Dyson ki->kaio_buffer_finished_count = 0; 26584af4da6SJohn Dyson ki->kaio_p = p; 2662244ea07SJohn Dyson TAILQ_INIT(&ki->kaio_jobdone); 2672244ea07SJohn Dyson TAILQ_INIT(&ki->kaio_jobqueue); 268fd3bf775SJohn Dyson TAILQ_INIT(&ki->kaio_bufdone); 269fd3bf775SJohn Dyson TAILQ_INIT(&ki->kaio_bufqueue); 27084af4da6SJohn Dyson TAILQ_INIT(&ki->kaio_liojoblist); 271bfbbc4aaSJason Evans TAILQ_INIT(&ki->kaio_sockqueue); 2722244ea07SJohn Dyson } 273bfbbc4aaSJason Evans 274bfbbc4aaSJason Evans while (num_aio_procs < target_aio_procs) 275bfbbc4aaSJason Evans aio_newproc(); 2762244ea07SJohn Dyson } 2772244ea07SJohn Dyson 2782244ea07SJohn Dyson /* 279bfbbc4aaSJason Evans * Free a job entry. Wait for completion if it is currently active, but don't 280bfbbc4aaSJason Evans * delay forever. If we delay, we return a flag that says that we have to 281bfbbc4aaSJason Evans * restart the queue scan. 2822244ea07SJohn Dyson */ 28388ed460eSAlan Cox static int 284fd3bf775SJohn Dyson aio_free_entry(struct aiocblist *aiocbe) 285fd3bf775SJohn Dyson { 2862244ea07SJohn Dyson struct kaioinfo *ki; 287b40ce416SJulian Elischer struct aiothreadlist *aiop; 28884af4da6SJohn Dyson struct aio_liojob *lj; 2892244ea07SJohn Dyson struct proc *p; 290fd3bf775SJohn Dyson int error; 29111783b14SJohn Dyson int s; 2922244ea07SJohn Dyson 2932244ea07SJohn Dyson if (aiocbe->jobstate == JOBST_NULL) 2942244ea07SJohn Dyson panic("aio_free_entry: freeing already free job"); 2952244ea07SJohn Dyson 2962244ea07SJohn Dyson p = aiocbe->userproc; 2972244ea07SJohn Dyson ki = p->p_aioinfo; 29884af4da6SJohn Dyson lj = aiocbe->lio; 2992244ea07SJohn Dyson if (ki == NULL) 3002244ea07SJohn Dyson panic("aio_free_entry: missing p->p_aioinfo"); 3012244ea07SJohn Dyson 30213644654SAlan Cox while (aiocbe->jobstate == JOBST_JOBRUNNING) { 3032244ea07SJohn Dyson if (aiocbe->jobflags & AIOCBLIST_ASYNCFREE) 3042244ea07SJohn Dyson return 0; 3052244ea07SJohn Dyson aiocbe->jobflags |= AIOCBLIST_RUNDOWN; 30613644654SAlan Cox tsleep(aiocbe, PRIBIO, "jobwai", 0); 3072244ea07SJohn Dyson } 3082244ea07SJohn Dyson aiocbe->jobflags &= ~AIOCBLIST_ASYNCFREE; 3092244ea07SJohn Dyson 310fd3bf775SJohn Dyson if (aiocbe->bp == NULL) { 3112244ea07SJohn Dyson if (ki->kaio_queue_count <= 0) 3122244ea07SJohn Dyson panic("aio_free_entry: process queue size <= 0"); 3132244ea07SJohn Dyson if (num_queue_count <= 0) 3142244ea07SJohn Dyson panic("aio_free_entry: system wide queue size <= 0"); 3152244ea07SJohn Dyson 31684af4da6SJohn Dyson if (lj) { 31784af4da6SJohn Dyson lj->lioj_queue_count--; 31884af4da6SJohn Dyson if (aiocbe->jobflags & AIOCBLIST_DONE) 31984af4da6SJohn Dyson lj->lioj_queue_finished_count--; 32084af4da6SJohn Dyson } 32184af4da6SJohn Dyson ki->kaio_queue_count--; 32284af4da6SJohn Dyson if (aiocbe->jobflags & AIOCBLIST_DONE) 32384af4da6SJohn Dyson ki->kaio_queue_finished_count--; 32484af4da6SJohn Dyson num_queue_count--; 325fd3bf775SJohn Dyson } else { 32684af4da6SJohn Dyson if (lj) { 32784af4da6SJohn Dyson lj->lioj_buffer_count--; 32884af4da6SJohn Dyson if (aiocbe->jobflags & AIOCBLIST_DONE) 32984af4da6SJohn Dyson lj->lioj_buffer_finished_count--; 33084af4da6SJohn Dyson } 33184af4da6SJohn Dyson if (aiocbe->jobflags & AIOCBLIST_DONE) 33284af4da6SJohn Dyson ki->kaio_buffer_finished_count--; 33384af4da6SJohn Dyson ki->kaio_buffer_count--; 33484af4da6SJohn Dyson num_buf_aio--; 335fd3bf775SJohn Dyson } 336fd3bf775SJohn Dyson 337cb679c38SJonathan Lemon /* aiocbe is going away, we need to destroy any knotes */ 338b40ce416SJulian Elischer knote_remove(&p->p_thread, &aiocbe->klist); /* XXXKSE */ 339b40ce416SJulian Elischer /* XXXKSE Note the thread here is used to eventually find the 340b40ce416SJulian Elischer * owning process again, but it is also used to do a fo_close 341b40ce416SJulian Elischer * and that requires the thread. (but does it require the 342b40ce416SJulian Elischer * OWNING thread? (or maby the running thread?) 343b40ce416SJulian Elischer * There is a semantic problem here... 344b40ce416SJulian Elischer */ 345cb679c38SJonathan Lemon 346bfbbc4aaSJason Evans if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags & KAIO_RUNDOWN) 347bfbbc4aaSJason Evans && ((ki->kaio_buffer_count == 0) && (ki->kaio_queue_count == 0)))) { 348fd3bf775SJohn Dyson ki->kaio_flags &= ~KAIO_WAKEUP; 349fd3bf775SJohn Dyson wakeup(p); 350fd3bf775SJohn Dyson } 351fd3bf775SJohn Dyson 352fd3bf775SJohn Dyson if (aiocbe->jobstate == JOBST_JOBQBUF) { 3539c8a2647SAlan Cox if ((error = aio_fphysio(p, aiocbe)) != 0) 354fd3bf775SJohn Dyson return error; 355fd3bf775SJohn Dyson if (aiocbe->jobstate != JOBST_JOBBFINISHED) 356fd3bf775SJohn Dyson panic("aio_free_entry: invalid physio finish-up state"); 35711783b14SJohn Dyson s = splbio(); 358fd3bf775SJohn Dyson TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist); 35911783b14SJohn Dyson splx(s); 360fd3bf775SJohn Dyson } else if (aiocbe->jobstate == JOBST_JOBQPROC) { 361b40ce416SJulian Elischer aiop = aiocbe->jobaiothread; 3622244ea07SJohn Dyson TAILQ_REMOVE(&aiop->jobtorun, aiocbe, list); 36313644654SAlan Cox } else if (aiocbe->jobstate == JOBST_JOBQGLOBAL) { 3642244ea07SJohn Dyson TAILQ_REMOVE(&aio_jobs, aiocbe, list); 36513644654SAlan Cox TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist); 36613644654SAlan Cox } else if (aiocbe->jobstate == JOBST_JOBFINISHED) 3672244ea07SJohn Dyson TAILQ_REMOVE(&ki->kaio_jobdone, aiocbe, plist); 368bfbbc4aaSJason Evans else if (aiocbe->jobstate == JOBST_JOBBFINISHED) { 36911783b14SJohn Dyson s = splbio(); 370fd3bf775SJohn Dyson TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist); 37111783b14SJohn Dyson splx(s); 37284af4da6SJohn Dyson if (aiocbe->bp) { 37384af4da6SJohn Dyson vunmapbuf(aiocbe->bp); 3741c7c3c6aSMatthew Dillon relpbuf(aiocbe->bp, NULL); 37584af4da6SJohn Dyson aiocbe->bp = NULL; 37684af4da6SJohn Dyson } 37784af4da6SJohn Dyson } 37884af4da6SJohn Dyson if (lj && (lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 0)) { 37984af4da6SJohn Dyson TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list); 38084af4da6SJohn Dyson zfree(aiolio_zone, lj); 3812244ea07SJohn Dyson } 3822244ea07SJohn Dyson aiocbe->jobstate = JOBST_NULL; 38313644654SAlan Cox untimeout(process_signal, aiocbe, aiocbe->timeouthandle); 38488ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 3852244ea07SJohn Dyson return 0; 3862244ea07SJohn Dyson } 3875dec52baSPeter Wemm #endif /* VFS_AIO */ 3882244ea07SJohn Dyson 3892244ea07SJohn Dyson /* 3902244ea07SJohn Dyson * Rundown the jobs for a given process. 3912244ea07SJohn Dyson */ 3922244ea07SJohn Dyson void 393fd3bf775SJohn Dyson aio_proc_rundown(struct proc *p) 394fd3bf775SJohn Dyson { 3955dec52baSPeter Wemm #ifndef VFS_AIO 3965dec52baSPeter Wemm return; 3975dec52baSPeter Wemm #else 39884af4da6SJohn Dyson int s; 3992244ea07SJohn Dyson struct kaioinfo *ki; 40084af4da6SJohn Dyson struct aio_liojob *lj, *ljn; 4012244ea07SJohn Dyson struct aiocblist *aiocbe, *aiocbn; 402bfbbc4aaSJason Evans struct file *fp; 403bfbbc4aaSJason Evans struct filedesc *fdp; 404bfbbc4aaSJason Evans struct socket *so; 4052244ea07SJohn Dyson 4062244ea07SJohn Dyson ki = p->p_aioinfo; 4072244ea07SJohn Dyson if (ki == NULL) 4082244ea07SJohn Dyson return; 4092244ea07SJohn Dyson 41084af4da6SJohn Dyson ki->kaio_flags |= LIOJ_SIGNAL_POSTED; 411bfbbc4aaSJason Evans while ((ki->kaio_active_count > 0) || (ki->kaio_buffer_count > 412bfbbc4aaSJason Evans ki->kaio_buffer_finished_count)) { 413fd3bf775SJohn Dyson ki->kaio_flags |= KAIO_RUNDOWN; 41484af4da6SJohn Dyson if (tsleep(p, PRIBIO, "kaiowt", aiod_timeout)) 415a624e84fSJohn Dyson break; 416a624e84fSJohn Dyson } 417a624e84fSJohn Dyson 418bfbbc4aaSJason Evans /* 419bfbbc4aaSJason Evans * Move any aio ops that are waiting on socket I/O to the normal job 420bfbbc4aaSJason Evans * queues so they are cleaned up with any others. 421bfbbc4aaSJason Evans */ 422bfbbc4aaSJason Evans fdp = p->p_fd; 423bfbbc4aaSJason Evans 424bfbbc4aaSJason Evans s = splnet(); 425bfbbc4aaSJason Evans for (aiocbe = TAILQ_FIRST(&ki->kaio_sockqueue); aiocbe; aiocbe = 426bfbbc4aaSJason Evans aiocbn) { 427bfbbc4aaSJason Evans aiocbn = TAILQ_NEXT(aiocbe, plist); 428bfbbc4aaSJason Evans fp = fdp->fd_ofiles[aiocbe->uaiocb.aio_fildes]; 429bfbbc4aaSJason Evans 430bfbbc4aaSJason Evans /* 431bfbbc4aaSJason Evans * Under some circumstances, the aio_fildes and the file 432bfbbc4aaSJason Evans * structure don't match. This would leave aiocbe's in the 433bfbbc4aaSJason Evans * TAILQ associated with the socket and cause a panic later. 434bfbbc4aaSJason Evans * 435bfbbc4aaSJason Evans * Detect and fix. 436bfbbc4aaSJason Evans */ 437bfbbc4aaSJason Evans if ((fp == NULL) || (fp != aiocbe->fd_file)) 438bfbbc4aaSJason Evans fp = aiocbe->fd_file; 439bfbbc4aaSJason Evans if (fp) { 440bfbbc4aaSJason Evans so = (struct socket *)fp->f_data; 441bfbbc4aaSJason Evans TAILQ_REMOVE(&so->so_aiojobq, aiocbe, list); 442bfbbc4aaSJason Evans if (TAILQ_EMPTY(&so->so_aiojobq)) { 443bfbbc4aaSJason Evans so->so_snd.sb_flags &= ~SB_AIO; 444bfbbc4aaSJason Evans so->so_rcv.sb_flags &= ~SB_AIO; 445bfbbc4aaSJason Evans } 446bfbbc4aaSJason Evans } 447bfbbc4aaSJason Evans TAILQ_REMOVE(&ki->kaio_sockqueue, aiocbe, plist); 448bfbbc4aaSJason Evans TAILQ_INSERT_HEAD(&aio_jobs, aiocbe, list); 449bfbbc4aaSJason Evans TAILQ_INSERT_HEAD(&ki->kaio_jobqueue, aiocbe, plist); 450bfbbc4aaSJason Evans } 451bfbbc4aaSJason Evans splx(s); 452bfbbc4aaSJason Evans 4532244ea07SJohn Dyson restart1: 454bfbbc4aaSJason Evans for (aiocbe = TAILQ_FIRST(&ki->kaio_jobdone); aiocbe; aiocbe = aiocbn) { 4552244ea07SJohn Dyson aiocbn = TAILQ_NEXT(aiocbe, plist); 4562244ea07SJohn Dyson if (aio_free_entry(aiocbe)) 4572244ea07SJohn Dyson goto restart1; 4582244ea07SJohn Dyson } 4592244ea07SJohn Dyson 4602244ea07SJohn Dyson restart2: 461bfbbc4aaSJason Evans for (aiocbe = TAILQ_FIRST(&ki->kaio_jobqueue); aiocbe; aiocbe = 462bfbbc4aaSJason Evans aiocbn) { 4632244ea07SJohn Dyson aiocbn = TAILQ_NEXT(aiocbe, plist); 4642244ea07SJohn Dyson if (aio_free_entry(aiocbe)) 4652244ea07SJohn Dyson goto restart2; 4662244ea07SJohn Dyson } 46784af4da6SJohn Dyson 468c5efdcbdSJohn Dyson /* 469bfbbc4aaSJason Evans * Note the use of lots of splbio here, trying to avoid splbio for long chains 470bfbbc4aaSJason Evans * of I/O. Probably unnecessary. 471c5efdcbdSJohn Dyson */ 47284af4da6SJohn Dyson restart3: 47384af4da6SJohn Dyson s = splbio(); 47484af4da6SJohn Dyson while (TAILQ_FIRST(&ki->kaio_bufqueue)) { 47584af4da6SJohn Dyson ki->kaio_flags |= KAIO_WAKEUP; 47684af4da6SJohn Dyson tsleep(p, PRIBIO, "aioprn", 0); 47784af4da6SJohn Dyson splx(s); 47884af4da6SJohn Dyson goto restart3; 47984af4da6SJohn Dyson } 480c5efdcbdSJohn Dyson splx(s); 48184af4da6SJohn Dyson 48284af4da6SJohn Dyson restart4: 48384af4da6SJohn Dyson s = splbio(); 484bfbbc4aaSJason Evans for (aiocbe = TAILQ_FIRST(&ki->kaio_bufdone); aiocbe; aiocbe = aiocbn) { 48584af4da6SJohn Dyson aiocbn = TAILQ_NEXT(aiocbe, plist); 48684af4da6SJohn Dyson if (aio_free_entry(aiocbe)) { 48784af4da6SJohn Dyson splx(s); 48884af4da6SJohn Dyson goto restart4; 48984af4da6SJohn Dyson } 49084af4da6SJohn Dyson } 49184af4da6SJohn Dyson splx(s); 49284af4da6SJohn Dyson 49313644654SAlan Cox /* 49413644654SAlan Cox * If we've slept, jobs might have moved from one queue to another. 49513644654SAlan Cox * Retry rundown if we didn't manage to empty the queues. 49613644654SAlan Cox */ 49713644654SAlan Cox if (TAILQ_FIRST(&ki->kaio_jobdone) != NULL || 49813644654SAlan Cox TAILQ_FIRST(&ki->kaio_jobqueue) != NULL || 49913644654SAlan Cox TAILQ_FIRST(&ki->kaio_bufqueue) != NULL || 50013644654SAlan Cox TAILQ_FIRST(&ki->kaio_bufdone) != NULL) 50113644654SAlan Cox goto restart1; 50213644654SAlan Cox 503bfbbc4aaSJason Evans for (lj = TAILQ_FIRST(&ki->kaio_liojoblist); lj; lj = ljn) { 50484af4da6SJohn Dyson ljn = TAILQ_NEXT(lj, lioj_list); 505bfbbc4aaSJason Evans if ((lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 506bfbbc4aaSJason Evans 0)) { 50784af4da6SJohn Dyson TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list); 50884af4da6SJohn Dyson zfree(aiolio_zone, lj); 509f4f0ecefSJohn Dyson } else { 510bfbbc4aaSJason Evans #ifdef DIAGNOSTIC 511bfbbc4aaSJason Evans printf("LIO job not cleaned up: B:%d, BF:%d, Q:%d, " 512bfbbc4aaSJason Evans "QF:%d\n", lj->lioj_buffer_count, 513bfbbc4aaSJason Evans lj->lioj_buffer_finished_count, 514bfbbc4aaSJason Evans lj->lioj_queue_count, 515bfbbc4aaSJason Evans lj->lioj_queue_finished_count); 51611783b14SJohn Dyson #endif 51784af4da6SJohn Dyson } 518f4f0ecefSJohn Dyson } 51984af4da6SJohn Dyson 520fd3bf775SJohn Dyson zfree(kaio_zone, ki); 521a624e84fSJohn Dyson p->p_aioinfo = NULL; 5225dec52baSPeter Wemm #endif /* VFS_AIO */ 5232244ea07SJohn Dyson } 5242244ea07SJohn Dyson 5255dec52baSPeter Wemm #ifdef VFS_AIO 5262244ea07SJohn Dyson /* 527bfbbc4aaSJason Evans * Select a job to run (called by an AIO daemon). 5282244ea07SJohn Dyson */ 5292244ea07SJohn Dyson static struct aiocblist * 530b40ce416SJulian Elischer aio_selectjob(struct aiothreadlist *aiop) 531fd3bf775SJohn Dyson { 532bfbbc4aaSJason Evans int s; 5332244ea07SJohn Dyson struct aiocblist *aiocbe; 534bfbbc4aaSJason Evans struct kaioinfo *ki; 535bfbbc4aaSJason Evans struct proc *userp; 5362244ea07SJohn Dyson 5372244ea07SJohn Dyson aiocbe = TAILQ_FIRST(&aiop->jobtorun); 5382244ea07SJohn Dyson if (aiocbe) { 5392244ea07SJohn Dyson TAILQ_REMOVE(&aiop->jobtorun, aiocbe, list); 5402244ea07SJohn Dyson return aiocbe; 5412244ea07SJohn Dyson } 5422244ea07SJohn Dyson 543bfbbc4aaSJason Evans s = splnet(); 544bfbbc4aaSJason Evans for (aiocbe = TAILQ_FIRST(&aio_jobs); aiocbe; aiocbe = 545bfbbc4aaSJason Evans TAILQ_NEXT(aiocbe, list)) { 5462244ea07SJohn Dyson userp = aiocbe->userproc; 5472244ea07SJohn Dyson ki = userp->p_aioinfo; 5482244ea07SJohn Dyson 5492244ea07SJohn Dyson if (ki->kaio_active_count < ki->kaio_maxactive_count) { 5502244ea07SJohn Dyson TAILQ_REMOVE(&aio_jobs, aiocbe, list); 551bfbbc4aaSJason Evans splx(s); 5522244ea07SJohn Dyson return aiocbe; 5532244ea07SJohn Dyson } 5542244ea07SJohn Dyson } 555bfbbc4aaSJason Evans splx(s); 5562244ea07SJohn Dyson 5572244ea07SJohn Dyson return NULL; 5582244ea07SJohn Dyson } 5592244ea07SJohn Dyson 5602244ea07SJohn Dyson /* 561bfbbc4aaSJason Evans * The AIO processing activity. This is the code that does the I/O request for 562bfbbc4aaSJason Evans * the non-physio version of the operations. The normal vn operations are used, 563bfbbc4aaSJason Evans * and this code should work in all instances for every type of file, including 564bfbbc4aaSJason Evans * pipes, sockets, fifos, and regular files. 5652244ea07SJohn Dyson */ 56688ed460eSAlan Cox static void 567fd3bf775SJohn Dyson aio_process(struct aiocblist *aiocbe) 568fd3bf775SJohn Dyson { 5692244ea07SJohn Dyson struct filedesc *fdp; 570b40ce416SJulian Elischer struct thread *td; 571b40ce416SJulian Elischer struct proc *userp; 572b40ce416SJulian Elischer struct proc *mycp; 5732244ea07SJohn Dyson struct aiocb *cb; 5742244ea07SJohn Dyson struct file *fp; 5752244ea07SJohn Dyson struct uio auio; 5762244ea07SJohn Dyson struct iovec aiov; 5772244ea07SJohn Dyson unsigned int fd; 5782244ea07SJohn Dyson int cnt; 5792244ea07SJohn Dyson int error; 580a624e84fSJohn Dyson off_t offset; 581fd3bf775SJohn Dyson int oublock_st, oublock_end; 582fd3bf775SJohn Dyson int inblock_st, inblock_end; 5832244ea07SJohn Dyson 5842244ea07SJohn Dyson userp = aiocbe->userproc; 585b40ce416SJulian Elischer td = curthread; 586b40ce416SJulian Elischer mycp = td->td_proc; 5872244ea07SJohn Dyson cb = &aiocbe->uaiocb; 5882244ea07SJohn Dyson 589fd3bf775SJohn Dyson fdp = mycp->p_fd; 5902244ea07SJohn Dyson fd = cb->aio_fildes; 5912244ea07SJohn Dyson fp = fdp->fd_ofiles[fd]; 5922244ea07SJohn Dyson 593bfbbc4aaSJason Evans if ((fp == NULL) || (fp != aiocbe->fd_file)) { 594bfbbc4aaSJason Evans cb->_aiocb_private.error = EBADF; 595bfbbc4aaSJason Evans cb->_aiocb_private.status = -1; 596bfbbc4aaSJason Evans return; 597bfbbc4aaSJason Evans } 598bfbbc4aaSJason Evans 599b40ce416SJulian Elischer aiov.iov_base = cb->aio_buf; 6002244ea07SJohn Dyson aiov.iov_len = cb->aio_nbytes; 6012244ea07SJohn Dyson 6022244ea07SJohn Dyson auio.uio_iov = &aiov; 6032244ea07SJohn Dyson auio.uio_iovcnt = 1; 604a624e84fSJohn Dyson auio.uio_offset = offset = cb->aio_offset; 6052244ea07SJohn Dyson auio.uio_resid = cb->aio_nbytes; 6062244ea07SJohn Dyson cnt = cb->aio_nbytes; 6072244ea07SJohn Dyson auio.uio_segflg = UIO_USERSPACE; 608b40ce416SJulian Elischer auio.uio_td = td; 6092244ea07SJohn Dyson 610fd3bf775SJohn Dyson inblock_st = mycp->p_stats->p_ru.ru_inblock; 611fd3bf775SJohn Dyson oublock_st = mycp->p_stats->p_ru.ru_oublock; 612279d7226SMatthew Dillon /* 613279d7226SMatthew Dillon * Temporarily bump the ref count while reading to avoid the 614279d7226SMatthew Dillon * descriptor being ripped out from under us. 615279d7226SMatthew Dillon */ 616279d7226SMatthew Dillon fhold(fp); 6172244ea07SJohn Dyson if (cb->aio_lio_opcode == LIO_READ) { 6182244ea07SJohn Dyson auio.uio_rw = UIO_READ; 619b40ce416SJulian Elischer error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td); 6202244ea07SJohn Dyson } else { 6212244ea07SJohn Dyson auio.uio_rw = UIO_WRITE; 622b40ce416SJulian Elischer error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td); 6232244ea07SJohn Dyson } 624b40ce416SJulian Elischer fdrop(fp, td); 625fd3bf775SJohn Dyson inblock_end = mycp->p_stats->p_ru.ru_inblock; 626fd3bf775SJohn Dyson oublock_end = mycp->p_stats->p_ru.ru_oublock; 627fd3bf775SJohn Dyson 628fd3bf775SJohn Dyson aiocbe->inputcharge = inblock_end - inblock_st; 629fd3bf775SJohn Dyson aiocbe->outputcharge = oublock_end - oublock_st; 6302244ea07SJohn Dyson 631bfbbc4aaSJason Evans if ((error) && (auio.uio_resid != cnt)) { 6322244ea07SJohn Dyson if (error == ERESTART || error == EINTR || error == EWOULDBLOCK) 6332244ea07SJohn Dyson error = 0; 63419eb87d2SJohn Baldwin if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) { 63519eb87d2SJohn Baldwin PROC_LOCK(userp); 6362244ea07SJohn Dyson psignal(userp, SIGPIPE); 63719eb87d2SJohn Baldwin PROC_UNLOCK(userp); 63819eb87d2SJohn Baldwin } 6392244ea07SJohn Dyson } 6402244ea07SJohn Dyson 6412244ea07SJohn Dyson cnt -= auio.uio_resid; 6422244ea07SJohn Dyson cb->_aiocb_private.error = error; 6432244ea07SJohn Dyson cb->_aiocb_private.status = cnt; 6442244ea07SJohn Dyson } 6452244ea07SJohn Dyson 6462244ea07SJohn Dyson /* 64784af4da6SJohn Dyson * The AIO daemon, most of the actual work is done in aio_process, 64884af4da6SJohn Dyson * but the setup (and address space mgmt) is done in this routine. 6492244ea07SJohn Dyson */ 6502244ea07SJohn Dyson static void 6519c8b8baaSPeter Wemm aio_daemon(void *uproc) 6522244ea07SJohn Dyson { 65384af4da6SJohn Dyson int s; 654bfbbc4aaSJason Evans struct aio_liojob *lj; 655bfbbc4aaSJason Evans struct aiocb *cb; 656bfbbc4aaSJason Evans struct aiocblist *aiocbe; 657b40ce416SJulian Elischer struct aiothreadlist *aiop; 658bfbbc4aaSJason Evans struct kaioinfo *ki; 659bfbbc4aaSJason Evans struct proc *curcp, *mycp, *userp; 660bfbbc4aaSJason Evans struct vmspace *myvm, *tmpvm; 661b40ce416SJulian Elischer struct thread *td = curthread; 6622244ea07SJohn Dyson 6639ed346baSBosko Milekic mtx_lock(&Giant); 6642244ea07SJohn Dyson /* 665fd3bf775SJohn Dyson * Local copies of curproc (cp) and vmspace (myvm) 6662244ea07SJohn Dyson */ 667b40ce416SJulian Elischer mycp = td->td_proc; 668fd3bf775SJohn Dyson myvm = mycp->p_vmspace; 669fd3bf775SJohn Dyson 670fd3bf775SJohn Dyson if (mycp->p_textvp) { 671fd3bf775SJohn Dyson vrele(mycp->p_textvp); 672fd3bf775SJohn Dyson mycp->p_textvp = NULL; 673fd3bf775SJohn Dyson } 674fd3bf775SJohn Dyson 675fd3bf775SJohn Dyson /* 676bfbbc4aaSJason Evans * Allocate and ready the aio control info. There is one aiop structure 677bfbbc4aaSJason Evans * per daemon. 678fd3bf775SJohn Dyson */ 679fd3bf775SJohn Dyson aiop = zalloc(aiop_zone); 680b40ce416SJulian Elischer aiop->aiothread = td; 681b40ce416SJulian Elischer aiop->aiothreadflags |= AIOP_FREE; 6822244ea07SJohn Dyson TAILQ_INIT(&aiop->jobtorun); 6832244ea07SJohn Dyson 684bfbbc4aaSJason Evans s = splnet(); 685bfbbc4aaSJason Evans 6862244ea07SJohn Dyson /* 687bfbbc4aaSJason Evans * Place thread (lightweight process) onto the AIO free thread list. 6882244ea07SJohn Dyson */ 689fd3bf775SJohn Dyson if (TAILQ_EMPTY(&aio_freeproc)) 690fd3bf775SJohn Dyson wakeup(&aio_freeproc); 691fd3bf775SJohn Dyson TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list); 6922244ea07SJohn Dyson 693bfbbc4aaSJason Evans splx(s); 694bfbbc4aaSJason Evans 6952244ea07SJohn Dyson /* 696fd3bf775SJohn Dyson * Get rid of our current filedescriptors. AIOD's don't need any 697fd3bf775SJohn Dyson * filedescriptors, except as temporarily inherited from the client. 698bfbbc4aaSJason Evans * Credentials are also cloned, and made equivalent to "root". 6992244ea07SJohn Dyson */ 700b40ce416SJulian Elischer fdfree(td); 701fd3bf775SJohn Dyson mycp->p_fd = NULL; 702fd3bf775SJohn Dyson mycp->p_ucred = crcopy(mycp->p_ucred); 703fd3bf775SJohn Dyson mycp->p_ucred->cr_uid = 0; 704f535380cSDon Lewis uifree(mycp->p_ucred->cr_uidinfo); 705f535380cSDon Lewis mycp->p_ucred->cr_uidinfo = uifind(0); 706fd3bf775SJohn Dyson mycp->p_ucred->cr_ngroups = 1; 707fd3bf775SJohn Dyson mycp->p_ucred->cr_groups[0] = 1; 708fd3bf775SJohn Dyson 709bfbbc4aaSJason Evans /* The daemon resides in its own pgrp. */ 710fd3bf775SJohn Dyson enterpgrp(mycp, mycp->p_pid, 1); 711fd3bf775SJohn Dyson 712bfbbc4aaSJason Evans /* Mark special process type. */ 71386360feeSJake Burkholder mycp->p_flag |= P_SYSTEM; 7142244ea07SJohn Dyson 715fd3bf775SJohn Dyson /* 716fd3bf775SJohn Dyson * Wakeup parent process. (Parent sleeps to keep from blasting away 717b40ce416SJulian Elischer * and creating too many daemons.) 718fd3bf775SJohn Dyson */ 719fd3bf775SJohn Dyson wakeup(mycp); 7202244ea07SJohn Dyson 721bfbbc4aaSJason Evans for (;;) { 722fd3bf775SJohn Dyson /* 723fd3bf775SJohn Dyson * curcp is the current daemon process context. 724fd3bf775SJohn Dyson * userp is the current user process context. 725fd3bf775SJohn Dyson */ 726fd3bf775SJohn Dyson curcp = mycp; 727c4860686SJohn Dyson 728fd3bf775SJohn Dyson /* 729fd3bf775SJohn Dyson * Take daemon off of free queue 730fd3bf775SJohn Dyson */ 731b40ce416SJulian Elischer if (aiop->aiothreadflags & AIOP_FREE) { 732bfbbc4aaSJason Evans s = splnet(); 7332244ea07SJohn Dyson TAILQ_REMOVE(&aio_freeproc, aiop, list); 7342244ea07SJohn Dyson TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list); 735b40ce416SJulian Elischer aiop->aiothreadflags &= ~AIOP_FREE; 736bfbbc4aaSJason Evans splx(s); 7372244ea07SJohn Dyson } 738b40ce416SJulian Elischer aiop->aiothreadflags &= ~AIOP_SCHED; 7392244ea07SJohn Dyson 740fd3bf775SJohn Dyson /* 741bfbbc4aaSJason Evans * Check for jobs. 742fd3bf775SJohn Dyson */ 743d254af07SMatthew Dillon while ((aiocbe = aio_selectjob(aiop)) != NULL) { 7442244ea07SJohn Dyson cb = &aiocbe->uaiocb; 7452244ea07SJohn Dyson userp = aiocbe->userproc; 7462244ea07SJohn Dyson 7472244ea07SJohn Dyson aiocbe->jobstate = JOBST_JOBRUNNING; 748fd3bf775SJohn Dyson 749fd3bf775SJohn Dyson /* 750bfbbc4aaSJason Evans * Connect to process address space for user program. 751fd3bf775SJohn Dyson */ 752fd3bf775SJohn Dyson if (userp != curcp) { 753fd3bf775SJohn Dyson /* 754bfbbc4aaSJason Evans * Save the current address space that we are 755bfbbc4aaSJason Evans * connected to. 756fd3bf775SJohn Dyson */ 757fd3bf775SJohn Dyson tmpvm = mycp->p_vmspace; 758bfbbc4aaSJason Evans 759fd3bf775SJohn Dyson /* 760bfbbc4aaSJason Evans * Point to the new user address space, and 761bfbbc4aaSJason Evans * refer to it. 762fd3bf775SJohn Dyson */ 763fd3bf775SJohn Dyson mycp->p_vmspace = userp->p_vmspace; 76484af4da6SJohn Dyson mycp->p_vmspace->vm_refcnt++; 765bfbbc4aaSJason Evans 766bfbbc4aaSJason Evans /* Activate the new mapping. */ 767b40ce416SJulian Elischer pmap_activate(&mycp->p_thread); 768bfbbc4aaSJason Evans 769fd3bf775SJohn Dyson /* 770bfbbc4aaSJason Evans * If the old address space wasn't the daemons 771bfbbc4aaSJason Evans * own address space, then we need to remove the 772bfbbc4aaSJason Evans * daemon's reference from the other process 773bfbbc4aaSJason Evans * that it was acting on behalf of. 774fd3bf775SJohn Dyson */ 7752244ea07SJohn Dyson if (tmpvm != myvm) { 7762244ea07SJohn Dyson vmspace_free(tmpvm); 7772244ea07SJohn Dyson } 778bfbbc4aaSJason Evans 779fd3bf775SJohn Dyson /* 780bfbbc4aaSJason Evans * Disassociate from previous clients file 781bfbbc4aaSJason Evans * descriptors, and associate to the new clients 782bfbbc4aaSJason Evans * descriptors. Note that the daemon doesn't 783bfbbc4aaSJason Evans * need to worry about its orginal descriptors, 784bfbbc4aaSJason Evans * because they were originally freed. 785fd3bf775SJohn Dyson */ 786fd3bf775SJohn Dyson if (mycp->p_fd) 787b40ce416SJulian Elischer fdfree(td); 788fd3bf775SJohn Dyson mycp->p_fd = fdshare(userp); 789fd3bf775SJohn Dyson curcp = userp; 7902244ea07SJohn Dyson } 7912244ea07SJohn Dyson 792fd3bf775SJohn Dyson ki = userp->p_aioinfo; 79384af4da6SJohn Dyson lj = aiocbe->lio; 79484af4da6SJohn Dyson 795bfbbc4aaSJason Evans /* Account for currently active jobs. */ 7962244ea07SJohn Dyson ki->kaio_active_count++; 79711783b14SJohn Dyson 798bfbbc4aaSJason Evans /* Do the I/O function. */ 799b40ce416SJulian Elischer aiocbe->jobaiothread = aiop; 8002244ea07SJohn Dyson aio_process(aiocbe); 80184af4da6SJohn Dyson 802bfbbc4aaSJason Evans /* Decrement the active job count. */ 80384af4da6SJohn Dyson ki->kaio_active_count--; 80484af4da6SJohn Dyson 80584af4da6SJohn Dyson /* 806bfbbc4aaSJason Evans * Increment the completion count for wakeup/signal 807bfbbc4aaSJason Evans * comparisons. 80884af4da6SJohn Dyson */ 80984af4da6SJohn Dyson aiocbe->jobflags |= AIOCBLIST_DONE; 81084af4da6SJohn Dyson ki->kaio_queue_finished_count++; 811bfbbc4aaSJason Evans if (lj) 81284af4da6SJohn Dyson lj->lioj_queue_finished_count++; 813bfbbc4aaSJason Evans if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags 814bfbbc4aaSJason Evans & KAIO_RUNDOWN) && (ki->kaio_active_count == 0))) { 815fd3bf775SJohn Dyson ki->kaio_flags &= ~KAIO_WAKEUP; 816fd3bf775SJohn Dyson wakeup(userp); 817fd3bf775SJohn Dyson } 8182244ea07SJohn Dyson 81984af4da6SJohn Dyson s = splbio(); 820bfbbc4aaSJason Evans if (lj && (lj->lioj_flags & 821bfbbc4aaSJason Evans (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == LIOJ_SIGNAL) { 822bfbbc4aaSJason Evans if ((lj->lioj_queue_finished_count == 823bfbbc4aaSJason Evans lj->lioj_queue_count) && 824bfbbc4aaSJason Evans (lj->lioj_buffer_finished_count == 825bfbbc4aaSJason Evans lj->lioj_buffer_count)) { 82619eb87d2SJohn Baldwin PROC_LOCK(userp); 827bfbbc4aaSJason Evans psignal(userp, 828bfbbc4aaSJason Evans lj->lioj_signal.sigev_signo); 82919eb87d2SJohn Baldwin PROC_UNLOCK(userp); 83019eb87d2SJohn Baldwin lj->lioj_flags |= LIOJ_SIGNAL_POSTED; 83184af4da6SJohn Dyson } 83284af4da6SJohn Dyson } 83384af4da6SJohn Dyson splx(s); 83484af4da6SJohn Dyson 8352244ea07SJohn Dyson aiocbe->jobstate = JOBST_JOBFINISHED; 8362244ea07SJohn Dyson 837fd3bf775SJohn Dyson /* 838bfbbc4aaSJason Evans * If the I/O request should be automatically rundown, 839bfbbc4aaSJason Evans * do the needed cleanup. Otherwise, place the queue 840bfbbc4aaSJason Evans * entry for the just finished I/O request into the done 841bfbbc4aaSJason Evans * queue for the associated client. 842fd3bf775SJohn Dyson */ 843bfbbc4aaSJason Evans s = splnet(); 8442244ea07SJohn Dyson if (aiocbe->jobflags & AIOCBLIST_ASYNCFREE) { 8452244ea07SJohn Dyson aiocbe->jobflags &= ~AIOCBLIST_ASYNCFREE; 84688ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 8472244ea07SJohn Dyson } else { 848bfbbc4aaSJason Evans TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist); 849bfbbc4aaSJason Evans TAILQ_INSERT_TAIL(&ki->kaio_jobdone, aiocbe, 850bfbbc4aaSJason Evans plist); 8512244ea07SJohn Dyson } 852bfbbc4aaSJason Evans splx(s); 853cb679c38SJonathan Lemon KNOTE(&aiocbe->klist, 0); 8542244ea07SJohn Dyson 8552244ea07SJohn Dyson if (aiocbe->jobflags & AIOCBLIST_RUNDOWN) { 8562244ea07SJohn Dyson wakeup(aiocbe); 8572244ea07SJohn Dyson aiocbe->jobflags &= ~AIOCBLIST_RUNDOWN; 8582244ea07SJohn Dyson } 8592244ea07SJohn Dyson 8602244ea07SJohn Dyson if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) { 86119eb87d2SJohn Baldwin PROC_LOCK(userp); 8622244ea07SJohn Dyson psignal(userp, cb->aio_sigevent.sigev_signo); 86319eb87d2SJohn Baldwin PROC_UNLOCK(userp); 8642244ea07SJohn Dyson } 8652244ea07SJohn Dyson } 8662244ea07SJohn Dyson 867fd3bf775SJohn Dyson /* 868bfbbc4aaSJason Evans * Disconnect from user address space. 869fd3bf775SJohn Dyson */ 870fd3bf775SJohn Dyson if (curcp != mycp) { 871bfbbc4aaSJason Evans /* Get the user address space to disconnect from. */ 872fd3bf775SJohn Dyson tmpvm = mycp->p_vmspace; 873bfbbc4aaSJason Evans 874bfbbc4aaSJason Evans /* Get original address space for daemon. */ 875fd3bf775SJohn Dyson mycp->p_vmspace = myvm; 876bfbbc4aaSJason Evans 877bfbbc4aaSJason Evans /* Activate the daemon's address space. */ 878b40ce416SJulian Elischer pmap_activate(&mycp->p_thread); 879bfbbc4aaSJason Evans #ifdef DIAGNOSTIC 880bfbbc4aaSJason Evans if (tmpvm == myvm) { 881bfbbc4aaSJason Evans printf("AIOD: vmspace problem -- %d\n", 882bfbbc4aaSJason Evans mycp->p_pid); 883bfbbc4aaSJason Evans } 88411783b14SJohn Dyson #endif 885bfbbc4aaSJason Evans /* Remove our vmspace reference. */ 8862244ea07SJohn Dyson vmspace_free(tmpvm); 887bfbbc4aaSJason Evans 888fd3bf775SJohn Dyson /* 889bfbbc4aaSJason Evans * Disassociate from the user process's file 890bfbbc4aaSJason Evans * descriptors. 891fd3bf775SJohn Dyson */ 892fd3bf775SJohn Dyson if (mycp->p_fd) 893b40ce416SJulian Elischer fdfree(td); 894fd3bf775SJohn Dyson mycp->p_fd = NULL; 895fd3bf775SJohn Dyson curcp = mycp; 896fd3bf775SJohn Dyson } 897fd3bf775SJohn Dyson 898fd3bf775SJohn Dyson /* 899fd3bf775SJohn Dyson * If we are the first to be put onto the free queue, wakeup 900fd3bf775SJohn Dyson * anyone waiting for a daemon. 901fd3bf775SJohn Dyson */ 902bfbbc4aaSJason Evans s = splnet(); 903fd3bf775SJohn Dyson TAILQ_REMOVE(&aio_activeproc, aiop, list); 904fd3bf775SJohn Dyson if (TAILQ_EMPTY(&aio_freeproc)) 905fd3bf775SJohn Dyson wakeup(&aio_freeproc); 906fd3bf775SJohn Dyson TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list); 907b40ce416SJulian Elischer aiop->aiothreadflags |= AIOP_FREE; 908bfbbc4aaSJason Evans splx(s); 909fd3bf775SJohn Dyson 910fd3bf775SJohn Dyson /* 911bfbbc4aaSJason Evans * If daemon is inactive for a long time, allow it to exit, 912bfbbc4aaSJason Evans * thereby freeing resources. 913fd3bf775SJohn Dyson */ 914b40ce416SJulian Elischer if (((aiop->aiothreadflags & AIOP_SCHED) == 0) && tsleep(mycp, 915bfbbc4aaSJason Evans PRIBIO, "aiordy", aiod_lifetime)) { 916bfbbc4aaSJason Evans s = splnet(); 917fd3bf775SJohn Dyson if ((TAILQ_FIRST(&aio_jobs) == NULL) && 918fd3bf775SJohn Dyson (TAILQ_FIRST(&aiop->jobtorun) == NULL)) { 919b40ce416SJulian Elischer if ((aiop->aiothreadflags & AIOP_FREE) && 92084af4da6SJohn Dyson (num_aio_procs > target_aio_procs)) { 921fd3bf775SJohn Dyson TAILQ_REMOVE(&aio_freeproc, aiop, list); 922bfbbc4aaSJason Evans splx(s); 923fd3bf775SJohn Dyson zfree(aiop_zone, aiop); 92484af4da6SJohn Dyson num_aio_procs--; 925bfbbc4aaSJason Evans #ifdef DIAGNOSTIC 926bfbbc4aaSJason Evans if (mycp->p_vmspace->vm_refcnt <= 1) { 927bfbbc4aaSJason Evans printf("AIOD: bad vm refcnt for" 928bfbbc4aaSJason Evans " exiting daemon: %d\n", 929fd3bf775SJohn Dyson mycp->p_vmspace->vm_refcnt); 930bfbbc4aaSJason Evans } 93111783b14SJohn Dyson #endif 932c9a970a7SAlan Cox kthread_exit(0); 933fd3bf775SJohn Dyson } 934fd3bf775SJohn Dyson } 935bfbbc4aaSJason Evans splx(s); 9362244ea07SJohn Dyson } 9372244ea07SJohn Dyson } 9382244ea07SJohn Dyson } 9392244ea07SJohn Dyson 9402244ea07SJohn Dyson /* 941bfbbc4aaSJason Evans * Create a new AIO daemon. This is mostly a kernel-thread fork routine. The 942bfbbc4aaSJason Evans * AIO daemon modifies its environment itself. 9432244ea07SJohn Dyson */ 9442244ea07SJohn Dyson static int 945fd3bf775SJohn Dyson aio_newproc() 946fd3bf775SJohn Dyson { 9472244ea07SJohn Dyson int error; 948c9a970a7SAlan Cox struct proc *p; 9492244ea07SJohn Dyson 950c9a970a7SAlan Cox error = kthread_create(aio_daemon, curproc, &p, RFNOWAIT, "aiod%d", 951c9a970a7SAlan Cox num_aio_procs); 952d5558c00SPeter Wemm if (error) 9532244ea07SJohn Dyson return error; 9542244ea07SJohn Dyson 955fd3bf775SJohn Dyson /* 956bfbbc4aaSJason Evans * Wait until daemon is started, but continue on just in case to 957fd3bf775SJohn Dyson * handle error conditions. 958fd3bf775SJohn Dyson */ 959c9a970a7SAlan Cox error = tsleep(p, PZERO, "aiosta", aiod_timeout); 960c9a970a7SAlan Cox 96184af4da6SJohn Dyson num_aio_procs++; 9622244ea07SJohn Dyson 9632244ea07SJohn Dyson return error; 9642244ea07SJohn Dyson } 9652244ea07SJohn Dyson 9662244ea07SJohn Dyson /* 96788ed460eSAlan Cox * Try the high-performance, low-overhead physio method for eligible 96888ed460eSAlan Cox * VCHR devices. This method doesn't use an aio helper thread, and 96988ed460eSAlan Cox * thus has very low overhead. 97088ed460eSAlan Cox * 97188ed460eSAlan Cox * Assumes that the caller, _aio_aqueue(), has incremented the file 97288ed460eSAlan Cox * structure's reference count, preventing its deallocation for the 97388ed460eSAlan Cox * duration of this call. 974fd3bf775SJohn Dyson */ 97588ed460eSAlan Cox static int 976bfbbc4aaSJason Evans aio_qphysio(struct proc *p, struct aiocblist *aiocbe) 977fd3bf775SJohn Dyson { 978fd3bf775SJohn Dyson int error; 979fd3bf775SJohn Dyson struct aiocb *cb; 980fd3bf775SJohn Dyson struct file *fp; 981fd3bf775SJohn Dyson struct buf *bp; 982fd3bf775SJohn Dyson struct vnode *vp; 983fd3bf775SJohn Dyson struct kaioinfo *ki; 984fd3bf775SJohn Dyson struct filedesc *fdp; 98584af4da6SJohn Dyson struct aio_liojob *lj; 986fd3bf775SJohn Dyson int fd; 987fd3bf775SJohn Dyson int s; 988b92bb032SAlan Cox int notify; 989fd3bf775SJohn Dyson 99084af4da6SJohn Dyson cb = &aiocbe->uaiocb; 991fd3bf775SJohn Dyson fdp = p->p_fd; 992fd3bf775SJohn Dyson fd = cb->aio_fildes; 993fd3bf775SJohn Dyson fp = fdp->fd_ofiles[fd]; 994fd3bf775SJohn Dyson 995008626c3SPoul-Henning Kamp if (fp->f_type != DTYPE_VNODE) 996008626c3SPoul-Henning Kamp return (-1); 997fd3bf775SJohn Dyson 998fd3bf775SJohn Dyson vp = (struct vnode *)fp->f_data; 99911783b14SJohn Dyson 1000f582ac06SBrian Feldman /* 1001f582ac06SBrian Feldman * If its not a disk, we don't want to return a positive error. 1002f582ac06SBrian Feldman * It causes the aio code to not fall through to try the thread 1003f582ac06SBrian Feldman * way when you're talking to a regular file. 1004f582ac06SBrian Feldman */ 1005f582ac06SBrian Feldman if (!vn_isdisk(vp, &error)) { 1006f582ac06SBrian Feldman if (error == ENOTBLK) 1007f582ac06SBrian Feldman return (-1); 1008f582ac06SBrian Feldman else 1009ba4ad1fcSPoul-Henning Kamp return (error); 1010f582ac06SBrian Feldman } 1011fd3bf775SJohn Dyson 1012008626c3SPoul-Henning Kamp if (cb->aio_nbytes % vp->v_rdev->si_bsize_phys) 1013008626c3SPoul-Henning Kamp return (-1); 1014fd3bf775SJohn Dyson 101513644654SAlan Cox if (cb->aio_nbytes > 101613644654SAlan Cox MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK)) 1017008626c3SPoul-Henning Kamp return (-1); 1018fd3bf775SJohn Dyson 1019fd3bf775SJohn Dyson ki = p->p_aioinfo; 1020008626c3SPoul-Henning Kamp if (ki->kaio_buffer_count >= ki->kaio_ballowed_count) 1021008626c3SPoul-Henning Kamp return (-1); 1022fd3bf775SJohn Dyson 1023fd3bf775SJohn Dyson ki->kaio_buffer_count++; 102411783b14SJohn Dyson 102511783b14SJohn Dyson lj = aiocbe->lio; 1026bfbbc4aaSJason Evans if (lj) 102784af4da6SJohn Dyson lj->lioj_buffer_count++; 1028fd3bf775SJohn Dyson 1029bfbbc4aaSJason Evans /* Create and build a buffer header for a transfer. */ 10301c7c3c6aSMatthew Dillon bp = (struct buf *)getpbuf(NULL); 103113644654SAlan Cox BUF_KERNPROC(bp); 1032fd3bf775SJohn Dyson 1033fd3bf775SJohn Dyson /* 1034bfbbc4aaSJason Evans * Get a copy of the kva from the physical buffer. 1035fd3bf775SJohn Dyson */ 1036b0eeea20SPoul-Henning Kamp bp->b_caller1 = p; 103749ff4debSPoul-Henning Kamp bp->b_dev = vp->v_rdev; 1038fd3bf775SJohn Dyson error = bp->b_error = 0; 1039fd3bf775SJohn Dyson 1040fd3bf775SJohn Dyson bp->b_bcount = cb->aio_nbytes; 1041fd3bf775SJohn Dyson bp->b_bufsize = cb->aio_nbytes; 104221144e3bSPoul-Henning Kamp bp->b_flags = B_PHYS; 1043fd3bf775SJohn Dyson bp->b_iodone = aio_physwakeup; 1044fd3bf775SJohn Dyson bp->b_saveaddr = bp->b_data; 1045b40ce416SJulian Elischer bp->b_data = cb->aio_buf; 1046fd3bf775SJohn Dyson bp->b_blkno = btodb(cb->aio_offset); 1047fd3bf775SJohn Dyson 104802c58685SPoul-Henning Kamp if (cb->aio_lio_opcode == LIO_WRITE) { 104921144e3bSPoul-Henning Kamp bp->b_iocmd = BIO_WRITE; 105002c58685SPoul-Henning Kamp if (!useracc(bp->b_data, bp->b_bufsize, VM_PROT_READ)) { 1051fd3bf775SJohn Dyson error = EFAULT; 1052fd3bf775SJohn Dyson goto doerror; 1053fd3bf775SJohn Dyson } 105402c58685SPoul-Henning Kamp } else { 105521144e3bSPoul-Henning Kamp bp->b_iocmd = BIO_READ; 105602c58685SPoul-Henning Kamp if (!useracc(bp->b_data, bp->b_bufsize, VM_PROT_WRITE)) { 1057fd3bf775SJohn Dyson error = EFAULT; 1058fd3bf775SJohn Dyson goto doerror; 1059fd3bf775SJohn Dyson } 106002c58685SPoul-Henning Kamp } 1061fd3bf775SJohn Dyson 1062bfbbc4aaSJason Evans /* Bring buffer into kernel space. */ 1063fd3bf775SJohn Dyson vmapbuf(bp); 1064fd3bf775SJohn Dyson 106584af4da6SJohn Dyson s = splbio(); 1066fd3bf775SJohn Dyson aiocbe->bp = bp; 1067fd3bf775SJohn Dyson bp->b_spc = (void *)aiocbe; 1068fd3bf775SJohn Dyson TAILQ_INSERT_TAIL(&aio_bufjobs, aiocbe, list); 106984af4da6SJohn Dyson TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist); 1070fd3bf775SJohn Dyson aiocbe->jobstate = JOBST_JOBQBUF; 107184af4da6SJohn Dyson cb->_aiocb_private.status = cb->aio_nbytes; 107284af4da6SJohn Dyson num_buf_aio++; 1073fd3bf775SJohn Dyson bp->b_error = 0; 1074fd3bf775SJohn Dyson 107584af4da6SJohn Dyson splx(s); 1076bfbbc4aaSJason Evans 1077bfbbc4aaSJason Evans /* Perform transfer. */ 1078b99c307aSPoul-Henning Kamp DEV_STRATEGY(bp, 0); 1079fd3bf775SJohn Dyson 1080cb679c38SJonathan Lemon notify = 0; 108184af4da6SJohn Dyson s = splbio(); 1082bfbbc4aaSJason Evans 108311783b14SJohn Dyson /* 108411783b14SJohn Dyson * If we had an error invoking the request, or an error in processing 1085bfbbc4aaSJason Evans * the request before we have returned, we process it as an error in 1086bfbbc4aaSJason Evans * transfer. Note that such an I/O error is not indicated immediately, 1087bfbbc4aaSJason Evans * but is returned using the aio_error mechanism. In this case, 1088bfbbc4aaSJason Evans * aio_suspend will return immediately. 108911783b14SJohn Dyson */ 1090c244d2deSPoul-Henning Kamp if (bp->b_error || (bp->b_ioflags & BIO_ERROR)) { 109111783b14SJohn Dyson struct aiocb *job = aiocbe->uuaiocb; 109211783b14SJohn Dyson 109311783b14SJohn Dyson aiocbe->uaiocb._aiocb_private.status = 0; 109411783b14SJohn Dyson suword(&job->_aiocb_private.status, 0); 109511783b14SJohn Dyson aiocbe->uaiocb._aiocb_private.error = bp->b_error; 109611783b14SJohn Dyson suword(&job->_aiocb_private.error, bp->b_error); 109711783b14SJohn Dyson 109811783b14SJohn Dyson ki->kaio_buffer_finished_count++; 109911783b14SJohn Dyson 110011783b14SJohn Dyson if (aiocbe->jobstate != JOBST_JOBBFINISHED) { 110111783b14SJohn Dyson aiocbe->jobstate = JOBST_JOBBFINISHED; 110211783b14SJohn Dyson aiocbe->jobflags |= AIOCBLIST_DONE; 1103fd3bf775SJohn Dyson TAILQ_REMOVE(&aio_bufjobs, aiocbe, list); 110484af4da6SJohn Dyson TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist); 110511783b14SJohn Dyson TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist); 1106cb679c38SJonathan Lemon notify = 1; 110711783b14SJohn Dyson } 110811783b14SJohn Dyson } 110984af4da6SJohn Dyson splx(s); 1110cb679c38SJonathan Lemon if (notify) 1111cb679c38SJonathan Lemon KNOTE(&aiocbe->klist, 0); 1112fd3bf775SJohn Dyson return 0; 1113fd3bf775SJohn Dyson 1114fd3bf775SJohn Dyson doerror: 1115fd3bf775SJohn Dyson ki->kaio_buffer_count--; 1116bfbbc4aaSJason Evans if (lj) 111784af4da6SJohn Dyson lj->lioj_buffer_count--; 111884af4da6SJohn Dyson aiocbe->bp = NULL; 11191c7c3c6aSMatthew Dillon relpbuf(bp, NULL); 1120fd3bf775SJohn Dyson return error; 1121fd3bf775SJohn Dyson } 1122fd3bf775SJohn Dyson 112384af4da6SJohn Dyson /* 112484af4da6SJohn Dyson * This waits/tests physio completion. 112584af4da6SJohn Dyson */ 112688ed460eSAlan Cox static int 11279c8a2647SAlan Cox aio_fphysio(struct proc *p, struct aiocblist *iocb) 1128fd3bf775SJohn Dyson { 1129fd3bf775SJohn Dyson int s; 1130fd3bf775SJohn Dyson struct buf *bp; 1131fd3bf775SJohn Dyson int error; 1132fd3bf775SJohn Dyson 1133fd3bf775SJohn Dyson bp = iocb->bp; 1134fd3bf775SJohn Dyson 1135fd3bf775SJohn Dyson s = splbio(); 1136fd3bf775SJohn Dyson while ((bp->b_flags & B_DONE) == 0) { 11379c8a2647SAlan Cox if (tsleep(bp, PRIBIO, "physstr", aiod_timeout)) { 1138fd3bf775SJohn Dyson if ((bp->b_flags & B_DONE) == 0) { 1139fd3bf775SJohn Dyson splx(s); 1140fd3bf775SJohn Dyson return EINPROGRESS; 1141bfbbc4aaSJason Evans } else 1142fd3bf775SJohn Dyson break; 1143fd3bf775SJohn Dyson } 1144fd3bf775SJohn Dyson } 11459c8a2647SAlan Cox splx(s); 1146fd3bf775SJohn Dyson 1147bfbbc4aaSJason Evans /* Release mapping into kernel space. */ 1148fd3bf775SJohn Dyson vunmapbuf(bp); 1149fd3bf775SJohn Dyson iocb->bp = 0; 1150fd3bf775SJohn Dyson 1151fd3bf775SJohn Dyson error = 0; 1152bfbbc4aaSJason Evans 1153bfbbc4aaSJason Evans /* Check for an error. */ 1154c244d2deSPoul-Henning Kamp if (bp->b_ioflags & BIO_ERROR) 1155fd3bf775SJohn Dyson error = bp->b_error; 1156fd3bf775SJohn Dyson 11571c7c3c6aSMatthew Dillon relpbuf(bp, NULL); 1158fd3bf775SJohn Dyson return (error); 1159fd3bf775SJohn Dyson } 11605dec52baSPeter Wemm #endif /* VFS_AIO */ 1161fd3bf775SJohn Dyson 1162fd3bf775SJohn Dyson /* 1163bfbbc4aaSJason Evans * Wake up aio requests that may be serviceable now. 1164bfbbc4aaSJason Evans */ 1165bfbbc4aaSJason Evans void 1166bfbbc4aaSJason Evans aio_swake(struct socket *so, struct sockbuf *sb) 1167bfbbc4aaSJason Evans { 11685dec52baSPeter Wemm #ifndef VFS_AIO 11695dec52baSPeter Wemm return; 11705dec52baSPeter Wemm #else 1171bfbbc4aaSJason Evans struct aiocblist *cb,*cbn; 1172bfbbc4aaSJason Evans struct proc *p; 1173bfbbc4aaSJason Evans struct kaioinfo *ki = NULL; 1174bfbbc4aaSJason Evans int opcode, wakecount = 0; 1175b40ce416SJulian Elischer struct aiothreadlist *aiop; 1176bfbbc4aaSJason Evans 1177bfbbc4aaSJason Evans if (sb == &so->so_snd) { 1178bfbbc4aaSJason Evans opcode = LIO_WRITE; 1179bfbbc4aaSJason Evans so->so_snd.sb_flags &= ~SB_AIO; 1180bfbbc4aaSJason Evans } else { 1181bfbbc4aaSJason Evans opcode = LIO_READ; 1182bfbbc4aaSJason Evans so->so_rcv.sb_flags &= ~SB_AIO; 1183bfbbc4aaSJason Evans } 1184bfbbc4aaSJason Evans 1185bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&so->so_aiojobq); cb; cb = cbn) { 1186bfbbc4aaSJason Evans cbn = TAILQ_NEXT(cb, list); 1187bfbbc4aaSJason Evans if (opcode == cb->uaiocb.aio_lio_opcode) { 1188bfbbc4aaSJason Evans p = cb->userproc; 1189bfbbc4aaSJason Evans ki = p->p_aioinfo; 1190bfbbc4aaSJason Evans TAILQ_REMOVE(&so->so_aiojobq, cb, list); 1191bfbbc4aaSJason Evans TAILQ_REMOVE(&ki->kaio_sockqueue, cb, plist); 1192bfbbc4aaSJason Evans TAILQ_INSERT_TAIL(&aio_jobs, cb, list); 1193bfbbc4aaSJason Evans TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, cb, plist); 1194bfbbc4aaSJason Evans wakecount++; 1195bfbbc4aaSJason Evans if (cb->jobstate != JOBST_JOBQGLOBAL) 1196bfbbc4aaSJason Evans panic("invalid queue value"); 1197bfbbc4aaSJason Evans } 1198bfbbc4aaSJason Evans } 1199bfbbc4aaSJason Evans 1200bfbbc4aaSJason Evans while (wakecount--) { 1201bfbbc4aaSJason Evans if ((aiop = TAILQ_FIRST(&aio_freeproc)) != 0) { 1202bfbbc4aaSJason Evans TAILQ_REMOVE(&aio_freeproc, aiop, list); 1203bfbbc4aaSJason Evans TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list); 1204b40ce416SJulian Elischer aiop->aiothreadflags &= ~AIOP_FREE; 1205b40ce416SJulian Elischer wakeup(aiop->aiothread); 1206bfbbc4aaSJason Evans } 1207bfbbc4aaSJason Evans } 12085dec52baSPeter Wemm #endif /* VFS_AIO */ 1209bfbbc4aaSJason Evans } 1210bfbbc4aaSJason Evans 12115dec52baSPeter Wemm #ifdef VFS_AIO 1212bfbbc4aaSJason Evans /* 1213bfbbc4aaSJason Evans * Queue a new AIO request. Choosing either the threaded or direct physio VCHR 1214bfbbc4aaSJason Evans * technique is done in this code. 12152244ea07SJohn Dyson */ 12162244ea07SJohn Dyson static int 1217b40ce416SJulian Elischer _aio_aqueue(struct thread *td, struct aiocb *job, struct aio_liojob *lj, int type) 1218fd3bf775SJohn Dyson { 1219b40ce416SJulian Elischer struct proc *p = td->td_proc; 12202244ea07SJohn Dyson struct filedesc *fdp; 12212244ea07SJohn Dyson struct file *fp; 12222244ea07SJohn Dyson unsigned int fd; 1223bfbbc4aaSJason Evans struct socket *so; 1224bfbbc4aaSJason Evans int s; 1225b92bb032SAlan Cox int error; 12262244ea07SJohn Dyson int opcode; 12272244ea07SJohn Dyson struct aiocblist *aiocbe; 1228b40ce416SJulian Elischer struct aiothreadlist *aiop; 12292244ea07SJohn Dyson struct kaioinfo *ki; 1230c6fa9f78SAlan Cox struct kevent kev; 1231c6fa9f78SAlan Cox struct kqueue *kq; 1232c6fa9f78SAlan Cox struct file *kq_fp; 12332244ea07SJohn Dyson 1234fd3bf775SJohn Dyson aiocbe = zalloc(aiocb_zone); 1235fd3bf775SJohn Dyson aiocbe->inputcharge = 0; 1236fd3bf775SJohn Dyson aiocbe->outputcharge = 0; 123713644654SAlan Cox callout_handle_init(&aiocbe->timeouthandle); 1238cb679c38SJonathan Lemon SLIST_INIT(&aiocbe->klist); 1239fd3bf775SJohn Dyson 1240fd3bf775SJohn Dyson suword(&job->_aiocb_private.status, -1); 1241fd3bf775SJohn Dyson suword(&job->_aiocb_private.error, 0); 1242fd3bf775SJohn Dyson suword(&job->_aiocb_private.kernelinfo, -1); 1243fd3bf775SJohn Dyson 124488ed460eSAlan Cox error = copyin(job, &aiocbe->uaiocb, sizeof(aiocbe->uaiocb)); 12452244ea07SJohn Dyson if (error) { 1246fd3bf775SJohn Dyson suword(&job->_aiocb_private.error, error); 124788ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 12482244ea07SJohn Dyson return error; 12492244ea07SJohn Dyson } 12502f3cf918SAlfred Perlstein if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL && 12512f3cf918SAlfred Perlstein !_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) { 12522f3cf918SAlfred Perlstein zfree(aiocb_zone, aiocbe); 12532f3cf918SAlfred Perlstein return EINVAL; 12542f3cf918SAlfred Perlstein } 12552244ea07SJohn Dyson 1256bfbbc4aaSJason Evans /* Save userspace address of the job info. */ 125711783b14SJohn Dyson aiocbe->uuaiocb = job; 125811783b14SJohn Dyson 1259bfbbc4aaSJason Evans /* Get the opcode. */ 1260bfbbc4aaSJason Evans if (type != LIO_NOP) 1261a624e84fSJohn Dyson aiocbe->uaiocb.aio_lio_opcode = type; 1262a624e84fSJohn Dyson opcode = aiocbe->uaiocb.aio_lio_opcode; 12632244ea07SJohn Dyson 1264bfbbc4aaSJason Evans /* Get the fd info for process. */ 12652244ea07SJohn Dyson fdp = p->p_fd; 12662244ea07SJohn Dyson 12672244ea07SJohn Dyson /* 1268bfbbc4aaSJason Evans * Range check file descriptor. 12692244ea07SJohn Dyson */ 12702244ea07SJohn Dyson fd = aiocbe->uaiocb.aio_fildes; 12712244ea07SJohn Dyson if (fd >= fdp->fd_nfiles) { 127288ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 1273bfbbc4aaSJason Evans if (type == 0) 12742244ea07SJohn Dyson suword(&job->_aiocb_private.error, EBADF); 12752244ea07SJohn Dyson return EBADF; 12762244ea07SJohn Dyson } 12772244ea07SJohn Dyson 1278bfbbc4aaSJason Evans fp = aiocbe->fd_file = fdp->fd_ofiles[fd]; 1279bfbbc4aaSJason Evans if ((fp == NULL) || ((opcode == LIO_WRITE) && ((fp->f_flag & FWRITE) == 1280bfbbc4aaSJason Evans 0))) { 128188ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 1282bfbbc4aaSJason Evans if (type == 0) 12832244ea07SJohn Dyson suword(&job->_aiocb_private.error, EBADF); 12842244ea07SJohn Dyson return EBADF; 12852244ea07SJohn Dyson } 12862244ea07SJohn Dyson 12872244ea07SJohn Dyson if (aiocbe->uaiocb.aio_offset == -1LL) { 128888ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 1289bfbbc4aaSJason Evans if (type == 0) 12902244ea07SJohn Dyson suword(&job->_aiocb_private.error, EINVAL); 12912244ea07SJohn Dyson return EINVAL; 12922244ea07SJohn Dyson } 12932244ea07SJohn Dyson 12942244ea07SJohn Dyson error = suword(&job->_aiocb_private.kernelinfo, jobrefid); 12952244ea07SJohn Dyson if (error) { 129688ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 1297bfbbc4aaSJason Evans if (type == 0) 12982244ea07SJohn Dyson suword(&job->_aiocb_private.error, EINVAL); 12992244ea07SJohn Dyson return error; 13002244ea07SJohn Dyson } 13012244ea07SJohn Dyson 130230166fabSBruce Evans aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jobrefid; 13032d2f8ae7SBruce Evans if (jobrefid == LONG_MAX) 1304fd3bf775SJohn Dyson jobrefid = 1; 13052d2f8ae7SBruce Evans else 13062d2f8ae7SBruce Evans jobrefid++; 13072244ea07SJohn Dyson 13082244ea07SJohn Dyson if (opcode == LIO_NOP) { 130988ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 13102244ea07SJohn Dyson if (type == 0) { 13112244ea07SJohn Dyson suword(&job->_aiocb_private.error, 0); 1312fd3bf775SJohn Dyson suword(&job->_aiocb_private.status, 0); 1313fd3bf775SJohn Dyson suword(&job->_aiocb_private.kernelinfo, 0); 13142244ea07SJohn Dyson } 13152244ea07SJohn Dyson return 0; 13162244ea07SJohn Dyson } 13172244ea07SJohn Dyson 1318fd3bf775SJohn Dyson if ((opcode != LIO_READ) && (opcode != LIO_WRITE)) { 131988ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 13202244ea07SJohn Dyson if (type == 0) { 1321fd3bf775SJohn Dyson suword(&job->_aiocb_private.status, 0); 13222244ea07SJohn Dyson suword(&job->_aiocb_private.error, EINVAL); 13232244ea07SJohn Dyson } 13242244ea07SJohn Dyson return EINVAL; 13252244ea07SJohn Dyson } 13262244ea07SJohn Dyson 1327279d7226SMatthew Dillon fhold(fp); 1328279d7226SMatthew Dillon 1329c6fa9f78SAlan Cox if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_KEVENT) { 1330c6fa9f78SAlan Cox kev.ident = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue; 1331c6fa9f78SAlan Cox kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sigval_ptr; 1332c6fa9f78SAlan Cox } 1333c6fa9f78SAlan Cox else { 1334cb679c38SJonathan Lemon /* 1335c6fa9f78SAlan Cox * This method for requesting kevent-based notification won't 1336c6fa9f78SAlan Cox * work on the alpha, since we're passing in a pointer 1337c6fa9f78SAlan Cox * via aio_lio_opcode, which is an int. Use the SIGEV_KEVENT- 1338c6fa9f78SAlan Cox * based method instead. 1339cb679c38SJonathan Lemon */ 1340c6fa9f78SAlan Cox struct kevent *kevp; 1341cb679c38SJonathan Lemon 1342cb679c38SJonathan Lemon kevp = (struct kevent *)job->aio_lio_opcode; 1343cb679c38SJonathan Lemon if (kevp == NULL) 1344cb679c38SJonathan Lemon goto no_kqueue; 1345cb679c38SJonathan Lemon 134688ed460eSAlan Cox error = copyin(kevp, &kev, sizeof(kev)); 1347cb679c38SJonathan Lemon if (error) 1348cb679c38SJonathan Lemon goto aqueue_fail; 1349c6fa9f78SAlan Cox } 1350cb679c38SJonathan Lemon if ((u_int)kev.ident >= fdp->fd_nfiles || 135139b2b25fSAlan Cox (kq_fp = fdp->fd_ofiles[kev.ident]) == NULL || 135239b2b25fSAlan Cox (kq_fp->f_type != DTYPE_KQUEUE)) { 1353cb679c38SJonathan Lemon error = EBADF; 1354cb679c38SJonathan Lemon goto aqueue_fail; 1355cb679c38SJonathan Lemon } 135639b2b25fSAlan Cox kq = (struct kqueue *)kq_fp->f_data; 1357c6fa9f78SAlan Cox kev.ident = (uintptr_t)aiocbe; 1358cb679c38SJonathan Lemon kev.filter = EVFILT_AIO; 1359cb679c38SJonathan Lemon kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1; 1360b40ce416SJulian Elischer error = kqueue_register(kq, &kev, td); 1361cb679c38SJonathan Lemon aqueue_fail: 1362cb679c38SJonathan Lemon if (error) { 136388ed460eSAlan Cox zfree(aiocb_zone, aiocbe); 1364cb679c38SJonathan Lemon if (type == 0) 1365cb679c38SJonathan Lemon suword(&job->_aiocb_private.error, error); 1366279d7226SMatthew Dillon goto done; 1367cb679c38SJonathan Lemon } 1368cb679c38SJonathan Lemon no_kqueue: 1369cb679c38SJonathan Lemon 1370fd3bf775SJohn Dyson suword(&job->_aiocb_private.error, EINPROGRESS); 1371fd3bf775SJohn Dyson aiocbe->uaiocb._aiocb_private.error = EINPROGRESS; 13722244ea07SJohn Dyson aiocbe->userproc = p; 13732244ea07SJohn Dyson aiocbe->jobflags = 0; 137484af4da6SJohn Dyson aiocbe->lio = lj; 137584af4da6SJohn Dyson ki = p->p_aioinfo; 13762244ea07SJohn Dyson 1377bfbbc4aaSJason Evans if (fp->f_type == DTYPE_SOCKET) { 1378bfbbc4aaSJason Evans /* 1379bfbbc4aaSJason Evans * Alternate queueing for socket ops: Reach down into the 1380bfbbc4aaSJason Evans * descriptor to get the socket data. Then check to see if the 1381bfbbc4aaSJason Evans * socket is ready to be read or written (based on the requested 1382bfbbc4aaSJason Evans * operation). 1383bfbbc4aaSJason Evans * 1384bfbbc4aaSJason Evans * If it is not ready for io, then queue the aiocbe on the 1385bfbbc4aaSJason Evans * socket, and set the flags so we get a call when sbnotify() 1386bfbbc4aaSJason Evans * happens. 1387bfbbc4aaSJason Evans */ 1388bfbbc4aaSJason Evans so = (struct socket *)fp->f_data; 1389bfbbc4aaSJason Evans s = splnet(); 1390bfbbc4aaSJason Evans if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode == 1391bfbbc4aaSJason Evans LIO_WRITE) && (!sowriteable(so)))) { 1392bfbbc4aaSJason Evans TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list); 1393bfbbc4aaSJason Evans TAILQ_INSERT_TAIL(&ki->kaio_sockqueue, aiocbe, plist); 1394bfbbc4aaSJason Evans if (opcode == LIO_READ) 1395bfbbc4aaSJason Evans so->so_rcv.sb_flags |= SB_AIO; 1396bfbbc4aaSJason Evans else 1397bfbbc4aaSJason Evans so->so_snd.sb_flags |= SB_AIO; 1398bfbbc4aaSJason Evans aiocbe->jobstate = JOBST_JOBQGLOBAL; /* XXX */ 1399bfbbc4aaSJason Evans ki->kaio_queue_count++; 1400bfbbc4aaSJason Evans num_queue_count++; 1401bfbbc4aaSJason Evans splx(s); 1402279d7226SMatthew Dillon error = 0; 1403279d7226SMatthew Dillon goto done; 1404bfbbc4aaSJason Evans } 1405bfbbc4aaSJason Evans splx(s); 1406bfbbc4aaSJason Evans } 1407bfbbc4aaSJason Evans 1408bfbbc4aaSJason Evans if ((error = aio_qphysio(p, aiocbe)) == 0) 1409279d7226SMatthew Dillon goto done; 1410279d7226SMatthew Dillon if (error > 0) { 1411fd3bf775SJohn Dyson suword(&job->_aiocb_private.status, 0); 1412fd3bf775SJohn Dyson aiocbe->uaiocb._aiocb_private.error = error; 1413fd3bf775SJohn Dyson suword(&job->_aiocb_private.error, error); 1414279d7226SMatthew Dillon goto done; 1415fd3bf775SJohn Dyson } 1416fd3bf775SJohn Dyson 1417bfbbc4aaSJason Evans /* No buffer for daemon I/O. */ 141884af4da6SJohn Dyson aiocbe->bp = NULL; 141984af4da6SJohn Dyson 142084af4da6SJohn Dyson ki->kaio_queue_count++; 1421bfbbc4aaSJason Evans if (lj) 142284af4da6SJohn Dyson lj->lioj_queue_count++; 1423bfbbc4aaSJason Evans s = splnet(); 1424fd3bf775SJohn Dyson TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist); 1425fd3bf775SJohn Dyson TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list); 1426bfbbc4aaSJason Evans splx(s); 1427fd3bf775SJohn Dyson aiocbe->jobstate = JOBST_JOBQGLOBAL; 1428fd3bf775SJohn Dyson 142984af4da6SJohn Dyson num_queue_count++; 1430fd3bf775SJohn Dyson error = 0; 1431fd3bf775SJohn Dyson 1432fd3bf775SJohn Dyson /* 1433bfbbc4aaSJason Evans * If we don't have a free AIO process, and we are below our quota, then 1434bfbbc4aaSJason Evans * start one. Otherwise, depend on the subsequent I/O completions to 1435bfbbc4aaSJason Evans * pick-up this job. If we don't sucessfully create the new process 1436bfbbc4aaSJason Evans * (thread) due to resource issues, we return an error for now (EAGAIN), 1437bfbbc4aaSJason Evans * which is likely not the correct thing to do. 1438fd3bf775SJohn Dyson */ 14392244ea07SJohn Dyson retryproc: 1440bfbbc4aaSJason Evans s = splnet(); 1441d254af07SMatthew Dillon if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) { 14422244ea07SJohn Dyson TAILQ_REMOVE(&aio_freeproc, aiop, list); 14432244ea07SJohn Dyson TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list); 1444b40ce416SJulian Elischer aiop->aiothreadflags &= ~AIOP_FREE; 1445b40ce416SJulian Elischer wakeup(aiop->aiothread); 1446fd3bf775SJohn Dyson } else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) && 1447fd3bf775SJohn Dyson ((ki->kaio_active_count + num_aio_resv_start) < 1448fd3bf775SJohn Dyson ki->kaio_maxactive_count)) { 1449fd3bf775SJohn Dyson num_aio_resv_start++; 1450fd3bf775SJohn Dyson if ((error = aio_newproc()) == 0) { 145184af4da6SJohn Dyson num_aio_resv_start--; 1452b40ce416SJulian Elischer td->td_retval[0] = 0; 14532244ea07SJohn Dyson goto retryproc; 1454fd3bf775SJohn Dyson } 145584af4da6SJohn Dyson num_aio_resv_start--; 1456fd3bf775SJohn Dyson } 1457bfbbc4aaSJason Evans splx(s); 1458279d7226SMatthew Dillon done: 1459b40ce416SJulian Elischer fdrop(fp, td); 1460fd3bf775SJohn Dyson return error; 14612244ea07SJohn Dyson } 14622244ea07SJohn Dyson 1463fd3bf775SJohn Dyson /* 1464fd3bf775SJohn Dyson * This routine queues an AIO request, checking for quotas. 1465fd3bf775SJohn Dyson */ 14662244ea07SJohn Dyson static int 1467b40ce416SJulian Elischer aio_aqueue(struct thread *td, struct aiocb *job, int type) 1468fd3bf775SJohn Dyson { 1469b40ce416SJulian Elischer struct proc *p = td->td_proc; 14702244ea07SJohn Dyson struct kaioinfo *ki; 14712244ea07SJohn Dyson 1472bfbbc4aaSJason Evans if (p->p_aioinfo == NULL) 14732244ea07SJohn Dyson aio_init_aioinfo(p); 14742244ea07SJohn Dyson 14752244ea07SJohn Dyson if (num_queue_count >= max_queue_count) 14762244ea07SJohn Dyson return EAGAIN; 14772244ea07SJohn Dyson 14782244ea07SJohn Dyson ki = p->p_aioinfo; 14792244ea07SJohn Dyson if (ki->kaio_queue_count >= ki->kaio_qallowed_count) 14802244ea07SJohn Dyson return EAGAIN; 14812244ea07SJohn Dyson 1482b40ce416SJulian Elischer return _aio_aqueue(td, job, NULL, type); 14832244ea07SJohn Dyson } 14845dec52baSPeter Wemm #endif /* VFS_AIO */ 14852244ea07SJohn Dyson 14862244ea07SJohn Dyson /* 1487bfbbc4aaSJason Evans * Support the aio_return system call, as a side-effect, kernel resources are 1488bfbbc4aaSJason Evans * released. 14892244ea07SJohn Dyson */ 14902244ea07SJohn Dyson int 1491b40ce416SJulian Elischer aio_return(struct thread *td, struct aio_return_args *uap) 1492fd3bf775SJohn Dyson { 1493dd85920aSJason Evans #ifndef VFS_AIO 1494dd85920aSJason Evans return ENOSYS; 1495dd85920aSJason Evans #else 1496b40ce416SJulian Elischer struct proc *p = td->td_proc; 149784af4da6SJohn Dyson int s; 1498f5ef029eSPoul-Henning Kamp int jobref; 149984af4da6SJohn Dyson struct aiocblist *cb, *ncb; 150011783b14SJohn Dyson struct aiocb *ujob; 15012244ea07SJohn Dyson struct kaioinfo *ki; 15022244ea07SJohn Dyson 15032244ea07SJohn Dyson ki = p->p_aioinfo; 1504bfbbc4aaSJason Evans if (ki == NULL) 15052244ea07SJohn Dyson return EINVAL; 15062244ea07SJohn Dyson 150711783b14SJohn Dyson ujob = uap->aiocbp; 150811783b14SJohn Dyson 150911783b14SJohn Dyson jobref = fuword(&ujob->_aiocb_private.kernelinfo); 1510fd3bf775SJohn Dyson if (jobref == -1 || jobref == 0) 15112244ea07SJohn Dyson return EINVAL; 15122244ea07SJohn Dyson 1513bfbbc4aaSJason Evans s = splnet(); 1514bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_jobdone); cb; cb = TAILQ_NEXT(cb, 1515bfbbc4aaSJason Evans plist)) { 1516bfbbc4aaSJason Evans if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) == 1517bfbbc4aaSJason Evans jobref) { 1518bfbbc4aaSJason Evans splx(s); 151911783b14SJohn Dyson if (ujob == cb->uuaiocb) { 1520b40ce416SJulian Elischer td->td_retval[0] = 1521bfbbc4aaSJason Evans cb->uaiocb._aiocb_private.status; 1522bfbbc4aaSJason Evans } else 1523b40ce416SJulian Elischer td->td_retval[0] = EFAULT; 1524fd3bf775SJohn Dyson if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) { 1525bfbbc4aaSJason Evans curproc->p_stats->p_ru.ru_oublock += 1526bfbbc4aaSJason Evans cb->outputcharge; 1527fd3bf775SJohn Dyson cb->outputcharge = 0; 1528fd3bf775SJohn Dyson } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) { 1529bfbbc4aaSJason Evans curproc->p_stats->p_ru.ru_inblock += 1530bfbbc4aaSJason Evans cb->inputcharge; 1531fd3bf775SJohn Dyson cb->inputcharge = 0; 1532fd3bf775SJohn Dyson } 15332244ea07SJohn Dyson aio_free_entry(cb); 15342244ea07SJohn Dyson return 0; 15352244ea07SJohn Dyson } 15362244ea07SJohn Dyson } 1537bfbbc4aaSJason Evans splx(s); 15382244ea07SJohn Dyson 153984af4da6SJohn Dyson s = splbio(); 1540bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = ncb) { 154184af4da6SJohn Dyson ncb = TAILQ_NEXT(cb, plist); 1542bfbbc4aaSJason Evans if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) 1543bfbbc4aaSJason Evans == jobref) { 154484af4da6SJohn Dyson splx(s); 154511783b14SJohn Dyson if (ujob == cb->uuaiocb) { 1546b40ce416SJulian Elischer td->td_retval[0] = 1547bfbbc4aaSJason Evans cb->uaiocb._aiocb_private.status; 1548bfbbc4aaSJason Evans } else 1549b40ce416SJulian Elischer td->td_retval[0] = EFAULT; 155084af4da6SJohn Dyson aio_free_entry(cb); 155184af4da6SJohn Dyson return 0; 155284af4da6SJohn Dyson } 155384af4da6SJohn Dyson } 155484af4da6SJohn Dyson splx(s); 155584af4da6SJohn Dyson 15562244ea07SJohn Dyson return (EINVAL); 1557dd85920aSJason Evans #endif /* VFS_AIO */ 15582244ea07SJohn Dyson } 15592244ea07SJohn Dyson 15602244ea07SJohn Dyson /* 1561bfbbc4aaSJason Evans * Allow a process to wakeup when any of the I/O requests are completed. 15622244ea07SJohn Dyson */ 15632244ea07SJohn Dyson int 1564b40ce416SJulian Elischer aio_suspend(struct thread *td, struct aio_suspend_args *uap) 1565fd3bf775SJohn Dyson { 1566dd85920aSJason Evans #ifndef VFS_AIO 1567dd85920aSJason Evans return ENOSYS; 1568dd85920aSJason Evans #else 1569b40ce416SJulian Elischer struct proc *p = td->td_proc; 15704a11ca4eSPoul-Henning Kamp struct timeval atv; 15712244ea07SJohn Dyson struct timespec ts; 15722244ea07SJohn Dyson struct aiocb *const *cbptr, *cbp; 15732244ea07SJohn Dyson struct kaioinfo *ki; 15742244ea07SJohn Dyson struct aiocblist *cb; 15752244ea07SJohn Dyson int i; 157684af4da6SJohn Dyson int njoblist; 15772244ea07SJohn Dyson int error, s, timo; 157811783b14SJohn Dyson int *ijoblist; 157911783b14SJohn Dyson struct aiocb **ujoblist; 15802244ea07SJohn Dyson 1581fd3bf775SJohn Dyson if (uap->nent >= AIO_LISTIO_MAX) 1582fd3bf775SJohn Dyson return EINVAL; 15832244ea07SJohn Dyson 15842244ea07SJohn Dyson timo = 0; 15852244ea07SJohn Dyson if (uap->timeout) { 1586bfbbc4aaSJason Evans /* Get timespec struct. */ 1587bfbbc4aaSJason Evans if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0) 15882244ea07SJohn Dyson return error; 15892244ea07SJohn Dyson 15902244ea07SJohn Dyson if (ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000) 15912244ea07SJohn Dyson return (EINVAL); 15922244ea07SJohn Dyson 1593e3b3ba2dSDag-Erling Smørgrav TIMESPEC_TO_TIMEVAL(&atv, &ts); 15942244ea07SJohn Dyson if (itimerfix(&atv)) 15952244ea07SJohn Dyson return (EINVAL); 1596227ee8a1SPoul-Henning Kamp timo = tvtohz(&atv); 15972244ea07SJohn Dyson } 15982244ea07SJohn Dyson 15992244ea07SJohn Dyson ki = p->p_aioinfo; 16002244ea07SJohn Dyson if (ki == NULL) 16012244ea07SJohn Dyson return EAGAIN; 16022244ea07SJohn Dyson 160384af4da6SJohn Dyson njoblist = 0; 160411783b14SJohn Dyson ijoblist = zalloc(aiol_zone); 160511783b14SJohn Dyson ujoblist = zalloc(aiol_zone); 16062244ea07SJohn Dyson cbptr = uap->aiocbp; 16072244ea07SJohn Dyson 16082244ea07SJohn Dyson for (i = 0; i < uap->nent; i++) { 160930166fabSBruce Evans cbp = (struct aiocb *)(intptr_t)fuword((caddr_t)&cbptr[i]); 161084af4da6SJohn Dyson if (cbp == 0) 161184af4da6SJohn Dyson continue; 161211783b14SJohn Dyson ujoblist[njoblist] = cbp; 161311783b14SJohn Dyson ijoblist[njoblist] = fuword(&cbp->_aiocb_private.kernelinfo); 161484af4da6SJohn Dyson njoblist++; 16152244ea07SJohn Dyson } 1616bfbbc4aaSJason Evans 161711783b14SJohn Dyson if (njoblist == 0) { 161811783b14SJohn Dyson zfree(aiol_zone, ijoblist); 161911783b14SJohn Dyson zfree(aiol_zone, ujoblist); 162084af4da6SJohn Dyson return 0; 162111783b14SJohn Dyson } 16222244ea07SJohn Dyson 162311783b14SJohn Dyson error = 0; 1624bfbbc4aaSJason Evans for (;;) { 1625bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_jobdone); cb; cb = 1626bfbbc4aaSJason Evans TAILQ_NEXT(cb, plist)) { 162784af4da6SJohn Dyson for (i = 0; i < njoblist; i++) { 1628bfbbc4aaSJason Evans if (((intptr_t) 1629bfbbc4aaSJason Evans cb->uaiocb._aiocb_private.kernelinfo) == 163011783b14SJohn Dyson ijoblist[i]) { 163111783b14SJohn Dyson if (ujoblist[i] != cb->uuaiocb) 163211783b14SJohn Dyson error = EINVAL; 163311783b14SJohn Dyson zfree(aiol_zone, ijoblist); 163411783b14SJohn Dyson zfree(aiol_zone, ujoblist); 163511783b14SJohn Dyson return error; 163684af4da6SJohn Dyson } 163784af4da6SJohn Dyson } 163884af4da6SJohn Dyson } 163984af4da6SJohn Dyson 164084af4da6SJohn Dyson s = splbio(); 1641bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = 1642bfbbc4aaSJason Evans TAILQ_NEXT(cb, plist)) { 164384af4da6SJohn Dyson for (i = 0; i < njoblist; i++) { 1644bfbbc4aaSJason Evans if (((intptr_t) 1645bfbbc4aaSJason Evans cb->uaiocb._aiocb_private.kernelinfo) == 164611783b14SJohn Dyson ijoblist[i]) { 164784af4da6SJohn Dyson splx(s); 164811783b14SJohn Dyson if (ujoblist[i] != cb->uuaiocb) 164911783b14SJohn Dyson error = EINVAL; 165011783b14SJohn Dyson zfree(aiol_zone, ijoblist); 165111783b14SJohn Dyson zfree(aiol_zone, ujoblist); 165211783b14SJohn Dyson return error; 16532244ea07SJohn Dyson } 16542244ea07SJohn Dyson } 16552244ea07SJohn Dyson } 16562244ea07SJohn Dyson 1657fd3bf775SJohn Dyson ki->kaio_flags |= KAIO_WAKEUP; 16582244ea07SJohn Dyson error = tsleep(p, PRIBIO | PCATCH, "aiospn", timo); 1659b7592c7bSJason Evans splx(s); 16602244ea07SJohn Dyson 166160ffb019SJason Evans if (error == ERESTART || error == EINTR) { 166211783b14SJohn Dyson zfree(aiol_zone, ijoblist); 166311783b14SJohn Dyson zfree(aiol_zone, ujoblist); 16642244ea07SJohn Dyson return EINTR; 16652244ea07SJohn Dyson } else if (error == EWOULDBLOCK) { 166611783b14SJohn Dyson zfree(aiol_zone, ijoblist); 166711783b14SJohn Dyson zfree(aiol_zone, ujoblist); 16682244ea07SJohn Dyson return EAGAIN; 16692244ea07SJohn Dyson } 16702244ea07SJohn Dyson } 16712244ea07SJohn Dyson 16722244ea07SJohn Dyson /* NOTREACHED */ 16732244ea07SJohn Dyson return EINVAL; 1674dd85920aSJason Evans #endif /* VFS_AIO */ 16752244ea07SJohn Dyson } 1676ee877a35SJohn Dyson 1677ee877a35SJohn Dyson /* 1678dd85920aSJason Evans * aio_cancel cancels any non-physio aio operations not currently in 1679dd85920aSJason Evans * progress. 1680ee877a35SJohn Dyson */ 1681ee877a35SJohn Dyson int 1682b40ce416SJulian Elischer aio_cancel(struct thread *td, struct aio_cancel_args *uap) 1683fd3bf775SJohn Dyson { 1684dd85920aSJason Evans #ifndef VFS_AIO 168578922e41SJohn Dyson return ENOSYS; 1686dd85920aSJason Evans #else 1687b40ce416SJulian Elischer struct proc *p = td->td_proc; 1688dd85920aSJason Evans struct kaioinfo *ki; 1689dd85920aSJason Evans struct aiocblist *cbe, *cbn; 1690dd85920aSJason Evans struct file *fp; 1691dd85920aSJason Evans struct filedesc *fdp; 1692dd85920aSJason Evans struct socket *so; 1693dd85920aSJason Evans struct proc *po; 1694dd85920aSJason Evans int s,error; 1695dd85920aSJason Evans int cancelled=0; 1696dd85920aSJason Evans int notcancelled=0; 1697dd85920aSJason Evans struct vnode *vp; 1698dd85920aSJason Evans 1699dd85920aSJason Evans fdp = p->p_fd; 1700dd85920aSJason Evans 1701dd85920aSJason Evans fp = fdp->fd_ofiles[uap->fd]; 1702dd85920aSJason Evans 1703dd85920aSJason Evans if (fp == NULL) { 1704dd85920aSJason Evans return EBADF; 1705dd85920aSJason Evans } 1706dd85920aSJason Evans 1707dd85920aSJason Evans if (fp->f_type == DTYPE_VNODE) { 1708dd85920aSJason Evans vp = (struct vnode *)fp->f_data; 1709dd85920aSJason Evans 1710dd85920aSJason Evans if (vn_isdisk(vp,&error)) { 1711b40ce416SJulian Elischer td->td_retval[0] = AIO_NOTCANCELED; 1712dd85920aSJason Evans return 0; 1713dd85920aSJason Evans } 1714dd85920aSJason Evans } else if (fp->f_type == DTYPE_SOCKET) { 1715dd85920aSJason Evans so = (struct socket *)fp->f_data; 1716dd85920aSJason Evans 1717dd85920aSJason Evans s = splnet(); 1718dd85920aSJason Evans 1719dd85920aSJason Evans for (cbe = TAILQ_FIRST(&so->so_aiojobq); cbe; cbe = cbn) { 1720dd85920aSJason Evans cbn = TAILQ_NEXT(cbe, list); 1721dd85920aSJason Evans if ((uap->aiocbp == NULL) || 1722dd85920aSJason Evans (uap->aiocbp == cbe->uuaiocb) ) { 1723dd85920aSJason Evans po = cbe->userproc; 1724dd85920aSJason Evans ki = po->p_aioinfo; 1725dd85920aSJason Evans TAILQ_REMOVE(&so->so_aiojobq, cbe, list); 1726dd85920aSJason Evans TAILQ_REMOVE(&ki->kaio_sockqueue, cbe, plist); 1727dd85920aSJason Evans TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, plist); 1728dd85920aSJason Evans if (ki->kaio_flags & KAIO_WAKEUP) { 1729dd85920aSJason Evans wakeup(po); 1730dd85920aSJason Evans } 1731dd85920aSJason Evans cbe->jobstate = JOBST_JOBFINISHED; 1732dd85920aSJason Evans cbe->uaiocb._aiocb_private.status=-1; 1733dd85920aSJason Evans cbe->uaiocb._aiocb_private.error=ECANCELED; 1734dd85920aSJason Evans cancelled++; 1735cb679c38SJonathan Lemon /* XXX cancelled, knote? */ 1736dd85920aSJason Evans if (cbe->uaiocb.aio_sigevent.sigev_notify == 173719eb87d2SJohn Baldwin SIGEV_SIGNAL) { 173819eb87d2SJohn Baldwin PROC_LOCK(cbe->userproc); 1739dd85920aSJason Evans psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo); 174019eb87d2SJohn Baldwin PROC_UNLOCK(cbe->userproc); 174119eb87d2SJohn Baldwin } 1742dd85920aSJason Evans if (uap->aiocbp) 1743dd85920aSJason Evans break; 1744dd85920aSJason Evans } 1745dd85920aSJason Evans } 1746dd85920aSJason Evans 1747dd85920aSJason Evans splx(s); 1748dd85920aSJason Evans 1749dd85920aSJason Evans if ((cancelled) && (uap->aiocbp)) { 1750b40ce416SJulian Elischer td->td_retval[0] = AIO_CANCELED; 1751dd85920aSJason Evans return 0; 1752dd85920aSJason Evans } 1753dd85920aSJason Evans 1754dd85920aSJason Evans } 1755dd85920aSJason Evans 1756dd85920aSJason Evans ki=p->p_aioinfo; 1757dd85920aSJason Evans 1758dd85920aSJason Evans s = splnet(); 1759dd85920aSJason Evans 1760dd85920aSJason Evans for (cbe = TAILQ_FIRST(&ki->kaio_jobqueue); cbe; cbe = cbn) { 1761dd85920aSJason Evans cbn = TAILQ_NEXT(cbe, plist); 1762dd85920aSJason Evans 1763dd85920aSJason Evans if ((uap->fd == cbe->uaiocb.aio_fildes) && 1764dd85920aSJason Evans ((uap->aiocbp == NULL ) || 1765dd85920aSJason Evans (uap->aiocbp == cbe->uuaiocb))) { 1766dd85920aSJason Evans 1767dd85920aSJason Evans if (cbe->jobstate == JOBST_JOBQGLOBAL) { 1768dd85920aSJason Evans TAILQ_REMOVE(&aio_jobs, cbe, list); 1769dd85920aSJason Evans TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist); 1770dd85920aSJason Evans TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, 1771dd85920aSJason Evans plist); 1772dd85920aSJason Evans cancelled++; 1773dd85920aSJason Evans ki->kaio_queue_finished_count++; 1774dd85920aSJason Evans cbe->jobstate = JOBST_JOBFINISHED; 1775dd85920aSJason Evans cbe->uaiocb._aiocb_private.status = -1; 1776dd85920aSJason Evans cbe->uaiocb._aiocb_private.error = ECANCELED; 1777cb679c38SJonathan Lemon /* XXX cancelled, knote? */ 1778dd85920aSJason Evans if (cbe->uaiocb.aio_sigevent.sigev_notify == 177919eb87d2SJohn Baldwin SIGEV_SIGNAL) { 178019eb87d2SJohn Baldwin PROC_LOCK(cbe->userproc); 1781dd85920aSJason Evans psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo); 178219eb87d2SJohn Baldwin PROC_UNLOCK(cbe->userproc); 178319eb87d2SJohn Baldwin } 1784dd85920aSJason Evans } else { 1785dd85920aSJason Evans notcancelled++; 1786dd85920aSJason Evans } 1787dd85920aSJason Evans } 1788dd85920aSJason Evans } 1789dd85920aSJason Evans 1790dd85920aSJason Evans splx(s); 1791dd85920aSJason Evans 1792dd85920aSJason Evans 1793dd85920aSJason Evans if (notcancelled) { 1794b40ce416SJulian Elischer td->td_retval[0] = AIO_NOTCANCELED; 1795dd85920aSJason Evans return 0; 1796dd85920aSJason Evans } 1797dd85920aSJason Evans 1798dd85920aSJason Evans if (cancelled) { 1799b40ce416SJulian Elischer td->td_retval[0] = AIO_CANCELED; 1800dd85920aSJason Evans return 0; 1801dd85920aSJason Evans } 1802dd85920aSJason Evans 1803b40ce416SJulian Elischer td->td_retval[0] = AIO_ALLDONE; 1804dd85920aSJason Evans 1805dd85920aSJason Evans return 0; 1806dd85920aSJason Evans #endif /* VFS_AIO */ 1807ee877a35SJohn Dyson } 1808ee877a35SJohn Dyson 1809ee877a35SJohn Dyson /* 1810bfbbc4aaSJason Evans * aio_error is implemented in the kernel level for compatibility purposes only. 1811bfbbc4aaSJason Evans * For a user mode async implementation, it would be best to do it in a userland 1812bfbbc4aaSJason Evans * subroutine. 1813ee877a35SJohn Dyson */ 1814ee877a35SJohn Dyson int 1815b40ce416SJulian Elischer aio_error(struct thread *td, struct aio_error_args *uap) 1816fd3bf775SJohn Dyson { 1817dd85920aSJason Evans #ifndef VFS_AIO 1818dd85920aSJason Evans return ENOSYS; 1819dd85920aSJason Evans #else 1820b40ce416SJulian Elischer struct proc *p = td->td_proc; 182184af4da6SJohn Dyson int s; 18222244ea07SJohn Dyson struct aiocblist *cb; 18232244ea07SJohn Dyson struct kaioinfo *ki; 18242244ea07SJohn Dyson int jobref; 1825ee877a35SJohn Dyson 18262244ea07SJohn Dyson ki = p->p_aioinfo; 18272244ea07SJohn Dyson if (ki == NULL) 18282244ea07SJohn Dyson return EINVAL; 18292244ea07SJohn Dyson 18302244ea07SJohn Dyson jobref = fuword(&uap->aiocbp->_aiocb_private.kernelinfo); 1831fd3bf775SJohn Dyson if ((jobref == -1) || (jobref == 0)) 1832fd3bf775SJohn Dyson return EINVAL; 1833ee877a35SJohn Dyson 1834bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_jobdone); cb; cb = TAILQ_NEXT(cb, 1835bfbbc4aaSJason Evans plist)) { 1836bfbbc4aaSJason Evans if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1837bfbbc4aaSJason Evans jobref) { 1838b40ce416SJulian Elischer td->td_retval[0] = cb->uaiocb._aiocb_private.error; 18392244ea07SJohn Dyson return 0; 18402244ea07SJohn Dyson } 1841ee877a35SJohn Dyson } 1842ee877a35SJohn Dyson 1843bfbbc4aaSJason Evans s = splnet(); 18442244ea07SJohn Dyson 1845bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_jobqueue); cb; cb = TAILQ_NEXT(cb, 1846bfbbc4aaSJason Evans plist)) { 1847bfbbc4aaSJason Evans if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1848bfbbc4aaSJason Evans jobref) { 1849b40ce416SJulian Elischer td->td_retval[0] = EINPROGRESS; 1850bfbbc4aaSJason Evans splx(s); 18512244ea07SJohn Dyson return 0; 18522244ea07SJohn Dyson } 18532244ea07SJohn Dyson } 185460ffb019SJason Evans 185560ffb019SJason Evans for (cb = TAILQ_FIRST(&ki->kaio_sockqueue); cb; cb = TAILQ_NEXT(cb, 185660ffb019SJason Evans plist)) { 185760ffb019SJason Evans if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 185860ffb019SJason Evans jobref) { 1859b40ce416SJulian Elischer td->td_retval[0] = EINPROGRESS; 186060ffb019SJason Evans splx(s); 186160ffb019SJason Evans return 0; 186260ffb019SJason Evans } 186360ffb019SJason Evans } 1864bfbbc4aaSJason Evans splx(s); 18652244ea07SJohn Dyson 186684af4da6SJohn Dyson s = splbio(); 1867bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = TAILQ_NEXT(cb, 1868bfbbc4aaSJason Evans plist)) { 1869bfbbc4aaSJason Evans if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1870bfbbc4aaSJason Evans jobref) { 1871b40ce416SJulian Elischer td->td_retval[0] = cb->uaiocb._aiocb_private.error; 187284af4da6SJohn Dyson splx(s); 187384af4da6SJohn Dyson return 0; 187484af4da6SJohn Dyson } 187584af4da6SJohn Dyson } 187684af4da6SJohn Dyson 1877bfbbc4aaSJason Evans for (cb = TAILQ_FIRST(&ki->kaio_bufqueue); cb; cb = TAILQ_NEXT(cb, 1878bfbbc4aaSJason Evans plist)) { 1879bfbbc4aaSJason Evans if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1880bfbbc4aaSJason Evans jobref) { 1881b40ce416SJulian Elischer td->td_retval[0] = EINPROGRESS; 188284af4da6SJohn Dyson splx(s); 188384af4da6SJohn Dyson return 0; 188484af4da6SJohn Dyson } 188584af4da6SJohn Dyson } 188684af4da6SJohn Dyson splx(s); 188784af4da6SJohn Dyson 1888bfbbc4aaSJason Evans #if (0) 18892244ea07SJohn Dyson /* 1890bfbbc4aaSJason Evans * Hack for lio. 18912244ea07SJohn Dyson */ 18922244ea07SJohn Dyson status = fuword(&uap->aiocbp->_aiocb_private.status); 1893bfbbc4aaSJason Evans if (status == -1) 18942244ea07SJohn Dyson return fuword(&uap->aiocbp->_aiocb_private.error); 1895bfbbc4aaSJason Evans #endif 18962244ea07SJohn Dyson return EINVAL; 1897dd85920aSJason Evans #endif /* VFS_AIO */ 1898ee877a35SJohn Dyson } 1899ee877a35SJohn Dyson 1900ee877a35SJohn Dyson int 1901b40ce416SJulian Elischer aio_read(struct thread *td, struct aio_read_args *uap) 1902fd3bf775SJohn Dyson { 1903dd85920aSJason Evans #ifndef VFS_AIO 1904dd85920aSJason Evans return ENOSYS; 1905dd85920aSJason Evans #else 1906b40ce416SJulian Elischer return aio_aqueue(td, uap->aiocbp, LIO_READ); 1907dd85920aSJason Evans #endif /* VFS_AIO */ 1908ee877a35SJohn Dyson } 1909ee877a35SJohn Dyson 1910ee877a35SJohn Dyson int 1911b40ce416SJulian Elischer aio_write(struct thread *td, struct aio_write_args *uap) 1912fd3bf775SJohn Dyson { 1913dd85920aSJason Evans #ifndef VFS_AIO 1914dd85920aSJason Evans return ENOSYS; 1915dd85920aSJason Evans #else 1916b40ce416SJulian Elischer return aio_aqueue(td, uap->aiocbp, LIO_WRITE); 1917dd85920aSJason Evans #endif /* VFS_AIO */ 1918ee877a35SJohn Dyson } 1919ee877a35SJohn Dyson 1920ee877a35SJohn Dyson int 1921b40ce416SJulian Elischer lio_listio(struct thread *td, struct lio_listio_args *uap) 1922fd3bf775SJohn Dyson { 1923dd85920aSJason Evans #ifndef VFS_AIO 1924dd85920aSJason Evans return ENOSYS; 1925dd85920aSJason Evans #else 1926b40ce416SJulian Elischer struct proc *p = td->td_proc; 19274a11ca4eSPoul-Henning Kamp int nent, nentqueued; 19282244ea07SJohn Dyson struct aiocb *iocb, * const *cbptr; 19292244ea07SJohn Dyson struct aiocblist *cb; 19302244ea07SJohn Dyson struct kaioinfo *ki; 193184af4da6SJohn Dyson struct aio_liojob *lj; 19322244ea07SJohn Dyson int error, runningcode; 1933fd3bf775SJohn Dyson int nerror; 1934ee877a35SJohn Dyson int i; 193584af4da6SJohn Dyson int s; 1936ee877a35SJohn Dyson 1937bfbbc4aaSJason Evans if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) 1938ee877a35SJohn Dyson return EINVAL; 19392244ea07SJohn Dyson 19402244ea07SJohn Dyson nent = uap->nent; 1941bfbbc4aaSJason Evans if (nent > AIO_LISTIO_MAX) 19422244ea07SJohn Dyson return EINVAL; 19432244ea07SJohn Dyson 1944bfbbc4aaSJason Evans if (p->p_aioinfo == NULL) 19452244ea07SJohn Dyson aio_init_aioinfo(p); 19462244ea07SJohn Dyson 1947bfbbc4aaSJason Evans if ((nent + num_queue_count) > max_queue_count) 19482244ea07SJohn Dyson return EAGAIN; 19492244ea07SJohn Dyson 19502244ea07SJohn Dyson ki = p->p_aioinfo; 1951bfbbc4aaSJason Evans if ((nent + ki->kaio_queue_count) > ki->kaio_qallowed_count) 19522244ea07SJohn Dyson return EAGAIN; 19532244ea07SJohn Dyson 195484af4da6SJohn Dyson lj = zalloc(aiolio_zone); 1955bfbbc4aaSJason Evans if (!lj) 195684af4da6SJohn Dyson return EAGAIN; 195784af4da6SJohn Dyson 195884af4da6SJohn Dyson lj->lioj_flags = 0; 195984af4da6SJohn Dyson lj->lioj_buffer_count = 0; 196084af4da6SJohn Dyson lj->lioj_buffer_finished_count = 0; 196184af4da6SJohn Dyson lj->lioj_queue_count = 0; 196284af4da6SJohn Dyson lj->lioj_queue_finished_count = 0; 196384af4da6SJohn Dyson lj->lioj_ki = ki; 196484af4da6SJohn Dyson 196584af4da6SJohn Dyson /* 1966bfbbc4aaSJason Evans * Setup signal. 196784af4da6SJohn Dyson */ 196884af4da6SJohn Dyson if (uap->sig && (uap->mode == LIO_NOWAIT)) { 1969bfbbc4aaSJason Evans error = copyin(uap->sig, &lj->lioj_signal, 1970bfbbc4aaSJason Evans sizeof(lj->lioj_signal)); 19712f3cf918SAlfred Perlstein if (error) { 19722f3cf918SAlfred Perlstein zfree(aiolio_zone, lj); 197384af4da6SJohn Dyson return error; 19742f3cf918SAlfred Perlstein } 19752f3cf918SAlfred Perlstein if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) { 19762f3cf918SAlfred Perlstein zfree(aiolio_zone, lj); 19772f3cf918SAlfred Perlstein return EINVAL; 19782f3cf918SAlfred Perlstein } 197984af4da6SJohn Dyson lj->lioj_flags |= LIOJ_SIGNAL; 198084af4da6SJohn Dyson lj->lioj_flags &= ~LIOJ_SIGNAL_POSTED; 1981bfbbc4aaSJason Evans } else 198284af4da6SJohn Dyson lj->lioj_flags &= ~LIOJ_SIGNAL; 198384af4da6SJohn Dyson 19842f3cf918SAlfred Perlstein TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list); 19852244ea07SJohn Dyson /* 1986bfbbc4aaSJason Evans * Get pointers to the list of I/O requests. 19872244ea07SJohn Dyson */ 1988fd3bf775SJohn Dyson nerror = 0; 1989fd3bf775SJohn Dyson nentqueued = 0; 19902244ea07SJohn Dyson cbptr = uap->acb_list; 19912244ea07SJohn Dyson for (i = 0; i < uap->nent; i++) { 199230166fabSBruce Evans iocb = (struct aiocb *)(intptr_t)fuword((caddr_t)&cbptr[i]); 199330166fabSBruce Evans if (((intptr_t)iocb != -1) && ((intptr_t)iocb != NULL)) { 1994b40ce416SJulian Elischer error = _aio_aqueue(td, iocb, lj, 0); 1995bfbbc4aaSJason Evans if (error == 0) 19962244ea07SJohn Dyson nentqueued++; 1997bfbbc4aaSJason Evans else 1998fd3bf775SJohn Dyson nerror++; 1999fd3bf775SJohn Dyson } 2000fd3bf775SJohn Dyson } 20012244ea07SJohn Dyson 2002a624e84fSJohn Dyson /* 2003bfbbc4aaSJason Evans * If we haven't queued any, then just return error. 2004a624e84fSJohn Dyson */ 2005bfbbc4aaSJason Evans if (nentqueued == 0) 2006fd3bf775SJohn Dyson return 0; 20072244ea07SJohn Dyson 2008a624e84fSJohn Dyson /* 2009bfbbc4aaSJason Evans * Calculate the appropriate error return. 2010a624e84fSJohn Dyson */ 20112244ea07SJohn Dyson runningcode = 0; 2012fd3bf775SJohn Dyson if (nerror) 20132244ea07SJohn Dyson runningcode = EIO; 20142244ea07SJohn Dyson 20152244ea07SJohn Dyson if (uap->mode == LIO_WAIT) { 2016bfbbc4aaSJason Evans int command, found, jobref; 2017bfbbc4aaSJason Evans 2018bfbbc4aaSJason Evans for (;;) { 2019fd3bf775SJohn Dyson found = 0; 2020fd3bf775SJohn Dyson for (i = 0; i < uap->nent; i++) { 2021a624e84fSJohn Dyson /* 2022bfbbc4aaSJason Evans * Fetch address of the control buf pointer in 2023bfbbc4aaSJason Evans * user space. 2024a624e84fSJohn Dyson */ 202530166fabSBruce Evans iocb = (struct aiocb *)(intptr_t)fuword((caddr_t)&cbptr[i]); 2026bfbbc4aaSJason Evans if (((intptr_t)iocb == -1) || ((intptr_t)iocb 2027bfbbc4aaSJason Evans == 0)) 2028fd3bf775SJohn Dyson continue; 2029a624e84fSJohn Dyson 2030a624e84fSJohn Dyson /* 2031bfbbc4aaSJason Evans * Fetch the associated command from user space. 2032a624e84fSJohn Dyson */ 20332244ea07SJohn Dyson command = fuword(&iocb->aio_lio_opcode); 2034fd3bf775SJohn Dyson if (command == LIO_NOP) { 2035fd3bf775SJohn Dyson found++; 20362244ea07SJohn Dyson continue; 2037fd3bf775SJohn Dyson } 2038a624e84fSJohn Dyson 20392244ea07SJohn Dyson jobref = fuword(&iocb->_aiocb_private.kernelinfo); 20402244ea07SJohn Dyson 204137d40066SPoul-Henning Kamp TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) { 2042bfbbc4aaSJason Evans if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) 2043bfbbc4aaSJason Evans == jobref) { 2044bfbbc4aaSJason Evans if (cb->uaiocb.aio_lio_opcode 2045bfbbc4aaSJason Evans == LIO_WRITE) { 2046bfbbc4aaSJason Evans curproc->p_stats->p_ru.ru_oublock 2047bfbbc4aaSJason Evans += 204884af4da6SJohn Dyson cb->outputcharge; 204984af4da6SJohn Dyson cb->outputcharge = 0; 2050bfbbc4aaSJason Evans } else if (cb->uaiocb.aio_lio_opcode 2051bfbbc4aaSJason Evans == LIO_READ) { 2052bfbbc4aaSJason Evans curproc->p_stats->p_ru.ru_inblock 2053bfbbc4aaSJason Evans += cb->inputcharge; 205484af4da6SJohn Dyson cb->inputcharge = 0; 205584af4da6SJohn Dyson } 20562244ea07SJohn Dyson found++; 20572244ea07SJohn Dyson break; 20582244ea07SJohn Dyson } 20592244ea07SJohn Dyson } 2060fd3bf775SJohn Dyson 206184af4da6SJohn Dyson s = splbio(); 206237d40066SPoul-Henning Kamp TAILQ_FOREACH(cb, &ki->kaio_bufdone, plist) { 2063bfbbc4aaSJason Evans if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) 2064bfbbc4aaSJason Evans == jobref) { 206584af4da6SJohn Dyson found++; 206684af4da6SJohn Dyson break; 2067fd3bf775SJohn Dyson } 20682244ea07SJohn Dyson } 206984af4da6SJohn Dyson splx(s); 207084af4da6SJohn Dyson } 20712244ea07SJohn Dyson 2072a624e84fSJohn Dyson /* 2073bfbbc4aaSJason Evans * If all I/Os have been disposed of, then we can 2074bfbbc4aaSJason Evans * return. 2075a624e84fSJohn Dyson */ 2076bfbbc4aaSJason Evans if (found == nentqueued) 20772244ea07SJohn Dyson return runningcode; 20782244ea07SJohn Dyson 2079fd3bf775SJohn Dyson ki->kaio_flags |= KAIO_WAKEUP; 20802244ea07SJohn Dyson error = tsleep(p, PRIBIO | PCATCH, "aiospn", 0); 20812244ea07SJohn Dyson 2082bfbbc4aaSJason Evans if (error == EINTR) 20832244ea07SJohn Dyson return EINTR; 2084bfbbc4aaSJason Evans else if (error == EWOULDBLOCK) 20852244ea07SJohn Dyson return EAGAIN; 20862244ea07SJohn Dyson } 20872244ea07SJohn Dyson } 20882244ea07SJohn Dyson 20892244ea07SJohn Dyson return runningcode; 2090dd85920aSJason Evans #endif /* VFS_AIO */ 2091ee877a35SJohn Dyson } 2092fd3bf775SJohn Dyson 20935dec52baSPeter Wemm #ifdef VFS_AIO 209484af4da6SJohn Dyson /* 2095f09deb69SJeroen Ruigrok van der Werven * This is a weird hack so that we can post a signal. It is safe to do so from 2096bfbbc4aaSJason Evans * a timeout routine, but *not* from an interrupt routine. 209784af4da6SJohn Dyson */ 209884af4da6SJohn Dyson static void 2099bfbbc4aaSJason Evans process_signal(void *aioj) 210084af4da6SJohn Dyson { 2101bfbbc4aaSJason Evans struct aiocblist *aiocbe = aioj; 2102bfbbc4aaSJason Evans struct aio_liojob *lj = aiocbe->lio; 2103bfbbc4aaSJason Evans struct aiocb *cb = &aiocbe->uaiocb; 2104bfbbc4aaSJason Evans 2105bfbbc4aaSJason Evans if ((lj) && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL) && 2106bfbbc4aaSJason Evans (lj->lioj_queue_count == lj->lioj_queue_finished_count)) { 210719eb87d2SJohn Baldwin PROC_LOCK(lj->lioj_ki->kaio_p); 210884af4da6SJohn Dyson psignal(lj->lioj_ki->kaio_p, lj->lioj_signal.sigev_signo); 210919eb87d2SJohn Baldwin PROC_UNLOCK(lj->lioj_ki->kaio_p); 211084af4da6SJohn Dyson lj->lioj_flags |= LIOJ_SIGNAL_POSTED; 211184af4da6SJohn Dyson } 2112bfbbc4aaSJason Evans 211319eb87d2SJohn Baldwin if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) { 211419eb87d2SJohn Baldwin PROC_LOCK(aiocbe->userproc); 2115bfbbc4aaSJason Evans psignal(aiocbe->userproc, cb->aio_sigevent.sigev_signo); 211619eb87d2SJohn Baldwin PROC_UNLOCK(aiocbe->userproc); 211719eb87d2SJohn Baldwin } 211884af4da6SJohn Dyson } 211984af4da6SJohn Dyson 212084af4da6SJohn Dyson /* 2121bfbbc4aaSJason Evans * Interrupt handler for physio, performs the necessary process wakeups, and 2122bfbbc4aaSJason Evans * signals. 212384af4da6SJohn Dyson */ 2124fd3bf775SJohn Dyson static void 2125bfbbc4aaSJason Evans aio_physwakeup(struct buf *bp) 2126fd3bf775SJohn Dyson { 212784af4da6SJohn Dyson struct aiocblist *aiocbe; 2128fd3bf775SJohn Dyson struct proc *p; 2129fd3bf775SJohn Dyson struct kaioinfo *ki; 213084af4da6SJohn Dyson struct aio_liojob *lj; 2131fd3bf775SJohn Dyson 21329c8a2647SAlan Cox wakeup(bp); 2133fd3bf775SJohn Dyson 213484af4da6SJohn Dyson aiocbe = (struct aiocblist *)bp->b_spc; 213584af4da6SJohn Dyson if (aiocbe) { 2136b0eeea20SPoul-Henning Kamp p = bp->b_caller1; 213784af4da6SJohn Dyson 213884af4da6SJohn Dyson aiocbe->jobstate = JOBST_JOBBFINISHED; 213984af4da6SJohn Dyson aiocbe->uaiocb._aiocb_private.status -= bp->b_resid; 214084af4da6SJohn Dyson aiocbe->uaiocb._aiocb_private.error = 0; 214184af4da6SJohn Dyson aiocbe->jobflags |= AIOCBLIST_DONE; 214284af4da6SJohn Dyson 2143c244d2deSPoul-Henning Kamp if (bp->b_ioflags & BIO_ERROR) 214484af4da6SJohn Dyson aiocbe->uaiocb._aiocb_private.error = bp->b_error; 214584af4da6SJohn Dyson 214684af4da6SJohn Dyson lj = aiocbe->lio; 214784af4da6SJohn Dyson if (lj) { 214884af4da6SJohn Dyson lj->lioj_buffer_finished_count++; 2149bfbbc4aaSJason Evans 215084af4da6SJohn Dyson /* 2151bfbbc4aaSJason Evans * wakeup/signal if all of the interrupt jobs are done. 215284af4da6SJohn Dyson */ 2153bfbbc4aaSJason Evans if (lj->lioj_buffer_finished_count == 2154bfbbc4aaSJason Evans lj->lioj_buffer_count) { 215584af4da6SJohn Dyson /* 2156bfbbc4aaSJason Evans * Post a signal if it is called for. 215784af4da6SJohn Dyson */ 2158bfbbc4aaSJason Evans if ((lj->lioj_flags & 2159bfbbc4aaSJason Evans (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == 216084af4da6SJohn Dyson LIOJ_SIGNAL) { 216184af4da6SJohn Dyson lj->lioj_flags |= LIOJ_SIGNAL_POSTED; 216213644654SAlan Cox aiocbe->timeouthandle = 216313644654SAlan Cox timeout(process_signal, 216413644654SAlan Cox aiocbe, 0); 216584af4da6SJohn Dyson } 216684af4da6SJohn Dyson } 216784af4da6SJohn Dyson } 216884af4da6SJohn Dyson 2169fd3bf775SJohn Dyson ki = p->p_aioinfo; 217084af4da6SJohn Dyson if (ki) { 217184af4da6SJohn Dyson ki->kaio_buffer_finished_count++; 217284af4da6SJohn Dyson TAILQ_REMOVE(&aio_bufjobs, aiocbe, list); 217384af4da6SJohn Dyson TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist); 217484af4da6SJohn Dyson TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist); 2175bfbbc4aaSJason Evans 2176cb679c38SJonathan Lemon KNOTE(&aiocbe->klist, 0); 2177bfbbc4aaSJason Evans /* Do the wakeup. */ 217884af4da6SJohn Dyson if (ki->kaio_flags & (KAIO_RUNDOWN|KAIO_WAKEUP)) { 2179fd3bf775SJohn Dyson ki->kaio_flags &= ~KAIO_WAKEUP; 2180fd3bf775SJohn Dyson wakeup(p); 2181fd3bf775SJohn Dyson } 2182fd3bf775SJohn Dyson } 2183bfbbc4aaSJason Evans 2184bfbbc4aaSJason Evans if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL) 218513644654SAlan Cox aiocbe->timeouthandle = 2186bfbbc4aaSJason Evans timeout(process_signal, aiocbe, 0); 2187fd3bf775SJohn Dyson } 218884af4da6SJohn Dyson } 21895dec52baSPeter Wemm #endif /* VFS_AIO */ 2190bfbbc4aaSJason Evans 2191bfbbc4aaSJason Evans int 2192b40ce416SJulian Elischer aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap) 2193bfbbc4aaSJason Evans { 2194dd85920aSJason Evans #ifndef VFS_AIO 2195dd85920aSJason Evans return ENOSYS; 2196dd85920aSJason Evans #else 2197b40ce416SJulian Elischer struct proc *p = td->td_proc; 2198bfbbc4aaSJason Evans struct timeval atv; 2199bfbbc4aaSJason Evans struct timespec ts; 2200bfbbc4aaSJason Evans struct aiocb **cbptr; 2201bfbbc4aaSJason Evans struct kaioinfo *ki; 2202bfbbc4aaSJason Evans struct aiocblist *cb = NULL; 2203bfbbc4aaSJason Evans int error, s, timo; 2204bfbbc4aaSJason Evans 2205dd85920aSJason Evans suword(uap->aiocbp, (int)NULL); 2206dd85920aSJason Evans 2207bfbbc4aaSJason Evans timo = 0; 2208bfbbc4aaSJason Evans if (uap->timeout) { 2209bfbbc4aaSJason Evans /* Get timespec struct. */ 221088ed460eSAlan Cox error = copyin(uap->timeout, &ts, sizeof(ts)); 2211bfbbc4aaSJason Evans if (error) 2212bfbbc4aaSJason Evans return error; 2213bfbbc4aaSJason Evans 2214bfbbc4aaSJason Evans if ((ts.tv_nsec < 0) || (ts.tv_nsec >= 1000000000)) 2215bfbbc4aaSJason Evans return (EINVAL); 2216bfbbc4aaSJason Evans 2217bfbbc4aaSJason Evans TIMESPEC_TO_TIMEVAL(&atv, &ts); 2218bfbbc4aaSJason Evans if (itimerfix(&atv)) 2219bfbbc4aaSJason Evans return (EINVAL); 2220bfbbc4aaSJason Evans timo = tvtohz(&atv); 2221bfbbc4aaSJason Evans } 2222bfbbc4aaSJason Evans 2223bfbbc4aaSJason Evans ki = p->p_aioinfo; 2224bfbbc4aaSJason Evans if (ki == NULL) 2225bfbbc4aaSJason Evans return EAGAIN; 2226bfbbc4aaSJason Evans 2227bfbbc4aaSJason Evans cbptr = uap->aiocbp; 2228bfbbc4aaSJason Evans 2229bfbbc4aaSJason Evans for (;;) { 2230bfbbc4aaSJason Evans if ((cb = TAILQ_FIRST(&ki->kaio_jobdone)) != 0) { 2231bfbbc4aaSJason Evans suword(uap->aiocbp, (int)cb->uuaiocb); 2232b40ce416SJulian Elischer td->td_retval[0] = cb->uaiocb._aiocb_private.status; 2233bfbbc4aaSJason Evans if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) { 2234bfbbc4aaSJason Evans curproc->p_stats->p_ru.ru_oublock += 2235bfbbc4aaSJason Evans cb->outputcharge; 2236bfbbc4aaSJason Evans cb->outputcharge = 0; 2237bfbbc4aaSJason Evans } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) { 2238bfbbc4aaSJason Evans curproc->p_stats->p_ru.ru_inblock += 2239bfbbc4aaSJason Evans cb->inputcharge; 2240bfbbc4aaSJason Evans cb->inputcharge = 0; 2241bfbbc4aaSJason Evans } 2242bfbbc4aaSJason Evans aio_free_entry(cb); 2243dd85920aSJason Evans return cb->uaiocb._aiocb_private.error; 2244bfbbc4aaSJason Evans } 2245bfbbc4aaSJason Evans 2246bfbbc4aaSJason Evans s = splbio(); 2247bfbbc4aaSJason Evans if ((cb = TAILQ_FIRST(&ki->kaio_bufdone)) != 0 ) { 2248bfbbc4aaSJason Evans splx(s); 2249bfbbc4aaSJason Evans suword(uap->aiocbp, (int)cb->uuaiocb); 2250b40ce416SJulian Elischer td->td_retval[0] = cb->uaiocb._aiocb_private.status; 2251bfbbc4aaSJason Evans aio_free_entry(cb); 2252dd85920aSJason Evans return cb->uaiocb._aiocb_private.error; 2253bfbbc4aaSJason Evans } 2254bfbbc4aaSJason Evans 2255bfbbc4aaSJason Evans ki->kaio_flags |= KAIO_WAKEUP; 2256bfbbc4aaSJason Evans error = tsleep(p, PRIBIO | PCATCH, "aiowc", timo); 2257dd85920aSJason Evans splx(s); 2258bfbbc4aaSJason Evans 2259dd85920aSJason Evans if (error == ERESTART) 2260dd85920aSJason Evans return EINTR; 2261dd85920aSJason Evans else if (error < 0) 2262bfbbc4aaSJason Evans return error; 2263bfbbc4aaSJason Evans else if (error == EINTR) 2264bfbbc4aaSJason Evans return EINTR; 2265bfbbc4aaSJason Evans else if (error == EWOULDBLOCK) 2266bfbbc4aaSJason Evans return EAGAIN; 2267bfbbc4aaSJason Evans } 2268dd85920aSJason Evans #endif /* VFS_AIO */ 2269bfbbc4aaSJason Evans } 2270cb679c38SJonathan Lemon 22715dec52baSPeter Wemm 22725dec52baSPeter Wemm #ifndef VFS_AIO 22735dec52baSPeter Wemm static int 22745dec52baSPeter Wemm filt_aioattach(struct knote *kn) 22755dec52baSPeter Wemm { 22765dec52baSPeter Wemm 22775dec52baSPeter Wemm return (ENXIO); 22785dec52baSPeter Wemm } 22795dec52baSPeter Wemm 22805dec52baSPeter Wemm struct filterops aio_filtops = 22815dec52baSPeter Wemm { 0, filt_aioattach, NULL, NULL }; 22825dec52baSPeter Wemm 22835dec52baSPeter Wemm #else 2284cb679c38SJonathan Lemon static int 2285cb679c38SJonathan Lemon filt_aioattach(struct knote *kn) 2286cb679c38SJonathan Lemon { 2287cb679c38SJonathan Lemon struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_id; 2288cb679c38SJonathan Lemon 2289cb679c38SJonathan Lemon /* 2290cb679c38SJonathan Lemon * The aiocbe pointer must be validated before using it, so 2291cb679c38SJonathan Lemon * registration is restricted to the kernel; the user cannot 2292cb679c38SJonathan Lemon * set EV_FLAG1. 2293cb679c38SJonathan Lemon */ 2294cb679c38SJonathan Lemon if ((kn->kn_flags & EV_FLAG1) == 0) 2295cb679c38SJonathan Lemon return (EPERM); 2296cb679c38SJonathan Lemon kn->kn_flags &= ~EV_FLAG1; 2297cb679c38SJonathan Lemon 2298cb679c38SJonathan Lemon SLIST_INSERT_HEAD(&aiocbe->klist, kn, kn_selnext); 2299cb679c38SJonathan Lemon 2300cb679c38SJonathan Lemon return (0); 2301cb679c38SJonathan Lemon } 2302cb679c38SJonathan Lemon 2303cb679c38SJonathan Lemon static void 2304cb679c38SJonathan Lemon filt_aiodetach(struct knote *kn) 2305cb679c38SJonathan Lemon { 2306cb679c38SJonathan Lemon struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_id; 2307cb679c38SJonathan Lemon int s = splhigh(); /* XXX no clue, so overkill */ 2308cb679c38SJonathan Lemon 2309e3975643SJake Burkholder SLIST_REMOVE(&aiocbe->klist, kn, knote, kn_selnext); 2310cb679c38SJonathan Lemon splx(s); 2311cb679c38SJonathan Lemon } 2312cb679c38SJonathan Lemon 2313cb679c38SJonathan Lemon /*ARGSUSED*/ 2314cb679c38SJonathan Lemon static int 2315cb679c38SJonathan Lemon filt_aio(struct knote *kn, long hint) 2316cb679c38SJonathan Lemon { 2317cb679c38SJonathan Lemon struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_id; 2318cb679c38SJonathan Lemon 2319cb679c38SJonathan Lemon kn->kn_data = 0; /* XXX data returned? */ 2320b70158baSAlan Cox if (aiocbe->jobstate != JOBST_JOBFINISHED && 2321b70158baSAlan Cox aiocbe->jobstate != JOBST_JOBBFINISHED) 2322cb679c38SJonathan Lemon return (0); 2323cb679c38SJonathan Lemon kn->kn_flags |= EV_EOF; 2324cb679c38SJonathan Lemon return (1); 2325cb679c38SJonathan Lemon } 23265dec52baSPeter Wemm 23275dec52baSPeter Wemm struct filterops aio_filtops = 23285dec52baSPeter Wemm { 0, filt_aioattach, filt_aiodetach, filt_aio }; 23295dec52baSPeter Wemm #endif /* VFS_AIO */ 2330