xref: /freebsd/sys/kern/vfs_aio.c (revision 74bf4e164ba5851606a27d4feff27717452583e5)
1 /*
2  * Copyright (c) 1997 John S. Dyson.  All rights reserved.
3  *
4  * Redistribution and use in source and binary forms, with or without
5  * modification, are permitted provided that the following conditions
6  * are met:
7  * 1. Redistributions of source code must retain the above copyright
8  *    notice, this list of conditions and the following disclaimer.
9  * 2. John S. Dyson's name may not be used to endorse or promote products
10  *    derived from this software without specific prior written permission.
11  *
12  * DISCLAIMER:  This code isn't warranted to do anything useful.  Anything
13  * bad that happens because of using this software isn't the responsibility
14  * of the author.  This software is distributed AS-IS.
15  */
16 
17 /*
18  * This file contains support for the POSIX 1003.1B AIO/LIO facility.
19  */
20 
21 #include <sys/cdefs.h>
22 __FBSDID("$FreeBSD$");
23 
24 #include <sys/param.h>
25 #include <sys/systm.h>
26 #include <sys/malloc.h>
27 #include <sys/bio.h>
28 #include <sys/buf.h>
29 #include <sys/eventhandler.h>
30 #include <sys/sysproto.h>
31 #include <sys/filedesc.h>
32 #include <sys/kernel.h>
33 #include <sys/module.h>
34 #include <sys/kthread.h>
35 #include <sys/fcntl.h>
36 #include <sys/file.h>
37 #include <sys/limits.h>
38 #include <sys/lock.h>
39 #include <sys/mutex.h>
40 #include <sys/unistd.h>
41 #include <sys/proc.h>
42 #include <sys/resourcevar.h>
43 #include <sys/signalvar.h>
44 #include <sys/protosw.h>
45 #include <sys/socketvar.h>
46 #include <sys/syscall.h>
47 #include <sys/sysent.h>
48 #include <sys/sysctl.h>
49 #include <sys/sx.h>
50 #include <sys/vnode.h>
51 #include <sys/conf.h>
52 #include <sys/event.h>
53 
54 #include <posix4/posix4.h>
55 #include <vm/vm.h>
56 #include <vm/vm_extern.h>
57 #include <vm/pmap.h>
58 #include <vm/vm_map.h>
59 #include <vm/uma.h>
60 #include <sys/aio.h>
61 
62 #include "opt_vfs_aio.h"
63 
64 /*
65  * Counter for allocating reference ids to new jobs.  Wrapped to 1 on
66  * overflow.
67  */
68 static	long jobrefid;
69 
70 #define JOBST_NULL		0x0
71 #define JOBST_JOBQGLOBAL	0x2
72 #define JOBST_JOBRUNNING	0x3
73 #define JOBST_JOBFINISHED	0x4
74 #define	JOBST_JOBQBUF		0x5
75 #define	JOBST_JOBBFINISHED	0x6
76 
77 #ifndef MAX_AIO_PER_PROC
78 #define MAX_AIO_PER_PROC	32
79 #endif
80 
81 #ifndef MAX_AIO_QUEUE_PER_PROC
82 #define MAX_AIO_QUEUE_PER_PROC	256 /* Bigger than AIO_LISTIO_MAX */
83 #endif
84 
85 #ifndef MAX_AIO_PROCS
86 #define MAX_AIO_PROCS		32
87 #endif
88 
89 #ifndef MAX_AIO_QUEUE
90 #define	MAX_AIO_QUEUE		1024 /* Bigger than AIO_LISTIO_MAX */
91 #endif
92 
93 #ifndef TARGET_AIO_PROCS
94 #define TARGET_AIO_PROCS	4
95 #endif
96 
97 #ifndef MAX_BUF_AIO
98 #define MAX_BUF_AIO		16
99 #endif
100 
101 #ifndef AIOD_TIMEOUT_DEFAULT
102 #define	AIOD_TIMEOUT_DEFAULT	(10 * hz)
103 #endif
104 
105 #ifndef AIOD_LIFETIME_DEFAULT
106 #define AIOD_LIFETIME_DEFAULT	(30 * hz)
107 #endif
108 
109 SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "Async IO management");
110 
111 static int max_aio_procs = MAX_AIO_PROCS;
112 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs,
113 	CTLFLAG_RW, &max_aio_procs, 0,
114 	"Maximum number of kernel threads to use for handling async IO ");
115 
116 static int num_aio_procs = 0;
117 SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs,
118 	CTLFLAG_RD, &num_aio_procs, 0,
119 	"Number of presently active kernel threads for async IO");
120 
121 /*
122  * The code will adjust the actual number of AIO processes towards this
123  * number when it gets a chance.
124  */
125 static int target_aio_procs = TARGET_AIO_PROCS;
126 SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW, &target_aio_procs,
127 	0, "Preferred number of ready kernel threads for async IO");
128 
129 static int max_queue_count = MAX_AIO_QUEUE;
130 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW, &max_queue_count, 0,
131     "Maximum number of aio requests to queue, globally");
132 
133 static int num_queue_count = 0;
134 SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD, &num_queue_count, 0,
135     "Number of queued aio requests");
136 
137 static int num_buf_aio = 0;
138 SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD, &num_buf_aio, 0,
139     "Number of aio requests presently handled by the buf subsystem");
140 
141 /* Number of async I/O thread in the process of being started */
142 /* XXX This should be local to _aio_aqueue() */
143 static int num_aio_resv_start = 0;
144 
145 static int aiod_timeout;
146 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout, CTLFLAG_RW, &aiod_timeout, 0,
147     "Timeout value for synchronous aio operations");
148 
149 static int aiod_lifetime;
150 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW, &aiod_lifetime, 0,
151     "Maximum lifetime for idle aiod");
152 
153 static int unloadable = 0;
154 SYSCTL_INT(_vfs_aio, OID_AUTO, unloadable, CTLFLAG_RW, &unloadable, 0,
155     "Allow unload of aio (not recommended)");
156 
157 
158 static int max_aio_per_proc = MAX_AIO_PER_PROC;
159 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW, &max_aio_per_proc,
160     0, "Maximum active aio requests per process (stored in the process)");
161 
162 static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
163 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW,
164     &max_aio_queue_per_proc, 0,
165     "Maximum queued aio requests per process (stored in the process)");
166 
167 static int max_buf_aio = MAX_BUF_AIO;
168 SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0,
169     "Maximum buf aio requests per process (stored in the process)");
170 
171 struct aiocblist {
172 	TAILQ_ENTRY(aiocblist) list;	/* List of jobs */
173 	TAILQ_ENTRY(aiocblist) plist;	/* List of jobs for proc */
174 	int	jobflags;
175 	int	jobstate;
176 	int	inputcharge;
177 	int	outputcharge;
178 	struct	callout_handle timeouthandle;
179 	struct	buf *bp;		/* Buffer pointer */
180 	struct	proc *userproc;		/* User process */ /* Not td! */
181 	struct  ucred *cred;		/* Active credential when created */
182 	struct	file *fd_file;		/* Pointer to file structure */
183 	struct	aio_liojob *lio;	/* Optional lio job */
184 	struct	aiocb *uuaiocb;		/* Pointer in userspace of aiocb */
185 	struct	knlist klist;		/* list of knotes */
186 	struct	aiocb uaiocb;		/* Kernel I/O control block */
187 };
188 
189 /* jobflags */
190 #define AIOCBLIST_RUNDOWN	0x4
191 #define AIOCBLIST_DONE		0x10
192 
193 /*
194  * AIO process info
195  */
196 #define AIOP_FREE	0x1			/* proc on free queue */
197 #define AIOP_SCHED	0x2			/* proc explicitly scheduled */
198 
199 struct aiothreadlist {
200 	int aiothreadflags;			/* AIO proc flags */
201 	TAILQ_ENTRY(aiothreadlist) list;	/* List of processes */
202 	struct thread *aiothread;		/* The AIO thread */
203 };
204 
205 /*
206  * data-structure for lio signal management
207  */
208 struct aio_liojob {
209 	int	lioj_flags;
210 	int	lioj_buffer_count;
211 	int	lioj_buffer_finished_count;
212 	int	lioj_queue_count;
213 	int	lioj_queue_finished_count;
214 	struct	sigevent lioj_signal;	/* signal on all I/O done */
215 	TAILQ_ENTRY(aio_liojob) lioj_list;
216 	struct	kaioinfo *lioj_ki;
217 };
218 #define	LIOJ_SIGNAL		0x1	/* signal on all done (lio) */
219 #define	LIOJ_SIGNAL_POSTED	0x2	/* signal has been posted */
220 
221 /*
222  * per process aio data structure
223  */
224 struct kaioinfo {
225 	int	kaio_flags;		/* per process kaio flags */
226 	int	kaio_maxactive_count;	/* maximum number of AIOs */
227 	int	kaio_active_count;	/* number of currently used AIOs */
228 	int	kaio_qallowed_count;	/* maxiumu size of AIO queue */
229 	int	kaio_queue_count;	/* size of AIO queue */
230 	int	kaio_ballowed_count;	/* maximum number of buffers */
231 	int	kaio_queue_finished_count; /* number of daemon jobs finished */
232 	int	kaio_buffer_count;	/* number of physio buffers */
233 	int	kaio_buffer_finished_count; /* count of I/O done */
234 	struct 	proc *kaio_p;		/* process that uses this kaio block */
235 	TAILQ_HEAD(,aio_liojob) kaio_liojoblist; /* list of lio jobs */
236 	TAILQ_HEAD(,aiocblist) kaio_jobqueue;	/* job queue for process */
237 	TAILQ_HEAD(,aiocblist) kaio_jobdone;	/* done queue for process */
238 	TAILQ_HEAD(,aiocblist) kaio_bufqueue;	/* buffer job queue for process */
239 	TAILQ_HEAD(,aiocblist) kaio_bufdone;	/* buffer done queue for process */
240 	TAILQ_HEAD(,aiocblist) kaio_sockqueue;	/* queue for aios waiting on sockets */
241 };
242 
243 #define KAIO_RUNDOWN	0x1	/* process is being run down */
244 #define KAIO_WAKEUP	0x2	/* wakeup process when there is a significant event */
245 
246 static TAILQ_HEAD(,aiothreadlist) aio_activeproc;	/* Active daemons */
247 static TAILQ_HEAD(,aiothreadlist) aio_freeproc;		/* Idle daemons */
248 static TAILQ_HEAD(,aiocblist) aio_jobs;			/* Async job list */
249 static TAILQ_HEAD(,aiocblist) aio_bufjobs;		/* Phys I/O job list */
250 
251 static void	aio_init_aioinfo(struct proc *p);
252 static void	aio_onceonly(void);
253 static int	aio_free_entry(struct aiocblist *aiocbe);
254 static void	aio_process(struct aiocblist *aiocbe);
255 static int	aio_newproc(void);
256 static int	aio_aqueue(struct thread *td, struct aiocb *job, int type);
257 static void	aio_physwakeup(struct buf *bp);
258 static void	aio_proc_rundown(void *arg, struct proc *p);
259 static int	aio_fphysio(struct aiocblist *aiocbe);
260 static int	aio_qphysio(struct proc *p, struct aiocblist *iocb);
261 static void	aio_daemon(void *uproc);
262 static void	aio_swake_cb(struct socket *, struct sockbuf *);
263 static int	aio_unload(void);
264 static void	process_signal(void *aioj);
265 static int	filt_aioattach(struct knote *kn);
266 static void	filt_aiodetach(struct knote *kn);
267 static int	filt_aio(struct knote *kn, long hint);
268 
269 /*
270  * Zones for:
271  * 	kaio	Per process async io info
272  *	aiop	async io thread data
273  *	aiocb	async io jobs
274  *	aiol	list io job pointer - internal to aio_suspend XXX
275  *	aiolio	list io jobs
276  */
277 static uma_zone_t kaio_zone, aiop_zone, aiocb_zone, aiol_zone, aiolio_zone;
278 
279 /* kqueue filters for aio */
280 static struct filterops aio_filtops =
281 	{ 0, filt_aioattach, filt_aiodetach, filt_aio };
282 
283 static eventhandler_tag exit_tag, exec_tag;
284 
285 /*
286  * Main operations function for use as a kernel module.
287  */
288 static int
289 aio_modload(struct module *module, int cmd, void *arg)
290 {
291 	int error = 0;
292 
293 	switch (cmd) {
294 	case MOD_LOAD:
295 		aio_onceonly();
296 		break;
297 	case MOD_UNLOAD:
298 		error = aio_unload();
299 		break;
300 	case MOD_SHUTDOWN:
301 		break;
302 	default:
303 		error = EINVAL;
304 		break;
305 	}
306 	return (error);
307 }
308 
309 static moduledata_t aio_mod = {
310 	"aio",
311 	&aio_modload,
312 	NULL
313 };
314 
315 SYSCALL_MODULE_HELPER(aio_return);
316 SYSCALL_MODULE_HELPER(aio_suspend);
317 SYSCALL_MODULE_HELPER(aio_cancel);
318 SYSCALL_MODULE_HELPER(aio_error);
319 SYSCALL_MODULE_HELPER(aio_read);
320 SYSCALL_MODULE_HELPER(aio_write);
321 SYSCALL_MODULE_HELPER(aio_waitcomplete);
322 SYSCALL_MODULE_HELPER(lio_listio);
323 
324 DECLARE_MODULE(aio, aio_mod,
325 	SI_SUB_VFS, SI_ORDER_ANY);
326 MODULE_VERSION(aio, 1);
327 
328 /*
329  * Startup initialization
330  */
331 static void
332 aio_onceonly(void)
333 {
334 
335 	/* XXX: should probably just use so->callback */
336 	aio_swake = &aio_swake_cb;
337 	exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL,
338 	    EVENTHANDLER_PRI_ANY);
339 	exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown, NULL,
340 	    EVENTHANDLER_PRI_ANY);
341 	kqueue_add_filteropts(EVFILT_AIO, &aio_filtops);
342 	TAILQ_INIT(&aio_freeproc);
343 	TAILQ_INIT(&aio_activeproc);
344 	TAILQ_INIT(&aio_jobs);
345 	TAILQ_INIT(&aio_bufjobs);
346 	kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL,
347 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
348 	aiop_zone = uma_zcreate("AIOP", sizeof(struct aiothreadlist), NULL,
349 	    NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
350 	aiocb_zone = uma_zcreate("AIOCB", sizeof(struct aiocblist), NULL, NULL,
351 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
352 	aiol_zone = uma_zcreate("AIOL", AIO_LISTIO_MAX*sizeof(intptr_t) , NULL,
353 	    NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
354 	aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aio_liojob), NULL,
355 	    NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
356 	aiod_timeout = AIOD_TIMEOUT_DEFAULT;
357 	aiod_lifetime = AIOD_LIFETIME_DEFAULT;
358 	jobrefid = 1;
359 	async_io_version = _POSIX_VERSION;
360 	p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, AIO_LISTIO_MAX);
361 	p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE);
362 	p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0);
363 }
364 
365 /*
366  * Callback for unload of AIO when used as a module.
367  */
368 static int
369 aio_unload(void)
370 {
371 	int error;
372 
373 	/*
374 	 * XXX: no unloads by default, it's too dangerous.
375 	 * perhaps we could do it if locked out callers and then
376 	 * did an aio_proc_rundown() on each process.
377 	 */
378 	if (!unloadable)
379 		return (EOPNOTSUPP);
380 
381 	error = kqueue_del_filteropts(EVFILT_AIO);
382 	if (error)
383 		return error;
384 
385 	async_io_version = 0;
386 	aio_swake = NULL;
387 	EVENTHANDLER_DEREGISTER(process_exit, exit_tag);
388 	EVENTHANDLER_DEREGISTER(process_exec, exec_tag);
389 	p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, -1);
390 	p31b_setcfg(CTL_P1003_1B_AIO_MAX, -1);
391 	p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, -1);
392 	return (0);
393 }
394 
395 /*
396  * Init the per-process aioinfo structure.  The aioinfo limits are set
397  * per-process for user limit (resource) management.
398  */
399 static void
400 aio_init_aioinfo(struct proc *p)
401 {
402 	struct kaioinfo *ki;
403 
404 	if (p->p_aioinfo == NULL) {
405 		ki = uma_zalloc(kaio_zone, M_WAITOK);
406 		p->p_aioinfo = ki;
407 		ki->kaio_flags = 0;
408 		ki->kaio_maxactive_count = max_aio_per_proc;
409 		ki->kaio_active_count = 0;
410 		ki->kaio_qallowed_count = max_aio_queue_per_proc;
411 		ki->kaio_queue_count = 0;
412 		ki->kaio_ballowed_count = max_buf_aio;
413 		ki->kaio_buffer_count = 0;
414 		ki->kaio_buffer_finished_count = 0;
415 		ki->kaio_p = p;
416 		TAILQ_INIT(&ki->kaio_jobdone);
417 		TAILQ_INIT(&ki->kaio_jobqueue);
418 		TAILQ_INIT(&ki->kaio_bufdone);
419 		TAILQ_INIT(&ki->kaio_bufqueue);
420 		TAILQ_INIT(&ki->kaio_liojoblist);
421 		TAILQ_INIT(&ki->kaio_sockqueue);
422 	}
423 
424 	while (num_aio_procs < target_aio_procs)
425 		aio_newproc();
426 }
427 
428 /*
429  * Free a job entry.  Wait for completion if it is currently active, but don't
430  * delay forever.  If we delay, we return a flag that says that we have to
431  * restart the queue scan.
432  */
433 static int
434 aio_free_entry(struct aiocblist *aiocbe)
435 {
436 	struct kaioinfo *ki;
437 	struct aio_liojob *lj;
438 	struct proc *p;
439 	int error;
440 	int s;
441 
442 	if (aiocbe->jobstate == JOBST_NULL)
443 		panic("aio_free_entry: freeing already free job");
444 
445 	p = aiocbe->userproc;
446 	ki = p->p_aioinfo;
447 	lj = aiocbe->lio;
448 	if (ki == NULL)
449 		panic("aio_free_entry: missing p->p_aioinfo");
450 
451 	while (aiocbe->jobstate == JOBST_JOBRUNNING) {
452 		aiocbe->jobflags |= AIOCBLIST_RUNDOWN;
453 		tsleep(aiocbe, PRIBIO, "jobwai", 0);
454 	}
455 	if (aiocbe->bp == NULL) {
456 		if (ki->kaio_queue_count <= 0)
457 			panic("aio_free_entry: process queue size <= 0");
458 		if (num_queue_count <= 0)
459 			panic("aio_free_entry: system wide queue size <= 0");
460 
461 		if (lj) {
462 			lj->lioj_queue_count--;
463 			if (aiocbe->jobflags & AIOCBLIST_DONE)
464 				lj->lioj_queue_finished_count--;
465 		}
466 		ki->kaio_queue_count--;
467 		if (aiocbe->jobflags & AIOCBLIST_DONE)
468 			ki->kaio_queue_finished_count--;
469 		num_queue_count--;
470 	} else {
471 		if (lj) {
472 			lj->lioj_buffer_count--;
473 			if (aiocbe->jobflags & AIOCBLIST_DONE)
474 				lj->lioj_buffer_finished_count--;
475 		}
476 		if (aiocbe->jobflags & AIOCBLIST_DONE)
477 			ki->kaio_buffer_finished_count--;
478 		ki->kaio_buffer_count--;
479 		num_buf_aio--;
480 	}
481 
482 	/* aiocbe is going away, we need to destroy any knotes */
483 	/* XXXKSE Note the thread here is used to eventually find the
484 	 * owning process again, but it is also used to do a fo_close
485 	 * and that requires the thread. (but does it require the
486 	 * OWNING thread? (or maybe the running thread?)
487 	 * There is a semantic problem here...
488 	 */
489 	knlist_clear(&aiocbe->klist, 0); /* XXXKSE */
490 
491 	if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags & KAIO_RUNDOWN)
492 	    && ((ki->kaio_buffer_count == 0) && (ki->kaio_queue_count == 0)))) {
493 		ki->kaio_flags &= ~KAIO_WAKEUP;
494 		wakeup(p);
495 	}
496 
497 	if (aiocbe->jobstate == JOBST_JOBQBUF) {
498 		if ((error = aio_fphysio(aiocbe)) != 0)
499 			return (error);
500 		if (aiocbe->jobstate != JOBST_JOBBFINISHED)
501 			panic("aio_free_entry: invalid physio finish-up state");
502 		s = splbio();
503 		TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
504 		splx(s);
505 	} else if (aiocbe->jobstate == JOBST_JOBQGLOBAL) {
506 		s = splnet();
507 		TAILQ_REMOVE(&aio_jobs, aiocbe, list);
508 		TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
509 		splx(s);
510 	} else if (aiocbe->jobstate == JOBST_JOBFINISHED)
511 		TAILQ_REMOVE(&ki->kaio_jobdone, aiocbe, plist);
512 	else if (aiocbe->jobstate == JOBST_JOBBFINISHED) {
513 		s = splbio();
514 		TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
515 		splx(s);
516 		if (aiocbe->bp) {
517 			vunmapbuf(aiocbe->bp);
518 			relpbuf(aiocbe->bp, NULL);
519 			aiocbe->bp = NULL;
520 		}
521 	}
522 	if (lj && (lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 0)) {
523 		TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
524 		uma_zfree(aiolio_zone, lj);
525 	}
526 	aiocbe->jobstate = JOBST_NULL;
527 	untimeout(process_signal, aiocbe, aiocbe->timeouthandle);
528 	fdrop(aiocbe->fd_file, curthread);
529 	crfree(aiocbe->cred);
530 	uma_zfree(aiocb_zone, aiocbe);
531 	return (0);
532 }
533 
534 /*
535  * Rundown the jobs for a given process.
536  */
537 static void
538 aio_proc_rundown(void *arg, struct proc *p)
539 {
540 	int s;
541 	struct kaioinfo *ki;
542 	struct aio_liojob *lj, *ljn;
543 	struct aiocblist *aiocbe, *aiocbn;
544 	struct file *fp;
545 	struct socket *so;
546 
547 	ki = p->p_aioinfo;
548 	if (ki == NULL)
549 		return;
550 
551 	mtx_lock(&Giant);
552 	ki->kaio_flags |= LIOJ_SIGNAL_POSTED;
553 	while ((ki->kaio_active_count > 0) || (ki->kaio_buffer_count >
554 	    ki->kaio_buffer_finished_count)) {
555 		ki->kaio_flags |= KAIO_RUNDOWN;
556 		if (tsleep(p, PRIBIO, "kaiowt", aiod_timeout))
557 			break;
558 	}
559 
560 	/*
561 	 * Move any aio ops that are waiting on socket I/O to the normal job
562 	 * queues so they are cleaned up with any others.
563 	 */
564 	s = splnet();
565 	for (aiocbe = TAILQ_FIRST(&ki->kaio_sockqueue); aiocbe; aiocbe =
566 	    aiocbn) {
567 		aiocbn = TAILQ_NEXT(aiocbe, plist);
568 		fp = aiocbe->fd_file;
569 		if (fp != NULL) {
570 			so = fp->f_data;
571 			TAILQ_REMOVE(&so->so_aiojobq, aiocbe, list);
572 			if (TAILQ_EMPTY(&so->so_aiojobq)) {
573 				SOCKBUF_LOCK(&so->so_snd);
574 				so->so_snd.sb_flags &= ~SB_AIO;
575 				SOCKBUF_UNLOCK(&so->so_snd);
576 				SOCKBUF_LOCK(&so->so_rcv);
577 				so->so_rcv.sb_flags &= ~SB_AIO;
578 				SOCKBUF_UNLOCK(&so->so_rcv);
579 			}
580 		}
581 		TAILQ_REMOVE(&ki->kaio_sockqueue, aiocbe, plist);
582 		TAILQ_INSERT_HEAD(&aio_jobs, aiocbe, list);
583 		TAILQ_INSERT_HEAD(&ki->kaio_jobqueue, aiocbe, plist);
584 	}
585 	splx(s);
586 
587 restart1:
588 	for (aiocbe = TAILQ_FIRST(&ki->kaio_jobdone); aiocbe; aiocbe = aiocbn) {
589 		aiocbn = TAILQ_NEXT(aiocbe, plist);
590 		if (aio_free_entry(aiocbe))
591 			goto restart1;
592 	}
593 
594 restart2:
595 	for (aiocbe = TAILQ_FIRST(&ki->kaio_jobqueue); aiocbe; aiocbe =
596 	    aiocbn) {
597 		aiocbn = TAILQ_NEXT(aiocbe, plist);
598 		if (aio_free_entry(aiocbe))
599 			goto restart2;
600 	}
601 
602 /*
603  * Note the use of lots of splbio here, trying to avoid splbio for long chains
604  * of I/O.  Probably unnecessary.
605  */
606 restart3:
607 	s = splbio();
608 	while (TAILQ_FIRST(&ki->kaio_bufqueue)) {
609 		ki->kaio_flags |= KAIO_WAKEUP;
610 		tsleep(p, PRIBIO, "aioprn", 0);
611 		splx(s);
612 		goto restart3;
613 	}
614 	splx(s);
615 
616 restart4:
617 	s = splbio();
618 	for (aiocbe = TAILQ_FIRST(&ki->kaio_bufdone); aiocbe; aiocbe = aiocbn) {
619 		aiocbn = TAILQ_NEXT(aiocbe, plist);
620 		if (aio_free_entry(aiocbe)) {
621 			splx(s);
622 			goto restart4;
623 		}
624 	}
625 	splx(s);
626 
627 	/*
628 	 * If we've slept, jobs might have moved from one queue to another.
629 	 * Retry rundown if we didn't manage to empty the queues.
630 	 */
631 	if (TAILQ_FIRST(&ki->kaio_jobdone) != NULL ||
632 	    TAILQ_FIRST(&ki->kaio_jobqueue) != NULL ||
633 	    TAILQ_FIRST(&ki->kaio_bufqueue) != NULL ||
634 	    TAILQ_FIRST(&ki->kaio_bufdone) != NULL)
635 		goto restart1;
636 
637 	for (lj = TAILQ_FIRST(&ki->kaio_liojoblist); lj; lj = ljn) {
638 		ljn = TAILQ_NEXT(lj, lioj_list);
639 		if ((lj->lioj_buffer_count == 0) && (lj->lioj_queue_count ==
640 		    0)) {
641 			TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
642 			uma_zfree(aiolio_zone, lj);
643 		} else {
644 #ifdef DIAGNOSTIC
645 			printf("LIO job not cleaned up: B:%d, BF:%d, Q:%d, "
646 			    "QF:%d\n", lj->lioj_buffer_count,
647 			    lj->lioj_buffer_finished_count,
648 			    lj->lioj_queue_count,
649 			    lj->lioj_queue_finished_count);
650 #endif
651 		}
652 	}
653 
654 	uma_zfree(kaio_zone, ki);
655 	p->p_aioinfo = NULL;
656 	mtx_unlock(&Giant);
657 }
658 
659 /*
660  * Select a job to run (called by an AIO daemon).
661  */
662 static struct aiocblist *
663 aio_selectjob(struct aiothreadlist *aiop)
664 {
665 	int s;
666 	struct aiocblist *aiocbe;
667 	struct kaioinfo *ki;
668 	struct proc *userp;
669 
670 	s = splnet();
671 	for (aiocbe = TAILQ_FIRST(&aio_jobs); aiocbe; aiocbe =
672 	    TAILQ_NEXT(aiocbe, list)) {
673 		userp = aiocbe->userproc;
674 		ki = userp->p_aioinfo;
675 
676 		if (ki->kaio_active_count < ki->kaio_maxactive_count) {
677 			TAILQ_REMOVE(&aio_jobs, aiocbe, list);
678 			splx(s);
679 			return (aiocbe);
680 		}
681 	}
682 	splx(s);
683 
684 	return (NULL);
685 }
686 
687 /*
688  * The AIO processing activity.  This is the code that does the I/O request for
689  * the non-physio version of the operations.  The normal vn operations are used,
690  * and this code should work in all instances for every type of file, including
691  * pipes, sockets, fifos, and regular files.
692  */
693 static void
694 aio_process(struct aiocblist *aiocbe)
695 {
696 	struct ucred *td_savedcred;
697 	struct thread *td;
698 	struct proc *mycp;
699 	struct aiocb *cb;
700 	struct file *fp;
701 	struct uio auio;
702 	struct iovec aiov;
703 	int cnt;
704 	int error;
705 	int oublock_st, oublock_end;
706 	int inblock_st, inblock_end;
707 
708 	td = curthread;
709 	td_savedcred = td->td_ucred;
710 	td->td_ucred = aiocbe->cred;
711 	mycp = td->td_proc;
712 	cb = &aiocbe->uaiocb;
713 	fp = aiocbe->fd_file;
714 
715 	aiov.iov_base = (void *)(uintptr_t)cb->aio_buf;
716 	aiov.iov_len = cb->aio_nbytes;
717 
718 	auio.uio_iov = &aiov;
719 	auio.uio_iovcnt = 1;
720 	auio.uio_offset = cb->aio_offset;
721 	auio.uio_resid = cb->aio_nbytes;
722 	cnt = cb->aio_nbytes;
723 	auio.uio_segflg = UIO_USERSPACE;
724 	auio.uio_td = td;
725 
726 	inblock_st = mycp->p_stats->p_ru.ru_inblock;
727 	oublock_st = mycp->p_stats->p_ru.ru_oublock;
728 	/*
729 	 * _aio_aqueue() acquires a reference to the file that is
730 	 * released in aio_free_entry().
731 	 */
732 	if (cb->aio_lio_opcode == LIO_READ) {
733 		auio.uio_rw = UIO_READ;
734 		error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td);
735 	} else {
736 		auio.uio_rw = UIO_WRITE;
737 		error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td);
738 	}
739 	inblock_end = mycp->p_stats->p_ru.ru_inblock;
740 	oublock_end = mycp->p_stats->p_ru.ru_oublock;
741 
742 	aiocbe->inputcharge = inblock_end - inblock_st;
743 	aiocbe->outputcharge = oublock_end - oublock_st;
744 
745 	if ((error) && (auio.uio_resid != cnt)) {
746 		if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
747 			error = 0;
748 		if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) {
749 			PROC_LOCK(aiocbe->userproc);
750 			psignal(aiocbe->userproc, SIGPIPE);
751 			PROC_UNLOCK(aiocbe->userproc);
752 		}
753 	}
754 
755 	cnt -= auio.uio_resid;
756 	cb->_aiocb_private.error = error;
757 	cb->_aiocb_private.status = cnt;
758 	td->td_ucred = td_savedcred;
759 }
760 
761 /*
762  * The AIO daemon, most of the actual work is done in aio_process,
763  * but the setup (and address space mgmt) is done in this routine.
764  */
765 static void
766 aio_daemon(void *uproc)
767 {
768 	int s;
769 	struct aio_liojob *lj;
770 	struct aiocb *cb;
771 	struct aiocblist *aiocbe;
772 	struct aiothreadlist *aiop;
773 	struct kaioinfo *ki;
774 	struct proc *curcp, *mycp, *userp;
775 	struct vmspace *myvm, *tmpvm;
776 	struct thread *td = curthread;
777 	struct pgrp *newpgrp;
778 	struct session *newsess;
779 
780 	mtx_lock(&Giant);
781 	/*
782 	 * Local copies of curproc (cp) and vmspace (myvm)
783 	 */
784 	mycp = td->td_proc;
785 	myvm = mycp->p_vmspace;
786 
787 	KASSERT(mycp->p_textvp == NULL, ("kthread has a textvp"));
788 
789 	/*
790 	 * Allocate and ready the aio control info.  There is one aiop structure
791 	 * per daemon.
792 	 */
793 	aiop = uma_zalloc(aiop_zone, M_WAITOK);
794 	aiop->aiothread = td;
795 	aiop->aiothreadflags |= AIOP_FREE;
796 
797 	s = splnet();
798 
799 	/*
800 	 * Place thread (lightweight process) onto the AIO free thread list.
801 	 */
802 	if (TAILQ_EMPTY(&aio_freeproc))
803 		wakeup(&aio_freeproc);
804 	TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
805 
806 	splx(s);
807 
808 	/*
809 	 * Get rid of our current filedescriptors.  AIOD's don't need any
810 	 * filedescriptors, except as temporarily inherited from the client.
811 	 */
812 	fdfree(td);
813 
814 	mtx_unlock(&Giant);
815 	/* The daemon resides in its own pgrp. */
816 	MALLOC(newpgrp, struct pgrp *, sizeof(struct pgrp), M_PGRP,
817 		M_WAITOK | M_ZERO);
818 	MALLOC(newsess, struct session *, sizeof(struct session), M_SESSION,
819 		M_WAITOK | M_ZERO);
820 
821 	sx_xlock(&proctree_lock);
822 	enterpgrp(mycp, mycp->p_pid, newpgrp, newsess);
823 	sx_xunlock(&proctree_lock);
824 	mtx_lock(&Giant);
825 
826 	/*
827 	 * Wakeup parent process.  (Parent sleeps to keep from blasting away
828 	 * and creating too many daemons.)
829 	 */
830 	wakeup(mycp);
831 
832 	for (;;) {
833 		/*
834 		 * curcp is the current daemon process context.
835 		 * userp is the current user process context.
836 		 */
837 		curcp = mycp;
838 
839 		/*
840 		 * Take daemon off of free queue
841 		 */
842 		if (aiop->aiothreadflags & AIOP_FREE) {
843 			s = splnet();
844 			TAILQ_REMOVE(&aio_freeproc, aiop, list);
845 			TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list);
846 			aiop->aiothreadflags &= ~AIOP_FREE;
847 			splx(s);
848 		}
849 		aiop->aiothreadflags &= ~AIOP_SCHED;
850 
851 		/*
852 		 * Check for jobs.
853 		 */
854 		while ((aiocbe = aio_selectjob(aiop)) != NULL) {
855 			cb = &aiocbe->uaiocb;
856 			userp = aiocbe->userproc;
857 
858 			aiocbe->jobstate = JOBST_JOBRUNNING;
859 
860 			/*
861 			 * Connect to process address space for user program.
862 			 */
863 			if (userp != curcp) {
864 				/*
865 				 * Save the current address space that we are
866 				 * connected to.
867 				 */
868 				tmpvm = mycp->p_vmspace;
869 
870 				/*
871 				 * Point to the new user address space, and
872 				 * refer to it.
873 				 */
874 				mycp->p_vmspace = userp->p_vmspace;
875 				atomic_add_int(&mycp->p_vmspace->vm_refcnt, 1);
876 
877 				/* Activate the new mapping. */
878 				pmap_activate(FIRST_THREAD_IN_PROC(mycp));
879 
880 				/*
881 				 * If the old address space wasn't the daemons
882 				 * own address space, then we need to remove the
883 				 * daemon's reference from the other process
884 				 * that it was acting on behalf of.
885 				 */
886 				if (tmpvm != myvm) {
887 					vmspace_free(tmpvm);
888 				}
889 				curcp = userp;
890 			}
891 
892 			ki = userp->p_aioinfo;
893 			lj = aiocbe->lio;
894 
895 			/* Account for currently active jobs. */
896 			ki->kaio_active_count++;
897 
898 			/* Do the I/O function. */
899 			aio_process(aiocbe);
900 
901 			/* Decrement the active job count. */
902 			ki->kaio_active_count--;
903 
904 			/*
905 			 * Increment the completion count for wakeup/signal
906 			 * comparisons.
907 			 */
908 			aiocbe->jobflags |= AIOCBLIST_DONE;
909 			ki->kaio_queue_finished_count++;
910 			if (lj)
911 				lj->lioj_queue_finished_count++;
912 			if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags
913 			    & KAIO_RUNDOWN) && (ki->kaio_active_count == 0))) {
914 				ki->kaio_flags &= ~KAIO_WAKEUP;
915 				wakeup(userp);
916 			}
917 
918 			s = splbio();
919 			if (lj && (lj->lioj_flags &
920 			    (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == LIOJ_SIGNAL) {
921 				if ((lj->lioj_queue_finished_count ==
922 				    lj->lioj_queue_count) &&
923 				    (lj->lioj_buffer_finished_count ==
924 				    lj->lioj_buffer_count)) {
925 					PROC_LOCK(userp);
926 					psignal(userp,
927 					    lj->lioj_signal.sigev_signo);
928 					PROC_UNLOCK(userp);
929 					lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
930 				}
931 			}
932 			splx(s);
933 
934 			aiocbe->jobstate = JOBST_JOBFINISHED;
935 
936 			s = splnet();
937 			TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
938 			TAILQ_INSERT_TAIL(&ki->kaio_jobdone, aiocbe, plist);
939 			splx(s);
940 			KNOTE_UNLOCKED(&aiocbe->klist, 0);
941 
942 			if (aiocbe->jobflags & AIOCBLIST_RUNDOWN) {
943 				wakeup(aiocbe);
944 				aiocbe->jobflags &= ~AIOCBLIST_RUNDOWN;
945 			}
946 
947 			if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
948 				PROC_LOCK(userp);
949 				psignal(userp, cb->aio_sigevent.sigev_signo);
950 				PROC_UNLOCK(userp);
951 			}
952 		}
953 
954 		/*
955 		 * Disconnect from user address space.
956 		 */
957 		if (curcp != mycp) {
958 			/* Get the user address space to disconnect from. */
959 			tmpvm = mycp->p_vmspace;
960 
961 			/* Get original address space for daemon. */
962 			mycp->p_vmspace = myvm;
963 
964 			/* Activate the daemon's address space. */
965 			pmap_activate(FIRST_THREAD_IN_PROC(mycp));
966 #ifdef DIAGNOSTIC
967 			if (tmpvm == myvm) {
968 				printf("AIOD: vmspace problem -- %d\n",
969 				    mycp->p_pid);
970 			}
971 #endif
972 			/* Remove our vmspace reference. */
973 			vmspace_free(tmpvm);
974 
975 			curcp = mycp;
976 		}
977 
978 		/*
979 		 * If we are the first to be put onto the free queue, wakeup
980 		 * anyone waiting for a daemon.
981 		 */
982 		s = splnet();
983 		TAILQ_REMOVE(&aio_activeproc, aiop, list);
984 		if (TAILQ_EMPTY(&aio_freeproc))
985 			wakeup(&aio_freeproc);
986 		TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
987 		aiop->aiothreadflags |= AIOP_FREE;
988 		splx(s);
989 
990 		/*
991 		 * If daemon is inactive for a long time, allow it to exit,
992 		 * thereby freeing resources.
993 		 */
994 		if ((aiop->aiothreadflags & AIOP_SCHED) == 0 &&
995 		    tsleep(aiop->aiothread, PRIBIO, "aiordy", aiod_lifetime)) {
996 			s = splnet();
997 			if (TAILQ_EMPTY(&aio_jobs)) {
998 				if ((aiop->aiothreadflags & AIOP_FREE) &&
999 				    (num_aio_procs > target_aio_procs)) {
1000 					TAILQ_REMOVE(&aio_freeproc, aiop, list);
1001 					splx(s);
1002 					uma_zfree(aiop_zone, aiop);
1003 					num_aio_procs--;
1004 #ifdef DIAGNOSTIC
1005 					if (mycp->p_vmspace->vm_refcnt <= 1) {
1006 						printf("AIOD: bad vm refcnt for"
1007 						    " exiting daemon: %d\n",
1008 						    mycp->p_vmspace->vm_refcnt);
1009 					}
1010 #endif
1011 					kthread_exit(0);
1012 				}
1013 			}
1014 			splx(s);
1015 		}
1016 	}
1017 }
1018 
1019 /*
1020  * Create a new AIO daemon.  This is mostly a kernel-thread fork routine.  The
1021  * AIO daemon modifies its environment itself.
1022  */
1023 static int
1024 aio_newproc(void)
1025 {
1026 	int error;
1027 	struct proc *p;
1028 
1029 	error = kthread_create(aio_daemon, curproc, &p, RFNOWAIT, 0, "aiod%d",
1030 	    num_aio_procs);
1031 	if (error)
1032 		return (error);
1033 
1034 	/*
1035 	 * Wait until daemon is started, but continue on just in case to
1036 	 * handle error conditions.
1037 	 */
1038 	error = tsleep(p, PZERO, "aiosta", aiod_timeout);
1039 
1040 	num_aio_procs++;
1041 
1042 	return (error);
1043 }
1044 
1045 /*
1046  * Try the high-performance, low-overhead physio method for eligible
1047  * VCHR devices.  This method doesn't use an aio helper thread, and
1048  * thus has very low overhead.
1049  *
1050  * Assumes that the caller, _aio_aqueue(), has incremented the file
1051  * structure's reference count, preventing its deallocation for the
1052  * duration of this call.
1053  */
1054 static int
1055 aio_qphysio(struct proc *p, struct aiocblist *aiocbe)
1056 {
1057 	int error;
1058 	struct aiocb *cb;
1059 	struct file *fp;
1060 	struct buf *bp;
1061 	struct vnode *vp;
1062 	struct kaioinfo *ki;
1063 	struct aio_liojob *lj;
1064 	int s;
1065 	int notify;
1066 
1067 	cb = &aiocbe->uaiocb;
1068 	fp = aiocbe->fd_file;
1069 
1070 	if (fp->f_type != DTYPE_VNODE)
1071 		return (-1);
1072 
1073 	vp = fp->f_vnode;
1074 
1075 	/*
1076 	 * If its not a disk, we don't want to return a positive error.
1077 	 * It causes the aio code to not fall through to try the thread
1078 	 * way when you're talking to a regular file.
1079 	 */
1080 	if (!vn_isdisk(vp, &error)) {
1081 		if (error == ENOTBLK)
1082 			return (-1);
1083 		else
1084 			return (error);
1085 	}
1086 
1087  	if (cb->aio_nbytes % vp->v_rdev->si_bsize_phys)
1088 		return (-1);
1089 
1090 	if (cb->aio_nbytes >
1091 	    MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK))
1092 		return (-1);
1093 
1094 	ki = p->p_aioinfo;
1095 	if (ki->kaio_buffer_count >= ki->kaio_ballowed_count)
1096 		return (-1);
1097 
1098 	ki->kaio_buffer_count++;
1099 
1100 	lj = aiocbe->lio;
1101 	if (lj)
1102 		lj->lioj_buffer_count++;
1103 
1104 	/* Create and build a buffer header for a transfer. */
1105 	bp = (struct buf *)getpbuf(NULL);
1106 	BUF_KERNPROC(bp);
1107 
1108 	/*
1109 	 * Get a copy of the kva from the physical buffer.
1110 	 */
1111 	bp->b_dev = vp->v_rdev;
1112 	error = 0;
1113 
1114 	bp->b_bcount = cb->aio_nbytes;
1115 	bp->b_bufsize = cb->aio_nbytes;
1116 	bp->b_iodone = aio_physwakeup;
1117 	bp->b_saveaddr = bp->b_data;
1118 	bp->b_data = (void *)(uintptr_t)cb->aio_buf;
1119 	bp->b_offset = cb->aio_offset;
1120 	bp->b_iooffset = cb->aio_offset;
1121 	bp->b_blkno = btodb(cb->aio_offset);
1122 	bp->b_iocmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ;
1123 
1124 	/*
1125 	 * Bring buffer into kernel space.
1126 	 */
1127 	if (vmapbuf(bp) < 0) {
1128 		error = EFAULT;
1129 		goto doerror;
1130 	}
1131 
1132 	s = splbio();
1133 	aiocbe->bp = bp;
1134 	bp->b_caller1 = (void *)aiocbe;
1135 	TAILQ_INSERT_TAIL(&aio_bufjobs, aiocbe, list);
1136 	TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist);
1137 	aiocbe->jobstate = JOBST_JOBQBUF;
1138 	cb->_aiocb_private.status = cb->aio_nbytes;
1139 	num_buf_aio++;
1140 	bp->b_error = 0;
1141 
1142 	splx(s);
1143 
1144 	/* Perform transfer. */
1145 	DEV_STRATEGY(bp);
1146 
1147 	notify = 0;
1148 	s = splbio();
1149 
1150 	/*
1151 	 * If we had an error invoking the request, or an error in processing
1152 	 * the request before we have returned, we process it as an error in
1153 	 * transfer.  Note that such an I/O error is not indicated immediately,
1154 	 * but is returned using the aio_error mechanism.  In this case,
1155 	 * aio_suspend will return immediately.
1156 	 */
1157 	if (bp->b_error || (bp->b_ioflags & BIO_ERROR)) {
1158 		struct aiocb *job = aiocbe->uuaiocb;
1159 
1160 		aiocbe->uaiocb._aiocb_private.status = 0;
1161 		suword(&job->_aiocb_private.status, 0);
1162 		aiocbe->uaiocb._aiocb_private.error = bp->b_error;
1163 		suword(&job->_aiocb_private.error, bp->b_error);
1164 
1165 		ki->kaio_buffer_finished_count++;
1166 
1167 		if (aiocbe->jobstate != JOBST_JOBBFINISHED) {
1168 			aiocbe->jobstate = JOBST_JOBBFINISHED;
1169 			aiocbe->jobflags |= AIOCBLIST_DONE;
1170 			TAILQ_REMOVE(&aio_bufjobs, aiocbe, list);
1171 			TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
1172 			TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
1173 			notify = 1;
1174 		}
1175 	}
1176 	splx(s);
1177 	if (notify)
1178 		KNOTE_UNLOCKED(&aiocbe->klist, 0);
1179 	return (0);
1180 
1181 doerror:
1182 	ki->kaio_buffer_count--;
1183 	if (lj)
1184 		lj->lioj_buffer_count--;
1185 	aiocbe->bp = NULL;
1186 	relpbuf(bp, NULL);
1187 	return (error);
1188 }
1189 
1190 /*
1191  * This waits/tests physio completion.
1192  */
1193 static int
1194 aio_fphysio(struct aiocblist *iocb)
1195 {
1196 	int s;
1197 	struct buf *bp;
1198 	int error;
1199 
1200 	bp = iocb->bp;
1201 
1202 	s = splbio();
1203 	while ((bp->b_flags & B_DONE) == 0) {
1204 		if (tsleep(bp, PRIBIO, "physstr", aiod_timeout)) {
1205 			if ((bp->b_flags & B_DONE) == 0) {
1206 				splx(s);
1207 				return (EINPROGRESS);
1208 			} else
1209 				break;
1210 		}
1211 	}
1212 	splx(s);
1213 
1214 	/* Release mapping into kernel space. */
1215 	vunmapbuf(bp);
1216 	iocb->bp = 0;
1217 
1218 	error = 0;
1219 
1220 	/* Check for an error. */
1221 	if (bp->b_ioflags & BIO_ERROR)
1222 		error = bp->b_error;
1223 
1224 	relpbuf(bp, NULL);
1225 	return (error);
1226 }
1227 
1228 /*
1229  * Wake up aio requests that may be serviceable now.
1230  */
1231 static void
1232 aio_swake_cb(struct socket *so, struct sockbuf *sb)
1233 {
1234 	struct aiocblist *cb,*cbn;
1235 	struct proc *p;
1236 	struct kaioinfo *ki = NULL;
1237 	int opcode, wakecount = 0;
1238 	struct aiothreadlist *aiop;
1239 
1240 	if (sb == &so->so_snd) {
1241 		opcode = LIO_WRITE;
1242 		SOCKBUF_LOCK(&so->so_snd);
1243 		so->so_snd.sb_flags &= ~SB_AIO;
1244 		SOCKBUF_UNLOCK(&so->so_snd);
1245 	} else {
1246 		opcode = LIO_READ;
1247 		SOCKBUF_LOCK(&so->so_rcv);
1248 		so->so_rcv.sb_flags &= ~SB_AIO;
1249 		SOCKBUF_UNLOCK(&so->so_rcv);
1250 	}
1251 
1252 	for (cb = TAILQ_FIRST(&so->so_aiojobq); cb; cb = cbn) {
1253 		cbn = TAILQ_NEXT(cb, list);
1254 		if (opcode == cb->uaiocb.aio_lio_opcode) {
1255 			p = cb->userproc;
1256 			ki = p->p_aioinfo;
1257 			TAILQ_REMOVE(&so->so_aiojobq, cb, list);
1258 			TAILQ_REMOVE(&ki->kaio_sockqueue, cb, plist);
1259 			TAILQ_INSERT_TAIL(&aio_jobs, cb, list);
1260 			TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, cb, plist);
1261 			wakecount++;
1262 			if (cb->jobstate != JOBST_JOBQGLOBAL)
1263 				panic("invalid queue value");
1264 		}
1265 	}
1266 
1267 	while (wakecount--) {
1268 		if ((aiop = TAILQ_FIRST(&aio_freeproc)) != 0) {
1269 			TAILQ_REMOVE(&aio_freeproc, aiop, list);
1270 			TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list);
1271 			aiop->aiothreadflags &= ~AIOP_FREE;
1272 			wakeup(aiop->aiothread);
1273 		}
1274 	}
1275 }
1276 
1277 /*
1278  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
1279  * technique is done in this code.
1280  */
1281 static int
1282 _aio_aqueue(struct thread *td, struct aiocb *job, struct aio_liojob *lj, int type)
1283 {
1284 	struct proc *p = td->td_proc;
1285 	struct filedesc *fdp;
1286 	struct file *fp;
1287 	unsigned int fd;
1288 	struct socket *so;
1289 	int s;
1290 	int error;
1291 	int opcode, user_opcode;
1292 	struct aiocblist *aiocbe;
1293 	struct aiothreadlist *aiop;
1294 	struct kaioinfo *ki;
1295 	struct kevent kev;
1296 	struct kqueue *kq;
1297 	struct file *kq_fp;
1298 
1299 	aiocbe = uma_zalloc(aiocb_zone, M_WAITOK);
1300 	aiocbe->inputcharge = 0;
1301 	aiocbe->outputcharge = 0;
1302 	callout_handle_init(&aiocbe->timeouthandle);
1303 	/* XXX - need a lock */
1304 	knlist_init(&aiocbe->klist, NULL);
1305 
1306 	suword(&job->_aiocb_private.status, -1);
1307 	suword(&job->_aiocb_private.error, 0);
1308 	suword(&job->_aiocb_private.kernelinfo, -1);
1309 
1310 	error = copyin(job, &aiocbe->uaiocb, sizeof(aiocbe->uaiocb));
1311 	if (error) {
1312 		suword(&job->_aiocb_private.error, error);
1313 		uma_zfree(aiocb_zone, aiocbe);
1314 		return (error);
1315 	}
1316 	if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL &&
1317 		!_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) {
1318 		uma_zfree(aiocb_zone, aiocbe);
1319 		return (EINVAL);
1320 	}
1321 
1322 	/* Save userspace address of the job info. */
1323 	aiocbe->uuaiocb = job;
1324 
1325 	/* Get the opcode. */
1326 	user_opcode = aiocbe->uaiocb.aio_lio_opcode;
1327 	if (type != LIO_NOP)
1328 		aiocbe->uaiocb.aio_lio_opcode = type;
1329 	opcode = aiocbe->uaiocb.aio_lio_opcode;
1330 
1331 	/* Get the fd info for process. */
1332 	fdp = p->p_fd;
1333 
1334 	/*
1335 	 * Range check file descriptor.
1336 	 */
1337 	FILEDESC_LOCK(fdp);
1338 	fd = aiocbe->uaiocb.aio_fildes;
1339 	if (fd >= fdp->fd_nfiles) {
1340 		FILEDESC_UNLOCK(fdp);
1341 		uma_zfree(aiocb_zone, aiocbe);
1342 		if (type == 0)
1343 			suword(&job->_aiocb_private.error, EBADF);
1344 		return (EBADF);
1345 	}
1346 
1347 	fp = aiocbe->fd_file = fdp->fd_ofiles[fd];
1348 	if ((fp == NULL) ||
1349 	    ((opcode == LIO_WRITE) && ((fp->f_flag & FWRITE) == 0)) ||
1350 	    ((opcode == LIO_READ) && ((fp->f_flag & FREAD) == 0))) {
1351 		FILEDESC_UNLOCK(fdp);
1352 		uma_zfree(aiocb_zone, aiocbe);
1353 		if (type == 0)
1354 			suword(&job->_aiocb_private.error, EBADF);
1355 		return (EBADF);
1356 	}
1357 	fhold(fp);
1358 	FILEDESC_UNLOCK(fdp);
1359 
1360 	if (aiocbe->uaiocb.aio_offset == -1LL) {
1361 		error = EINVAL;
1362 		goto aqueue_fail;
1363 	}
1364 	error = suword(&job->_aiocb_private.kernelinfo, jobrefid);
1365 	if (error) {
1366 		error = EINVAL;
1367 		goto aqueue_fail;
1368 	}
1369 	aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jobrefid;
1370 	if (jobrefid == LONG_MAX)
1371 		jobrefid = 1;
1372 	else
1373 		jobrefid++;
1374 
1375 	if (opcode == LIO_NOP) {
1376 		fdrop(fp, td);
1377 		uma_zfree(aiocb_zone, aiocbe);
1378 		if (type == 0) {
1379 			suword(&job->_aiocb_private.error, 0);
1380 			suword(&job->_aiocb_private.status, 0);
1381 			suword(&job->_aiocb_private.kernelinfo, 0);
1382 		}
1383 		return (0);
1384 	}
1385 	if ((opcode != LIO_READ) && (opcode != LIO_WRITE)) {
1386 		if (type == 0)
1387 			suword(&job->_aiocb_private.status, 0);
1388 		error = EINVAL;
1389 		goto aqueue_fail;
1390 	}
1391 
1392 	if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_KEVENT) {
1393 		kev.ident = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue;
1394 		kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sigval_ptr;
1395 	}
1396 	else {
1397 		/*
1398 		 * This method for requesting kevent-based notification won't
1399 		 * work on the alpha, since we're passing in a pointer
1400 		 * via aio_lio_opcode, which is an int.  Use the SIGEV_KEVENT-
1401 		 * based method instead.
1402 		 */
1403 		if (user_opcode == LIO_NOP || user_opcode == LIO_READ ||
1404 		    user_opcode == LIO_WRITE)
1405 			goto no_kqueue;
1406 
1407 		error = copyin((struct kevent *)(uintptr_t)user_opcode,
1408 		    &kev, sizeof(kev));
1409 		if (error)
1410 			goto aqueue_fail;
1411 	}
1412 	if ((u_int)kev.ident >= fdp->fd_nfiles ||
1413 	    (kq_fp = fdp->fd_ofiles[kev.ident]) == NULL ||
1414 	    (kq_fp->f_type != DTYPE_KQUEUE)) {
1415 		error = EBADF;
1416 		goto aqueue_fail;
1417 	}
1418 	kq = kq_fp->f_data;
1419 	kev.ident = (uintptr_t)aiocbe->uuaiocb;
1420 	kev.filter = EVFILT_AIO;
1421 	kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
1422 	kev.data = (intptr_t)aiocbe;
1423 	error = kqueue_register(kq, &kev, td, 1);
1424 aqueue_fail:
1425 	if (error) {
1426 		fdrop(fp, td);
1427 		uma_zfree(aiocb_zone, aiocbe);
1428 		if (type == 0)
1429 			suword(&job->_aiocb_private.error, error);
1430 		goto done;
1431 	}
1432 no_kqueue:
1433 
1434 	suword(&job->_aiocb_private.error, EINPROGRESS);
1435 	aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
1436 	aiocbe->userproc = p;
1437 	aiocbe->cred = crhold(td->td_ucred);
1438 	aiocbe->jobflags = 0;
1439 	aiocbe->lio = lj;
1440 	ki = p->p_aioinfo;
1441 
1442 	if (fp->f_type == DTYPE_SOCKET) {
1443 		/*
1444 		 * Alternate queueing for socket ops: Reach down into the
1445 		 * descriptor to get the socket data.  Then check to see if the
1446 		 * socket is ready to be read or written (based on the requested
1447 		 * operation).
1448 		 *
1449 		 * If it is not ready for io, then queue the aiocbe on the
1450 		 * socket, and set the flags so we get a call when sbnotify()
1451 		 * happens.
1452 		 */
1453 		so = fp->f_data;
1454 		s = splnet();
1455 		if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode ==
1456 		    LIO_WRITE) && (!sowriteable(so)))) {
1457 			TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list);
1458 			TAILQ_INSERT_TAIL(&ki->kaio_sockqueue, aiocbe, plist);
1459 			if (opcode == LIO_READ) {
1460 				SOCKBUF_LOCK(&so->so_rcv);
1461 				so->so_rcv.sb_flags |= SB_AIO;
1462 				SOCKBUF_UNLOCK(&so->so_rcv);
1463 			} else {
1464 				SOCKBUF_LOCK(&so->so_snd);
1465 				so->so_snd.sb_flags |= SB_AIO;
1466 				SOCKBUF_UNLOCK(&so->so_snd);
1467 			}
1468 			aiocbe->jobstate = JOBST_JOBQGLOBAL; /* XXX */
1469 			ki->kaio_queue_count++;
1470 			num_queue_count++;
1471 			splx(s);
1472 			error = 0;
1473 			goto done;
1474 		}
1475 		splx(s);
1476 	}
1477 
1478 	if ((error = aio_qphysio(p, aiocbe)) == 0)
1479 		goto done;
1480 	if (error > 0) {
1481 		suword(&job->_aiocb_private.status, 0);
1482 		aiocbe->uaiocb._aiocb_private.error = error;
1483 		suword(&job->_aiocb_private.error, error);
1484 		goto done;
1485 	}
1486 
1487 	/* No buffer for daemon I/O. */
1488 	aiocbe->bp = NULL;
1489 
1490 	ki->kaio_queue_count++;
1491 	if (lj)
1492 		lj->lioj_queue_count++;
1493 	s = splnet();
1494 	TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
1495 	TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list);
1496 	splx(s);
1497 	aiocbe->jobstate = JOBST_JOBQGLOBAL;
1498 
1499 	num_queue_count++;
1500 	error = 0;
1501 
1502 	/*
1503 	 * If we don't have a free AIO process, and we are below our quota, then
1504 	 * start one.  Otherwise, depend on the subsequent I/O completions to
1505 	 * pick-up this job.  If we don't sucessfully create the new process
1506 	 * (thread) due to resource issues, we return an error for now (EAGAIN),
1507 	 * which is likely not the correct thing to do.
1508 	 */
1509 	s = splnet();
1510 retryproc:
1511 	if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
1512 		TAILQ_REMOVE(&aio_freeproc, aiop, list);
1513 		TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list);
1514 		aiop->aiothreadflags &= ~AIOP_FREE;
1515 		wakeup(aiop->aiothread);
1516 	} else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
1517 	    ((ki->kaio_active_count + num_aio_resv_start) <
1518 	    ki->kaio_maxactive_count)) {
1519 		num_aio_resv_start++;
1520 		if ((error = aio_newproc()) == 0) {
1521 			num_aio_resv_start--;
1522 			goto retryproc;
1523 		}
1524 		num_aio_resv_start--;
1525 	}
1526 	splx(s);
1527 done:
1528 	return (error);
1529 }
1530 
1531 /*
1532  * This routine queues an AIO request, checking for quotas.
1533  */
1534 static int
1535 aio_aqueue(struct thread *td, struct aiocb *job, int type)
1536 {
1537 	struct proc *p = td->td_proc;
1538 	struct kaioinfo *ki;
1539 
1540 	if (p->p_aioinfo == NULL)
1541 		aio_init_aioinfo(p);
1542 
1543 	if (num_queue_count >= max_queue_count)
1544 		return (EAGAIN);
1545 
1546 	ki = p->p_aioinfo;
1547 	if (ki->kaio_queue_count >= ki->kaio_qallowed_count)
1548 		return (EAGAIN);
1549 
1550 	return _aio_aqueue(td, job, NULL, type);
1551 }
1552 
1553 /*
1554  * Support the aio_return system call, as a side-effect, kernel resources are
1555  * released.
1556  */
1557 int
1558 aio_return(struct thread *td, struct aio_return_args *uap)
1559 {
1560 	struct proc *p = td->td_proc;
1561 	int s;
1562 	long jobref;
1563 	struct aiocblist *cb, *ncb;
1564 	struct aiocb *ujob;
1565 	struct kaioinfo *ki;
1566 
1567 	ujob = uap->aiocbp;
1568 	jobref = fuword(&ujob->_aiocb_private.kernelinfo);
1569 	if (jobref == -1 || jobref == 0)
1570 		return (EINVAL);
1571 
1572 	ki = p->p_aioinfo;
1573 	if (ki == NULL)
1574 		return (EINVAL);
1575 	TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
1576 		if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) ==
1577 		    jobref) {
1578 			if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
1579 				p->p_stats->p_ru.ru_oublock +=
1580 				    cb->outputcharge;
1581 				cb->outputcharge = 0;
1582 			} else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
1583 				p->p_stats->p_ru.ru_inblock += cb->inputcharge;
1584 				cb->inputcharge = 0;
1585 			}
1586 			goto done;
1587 		}
1588 	}
1589 	s = splbio();
1590 	for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = ncb) {
1591 		ncb = TAILQ_NEXT(cb, plist);
1592 		if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo)
1593 		    == jobref) {
1594 			break;
1595 		}
1596 	}
1597 	splx(s);
1598  done:
1599 	if (cb != NULL) {
1600 		if (ujob == cb->uuaiocb) {
1601 			td->td_retval[0] =
1602 			    cb->uaiocb._aiocb_private.status;
1603 		} else
1604 			td->td_retval[0] = EFAULT;
1605 		aio_free_entry(cb);
1606 		return (0);
1607 	}
1608 	return (EINVAL);
1609 }
1610 
1611 /*
1612  * Allow a process to wakeup when any of the I/O requests are completed.
1613  */
1614 int
1615 aio_suspend(struct thread *td, struct aio_suspend_args *uap)
1616 {
1617 	struct proc *p = td->td_proc;
1618 	struct timeval atv;
1619 	struct timespec ts;
1620 	struct aiocb *const *cbptr, *cbp;
1621 	struct kaioinfo *ki;
1622 	struct aiocblist *cb;
1623 	int i;
1624 	int njoblist;
1625 	int error, s, timo;
1626 	long *ijoblist;
1627 	struct aiocb **ujoblist;
1628 
1629 	if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
1630 		return (EINVAL);
1631 
1632 	timo = 0;
1633 	if (uap->timeout) {
1634 		/* Get timespec struct. */
1635 		if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
1636 			return (error);
1637 
1638 		if (ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000)
1639 			return (EINVAL);
1640 
1641 		TIMESPEC_TO_TIMEVAL(&atv, &ts);
1642 		if (itimerfix(&atv))
1643 			return (EINVAL);
1644 		timo = tvtohz(&atv);
1645 	}
1646 
1647 	ki = p->p_aioinfo;
1648 	if (ki == NULL)
1649 		return (EAGAIN);
1650 
1651 	njoblist = 0;
1652 	ijoblist = uma_zalloc(aiol_zone, M_WAITOK);
1653 	ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
1654 	cbptr = uap->aiocbp;
1655 
1656 	for (i = 0; i < uap->nent; i++) {
1657 		cbp = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
1658 		if (cbp == 0)
1659 			continue;
1660 		ujoblist[njoblist] = cbp;
1661 		ijoblist[njoblist] = fuword(&cbp->_aiocb_private.kernelinfo);
1662 		njoblist++;
1663 	}
1664 
1665 	if (njoblist == 0) {
1666 		uma_zfree(aiol_zone, ijoblist);
1667 		uma_zfree(aiol_zone, ujoblist);
1668 		return (0);
1669 	}
1670 
1671 	error = 0;
1672 	for (;;) {
1673 		TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
1674 			for (i = 0; i < njoblist; i++) {
1675 				if (((intptr_t)
1676 				    cb->uaiocb._aiocb_private.kernelinfo) ==
1677 				    ijoblist[i]) {
1678 					if (ujoblist[i] != cb->uuaiocb)
1679 						error = EINVAL;
1680 					uma_zfree(aiol_zone, ijoblist);
1681 					uma_zfree(aiol_zone, ujoblist);
1682 					return (error);
1683 				}
1684 			}
1685 		}
1686 
1687 		s = splbio();
1688 		for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb =
1689 		    TAILQ_NEXT(cb, plist)) {
1690 			for (i = 0; i < njoblist; i++) {
1691 				if (((intptr_t)
1692 				    cb->uaiocb._aiocb_private.kernelinfo) ==
1693 				    ijoblist[i]) {
1694 					splx(s);
1695 					if (ujoblist[i] != cb->uuaiocb)
1696 						error = EINVAL;
1697 					uma_zfree(aiol_zone, ijoblist);
1698 					uma_zfree(aiol_zone, ujoblist);
1699 					return (error);
1700 				}
1701 			}
1702 		}
1703 
1704 		ki->kaio_flags |= KAIO_WAKEUP;
1705 		error = tsleep(p, PRIBIO | PCATCH, "aiospn", timo);
1706 		splx(s);
1707 
1708 		if (error == ERESTART || error == EINTR) {
1709 			uma_zfree(aiol_zone, ijoblist);
1710 			uma_zfree(aiol_zone, ujoblist);
1711 			return (EINTR);
1712 		} else if (error == EWOULDBLOCK) {
1713 			uma_zfree(aiol_zone, ijoblist);
1714 			uma_zfree(aiol_zone, ujoblist);
1715 			return (EAGAIN);
1716 		}
1717 	}
1718 
1719 /* NOTREACHED */
1720 	return (EINVAL);
1721 }
1722 
1723 /*
1724  * aio_cancel cancels any non-physio aio operations not currently in
1725  * progress.
1726  */
1727 int
1728 aio_cancel(struct thread *td, struct aio_cancel_args *uap)
1729 {
1730 	struct proc *p = td->td_proc;
1731 	struct kaioinfo *ki;
1732 	struct aiocblist *cbe, *cbn;
1733 	struct file *fp;
1734 	struct filedesc *fdp;
1735 	struct socket *so;
1736 	struct proc *po;
1737 	int s,error;
1738 	int cancelled=0;
1739 	int notcancelled=0;
1740 	struct vnode *vp;
1741 
1742 	fdp = p->p_fd;
1743 	if ((u_int)uap->fd >= fdp->fd_nfiles ||
1744 	    (fp = fdp->fd_ofiles[uap->fd]) == NULL)
1745 		return (EBADF);
1746 
1747 	if (fp->f_type == DTYPE_VNODE) {
1748 		vp = fp->f_vnode;
1749 
1750 		if (vn_isdisk(vp,&error)) {
1751 			td->td_retval[0] = AIO_NOTCANCELED;
1752 			return (0);
1753 		}
1754 	} else if (fp->f_type == DTYPE_SOCKET) {
1755 		so = fp->f_data;
1756 
1757 		s = splnet();
1758 
1759 		for (cbe = TAILQ_FIRST(&so->so_aiojobq); cbe; cbe = cbn) {
1760 			cbn = TAILQ_NEXT(cbe, list);
1761 			if ((uap->aiocbp == NULL) ||
1762 				(uap->aiocbp == cbe->uuaiocb) ) {
1763 				po = cbe->userproc;
1764 				ki = po->p_aioinfo;
1765 				TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
1766 				TAILQ_REMOVE(&ki->kaio_sockqueue, cbe, plist);
1767 				TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, plist);
1768 				if (ki->kaio_flags & KAIO_WAKEUP) {
1769 					wakeup(po);
1770 				}
1771 				cbe->jobstate = JOBST_JOBFINISHED;
1772 				cbe->uaiocb._aiocb_private.status=-1;
1773 				cbe->uaiocb._aiocb_private.error=ECANCELED;
1774 				cancelled++;
1775 /* XXX cancelled, knote? */
1776 				if (cbe->uaiocb.aio_sigevent.sigev_notify ==
1777 				    SIGEV_SIGNAL) {
1778 					PROC_LOCK(cbe->userproc);
1779 					psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
1780 					PROC_UNLOCK(cbe->userproc);
1781 				}
1782 				if (uap->aiocbp)
1783 					break;
1784 			}
1785 		}
1786 		splx(s);
1787 
1788 		if ((cancelled) && (uap->aiocbp)) {
1789 			td->td_retval[0] = AIO_CANCELED;
1790 			return (0);
1791 		}
1792 	}
1793 	ki=p->p_aioinfo;
1794 	if (ki == NULL)
1795 		goto done;
1796 	s = splnet();
1797 
1798 	for (cbe = TAILQ_FIRST(&ki->kaio_jobqueue); cbe; cbe = cbn) {
1799 		cbn = TAILQ_NEXT(cbe, plist);
1800 
1801 		if ((uap->fd == cbe->uaiocb.aio_fildes) &&
1802 		    ((uap->aiocbp == NULL ) ||
1803 		     (uap->aiocbp == cbe->uuaiocb))) {
1804 
1805 			if (cbe->jobstate == JOBST_JOBQGLOBAL) {
1806 				TAILQ_REMOVE(&aio_jobs, cbe, list);
1807 				TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
1808 				TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe,
1809 				    plist);
1810 				cancelled++;
1811 				ki->kaio_queue_finished_count++;
1812 				cbe->jobstate = JOBST_JOBFINISHED;
1813 				cbe->uaiocb._aiocb_private.status = -1;
1814 				cbe->uaiocb._aiocb_private.error = ECANCELED;
1815 /* XXX cancelled, knote? */
1816 				if (cbe->uaiocb.aio_sigevent.sigev_notify ==
1817 				    SIGEV_SIGNAL) {
1818 					PROC_LOCK(cbe->userproc);
1819 					psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
1820 					PROC_UNLOCK(cbe->userproc);
1821 				}
1822 			} else {
1823 				notcancelled++;
1824 			}
1825 		}
1826 	}
1827 	splx(s);
1828 done:
1829 	if (notcancelled) {
1830 		td->td_retval[0] = AIO_NOTCANCELED;
1831 		return (0);
1832 	}
1833 	if (cancelled) {
1834 		td->td_retval[0] = AIO_CANCELED;
1835 		return (0);
1836 	}
1837 	td->td_retval[0] = AIO_ALLDONE;
1838 
1839 	return (0);
1840 }
1841 
1842 /*
1843  * aio_error is implemented in the kernel level for compatibility purposes only.
1844  * For a user mode async implementation, it would be best to do it in a userland
1845  * subroutine.
1846  */
1847 int
1848 aio_error(struct thread *td, struct aio_error_args *uap)
1849 {
1850 	struct proc *p = td->td_proc;
1851 	int s;
1852 	struct aiocblist *cb;
1853 	struct kaioinfo *ki;
1854 	long jobref;
1855 
1856 	ki = p->p_aioinfo;
1857 	if (ki == NULL)
1858 		return (EINVAL);
1859 
1860 	jobref = fuword(&uap->aiocbp->_aiocb_private.kernelinfo);
1861 	if ((jobref == -1) || (jobref == 0))
1862 		return (EINVAL);
1863 
1864 	TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
1865 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1866 		    jobref) {
1867 			td->td_retval[0] = cb->uaiocb._aiocb_private.error;
1868 			return (0);
1869 		}
1870 	}
1871 
1872 	s = splnet();
1873 
1874 	for (cb = TAILQ_FIRST(&ki->kaio_jobqueue); cb; cb = TAILQ_NEXT(cb,
1875 	    plist)) {
1876 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1877 		    jobref) {
1878 			td->td_retval[0] = EINPROGRESS;
1879 			splx(s);
1880 			return (0);
1881 		}
1882 	}
1883 
1884 	for (cb = TAILQ_FIRST(&ki->kaio_sockqueue); cb; cb = TAILQ_NEXT(cb,
1885 	    plist)) {
1886 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1887 		    jobref) {
1888 			td->td_retval[0] = EINPROGRESS;
1889 			splx(s);
1890 			return (0);
1891 		}
1892 	}
1893 	splx(s);
1894 
1895 	s = splbio();
1896 	for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = TAILQ_NEXT(cb,
1897 	    plist)) {
1898 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1899 		    jobref) {
1900 			td->td_retval[0] = cb->uaiocb._aiocb_private.error;
1901 			splx(s);
1902 			return (0);
1903 		}
1904 	}
1905 
1906 	for (cb = TAILQ_FIRST(&ki->kaio_bufqueue); cb; cb = TAILQ_NEXT(cb,
1907 	    plist)) {
1908 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1909 		    jobref) {
1910 			td->td_retval[0] = EINPROGRESS;
1911 			splx(s);
1912 			return (0);
1913 		}
1914 	}
1915 	splx(s);
1916 
1917 #if (0)
1918 	/*
1919 	 * Hack for lio.
1920 	 */
1921 	status = fuword(&uap->aiocbp->_aiocb_private.status);
1922 	if (status == -1)
1923 		return fuword(&uap->aiocbp->_aiocb_private.error);
1924 #endif
1925 	return (EINVAL);
1926 }
1927 
1928 /* syscall - asynchronous read from a file (REALTIME) */
1929 int
1930 aio_read(struct thread *td, struct aio_read_args *uap)
1931 {
1932 
1933 	return aio_aqueue(td, uap->aiocbp, LIO_READ);
1934 }
1935 
1936 /* syscall - asynchronous write to a file (REALTIME) */
1937 int
1938 aio_write(struct thread *td, struct aio_write_args *uap)
1939 {
1940 
1941 	return aio_aqueue(td, uap->aiocbp, LIO_WRITE);
1942 }
1943 
1944 /* syscall - list directed I/O (REALTIME) */
1945 int
1946 lio_listio(struct thread *td, struct lio_listio_args *uap)
1947 {
1948 	struct proc *p = td->td_proc;
1949 	int nent, nentqueued;
1950 	struct aiocb *iocb, * const *cbptr;
1951 	struct aiocblist *cb;
1952 	struct kaioinfo *ki;
1953 	struct aio_liojob *lj;
1954 	int error, runningcode;
1955 	int nerror;
1956 	int i;
1957 	int s;
1958 
1959 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
1960 		return (EINVAL);
1961 
1962 	nent = uap->nent;
1963 	if (nent < 0 || nent > AIO_LISTIO_MAX)
1964 		return (EINVAL);
1965 
1966 	if (p->p_aioinfo == NULL)
1967 		aio_init_aioinfo(p);
1968 
1969 	if ((nent + num_queue_count) > max_queue_count)
1970 		return (EAGAIN);
1971 
1972 	ki = p->p_aioinfo;
1973 	if ((nent + ki->kaio_queue_count) > ki->kaio_qallowed_count)
1974 		return (EAGAIN);
1975 
1976 	lj = uma_zalloc(aiolio_zone, M_WAITOK);
1977 	if (!lj)
1978 		return (EAGAIN);
1979 
1980 	lj->lioj_flags = 0;
1981 	lj->lioj_buffer_count = 0;
1982 	lj->lioj_buffer_finished_count = 0;
1983 	lj->lioj_queue_count = 0;
1984 	lj->lioj_queue_finished_count = 0;
1985 	lj->lioj_ki = ki;
1986 
1987 	/*
1988 	 * Setup signal.
1989 	 */
1990 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
1991 		error = copyin(uap->sig, &lj->lioj_signal,
1992 		    sizeof(lj->lioj_signal));
1993 		if (error) {
1994 			uma_zfree(aiolio_zone, lj);
1995 			return (error);
1996 		}
1997 		if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) {
1998 			uma_zfree(aiolio_zone, lj);
1999 			return (EINVAL);
2000 		}
2001 		lj->lioj_flags |= LIOJ_SIGNAL;
2002 	}
2003 	TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list);
2004 	/*
2005 	 * Get pointers to the list of I/O requests.
2006 	 */
2007 	nerror = 0;
2008 	nentqueued = 0;
2009 	cbptr = uap->acb_list;
2010 	for (i = 0; i < uap->nent; i++) {
2011 		iocb = (struct aiocb *)(intptr_t)fuword(&cbptr[i]);
2012 		if (((intptr_t)iocb != -1) && ((intptr_t)iocb != 0)) {
2013 			error = _aio_aqueue(td, iocb, lj, 0);
2014 			if (error == 0)
2015 				nentqueued++;
2016 			else
2017 				nerror++;
2018 		}
2019 	}
2020 
2021 	/*
2022 	 * If we haven't queued any, then just return error.
2023 	 */
2024 	if (nentqueued == 0)
2025 		return (0);
2026 
2027 	/*
2028 	 * Calculate the appropriate error return.
2029 	 */
2030 	runningcode = 0;
2031 	if (nerror)
2032 		runningcode = EIO;
2033 
2034 	if (uap->mode == LIO_WAIT) {
2035 		int command, found, jobref;
2036 
2037 		for (;;) {
2038 			found = 0;
2039 			for (i = 0; i < uap->nent; i++) {
2040 				/*
2041 				 * Fetch address of the control buf pointer in
2042 				 * user space.
2043 				 */
2044 				iocb = (struct aiocb *)
2045 				    (intptr_t)fuword(&cbptr[i]);
2046 				if (((intptr_t)iocb == -1) || ((intptr_t)iocb
2047 				    == 0))
2048 					continue;
2049 
2050 				/*
2051 				 * Fetch the associated command from user space.
2052 				 */
2053 				command = fuword(&iocb->aio_lio_opcode);
2054 				if (command == LIO_NOP) {
2055 					found++;
2056 					continue;
2057 				}
2058 
2059 				jobref =
2060 				    fuword(&iocb->_aiocb_private.kernelinfo);
2061 
2062 				TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
2063 					if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
2064 					    == jobref) {
2065 						if (cb->uaiocb.aio_lio_opcode
2066 						    == LIO_WRITE) {
2067 							p->p_stats->p_ru.ru_oublock
2068 							    +=
2069 							    cb->outputcharge;
2070 							cb->outputcharge = 0;
2071 						} else if (cb->uaiocb.aio_lio_opcode
2072 						    == LIO_READ) {
2073 							p->p_stats->p_ru.ru_inblock
2074 							    += cb->inputcharge;
2075 							cb->inputcharge = 0;
2076 						}
2077 						found++;
2078 						break;
2079 					}
2080 				}
2081 
2082 				s = splbio();
2083 				TAILQ_FOREACH(cb, &ki->kaio_bufdone, plist) {
2084 					if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
2085 					    == jobref) {
2086 						found++;
2087 						break;
2088 					}
2089 				}
2090 				splx(s);
2091 			}
2092 
2093 			/*
2094 			 * If all I/Os have been disposed of, then we can
2095 			 * return.
2096 			 */
2097 			if (found == nentqueued)
2098 				return (runningcode);
2099 
2100 			ki->kaio_flags |= KAIO_WAKEUP;
2101 			error = tsleep(p, PRIBIO | PCATCH, "aiospn", 0);
2102 
2103 			if (error == EINTR)
2104 				return (EINTR);
2105 			else if (error == EWOULDBLOCK)
2106 				return (EAGAIN);
2107 		}
2108 	}
2109 
2110 	return (runningcode);
2111 }
2112 
2113 /*
2114  * This is a weird hack so that we can post a signal.  It is safe to do so from
2115  * a timeout routine, but *not* from an interrupt routine.
2116  */
2117 static void
2118 process_signal(void *aioj)
2119 {
2120 	struct aiocblist *aiocbe = aioj;
2121 	struct aio_liojob *lj = aiocbe->lio;
2122 	struct aiocb *cb = &aiocbe->uaiocb;
2123 
2124 	if ((lj) && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL) &&
2125 		(lj->lioj_queue_count == lj->lioj_queue_finished_count)) {
2126 		PROC_LOCK(lj->lioj_ki->kaio_p);
2127 		psignal(lj->lioj_ki->kaio_p, lj->lioj_signal.sigev_signo);
2128 		PROC_UNLOCK(lj->lioj_ki->kaio_p);
2129 		lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
2130 	}
2131 
2132 	if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
2133 		PROC_LOCK(aiocbe->userproc);
2134 		psignal(aiocbe->userproc, cb->aio_sigevent.sigev_signo);
2135 		PROC_UNLOCK(aiocbe->userproc);
2136 	}
2137 }
2138 
2139 /*
2140  * Interrupt handler for physio, performs the necessary process wakeups, and
2141  * signals.
2142  */
2143 static void
2144 aio_physwakeup(struct buf *bp)
2145 {
2146 	struct aiocblist *aiocbe;
2147 	struct proc *p;
2148 	struct kaioinfo *ki;
2149 	struct aio_liojob *lj;
2150 
2151 	wakeup(bp);
2152 
2153 	aiocbe = (struct aiocblist *)bp->b_caller1;
2154 	if (aiocbe) {
2155 		p = aiocbe->userproc;
2156 
2157 		aiocbe->jobstate = JOBST_JOBBFINISHED;
2158 		aiocbe->uaiocb._aiocb_private.status -= bp->b_resid;
2159 		aiocbe->uaiocb._aiocb_private.error = 0;
2160 		aiocbe->jobflags |= AIOCBLIST_DONE;
2161 
2162 		if (bp->b_ioflags & BIO_ERROR)
2163 			aiocbe->uaiocb._aiocb_private.error = bp->b_error;
2164 
2165 		lj = aiocbe->lio;
2166 		if (lj) {
2167 			lj->lioj_buffer_finished_count++;
2168 
2169 			/*
2170 			 * wakeup/signal if all of the interrupt jobs are done.
2171 			 */
2172 			if (lj->lioj_buffer_finished_count ==
2173 			    lj->lioj_buffer_count) {
2174 				/*
2175 				 * Post a signal if it is called for.
2176 				 */
2177 				if ((lj->lioj_flags &
2178 				    (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) ==
2179 				    LIOJ_SIGNAL) {
2180 					lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
2181 					aiocbe->timeouthandle =
2182 						timeout(process_signal,
2183 							aiocbe, 0);
2184 				}
2185 			}
2186 		}
2187 
2188 		ki = p->p_aioinfo;
2189 		if (ki) {
2190 			ki->kaio_buffer_finished_count++;
2191 			TAILQ_REMOVE(&aio_bufjobs, aiocbe, list);
2192 			TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
2193 			TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
2194 
2195 			KNOTE_UNLOCKED(&aiocbe->klist, 0);
2196 			/* Do the wakeup. */
2197 			if (ki->kaio_flags & (KAIO_RUNDOWN|KAIO_WAKEUP)) {
2198 				ki->kaio_flags &= ~KAIO_WAKEUP;
2199 				wakeup(p);
2200 			}
2201 		}
2202 
2203 		if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL)
2204 			aiocbe->timeouthandle =
2205 				timeout(process_signal, aiocbe, 0);
2206 	}
2207 }
2208 
2209 /* syscall - wait for the next completion of an aio request */
2210 int
2211 aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
2212 {
2213 	struct proc *p = td->td_proc;
2214 	struct timeval atv;
2215 	struct timespec ts;
2216 	struct kaioinfo *ki;
2217 	struct aiocblist *cb = NULL;
2218 	int error, s, timo;
2219 
2220 	suword(uap->aiocbp, (int)NULL);
2221 
2222 	timo = 0;
2223 	if (uap->timeout) {
2224 		/* Get timespec struct. */
2225 		error = copyin(uap->timeout, &ts, sizeof(ts));
2226 		if (error)
2227 			return (error);
2228 
2229 		if ((ts.tv_nsec < 0) || (ts.tv_nsec >= 1000000000))
2230 			return (EINVAL);
2231 
2232 		TIMESPEC_TO_TIMEVAL(&atv, &ts);
2233 		if (itimerfix(&atv))
2234 			return (EINVAL);
2235 		timo = tvtohz(&atv);
2236 	}
2237 
2238 	ki = p->p_aioinfo;
2239 	if (ki == NULL)
2240 		return (EAGAIN);
2241 
2242 	for (;;) {
2243 		if ((cb = TAILQ_FIRST(&ki->kaio_jobdone)) != 0) {
2244 			suword(uap->aiocbp, (uintptr_t)cb->uuaiocb);
2245 			td->td_retval[0] = cb->uaiocb._aiocb_private.status;
2246 			if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
2247 				p->p_stats->p_ru.ru_oublock +=
2248 				    cb->outputcharge;
2249 				cb->outputcharge = 0;
2250 			} else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
2251 				p->p_stats->p_ru.ru_inblock += cb->inputcharge;
2252 				cb->inputcharge = 0;
2253 			}
2254 			aio_free_entry(cb);
2255 			return (cb->uaiocb._aiocb_private.error);
2256 		}
2257 
2258 		s = splbio();
2259  		if ((cb = TAILQ_FIRST(&ki->kaio_bufdone)) != 0 ) {
2260 			splx(s);
2261 			suword(uap->aiocbp, (uintptr_t)cb->uuaiocb);
2262 			td->td_retval[0] = cb->uaiocb._aiocb_private.status;
2263 			aio_free_entry(cb);
2264 			return (cb->uaiocb._aiocb_private.error);
2265 		}
2266 
2267 		ki->kaio_flags |= KAIO_WAKEUP;
2268 		error = tsleep(p, PRIBIO | PCATCH, "aiowc", timo);
2269 		splx(s);
2270 
2271 		if (error == ERESTART)
2272 			return (EINTR);
2273 		else if (error < 0)
2274 			return (error);
2275 		else if (error == EINTR)
2276 			return (EINTR);
2277 		else if (error == EWOULDBLOCK)
2278 			return (EAGAIN);
2279 	}
2280 }
2281 
2282 /* kqueue attach function */
2283 static int
2284 filt_aioattach(struct knote *kn)
2285 {
2286 	struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
2287 
2288 	/*
2289 	 * The aiocbe pointer must be validated before using it, so
2290 	 * registration is restricted to the kernel; the user cannot
2291 	 * set EV_FLAG1.
2292 	 */
2293 	if ((kn->kn_flags & EV_FLAG1) == 0)
2294 		return (EPERM);
2295 	kn->kn_flags &= ~EV_FLAG1;
2296 
2297 	knlist_add(&aiocbe->klist, kn, 0);
2298 
2299 	return (0);
2300 }
2301 
2302 /* kqueue detach function */
2303 static void
2304 filt_aiodetach(struct knote *kn)
2305 {
2306 	struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
2307 
2308 	knlist_remove(&aiocbe->klist, kn, 0);
2309 }
2310 
2311 /* kqueue filter function */
2312 /*ARGSUSED*/
2313 static int
2314 filt_aio(struct knote *kn, long hint)
2315 {
2316 	struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
2317 
2318 	kn->kn_data = aiocbe->uaiocb._aiocb_private.error;
2319 	if (aiocbe->jobstate != JOBST_JOBFINISHED &&
2320 	    aiocbe->jobstate != JOBST_JOBBFINISHED)
2321 		return (0);
2322 	kn->kn_flags |= EV_EOF;
2323 	return (1);
2324 }
2325