xref: /freebsd/sys/kern/vfs_aio.c (revision b40ce4165d5eb3a5de1515245055350ae3dbab8e)
1ee877a35SJohn Dyson /*
2ee877a35SJohn Dyson  * Copyright (c) 1997 John S. Dyson.  All rights reserved.
3ee877a35SJohn Dyson  *
4ee877a35SJohn Dyson  * Redistribution and use in source and binary forms, with or without
5ee877a35SJohn Dyson  * modification, are permitted provided that the following conditions
6ee877a35SJohn Dyson  * are met:
7ee877a35SJohn Dyson  * 1. Redistributions of source code must retain the above copyright
8ee877a35SJohn Dyson  *    notice, this list of conditions and the following disclaimer.
9ee877a35SJohn Dyson  * 2. John S. Dyson's name may not be used to endorse or promote products
10ee877a35SJohn Dyson  *    derived from this software without specific prior written permission.
11ee877a35SJohn Dyson  *
12ee877a35SJohn Dyson  * DISCLAIMER:  This code isn't warranted to do anything useful.  Anything
13ee877a35SJohn Dyson  * bad that happens because of using this software isn't the responsibility
14ee877a35SJohn Dyson  * of the author.  This software is distributed AS-IS.
15ee877a35SJohn Dyson  *
16c3aac50fSPeter Wemm  * $FreeBSD$
17ee877a35SJohn Dyson  */
18ee877a35SJohn Dyson 
19ee877a35SJohn Dyson /*
208a6472b7SPeter Dufault  * This file contains support for the POSIX 1003.1B AIO/LIO facility.
21ee877a35SJohn Dyson  */
22ee877a35SJohn Dyson 
23ee877a35SJohn Dyson #include <sys/param.h>
24ee877a35SJohn Dyson #include <sys/systm.h>
259626b608SPoul-Henning Kamp #include <sys/bio.h>
26a5c9bce7SBruce Evans #include <sys/buf.h>
27ee877a35SJohn Dyson #include <sys/sysproto.h>
28ee877a35SJohn Dyson #include <sys/filedesc.h>
29ee877a35SJohn Dyson #include <sys/kernel.h>
30c9a970a7SAlan Cox #include <sys/kthread.h>
31ee877a35SJohn Dyson #include <sys/fcntl.h>
32ee877a35SJohn Dyson #include <sys/file.h>
33fdebd4f0SBruce Evans #include <sys/lock.h>
3435e0e5b3SJohn Baldwin #include <sys/mutex.h>
35ee877a35SJohn Dyson #include <sys/unistd.h>
36ee877a35SJohn Dyson #include <sys/proc.h>
372d2f8ae7SBruce Evans #include <sys/resourcevar.h>
38ee877a35SJohn Dyson #include <sys/signalvar.h>
39bfbbc4aaSJason Evans #include <sys/protosw.h>
40bfbbc4aaSJason Evans #include <sys/socketvar.h>
41a624e84fSJohn Dyson #include <sys/sysctl.h>
42fd3bf775SJohn Dyson #include <sys/vnode.h>
43fd3bf775SJohn Dyson #include <sys/conf.h>
44cb679c38SJonathan Lemon #include <sys/event.h>
45ee877a35SJohn Dyson 
46ee877a35SJohn Dyson #include <vm/vm.h>
47ee877a35SJohn Dyson #include <vm/vm_extern.h>
482244ea07SJohn Dyson #include <vm/pmap.h>
492244ea07SJohn Dyson #include <vm/vm_map.h>
50fd3bf775SJohn Dyson #include <vm/vm_zone.h>
51ee877a35SJohn Dyson #include <sys/aio.h>
525aaef07cSJohn Dyson 
5308637435SBruce Evans #include <machine/limits.h>
54621dbe43SBruce Evans 
55dd85920aSJason Evans #include "opt_vfs_aio.h"
56ee877a35SJohn Dyson 
575dec52baSPeter Wemm #ifdef VFS_AIO
585dec52baSPeter Wemm 
598c12612cSDoug Rabson static	long jobrefid;
602244ea07SJohn Dyson 
612244ea07SJohn Dyson #define JOBST_NULL		0x0
622244ea07SJohn Dyson #define	JOBST_JOBQPROC		0x1
632244ea07SJohn Dyson #define JOBST_JOBQGLOBAL	0x2
642244ea07SJohn Dyson #define JOBST_JOBRUNNING	0x3
652244ea07SJohn Dyson #define JOBST_JOBFINISHED	0x4
66fd3bf775SJohn Dyson #define	JOBST_JOBQBUF		0x5
67fd3bf775SJohn Dyson #define	JOBST_JOBBFINISHED	0x6
682244ea07SJohn Dyson 
6984af4da6SJohn Dyson #ifndef MAX_AIO_PER_PROC
702244ea07SJohn Dyson #define MAX_AIO_PER_PROC	32
7184af4da6SJohn Dyson #endif
7284af4da6SJohn Dyson 
7384af4da6SJohn Dyson #ifndef MAX_AIO_QUEUE_PER_PROC
742244ea07SJohn Dyson #define MAX_AIO_QUEUE_PER_PROC	256 /* Bigger than AIO_LISTIO_MAX */
7584af4da6SJohn Dyson #endif
7684af4da6SJohn Dyson 
7784af4da6SJohn Dyson #ifndef MAX_AIO_PROCS
78fd3bf775SJohn Dyson #define MAX_AIO_PROCS		32
7984af4da6SJohn Dyson #endif
8084af4da6SJohn Dyson 
8184af4da6SJohn Dyson #ifndef MAX_AIO_QUEUE
822244ea07SJohn Dyson #define	MAX_AIO_QUEUE		1024 /* Bigger than AIO_LISTIO_MAX */
8384af4da6SJohn Dyson #endif
8484af4da6SJohn Dyson 
8584af4da6SJohn Dyson #ifndef TARGET_AIO_PROCS
86bfbbc4aaSJason Evans #define TARGET_AIO_PROCS	4
8784af4da6SJohn Dyson #endif
8884af4da6SJohn Dyson 
8984af4da6SJohn Dyson #ifndef MAX_BUF_AIO
9084af4da6SJohn Dyson #define MAX_BUF_AIO		16
9184af4da6SJohn Dyson #endif
9284af4da6SJohn Dyson 
9384af4da6SJohn Dyson #ifndef AIOD_TIMEOUT_DEFAULT
9484af4da6SJohn Dyson #define	AIOD_TIMEOUT_DEFAULT	(10 * hz)
9584af4da6SJohn Dyson #endif
9684af4da6SJohn Dyson 
9784af4da6SJohn Dyson #ifndef AIOD_LIFETIME_DEFAULT
9884af4da6SJohn Dyson #define AIOD_LIFETIME_DEFAULT	(30 * hz)
9984af4da6SJohn Dyson #endif
1002244ea07SJohn Dyson 
101303b270bSEivind Eklund static int max_aio_procs = MAX_AIO_PROCS;
102303b270bSEivind Eklund static int num_aio_procs = 0;
103303b270bSEivind Eklund static int target_aio_procs = TARGET_AIO_PROCS;
104303b270bSEivind Eklund static int max_queue_count = MAX_AIO_QUEUE;
105303b270bSEivind Eklund static int num_queue_count = 0;
106303b270bSEivind Eklund static int num_buf_aio = 0;
107303b270bSEivind Eklund static int num_aio_resv_start = 0;
108303b270bSEivind Eklund static int aiod_timeout;
109303b270bSEivind Eklund static int aiod_lifetime;
110a624e84fSJohn Dyson 
111bfbbc4aaSJason Evans static int max_aio_per_proc = MAX_AIO_PER_PROC;
112bfbbc4aaSJason Evans static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
113303b270bSEivind Eklund static int max_buf_aio = MAX_BUF_AIO;
114a624e84fSJohn Dyson 
115a624e84fSJohn Dyson SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "AIO mgmt");
116a624e84fSJohn Dyson 
117a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc,
118a624e84fSJohn Dyson 	CTLFLAG_RW, &max_aio_per_proc, 0, "");
119a624e84fSJohn Dyson 
120a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc,
121a624e84fSJohn Dyson 	CTLFLAG_RW, &max_aio_queue_per_proc, 0, "");
122a624e84fSJohn Dyson 
123a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs,
124a624e84fSJohn Dyson 	CTLFLAG_RW, &max_aio_procs, 0, "");
125a624e84fSJohn Dyson 
126a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs,
127a624e84fSJohn Dyson 	CTLFLAG_RD, &num_aio_procs, 0, "");
128a624e84fSJohn Dyson 
129a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count,
130a624e84fSJohn Dyson 	CTLFLAG_RD, &num_queue_count, 0, "");
131a624e84fSJohn Dyson 
132a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue,
133a624e84fSJohn Dyson 	CTLFLAG_RW, &max_queue_count, 0, "");
134a624e84fSJohn Dyson 
135a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs,
136a624e84fSJohn Dyson 	CTLFLAG_RW, &target_aio_procs, 0, "");
137a624e84fSJohn Dyson 
13884af4da6SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio,
13984af4da6SJohn Dyson 	CTLFLAG_RW, &max_buf_aio, 0, "");
140fd3bf775SJohn Dyson 
141fd3bf775SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio,
142fd3bf775SJohn Dyson 	CTLFLAG_RD, &num_buf_aio, 0, "");
143fd3bf775SJohn Dyson 
14484af4da6SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime,
14584af4da6SJohn Dyson 	CTLFLAG_RW, &aiod_lifetime, 0, "");
14684af4da6SJohn Dyson 
14784af4da6SJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout,
14884af4da6SJohn Dyson 	CTLFLAG_RW, &aiod_timeout, 0, "");
14984af4da6SJohn Dyson 
1502244ea07SJohn Dyson /*
1512244ea07SJohn Dyson  * AIO process info
1522244ea07SJohn Dyson  */
15384af4da6SJohn Dyson #define AIOP_FREE	0x1			/* proc on free queue */
15484af4da6SJohn Dyson #define AIOP_SCHED	0x2			/* proc explicitly scheduled */
15584af4da6SJohn Dyson 
156b40ce416SJulian Elischer struct aiothreadlist {
157b40ce416SJulian Elischer 	int aiothreadflags;			/* AIO proc flags */
158b40ce416SJulian Elischer 	TAILQ_ENTRY(aiothreadlist) list;	/* List of processes */
159b40ce416SJulian Elischer 	struct thread *aiothread;		/* The AIO thread */
160e3975643SJake Burkholder 	TAILQ_HEAD (,aiocblist) jobtorun;	/* suggested job to run */
1612244ea07SJohn Dyson };
1622244ea07SJohn Dyson 
16384af4da6SJohn Dyson /*
16484af4da6SJohn Dyson  * data-structure for lio signal management
16584af4da6SJohn Dyson  */
16684af4da6SJohn Dyson struct aio_liojob {
16784af4da6SJohn Dyson 	int	lioj_flags;
16884af4da6SJohn Dyson 	int	lioj_buffer_count;
16984af4da6SJohn Dyson 	int	lioj_buffer_finished_count;
17084af4da6SJohn Dyson 	int	lioj_queue_count;
17184af4da6SJohn Dyson 	int	lioj_queue_finished_count;
17284af4da6SJohn Dyson 	struct	sigevent lioj_signal;	/* signal on all I/O done */
173e3975643SJake Burkholder 	TAILQ_ENTRY	(aio_liojob) lioj_list;
17484af4da6SJohn Dyson 	struct	kaioinfo *lioj_ki;
17584af4da6SJohn Dyson };
17684af4da6SJohn Dyson #define	LIOJ_SIGNAL		0x1	/* signal on all done (lio) */
17784af4da6SJohn Dyson #define	LIOJ_SIGNAL_POSTED	0x2	/* signal has been posted */
17884af4da6SJohn Dyson 
17984af4da6SJohn Dyson /*
18084af4da6SJohn Dyson  * per process aio data structure
18184af4da6SJohn Dyson  */
1822244ea07SJohn Dyson struct kaioinfo {
183fd3bf775SJohn Dyson 	int	kaio_flags;		/* per process kaio flags */
1842244ea07SJohn Dyson 	int	kaio_maxactive_count;	/* maximum number of AIOs */
1852244ea07SJohn Dyson 	int	kaio_active_count;	/* number of currently used AIOs */
1862244ea07SJohn Dyson 	int	kaio_qallowed_count;	/* maxiumu size of AIO queue */
1872244ea07SJohn Dyson 	int	kaio_queue_count;	/* size of AIO queue */
188fd3bf775SJohn Dyson 	int	kaio_ballowed_count;	/* maximum number of buffers */
18984af4da6SJohn Dyson 	int	kaio_queue_finished_count; /* number of daemon jobs finished */
190fd3bf775SJohn Dyson 	int	kaio_buffer_count;	/* number of physio buffers */
19184af4da6SJohn Dyson 	int	kaio_buffer_finished_count; /* count of I/O done */
19284af4da6SJohn Dyson 	struct 	proc *kaio_p;		/* process that uses this kaio block */
193e3975643SJake Burkholder 	TAILQ_HEAD (,aio_liojob) kaio_liojoblist; /* list of lio jobs */
194e3975643SJake Burkholder 	TAILQ_HEAD (,aiocblist)	kaio_jobqueue;	/* job queue for process */
195e3975643SJake Burkholder 	TAILQ_HEAD (,aiocblist)	kaio_jobdone;	/* done queue for process */
196e3975643SJake Burkholder 	TAILQ_HEAD (,aiocblist)	kaio_bufqueue;	/* buffer job queue for process */
197e3975643SJake Burkholder 	TAILQ_HEAD (,aiocblist)	kaio_bufdone;	/* buffer done queue for process */
198e3975643SJake Burkholder 	TAILQ_HEAD (,aiocblist) kaio_sockqueue; /* queue for aios waiting on sockets */
1992244ea07SJohn Dyson };
2002244ea07SJohn Dyson 
20184af4da6SJohn Dyson #define KAIO_RUNDOWN	0x1	/* process is being run down */
202bfbbc4aaSJason Evans #define KAIO_WAKEUP	0x2	/* wakeup process when there is a significant event */
203fd3bf775SJohn Dyson 
204b40ce416SJulian Elischer static TAILQ_HEAD(,aiothreadlist) aio_freeproc, aio_activeproc;
205e3975643SJake Burkholder static TAILQ_HEAD(,aiocblist) aio_jobs;			/* Async job list */
206e3975643SJake Burkholder static TAILQ_HEAD(,aiocblist) aio_bufjobs;		/* Phys I/O job list */
2072244ea07SJohn Dyson 
208fd3bf775SJohn Dyson static void	aio_init_aioinfo(struct proc *p);
209fd3bf775SJohn Dyson static void	aio_onceonly(void *);
210fd3bf775SJohn Dyson static int	aio_free_entry(struct aiocblist *aiocbe);
211fd3bf775SJohn Dyson static void	aio_process(struct aiocblist *aiocbe);
2122244ea07SJohn Dyson static int	aio_newproc(void);
213b40ce416SJulian Elischer static int	aio_aqueue(struct thread *td, struct aiocb *job, int type);
214fd3bf775SJohn Dyson static void	aio_physwakeup(struct buf *bp);
2159c8a2647SAlan Cox static int	aio_fphysio(struct proc *p, struct aiocblist *aiocbe);
216fd3bf775SJohn Dyson static int	aio_qphysio(struct proc *p, struct aiocblist *iocb);
2179c8b8baaSPeter Wemm static void	aio_daemon(void *uproc);
21813644654SAlan Cox static void	process_signal(void *aioj);
2192244ea07SJohn Dyson 
2202244ea07SJohn Dyson SYSINIT(aio, SI_SUB_VFS, SI_ORDER_ANY, aio_onceonly, NULL);
2212244ea07SJohn Dyson 
222bfbbc4aaSJason Evans static vm_zone_t kaio_zone = 0, aiop_zone = 0, aiocb_zone = 0, aiol_zone = 0;
223bfbbc4aaSJason Evans static vm_zone_t aiolio_zone = 0;
224fd3bf775SJohn Dyson 
225fd3bf775SJohn Dyson /*
2262244ea07SJohn Dyson  * Startup initialization
2272244ea07SJohn Dyson  */
22888ed460eSAlan Cox static void
229fd3bf775SJohn Dyson aio_onceonly(void *na)
230fd3bf775SJohn Dyson {
2312244ea07SJohn Dyson 	TAILQ_INIT(&aio_freeproc);
2322244ea07SJohn Dyson 	TAILQ_INIT(&aio_activeproc);
2332244ea07SJohn Dyson 	TAILQ_INIT(&aio_jobs);
234fd3bf775SJohn Dyson 	TAILQ_INIT(&aio_bufjobs);
235fd3bf775SJohn Dyson 	kaio_zone = zinit("AIO", sizeof (struct kaioinfo), 0, 0, 1);
236b40ce416SJulian Elischer 	aiop_zone = zinit("AIOP", sizeof (struct aiothreadlist), 0, 0, 1);
237fd3bf775SJohn Dyson 	aiocb_zone = zinit("AIOCB", sizeof (struct aiocblist), 0, 0, 1);
238fd3bf775SJohn Dyson 	aiol_zone = zinit("AIOL", AIO_LISTIO_MAX * sizeof (int), 0, 0, 1);
239bfbbc4aaSJason Evans 	aiolio_zone = zinit("AIOLIO", AIO_LISTIO_MAX * sizeof (struct
240bfbbc4aaSJason Evans 	    aio_liojob), 0, 0, 1);
24184af4da6SJohn Dyson 	aiod_timeout = AIOD_TIMEOUT_DEFAULT;
24284af4da6SJohn Dyson 	aiod_lifetime = AIOD_LIFETIME_DEFAULT;
243fd3bf775SJohn Dyson 	jobrefid = 1;
2442244ea07SJohn Dyson }
2452244ea07SJohn Dyson 
2462244ea07SJohn Dyson /*
247bfbbc4aaSJason Evans  * Init the per-process aioinfo structure.  The aioinfo limits are set
248bfbbc4aaSJason Evans  * per-process for user limit (resource) management.
2492244ea07SJohn Dyson  */
25088ed460eSAlan Cox static void
251fd3bf775SJohn Dyson aio_init_aioinfo(struct proc *p)
252fd3bf775SJohn Dyson {
2532244ea07SJohn Dyson 	struct kaioinfo *ki;
2542244ea07SJohn Dyson 	if (p->p_aioinfo == NULL) {
255fd3bf775SJohn Dyson 		ki = zalloc(kaio_zone);
2562244ea07SJohn Dyson 		p->p_aioinfo = ki;
25784af4da6SJohn Dyson 		ki->kaio_flags = 0;
258a624e84fSJohn Dyson 		ki->kaio_maxactive_count = max_aio_per_proc;
2592244ea07SJohn Dyson 		ki->kaio_active_count = 0;
260a624e84fSJohn Dyson 		ki->kaio_qallowed_count = max_aio_queue_per_proc;
2612244ea07SJohn Dyson 		ki->kaio_queue_count = 0;
26284af4da6SJohn Dyson 		ki->kaio_ballowed_count = max_buf_aio;
263fd3bf775SJohn Dyson 		ki->kaio_buffer_count = 0;
26484af4da6SJohn Dyson 		ki->kaio_buffer_finished_count = 0;
26584af4da6SJohn Dyson 		ki->kaio_p = p;
2662244ea07SJohn Dyson 		TAILQ_INIT(&ki->kaio_jobdone);
2672244ea07SJohn Dyson 		TAILQ_INIT(&ki->kaio_jobqueue);
268fd3bf775SJohn Dyson 		TAILQ_INIT(&ki->kaio_bufdone);
269fd3bf775SJohn Dyson 		TAILQ_INIT(&ki->kaio_bufqueue);
27084af4da6SJohn Dyson 		TAILQ_INIT(&ki->kaio_liojoblist);
271bfbbc4aaSJason Evans 		TAILQ_INIT(&ki->kaio_sockqueue);
2722244ea07SJohn Dyson 	}
273bfbbc4aaSJason Evans 
274bfbbc4aaSJason Evans 	while (num_aio_procs < target_aio_procs)
275bfbbc4aaSJason Evans 		aio_newproc();
2762244ea07SJohn Dyson }
2772244ea07SJohn Dyson 
2782244ea07SJohn Dyson /*
279bfbbc4aaSJason Evans  * Free a job entry.  Wait for completion if it is currently active, but don't
280bfbbc4aaSJason Evans  * delay forever.  If we delay, we return a flag that says that we have to
281bfbbc4aaSJason Evans  * restart the queue scan.
2822244ea07SJohn Dyson  */
28388ed460eSAlan Cox static int
284fd3bf775SJohn Dyson aio_free_entry(struct aiocblist *aiocbe)
285fd3bf775SJohn Dyson {
2862244ea07SJohn Dyson 	struct kaioinfo *ki;
287b40ce416SJulian Elischer 	struct aiothreadlist *aiop;
28884af4da6SJohn Dyson 	struct aio_liojob *lj;
2892244ea07SJohn Dyson 	struct proc *p;
290fd3bf775SJohn Dyson 	int error;
29111783b14SJohn Dyson 	int s;
2922244ea07SJohn Dyson 
2932244ea07SJohn Dyson 	if (aiocbe->jobstate == JOBST_NULL)
2942244ea07SJohn Dyson 		panic("aio_free_entry: freeing already free job");
2952244ea07SJohn Dyson 
2962244ea07SJohn Dyson 	p = aiocbe->userproc;
2972244ea07SJohn Dyson 	ki = p->p_aioinfo;
29884af4da6SJohn Dyson 	lj = aiocbe->lio;
2992244ea07SJohn Dyson 	if (ki == NULL)
3002244ea07SJohn Dyson 		panic("aio_free_entry: missing p->p_aioinfo");
3012244ea07SJohn Dyson 
30213644654SAlan Cox 	while (aiocbe->jobstate == JOBST_JOBRUNNING) {
3032244ea07SJohn Dyson 		if (aiocbe->jobflags & AIOCBLIST_ASYNCFREE)
3042244ea07SJohn Dyson 			return 0;
3052244ea07SJohn Dyson 		aiocbe->jobflags |= AIOCBLIST_RUNDOWN;
30613644654SAlan Cox 		tsleep(aiocbe, PRIBIO, "jobwai", 0);
3072244ea07SJohn Dyson 	}
3082244ea07SJohn Dyson 	aiocbe->jobflags &= ~AIOCBLIST_ASYNCFREE;
3092244ea07SJohn Dyson 
310fd3bf775SJohn Dyson 	if (aiocbe->bp == NULL) {
3112244ea07SJohn Dyson 		if (ki->kaio_queue_count <= 0)
3122244ea07SJohn Dyson 			panic("aio_free_entry: process queue size <= 0");
3132244ea07SJohn Dyson 		if (num_queue_count <= 0)
3142244ea07SJohn Dyson 			panic("aio_free_entry: system wide queue size <= 0");
3152244ea07SJohn Dyson 
31684af4da6SJohn Dyson 		if (lj) {
31784af4da6SJohn Dyson 			lj->lioj_queue_count--;
31884af4da6SJohn Dyson 			if (aiocbe->jobflags & AIOCBLIST_DONE)
31984af4da6SJohn Dyson 				lj->lioj_queue_finished_count--;
32084af4da6SJohn Dyson 		}
32184af4da6SJohn Dyson 		ki->kaio_queue_count--;
32284af4da6SJohn Dyson 		if (aiocbe->jobflags & AIOCBLIST_DONE)
32384af4da6SJohn Dyson 			ki->kaio_queue_finished_count--;
32484af4da6SJohn Dyson 		num_queue_count--;
325fd3bf775SJohn Dyson 	} else {
32684af4da6SJohn Dyson 		if (lj) {
32784af4da6SJohn Dyson 			lj->lioj_buffer_count--;
32884af4da6SJohn Dyson 			if (aiocbe->jobflags & AIOCBLIST_DONE)
32984af4da6SJohn Dyson 				lj->lioj_buffer_finished_count--;
33084af4da6SJohn Dyson 		}
33184af4da6SJohn Dyson 		if (aiocbe->jobflags & AIOCBLIST_DONE)
33284af4da6SJohn Dyson 			ki->kaio_buffer_finished_count--;
33384af4da6SJohn Dyson 		ki->kaio_buffer_count--;
33484af4da6SJohn Dyson 		num_buf_aio--;
335fd3bf775SJohn Dyson 	}
336fd3bf775SJohn Dyson 
337cb679c38SJonathan Lemon 	/* aiocbe is going away, we need to destroy any knotes */
338b40ce416SJulian Elischer 	knote_remove(&p->p_thread, &aiocbe->klist); /* XXXKSE */
339b40ce416SJulian Elischer 	/* XXXKSE Note the thread here is used to eventually find the
340b40ce416SJulian Elischer 	 * owning process again, but it is also used to do a fo_close
341b40ce416SJulian Elischer 	 * and that requires the thread. (but does it require the
342b40ce416SJulian Elischer 	 * OWNING thread? (or maby the running thread?)
343b40ce416SJulian Elischer 	 * There is a semantic problem here...
344b40ce416SJulian Elischer 	 */
345cb679c38SJonathan Lemon 
346bfbbc4aaSJason Evans 	if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags & KAIO_RUNDOWN)
347bfbbc4aaSJason Evans 	    && ((ki->kaio_buffer_count == 0) && (ki->kaio_queue_count == 0)))) {
348fd3bf775SJohn Dyson 		ki->kaio_flags &= ~KAIO_WAKEUP;
349fd3bf775SJohn Dyson 		wakeup(p);
350fd3bf775SJohn Dyson 	}
351fd3bf775SJohn Dyson 
352fd3bf775SJohn Dyson 	if (aiocbe->jobstate == JOBST_JOBQBUF) {
3539c8a2647SAlan Cox 		if ((error = aio_fphysio(p, aiocbe)) != 0)
354fd3bf775SJohn Dyson 			return error;
355fd3bf775SJohn Dyson 		if (aiocbe->jobstate != JOBST_JOBBFINISHED)
356fd3bf775SJohn Dyson 			panic("aio_free_entry: invalid physio finish-up state");
35711783b14SJohn Dyson 		s = splbio();
358fd3bf775SJohn Dyson 		TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
35911783b14SJohn Dyson 		splx(s);
360fd3bf775SJohn Dyson 	} else if (aiocbe->jobstate == JOBST_JOBQPROC) {
361b40ce416SJulian Elischer 		aiop = aiocbe->jobaiothread;
3622244ea07SJohn Dyson 		TAILQ_REMOVE(&aiop->jobtorun, aiocbe, list);
36313644654SAlan Cox 	} else if (aiocbe->jobstate == JOBST_JOBQGLOBAL) {
3642244ea07SJohn Dyson 		TAILQ_REMOVE(&aio_jobs, aiocbe, list);
36513644654SAlan Cox 		TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
36613644654SAlan Cox 	} else if (aiocbe->jobstate == JOBST_JOBFINISHED)
3672244ea07SJohn Dyson 		TAILQ_REMOVE(&ki->kaio_jobdone, aiocbe, plist);
368bfbbc4aaSJason Evans 	else if (aiocbe->jobstate == JOBST_JOBBFINISHED) {
36911783b14SJohn Dyson 		s = splbio();
370fd3bf775SJohn Dyson 		TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist);
37111783b14SJohn Dyson 		splx(s);
37284af4da6SJohn Dyson 		if (aiocbe->bp) {
37384af4da6SJohn Dyson 			vunmapbuf(aiocbe->bp);
3741c7c3c6aSMatthew Dillon 			relpbuf(aiocbe->bp, NULL);
37584af4da6SJohn Dyson 			aiocbe->bp = NULL;
37684af4da6SJohn Dyson 		}
37784af4da6SJohn Dyson 	}
37884af4da6SJohn Dyson 	if (lj && (lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 0)) {
37984af4da6SJohn Dyson 		TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
38084af4da6SJohn Dyson 		zfree(aiolio_zone, lj);
3812244ea07SJohn Dyson 	}
3822244ea07SJohn Dyson 	aiocbe->jobstate = JOBST_NULL;
38313644654SAlan Cox 	untimeout(process_signal, aiocbe, aiocbe->timeouthandle);
38488ed460eSAlan Cox 	zfree(aiocb_zone, aiocbe);
3852244ea07SJohn Dyson 	return 0;
3862244ea07SJohn Dyson }
3875dec52baSPeter Wemm #endif /* VFS_AIO */
3882244ea07SJohn Dyson 
3892244ea07SJohn Dyson /*
3902244ea07SJohn Dyson  * Rundown the jobs for a given process.
3912244ea07SJohn Dyson  */
3922244ea07SJohn Dyson void
393fd3bf775SJohn Dyson aio_proc_rundown(struct proc *p)
394fd3bf775SJohn Dyson {
3955dec52baSPeter Wemm #ifndef VFS_AIO
3965dec52baSPeter Wemm 	return;
3975dec52baSPeter Wemm #else
39884af4da6SJohn Dyson 	int s;
3992244ea07SJohn Dyson 	struct kaioinfo *ki;
40084af4da6SJohn Dyson 	struct aio_liojob *lj, *ljn;
4012244ea07SJohn Dyson 	struct aiocblist *aiocbe, *aiocbn;
402bfbbc4aaSJason Evans 	struct file *fp;
403bfbbc4aaSJason Evans 	struct filedesc *fdp;
404bfbbc4aaSJason Evans 	struct socket *so;
4052244ea07SJohn Dyson 
4062244ea07SJohn Dyson 	ki = p->p_aioinfo;
4072244ea07SJohn Dyson 	if (ki == NULL)
4082244ea07SJohn Dyson 		return;
4092244ea07SJohn Dyson 
41084af4da6SJohn Dyson 	ki->kaio_flags |= LIOJ_SIGNAL_POSTED;
411bfbbc4aaSJason Evans 	while ((ki->kaio_active_count > 0) || (ki->kaio_buffer_count >
412bfbbc4aaSJason Evans 	    ki->kaio_buffer_finished_count)) {
413fd3bf775SJohn Dyson 		ki->kaio_flags |= KAIO_RUNDOWN;
41484af4da6SJohn Dyson 		if (tsleep(p, PRIBIO, "kaiowt", aiod_timeout))
415a624e84fSJohn Dyson 			break;
416a624e84fSJohn Dyson 	}
417a624e84fSJohn Dyson 
418bfbbc4aaSJason Evans 	/*
419bfbbc4aaSJason Evans 	 * Move any aio ops that are waiting on socket I/O to the normal job
420bfbbc4aaSJason Evans 	 * queues so they are cleaned up with any others.
421bfbbc4aaSJason Evans 	 */
422bfbbc4aaSJason Evans 	fdp = p->p_fd;
423bfbbc4aaSJason Evans 
424bfbbc4aaSJason Evans 	s = splnet();
425bfbbc4aaSJason Evans 	for (aiocbe = TAILQ_FIRST(&ki->kaio_sockqueue); aiocbe; aiocbe =
426bfbbc4aaSJason Evans 	    aiocbn) {
427bfbbc4aaSJason Evans 		aiocbn = TAILQ_NEXT(aiocbe, plist);
428bfbbc4aaSJason Evans 		fp = fdp->fd_ofiles[aiocbe->uaiocb.aio_fildes];
429bfbbc4aaSJason Evans 
430bfbbc4aaSJason Evans 		/*
431bfbbc4aaSJason Evans 		 * Under some circumstances, the aio_fildes and the file
432bfbbc4aaSJason Evans 		 * structure don't match.  This would leave aiocbe's in the
433bfbbc4aaSJason Evans 		 * TAILQ associated with the socket and cause a panic later.
434bfbbc4aaSJason Evans 		 *
435bfbbc4aaSJason Evans 		 * Detect and fix.
436bfbbc4aaSJason Evans 		 */
437bfbbc4aaSJason Evans 		if ((fp == NULL) || (fp != aiocbe->fd_file))
438bfbbc4aaSJason Evans 			fp = aiocbe->fd_file;
439bfbbc4aaSJason Evans 		if (fp) {
440bfbbc4aaSJason Evans 			so = (struct socket *)fp->f_data;
441bfbbc4aaSJason Evans 			TAILQ_REMOVE(&so->so_aiojobq, aiocbe, list);
442bfbbc4aaSJason Evans 			if (TAILQ_EMPTY(&so->so_aiojobq)) {
443bfbbc4aaSJason Evans 				so->so_snd.sb_flags &= ~SB_AIO;
444bfbbc4aaSJason Evans 				so->so_rcv.sb_flags &= ~SB_AIO;
445bfbbc4aaSJason Evans 			}
446bfbbc4aaSJason Evans 		}
447bfbbc4aaSJason Evans 		TAILQ_REMOVE(&ki->kaio_sockqueue, aiocbe, plist);
448bfbbc4aaSJason Evans 		TAILQ_INSERT_HEAD(&aio_jobs, aiocbe, list);
449bfbbc4aaSJason Evans 		TAILQ_INSERT_HEAD(&ki->kaio_jobqueue, aiocbe, plist);
450bfbbc4aaSJason Evans 	}
451bfbbc4aaSJason Evans 	splx(s);
452bfbbc4aaSJason Evans 
4532244ea07SJohn Dyson restart1:
454bfbbc4aaSJason Evans 	for (aiocbe = TAILQ_FIRST(&ki->kaio_jobdone); aiocbe; aiocbe = aiocbn) {
4552244ea07SJohn Dyson 		aiocbn = TAILQ_NEXT(aiocbe, plist);
4562244ea07SJohn Dyson 		if (aio_free_entry(aiocbe))
4572244ea07SJohn Dyson 			goto restart1;
4582244ea07SJohn Dyson 	}
4592244ea07SJohn Dyson 
4602244ea07SJohn Dyson restart2:
461bfbbc4aaSJason Evans 	for (aiocbe = TAILQ_FIRST(&ki->kaio_jobqueue); aiocbe; aiocbe =
462bfbbc4aaSJason Evans 	    aiocbn) {
4632244ea07SJohn Dyson 		aiocbn = TAILQ_NEXT(aiocbe, plist);
4642244ea07SJohn Dyson 		if (aio_free_entry(aiocbe))
4652244ea07SJohn Dyson 			goto restart2;
4662244ea07SJohn Dyson 	}
46784af4da6SJohn Dyson 
468c5efdcbdSJohn Dyson /*
469bfbbc4aaSJason Evans  * Note the use of lots of splbio here, trying to avoid splbio for long chains
470bfbbc4aaSJason Evans  * of I/O.  Probably unnecessary.
471c5efdcbdSJohn Dyson  */
47284af4da6SJohn Dyson restart3:
47384af4da6SJohn Dyson 	s = splbio();
47484af4da6SJohn Dyson 	while (TAILQ_FIRST(&ki->kaio_bufqueue)) {
47584af4da6SJohn Dyson 		ki->kaio_flags |= KAIO_WAKEUP;
47684af4da6SJohn Dyson 		tsleep(p, PRIBIO, "aioprn", 0);
47784af4da6SJohn Dyson 		splx(s);
47884af4da6SJohn Dyson 		goto restart3;
47984af4da6SJohn Dyson 	}
480c5efdcbdSJohn Dyson 	splx(s);
48184af4da6SJohn Dyson 
48284af4da6SJohn Dyson restart4:
48384af4da6SJohn Dyson 	s = splbio();
484bfbbc4aaSJason Evans 	for (aiocbe = TAILQ_FIRST(&ki->kaio_bufdone); aiocbe; aiocbe = aiocbn) {
48584af4da6SJohn Dyson 		aiocbn = TAILQ_NEXT(aiocbe, plist);
48684af4da6SJohn Dyson 		if (aio_free_entry(aiocbe)) {
48784af4da6SJohn Dyson 			splx(s);
48884af4da6SJohn Dyson 			goto restart4;
48984af4da6SJohn Dyson 		}
49084af4da6SJohn Dyson 	}
49184af4da6SJohn Dyson 	splx(s);
49284af4da6SJohn Dyson 
49313644654SAlan Cox         /*
49413644654SAlan Cox          * If we've slept, jobs might have moved from one queue to another.
49513644654SAlan Cox          * Retry rundown if we didn't manage to empty the queues.
49613644654SAlan Cox          */
49713644654SAlan Cox         if (TAILQ_FIRST(&ki->kaio_jobdone) != NULL ||
49813644654SAlan Cox 	    TAILQ_FIRST(&ki->kaio_jobqueue) != NULL ||
49913644654SAlan Cox 	    TAILQ_FIRST(&ki->kaio_bufqueue) != NULL ||
50013644654SAlan Cox 	    TAILQ_FIRST(&ki->kaio_bufdone) != NULL)
50113644654SAlan Cox 		goto restart1;
50213644654SAlan Cox 
503bfbbc4aaSJason Evans 	for (lj = TAILQ_FIRST(&ki->kaio_liojoblist); lj; lj = ljn) {
50484af4da6SJohn Dyson 		ljn = TAILQ_NEXT(lj, lioj_list);
505bfbbc4aaSJason Evans 		if ((lj->lioj_buffer_count == 0) && (lj->lioj_queue_count ==
506bfbbc4aaSJason Evans 		    0)) {
50784af4da6SJohn Dyson 			TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
50884af4da6SJohn Dyson 			zfree(aiolio_zone, lj);
509f4f0ecefSJohn Dyson 		} else {
510bfbbc4aaSJason Evans #ifdef DIAGNOSTIC
511bfbbc4aaSJason Evans 			printf("LIO job not cleaned up: B:%d, BF:%d, Q:%d, "
512bfbbc4aaSJason Evans 			    "QF:%d\n", lj->lioj_buffer_count,
513bfbbc4aaSJason Evans 			    lj->lioj_buffer_finished_count,
514bfbbc4aaSJason Evans 			    lj->lioj_queue_count,
515bfbbc4aaSJason Evans 			    lj->lioj_queue_finished_count);
51611783b14SJohn Dyson #endif
51784af4da6SJohn Dyson 		}
518f4f0ecefSJohn Dyson 	}
51984af4da6SJohn Dyson 
520fd3bf775SJohn Dyson 	zfree(kaio_zone, ki);
521a624e84fSJohn Dyson 	p->p_aioinfo = NULL;
5225dec52baSPeter Wemm #endif /* VFS_AIO */
5232244ea07SJohn Dyson }
5242244ea07SJohn Dyson 
5255dec52baSPeter Wemm #ifdef VFS_AIO
5262244ea07SJohn Dyson /*
527bfbbc4aaSJason Evans  * Select a job to run (called by an AIO daemon).
5282244ea07SJohn Dyson  */
5292244ea07SJohn Dyson static struct aiocblist *
530b40ce416SJulian Elischer aio_selectjob(struct aiothreadlist *aiop)
531fd3bf775SJohn Dyson {
532bfbbc4aaSJason Evans 	int s;
5332244ea07SJohn Dyson 	struct aiocblist *aiocbe;
534bfbbc4aaSJason Evans 	struct kaioinfo *ki;
535bfbbc4aaSJason Evans 	struct proc *userp;
5362244ea07SJohn Dyson 
5372244ea07SJohn Dyson 	aiocbe = TAILQ_FIRST(&aiop->jobtorun);
5382244ea07SJohn Dyson 	if (aiocbe) {
5392244ea07SJohn Dyson 		TAILQ_REMOVE(&aiop->jobtorun, aiocbe, list);
5402244ea07SJohn Dyson 		return aiocbe;
5412244ea07SJohn Dyson 	}
5422244ea07SJohn Dyson 
543bfbbc4aaSJason Evans 	s = splnet();
544bfbbc4aaSJason Evans 	for (aiocbe = TAILQ_FIRST(&aio_jobs); aiocbe; aiocbe =
545bfbbc4aaSJason Evans 	    TAILQ_NEXT(aiocbe, list)) {
5462244ea07SJohn Dyson 		userp = aiocbe->userproc;
5472244ea07SJohn Dyson 		ki = userp->p_aioinfo;
5482244ea07SJohn Dyson 
5492244ea07SJohn Dyson 		if (ki->kaio_active_count < ki->kaio_maxactive_count) {
5502244ea07SJohn Dyson 			TAILQ_REMOVE(&aio_jobs, aiocbe, list);
551bfbbc4aaSJason Evans 			splx(s);
5522244ea07SJohn Dyson 			return aiocbe;
5532244ea07SJohn Dyson 		}
5542244ea07SJohn Dyson 	}
555bfbbc4aaSJason Evans 	splx(s);
5562244ea07SJohn Dyson 
5572244ea07SJohn Dyson 	return NULL;
5582244ea07SJohn Dyson }
5592244ea07SJohn Dyson 
5602244ea07SJohn Dyson /*
561bfbbc4aaSJason Evans  * The AIO processing activity.  This is the code that does the I/O request for
562bfbbc4aaSJason Evans  * the non-physio version of the operations.  The normal vn operations are used,
563bfbbc4aaSJason Evans  * and this code should work in all instances for every type of file, including
564bfbbc4aaSJason Evans  * pipes, sockets, fifos, and regular files.
5652244ea07SJohn Dyson  */
56688ed460eSAlan Cox static void
567fd3bf775SJohn Dyson aio_process(struct aiocblist *aiocbe)
568fd3bf775SJohn Dyson {
5692244ea07SJohn Dyson 	struct filedesc *fdp;
570b40ce416SJulian Elischer 	struct thread *td;
571b40ce416SJulian Elischer 	struct proc *userp;
572b40ce416SJulian Elischer 	struct proc *mycp;
5732244ea07SJohn Dyson 	struct aiocb *cb;
5742244ea07SJohn Dyson 	struct file *fp;
5752244ea07SJohn Dyson 	struct uio auio;
5762244ea07SJohn Dyson 	struct iovec aiov;
5772244ea07SJohn Dyson 	unsigned int fd;
5782244ea07SJohn Dyson 	int cnt;
5792244ea07SJohn Dyson 	int error;
580a624e84fSJohn Dyson 	off_t offset;
581fd3bf775SJohn Dyson 	int oublock_st, oublock_end;
582fd3bf775SJohn Dyson 	int inblock_st, inblock_end;
5832244ea07SJohn Dyson 
5842244ea07SJohn Dyson 	userp = aiocbe->userproc;
585b40ce416SJulian Elischer 	td = curthread;
586b40ce416SJulian Elischer 	mycp = td->td_proc;
5872244ea07SJohn Dyson 	cb = &aiocbe->uaiocb;
5882244ea07SJohn Dyson 
589fd3bf775SJohn Dyson 	fdp = mycp->p_fd;
5902244ea07SJohn Dyson 	fd = cb->aio_fildes;
5912244ea07SJohn Dyson 	fp = fdp->fd_ofiles[fd];
5922244ea07SJohn Dyson 
593bfbbc4aaSJason Evans 	if ((fp == NULL) || (fp != aiocbe->fd_file)) {
594bfbbc4aaSJason Evans 		cb->_aiocb_private.error = EBADF;
595bfbbc4aaSJason Evans 		cb->_aiocb_private.status = -1;
596bfbbc4aaSJason Evans 		return;
597bfbbc4aaSJason Evans 	}
598bfbbc4aaSJason Evans 
599b40ce416SJulian Elischer 	aiov.iov_base = cb->aio_buf;
6002244ea07SJohn Dyson 	aiov.iov_len = cb->aio_nbytes;
6012244ea07SJohn Dyson 
6022244ea07SJohn Dyson 	auio.uio_iov = &aiov;
6032244ea07SJohn Dyson 	auio.uio_iovcnt = 1;
604a624e84fSJohn Dyson 	auio.uio_offset = offset = cb->aio_offset;
6052244ea07SJohn Dyson 	auio.uio_resid = cb->aio_nbytes;
6062244ea07SJohn Dyson 	cnt = cb->aio_nbytes;
6072244ea07SJohn Dyson 	auio.uio_segflg = UIO_USERSPACE;
608b40ce416SJulian Elischer 	auio.uio_td = td;
6092244ea07SJohn Dyson 
610fd3bf775SJohn Dyson 	inblock_st = mycp->p_stats->p_ru.ru_inblock;
611fd3bf775SJohn Dyson 	oublock_st = mycp->p_stats->p_ru.ru_oublock;
612279d7226SMatthew Dillon 	/*
613279d7226SMatthew Dillon 	 * Temporarily bump the ref count while reading to avoid the
614279d7226SMatthew Dillon 	 * descriptor being ripped out from under us.
615279d7226SMatthew Dillon 	 */
616279d7226SMatthew Dillon 	fhold(fp);
6172244ea07SJohn Dyson 	if (cb->aio_lio_opcode == LIO_READ) {
6182244ea07SJohn Dyson 		auio.uio_rw = UIO_READ;
619b40ce416SJulian Elischer 		error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td);
6202244ea07SJohn Dyson 	} else {
6212244ea07SJohn Dyson 		auio.uio_rw = UIO_WRITE;
622b40ce416SJulian Elischer 		error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td);
6232244ea07SJohn Dyson 	}
624b40ce416SJulian Elischer 	fdrop(fp, td);
625fd3bf775SJohn Dyson 	inblock_end = mycp->p_stats->p_ru.ru_inblock;
626fd3bf775SJohn Dyson 	oublock_end = mycp->p_stats->p_ru.ru_oublock;
627fd3bf775SJohn Dyson 
628fd3bf775SJohn Dyson 	aiocbe->inputcharge = inblock_end - inblock_st;
629fd3bf775SJohn Dyson 	aiocbe->outputcharge = oublock_end - oublock_st;
6302244ea07SJohn Dyson 
631bfbbc4aaSJason Evans 	if ((error) && (auio.uio_resid != cnt)) {
6322244ea07SJohn Dyson 		if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
6332244ea07SJohn Dyson 			error = 0;
63419eb87d2SJohn Baldwin 		if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) {
63519eb87d2SJohn Baldwin 			PROC_LOCK(userp);
6362244ea07SJohn Dyson 			psignal(userp, SIGPIPE);
63719eb87d2SJohn Baldwin 			PROC_UNLOCK(userp);
63819eb87d2SJohn Baldwin 		}
6392244ea07SJohn Dyson 	}
6402244ea07SJohn Dyson 
6412244ea07SJohn Dyson 	cnt -= auio.uio_resid;
6422244ea07SJohn Dyson 	cb->_aiocb_private.error = error;
6432244ea07SJohn Dyson 	cb->_aiocb_private.status = cnt;
6442244ea07SJohn Dyson }
6452244ea07SJohn Dyson 
6462244ea07SJohn Dyson /*
64784af4da6SJohn Dyson  * The AIO daemon, most of the actual work is done in aio_process,
64884af4da6SJohn Dyson  * but the setup (and address space mgmt) is done in this routine.
6492244ea07SJohn Dyson  */
6502244ea07SJohn Dyson static void
6519c8b8baaSPeter Wemm aio_daemon(void *uproc)
6522244ea07SJohn Dyson {
65384af4da6SJohn Dyson 	int s;
654bfbbc4aaSJason Evans 	struct aio_liojob *lj;
655bfbbc4aaSJason Evans 	struct aiocb *cb;
656bfbbc4aaSJason Evans 	struct aiocblist *aiocbe;
657b40ce416SJulian Elischer 	struct aiothreadlist *aiop;
658bfbbc4aaSJason Evans 	struct kaioinfo *ki;
659bfbbc4aaSJason Evans 	struct proc *curcp, *mycp, *userp;
660bfbbc4aaSJason Evans 	struct vmspace *myvm, *tmpvm;
661b40ce416SJulian Elischer 	struct thread *td = curthread;
6622244ea07SJohn Dyson 
6639ed346baSBosko Milekic 	mtx_lock(&Giant);
6642244ea07SJohn Dyson 	/*
665fd3bf775SJohn Dyson 	 * Local copies of curproc (cp) and vmspace (myvm)
6662244ea07SJohn Dyson 	 */
667b40ce416SJulian Elischer 	mycp = td->td_proc;
668fd3bf775SJohn Dyson 	myvm = mycp->p_vmspace;
669fd3bf775SJohn Dyson 
670fd3bf775SJohn Dyson 	if (mycp->p_textvp) {
671fd3bf775SJohn Dyson 		vrele(mycp->p_textvp);
672fd3bf775SJohn Dyson 		mycp->p_textvp = NULL;
673fd3bf775SJohn Dyson 	}
674fd3bf775SJohn Dyson 
675fd3bf775SJohn Dyson 	/*
676bfbbc4aaSJason Evans 	 * Allocate and ready the aio control info.  There is one aiop structure
677bfbbc4aaSJason Evans 	 * per daemon.
678fd3bf775SJohn Dyson 	 */
679fd3bf775SJohn Dyson 	aiop = zalloc(aiop_zone);
680b40ce416SJulian Elischer 	aiop->aiothread = td;
681b40ce416SJulian Elischer 	aiop->aiothreadflags |= AIOP_FREE;
6822244ea07SJohn Dyson 	TAILQ_INIT(&aiop->jobtorun);
6832244ea07SJohn Dyson 
684bfbbc4aaSJason Evans 	s = splnet();
685bfbbc4aaSJason Evans 
6862244ea07SJohn Dyson 	/*
687bfbbc4aaSJason Evans 	 * Place thread (lightweight process) onto the AIO free thread list.
6882244ea07SJohn Dyson 	 */
689fd3bf775SJohn Dyson 	if (TAILQ_EMPTY(&aio_freeproc))
690fd3bf775SJohn Dyson 		wakeup(&aio_freeproc);
691fd3bf775SJohn Dyson 	TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
6922244ea07SJohn Dyson 
693bfbbc4aaSJason Evans 	splx(s);
694bfbbc4aaSJason Evans 
6952244ea07SJohn Dyson 	/*
696fd3bf775SJohn Dyson 	 * Get rid of our current filedescriptors.  AIOD's don't need any
697fd3bf775SJohn Dyson 	 * filedescriptors, except as temporarily inherited from the client.
698bfbbc4aaSJason Evans 	 * Credentials are also cloned, and made equivalent to "root".
6992244ea07SJohn Dyson 	 */
700b40ce416SJulian Elischer 	fdfree(td);
701fd3bf775SJohn Dyson 	mycp->p_fd = NULL;
702fd3bf775SJohn Dyson 	mycp->p_ucred = crcopy(mycp->p_ucred);
703fd3bf775SJohn Dyson 	mycp->p_ucred->cr_uid = 0;
704f535380cSDon Lewis 	uifree(mycp->p_ucred->cr_uidinfo);
705f535380cSDon Lewis 	mycp->p_ucred->cr_uidinfo = uifind(0);
706fd3bf775SJohn Dyson 	mycp->p_ucred->cr_ngroups = 1;
707fd3bf775SJohn Dyson 	mycp->p_ucred->cr_groups[0] = 1;
708fd3bf775SJohn Dyson 
709bfbbc4aaSJason Evans 	/* The daemon resides in its own pgrp. */
710fd3bf775SJohn Dyson 	enterpgrp(mycp, mycp->p_pid, 1);
711fd3bf775SJohn Dyson 
712bfbbc4aaSJason Evans 	/* Mark special process type. */
71386360feeSJake Burkholder 	mycp->p_flag |= P_SYSTEM;
7142244ea07SJohn Dyson 
715fd3bf775SJohn Dyson 	/*
716fd3bf775SJohn Dyson 	 * Wakeup parent process.  (Parent sleeps to keep from blasting away
717b40ce416SJulian Elischer 	 * and creating too many daemons.)
718fd3bf775SJohn Dyson 	 */
719fd3bf775SJohn Dyson 	wakeup(mycp);
7202244ea07SJohn Dyson 
721bfbbc4aaSJason Evans 	for (;;) {
722fd3bf775SJohn Dyson 		/*
723fd3bf775SJohn Dyson 		 * curcp is the current daemon process context.
724fd3bf775SJohn Dyson 		 * userp is the current user process context.
725fd3bf775SJohn Dyson 		 */
726fd3bf775SJohn Dyson 		curcp = mycp;
727c4860686SJohn Dyson 
728fd3bf775SJohn Dyson 		/*
729fd3bf775SJohn Dyson 		 * Take daemon off of free queue
730fd3bf775SJohn Dyson 		 */
731b40ce416SJulian Elischer 		if (aiop->aiothreadflags & AIOP_FREE) {
732bfbbc4aaSJason Evans 			s = splnet();
7332244ea07SJohn Dyson 			TAILQ_REMOVE(&aio_freeproc, aiop, list);
7342244ea07SJohn Dyson 			TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list);
735b40ce416SJulian Elischer 			aiop->aiothreadflags &= ~AIOP_FREE;
736bfbbc4aaSJason Evans 			splx(s);
7372244ea07SJohn Dyson 		}
738b40ce416SJulian Elischer 		aiop->aiothreadflags &= ~AIOP_SCHED;
7392244ea07SJohn Dyson 
740fd3bf775SJohn Dyson 		/*
741bfbbc4aaSJason Evans 		 * Check for jobs.
742fd3bf775SJohn Dyson 		 */
743d254af07SMatthew Dillon 		while ((aiocbe = aio_selectjob(aiop)) != NULL) {
7442244ea07SJohn Dyson 			cb = &aiocbe->uaiocb;
7452244ea07SJohn Dyson 			userp = aiocbe->userproc;
7462244ea07SJohn Dyson 
7472244ea07SJohn Dyson 			aiocbe->jobstate = JOBST_JOBRUNNING;
748fd3bf775SJohn Dyson 
749fd3bf775SJohn Dyson 			/*
750bfbbc4aaSJason Evans 			 * Connect to process address space for user program.
751fd3bf775SJohn Dyson 			 */
752fd3bf775SJohn Dyson 			if (userp != curcp) {
753fd3bf775SJohn Dyson 				/*
754bfbbc4aaSJason Evans 				 * Save the current address space that we are
755bfbbc4aaSJason Evans 				 * connected to.
756fd3bf775SJohn Dyson 				 */
757fd3bf775SJohn Dyson 				tmpvm = mycp->p_vmspace;
758bfbbc4aaSJason Evans 
759fd3bf775SJohn Dyson 				/*
760bfbbc4aaSJason Evans 				 * Point to the new user address space, and
761bfbbc4aaSJason Evans 				 * refer to it.
762fd3bf775SJohn Dyson 				 */
763fd3bf775SJohn Dyson 				mycp->p_vmspace = userp->p_vmspace;
76484af4da6SJohn Dyson 				mycp->p_vmspace->vm_refcnt++;
765bfbbc4aaSJason Evans 
766bfbbc4aaSJason Evans 				/* Activate the new mapping. */
767b40ce416SJulian Elischer 				pmap_activate(&mycp->p_thread);
768bfbbc4aaSJason Evans 
769fd3bf775SJohn Dyson 				/*
770bfbbc4aaSJason Evans 				 * If the old address space wasn't the daemons
771bfbbc4aaSJason Evans 				 * own address space, then we need to remove the
772bfbbc4aaSJason Evans 				 * daemon's reference from the other process
773bfbbc4aaSJason Evans 				 * that it was acting on behalf of.
774fd3bf775SJohn Dyson 				 */
7752244ea07SJohn Dyson 				if (tmpvm != myvm) {
7762244ea07SJohn Dyson 					vmspace_free(tmpvm);
7772244ea07SJohn Dyson 				}
778bfbbc4aaSJason Evans 
779fd3bf775SJohn Dyson 				/*
780bfbbc4aaSJason Evans 				 * Disassociate from previous clients file
781bfbbc4aaSJason Evans 				 * descriptors, and associate to the new clients
782bfbbc4aaSJason Evans 				 * descriptors.  Note that the daemon doesn't
783bfbbc4aaSJason Evans 				 * need to worry about its orginal descriptors,
784bfbbc4aaSJason Evans 				 * because they were originally freed.
785fd3bf775SJohn Dyson 				 */
786fd3bf775SJohn Dyson 				if (mycp->p_fd)
787b40ce416SJulian Elischer 					fdfree(td);
788fd3bf775SJohn Dyson 				mycp->p_fd = fdshare(userp);
789fd3bf775SJohn Dyson 				curcp = userp;
7902244ea07SJohn Dyson 			}
7912244ea07SJohn Dyson 
792fd3bf775SJohn Dyson 			ki = userp->p_aioinfo;
79384af4da6SJohn Dyson 			lj = aiocbe->lio;
79484af4da6SJohn Dyson 
795bfbbc4aaSJason Evans 			/* Account for currently active jobs. */
7962244ea07SJohn Dyson 			ki->kaio_active_count++;
79711783b14SJohn Dyson 
798bfbbc4aaSJason Evans 			/* Do the I/O function. */
799b40ce416SJulian Elischer 			aiocbe->jobaiothread = aiop;
8002244ea07SJohn Dyson 			aio_process(aiocbe);
80184af4da6SJohn Dyson 
802bfbbc4aaSJason Evans 			/* Decrement the active job count. */
80384af4da6SJohn Dyson 			ki->kaio_active_count--;
80484af4da6SJohn Dyson 
80584af4da6SJohn Dyson 			/*
806bfbbc4aaSJason Evans 			 * Increment the completion count for wakeup/signal
807bfbbc4aaSJason Evans 			 * comparisons.
80884af4da6SJohn Dyson 			 */
80984af4da6SJohn Dyson 			aiocbe->jobflags |= AIOCBLIST_DONE;
81084af4da6SJohn Dyson 			ki->kaio_queue_finished_count++;
811bfbbc4aaSJason Evans 			if (lj)
81284af4da6SJohn Dyson 				lj->lioj_queue_finished_count++;
813bfbbc4aaSJason Evans 			if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags
814bfbbc4aaSJason Evans 			    & KAIO_RUNDOWN) && (ki->kaio_active_count == 0))) {
815fd3bf775SJohn Dyson 				ki->kaio_flags &= ~KAIO_WAKEUP;
816fd3bf775SJohn Dyson 				wakeup(userp);
817fd3bf775SJohn Dyson 			}
8182244ea07SJohn Dyson 
81984af4da6SJohn Dyson 			s = splbio();
820bfbbc4aaSJason Evans 			if (lj && (lj->lioj_flags &
821bfbbc4aaSJason Evans 			    (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == LIOJ_SIGNAL) {
822bfbbc4aaSJason Evans 				if ((lj->lioj_queue_finished_count ==
823bfbbc4aaSJason Evans 				    lj->lioj_queue_count) &&
824bfbbc4aaSJason Evans 				    (lj->lioj_buffer_finished_count ==
825bfbbc4aaSJason Evans 				    lj->lioj_buffer_count)) {
82619eb87d2SJohn Baldwin 					PROC_LOCK(userp);
827bfbbc4aaSJason Evans 					psignal(userp,
828bfbbc4aaSJason Evans 					    lj->lioj_signal.sigev_signo);
82919eb87d2SJohn Baldwin 					PROC_UNLOCK(userp);
83019eb87d2SJohn Baldwin 					lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
83184af4da6SJohn Dyson 				}
83284af4da6SJohn Dyson 			}
83384af4da6SJohn Dyson 			splx(s);
83484af4da6SJohn Dyson 
8352244ea07SJohn Dyson 			aiocbe->jobstate = JOBST_JOBFINISHED;
8362244ea07SJohn Dyson 
837fd3bf775SJohn Dyson 			/*
838bfbbc4aaSJason Evans 			 * If the I/O request should be automatically rundown,
839bfbbc4aaSJason Evans 			 * do the needed cleanup.  Otherwise, place the queue
840bfbbc4aaSJason Evans 			 * entry for the just finished I/O request into the done
841bfbbc4aaSJason Evans 			 * queue for the associated client.
842fd3bf775SJohn Dyson 			 */
843bfbbc4aaSJason Evans 			s = splnet();
8442244ea07SJohn Dyson 			if (aiocbe->jobflags & AIOCBLIST_ASYNCFREE) {
8452244ea07SJohn Dyson 				aiocbe->jobflags &= ~AIOCBLIST_ASYNCFREE;
84688ed460eSAlan Cox 				zfree(aiocb_zone, aiocbe);
8472244ea07SJohn Dyson 			} else {
848bfbbc4aaSJason Evans 				TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
849bfbbc4aaSJason Evans 				TAILQ_INSERT_TAIL(&ki->kaio_jobdone, aiocbe,
850bfbbc4aaSJason Evans 				    plist);
8512244ea07SJohn Dyson 			}
852bfbbc4aaSJason Evans 			splx(s);
853cb679c38SJonathan Lemon 			KNOTE(&aiocbe->klist, 0);
8542244ea07SJohn Dyson 
8552244ea07SJohn Dyson 			if (aiocbe->jobflags & AIOCBLIST_RUNDOWN) {
8562244ea07SJohn Dyson 				wakeup(aiocbe);
8572244ea07SJohn Dyson 				aiocbe->jobflags &= ~AIOCBLIST_RUNDOWN;
8582244ea07SJohn Dyson 			}
8592244ea07SJohn Dyson 
8602244ea07SJohn Dyson 			if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
86119eb87d2SJohn Baldwin 				PROC_LOCK(userp);
8622244ea07SJohn Dyson 				psignal(userp, cb->aio_sigevent.sigev_signo);
86319eb87d2SJohn Baldwin 				PROC_UNLOCK(userp);
8642244ea07SJohn Dyson 			}
8652244ea07SJohn Dyson 		}
8662244ea07SJohn Dyson 
867fd3bf775SJohn Dyson 		/*
868bfbbc4aaSJason Evans 		 * Disconnect from user address space.
869fd3bf775SJohn Dyson 		 */
870fd3bf775SJohn Dyson 		if (curcp != mycp) {
871bfbbc4aaSJason Evans 			/* Get the user address space to disconnect from. */
872fd3bf775SJohn Dyson 			tmpvm = mycp->p_vmspace;
873bfbbc4aaSJason Evans 
874bfbbc4aaSJason Evans 			/* Get original address space for daemon. */
875fd3bf775SJohn Dyson 			mycp->p_vmspace = myvm;
876bfbbc4aaSJason Evans 
877bfbbc4aaSJason Evans 			/* Activate the daemon's address space. */
878b40ce416SJulian Elischer 			pmap_activate(&mycp->p_thread);
879bfbbc4aaSJason Evans #ifdef DIAGNOSTIC
880bfbbc4aaSJason Evans 			if (tmpvm == myvm) {
881bfbbc4aaSJason Evans 				printf("AIOD: vmspace problem -- %d\n",
882bfbbc4aaSJason Evans 				    mycp->p_pid);
883bfbbc4aaSJason Evans 			}
88411783b14SJohn Dyson #endif
885bfbbc4aaSJason Evans 			/* Remove our vmspace reference. */
8862244ea07SJohn Dyson 			vmspace_free(tmpvm);
887bfbbc4aaSJason Evans 
888fd3bf775SJohn Dyson 			/*
889bfbbc4aaSJason Evans 			 * Disassociate from the user process's file
890bfbbc4aaSJason Evans 			 * descriptors.
891fd3bf775SJohn Dyson 			 */
892fd3bf775SJohn Dyson 			if (mycp->p_fd)
893b40ce416SJulian Elischer 				fdfree(td);
894fd3bf775SJohn Dyson 			mycp->p_fd = NULL;
895fd3bf775SJohn Dyson 			curcp = mycp;
896fd3bf775SJohn Dyson 		}
897fd3bf775SJohn Dyson 
898fd3bf775SJohn Dyson 		/*
899fd3bf775SJohn Dyson 		 * If we are the first to be put onto the free queue, wakeup
900fd3bf775SJohn Dyson 		 * anyone waiting for a daemon.
901fd3bf775SJohn Dyson 		 */
902bfbbc4aaSJason Evans 		s = splnet();
903fd3bf775SJohn Dyson 		TAILQ_REMOVE(&aio_activeproc, aiop, list);
904fd3bf775SJohn Dyson 		if (TAILQ_EMPTY(&aio_freeproc))
905fd3bf775SJohn Dyson 			wakeup(&aio_freeproc);
906fd3bf775SJohn Dyson 		TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
907b40ce416SJulian Elischer 		aiop->aiothreadflags |= AIOP_FREE;
908bfbbc4aaSJason Evans 		splx(s);
909fd3bf775SJohn Dyson 
910fd3bf775SJohn Dyson 		/*
911bfbbc4aaSJason Evans 		 * If daemon is inactive for a long time, allow it to exit,
912bfbbc4aaSJason Evans 		 * thereby freeing resources.
913fd3bf775SJohn Dyson 		 */
914b40ce416SJulian Elischer 		if (((aiop->aiothreadflags & AIOP_SCHED) == 0) && tsleep(mycp,
915bfbbc4aaSJason Evans 		    PRIBIO, "aiordy", aiod_lifetime)) {
916bfbbc4aaSJason Evans 			s = splnet();
917fd3bf775SJohn Dyson 			if ((TAILQ_FIRST(&aio_jobs) == NULL) &&
918fd3bf775SJohn Dyson 			    (TAILQ_FIRST(&aiop->jobtorun) == NULL)) {
919b40ce416SJulian Elischer 				if ((aiop->aiothreadflags & AIOP_FREE) &&
92084af4da6SJohn Dyson 				    (num_aio_procs > target_aio_procs)) {
921fd3bf775SJohn Dyson 					TAILQ_REMOVE(&aio_freeproc, aiop, list);
922bfbbc4aaSJason Evans 					splx(s);
923fd3bf775SJohn Dyson 					zfree(aiop_zone, aiop);
92484af4da6SJohn Dyson 					num_aio_procs--;
925bfbbc4aaSJason Evans #ifdef DIAGNOSTIC
926bfbbc4aaSJason Evans 					if (mycp->p_vmspace->vm_refcnt <= 1) {
927bfbbc4aaSJason Evans 						printf("AIOD: bad vm refcnt for"
928bfbbc4aaSJason Evans 						    " exiting daemon: %d\n",
929fd3bf775SJohn Dyson 						    mycp->p_vmspace->vm_refcnt);
930bfbbc4aaSJason Evans 					}
93111783b14SJohn Dyson #endif
932c9a970a7SAlan Cox 					kthread_exit(0);
933fd3bf775SJohn Dyson 				}
934fd3bf775SJohn Dyson 			}
935bfbbc4aaSJason Evans 			splx(s);
9362244ea07SJohn Dyson 		}
9372244ea07SJohn Dyson 	}
9382244ea07SJohn Dyson }
9392244ea07SJohn Dyson 
9402244ea07SJohn Dyson /*
941bfbbc4aaSJason Evans  * Create a new AIO daemon.  This is mostly a kernel-thread fork routine.  The
942bfbbc4aaSJason Evans  * AIO daemon modifies its environment itself.
9432244ea07SJohn Dyson  */
9442244ea07SJohn Dyson static int
945fd3bf775SJohn Dyson aio_newproc()
946fd3bf775SJohn Dyson {
9472244ea07SJohn Dyson 	int error;
948c9a970a7SAlan Cox 	struct proc *p;
9492244ea07SJohn Dyson 
950c9a970a7SAlan Cox 	error = kthread_create(aio_daemon, curproc, &p, RFNOWAIT, "aiod%d",
951c9a970a7SAlan Cox 			       num_aio_procs);
952d5558c00SPeter Wemm 	if (error)
9532244ea07SJohn Dyson 		return error;
9542244ea07SJohn Dyson 
955fd3bf775SJohn Dyson 	/*
956bfbbc4aaSJason Evans 	 * Wait until daemon is started, but continue on just in case to
957fd3bf775SJohn Dyson 	 * handle error conditions.
958fd3bf775SJohn Dyson 	 */
959c9a970a7SAlan Cox 	error = tsleep(p, PZERO, "aiosta", aiod_timeout);
960c9a970a7SAlan Cox 
96184af4da6SJohn Dyson 	num_aio_procs++;
9622244ea07SJohn Dyson 
9632244ea07SJohn Dyson 	return error;
9642244ea07SJohn Dyson }
9652244ea07SJohn Dyson 
9662244ea07SJohn Dyson /*
96788ed460eSAlan Cox  * Try the high-performance, low-overhead physio method for eligible
96888ed460eSAlan Cox  * VCHR devices.  This method doesn't use an aio helper thread, and
96988ed460eSAlan Cox  * thus has very low overhead.
97088ed460eSAlan Cox  *
97188ed460eSAlan Cox  * Assumes that the caller, _aio_aqueue(), has incremented the file
97288ed460eSAlan Cox  * structure's reference count, preventing its deallocation for the
97388ed460eSAlan Cox  * duration of this call.
974fd3bf775SJohn Dyson  */
97588ed460eSAlan Cox static int
976bfbbc4aaSJason Evans aio_qphysio(struct proc *p, struct aiocblist *aiocbe)
977fd3bf775SJohn Dyson {
978fd3bf775SJohn Dyson 	int error;
979fd3bf775SJohn Dyson 	struct aiocb *cb;
980fd3bf775SJohn Dyson 	struct file *fp;
981fd3bf775SJohn Dyson 	struct buf *bp;
982fd3bf775SJohn Dyson 	struct vnode *vp;
983fd3bf775SJohn Dyson 	struct kaioinfo *ki;
984fd3bf775SJohn Dyson 	struct filedesc *fdp;
98584af4da6SJohn Dyson 	struct aio_liojob *lj;
986fd3bf775SJohn Dyson 	int fd;
987fd3bf775SJohn Dyson 	int s;
988b92bb032SAlan Cox 	int notify;
989fd3bf775SJohn Dyson 
99084af4da6SJohn Dyson 	cb = &aiocbe->uaiocb;
991fd3bf775SJohn Dyson 	fdp = p->p_fd;
992fd3bf775SJohn Dyson 	fd = cb->aio_fildes;
993fd3bf775SJohn Dyson 	fp = fdp->fd_ofiles[fd];
994fd3bf775SJohn Dyson 
995008626c3SPoul-Henning Kamp 	if (fp->f_type != DTYPE_VNODE)
996008626c3SPoul-Henning Kamp 		return (-1);
997fd3bf775SJohn Dyson 
998fd3bf775SJohn Dyson 	vp = (struct vnode *)fp->f_data;
99911783b14SJohn Dyson 
1000f582ac06SBrian Feldman 	/*
1001f582ac06SBrian Feldman 	 * If its not a disk, we don't want to return a positive error.
1002f582ac06SBrian Feldman 	 * It causes the aio code to not fall through to try the thread
1003f582ac06SBrian Feldman 	 * way when you're talking to a regular file.
1004f582ac06SBrian Feldman 	 */
1005f582ac06SBrian Feldman 	if (!vn_isdisk(vp, &error)) {
1006f582ac06SBrian Feldman 		if (error == ENOTBLK)
1007f582ac06SBrian Feldman 			return (-1);
1008f582ac06SBrian Feldman 		else
1009ba4ad1fcSPoul-Henning Kamp 			return (error);
1010f582ac06SBrian Feldman 	}
1011fd3bf775SJohn Dyson 
1012008626c3SPoul-Henning Kamp  	if (cb->aio_nbytes % vp->v_rdev->si_bsize_phys)
1013008626c3SPoul-Henning Kamp 		return (-1);
1014fd3bf775SJohn Dyson 
101513644654SAlan Cox 	if (cb->aio_nbytes >
101613644654SAlan Cox 	    MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK))
1017008626c3SPoul-Henning Kamp 		return (-1);
1018fd3bf775SJohn Dyson 
1019fd3bf775SJohn Dyson 	ki = p->p_aioinfo;
1020008626c3SPoul-Henning Kamp 	if (ki->kaio_buffer_count >= ki->kaio_ballowed_count)
1021008626c3SPoul-Henning Kamp 		return (-1);
1022fd3bf775SJohn Dyson 
1023fd3bf775SJohn Dyson 	ki->kaio_buffer_count++;
102411783b14SJohn Dyson 
102511783b14SJohn Dyson 	lj = aiocbe->lio;
1026bfbbc4aaSJason Evans 	if (lj)
102784af4da6SJohn Dyson 		lj->lioj_buffer_count++;
1028fd3bf775SJohn Dyson 
1029bfbbc4aaSJason Evans 	/* Create and build a buffer header for a transfer. */
10301c7c3c6aSMatthew Dillon 	bp = (struct buf *)getpbuf(NULL);
103113644654SAlan Cox 	BUF_KERNPROC(bp);
1032fd3bf775SJohn Dyson 
1033fd3bf775SJohn Dyson 	/*
1034bfbbc4aaSJason Evans 	 * Get a copy of the kva from the physical buffer.
1035fd3bf775SJohn Dyson 	 */
1036b0eeea20SPoul-Henning Kamp 	bp->b_caller1 = p;
103749ff4debSPoul-Henning Kamp 	bp->b_dev = vp->v_rdev;
1038fd3bf775SJohn Dyson 	error = bp->b_error = 0;
1039fd3bf775SJohn Dyson 
1040fd3bf775SJohn Dyson 	bp->b_bcount = cb->aio_nbytes;
1041fd3bf775SJohn Dyson 	bp->b_bufsize = cb->aio_nbytes;
104221144e3bSPoul-Henning Kamp 	bp->b_flags = B_PHYS;
1043fd3bf775SJohn Dyson 	bp->b_iodone = aio_physwakeup;
1044fd3bf775SJohn Dyson 	bp->b_saveaddr = bp->b_data;
1045b40ce416SJulian Elischer 	bp->b_data = cb->aio_buf;
1046fd3bf775SJohn Dyson 	bp->b_blkno = btodb(cb->aio_offset);
1047fd3bf775SJohn Dyson 
104802c58685SPoul-Henning Kamp 	if (cb->aio_lio_opcode == LIO_WRITE) {
104921144e3bSPoul-Henning Kamp 		bp->b_iocmd = BIO_WRITE;
105002c58685SPoul-Henning Kamp 		if (!useracc(bp->b_data, bp->b_bufsize, VM_PROT_READ)) {
1051fd3bf775SJohn Dyson 			error = EFAULT;
1052fd3bf775SJohn Dyson 			goto doerror;
1053fd3bf775SJohn Dyson 		}
105402c58685SPoul-Henning Kamp 	} else {
105521144e3bSPoul-Henning Kamp 		bp->b_iocmd = BIO_READ;
105602c58685SPoul-Henning Kamp 		if (!useracc(bp->b_data, bp->b_bufsize, VM_PROT_WRITE)) {
1057fd3bf775SJohn Dyson 			error = EFAULT;
1058fd3bf775SJohn Dyson 			goto doerror;
1059fd3bf775SJohn Dyson 		}
106002c58685SPoul-Henning Kamp 	}
1061fd3bf775SJohn Dyson 
1062bfbbc4aaSJason Evans 	/* Bring buffer into kernel space. */
1063fd3bf775SJohn Dyson 	vmapbuf(bp);
1064fd3bf775SJohn Dyson 
106584af4da6SJohn Dyson 	s = splbio();
1066fd3bf775SJohn Dyson 	aiocbe->bp = bp;
1067fd3bf775SJohn Dyson 	bp->b_spc = (void *)aiocbe;
1068fd3bf775SJohn Dyson 	TAILQ_INSERT_TAIL(&aio_bufjobs, aiocbe, list);
106984af4da6SJohn Dyson 	TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist);
1070fd3bf775SJohn Dyson 	aiocbe->jobstate = JOBST_JOBQBUF;
107184af4da6SJohn Dyson 	cb->_aiocb_private.status = cb->aio_nbytes;
107284af4da6SJohn Dyson 	num_buf_aio++;
1073fd3bf775SJohn Dyson 	bp->b_error = 0;
1074fd3bf775SJohn Dyson 
107584af4da6SJohn Dyson 	splx(s);
1076bfbbc4aaSJason Evans 
1077bfbbc4aaSJason Evans 	/* Perform transfer. */
1078b99c307aSPoul-Henning Kamp 	DEV_STRATEGY(bp, 0);
1079fd3bf775SJohn Dyson 
1080cb679c38SJonathan Lemon 	notify = 0;
108184af4da6SJohn Dyson 	s = splbio();
1082bfbbc4aaSJason Evans 
108311783b14SJohn Dyson 	/*
108411783b14SJohn Dyson 	 * If we had an error invoking the request, or an error in processing
1085bfbbc4aaSJason Evans 	 * the request before we have returned, we process it as an error in
1086bfbbc4aaSJason Evans 	 * transfer.  Note that such an I/O error is not indicated immediately,
1087bfbbc4aaSJason Evans 	 * but is returned using the aio_error mechanism.  In this case,
1088bfbbc4aaSJason Evans 	 * aio_suspend will return immediately.
108911783b14SJohn Dyson 	 */
1090c244d2deSPoul-Henning Kamp 	if (bp->b_error || (bp->b_ioflags & BIO_ERROR)) {
109111783b14SJohn Dyson 		struct aiocb *job = aiocbe->uuaiocb;
109211783b14SJohn Dyson 
109311783b14SJohn Dyson 		aiocbe->uaiocb._aiocb_private.status = 0;
109411783b14SJohn Dyson 		suword(&job->_aiocb_private.status, 0);
109511783b14SJohn Dyson 		aiocbe->uaiocb._aiocb_private.error = bp->b_error;
109611783b14SJohn Dyson 		suword(&job->_aiocb_private.error, bp->b_error);
109711783b14SJohn Dyson 
109811783b14SJohn Dyson 		ki->kaio_buffer_finished_count++;
109911783b14SJohn Dyson 
110011783b14SJohn Dyson 		if (aiocbe->jobstate != JOBST_JOBBFINISHED) {
110111783b14SJohn Dyson 			aiocbe->jobstate = JOBST_JOBBFINISHED;
110211783b14SJohn Dyson 			aiocbe->jobflags |= AIOCBLIST_DONE;
1103fd3bf775SJohn Dyson 			TAILQ_REMOVE(&aio_bufjobs, aiocbe, list);
110484af4da6SJohn Dyson 			TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
110511783b14SJohn Dyson 			TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
1106cb679c38SJonathan Lemon 			notify = 1;
110711783b14SJohn Dyson 		}
110811783b14SJohn Dyson 	}
110984af4da6SJohn Dyson 	splx(s);
1110cb679c38SJonathan Lemon 	if (notify)
1111cb679c38SJonathan Lemon 		KNOTE(&aiocbe->klist, 0);
1112fd3bf775SJohn Dyson 	return 0;
1113fd3bf775SJohn Dyson 
1114fd3bf775SJohn Dyson doerror:
1115fd3bf775SJohn Dyson 	ki->kaio_buffer_count--;
1116bfbbc4aaSJason Evans 	if (lj)
111784af4da6SJohn Dyson 		lj->lioj_buffer_count--;
111884af4da6SJohn Dyson 	aiocbe->bp = NULL;
11191c7c3c6aSMatthew Dillon 	relpbuf(bp, NULL);
1120fd3bf775SJohn Dyson 	return error;
1121fd3bf775SJohn Dyson }
1122fd3bf775SJohn Dyson 
112384af4da6SJohn Dyson /*
112484af4da6SJohn Dyson  * This waits/tests physio completion.
112584af4da6SJohn Dyson  */
112688ed460eSAlan Cox static int
11279c8a2647SAlan Cox aio_fphysio(struct proc *p, struct aiocblist *iocb)
1128fd3bf775SJohn Dyson {
1129fd3bf775SJohn Dyson 	int s;
1130fd3bf775SJohn Dyson 	struct buf *bp;
1131fd3bf775SJohn Dyson 	int error;
1132fd3bf775SJohn Dyson 
1133fd3bf775SJohn Dyson 	bp = iocb->bp;
1134fd3bf775SJohn Dyson 
1135fd3bf775SJohn Dyson 	s = splbio();
1136fd3bf775SJohn Dyson 	while ((bp->b_flags & B_DONE) == 0) {
11379c8a2647SAlan Cox 		if (tsleep(bp, PRIBIO, "physstr", aiod_timeout)) {
1138fd3bf775SJohn Dyson 			if ((bp->b_flags & B_DONE) == 0) {
1139fd3bf775SJohn Dyson 				splx(s);
1140fd3bf775SJohn Dyson 				return EINPROGRESS;
1141bfbbc4aaSJason Evans 			} else
1142fd3bf775SJohn Dyson 				break;
1143fd3bf775SJohn Dyson 		}
1144fd3bf775SJohn Dyson 	}
11459c8a2647SAlan Cox 	splx(s);
1146fd3bf775SJohn Dyson 
1147bfbbc4aaSJason Evans 	/* Release mapping into kernel space. */
1148fd3bf775SJohn Dyson 	vunmapbuf(bp);
1149fd3bf775SJohn Dyson 	iocb->bp = 0;
1150fd3bf775SJohn Dyson 
1151fd3bf775SJohn Dyson 	error = 0;
1152bfbbc4aaSJason Evans 
1153bfbbc4aaSJason Evans 	/* Check for an error. */
1154c244d2deSPoul-Henning Kamp 	if (bp->b_ioflags & BIO_ERROR)
1155fd3bf775SJohn Dyson 		error = bp->b_error;
1156fd3bf775SJohn Dyson 
11571c7c3c6aSMatthew Dillon 	relpbuf(bp, NULL);
1158fd3bf775SJohn Dyson 	return (error);
1159fd3bf775SJohn Dyson }
11605dec52baSPeter Wemm #endif /* VFS_AIO */
1161fd3bf775SJohn Dyson 
1162fd3bf775SJohn Dyson /*
1163bfbbc4aaSJason Evans  * Wake up aio requests that may be serviceable now.
1164bfbbc4aaSJason Evans  */
1165bfbbc4aaSJason Evans void
1166bfbbc4aaSJason Evans aio_swake(struct socket *so, struct sockbuf *sb)
1167bfbbc4aaSJason Evans {
11685dec52baSPeter Wemm #ifndef VFS_AIO
11695dec52baSPeter Wemm 	return;
11705dec52baSPeter Wemm #else
1171bfbbc4aaSJason Evans 	struct aiocblist *cb,*cbn;
1172bfbbc4aaSJason Evans 	struct proc *p;
1173bfbbc4aaSJason Evans 	struct kaioinfo *ki = NULL;
1174bfbbc4aaSJason Evans 	int opcode, wakecount = 0;
1175b40ce416SJulian Elischer 	struct aiothreadlist *aiop;
1176bfbbc4aaSJason Evans 
1177bfbbc4aaSJason Evans 	if (sb == &so->so_snd) {
1178bfbbc4aaSJason Evans 		opcode = LIO_WRITE;
1179bfbbc4aaSJason Evans 		so->so_snd.sb_flags &= ~SB_AIO;
1180bfbbc4aaSJason Evans 	} else {
1181bfbbc4aaSJason Evans 		opcode = LIO_READ;
1182bfbbc4aaSJason Evans 		so->so_rcv.sb_flags &= ~SB_AIO;
1183bfbbc4aaSJason Evans 	}
1184bfbbc4aaSJason Evans 
1185bfbbc4aaSJason Evans 	for (cb = TAILQ_FIRST(&so->so_aiojobq); cb; cb = cbn) {
1186bfbbc4aaSJason Evans 		cbn = TAILQ_NEXT(cb, list);
1187bfbbc4aaSJason Evans 		if (opcode == cb->uaiocb.aio_lio_opcode) {
1188bfbbc4aaSJason Evans 			p = cb->userproc;
1189bfbbc4aaSJason Evans 			ki = p->p_aioinfo;
1190bfbbc4aaSJason Evans 			TAILQ_REMOVE(&so->so_aiojobq, cb, list);
1191bfbbc4aaSJason Evans 			TAILQ_REMOVE(&ki->kaio_sockqueue, cb, plist);
1192bfbbc4aaSJason Evans 			TAILQ_INSERT_TAIL(&aio_jobs, cb, list);
1193bfbbc4aaSJason Evans 			TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, cb, plist);
1194bfbbc4aaSJason Evans 			wakecount++;
1195bfbbc4aaSJason Evans 			if (cb->jobstate != JOBST_JOBQGLOBAL)
1196bfbbc4aaSJason Evans 				panic("invalid queue value");
1197bfbbc4aaSJason Evans 		}
1198bfbbc4aaSJason Evans 	}
1199bfbbc4aaSJason Evans 
1200bfbbc4aaSJason Evans 	while (wakecount--) {
1201bfbbc4aaSJason Evans 		if ((aiop = TAILQ_FIRST(&aio_freeproc)) != 0) {
1202bfbbc4aaSJason Evans 			TAILQ_REMOVE(&aio_freeproc, aiop, list);
1203bfbbc4aaSJason Evans 			TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list);
1204b40ce416SJulian Elischer 			aiop->aiothreadflags &= ~AIOP_FREE;
1205b40ce416SJulian Elischer 			wakeup(aiop->aiothread);
1206bfbbc4aaSJason Evans 		}
1207bfbbc4aaSJason Evans 	}
12085dec52baSPeter Wemm #endif /* VFS_AIO */
1209bfbbc4aaSJason Evans }
1210bfbbc4aaSJason Evans 
12115dec52baSPeter Wemm #ifdef VFS_AIO
1212bfbbc4aaSJason Evans /*
1213bfbbc4aaSJason Evans  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
1214bfbbc4aaSJason Evans  * technique is done in this code.
12152244ea07SJohn Dyson  */
12162244ea07SJohn Dyson static int
1217b40ce416SJulian Elischer _aio_aqueue(struct thread *td, struct aiocb *job, struct aio_liojob *lj, int type)
1218fd3bf775SJohn Dyson {
1219b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
12202244ea07SJohn Dyson 	struct filedesc *fdp;
12212244ea07SJohn Dyson 	struct file *fp;
12222244ea07SJohn Dyson 	unsigned int fd;
1223bfbbc4aaSJason Evans 	struct socket *so;
1224bfbbc4aaSJason Evans 	int s;
1225b92bb032SAlan Cox 	int error;
12262244ea07SJohn Dyson 	int opcode;
12272244ea07SJohn Dyson 	struct aiocblist *aiocbe;
1228b40ce416SJulian Elischer 	struct aiothreadlist *aiop;
12292244ea07SJohn Dyson 	struct kaioinfo *ki;
1230c6fa9f78SAlan Cox 	struct kevent kev;
1231c6fa9f78SAlan Cox 	struct kqueue *kq;
1232c6fa9f78SAlan Cox 	struct file *kq_fp;
12332244ea07SJohn Dyson 
1234fd3bf775SJohn Dyson 	aiocbe = zalloc(aiocb_zone);
1235fd3bf775SJohn Dyson 	aiocbe->inputcharge = 0;
1236fd3bf775SJohn Dyson 	aiocbe->outputcharge = 0;
123713644654SAlan Cox 	callout_handle_init(&aiocbe->timeouthandle);
1238cb679c38SJonathan Lemon 	SLIST_INIT(&aiocbe->klist);
1239fd3bf775SJohn Dyson 
1240fd3bf775SJohn Dyson 	suword(&job->_aiocb_private.status, -1);
1241fd3bf775SJohn Dyson 	suword(&job->_aiocb_private.error, 0);
1242fd3bf775SJohn Dyson 	suword(&job->_aiocb_private.kernelinfo, -1);
1243fd3bf775SJohn Dyson 
124488ed460eSAlan Cox 	error = copyin(job, &aiocbe->uaiocb, sizeof(aiocbe->uaiocb));
12452244ea07SJohn Dyson 	if (error) {
1246fd3bf775SJohn Dyson 		suword(&job->_aiocb_private.error, error);
124788ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
12482244ea07SJohn Dyson 		return error;
12492244ea07SJohn Dyson 	}
12502f3cf918SAlfred Perlstein 	if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL &&
12512f3cf918SAlfred Perlstein 		!_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) {
12522f3cf918SAlfred Perlstein 		zfree(aiocb_zone, aiocbe);
12532f3cf918SAlfred Perlstein 		return EINVAL;
12542f3cf918SAlfred Perlstein 	}
12552244ea07SJohn Dyson 
1256bfbbc4aaSJason Evans 	/* Save userspace address of the job info. */
125711783b14SJohn Dyson 	aiocbe->uuaiocb = job;
125811783b14SJohn Dyson 
1259bfbbc4aaSJason Evans 	/* Get the opcode. */
1260bfbbc4aaSJason Evans 	if (type != LIO_NOP)
1261a624e84fSJohn Dyson 		aiocbe->uaiocb.aio_lio_opcode = type;
1262a624e84fSJohn Dyson 	opcode = aiocbe->uaiocb.aio_lio_opcode;
12632244ea07SJohn Dyson 
1264bfbbc4aaSJason Evans 	/* Get the fd info for process. */
12652244ea07SJohn Dyson 	fdp = p->p_fd;
12662244ea07SJohn Dyson 
12672244ea07SJohn Dyson 	/*
1268bfbbc4aaSJason Evans 	 * Range check file descriptor.
12692244ea07SJohn Dyson 	 */
12702244ea07SJohn Dyson 	fd = aiocbe->uaiocb.aio_fildes;
12712244ea07SJohn Dyson 	if (fd >= fdp->fd_nfiles) {
127288ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
1273bfbbc4aaSJason Evans 		if (type == 0)
12742244ea07SJohn Dyson 			suword(&job->_aiocb_private.error, EBADF);
12752244ea07SJohn Dyson 		return EBADF;
12762244ea07SJohn Dyson 	}
12772244ea07SJohn Dyson 
1278bfbbc4aaSJason Evans 	fp = aiocbe->fd_file = fdp->fd_ofiles[fd];
1279bfbbc4aaSJason Evans 	if ((fp == NULL) || ((opcode == LIO_WRITE) && ((fp->f_flag & FWRITE) ==
1280bfbbc4aaSJason Evans 	    0))) {
128188ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
1282bfbbc4aaSJason Evans 		if (type == 0)
12832244ea07SJohn Dyson 			suword(&job->_aiocb_private.error, EBADF);
12842244ea07SJohn Dyson 		return EBADF;
12852244ea07SJohn Dyson 	}
12862244ea07SJohn Dyson 
12872244ea07SJohn Dyson 	if (aiocbe->uaiocb.aio_offset == -1LL) {
128888ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
1289bfbbc4aaSJason Evans 		if (type == 0)
12902244ea07SJohn Dyson 			suword(&job->_aiocb_private.error, EINVAL);
12912244ea07SJohn Dyson 		return EINVAL;
12922244ea07SJohn Dyson 	}
12932244ea07SJohn Dyson 
12942244ea07SJohn Dyson 	error = suword(&job->_aiocb_private.kernelinfo, jobrefid);
12952244ea07SJohn Dyson 	if (error) {
129688ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
1297bfbbc4aaSJason Evans 		if (type == 0)
12982244ea07SJohn Dyson 			suword(&job->_aiocb_private.error, EINVAL);
12992244ea07SJohn Dyson 		return error;
13002244ea07SJohn Dyson 	}
13012244ea07SJohn Dyson 
130230166fabSBruce Evans 	aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jobrefid;
13032d2f8ae7SBruce Evans 	if (jobrefid == LONG_MAX)
1304fd3bf775SJohn Dyson 		jobrefid = 1;
13052d2f8ae7SBruce Evans 	else
13062d2f8ae7SBruce Evans 		jobrefid++;
13072244ea07SJohn Dyson 
13082244ea07SJohn Dyson 	if (opcode == LIO_NOP) {
130988ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
13102244ea07SJohn Dyson 		if (type == 0) {
13112244ea07SJohn Dyson 			suword(&job->_aiocb_private.error, 0);
1312fd3bf775SJohn Dyson 			suword(&job->_aiocb_private.status, 0);
1313fd3bf775SJohn Dyson 			suword(&job->_aiocb_private.kernelinfo, 0);
13142244ea07SJohn Dyson 		}
13152244ea07SJohn Dyson 		return 0;
13162244ea07SJohn Dyson 	}
13172244ea07SJohn Dyson 
1318fd3bf775SJohn Dyson 	if ((opcode != LIO_READ) && (opcode != LIO_WRITE)) {
131988ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
13202244ea07SJohn Dyson 		if (type == 0) {
1321fd3bf775SJohn Dyson 			suword(&job->_aiocb_private.status, 0);
13222244ea07SJohn Dyson 			suword(&job->_aiocb_private.error, EINVAL);
13232244ea07SJohn Dyson 		}
13242244ea07SJohn Dyson 		return EINVAL;
13252244ea07SJohn Dyson 	}
13262244ea07SJohn Dyson 
1327279d7226SMatthew Dillon 	fhold(fp);
1328279d7226SMatthew Dillon 
1329c6fa9f78SAlan Cox 	if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_KEVENT) {
1330c6fa9f78SAlan Cox 		kev.ident = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue;
1331c6fa9f78SAlan Cox 		kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sigval_ptr;
1332c6fa9f78SAlan Cox 	}
1333c6fa9f78SAlan Cox 	else {
1334cb679c38SJonathan Lemon 		/*
1335c6fa9f78SAlan Cox 		 * This method for requesting kevent-based notification won't
1336c6fa9f78SAlan Cox 		 * work on the alpha, since we're passing in a pointer
1337c6fa9f78SAlan Cox 		 * via aio_lio_opcode, which is an int.  Use the SIGEV_KEVENT-
1338c6fa9f78SAlan Cox 		 * based method instead.
1339cb679c38SJonathan Lemon 		 */
1340c6fa9f78SAlan Cox 		struct kevent *kevp;
1341cb679c38SJonathan Lemon 
1342cb679c38SJonathan Lemon 		kevp = (struct kevent *)job->aio_lio_opcode;
1343cb679c38SJonathan Lemon 		if (kevp == NULL)
1344cb679c38SJonathan Lemon 			goto no_kqueue;
1345cb679c38SJonathan Lemon 
134688ed460eSAlan Cox 		error = copyin(kevp, &kev, sizeof(kev));
1347cb679c38SJonathan Lemon 		if (error)
1348cb679c38SJonathan Lemon 			goto aqueue_fail;
1349c6fa9f78SAlan Cox 	}
1350cb679c38SJonathan Lemon 	if ((u_int)kev.ident >= fdp->fd_nfiles ||
135139b2b25fSAlan Cox 	    (kq_fp = fdp->fd_ofiles[kev.ident]) == NULL ||
135239b2b25fSAlan Cox 	    (kq_fp->f_type != DTYPE_KQUEUE)) {
1353cb679c38SJonathan Lemon 		error = EBADF;
1354cb679c38SJonathan Lemon 		goto aqueue_fail;
1355cb679c38SJonathan Lemon 	}
135639b2b25fSAlan Cox 	kq = (struct kqueue *)kq_fp->f_data;
1357c6fa9f78SAlan Cox 	kev.ident = (uintptr_t)aiocbe;
1358cb679c38SJonathan Lemon 	kev.filter = EVFILT_AIO;
1359cb679c38SJonathan Lemon 	kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
1360b40ce416SJulian Elischer 	error = kqueue_register(kq, &kev, td);
1361cb679c38SJonathan Lemon aqueue_fail:
1362cb679c38SJonathan Lemon 	if (error) {
136388ed460eSAlan Cox 		zfree(aiocb_zone, aiocbe);
1364cb679c38SJonathan Lemon 		if (type == 0)
1365cb679c38SJonathan Lemon 			suword(&job->_aiocb_private.error, error);
1366279d7226SMatthew Dillon 		goto done;
1367cb679c38SJonathan Lemon 	}
1368cb679c38SJonathan Lemon no_kqueue:
1369cb679c38SJonathan Lemon 
1370fd3bf775SJohn Dyson 	suword(&job->_aiocb_private.error, EINPROGRESS);
1371fd3bf775SJohn Dyson 	aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
13722244ea07SJohn Dyson 	aiocbe->userproc = p;
13732244ea07SJohn Dyson 	aiocbe->jobflags = 0;
137484af4da6SJohn Dyson 	aiocbe->lio = lj;
137584af4da6SJohn Dyson 	ki = p->p_aioinfo;
13762244ea07SJohn Dyson 
1377bfbbc4aaSJason Evans 	if (fp->f_type == DTYPE_SOCKET) {
1378bfbbc4aaSJason Evans 		/*
1379bfbbc4aaSJason Evans 		 * Alternate queueing for socket ops: Reach down into the
1380bfbbc4aaSJason Evans 		 * descriptor to get the socket data.  Then check to see if the
1381bfbbc4aaSJason Evans 		 * socket is ready to be read or written (based on the requested
1382bfbbc4aaSJason Evans 		 * operation).
1383bfbbc4aaSJason Evans 		 *
1384bfbbc4aaSJason Evans 		 * If it is not ready for io, then queue the aiocbe on the
1385bfbbc4aaSJason Evans 		 * socket, and set the flags so we get a call when sbnotify()
1386bfbbc4aaSJason Evans 		 * happens.
1387bfbbc4aaSJason Evans 		 */
1388bfbbc4aaSJason Evans 		so = (struct socket *)fp->f_data;
1389bfbbc4aaSJason Evans 		s = splnet();
1390bfbbc4aaSJason Evans 		if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode ==
1391bfbbc4aaSJason Evans 		    LIO_WRITE) && (!sowriteable(so)))) {
1392bfbbc4aaSJason Evans 			TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list);
1393bfbbc4aaSJason Evans 			TAILQ_INSERT_TAIL(&ki->kaio_sockqueue, aiocbe, plist);
1394bfbbc4aaSJason Evans 			if (opcode == LIO_READ)
1395bfbbc4aaSJason Evans 				so->so_rcv.sb_flags |= SB_AIO;
1396bfbbc4aaSJason Evans 			else
1397bfbbc4aaSJason Evans 				so->so_snd.sb_flags |= SB_AIO;
1398bfbbc4aaSJason Evans 			aiocbe->jobstate = JOBST_JOBQGLOBAL; /* XXX */
1399bfbbc4aaSJason Evans 			ki->kaio_queue_count++;
1400bfbbc4aaSJason Evans 			num_queue_count++;
1401bfbbc4aaSJason Evans 			splx(s);
1402279d7226SMatthew Dillon 			error = 0;
1403279d7226SMatthew Dillon 			goto done;
1404bfbbc4aaSJason Evans 		}
1405bfbbc4aaSJason Evans 		splx(s);
1406bfbbc4aaSJason Evans 	}
1407bfbbc4aaSJason Evans 
1408bfbbc4aaSJason Evans 	if ((error = aio_qphysio(p, aiocbe)) == 0)
1409279d7226SMatthew Dillon 		goto done;
1410279d7226SMatthew Dillon 	if (error > 0) {
1411fd3bf775SJohn Dyson 		suword(&job->_aiocb_private.status, 0);
1412fd3bf775SJohn Dyson 		aiocbe->uaiocb._aiocb_private.error = error;
1413fd3bf775SJohn Dyson 		suword(&job->_aiocb_private.error, error);
1414279d7226SMatthew Dillon 		goto done;
1415fd3bf775SJohn Dyson 	}
1416fd3bf775SJohn Dyson 
1417bfbbc4aaSJason Evans 	/* No buffer for daemon I/O. */
141884af4da6SJohn Dyson 	aiocbe->bp = NULL;
141984af4da6SJohn Dyson 
142084af4da6SJohn Dyson 	ki->kaio_queue_count++;
1421bfbbc4aaSJason Evans 	if (lj)
142284af4da6SJohn Dyson 		lj->lioj_queue_count++;
1423bfbbc4aaSJason Evans 	s = splnet();
1424fd3bf775SJohn Dyson 	TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
1425fd3bf775SJohn Dyson 	TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list);
1426bfbbc4aaSJason Evans 	splx(s);
1427fd3bf775SJohn Dyson 	aiocbe->jobstate = JOBST_JOBQGLOBAL;
1428fd3bf775SJohn Dyson 
142984af4da6SJohn Dyson 	num_queue_count++;
1430fd3bf775SJohn Dyson 	error = 0;
1431fd3bf775SJohn Dyson 
1432fd3bf775SJohn Dyson 	/*
1433bfbbc4aaSJason Evans 	 * If we don't have a free AIO process, and we are below our quota, then
1434bfbbc4aaSJason Evans 	 * start one.  Otherwise, depend on the subsequent I/O completions to
1435bfbbc4aaSJason Evans 	 * pick-up this job.  If we don't sucessfully create the new process
1436bfbbc4aaSJason Evans 	 * (thread) due to resource issues, we return an error for now (EAGAIN),
1437bfbbc4aaSJason Evans 	 * which is likely not the correct thing to do.
1438fd3bf775SJohn Dyson 	 */
14392244ea07SJohn Dyson retryproc:
1440bfbbc4aaSJason Evans 	s = splnet();
1441d254af07SMatthew Dillon 	if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
14422244ea07SJohn Dyson 		TAILQ_REMOVE(&aio_freeproc, aiop, list);
14432244ea07SJohn Dyson 		TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list);
1444b40ce416SJulian Elischer 		aiop->aiothreadflags &= ~AIOP_FREE;
1445b40ce416SJulian Elischer 		wakeup(aiop->aiothread);
1446fd3bf775SJohn Dyson 	} else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
1447fd3bf775SJohn Dyson 	    ((ki->kaio_active_count + num_aio_resv_start) <
1448fd3bf775SJohn Dyson 	    ki->kaio_maxactive_count)) {
1449fd3bf775SJohn Dyson 		num_aio_resv_start++;
1450fd3bf775SJohn Dyson 		if ((error = aio_newproc()) == 0) {
145184af4da6SJohn Dyson 			num_aio_resv_start--;
1452b40ce416SJulian Elischer 			td->td_retval[0] = 0;
14532244ea07SJohn Dyson 			goto retryproc;
1454fd3bf775SJohn Dyson 		}
145584af4da6SJohn Dyson 		num_aio_resv_start--;
1456fd3bf775SJohn Dyson 	}
1457bfbbc4aaSJason Evans 	splx(s);
1458279d7226SMatthew Dillon done:
1459b40ce416SJulian Elischer 	fdrop(fp, td);
1460fd3bf775SJohn Dyson 	return error;
14612244ea07SJohn Dyson }
14622244ea07SJohn Dyson 
1463fd3bf775SJohn Dyson /*
1464fd3bf775SJohn Dyson  * This routine queues an AIO request, checking for quotas.
1465fd3bf775SJohn Dyson  */
14662244ea07SJohn Dyson static int
1467b40ce416SJulian Elischer aio_aqueue(struct thread *td, struct aiocb *job, int type)
1468fd3bf775SJohn Dyson {
1469b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
14702244ea07SJohn Dyson 	struct kaioinfo *ki;
14712244ea07SJohn Dyson 
1472bfbbc4aaSJason Evans 	if (p->p_aioinfo == NULL)
14732244ea07SJohn Dyson 		aio_init_aioinfo(p);
14742244ea07SJohn Dyson 
14752244ea07SJohn Dyson 	if (num_queue_count >= max_queue_count)
14762244ea07SJohn Dyson 		return EAGAIN;
14772244ea07SJohn Dyson 
14782244ea07SJohn Dyson 	ki = p->p_aioinfo;
14792244ea07SJohn Dyson 	if (ki->kaio_queue_count >= ki->kaio_qallowed_count)
14802244ea07SJohn Dyson 		return EAGAIN;
14812244ea07SJohn Dyson 
1482b40ce416SJulian Elischer 	return _aio_aqueue(td, job, NULL, type);
14832244ea07SJohn Dyson }
14845dec52baSPeter Wemm #endif /* VFS_AIO */
14852244ea07SJohn Dyson 
14862244ea07SJohn Dyson /*
1487bfbbc4aaSJason Evans  * Support the aio_return system call, as a side-effect, kernel resources are
1488bfbbc4aaSJason Evans  * released.
14892244ea07SJohn Dyson  */
14902244ea07SJohn Dyson int
1491b40ce416SJulian Elischer aio_return(struct thread *td, struct aio_return_args *uap)
1492fd3bf775SJohn Dyson {
1493dd85920aSJason Evans #ifndef VFS_AIO
1494dd85920aSJason Evans 	return ENOSYS;
1495dd85920aSJason Evans #else
1496b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
149784af4da6SJohn Dyson 	int s;
1498f5ef029eSPoul-Henning Kamp 	int jobref;
149984af4da6SJohn Dyson 	struct aiocblist *cb, *ncb;
150011783b14SJohn Dyson 	struct aiocb *ujob;
15012244ea07SJohn Dyson 	struct kaioinfo *ki;
15022244ea07SJohn Dyson 
15032244ea07SJohn Dyson 	ki = p->p_aioinfo;
1504bfbbc4aaSJason Evans 	if (ki == NULL)
15052244ea07SJohn Dyson 		return EINVAL;
15062244ea07SJohn Dyson 
150711783b14SJohn Dyson 	ujob = uap->aiocbp;
150811783b14SJohn Dyson 
150911783b14SJohn Dyson 	jobref = fuword(&ujob->_aiocb_private.kernelinfo);
1510fd3bf775SJohn Dyson 	if (jobref == -1 || jobref == 0)
15112244ea07SJohn Dyson 		return EINVAL;
15122244ea07SJohn Dyson 
1513bfbbc4aaSJason Evans 	s = splnet();
1514bfbbc4aaSJason Evans 	for (cb = TAILQ_FIRST(&ki->kaio_jobdone); cb; cb = TAILQ_NEXT(cb,
1515bfbbc4aaSJason Evans 	    plist)) {
1516bfbbc4aaSJason Evans 		if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) ==
1517bfbbc4aaSJason Evans 		    jobref) {
1518bfbbc4aaSJason Evans 			splx(s);
151911783b14SJohn Dyson 			if (ujob == cb->uuaiocb) {
1520b40ce416SJulian Elischer 				td->td_retval[0] =
1521bfbbc4aaSJason Evans 				    cb->uaiocb._aiocb_private.status;
1522bfbbc4aaSJason Evans 			} else
1523b40ce416SJulian Elischer 				td->td_retval[0] = EFAULT;
1524fd3bf775SJohn Dyson 			if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
1525bfbbc4aaSJason Evans 				curproc->p_stats->p_ru.ru_oublock +=
1526bfbbc4aaSJason Evans 				    cb->outputcharge;
1527fd3bf775SJohn Dyson 				cb->outputcharge = 0;
1528fd3bf775SJohn Dyson 			} else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
1529bfbbc4aaSJason Evans 				curproc->p_stats->p_ru.ru_inblock +=
1530bfbbc4aaSJason Evans 				    cb->inputcharge;
1531fd3bf775SJohn Dyson 				cb->inputcharge = 0;
1532fd3bf775SJohn Dyson 			}
15332244ea07SJohn Dyson 			aio_free_entry(cb);
15342244ea07SJohn Dyson 			return 0;
15352244ea07SJohn Dyson 		}
15362244ea07SJohn Dyson 	}
1537bfbbc4aaSJason Evans 	splx(s);
15382244ea07SJohn Dyson 
153984af4da6SJohn Dyson 	s = splbio();
1540bfbbc4aaSJason Evans 	for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = ncb) {
154184af4da6SJohn Dyson 		ncb = TAILQ_NEXT(cb, plist);
1542bfbbc4aaSJason Evans 		if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo)
1543bfbbc4aaSJason Evans 		    == jobref) {
154484af4da6SJohn Dyson 			splx(s);
154511783b14SJohn Dyson 			if (ujob == cb->uuaiocb) {
1546b40ce416SJulian Elischer 				td->td_retval[0] =
1547bfbbc4aaSJason Evans 				    cb->uaiocb._aiocb_private.status;
1548bfbbc4aaSJason Evans 			} else
1549b40ce416SJulian Elischer 				td->td_retval[0] = EFAULT;
155084af4da6SJohn Dyson 			aio_free_entry(cb);
155184af4da6SJohn Dyson 			return 0;
155284af4da6SJohn Dyson 		}
155384af4da6SJohn Dyson 	}
155484af4da6SJohn Dyson 	splx(s);
155584af4da6SJohn Dyson 
15562244ea07SJohn Dyson 	return (EINVAL);
1557dd85920aSJason Evans #endif /* VFS_AIO */
15582244ea07SJohn Dyson }
15592244ea07SJohn Dyson 
15602244ea07SJohn Dyson /*
1561bfbbc4aaSJason Evans  * Allow a process to wakeup when any of the I/O requests are completed.
15622244ea07SJohn Dyson  */
15632244ea07SJohn Dyson int
1564b40ce416SJulian Elischer aio_suspend(struct thread *td, struct aio_suspend_args *uap)
1565fd3bf775SJohn Dyson {
1566dd85920aSJason Evans #ifndef VFS_AIO
1567dd85920aSJason Evans 	return ENOSYS;
1568dd85920aSJason Evans #else
1569b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
15704a11ca4eSPoul-Henning Kamp 	struct timeval atv;
15712244ea07SJohn Dyson 	struct timespec ts;
15722244ea07SJohn Dyson 	struct aiocb *const *cbptr, *cbp;
15732244ea07SJohn Dyson 	struct kaioinfo *ki;
15742244ea07SJohn Dyson 	struct aiocblist *cb;
15752244ea07SJohn Dyson 	int i;
157684af4da6SJohn Dyson 	int njoblist;
15772244ea07SJohn Dyson 	int error, s, timo;
157811783b14SJohn Dyson 	int *ijoblist;
157911783b14SJohn Dyson 	struct aiocb **ujoblist;
15802244ea07SJohn Dyson 
1581fd3bf775SJohn Dyson 	if (uap->nent >= AIO_LISTIO_MAX)
1582fd3bf775SJohn Dyson 		return EINVAL;
15832244ea07SJohn Dyson 
15842244ea07SJohn Dyson 	timo = 0;
15852244ea07SJohn Dyson 	if (uap->timeout) {
1586bfbbc4aaSJason Evans 		/* Get timespec struct. */
1587bfbbc4aaSJason Evans 		if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
15882244ea07SJohn Dyson 			return error;
15892244ea07SJohn Dyson 
15902244ea07SJohn Dyson 		if (ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000)
15912244ea07SJohn Dyson 			return (EINVAL);
15922244ea07SJohn Dyson 
1593e3b3ba2dSDag-Erling Smørgrav 		TIMESPEC_TO_TIMEVAL(&atv, &ts);
15942244ea07SJohn Dyson 		if (itimerfix(&atv))
15952244ea07SJohn Dyson 			return (EINVAL);
1596227ee8a1SPoul-Henning Kamp 		timo = tvtohz(&atv);
15972244ea07SJohn Dyson 	}
15982244ea07SJohn Dyson 
15992244ea07SJohn Dyson 	ki = p->p_aioinfo;
16002244ea07SJohn Dyson 	if (ki == NULL)
16012244ea07SJohn Dyson 		return EAGAIN;
16022244ea07SJohn Dyson 
160384af4da6SJohn Dyson 	njoblist = 0;
160411783b14SJohn Dyson 	ijoblist = zalloc(aiol_zone);
160511783b14SJohn Dyson 	ujoblist = zalloc(aiol_zone);
16062244ea07SJohn Dyson 	cbptr = uap->aiocbp;
16072244ea07SJohn Dyson 
16082244ea07SJohn Dyson 	for (i = 0; i < uap->nent; i++) {
160930166fabSBruce Evans 		cbp = (struct aiocb *)(intptr_t)fuword((caddr_t)&cbptr[i]);
161084af4da6SJohn Dyson 		if (cbp == 0)
161184af4da6SJohn Dyson 			continue;
161211783b14SJohn Dyson 		ujoblist[njoblist] = cbp;
161311783b14SJohn Dyson 		ijoblist[njoblist] = fuword(&cbp->_aiocb_private.kernelinfo);
161484af4da6SJohn Dyson 		njoblist++;
16152244ea07SJohn Dyson 	}
1616bfbbc4aaSJason Evans 
161711783b14SJohn Dyson 	if (njoblist == 0) {
161811783b14SJohn Dyson 		zfree(aiol_zone, ijoblist);
161911783b14SJohn Dyson 		zfree(aiol_zone, ujoblist);
162084af4da6SJohn Dyson 		return 0;
162111783b14SJohn Dyson 	}
16222244ea07SJohn Dyson 
162311783b14SJohn Dyson 	error = 0;
1624bfbbc4aaSJason Evans 	for (;;) {
1625bfbbc4aaSJason Evans 		for (cb = TAILQ_FIRST(&ki->kaio_jobdone); cb; cb =
1626bfbbc4aaSJason Evans 		    TAILQ_NEXT(cb, plist)) {
162784af4da6SJohn Dyson 			for (i = 0; i < njoblist; i++) {
1628bfbbc4aaSJason Evans 				if (((intptr_t)
1629bfbbc4aaSJason Evans 				    cb->uaiocb._aiocb_private.kernelinfo) ==
163011783b14SJohn Dyson 				    ijoblist[i]) {
163111783b14SJohn Dyson 					if (ujoblist[i] != cb->uuaiocb)
163211783b14SJohn Dyson 						error = EINVAL;
163311783b14SJohn Dyson 					zfree(aiol_zone, ijoblist);
163411783b14SJohn Dyson 					zfree(aiol_zone, ujoblist);
163511783b14SJohn Dyson 					return error;
163684af4da6SJohn Dyson 				}
163784af4da6SJohn Dyson 			}
163884af4da6SJohn Dyson 		}
163984af4da6SJohn Dyson 
164084af4da6SJohn Dyson 		s = splbio();
1641bfbbc4aaSJason Evans 		for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb =
1642bfbbc4aaSJason Evans 		    TAILQ_NEXT(cb, plist)) {
164384af4da6SJohn Dyson 			for (i = 0; i < njoblist; i++) {
1644bfbbc4aaSJason Evans 				if (((intptr_t)
1645bfbbc4aaSJason Evans 				    cb->uaiocb._aiocb_private.kernelinfo) ==
164611783b14SJohn Dyson 				    ijoblist[i]) {
164784af4da6SJohn Dyson 					splx(s);
164811783b14SJohn Dyson 					if (ujoblist[i] != cb->uuaiocb)
164911783b14SJohn Dyson 						error = EINVAL;
165011783b14SJohn Dyson 					zfree(aiol_zone, ijoblist);
165111783b14SJohn Dyson 					zfree(aiol_zone, ujoblist);
165211783b14SJohn Dyson 					return error;
16532244ea07SJohn Dyson 				}
16542244ea07SJohn Dyson 			}
16552244ea07SJohn Dyson 		}
16562244ea07SJohn Dyson 
1657fd3bf775SJohn Dyson 		ki->kaio_flags |= KAIO_WAKEUP;
16582244ea07SJohn Dyson 		error = tsleep(p, PRIBIO | PCATCH, "aiospn", timo);
1659b7592c7bSJason Evans 		splx(s);
16602244ea07SJohn Dyson 
166160ffb019SJason Evans 		if (error == ERESTART || error == EINTR) {
166211783b14SJohn Dyson 			zfree(aiol_zone, ijoblist);
166311783b14SJohn Dyson 			zfree(aiol_zone, ujoblist);
16642244ea07SJohn Dyson 			return EINTR;
16652244ea07SJohn Dyson 		} else if (error == EWOULDBLOCK) {
166611783b14SJohn Dyson 			zfree(aiol_zone, ijoblist);
166711783b14SJohn Dyson 			zfree(aiol_zone, ujoblist);
16682244ea07SJohn Dyson 			return EAGAIN;
16692244ea07SJohn Dyson 		}
16702244ea07SJohn Dyson 	}
16712244ea07SJohn Dyson 
16722244ea07SJohn Dyson /* NOTREACHED */
16732244ea07SJohn Dyson 	return EINVAL;
1674dd85920aSJason Evans #endif /* VFS_AIO */
16752244ea07SJohn Dyson }
1676ee877a35SJohn Dyson 
1677ee877a35SJohn Dyson /*
1678dd85920aSJason Evans  * aio_cancel cancels any non-physio aio operations not currently in
1679dd85920aSJason Evans  * progress.
1680ee877a35SJohn Dyson  */
1681ee877a35SJohn Dyson int
1682b40ce416SJulian Elischer aio_cancel(struct thread *td, struct aio_cancel_args *uap)
1683fd3bf775SJohn Dyson {
1684dd85920aSJason Evans #ifndef VFS_AIO
168578922e41SJohn Dyson 	return ENOSYS;
1686dd85920aSJason Evans #else
1687b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
1688dd85920aSJason Evans 	struct kaioinfo *ki;
1689dd85920aSJason Evans 	struct aiocblist *cbe, *cbn;
1690dd85920aSJason Evans 	struct file *fp;
1691dd85920aSJason Evans 	struct filedesc *fdp;
1692dd85920aSJason Evans 	struct socket *so;
1693dd85920aSJason Evans 	struct proc *po;
1694dd85920aSJason Evans 	int s,error;
1695dd85920aSJason Evans 	int cancelled=0;
1696dd85920aSJason Evans 	int notcancelled=0;
1697dd85920aSJason Evans 	struct vnode *vp;
1698dd85920aSJason Evans 
1699dd85920aSJason Evans 	fdp = p->p_fd;
1700dd85920aSJason Evans 
1701dd85920aSJason Evans 	fp = fdp->fd_ofiles[uap->fd];
1702dd85920aSJason Evans 
1703dd85920aSJason Evans 	if (fp == NULL) {
1704dd85920aSJason Evans 		return EBADF;
1705dd85920aSJason Evans 	}
1706dd85920aSJason Evans 
1707dd85920aSJason Evans         if (fp->f_type == DTYPE_VNODE) {
1708dd85920aSJason Evans 		vp = (struct vnode *)fp->f_data;
1709dd85920aSJason Evans 
1710dd85920aSJason Evans 		if (vn_isdisk(vp,&error)) {
1711b40ce416SJulian Elischer 			td->td_retval[0] = AIO_NOTCANCELED;
1712dd85920aSJason Evans         	        return 0;
1713dd85920aSJason Evans 		}
1714dd85920aSJason Evans 	} else if (fp->f_type == DTYPE_SOCKET) {
1715dd85920aSJason Evans 		so = (struct socket *)fp->f_data;
1716dd85920aSJason Evans 
1717dd85920aSJason Evans 		s = splnet();
1718dd85920aSJason Evans 
1719dd85920aSJason Evans 		for (cbe = TAILQ_FIRST(&so->so_aiojobq); cbe; cbe = cbn) {
1720dd85920aSJason Evans 			cbn = TAILQ_NEXT(cbe, list);
1721dd85920aSJason Evans 			if ((uap->aiocbp == NULL) ||
1722dd85920aSJason Evans 				(uap->aiocbp == cbe->uuaiocb) ) {
1723dd85920aSJason Evans 				po = cbe->userproc;
1724dd85920aSJason Evans 				ki = po->p_aioinfo;
1725dd85920aSJason Evans 				TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
1726dd85920aSJason Evans 				TAILQ_REMOVE(&ki->kaio_sockqueue, cbe, plist);
1727dd85920aSJason Evans 				TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, plist);
1728dd85920aSJason Evans 				if (ki->kaio_flags & KAIO_WAKEUP) {
1729dd85920aSJason Evans 					wakeup(po);
1730dd85920aSJason Evans 				}
1731dd85920aSJason Evans 				cbe->jobstate = JOBST_JOBFINISHED;
1732dd85920aSJason Evans 				cbe->uaiocb._aiocb_private.status=-1;
1733dd85920aSJason Evans 				cbe->uaiocb._aiocb_private.error=ECANCELED;
1734dd85920aSJason Evans 				cancelled++;
1735cb679c38SJonathan Lemon /* XXX cancelled, knote? */
1736dd85920aSJason Evans 			        if (cbe->uaiocb.aio_sigevent.sigev_notify ==
173719eb87d2SJohn Baldwin 				    SIGEV_SIGNAL) {
173819eb87d2SJohn Baldwin 					PROC_LOCK(cbe->userproc);
1739dd85920aSJason Evans 					psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
174019eb87d2SJohn Baldwin 					PROC_UNLOCK(cbe->userproc);
174119eb87d2SJohn Baldwin 				}
1742dd85920aSJason Evans 				if (uap->aiocbp)
1743dd85920aSJason Evans 					break;
1744dd85920aSJason Evans 			}
1745dd85920aSJason Evans 		}
1746dd85920aSJason Evans 
1747dd85920aSJason Evans 		splx(s);
1748dd85920aSJason Evans 
1749dd85920aSJason Evans 		if ((cancelled) && (uap->aiocbp)) {
1750b40ce416SJulian Elischer 			td->td_retval[0] = AIO_CANCELED;
1751dd85920aSJason Evans 			return 0;
1752dd85920aSJason Evans 		}
1753dd85920aSJason Evans 
1754dd85920aSJason Evans 	}
1755dd85920aSJason Evans 
1756dd85920aSJason Evans 	ki=p->p_aioinfo;
1757dd85920aSJason Evans 
1758dd85920aSJason Evans 	s = splnet();
1759dd85920aSJason Evans 
1760dd85920aSJason Evans 	for (cbe = TAILQ_FIRST(&ki->kaio_jobqueue); cbe; cbe = cbn) {
1761dd85920aSJason Evans 		cbn = TAILQ_NEXT(cbe, plist);
1762dd85920aSJason Evans 
1763dd85920aSJason Evans 		if ((uap->fd == cbe->uaiocb.aio_fildes) &&
1764dd85920aSJason Evans 		    ((uap->aiocbp == NULL ) ||
1765dd85920aSJason Evans 		     (uap->aiocbp == cbe->uuaiocb))) {
1766dd85920aSJason Evans 
1767dd85920aSJason Evans 			if (cbe->jobstate == JOBST_JOBQGLOBAL) {
1768dd85920aSJason Evans 				TAILQ_REMOVE(&aio_jobs, cbe, list);
1769dd85920aSJason Evans                                 TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
1770dd85920aSJason Evans                                 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe,
1771dd85920aSJason Evans                                     plist);
1772dd85920aSJason Evans 				cancelled++;
1773dd85920aSJason Evans 				ki->kaio_queue_finished_count++;
1774dd85920aSJason Evans 				cbe->jobstate = JOBST_JOBFINISHED;
1775dd85920aSJason Evans 				cbe->uaiocb._aiocb_private.status = -1;
1776dd85920aSJason Evans 				cbe->uaiocb._aiocb_private.error = ECANCELED;
1777cb679c38SJonathan Lemon /* XXX cancelled, knote? */
1778dd85920aSJason Evans 			        if (cbe->uaiocb.aio_sigevent.sigev_notify ==
177919eb87d2SJohn Baldwin 				    SIGEV_SIGNAL) {
178019eb87d2SJohn Baldwin 					PROC_LOCK(cbe->userproc);
1781dd85920aSJason Evans 					psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo);
178219eb87d2SJohn Baldwin 					PROC_UNLOCK(cbe->userproc);
178319eb87d2SJohn Baldwin 				}
1784dd85920aSJason Evans 			} else {
1785dd85920aSJason Evans 				notcancelled++;
1786dd85920aSJason Evans 			}
1787dd85920aSJason Evans 		}
1788dd85920aSJason Evans 	}
1789dd85920aSJason Evans 
1790dd85920aSJason Evans 	splx(s);
1791dd85920aSJason Evans 
1792dd85920aSJason Evans 
1793dd85920aSJason Evans 	if (notcancelled) {
1794b40ce416SJulian Elischer 		td->td_retval[0] = AIO_NOTCANCELED;
1795dd85920aSJason Evans 		return 0;
1796dd85920aSJason Evans 	}
1797dd85920aSJason Evans 
1798dd85920aSJason Evans 	if (cancelled) {
1799b40ce416SJulian Elischer 		td->td_retval[0] = AIO_CANCELED;
1800dd85920aSJason Evans 		return 0;
1801dd85920aSJason Evans 	}
1802dd85920aSJason Evans 
1803b40ce416SJulian Elischer 	td->td_retval[0] = AIO_ALLDONE;
1804dd85920aSJason Evans 
1805dd85920aSJason Evans 	return 0;
1806dd85920aSJason Evans #endif /* VFS_AIO */
1807ee877a35SJohn Dyson }
1808ee877a35SJohn Dyson 
1809ee877a35SJohn Dyson /*
1810bfbbc4aaSJason Evans  * aio_error is implemented in the kernel level for compatibility purposes only.
1811bfbbc4aaSJason Evans  * For a user mode async implementation, it would be best to do it in a userland
1812bfbbc4aaSJason Evans  * subroutine.
1813ee877a35SJohn Dyson  */
1814ee877a35SJohn Dyson int
1815b40ce416SJulian Elischer aio_error(struct thread *td, struct aio_error_args *uap)
1816fd3bf775SJohn Dyson {
1817dd85920aSJason Evans #ifndef VFS_AIO
1818dd85920aSJason Evans 	return ENOSYS;
1819dd85920aSJason Evans #else
1820b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
182184af4da6SJohn Dyson 	int s;
18222244ea07SJohn Dyson 	struct aiocblist *cb;
18232244ea07SJohn Dyson 	struct kaioinfo *ki;
18242244ea07SJohn Dyson 	int jobref;
1825ee877a35SJohn Dyson 
18262244ea07SJohn Dyson 	ki = p->p_aioinfo;
18272244ea07SJohn Dyson 	if (ki == NULL)
18282244ea07SJohn Dyson 		return EINVAL;
18292244ea07SJohn Dyson 
18302244ea07SJohn Dyson 	jobref = fuword(&uap->aiocbp->_aiocb_private.kernelinfo);
1831fd3bf775SJohn Dyson 	if ((jobref == -1) || (jobref == 0))
1832fd3bf775SJohn Dyson 		return EINVAL;
1833ee877a35SJohn Dyson 
1834bfbbc4aaSJason Evans 	for (cb = TAILQ_FIRST(&ki->kaio_jobdone); cb; cb = TAILQ_NEXT(cb,
1835bfbbc4aaSJason Evans 	    plist)) {
1836bfbbc4aaSJason Evans 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1837bfbbc4aaSJason Evans 		    jobref) {
1838b40ce416SJulian Elischer 			td->td_retval[0] = cb->uaiocb._aiocb_private.error;
18392244ea07SJohn Dyson 			return 0;
18402244ea07SJohn Dyson 		}
1841ee877a35SJohn Dyson 	}
1842ee877a35SJohn Dyson 
1843bfbbc4aaSJason Evans 	s = splnet();
18442244ea07SJohn Dyson 
1845bfbbc4aaSJason Evans 	for (cb = TAILQ_FIRST(&ki->kaio_jobqueue); cb; cb = TAILQ_NEXT(cb,
1846bfbbc4aaSJason Evans 	    plist)) {
1847bfbbc4aaSJason Evans 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1848bfbbc4aaSJason Evans 		    jobref) {
1849b40ce416SJulian Elischer 			td->td_retval[0] = EINPROGRESS;
1850bfbbc4aaSJason Evans 			splx(s);
18512244ea07SJohn Dyson 			return 0;
18522244ea07SJohn Dyson 		}
18532244ea07SJohn Dyson 	}
185460ffb019SJason Evans 
185560ffb019SJason Evans 	for (cb = TAILQ_FIRST(&ki->kaio_sockqueue); cb; cb = TAILQ_NEXT(cb,
185660ffb019SJason Evans 	    plist)) {
185760ffb019SJason Evans 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
185860ffb019SJason Evans 		    jobref) {
1859b40ce416SJulian Elischer 			td->td_retval[0] = EINPROGRESS;
186060ffb019SJason Evans 			splx(s);
186160ffb019SJason Evans 			return 0;
186260ffb019SJason Evans 		}
186360ffb019SJason Evans 	}
1864bfbbc4aaSJason Evans 	splx(s);
18652244ea07SJohn Dyson 
186684af4da6SJohn Dyson 	s = splbio();
1867bfbbc4aaSJason Evans 	for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = TAILQ_NEXT(cb,
1868bfbbc4aaSJason Evans 	    plist)) {
1869bfbbc4aaSJason Evans 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1870bfbbc4aaSJason Evans 		    jobref) {
1871b40ce416SJulian Elischer 			td->td_retval[0] = cb->uaiocb._aiocb_private.error;
187284af4da6SJohn Dyson 			splx(s);
187384af4da6SJohn Dyson 			return 0;
187484af4da6SJohn Dyson 		}
187584af4da6SJohn Dyson 	}
187684af4da6SJohn Dyson 
1877bfbbc4aaSJason Evans 	for (cb = TAILQ_FIRST(&ki->kaio_bufqueue); cb; cb = TAILQ_NEXT(cb,
1878bfbbc4aaSJason Evans 	    plist)) {
1879bfbbc4aaSJason Evans 		if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) ==
1880bfbbc4aaSJason Evans 		    jobref) {
1881b40ce416SJulian Elischer 			td->td_retval[0] = EINPROGRESS;
188284af4da6SJohn Dyson 			splx(s);
188384af4da6SJohn Dyson 			return 0;
188484af4da6SJohn Dyson 		}
188584af4da6SJohn Dyson 	}
188684af4da6SJohn Dyson 	splx(s);
188784af4da6SJohn Dyson 
1888bfbbc4aaSJason Evans #if (0)
18892244ea07SJohn Dyson 	/*
1890bfbbc4aaSJason Evans 	 * Hack for lio.
18912244ea07SJohn Dyson 	 */
18922244ea07SJohn Dyson 	status = fuword(&uap->aiocbp->_aiocb_private.status);
1893bfbbc4aaSJason Evans 	if (status == -1)
18942244ea07SJohn Dyson 		return fuword(&uap->aiocbp->_aiocb_private.error);
1895bfbbc4aaSJason Evans #endif
18962244ea07SJohn Dyson 	return EINVAL;
1897dd85920aSJason Evans #endif /* VFS_AIO */
1898ee877a35SJohn Dyson }
1899ee877a35SJohn Dyson 
1900ee877a35SJohn Dyson int
1901b40ce416SJulian Elischer aio_read(struct thread *td, struct aio_read_args *uap)
1902fd3bf775SJohn Dyson {
1903dd85920aSJason Evans #ifndef VFS_AIO
1904dd85920aSJason Evans 	return ENOSYS;
1905dd85920aSJason Evans #else
1906b40ce416SJulian Elischer 	return aio_aqueue(td, uap->aiocbp, LIO_READ);
1907dd85920aSJason Evans #endif /* VFS_AIO */
1908ee877a35SJohn Dyson }
1909ee877a35SJohn Dyson 
1910ee877a35SJohn Dyson int
1911b40ce416SJulian Elischer aio_write(struct thread *td, struct aio_write_args *uap)
1912fd3bf775SJohn Dyson {
1913dd85920aSJason Evans #ifndef VFS_AIO
1914dd85920aSJason Evans 	return ENOSYS;
1915dd85920aSJason Evans #else
1916b40ce416SJulian Elischer 	return aio_aqueue(td, uap->aiocbp, LIO_WRITE);
1917dd85920aSJason Evans #endif /* VFS_AIO */
1918ee877a35SJohn Dyson }
1919ee877a35SJohn Dyson 
1920ee877a35SJohn Dyson int
1921b40ce416SJulian Elischer lio_listio(struct thread *td, struct lio_listio_args *uap)
1922fd3bf775SJohn Dyson {
1923dd85920aSJason Evans #ifndef VFS_AIO
1924dd85920aSJason Evans 	return ENOSYS;
1925dd85920aSJason Evans #else
1926b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
19274a11ca4eSPoul-Henning Kamp 	int nent, nentqueued;
19282244ea07SJohn Dyson 	struct aiocb *iocb, * const *cbptr;
19292244ea07SJohn Dyson 	struct aiocblist *cb;
19302244ea07SJohn Dyson 	struct kaioinfo *ki;
193184af4da6SJohn Dyson 	struct aio_liojob *lj;
19322244ea07SJohn Dyson 	int error, runningcode;
1933fd3bf775SJohn Dyson 	int nerror;
1934ee877a35SJohn Dyson 	int i;
193584af4da6SJohn Dyson 	int s;
1936ee877a35SJohn Dyson 
1937bfbbc4aaSJason Evans 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
1938ee877a35SJohn Dyson 		return EINVAL;
19392244ea07SJohn Dyson 
19402244ea07SJohn Dyson 	nent = uap->nent;
1941bfbbc4aaSJason Evans 	if (nent > AIO_LISTIO_MAX)
19422244ea07SJohn Dyson 		return EINVAL;
19432244ea07SJohn Dyson 
1944bfbbc4aaSJason Evans 	if (p->p_aioinfo == NULL)
19452244ea07SJohn Dyson 		aio_init_aioinfo(p);
19462244ea07SJohn Dyson 
1947bfbbc4aaSJason Evans 	if ((nent + num_queue_count) > max_queue_count)
19482244ea07SJohn Dyson 		return EAGAIN;
19492244ea07SJohn Dyson 
19502244ea07SJohn Dyson 	ki = p->p_aioinfo;
1951bfbbc4aaSJason Evans 	if ((nent + ki->kaio_queue_count) > ki->kaio_qallowed_count)
19522244ea07SJohn Dyson 		return EAGAIN;
19532244ea07SJohn Dyson 
195484af4da6SJohn Dyson 	lj = zalloc(aiolio_zone);
1955bfbbc4aaSJason Evans 	if (!lj)
195684af4da6SJohn Dyson 		return EAGAIN;
195784af4da6SJohn Dyson 
195884af4da6SJohn Dyson 	lj->lioj_flags = 0;
195984af4da6SJohn Dyson 	lj->lioj_buffer_count = 0;
196084af4da6SJohn Dyson 	lj->lioj_buffer_finished_count = 0;
196184af4da6SJohn Dyson 	lj->lioj_queue_count = 0;
196284af4da6SJohn Dyson 	lj->lioj_queue_finished_count = 0;
196384af4da6SJohn Dyson 	lj->lioj_ki = ki;
196484af4da6SJohn Dyson 
196584af4da6SJohn Dyson 	/*
1966bfbbc4aaSJason Evans 	 * Setup signal.
196784af4da6SJohn Dyson 	 */
196884af4da6SJohn Dyson 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
1969bfbbc4aaSJason Evans 		error = copyin(uap->sig, &lj->lioj_signal,
1970bfbbc4aaSJason Evans 			       sizeof(lj->lioj_signal));
19712f3cf918SAlfred Perlstein 		if (error) {
19722f3cf918SAlfred Perlstein 			zfree(aiolio_zone, lj);
197384af4da6SJohn Dyson 			return error;
19742f3cf918SAlfred Perlstein 		}
19752f3cf918SAlfred Perlstein 		if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) {
19762f3cf918SAlfred Perlstein 			zfree(aiolio_zone, lj);
19772f3cf918SAlfred Perlstein 			return EINVAL;
19782f3cf918SAlfred Perlstein 		}
197984af4da6SJohn Dyson 		lj->lioj_flags |= LIOJ_SIGNAL;
198084af4da6SJohn Dyson 		lj->lioj_flags &= ~LIOJ_SIGNAL_POSTED;
1981bfbbc4aaSJason Evans 	} else
198284af4da6SJohn Dyson 		lj->lioj_flags &= ~LIOJ_SIGNAL;
198384af4da6SJohn Dyson 
19842f3cf918SAlfred Perlstein 	TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list);
19852244ea07SJohn Dyson 	/*
1986bfbbc4aaSJason Evans 	 * Get pointers to the list of I/O requests.
19872244ea07SJohn Dyson 	 */
1988fd3bf775SJohn Dyson 	nerror = 0;
1989fd3bf775SJohn Dyson 	nentqueued = 0;
19902244ea07SJohn Dyson 	cbptr = uap->acb_list;
19912244ea07SJohn Dyson 	for (i = 0; i < uap->nent; i++) {
199230166fabSBruce Evans 		iocb = (struct aiocb *)(intptr_t)fuword((caddr_t)&cbptr[i]);
199330166fabSBruce Evans 		if (((intptr_t)iocb != -1) && ((intptr_t)iocb != NULL)) {
1994b40ce416SJulian Elischer 			error = _aio_aqueue(td, iocb, lj, 0);
1995bfbbc4aaSJason Evans 			if (error == 0)
19962244ea07SJohn Dyson 				nentqueued++;
1997bfbbc4aaSJason Evans 			else
1998fd3bf775SJohn Dyson 				nerror++;
1999fd3bf775SJohn Dyson 		}
2000fd3bf775SJohn Dyson 	}
20012244ea07SJohn Dyson 
2002a624e84fSJohn Dyson 	/*
2003bfbbc4aaSJason Evans 	 * If we haven't queued any, then just return error.
2004a624e84fSJohn Dyson 	 */
2005bfbbc4aaSJason Evans 	if (nentqueued == 0)
2006fd3bf775SJohn Dyson 		return 0;
20072244ea07SJohn Dyson 
2008a624e84fSJohn Dyson 	/*
2009bfbbc4aaSJason Evans 	 * Calculate the appropriate error return.
2010a624e84fSJohn Dyson 	 */
20112244ea07SJohn Dyson 	runningcode = 0;
2012fd3bf775SJohn Dyson 	if (nerror)
20132244ea07SJohn Dyson 		runningcode = EIO;
20142244ea07SJohn Dyson 
20152244ea07SJohn Dyson 	if (uap->mode == LIO_WAIT) {
2016bfbbc4aaSJason Evans 		int command, found, jobref;
2017bfbbc4aaSJason Evans 
2018bfbbc4aaSJason Evans 		for (;;) {
2019fd3bf775SJohn Dyson 			found = 0;
2020fd3bf775SJohn Dyson 			for (i = 0; i < uap->nent; i++) {
2021a624e84fSJohn Dyson 				/*
2022bfbbc4aaSJason Evans 				 * Fetch address of the control buf pointer in
2023bfbbc4aaSJason Evans 				 * user space.
2024a624e84fSJohn Dyson 				 */
202530166fabSBruce Evans 				iocb = (struct aiocb *)(intptr_t)fuword((caddr_t)&cbptr[i]);
2026bfbbc4aaSJason Evans 				if (((intptr_t)iocb == -1) || ((intptr_t)iocb
2027bfbbc4aaSJason Evans 				    == 0))
2028fd3bf775SJohn Dyson 					continue;
2029a624e84fSJohn Dyson 
2030a624e84fSJohn Dyson 				/*
2031bfbbc4aaSJason Evans 				 * Fetch the associated command from user space.
2032a624e84fSJohn Dyson 				 */
20332244ea07SJohn Dyson 				command = fuword(&iocb->aio_lio_opcode);
2034fd3bf775SJohn Dyson 				if (command == LIO_NOP) {
2035fd3bf775SJohn Dyson 					found++;
20362244ea07SJohn Dyson 					continue;
2037fd3bf775SJohn Dyson 				}
2038a624e84fSJohn Dyson 
20392244ea07SJohn Dyson 				jobref = fuword(&iocb->_aiocb_private.kernelinfo);
20402244ea07SJohn Dyson 
204137d40066SPoul-Henning Kamp 				TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) {
2042bfbbc4aaSJason Evans 					if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
2043bfbbc4aaSJason Evans 					    == jobref) {
2044bfbbc4aaSJason Evans 						if (cb->uaiocb.aio_lio_opcode
2045bfbbc4aaSJason Evans 						    == LIO_WRITE) {
2046bfbbc4aaSJason Evans 							curproc->p_stats->p_ru.ru_oublock
2047bfbbc4aaSJason Evans 							    +=
204884af4da6SJohn Dyson 							    cb->outputcharge;
204984af4da6SJohn Dyson 							cb->outputcharge = 0;
2050bfbbc4aaSJason Evans 						} else if (cb->uaiocb.aio_lio_opcode
2051bfbbc4aaSJason Evans 						    == LIO_READ) {
2052bfbbc4aaSJason Evans 							curproc->p_stats->p_ru.ru_inblock
2053bfbbc4aaSJason Evans 							    += cb->inputcharge;
205484af4da6SJohn Dyson 							cb->inputcharge = 0;
205584af4da6SJohn Dyson 						}
20562244ea07SJohn Dyson 						found++;
20572244ea07SJohn Dyson 						break;
20582244ea07SJohn Dyson 					}
20592244ea07SJohn Dyson 				}
2060fd3bf775SJohn Dyson 
206184af4da6SJohn Dyson 				s = splbio();
206237d40066SPoul-Henning Kamp 				TAILQ_FOREACH(cb, &ki->kaio_bufdone, plist) {
2063bfbbc4aaSJason Evans 					if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo)
2064bfbbc4aaSJason Evans 					    == jobref) {
206584af4da6SJohn Dyson 						found++;
206684af4da6SJohn Dyson 						break;
2067fd3bf775SJohn Dyson 					}
20682244ea07SJohn Dyson 				}
206984af4da6SJohn Dyson 				splx(s);
207084af4da6SJohn Dyson 			}
20712244ea07SJohn Dyson 
2072a624e84fSJohn Dyson 			/*
2073bfbbc4aaSJason Evans 			 * If all I/Os have been disposed of, then we can
2074bfbbc4aaSJason Evans 			 * return.
2075a624e84fSJohn Dyson 			 */
2076bfbbc4aaSJason Evans 			if (found == nentqueued)
20772244ea07SJohn Dyson 				return runningcode;
20782244ea07SJohn Dyson 
2079fd3bf775SJohn Dyson 			ki->kaio_flags |= KAIO_WAKEUP;
20802244ea07SJohn Dyson 			error = tsleep(p, PRIBIO | PCATCH, "aiospn", 0);
20812244ea07SJohn Dyson 
2082bfbbc4aaSJason Evans 			if (error == EINTR)
20832244ea07SJohn Dyson 				return EINTR;
2084bfbbc4aaSJason Evans 			else if (error == EWOULDBLOCK)
20852244ea07SJohn Dyson 				return EAGAIN;
20862244ea07SJohn Dyson 		}
20872244ea07SJohn Dyson 	}
20882244ea07SJohn Dyson 
20892244ea07SJohn Dyson 	return runningcode;
2090dd85920aSJason Evans #endif /* VFS_AIO */
2091ee877a35SJohn Dyson }
2092fd3bf775SJohn Dyson 
20935dec52baSPeter Wemm #ifdef VFS_AIO
209484af4da6SJohn Dyson /*
2095f09deb69SJeroen Ruigrok van der Werven  * This is a weird hack so that we can post a signal.  It is safe to do so from
2096bfbbc4aaSJason Evans  * a timeout routine, but *not* from an interrupt routine.
209784af4da6SJohn Dyson  */
209884af4da6SJohn Dyson static void
2099bfbbc4aaSJason Evans process_signal(void *aioj)
210084af4da6SJohn Dyson {
2101bfbbc4aaSJason Evans 	struct aiocblist *aiocbe = aioj;
2102bfbbc4aaSJason Evans 	struct aio_liojob *lj = aiocbe->lio;
2103bfbbc4aaSJason Evans 	struct aiocb *cb = &aiocbe->uaiocb;
2104bfbbc4aaSJason Evans 
2105bfbbc4aaSJason Evans 	if ((lj) && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL) &&
2106bfbbc4aaSJason Evans 		(lj->lioj_queue_count == lj->lioj_queue_finished_count)) {
210719eb87d2SJohn Baldwin 		PROC_LOCK(lj->lioj_ki->kaio_p);
210884af4da6SJohn Dyson 		psignal(lj->lioj_ki->kaio_p, lj->lioj_signal.sigev_signo);
210919eb87d2SJohn Baldwin 		PROC_UNLOCK(lj->lioj_ki->kaio_p);
211084af4da6SJohn Dyson 		lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
211184af4da6SJohn Dyson 	}
2112bfbbc4aaSJason Evans 
211319eb87d2SJohn Baldwin 	if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) {
211419eb87d2SJohn Baldwin 		PROC_LOCK(aiocbe->userproc);
2115bfbbc4aaSJason Evans 		psignal(aiocbe->userproc, cb->aio_sigevent.sigev_signo);
211619eb87d2SJohn Baldwin 		PROC_UNLOCK(aiocbe->userproc);
211719eb87d2SJohn Baldwin 	}
211884af4da6SJohn Dyson }
211984af4da6SJohn Dyson 
212084af4da6SJohn Dyson /*
2121bfbbc4aaSJason Evans  * Interrupt handler for physio, performs the necessary process wakeups, and
2122bfbbc4aaSJason Evans  * signals.
212384af4da6SJohn Dyson  */
2124fd3bf775SJohn Dyson static void
2125bfbbc4aaSJason Evans aio_physwakeup(struct buf *bp)
2126fd3bf775SJohn Dyson {
212784af4da6SJohn Dyson 	struct aiocblist *aiocbe;
2128fd3bf775SJohn Dyson 	struct proc *p;
2129fd3bf775SJohn Dyson 	struct kaioinfo *ki;
213084af4da6SJohn Dyson 	struct aio_liojob *lj;
2131fd3bf775SJohn Dyson 
21329c8a2647SAlan Cox 	wakeup(bp);
2133fd3bf775SJohn Dyson 
213484af4da6SJohn Dyson 	aiocbe = (struct aiocblist *)bp->b_spc;
213584af4da6SJohn Dyson 	if (aiocbe) {
2136b0eeea20SPoul-Henning Kamp 		p = bp->b_caller1;
213784af4da6SJohn Dyson 
213884af4da6SJohn Dyson 		aiocbe->jobstate = JOBST_JOBBFINISHED;
213984af4da6SJohn Dyson 		aiocbe->uaiocb._aiocb_private.status -= bp->b_resid;
214084af4da6SJohn Dyson 		aiocbe->uaiocb._aiocb_private.error = 0;
214184af4da6SJohn Dyson 		aiocbe->jobflags |= AIOCBLIST_DONE;
214284af4da6SJohn Dyson 
2143c244d2deSPoul-Henning Kamp 		if (bp->b_ioflags & BIO_ERROR)
214484af4da6SJohn Dyson 			aiocbe->uaiocb._aiocb_private.error = bp->b_error;
214584af4da6SJohn Dyson 
214684af4da6SJohn Dyson 		lj = aiocbe->lio;
214784af4da6SJohn Dyson 		if (lj) {
214884af4da6SJohn Dyson 			lj->lioj_buffer_finished_count++;
2149bfbbc4aaSJason Evans 
215084af4da6SJohn Dyson 			/*
2151bfbbc4aaSJason Evans 			 * wakeup/signal if all of the interrupt jobs are done.
215284af4da6SJohn Dyson 			 */
2153bfbbc4aaSJason Evans 			if (lj->lioj_buffer_finished_count ==
2154bfbbc4aaSJason Evans 			    lj->lioj_buffer_count) {
215584af4da6SJohn Dyson 				/*
2156bfbbc4aaSJason Evans 				 * Post a signal if it is called for.
215784af4da6SJohn Dyson 				 */
2158bfbbc4aaSJason Evans 				if ((lj->lioj_flags &
2159bfbbc4aaSJason Evans 				    (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) ==
216084af4da6SJohn Dyson 				    LIOJ_SIGNAL) {
216184af4da6SJohn Dyson 					lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
216213644654SAlan Cox 					aiocbe->timeouthandle =
216313644654SAlan Cox 						timeout(process_signal,
216413644654SAlan Cox 							aiocbe, 0);
216584af4da6SJohn Dyson 				}
216684af4da6SJohn Dyson 			}
216784af4da6SJohn Dyson 		}
216884af4da6SJohn Dyson 
2169fd3bf775SJohn Dyson 		ki = p->p_aioinfo;
217084af4da6SJohn Dyson 		if (ki) {
217184af4da6SJohn Dyson 			ki->kaio_buffer_finished_count++;
217284af4da6SJohn Dyson 			TAILQ_REMOVE(&aio_bufjobs, aiocbe, list);
217384af4da6SJohn Dyson 			TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist);
217484af4da6SJohn Dyson 			TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist);
2175bfbbc4aaSJason Evans 
2176cb679c38SJonathan Lemon 			KNOTE(&aiocbe->klist, 0);
2177bfbbc4aaSJason Evans 			/* Do the wakeup. */
217884af4da6SJohn Dyson 			if (ki->kaio_flags & (KAIO_RUNDOWN|KAIO_WAKEUP)) {
2179fd3bf775SJohn Dyson 				ki->kaio_flags &= ~KAIO_WAKEUP;
2180fd3bf775SJohn Dyson 				wakeup(p);
2181fd3bf775SJohn Dyson 			}
2182fd3bf775SJohn Dyson 		}
2183bfbbc4aaSJason Evans 
2184bfbbc4aaSJason Evans 		if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL)
218513644654SAlan Cox 			aiocbe->timeouthandle =
2186bfbbc4aaSJason Evans 				timeout(process_signal, aiocbe, 0);
2187fd3bf775SJohn Dyson 	}
218884af4da6SJohn Dyson }
21895dec52baSPeter Wemm #endif /* VFS_AIO */
2190bfbbc4aaSJason Evans 
2191bfbbc4aaSJason Evans int
2192b40ce416SJulian Elischer aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
2193bfbbc4aaSJason Evans {
2194dd85920aSJason Evans #ifndef VFS_AIO
2195dd85920aSJason Evans 	return ENOSYS;
2196dd85920aSJason Evans #else
2197b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
2198bfbbc4aaSJason Evans 	struct timeval atv;
2199bfbbc4aaSJason Evans 	struct timespec ts;
2200bfbbc4aaSJason Evans 	struct aiocb **cbptr;
2201bfbbc4aaSJason Evans 	struct kaioinfo *ki;
2202bfbbc4aaSJason Evans 	struct aiocblist *cb = NULL;
2203bfbbc4aaSJason Evans 	int error, s, timo;
2204bfbbc4aaSJason Evans 
2205dd85920aSJason Evans 	suword(uap->aiocbp, (int)NULL);
2206dd85920aSJason Evans 
2207bfbbc4aaSJason Evans 	timo = 0;
2208bfbbc4aaSJason Evans 	if (uap->timeout) {
2209bfbbc4aaSJason Evans 		/* Get timespec struct. */
221088ed460eSAlan Cox 		error = copyin(uap->timeout, &ts, sizeof(ts));
2211bfbbc4aaSJason Evans 		if (error)
2212bfbbc4aaSJason Evans 			return error;
2213bfbbc4aaSJason Evans 
2214bfbbc4aaSJason Evans 		if ((ts.tv_nsec < 0) || (ts.tv_nsec >= 1000000000))
2215bfbbc4aaSJason Evans 			return (EINVAL);
2216bfbbc4aaSJason Evans 
2217bfbbc4aaSJason Evans 		TIMESPEC_TO_TIMEVAL(&atv, &ts);
2218bfbbc4aaSJason Evans 		if (itimerfix(&atv))
2219bfbbc4aaSJason Evans 			return (EINVAL);
2220bfbbc4aaSJason Evans 		timo = tvtohz(&atv);
2221bfbbc4aaSJason Evans 	}
2222bfbbc4aaSJason Evans 
2223bfbbc4aaSJason Evans 	ki = p->p_aioinfo;
2224bfbbc4aaSJason Evans 	if (ki == NULL)
2225bfbbc4aaSJason Evans 		return EAGAIN;
2226bfbbc4aaSJason Evans 
2227bfbbc4aaSJason Evans 	cbptr = uap->aiocbp;
2228bfbbc4aaSJason Evans 
2229bfbbc4aaSJason Evans 	for (;;) {
2230bfbbc4aaSJason Evans 		if ((cb = TAILQ_FIRST(&ki->kaio_jobdone)) != 0) {
2231bfbbc4aaSJason Evans 			suword(uap->aiocbp, (int)cb->uuaiocb);
2232b40ce416SJulian Elischer 			td->td_retval[0] = cb->uaiocb._aiocb_private.status;
2233bfbbc4aaSJason Evans 			if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
2234bfbbc4aaSJason Evans 				curproc->p_stats->p_ru.ru_oublock +=
2235bfbbc4aaSJason Evans 				    cb->outputcharge;
2236bfbbc4aaSJason Evans 				cb->outputcharge = 0;
2237bfbbc4aaSJason Evans 			} else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
2238bfbbc4aaSJason Evans 				curproc->p_stats->p_ru.ru_inblock +=
2239bfbbc4aaSJason Evans 				    cb->inputcharge;
2240bfbbc4aaSJason Evans 				cb->inputcharge = 0;
2241bfbbc4aaSJason Evans 			}
2242bfbbc4aaSJason Evans 			aio_free_entry(cb);
2243dd85920aSJason Evans 			return cb->uaiocb._aiocb_private.error;
2244bfbbc4aaSJason Evans 		}
2245bfbbc4aaSJason Evans 
2246bfbbc4aaSJason Evans 		s = splbio();
2247bfbbc4aaSJason Evans  		if ((cb = TAILQ_FIRST(&ki->kaio_bufdone)) != 0 ) {
2248bfbbc4aaSJason Evans 			splx(s);
2249bfbbc4aaSJason Evans 			suword(uap->aiocbp, (int)cb->uuaiocb);
2250b40ce416SJulian Elischer 			td->td_retval[0] = cb->uaiocb._aiocb_private.status;
2251bfbbc4aaSJason Evans 			aio_free_entry(cb);
2252dd85920aSJason Evans 			return cb->uaiocb._aiocb_private.error;
2253bfbbc4aaSJason Evans 		}
2254bfbbc4aaSJason Evans 
2255bfbbc4aaSJason Evans 		ki->kaio_flags |= KAIO_WAKEUP;
2256bfbbc4aaSJason Evans 		error = tsleep(p, PRIBIO | PCATCH, "aiowc", timo);
2257dd85920aSJason Evans 		splx(s);
2258bfbbc4aaSJason Evans 
2259dd85920aSJason Evans 		if (error == ERESTART)
2260dd85920aSJason Evans 			return EINTR;
2261dd85920aSJason Evans 		else if (error < 0)
2262bfbbc4aaSJason Evans 			return error;
2263bfbbc4aaSJason Evans 		else if (error == EINTR)
2264bfbbc4aaSJason Evans 			return EINTR;
2265bfbbc4aaSJason Evans 		else if (error == EWOULDBLOCK)
2266bfbbc4aaSJason Evans 			return EAGAIN;
2267bfbbc4aaSJason Evans 	}
2268dd85920aSJason Evans #endif /* VFS_AIO */
2269bfbbc4aaSJason Evans }
2270cb679c38SJonathan Lemon 
22715dec52baSPeter Wemm 
22725dec52baSPeter Wemm #ifndef VFS_AIO
22735dec52baSPeter Wemm static int
22745dec52baSPeter Wemm filt_aioattach(struct knote *kn)
22755dec52baSPeter Wemm {
22765dec52baSPeter Wemm 
22775dec52baSPeter Wemm 	return (ENXIO);
22785dec52baSPeter Wemm }
22795dec52baSPeter Wemm 
22805dec52baSPeter Wemm struct filterops aio_filtops =
22815dec52baSPeter Wemm 	{ 0, filt_aioattach, NULL, NULL };
22825dec52baSPeter Wemm 
22835dec52baSPeter Wemm #else
2284cb679c38SJonathan Lemon static int
2285cb679c38SJonathan Lemon filt_aioattach(struct knote *kn)
2286cb679c38SJonathan Lemon {
2287cb679c38SJonathan Lemon 	struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_id;
2288cb679c38SJonathan Lemon 
2289cb679c38SJonathan Lemon 	/*
2290cb679c38SJonathan Lemon 	 * The aiocbe pointer must be validated before using it, so
2291cb679c38SJonathan Lemon 	 * registration is restricted to the kernel; the user cannot
2292cb679c38SJonathan Lemon 	 * set EV_FLAG1.
2293cb679c38SJonathan Lemon 	 */
2294cb679c38SJonathan Lemon 	if ((kn->kn_flags & EV_FLAG1) == 0)
2295cb679c38SJonathan Lemon 		return (EPERM);
2296cb679c38SJonathan Lemon 	kn->kn_flags &= ~EV_FLAG1;
2297cb679c38SJonathan Lemon 
2298cb679c38SJonathan Lemon 	SLIST_INSERT_HEAD(&aiocbe->klist, kn, kn_selnext);
2299cb679c38SJonathan Lemon 
2300cb679c38SJonathan Lemon 	return (0);
2301cb679c38SJonathan Lemon }
2302cb679c38SJonathan Lemon 
2303cb679c38SJonathan Lemon static void
2304cb679c38SJonathan Lemon filt_aiodetach(struct knote *kn)
2305cb679c38SJonathan Lemon {
2306cb679c38SJonathan Lemon 	struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_id;
2307cb679c38SJonathan Lemon 	int s = splhigh();	 /* XXX no clue, so overkill */
2308cb679c38SJonathan Lemon 
2309e3975643SJake Burkholder 	SLIST_REMOVE(&aiocbe->klist, kn, knote, kn_selnext);
2310cb679c38SJonathan Lemon 	splx(s);
2311cb679c38SJonathan Lemon }
2312cb679c38SJonathan Lemon 
2313cb679c38SJonathan Lemon /*ARGSUSED*/
2314cb679c38SJonathan Lemon static int
2315cb679c38SJonathan Lemon filt_aio(struct knote *kn, long hint)
2316cb679c38SJonathan Lemon {
2317cb679c38SJonathan Lemon 	struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_id;
2318cb679c38SJonathan Lemon 
2319cb679c38SJonathan Lemon 	kn->kn_data = 0;		/* XXX data returned? */
2320b70158baSAlan Cox 	if (aiocbe->jobstate != JOBST_JOBFINISHED &&
2321b70158baSAlan Cox 	    aiocbe->jobstate != JOBST_JOBBFINISHED)
2322cb679c38SJonathan Lemon 		return (0);
2323cb679c38SJonathan Lemon 	kn->kn_flags |= EV_EOF;
2324cb679c38SJonathan Lemon 	return (1);
2325cb679c38SJonathan Lemon }
23265dec52baSPeter Wemm 
23275dec52baSPeter Wemm struct filterops aio_filtops =
23285dec52baSPeter Wemm 	{ 0, filt_aioattach, filt_aiodetach, filt_aio };
23295dec52baSPeter Wemm #endif /* VFS_AIO */
2330