xref: /freebsd/sys/kern/vfs_aio.c (revision e588eeb1dc80089dd71e522f3d34ba84c38fbd76)
19454b2d8SWarner Losh /*-
2ee877a35SJohn Dyson  * Copyright (c) 1997 John S. Dyson.  All rights reserved.
3ee877a35SJohn Dyson  *
4ee877a35SJohn Dyson  * Redistribution and use in source and binary forms, with or without
5ee877a35SJohn Dyson  * modification, are permitted provided that the following conditions
6ee877a35SJohn Dyson  * are met:
7ee877a35SJohn Dyson  * 1. Redistributions of source code must retain the above copyright
8ee877a35SJohn Dyson  *    notice, this list of conditions and the following disclaimer.
9ee877a35SJohn Dyson  * 2. John S. Dyson's name may not be used to endorse or promote products
10ee877a35SJohn Dyson  *    derived from this software without specific prior written permission.
11ee877a35SJohn Dyson  *
12ee877a35SJohn Dyson  * DISCLAIMER:  This code isn't warranted to do anything useful.  Anything
13ee877a35SJohn Dyson  * bad that happens because of using this software isn't the responsibility
14ee877a35SJohn Dyson  * of the author.  This software is distributed AS-IS.
15ee877a35SJohn Dyson  */
16ee877a35SJohn Dyson 
17ee877a35SJohn Dyson /*
188a6472b7SPeter Dufault  * This file contains support for the POSIX 1003.1B AIO/LIO facility.
19ee877a35SJohn Dyson  */
20ee877a35SJohn Dyson 
21677b542eSDavid E. O'Brien #include <sys/cdefs.h>
22677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
23677b542eSDavid E. O'Brien 
243858a1f4SJohn Baldwin #include "opt_compat.h"
253858a1f4SJohn Baldwin 
26ee877a35SJohn Dyson #include <sys/param.h>
27ee877a35SJohn Dyson #include <sys/systm.h>
28f591779bSSeigo Tanimura #include <sys/malloc.h>
299626b608SPoul-Henning Kamp #include <sys/bio.h>
30a5c9bce7SBruce Evans #include <sys/buf.h>
3175b8b3b2SJohn Baldwin #include <sys/eventhandler.h>
32ee877a35SJohn Dyson #include <sys/sysproto.h>
33ee877a35SJohn Dyson #include <sys/filedesc.h>
34ee877a35SJohn Dyson #include <sys/kernel.h>
3577409fe1SPoul-Henning Kamp #include <sys/module.h>
36c9a970a7SAlan Cox #include <sys/kthread.h>
37ee877a35SJohn Dyson #include <sys/fcntl.h>
38ee877a35SJohn Dyson #include <sys/file.h>
39104a9b7eSAlexander Kabaev #include <sys/limits.h>
40fdebd4f0SBruce Evans #include <sys/lock.h>
4135e0e5b3SJohn Baldwin #include <sys/mutex.h>
42ee877a35SJohn Dyson #include <sys/unistd.h>
436aeb05d7STom Rhodes #include <sys/posix4.h>
44ee877a35SJohn Dyson #include <sys/proc.h>
452d2f8ae7SBruce Evans #include <sys/resourcevar.h>
46ee877a35SJohn Dyson #include <sys/signalvar.h>
47bfbbc4aaSJason Evans #include <sys/protosw.h>
481ce91824SDavid Xu #include <sys/sema.h>
491ce91824SDavid Xu #include <sys/socket.h>
50bfbbc4aaSJason Evans #include <sys/socketvar.h>
5121d56e9cSAlfred Perlstein #include <sys/syscall.h>
5221d56e9cSAlfred Perlstein #include <sys/sysent.h>
53a624e84fSJohn Dyson #include <sys/sysctl.h>
54ee99e978SBruce Evans #include <sys/sx.h>
551ce91824SDavid Xu #include <sys/taskqueue.h>
56fd3bf775SJohn Dyson #include <sys/vnode.h>
57fd3bf775SJohn Dyson #include <sys/conf.h>
58cb679c38SJonathan Lemon #include <sys/event.h>
5999eee864SDavid Xu #include <sys/mount.h>
60ee877a35SJohn Dyson 
611ce91824SDavid Xu #include <machine/atomic.h>
621ce91824SDavid Xu 
63ee877a35SJohn Dyson #include <vm/vm.h>
64ee877a35SJohn Dyson #include <vm/vm_extern.h>
652244ea07SJohn Dyson #include <vm/pmap.h>
662244ea07SJohn Dyson #include <vm/vm_map.h>
6799eee864SDavid Xu #include <vm/vm_object.h>
68c897b813SJeff Roberson #include <vm/uma.h>
69ee877a35SJohn Dyson #include <sys/aio.h>
705aaef07cSJohn Dyson 
71dd85920aSJason Evans #include "opt_vfs_aio.h"
72ee877a35SJohn Dyson 
73eb8e6d52SEivind Eklund /*
74eb8e6d52SEivind Eklund  * Counter for allocating reference ids to new jobs.  Wrapped to 1 on
7599eee864SDavid Xu  * overflow. (XXX will be removed soon.)
76eb8e6d52SEivind Eklund  */
7799eee864SDavid Xu static u_long jobrefid;
782244ea07SJohn Dyson 
7999eee864SDavid Xu /*
8099eee864SDavid Xu  * Counter for aio_fsync.
8199eee864SDavid Xu  */
8299eee864SDavid Xu static uint64_t jobseqno;
8399eee864SDavid Xu 
8499eee864SDavid Xu #define JOBST_NULL		0
8599eee864SDavid Xu #define JOBST_JOBQSOCK		1
8699eee864SDavid Xu #define JOBST_JOBQGLOBAL	2
8799eee864SDavid Xu #define JOBST_JOBRUNNING	3
8899eee864SDavid Xu #define JOBST_JOBFINISHED	4
8999eee864SDavid Xu #define JOBST_JOBQBUF		5
9099eee864SDavid Xu #define JOBST_JOBQSYNC		6
912244ea07SJohn Dyson 
9284af4da6SJohn Dyson #ifndef MAX_AIO_PER_PROC
932244ea07SJohn Dyson #define MAX_AIO_PER_PROC	32
9484af4da6SJohn Dyson #endif
9584af4da6SJohn Dyson 
9684af4da6SJohn Dyson #ifndef MAX_AIO_QUEUE_PER_PROC
972244ea07SJohn Dyson #define MAX_AIO_QUEUE_PER_PROC	256 /* Bigger than AIO_LISTIO_MAX */
9884af4da6SJohn Dyson #endif
9984af4da6SJohn Dyson 
10084af4da6SJohn Dyson #ifndef MAX_AIO_PROCS
101fd3bf775SJohn Dyson #define MAX_AIO_PROCS		32
10284af4da6SJohn Dyson #endif
10384af4da6SJohn Dyson 
10484af4da6SJohn Dyson #ifndef MAX_AIO_QUEUE
1052244ea07SJohn Dyson #define	MAX_AIO_QUEUE		1024 /* Bigger than AIO_LISTIO_MAX */
10684af4da6SJohn Dyson #endif
10784af4da6SJohn Dyson 
10884af4da6SJohn Dyson #ifndef TARGET_AIO_PROCS
109bfbbc4aaSJason Evans #define TARGET_AIO_PROCS	4
11084af4da6SJohn Dyson #endif
11184af4da6SJohn Dyson 
11284af4da6SJohn Dyson #ifndef MAX_BUF_AIO
11384af4da6SJohn Dyson #define MAX_BUF_AIO		16
11484af4da6SJohn Dyson #endif
11584af4da6SJohn Dyson 
11684af4da6SJohn Dyson #ifndef AIOD_TIMEOUT_DEFAULT
11784af4da6SJohn Dyson #define	AIOD_TIMEOUT_DEFAULT	(10 * hz)
11884af4da6SJohn Dyson #endif
11984af4da6SJohn Dyson 
12084af4da6SJohn Dyson #ifndef AIOD_LIFETIME_DEFAULT
12184af4da6SJohn Dyson #define AIOD_LIFETIME_DEFAULT	(30 * hz)
12284af4da6SJohn Dyson #endif
1232244ea07SJohn Dyson 
124e603be7aSRobert Watson FEATURE(aio, "Asynchronous I/O");
125e603be7aSRobert Watson 
1263858a1f4SJohn Baldwin static MALLOC_DEFINE(M_LIO, "lio", "listio aio control block list");
1273858a1f4SJohn Baldwin 
1285ece08f5SPoul-Henning Kamp static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "Async IO management");
129eb8e6d52SEivind Eklund 
130303b270bSEivind Eklund static int max_aio_procs = MAX_AIO_PROCS;
131a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs,
132eb8e6d52SEivind Eklund 	CTLFLAG_RW, &max_aio_procs, 0,
133eb8e6d52SEivind Eklund 	"Maximum number of kernel threads to use for handling async IO ");
134a624e84fSJohn Dyson 
135eb8e6d52SEivind Eklund static int num_aio_procs = 0;
136a624e84fSJohn Dyson SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs,
137eb8e6d52SEivind Eklund 	CTLFLAG_RD, &num_aio_procs, 0,
138eb8e6d52SEivind Eklund 	"Number of presently active kernel threads for async IO");
139a624e84fSJohn Dyson 
140eb8e6d52SEivind Eklund /*
141eb8e6d52SEivind Eklund  * The code will adjust the actual number of AIO processes towards this
142eb8e6d52SEivind Eklund  * number when it gets a chance.
143eb8e6d52SEivind Eklund  */
144eb8e6d52SEivind Eklund static int target_aio_procs = TARGET_AIO_PROCS;
145eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW, &target_aio_procs,
146eb8e6d52SEivind Eklund 	0, "Preferred number of ready kernel threads for async IO");
147a624e84fSJohn Dyson 
148eb8e6d52SEivind Eklund static int max_queue_count = MAX_AIO_QUEUE;
149eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW, &max_queue_count, 0,
150eb8e6d52SEivind Eklund     "Maximum number of aio requests to queue, globally");
151a624e84fSJohn Dyson 
152eb8e6d52SEivind Eklund static int num_queue_count = 0;
153eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD, &num_queue_count, 0,
154eb8e6d52SEivind Eklund     "Number of queued aio requests");
155a624e84fSJohn Dyson 
156eb8e6d52SEivind Eklund static int num_buf_aio = 0;
157eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD, &num_buf_aio, 0,
158eb8e6d52SEivind Eklund     "Number of aio requests presently handled by the buf subsystem");
159fd3bf775SJohn Dyson 
160eb8e6d52SEivind Eklund /* Number of async I/O thread in the process of being started */
161a9bf5e37SDavid Xu /* XXX This should be local to aio_aqueue() */
162eb8e6d52SEivind Eklund static int num_aio_resv_start = 0;
163fd3bf775SJohn Dyson 
164eb8e6d52SEivind Eklund static int aiod_timeout;
165eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout, CTLFLAG_RW, &aiod_timeout, 0,
166eb8e6d52SEivind Eklund     "Timeout value for synchronous aio operations");
16784af4da6SJohn Dyson 
168eb8e6d52SEivind Eklund static int aiod_lifetime;
169eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW, &aiod_lifetime, 0,
170eb8e6d52SEivind Eklund     "Maximum lifetime for idle aiod");
17184af4da6SJohn Dyson 
172eb8e6d52SEivind Eklund static int unloadable = 0;
17321d56e9cSAlfred Perlstein SYSCTL_INT(_vfs_aio, OID_AUTO, unloadable, CTLFLAG_RW, &unloadable, 0,
17421d56e9cSAlfred Perlstein     "Allow unload of aio (not recommended)");
17521d56e9cSAlfred Perlstein 
176eb8e6d52SEivind Eklund 
177eb8e6d52SEivind Eklund static int max_aio_per_proc = MAX_AIO_PER_PROC;
178eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW, &max_aio_per_proc,
179eb8e6d52SEivind Eklund     0, "Maximum active aio requests per process (stored in the process)");
180eb8e6d52SEivind Eklund 
181eb8e6d52SEivind Eklund static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
182eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW,
183eb8e6d52SEivind Eklund     &max_aio_queue_per_proc, 0,
184eb8e6d52SEivind Eklund     "Maximum queued aio requests per process (stored in the process)");
185eb8e6d52SEivind Eklund 
186eb8e6d52SEivind Eklund static int max_buf_aio = MAX_BUF_AIO;
187eb8e6d52SEivind Eklund SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0,
188eb8e6d52SEivind Eklund     "Maximum buf aio requests per process (stored in the process)");
189eb8e6d52SEivind Eklund 
1900972628aSDavid Xu typedef struct oaiocb {
1910972628aSDavid Xu 	int	aio_fildes;		/* File descriptor */
1920972628aSDavid Xu 	off_t	aio_offset;		/* File offset for I/O */
1930972628aSDavid Xu 	volatile void *aio_buf;         /* I/O buffer in process space */
1940972628aSDavid Xu 	size_t	aio_nbytes;		/* Number of bytes for I/O */
1950972628aSDavid Xu 	struct	osigevent aio_sigevent;	/* Signal to deliver */
1960972628aSDavid Xu 	int	aio_lio_opcode;		/* LIO opcode */
1970972628aSDavid Xu 	int	aio_reqprio;		/* Request priority -- ignored */
1980972628aSDavid Xu 	struct	__aiocb_private	_aiocb_private;
1990972628aSDavid Xu } oaiocb_t;
2000972628aSDavid Xu 
2011aa4c324SDavid Xu /*
2021aa4c324SDavid Xu  * Below is a key of locks used to protect each member of struct aiocblist
2031aa4c324SDavid Xu  * aioliojob and kaioinfo and any backends.
2041aa4c324SDavid Xu  *
2051aa4c324SDavid Xu  * * - need not protected
206759ccccaSDavid Xu  * a - locked by kaioinfo lock
2071aa4c324SDavid Xu  * b - locked by backend lock, the backend lock can be null in some cases,
2081aa4c324SDavid Xu  *     for example, BIO belongs to this type, in this case, proc lock is
2091aa4c324SDavid Xu  *     reused.
2101aa4c324SDavid Xu  * c - locked by aio_job_mtx, the lock for the generic file I/O backend.
2111aa4c324SDavid Xu  */
2121aa4c324SDavid Xu 
2131aa4c324SDavid Xu /*
2141aa4c324SDavid Xu  * Current, there is only two backends: BIO and generic file I/O.
2151aa4c324SDavid Xu  * socket I/O is served by generic file I/O, this is not a good idea, since
2161aa4c324SDavid Xu  * disk file I/O and any other types without O_NONBLOCK flag can block daemon
2171aa4c324SDavid Xu  * threads, if there is no thread to serve socket I/O, the socket I/O will be
2181aa4c324SDavid Xu  * delayed too long or starved, we should create some threads dedicated to
2191aa4c324SDavid Xu  * sockets to do non-blocking I/O, same for pipe and fifo, for these I/O
2201aa4c324SDavid Xu  * systems we really need non-blocking interface, fiddling O_NONBLOCK in file
2211aa4c324SDavid Xu  * structure is not safe because there is race between userland and aio
2221aa4c324SDavid Xu  * daemons.
2231aa4c324SDavid Xu  */
2241aa4c324SDavid Xu 
22548dac059SAlan Cox struct aiocblist {
2261aa4c324SDavid Xu 	TAILQ_ENTRY(aiocblist) list;	/* (b) internal list of for backend */
2271aa4c324SDavid Xu 	TAILQ_ENTRY(aiocblist) plist;	/* (a) list of jobs for each backend */
2281aa4c324SDavid Xu 	TAILQ_ENTRY(aiocblist) allist;  /* (a) list of all jobs in proc */
2291aa4c324SDavid Xu 	int	jobflags;		/* (a) job flags */
2301aa4c324SDavid Xu 	int	jobstate;		/* (b) job state */
2311aa4c324SDavid Xu 	int	inputcharge;		/* (*) input blockes */
2321aa4c324SDavid Xu 	int	outputcharge;		/* (*) output blockes */
2331aa4c324SDavid Xu 	struct	buf *bp;		/* (*) private to BIO backend,
2341aa4c324SDavid Xu 				  	 * buffer pointer
2351aa4c324SDavid Xu 					 */
2361aa4c324SDavid Xu 	struct	proc *userproc;		/* (*) user process */
2371aa4c324SDavid Xu 	struct  ucred *cred;		/* (*) active credential when created */
2381aa4c324SDavid Xu 	struct	file *fd_file;		/* (*) pointer to file structure */
2391aa4c324SDavid Xu 	struct	aioliojob *lio;		/* (*) optional lio job */
2401aa4c324SDavid Xu 	struct	aiocb *uuaiocb;		/* (*) pointer in userspace of aiocb */
2411aa4c324SDavid Xu 	struct	knlist klist;		/* (a) list of knotes */
2421aa4c324SDavid Xu 	struct	aiocb uaiocb;		/* (*) kernel I/O control block */
2431aa4c324SDavid Xu 	ksiginfo_t ksi;			/* (a) realtime signal info */
2441aa4c324SDavid Xu 	struct	task biotask;		/* (*) private to BIO backend */
24599eee864SDavid Xu 	uint64_t seqno;			/* (*) job number */
24699eee864SDavid Xu 	int	pending;		/* (a) number of pending I/O, aio_fsync only */
24748dac059SAlan Cox };
24848dac059SAlan Cox 
24948dac059SAlan Cox /* jobflags */
25099eee864SDavid Xu #define AIOCBLIST_DONE		0x01
25199eee864SDavid Xu #define AIOCBLIST_BUFDONE	0x02
2521ce91824SDavid Xu #define AIOCBLIST_RUNDOWN	0x04
25399eee864SDavid Xu #define AIOCBLIST_CHECKSYNC	0x08
25448dac059SAlan Cox 
2552244ea07SJohn Dyson /*
2562244ea07SJohn Dyson  * AIO process info
2572244ea07SJohn Dyson  */
25884af4da6SJohn Dyson #define AIOP_FREE	0x1			/* proc on free queue */
25984af4da6SJohn Dyson 
260b40ce416SJulian Elischer struct aiothreadlist {
2611aa4c324SDavid Xu 	int aiothreadflags;			/* (c) AIO proc flags */
2621aa4c324SDavid Xu 	TAILQ_ENTRY(aiothreadlist) list;	/* (c) list of processes */
2631aa4c324SDavid Xu 	struct thread *aiothread;		/* (*) the AIO thread */
2642244ea07SJohn Dyson };
2652244ea07SJohn Dyson 
26684af4da6SJohn Dyson /*
26784af4da6SJohn Dyson  * data-structure for lio signal management
26884af4da6SJohn Dyson  */
2691ce91824SDavid Xu struct aioliojob {
2701aa4c324SDavid Xu 	int	lioj_flags;			/* (a) listio flags */
2711aa4c324SDavid Xu 	int	lioj_count;			/* (a) listio flags */
2721aa4c324SDavid Xu 	int	lioj_finished_count;		/* (a) listio flags */
2731aa4c324SDavid Xu 	struct	sigevent lioj_signal;		/* (a) signal on all I/O done */
2741aa4c324SDavid Xu 	TAILQ_ENTRY(aioliojob) lioj_list;	/* (a) lio list */
2751aa4c324SDavid Xu 	struct  knlist klist;			/* (a) list of knotes */
2761aa4c324SDavid Xu 	ksiginfo_t lioj_ksi;			/* (a) Realtime signal info */
27784af4da6SJohn Dyson };
2781ce91824SDavid Xu 
27984af4da6SJohn Dyson #define	LIOJ_SIGNAL		0x1	/* signal on all done (lio) */
28084af4da6SJohn Dyson #define	LIOJ_SIGNAL_POSTED	0x2	/* signal has been posted */
28169cd28daSDoug Ambrisko #define LIOJ_KEVENT_POSTED	0x4	/* kevent triggered */
28284af4da6SJohn Dyson 
28384af4da6SJohn Dyson /*
28484af4da6SJohn Dyson  * per process aio data structure
28584af4da6SJohn Dyson  */
2862244ea07SJohn Dyson struct kaioinfo {
287759ccccaSDavid Xu 	struct mtx	kaio_mtx;	/* the lock to protect this struct */
2881aa4c324SDavid Xu 	int	kaio_flags;		/* (a) per process kaio flags */
2891aa4c324SDavid Xu 	int	kaio_maxactive_count;	/* (*) maximum number of AIOs */
2901aa4c324SDavid Xu 	int	kaio_active_count;	/* (c) number of currently used AIOs */
2911aa4c324SDavid Xu 	int	kaio_qallowed_count;	/* (*) maxiumu size of AIO queue */
2921aa4c324SDavid Xu 	int	kaio_count;		/* (a) size of AIO queue */
2931aa4c324SDavid Xu 	int	kaio_ballowed_count;	/* (*) maximum number of buffers */
2941aa4c324SDavid Xu 	int	kaio_buffer_count;	/* (a) number of physio buffers */
2951aa4c324SDavid Xu 	TAILQ_HEAD(,aiocblist) kaio_all;	/* (a) all AIOs in the process */
2961aa4c324SDavid Xu 	TAILQ_HEAD(,aiocblist) kaio_done;	/* (a) done queue for process */
2971aa4c324SDavid Xu 	TAILQ_HEAD(,aioliojob) kaio_liojoblist; /* (a) list of lio jobs */
2981aa4c324SDavid Xu 	TAILQ_HEAD(,aiocblist) kaio_jobqueue;	/* (a) job queue for process */
2991aa4c324SDavid Xu 	TAILQ_HEAD(,aiocblist) kaio_bufqueue;	/* (a) buffer job queue for process */
3001aa4c324SDavid Xu 	TAILQ_HEAD(,aiocblist) kaio_sockqueue;  /* (a) queue for aios waiting on sockets,
30199eee864SDavid Xu 						 *  NOT USED YET.
3021aa4c324SDavid Xu 						 */
30399eee864SDavid Xu 	TAILQ_HEAD(,aiocblist) kaio_syncqueue;	/* (a) queue for aio_fsync */
30499eee864SDavid Xu 	struct	task	kaio_task;	/* (*) task to kick aio threads */
3052244ea07SJohn Dyson };
3062244ea07SJohn Dyson 
307759ccccaSDavid Xu #define AIO_LOCK(ki)		mtx_lock(&(ki)->kaio_mtx)
308759ccccaSDavid Xu #define AIO_UNLOCK(ki)		mtx_unlock(&(ki)->kaio_mtx)
309759ccccaSDavid Xu #define AIO_LOCK_ASSERT(ki, f)	mtx_assert(&(ki)->kaio_mtx, (f))
310759ccccaSDavid Xu #define AIO_MTX(ki)		(&(ki)->kaio_mtx)
311759ccccaSDavid Xu 
31284af4da6SJohn Dyson #define KAIO_RUNDOWN	0x1	/* process is being run down */
313bfbbc4aaSJason Evans #define KAIO_WAKEUP	0x2	/* wakeup process when there is a significant event */
314fd3bf775SJohn Dyson 
3153858a1f4SJohn Baldwin /*
3163858a1f4SJohn Baldwin  * Operations used to interact with userland aio control blocks.
3173858a1f4SJohn Baldwin  * Different ABIs provide their own operations.
3183858a1f4SJohn Baldwin  */
3193858a1f4SJohn Baldwin struct aiocb_ops {
3203858a1f4SJohn Baldwin 	int	(*copyin)(struct aiocb *ujob, struct aiocb *kjob);
3213858a1f4SJohn Baldwin 	long	(*fetch_status)(struct aiocb *ujob);
3223858a1f4SJohn Baldwin 	long	(*fetch_error)(struct aiocb *ujob);
3233858a1f4SJohn Baldwin 	int	(*store_status)(struct aiocb *ujob, long status);
3243858a1f4SJohn Baldwin 	int	(*store_error)(struct aiocb *ujob, long error);
3253858a1f4SJohn Baldwin 	int	(*store_kernelinfo)(struct aiocb *ujob, long jobref);
3263858a1f4SJohn Baldwin 	int	(*store_aiocb)(struct aiocb **ujobp, struct aiocb *ujob);
3273858a1f4SJohn Baldwin };
3283858a1f4SJohn Baldwin 
3291aa4c324SDavid Xu static TAILQ_HEAD(,aiothreadlist) aio_freeproc;		/* (c) Idle daemons */
3301ce91824SDavid Xu static struct sema aio_newproc_sem;
3311ce91824SDavid Xu static struct mtx aio_job_mtx;
3321ce91824SDavid Xu static struct mtx aio_sock_mtx;
3331aa4c324SDavid Xu static TAILQ_HEAD(,aiocblist) aio_jobs;			/* (c) Async job list */
3341ce91824SDavid Xu static struct unrhdr *aiod_unr;
3352244ea07SJohn Dyson 
3366a1162d4SAlexander Leidinger void		aio_init_aioinfo(struct proc *p);
33721d56e9cSAlfred Perlstein static void	aio_onceonly(void);
338fd3bf775SJohn Dyson static int	aio_free_entry(struct aiocblist *aiocbe);
339fd3bf775SJohn Dyson static void	aio_process(struct aiocblist *aiocbe);
3401ce91824SDavid Xu static int	aio_newproc(int *);
3416a1162d4SAlexander Leidinger int		aio_aqueue(struct thread *td, struct aiocb *job,
3423858a1f4SJohn Baldwin 			struct aioliojob *lio, int type, struct aiocb_ops *ops);
343fd3bf775SJohn Dyson static void	aio_physwakeup(struct buf *bp);
34475b8b3b2SJohn Baldwin static void	aio_proc_rundown(void *arg, struct proc *p);
345993182e5SAlexander Leidinger static void	aio_proc_rundown_exec(void *arg, struct proc *p, struct image_params *imgp);
346fd3bf775SJohn Dyson static int	aio_qphysio(struct proc *p, struct aiocblist *iocb);
3471ce91824SDavid Xu static void	biohelper(void *, int);
3481ce91824SDavid Xu static void	aio_daemon(void *param);
34948dac059SAlan Cox static void	aio_swake_cb(struct socket *, struct sockbuf *);
35021d56e9cSAlfred Perlstein static int	aio_unload(void);
35199eee864SDavid Xu static void	aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type);
35299eee864SDavid Xu #define DONE_BUF	1
35399eee864SDavid Xu #define DONE_QUEUE	2
354dbbccfe9SDavid Xu static int	aio_kick(struct proc *userp);
35599eee864SDavid Xu static void	aio_kick_nowait(struct proc *userp);
35699eee864SDavid Xu static void	aio_kick_helper(void *context, int pending);
35721d56e9cSAlfred Perlstein static int	filt_aioattach(struct knote *kn);
35821d56e9cSAlfred Perlstein static void	filt_aiodetach(struct knote *kn);
35921d56e9cSAlfred Perlstein static int	filt_aio(struct knote *kn, long hint);
36069cd28daSDoug Ambrisko static int	filt_lioattach(struct knote *kn);
36169cd28daSDoug Ambrisko static void	filt_liodetach(struct knote *kn);
36269cd28daSDoug Ambrisko static int	filt_lio(struct knote *kn, long hint);
3632244ea07SJohn Dyson 
364eb8e6d52SEivind Eklund /*
365eb8e6d52SEivind Eklund  * Zones for:
366eb8e6d52SEivind Eklund  * 	kaio	Per process async io info
367eb8e6d52SEivind Eklund  *	aiop	async io thread data
368eb8e6d52SEivind Eklund  *	aiocb	async io jobs
369eb8e6d52SEivind Eklund  *	aiol	list io job pointer - internal to aio_suspend XXX
370eb8e6d52SEivind Eklund  *	aiolio	list io jobs
371eb8e6d52SEivind Eklund  */
372c897b813SJeff Roberson static uma_zone_t kaio_zone, aiop_zone, aiocb_zone, aiol_zone, aiolio_zone;
373fd3bf775SJohn Dyson 
374eb8e6d52SEivind Eklund /* kqueue filters for aio */
37521d56e9cSAlfred Perlstein static struct filterops aio_filtops =
37621d56e9cSAlfred Perlstein 	{ 0, filt_aioattach, filt_aiodetach, filt_aio };
37769cd28daSDoug Ambrisko static struct filterops lio_filtops =
37869cd28daSDoug Ambrisko 	{ 0, filt_lioattach, filt_liodetach, filt_lio };
37921d56e9cSAlfred Perlstein 
38075b8b3b2SJohn Baldwin static eventhandler_tag exit_tag, exec_tag;
38175b8b3b2SJohn Baldwin 
3821ce91824SDavid Xu TASKQUEUE_DEFINE_THREAD(aiod_bio);
3831ce91824SDavid Xu 
384eb8e6d52SEivind Eklund /*
385eb8e6d52SEivind Eklund  * Main operations function for use as a kernel module.
386eb8e6d52SEivind Eklund  */
38721d56e9cSAlfred Perlstein static int
38821d56e9cSAlfred Perlstein aio_modload(struct module *module, int cmd, void *arg)
38921d56e9cSAlfred Perlstein {
39021d56e9cSAlfred Perlstein 	int error = 0;
39121d56e9cSAlfred Perlstein 
39221d56e9cSAlfred Perlstein 	switch (cmd) {
39321d56e9cSAlfred Perlstein 	case MOD_LOAD:
39421d56e9cSAlfred Perlstein 		aio_onceonly();
39521d56e9cSAlfred Perlstein 		break;
39621d56e9cSAlfred Perlstein 	case MOD_UNLOAD:
39721d56e9cSAlfred Perlstein 		error = aio_unload();
39821d56e9cSAlfred Perlstein 		break;
39921d56e9cSAlfred Perlstein 	case MOD_SHUTDOWN:
40021d56e9cSAlfred Perlstein 		break;
40121d56e9cSAlfred Perlstein 	default:
40221d56e9cSAlfred Perlstein 		error = EINVAL;
40321d56e9cSAlfred Perlstein 		break;
40421d56e9cSAlfred Perlstein 	}
40521d56e9cSAlfred Perlstein 	return (error);
40621d56e9cSAlfred Perlstein }
40721d56e9cSAlfred Perlstein 
40821d56e9cSAlfred Perlstein static moduledata_t aio_mod = {
40921d56e9cSAlfred Perlstein 	"aio",
41021d56e9cSAlfred Perlstein 	&aio_modload,
41121d56e9cSAlfred Perlstein 	NULL
41221d56e9cSAlfred Perlstein };
41321d56e9cSAlfred Perlstein 
41421d56e9cSAlfred Perlstein SYSCALL_MODULE_HELPER(aio_cancel);
41521d56e9cSAlfred Perlstein SYSCALL_MODULE_HELPER(aio_error);
41699eee864SDavid Xu SYSCALL_MODULE_HELPER(aio_fsync);
41721d56e9cSAlfred Perlstein SYSCALL_MODULE_HELPER(aio_read);
41899eee864SDavid Xu SYSCALL_MODULE_HELPER(aio_return);
41999eee864SDavid Xu SYSCALL_MODULE_HELPER(aio_suspend);
42021d56e9cSAlfred Perlstein SYSCALL_MODULE_HELPER(aio_waitcomplete);
42199eee864SDavid Xu SYSCALL_MODULE_HELPER(aio_write);
42221d56e9cSAlfred Perlstein SYSCALL_MODULE_HELPER(lio_listio);
4230972628aSDavid Xu SYSCALL_MODULE_HELPER(oaio_read);
4240972628aSDavid Xu SYSCALL_MODULE_HELPER(oaio_write);
4250972628aSDavid Xu SYSCALL_MODULE_HELPER(olio_listio);
42621d56e9cSAlfred Perlstein 
42721d56e9cSAlfred Perlstein DECLARE_MODULE(aio, aio_mod,
42821d56e9cSAlfred Perlstein 	SI_SUB_VFS, SI_ORDER_ANY);
42921d56e9cSAlfred Perlstein MODULE_VERSION(aio, 1);
43021d56e9cSAlfred Perlstein 
431fd3bf775SJohn Dyson /*
4322244ea07SJohn Dyson  * Startup initialization
4332244ea07SJohn Dyson  */
43488ed460eSAlan Cox static void
43521d56e9cSAlfred Perlstein aio_onceonly(void)
436fd3bf775SJohn Dyson {
43721d56e9cSAlfred Perlstein 
43821d56e9cSAlfred Perlstein 	/* XXX: should probably just use so->callback */
43921d56e9cSAlfred Perlstein 	aio_swake = &aio_swake_cb;
44075b8b3b2SJohn Baldwin 	exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL,
44175b8b3b2SJohn Baldwin 	    EVENTHANDLER_PRI_ANY);
442993182e5SAlexander Leidinger 	exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown_exec, NULL,
44375b8b3b2SJohn Baldwin 	    EVENTHANDLER_PRI_ANY);
44421d56e9cSAlfred Perlstein 	kqueue_add_filteropts(EVFILT_AIO, &aio_filtops);
44569cd28daSDoug Ambrisko 	kqueue_add_filteropts(EVFILT_LIO, &lio_filtops);
4462244ea07SJohn Dyson 	TAILQ_INIT(&aio_freeproc);
4471ce91824SDavid Xu 	sema_init(&aio_newproc_sem, 0, "aio_new_proc");
4481ce91824SDavid Xu 	mtx_init(&aio_job_mtx, "aio_job", NULL, MTX_DEF);
4491ce91824SDavid Xu 	mtx_init(&aio_sock_mtx, "aio_sock", NULL, MTX_DEF);
4502244ea07SJohn Dyson 	TAILQ_INIT(&aio_jobs);
4511ce91824SDavid Xu 	aiod_unr = new_unrhdr(1, INT_MAX, NULL);
452c897b813SJeff Roberson 	kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL,
453c897b813SJeff Roberson 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
454c897b813SJeff Roberson 	aiop_zone = uma_zcreate("AIOP", sizeof(struct aiothreadlist), NULL,
455c897b813SJeff Roberson 	    NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
456c897b813SJeff Roberson 	aiocb_zone = uma_zcreate("AIOCB", sizeof(struct aiocblist), NULL, NULL,
457c897b813SJeff Roberson 	    NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
458c897b813SJeff Roberson 	aiol_zone = uma_zcreate("AIOL", AIO_LISTIO_MAX*sizeof(intptr_t) , NULL,
459c897b813SJeff Roberson 	    NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
4601ce91824SDavid Xu 	aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aioliojob), NULL,
461c897b813SJeff Roberson 	    NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE);
46284af4da6SJohn Dyson 	aiod_timeout = AIOD_TIMEOUT_DEFAULT;
46384af4da6SJohn Dyson 	aiod_lifetime = AIOD_LIFETIME_DEFAULT;
464fd3bf775SJohn Dyson 	jobrefid = 1;
465c7047e52SGarrett Wollman 	async_io_version = _POSIX_VERSION;
466c844abc9SAlfred Perlstein 	p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, AIO_LISTIO_MAX);
46786d52125SAlfred Perlstein 	p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE);
46886d52125SAlfred Perlstein 	p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0);
4692244ea07SJohn Dyson }
4702244ea07SJohn Dyson 
471eb8e6d52SEivind Eklund /*
472eb8e6d52SEivind Eklund  * Callback for unload of AIO when used as a module.
473eb8e6d52SEivind Eklund  */
47421d56e9cSAlfred Perlstein static int
47521d56e9cSAlfred Perlstein aio_unload(void)
47621d56e9cSAlfred Perlstein {
477ad3b9257SJohn-Mark Gurney 	int error;
47821d56e9cSAlfred Perlstein 
47921d56e9cSAlfred Perlstein 	/*
48021d56e9cSAlfred Perlstein 	 * XXX: no unloads by default, it's too dangerous.
48121d56e9cSAlfred Perlstein 	 * perhaps we could do it if locked out callers and then
48221d56e9cSAlfred Perlstein 	 * did an aio_proc_rundown() on each process.
4832a522eb9SJohn Baldwin 	 *
4842a522eb9SJohn Baldwin 	 * jhb: aio_proc_rundown() needs to run on curproc though,
4852a522eb9SJohn Baldwin 	 * so I don't think that would fly.
48621d56e9cSAlfred Perlstein 	 */
48721d56e9cSAlfred Perlstein 	if (!unloadable)
48821d56e9cSAlfred Perlstein 		return (EOPNOTSUPP);
48921d56e9cSAlfred Perlstein 
490ad3b9257SJohn-Mark Gurney 	error = kqueue_del_filteropts(EVFILT_AIO);
491ad3b9257SJohn-Mark Gurney 	if (error)
492ad3b9257SJohn-Mark Gurney 		return error;
493bd793be3SDavid Xu 	error = kqueue_del_filteropts(EVFILT_LIO);
494bd793be3SDavid Xu 	if (error)
495bd793be3SDavid Xu 		return error;
496c7047e52SGarrett Wollman 	async_io_version = 0;
49721d56e9cSAlfred Perlstein 	aio_swake = NULL;
4981ce91824SDavid Xu 	taskqueue_free(taskqueue_aiod_bio);
4991ce91824SDavid Xu 	delete_unrhdr(aiod_unr);
500bd793be3SDavid Xu 	uma_zdestroy(kaio_zone);
501bd793be3SDavid Xu 	uma_zdestroy(aiop_zone);
502bd793be3SDavid Xu 	uma_zdestroy(aiocb_zone);
503bd793be3SDavid Xu 	uma_zdestroy(aiol_zone);
504bd793be3SDavid Xu 	uma_zdestroy(aiolio_zone);
50575b8b3b2SJohn Baldwin 	EVENTHANDLER_DEREGISTER(process_exit, exit_tag);
50675b8b3b2SJohn Baldwin 	EVENTHANDLER_DEREGISTER(process_exec, exec_tag);
5071ce91824SDavid Xu 	mtx_destroy(&aio_job_mtx);
5081ce91824SDavid Xu 	mtx_destroy(&aio_sock_mtx);
5091ce91824SDavid Xu 	sema_destroy(&aio_newproc_sem);
510f51c1e89SAlfred Perlstein 	p31b_setcfg(CTL_P1003_1B_AIO_LISTIO_MAX, -1);
511f51c1e89SAlfred Perlstein 	p31b_setcfg(CTL_P1003_1B_AIO_MAX, -1);
512f51c1e89SAlfred Perlstein 	p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, -1);
51321d56e9cSAlfred Perlstein 	return (0);
51421d56e9cSAlfred Perlstein }
51521d56e9cSAlfred Perlstein 
5162244ea07SJohn Dyson /*
517bfbbc4aaSJason Evans  * Init the per-process aioinfo structure.  The aioinfo limits are set
518bfbbc4aaSJason Evans  * per-process for user limit (resource) management.
5192244ea07SJohn Dyson  */
5206a1162d4SAlexander Leidinger void
521fd3bf775SJohn Dyson aio_init_aioinfo(struct proc *p)
522fd3bf775SJohn Dyson {
5232244ea07SJohn Dyson 	struct kaioinfo *ki;
524ac41f2efSAlfred Perlstein 
525a163d034SWarner Losh 	ki = uma_zalloc(kaio_zone, M_WAITOK);
526759ccccaSDavid Xu 	mtx_init(&ki->kaio_mtx, "aiomtx", NULL, MTX_DEF);
52784af4da6SJohn Dyson 	ki->kaio_flags = 0;
528a624e84fSJohn Dyson 	ki->kaio_maxactive_count = max_aio_per_proc;
5292244ea07SJohn Dyson 	ki->kaio_active_count = 0;
530a624e84fSJohn Dyson 	ki->kaio_qallowed_count = max_aio_queue_per_proc;
5311ce91824SDavid Xu 	ki->kaio_count = 0;
53284af4da6SJohn Dyson 	ki->kaio_ballowed_count = max_buf_aio;
533fd3bf775SJohn Dyson 	ki->kaio_buffer_count = 0;
5341ce91824SDavid Xu 	TAILQ_INIT(&ki->kaio_all);
5351ce91824SDavid Xu 	TAILQ_INIT(&ki->kaio_done);
5362244ea07SJohn Dyson 	TAILQ_INIT(&ki->kaio_jobqueue);
537fd3bf775SJohn Dyson 	TAILQ_INIT(&ki->kaio_bufqueue);
53884af4da6SJohn Dyson 	TAILQ_INIT(&ki->kaio_liojoblist);
539bfbbc4aaSJason Evans 	TAILQ_INIT(&ki->kaio_sockqueue);
54099eee864SDavid Xu 	TAILQ_INIT(&ki->kaio_syncqueue);
54199eee864SDavid Xu 	TASK_INIT(&ki->kaio_task, 0, aio_kick_helper, p);
5423999ebe3SAlan Cox 	PROC_LOCK(p);
5433999ebe3SAlan Cox 	if (p->p_aioinfo == NULL) {
5443999ebe3SAlan Cox 		p->p_aioinfo = ki;
5453999ebe3SAlan Cox 		PROC_UNLOCK(p);
5463999ebe3SAlan Cox 	} else {
5473999ebe3SAlan Cox 		PROC_UNLOCK(p);
548759ccccaSDavid Xu 		mtx_destroy(&ki->kaio_mtx);
5493999ebe3SAlan Cox 		uma_zfree(kaio_zone, ki);
5502244ea07SJohn Dyson 	}
551bfbbc4aaSJason Evans 
55222035f47SOleksandr Tymoshenko 	while (num_aio_procs < MIN(target_aio_procs, max_aio_procs))
5531ce91824SDavid Xu 		aio_newproc(NULL);
5542244ea07SJohn Dyson }
5552244ea07SJohn Dyson 
5564c0fb2cfSDavid Xu static int
5574c0fb2cfSDavid Xu aio_sendsig(struct proc *p, struct sigevent *sigev, ksiginfo_t *ksi)
5584c0fb2cfSDavid Xu {
559759ccccaSDavid Xu 	int ret = 0;
560759ccccaSDavid Xu 
561759ccccaSDavid Xu 	PROC_LOCK(p);
5624c0fb2cfSDavid Xu 	if (!KSI_ONQ(ksi)) {
5634c0fb2cfSDavid Xu 		ksi->ksi_code = SI_ASYNCIO;
5644c0fb2cfSDavid Xu 		ksi->ksi_flags |= KSI_EXT | KSI_INS;
565759ccccaSDavid Xu 		ret = psignal_event(p, sigev, ksi);
5664c0fb2cfSDavid Xu 	}
567759ccccaSDavid Xu 	PROC_UNLOCK(p);
568759ccccaSDavid Xu 	return (ret);
5694c0fb2cfSDavid Xu }
5704c0fb2cfSDavid Xu 
5712244ea07SJohn Dyson /*
572bfbbc4aaSJason Evans  * Free a job entry.  Wait for completion if it is currently active, but don't
573bfbbc4aaSJason Evans  * delay forever.  If we delay, we return a flag that says that we have to
574bfbbc4aaSJason Evans  * restart the queue scan.
5752244ea07SJohn Dyson  */
57688ed460eSAlan Cox static int
577fd3bf775SJohn Dyson aio_free_entry(struct aiocblist *aiocbe)
578fd3bf775SJohn Dyson {
5792244ea07SJohn Dyson 	struct kaioinfo *ki;
5801ce91824SDavid Xu 	struct aioliojob *lj;
5812244ea07SJohn Dyson 	struct proc *p;
5822244ea07SJohn Dyson 
5832244ea07SJohn Dyson 	p = aiocbe->userproc;
5841ce91824SDavid Xu 	MPASS(curproc == p);
5852244ea07SJohn Dyson 	ki = p->p_aioinfo;
5861ce91824SDavid Xu 	MPASS(ki != NULL);
5871ce91824SDavid Xu 
588759ccccaSDavid Xu 	AIO_LOCK_ASSERT(ki, MA_OWNED);
589759ccccaSDavid Xu 	MPASS(aiocbe->jobstate == JOBST_JOBFINISHED);
590759ccccaSDavid Xu 
5911ce91824SDavid Xu 	atomic_subtract_int(&num_queue_count, 1);
5921ce91824SDavid Xu 
5931ce91824SDavid Xu 	ki->kaio_count--;
5941ce91824SDavid Xu 	MPASS(ki->kaio_count >= 0);
5951ce91824SDavid Xu 
59627b8220dSDavid Xu 	TAILQ_REMOVE(&ki->kaio_done, aiocbe, plist);
59727b8220dSDavid Xu 	TAILQ_REMOVE(&ki->kaio_all, aiocbe, allist);
59827b8220dSDavid Xu 
59984af4da6SJohn Dyson 	lj = aiocbe->lio;
60084af4da6SJohn Dyson 	if (lj) {
6011ce91824SDavid Xu 		lj->lioj_count--;
6021ce91824SDavid Xu 		lj->lioj_finished_count--;
6031ce91824SDavid Xu 
604a9bf5e37SDavid Xu 		if (lj->lioj_count == 0) {
6051ce91824SDavid Xu 			TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
6061ce91824SDavid Xu 			/* lio is going away, we need to destroy any knotes */
6071ce91824SDavid Xu 			knlist_delete(&lj->klist, curthread, 1);
608759ccccaSDavid Xu 			PROC_LOCK(p);
6091ce91824SDavid Xu 			sigqueue_take(&lj->lioj_ksi);
610759ccccaSDavid Xu 			PROC_UNLOCK(p);
6111ce91824SDavid Xu 			uma_zfree(aiolio_zone, lj);
61284af4da6SJohn Dyson 		}
61384af4da6SJohn Dyson 	}
6141ce91824SDavid Xu 
615cb679c38SJonathan Lemon 	/* aiocbe is going away, we need to destroy any knotes */
6161ce91824SDavid Xu 	knlist_delete(&aiocbe->klist, curthread, 1);
617759ccccaSDavid Xu 	PROC_LOCK(p);
6181ce91824SDavid Xu 	sigqueue_take(&aiocbe->ksi);
619759ccccaSDavid Xu 	PROC_UNLOCK(p);
6201ce91824SDavid Xu 
6211ce91824SDavid Xu 	MPASS(aiocbe->bp == NULL);
6221ce91824SDavid Xu 	aiocbe->jobstate = JOBST_NULL;
623759ccccaSDavid Xu 	AIO_UNLOCK(ki);
6242a522eb9SJohn Baldwin 
6252a522eb9SJohn Baldwin 	/*
6262a522eb9SJohn Baldwin 	 * The thread argument here is used to find the owning process
6272a522eb9SJohn Baldwin 	 * and is also passed to fo_close() which may pass it to various
6282a522eb9SJohn Baldwin 	 * places such as devsw close() routines.  Because of that, we
6292a522eb9SJohn Baldwin 	 * need a thread pointer from the process owning the job that is
6302a522eb9SJohn Baldwin 	 * persistent and won't disappear out from under us or move to
6312a522eb9SJohn Baldwin 	 * another process.
6322a522eb9SJohn Baldwin 	 *
6332a522eb9SJohn Baldwin 	 * Currently, all the callers of this function call it to remove
6342a522eb9SJohn Baldwin 	 * an aiocblist from the current process' job list either via a
6352a522eb9SJohn Baldwin 	 * syscall or due to the current process calling exit() or
6362a522eb9SJohn Baldwin 	 * execve().  Thus, we know that p == curproc.  We also know that
6372a522eb9SJohn Baldwin 	 * curthread can't exit since we are curthread.
6382a522eb9SJohn Baldwin 	 *
6392a522eb9SJohn Baldwin 	 * Therefore, we use curthread as the thread to pass to
6402a522eb9SJohn Baldwin 	 * knlist_delete().  This does mean that it is possible for the
6412a522eb9SJohn Baldwin 	 * thread pointer at close time to differ from the thread pointer
6422a522eb9SJohn Baldwin 	 * at open time, but this is already true of file descriptors in
6432a522eb9SJohn Baldwin 	 * a multithreaded process.
644b40ce416SJulian Elischer 	 */
645a5c0b1c0SAlan Cox 	fdrop(aiocbe->fd_file, curthread);
646f8f750c5SRobert Watson 	crfree(aiocbe->cred);
647c897b813SJeff Roberson 	uma_zfree(aiocb_zone, aiocbe);
648759ccccaSDavid Xu 	AIO_LOCK(ki);
6491ce91824SDavid Xu 
650ac41f2efSAlfred Perlstein 	return (0);
6512244ea07SJohn Dyson }
6522244ea07SJohn Dyson 
653993182e5SAlexander Leidinger static void
654993182e5SAlexander Leidinger aio_proc_rundown_exec(void *arg, struct proc *p, struct image_params *imgp __unused)
655993182e5SAlexander Leidinger {
656993182e5SAlexander Leidinger    	aio_proc_rundown(arg, p);
657993182e5SAlexander Leidinger }
658993182e5SAlexander Leidinger 
6592244ea07SJohn Dyson /*
6602244ea07SJohn Dyson  * Rundown the jobs for a given process.
6612244ea07SJohn Dyson  */
66221d56e9cSAlfred Perlstein static void
66375b8b3b2SJohn Baldwin aio_proc_rundown(void *arg, struct proc *p)
664fd3bf775SJohn Dyson {
6652244ea07SJohn Dyson 	struct kaioinfo *ki;
6661ce91824SDavid Xu 	struct aioliojob *lj;
6671ce91824SDavid Xu 	struct aiocblist *cbe, *cbn;
668bfbbc4aaSJason Evans 	struct file *fp;
669bfbbc4aaSJason Evans 	struct socket *so;
6701aa4c324SDavid Xu 	int remove;
6712244ea07SJohn Dyson 
6722a522eb9SJohn Baldwin 	KASSERT(curthread->td_proc == p,
6732a522eb9SJohn Baldwin 	    ("%s: called on non-curproc", __func__));
6742244ea07SJohn Dyson 	ki = p->p_aioinfo;
6752244ea07SJohn Dyson 	if (ki == NULL)
6762244ea07SJohn Dyson 		return;
6772244ea07SJohn Dyson 
678759ccccaSDavid Xu 	AIO_LOCK(ki);
67927b8220dSDavid Xu 	ki->kaio_flags |= KAIO_RUNDOWN;
6801ce91824SDavid Xu 
6811ce91824SDavid Xu restart:
682a624e84fSJohn Dyson 
683bfbbc4aaSJason Evans 	/*
6841ce91824SDavid Xu 	 * Try to cancel all pending requests. This code simulates
6851ce91824SDavid Xu 	 * aio_cancel on all pending I/O requests.
686bfbbc4aaSJason Evans 	 */
6871ce91824SDavid Xu 	TAILQ_FOREACH_SAFE(cbe, &ki->kaio_jobqueue, plist, cbn) {
6881aa4c324SDavid Xu 		remove = 0;
6891ce91824SDavid Xu 		mtx_lock(&aio_job_mtx);
6901ce91824SDavid Xu 		if (cbe->jobstate == JOBST_JOBQGLOBAL) {
6911ce91824SDavid Xu 			TAILQ_REMOVE(&aio_jobs, cbe, list);
6921aa4c324SDavid Xu 			remove = 1;
6931aa4c324SDavid Xu 		} else if (cbe->jobstate == JOBST_JOBQSOCK) {
6941aa4c324SDavid Xu 			fp = cbe->fd_file;
6951aa4c324SDavid Xu 			MPASS(fp->f_type == DTYPE_SOCKET);
6961aa4c324SDavid Xu 			so = fp->f_data;
6971aa4c324SDavid Xu 			TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
6981aa4c324SDavid Xu 			remove = 1;
69999eee864SDavid Xu 		} else if (cbe->jobstate == JOBST_JOBQSYNC) {
70099eee864SDavid Xu 			TAILQ_REMOVE(&ki->kaio_syncqueue, cbe, list);
70199eee864SDavid Xu 			remove = 1;
7021aa4c324SDavid Xu 		}
7031ce91824SDavid Xu 		mtx_unlock(&aio_job_mtx);
7041aa4c324SDavid Xu 
7051aa4c324SDavid Xu 		if (remove) {
7061ce91824SDavid Xu 			cbe->jobstate = JOBST_JOBFINISHED;
7071ce91824SDavid Xu 			cbe->uaiocb._aiocb_private.status = -1;
7081ce91824SDavid Xu 			cbe->uaiocb._aiocb_private.error = ECANCELED;
7091ce91824SDavid Xu 			TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
7101ce91824SDavid Xu 			aio_bio_done_notify(p, cbe, DONE_QUEUE);
7111ce91824SDavid Xu 		}
7122244ea07SJohn Dyson 	}
71384af4da6SJohn Dyson 
7141ce91824SDavid Xu 	/* Wait for all running I/O to be finished */
7151ce91824SDavid Xu 	if (TAILQ_FIRST(&ki->kaio_bufqueue) ||
7161ce91824SDavid Xu 	    TAILQ_FIRST(&ki->kaio_jobqueue)) {
71784af4da6SJohn Dyson 		ki->kaio_flags |= KAIO_WAKEUP;
718759ccccaSDavid Xu 		msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO, "aioprn", hz);
7191ce91824SDavid Xu 		goto restart;
72084af4da6SJohn Dyson 	}
72184af4da6SJohn Dyson 
7221ce91824SDavid Xu 	/* Free all completed I/O requests. */
7231ce91824SDavid Xu 	while ((cbe = TAILQ_FIRST(&ki->kaio_done)) != NULL)
7241ce91824SDavid Xu 		aio_free_entry(cbe);
72584af4da6SJohn Dyson 
7261ce91824SDavid Xu 	while ((lj = TAILQ_FIRST(&ki->kaio_liojoblist)) != NULL) {
727a9bf5e37SDavid Xu 		if (lj->lioj_count == 0) {
72884af4da6SJohn Dyson 			TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
7291ce91824SDavid Xu 			knlist_delete(&lj->klist, curthread, 1);
730759ccccaSDavid Xu 			PROC_LOCK(p);
7311ce91824SDavid Xu 			sigqueue_take(&lj->lioj_ksi);
732759ccccaSDavid Xu 			PROC_UNLOCK(p);
733c897b813SJeff Roberson 			uma_zfree(aiolio_zone, lj);
734f4f0ecefSJohn Dyson 		} else {
735a9bf5e37SDavid Xu 			panic("LIO job not cleaned up: C:%d, FC:%d\n",
736a9bf5e37SDavid Xu 			    lj->lioj_count, lj->lioj_finished_count);
73784af4da6SJohn Dyson 		}
738f4f0ecefSJohn Dyson 	}
739759ccccaSDavid Xu 	AIO_UNLOCK(ki);
74099eee864SDavid Xu 	taskqueue_drain(taskqueue_aiod_bio, &ki->kaio_task);
7415114048bSKonstantin Belousov 	mtx_destroy(&ki->kaio_mtx);
742c897b813SJeff Roberson 	uma_zfree(kaio_zone, ki);
743a624e84fSJohn Dyson 	p->p_aioinfo = NULL;
7442244ea07SJohn Dyson }
7452244ea07SJohn Dyson 
7462244ea07SJohn Dyson /*
747bfbbc4aaSJason Evans  * Select a job to run (called by an AIO daemon).
7482244ea07SJohn Dyson  */
7492244ea07SJohn Dyson static struct aiocblist *
750b40ce416SJulian Elischer aio_selectjob(struct aiothreadlist *aiop)
751fd3bf775SJohn Dyson {
7522244ea07SJohn Dyson 	struct aiocblist *aiocbe;
753bfbbc4aaSJason Evans 	struct kaioinfo *ki;
754bfbbc4aaSJason Evans 	struct proc *userp;
7552244ea07SJohn Dyson 
7561ce91824SDavid Xu 	mtx_assert(&aio_job_mtx, MA_OWNED);
7572a522eb9SJohn Baldwin 	TAILQ_FOREACH(aiocbe, &aio_jobs, list) {
7582244ea07SJohn Dyson 		userp = aiocbe->userproc;
7592244ea07SJohn Dyson 		ki = userp->p_aioinfo;
7602244ea07SJohn Dyson 
7612244ea07SJohn Dyson 		if (ki->kaio_active_count < ki->kaio_maxactive_count) {
7622244ea07SJohn Dyson 			TAILQ_REMOVE(&aio_jobs, aiocbe, list);
7631ce91824SDavid Xu 			/* Account for currently active jobs. */
7641ce91824SDavid Xu 			ki->kaio_active_count++;
7651ce91824SDavid Xu 			aiocbe->jobstate = JOBST_JOBRUNNING;
7661ce91824SDavid Xu 			break;
7671ce91824SDavid Xu 		}
7681ce91824SDavid Xu 	}
769ac41f2efSAlfred Perlstein 	return (aiocbe);
7702244ea07SJohn Dyson }
7712244ea07SJohn Dyson 
7722244ea07SJohn Dyson /*
77399eee864SDavid Xu  *  Move all data to a permanent storage device, this code
77499eee864SDavid Xu  *  simulates fsync syscall.
77599eee864SDavid Xu  */
77699eee864SDavid Xu static int
77799eee864SDavid Xu aio_fsync_vnode(struct thread *td, struct vnode *vp)
77899eee864SDavid Xu {
77999eee864SDavid Xu 	struct mount *mp;
78099eee864SDavid Xu 	int vfslocked;
78199eee864SDavid Xu 	int error;
78299eee864SDavid Xu 
78399eee864SDavid Xu 	vfslocked = VFS_LOCK_GIANT(vp->v_mount);
78499eee864SDavid Xu 	if ((error = vn_start_write(vp, &mp, V_WAIT | PCATCH)) != 0)
78599eee864SDavid Xu 		goto drop;
786cb05b60aSAttilio Rao 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
78799eee864SDavid Xu 	if (vp->v_object != NULL) {
78899eee864SDavid Xu 		VM_OBJECT_LOCK(vp->v_object);
78999eee864SDavid Xu 		vm_object_page_clean(vp->v_object, 0, 0, 0);
79099eee864SDavid Xu 		VM_OBJECT_UNLOCK(vp->v_object);
79199eee864SDavid Xu 	}
79299eee864SDavid Xu 	error = VOP_FSYNC(vp, MNT_WAIT, td);
79399eee864SDavid Xu 
79422db15c0SAttilio Rao 	VOP_UNLOCK(vp, 0);
79599eee864SDavid Xu 	vn_finished_write(mp);
79699eee864SDavid Xu drop:
79799eee864SDavid Xu 	VFS_UNLOCK_GIANT(vfslocked);
79899eee864SDavid Xu 	return (error);
79999eee864SDavid Xu }
80099eee864SDavid Xu 
80199eee864SDavid Xu /*
802bfbbc4aaSJason Evans  * The AIO processing activity.  This is the code that does the I/O request for
803bfbbc4aaSJason Evans  * the non-physio version of the operations.  The normal vn operations are used,
804bfbbc4aaSJason Evans  * and this code should work in all instances for every type of file, including
805bfbbc4aaSJason Evans  * pipes, sockets, fifos, and regular files.
8061ce91824SDavid Xu  *
8071aa4c324SDavid Xu  * XXX I don't think it works well for socket, pipe, and fifo.
8082244ea07SJohn Dyson  */
80988ed460eSAlan Cox static void
810fd3bf775SJohn Dyson aio_process(struct aiocblist *aiocbe)
811fd3bf775SJohn Dyson {
812f8f750c5SRobert Watson 	struct ucred *td_savedcred;
813b40ce416SJulian Elischer 	struct thread *td;
8142244ea07SJohn Dyson 	struct aiocb *cb;
8152244ea07SJohn Dyson 	struct file *fp;
8161ce91824SDavid Xu 	struct socket *so;
8172244ea07SJohn Dyson 	struct uio auio;
8182244ea07SJohn Dyson 	struct iovec aiov;
8192244ea07SJohn Dyson 	int cnt;
8202244ea07SJohn Dyson 	int error;
821fd3bf775SJohn Dyson 	int oublock_st, oublock_end;
822fd3bf775SJohn Dyson 	int inblock_st, inblock_end;
8232244ea07SJohn Dyson 
824b40ce416SJulian Elischer 	td = curthread;
825f8f750c5SRobert Watson 	td_savedcred = td->td_ucred;
826f8f750c5SRobert Watson 	td->td_ucred = aiocbe->cred;
8272244ea07SJohn Dyson 	cb = &aiocbe->uaiocb;
82800e73160SAlan Cox 	fp = aiocbe->fd_file;
829bfbbc4aaSJason Evans 
83099eee864SDavid Xu 	if (cb->aio_lio_opcode == LIO_SYNC) {
83199eee864SDavid Xu 		error = 0;
83299eee864SDavid Xu 		cnt = 0;
83399eee864SDavid Xu 		if (fp->f_vnode != NULL)
83499eee864SDavid Xu 			error = aio_fsync_vnode(td, fp->f_vnode);
83599eee864SDavid Xu 		cb->_aiocb_private.error = error;
83699eee864SDavid Xu 		cb->_aiocb_private.status = 0;
83799eee864SDavid Xu 		td->td_ucred = td_savedcred;
83899eee864SDavid Xu 		return;
83999eee864SDavid Xu 	}
84099eee864SDavid Xu 
84191369fc7SAlan Cox 	aiov.iov_base = (void *)(uintptr_t)cb->aio_buf;
8422244ea07SJohn Dyson 	aiov.iov_len = cb->aio_nbytes;
8432244ea07SJohn Dyson 
8442244ea07SJohn Dyson 	auio.uio_iov = &aiov;
8452244ea07SJohn Dyson 	auio.uio_iovcnt = 1;
8469b16adc1SAlan Cox 	auio.uio_offset = cb->aio_offset;
8472244ea07SJohn Dyson 	auio.uio_resid = cb->aio_nbytes;
8482244ea07SJohn Dyson 	cnt = cb->aio_nbytes;
8492244ea07SJohn Dyson 	auio.uio_segflg = UIO_USERSPACE;
850b40ce416SJulian Elischer 	auio.uio_td = td;
8512244ea07SJohn Dyson 
8521c4bcd05SJeff Roberson 	inblock_st = td->td_ru.ru_inblock;
8531c4bcd05SJeff Roberson 	oublock_st = td->td_ru.ru_oublock;
854279d7226SMatthew Dillon 	/*
855a9bf5e37SDavid Xu 	 * aio_aqueue() acquires a reference to the file that is
8569b16adc1SAlan Cox 	 * released in aio_free_entry().
857279d7226SMatthew Dillon 	 */
8582244ea07SJohn Dyson 	if (cb->aio_lio_opcode == LIO_READ) {
8592244ea07SJohn Dyson 		auio.uio_rw = UIO_READ;
8605114048bSKonstantin Belousov 		if (auio.uio_resid == 0)
8615114048bSKonstantin Belousov 			error = 0;
8625114048bSKonstantin Belousov 		else
863b40ce416SJulian Elischer 			error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, td);
8642244ea07SJohn Dyson 	} else {
8656d53aa62SDavid Xu 		if (fp->f_type == DTYPE_VNODE)
8666d53aa62SDavid Xu 			bwillwrite();
8672244ea07SJohn Dyson 		auio.uio_rw = UIO_WRITE;
868b40ce416SJulian Elischer 		error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, td);
8692244ea07SJohn Dyson 	}
8701c4bcd05SJeff Roberson 	inblock_end = td->td_ru.ru_inblock;
8711c4bcd05SJeff Roberson 	oublock_end = td->td_ru.ru_oublock;
872fd3bf775SJohn Dyson 
873fd3bf775SJohn Dyson 	aiocbe->inputcharge = inblock_end - inblock_st;
874fd3bf775SJohn Dyson 	aiocbe->outputcharge = oublock_end - oublock_st;
8752244ea07SJohn Dyson 
876bfbbc4aaSJason Evans 	if ((error) && (auio.uio_resid != cnt)) {
8772244ea07SJohn Dyson 		if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
8782244ea07SJohn Dyson 			error = 0;
87919eb87d2SJohn Baldwin 		if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) {
8801ce91824SDavid Xu 			int sigpipe = 1;
8811ce91824SDavid Xu 			if (fp->f_type == DTYPE_SOCKET) {
8821ce91824SDavid Xu 				so = fp->f_data;
8831ce91824SDavid Xu 				if (so->so_options & SO_NOSIGPIPE)
8841ce91824SDavid Xu 					sigpipe = 0;
8851ce91824SDavid Xu 			}
8861ce91824SDavid Xu 			if (sigpipe) {
8879b16adc1SAlan Cox 				PROC_LOCK(aiocbe->userproc);
8889b16adc1SAlan Cox 				psignal(aiocbe->userproc, SIGPIPE);
8899b16adc1SAlan Cox 				PROC_UNLOCK(aiocbe->userproc);
89019eb87d2SJohn Baldwin 			}
8912244ea07SJohn Dyson 		}
8921ce91824SDavid Xu 	}
8932244ea07SJohn Dyson 
8942244ea07SJohn Dyson 	cnt -= auio.uio_resid;
8952244ea07SJohn Dyson 	cb->_aiocb_private.error = error;
8962244ea07SJohn Dyson 	cb->_aiocb_private.status = cnt;
897f8f750c5SRobert Watson 	td->td_ucred = td_savedcred;
8982244ea07SJohn Dyson }
8992244ea07SJohn Dyson 
90069cd28daSDoug Ambrisko static void
9011ce91824SDavid Xu aio_bio_done_notify(struct proc *userp, struct aiocblist *aiocbe, int type)
9021ce91824SDavid Xu {
9031ce91824SDavid Xu 	struct aioliojob *lj;
90469cd28daSDoug Ambrisko 	struct kaioinfo *ki;
90599eee864SDavid Xu 	struct aiocblist *scb, *scbn;
9061ce91824SDavid Xu 	int lj_done;
90769cd28daSDoug Ambrisko 
90869cd28daSDoug Ambrisko 	ki = userp->p_aioinfo;
909759ccccaSDavid Xu 	AIO_LOCK_ASSERT(ki, MA_OWNED);
91069cd28daSDoug Ambrisko 	lj = aiocbe->lio;
91169cd28daSDoug Ambrisko 	lj_done = 0;
91269cd28daSDoug Ambrisko 	if (lj) {
9131ce91824SDavid Xu 		lj->lioj_finished_count++;
9141ce91824SDavid Xu 		if (lj->lioj_count == lj->lioj_finished_count)
91569cd28daSDoug Ambrisko 			lj_done = 1;
91669cd28daSDoug Ambrisko 	}
91769cd28daSDoug Ambrisko 	if (type == DONE_QUEUE) {
9181ce91824SDavid Xu 		aiocbe->jobflags |= AIOCBLIST_DONE;
91969cd28daSDoug Ambrisko 	} else {
9201ce91824SDavid Xu 		aiocbe->jobflags |= AIOCBLIST_BUFDONE;
92169cd28daSDoug Ambrisko 	}
9221ce91824SDavid Xu 	TAILQ_INSERT_TAIL(&ki->kaio_done, aiocbe, plist);
9231ce91824SDavid Xu 	aiocbe->jobstate = JOBST_JOBFINISHED;
92427b8220dSDavid Xu 
92527b8220dSDavid Xu 	if (ki->kaio_flags & KAIO_RUNDOWN)
92627b8220dSDavid Xu 		goto notification_done;
92727b8220dSDavid Xu 
9281ce91824SDavid Xu 	if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL ||
9291ce91824SDavid Xu 	    aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID)
9301ce91824SDavid Xu 		aio_sendsig(userp, &aiocbe->uaiocb.aio_sigevent, &aiocbe->ksi);
9311ce91824SDavid Xu 
9321ce91824SDavid Xu 	KNOTE_LOCKED(&aiocbe->klist, 1);
9331ce91824SDavid Xu 
93469cd28daSDoug Ambrisko 	if (lj_done) {
9351ce91824SDavid Xu 		if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
93669cd28daSDoug Ambrisko 			lj->lioj_flags |= LIOJ_KEVENT_POSTED;
9371ce91824SDavid Xu 			KNOTE_LOCKED(&lj->klist, 1);
93869cd28daSDoug Ambrisko 		}
9391ce91824SDavid Xu 		if ((lj->lioj_flags & (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED))
94069cd28daSDoug Ambrisko 		    == LIOJ_SIGNAL
9414c0fb2cfSDavid Xu 		    && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
9424c0fb2cfSDavid Xu 		        lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID)) {
9434c0fb2cfSDavid Xu 			aio_sendsig(userp, &lj->lioj_signal, &lj->lioj_ksi);
94469cd28daSDoug Ambrisko 			lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
94569cd28daSDoug Ambrisko 		}
94669cd28daSDoug Ambrisko 	}
94727b8220dSDavid Xu 
94827b8220dSDavid Xu notification_done:
94999eee864SDavid Xu 	if (aiocbe->jobflags & AIOCBLIST_CHECKSYNC) {
95099eee864SDavid Xu 		TAILQ_FOREACH_SAFE(scb, &ki->kaio_syncqueue, list, scbn) {
951dbbccfe9SDavid Xu 			if (aiocbe->fd_file == scb->fd_file &&
95299eee864SDavid Xu 			    aiocbe->seqno < scb->seqno) {
95399eee864SDavid Xu 				if (--scb->pending == 0) {
95499eee864SDavid Xu 					mtx_lock(&aio_job_mtx);
95599eee864SDavid Xu 					scb->jobstate = JOBST_JOBQGLOBAL;
95699eee864SDavid Xu 					TAILQ_REMOVE(&ki->kaio_syncqueue, scb, list);
95799eee864SDavid Xu 					TAILQ_INSERT_TAIL(&aio_jobs, scb, list);
95899eee864SDavid Xu 					aio_kick_nowait(userp);
95999eee864SDavid Xu 					mtx_unlock(&aio_job_mtx);
96099eee864SDavid Xu 				}
96199eee864SDavid Xu 			}
96299eee864SDavid Xu 		}
96399eee864SDavid Xu 	}
96427b8220dSDavid Xu 	if (ki->kaio_flags & KAIO_WAKEUP) {
96569cd28daSDoug Ambrisko 		ki->kaio_flags &= ~KAIO_WAKEUP;
9661ce91824SDavid Xu 		wakeup(&userp->p_aioinfo);
96769cd28daSDoug Ambrisko 	}
96869cd28daSDoug Ambrisko }
96969cd28daSDoug Ambrisko 
9702244ea07SJohn Dyson /*
97184af4da6SJohn Dyson  * The AIO daemon, most of the actual work is done in aio_process,
97284af4da6SJohn Dyson  * but the setup (and address space mgmt) is done in this routine.
9732244ea07SJohn Dyson  */
9742244ea07SJohn Dyson static void
9751ce91824SDavid Xu aio_daemon(void *_id)
9762244ea07SJohn Dyson {
977bfbbc4aaSJason Evans 	struct aiocblist *aiocbe;
978b40ce416SJulian Elischer 	struct aiothreadlist *aiop;
979bfbbc4aaSJason Evans 	struct kaioinfo *ki;
980bfbbc4aaSJason Evans 	struct proc *curcp, *mycp, *userp;
981bfbbc4aaSJason Evans 	struct vmspace *myvm, *tmpvm;
982b40ce416SJulian Elischer 	struct thread *td = curthread;
9831ce91824SDavid Xu 	int id = (intptr_t)_id;
9842244ea07SJohn Dyson 
9852244ea07SJohn Dyson 	/*
986fd3bf775SJohn Dyson 	 * Local copies of curproc (cp) and vmspace (myvm)
9872244ea07SJohn Dyson 	 */
988b40ce416SJulian Elischer 	mycp = td->td_proc;
989fd3bf775SJohn Dyson 	myvm = mycp->p_vmspace;
990fd3bf775SJohn Dyson 
991cd4ed3b5SJohn Baldwin 	KASSERT(mycp->p_textvp == NULL, ("kthread has a textvp"));
992fd3bf775SJohn Dyson 
993fd3bf775SJohn Dyson 	/*
994bfbbc4aaSJason Evans 	 * Allocate and ready the aio control info.  There is one aiop structure
995bfbbc4aaSJason Evans 	 * per daemon.
996fd3bf775SJohn Dyson 	 */
997a163d034SWarner Losh 	aiop = uma_zalloc(aiop_zone, M_WAITOK);
998b40ce416SJulian Elischer 	aiop->aiothread = td;
99999eee864SDavid Xu 	aiop->aiothreadflags = 0;
1000bfbbc4aaSJason Evans 
1001e6bdc05fSDavid Xu 	/* The daemon resides in its own pgrp. */
1002bd793be3SDavid Xu 	setsid(td, NULL);
1003fd3bf775SJohn Dyson 
1004fd3bf775SJohn Dyson 	/*
1005fd3bf775SJohn Dyson 	 * Wakeup parent process.  (Parent sleeps to keep from blasting away
1006b40ce416SJulian Elischer 	 * and creating too many daemons.)
1007fd3bf775SJohn Dyson 	 */
10081ce91824SDavid Xu 	sema_post(&aio_newproc_sem);
10092244ea07SJohn Dyson 
10101ce91824SDavid Xu 	mtx_lock(&aio_job_mtx);
1011bfbbc4aaSJason Evans 	for (;;) {
1012fd3bf775SJohn Dyson 		/*
1013fd3bf775SJohn Dyson 		 * curcp is the current daemon process context.
1014fd3bf775SJohn Dyson 		 * userp is the current user process context.
1015fd3bf775SJohn Dyson 		 */
1016fd3bf775SJohn Dyson 		curcp = mycp;
1017c4860686SJohn Dyson 
1018fd3bf775SJohn Dyson 		/*
1019fd3bf775SJohn Dyson 		 * Take daemon off of free queue
1020fd3bf775SJohn Dyson 		 */
1021b40ce416SJulian Elischer 		if (aiop->aiothreadflags & AIOP_FREE) {
10222244ea07SJohn Dyson 			TAILQ_REMOVE(&aio_freeproc, aiop, list);
1023b40ce416SJulian Elischer 			aiop->aiothreadflags &= ~AIOP_FREE;
10242244ea07SJohn Dyson 		}
10252244ea07SJohn Dyson 
1026fd3bf775SJohn Dyson 		/*
1027bfbbc4aaSJason Evans 		 * Check for jobs.
1028fd3bf775SJohn Dyson 		 */
1029d254af07SMatthew Dillon 		while ((aiocbe = aio_selectjob(aiop)) != NULL) {
10301ce91824SDavid Xu 			mtx_unlock(&aio_job_mtx);
10312244ea07SJohn Dyson 			userp = aiocbe->userproc;
10322244ea07SJohn Dyson 
1033fd3bf775SJohn Dyson 			/*
1034bfbbc4aaSJason Evans 			 * Connect to process address space for user program.
1035fd3bf775SJohn Dyson 			 */
1036fd3bf775SJohn Dyson 			if (userp != curcp) {
1037fd3bf775SJohn Dyson 				/*
1038bfbbc4aaSJason Evans 				 * Save the current address space that we are
1039bfbbc4aaSJason Evans 				 * connected to.
1040fd3bf775SJohn Dyson 				 */
1041fd3bf775SJohn Dyson 				tmpvm = mycp->p_vmspace;
1042bfbbc4aaSJason Evans 
1043fd3bf775SJohn Dyson 				/*
1044bfbbc4aaSJason Evans 				 * Point to the new user address space, and
1045bfbbc4aaSJason Evans 				 * refer to it.
1046fd3bf775SJohn Dyson 				 */
1047fd3bf775SJohn Dyson 				mycp->p_vmspace = userp->p_vmspace;
10481a276a3fSAlan Cox 				atomic_add_int(&mycp->p_vmspace->vm_refcnt, 1);
1049bfbbc4aaSJason Evans 
1050bfbbc4aaSJason Evans 				/* Activate the new mapping. */
1051079b7badSJulian Elischer 				pmap_activate(FIRST_THREAD_IN_PROC(mycp));
1052bfbbc4aaSJason Evans 
1053fd3bf775SJohn Dyson 				/*
1054bfbbc4aaSJason Evans 				 * If the old address space wasn't the daemons
1055bfbbc4aaSJason Evans 				 * own address space, then we need to remove the
1056bfbbc4aaSJason Evans 				 * daemon's reference from the other process
1057bfbbc4aaSJason Evans 				 * that it was acting on behalf of.
1058fd3bf775SJohn Dyson 				 */
10592244ea07SJohn Dyson 				if (tmpvm != myvm) {
10602244ea07SJohn Dyson 					vmspace_free(tmpvm);
10612244ea07SJohn Dyson 				}
1062fd3bf775SJohn Dyson 				curcp = userp;
10632244ea07SJohn Dyson 			}
10642244ea07SJohn Dyson 
1065fd3bf775SJohn Dyson 			ki = userp->p_aioinfo;
106684af4da6SJohn Dyson 
1067bfbbc4aaSJason Evans 			/* Do the I/O function. */
10682244ea07SJohn Dyson 			aio_process(aiocbe);
106984af4da6SJohn Dyson 
10709b84335cSDavid Xu 			mtx_lock(&aio_job_mtx);
10719b84335cSDavid Xu 			/* Decrement the active job count. */
10729b84335cSDavid Xu 			ki->kaio_active_count--;
10739b84335cSDavid Xu 			mtx_unlock(&aio_job_mtx);
10749b84335cSDavid Xu 
1075759ccccaSDavid Xu 			AIO_LOCK(ki);
10761ce91824SDavid Xu 			TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist);
107769cd28daSDoug Ambrisko 			aio_bio_done_notify(userp, aiocbe, DONE_QUEUE);
1078759ccccaSDavid Xu 			AIO_UNLOCK(ki);
10791ce91824SDavid Xu 
10801ce91824SDavid Xu 			mtx_lock(&aio_job_mtx);
10812244ea07SJohn Dyson 		}
10822244ea07SJohn Dyson 
1083fd3bf775SJohn Dyson 		/*
1084bfbbc4aaSJason Evans 		 * Disconnect from user address space.
1085fd3bf775SJohn Dyson 		 */
1086fd3bf775SJohn Dyson 		if (curcp != mycp) {
10871ce91824SDavid Xu 
10881ce91824SDavid Xu 			mtx_unlock(&aio_job_mtx);
10891ce91824SDavid Xu 
1090bfbbc4aaSJason Evans 			/* Get the user address space to disconnect from. */
1091fd3bf775SJohn Dyson 			tmpvm = mycp->p_vmspace;
1092bfbbc4aaSJason Evans 
1093bfbbc4aaSJason Evans 			/* Get original address space for daemon. */
1094fd3bf775SJohn Dyson 			mycp->p_vmspace = myvm;
1095bfbbc4aaSJason Evans 
1096bfbbc4aaSJason Evans 			/* Activate the daemon's address space. */
1097079b7badSJulian Elischer 			pmap_activate(FIRST_THREAD_IN_PROC(mycp));
1098bfbbc4aaSJason Evans #ifdef DIAGNOSTIC
1099bfbbc4aaSJason Evans 			if (tmpvm == myvm) {
1100bfbbc4aaSJason Evans 				printf("AIOD: vmspace problem -- %d\n",
1101bfbbc4aaSJason Evans 				    mycp->p_pid);
1102bfbbc4aaSJason Evans 			}
110311783b14SJohn Dyson #endif
1104bfbbc4aaSJason Evans 			/* Remove our vmspace reference. */
11052244ea07SJohn Dyson 			vmspace_free(tmpvm);
1106bfbbc4aaSJason Evans 
1107fd3bf775SJohn Dyson 			curcp = mycp;
11081ce91824SDavid Xu 
11091ce91824SDavid Xu 			mtx_lock(&aio_job_mtx);
11101ce91824SDavid Xu 			/*
11111ce91824SDavid Xu 			 * We have to restart to avoid race, we only sleep if
11121ce91824SDavid Xu 			 * no job can be selected, that should be
11131ce91824SDavid Xu 			 * curcp == mycp.
11141ce91824SDavid Xu 			 */
11151ce91824SDavid Xu 			continue;
1116fd3bf775SJohn Dyson 		}
1117fd3bf775SJohn Dyson 
11181ce91824SDavid Xu 		mtx_assert(&aio_job_mtx, MA_OWNED);
11191ce91824SDavid Xu 
1120fd3bf775SJohn Dyson 		TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
1121b40ce416SJulian Elischer 		aiop->aiothreadflags |= AIOP_FREE;
1122fd3bf775SJohn Dyson 
1123fd3bf775SJohn Dyson 		/*
1124bfbbc4aaSJason Evans 		 * If daemon is inactive for a long time, allow it to exit,
1125bfbbc4aaSJason Evans 		 * thereby freeing resources.
1126fd3bf775SJohn Dyson 		 */
11271ce91824SDavid Xu 		if (msleep(aiop->aiothread, &aio_job_mtx, PRIBIO, "aiordy",
11281ce91824SDavid Xu 		    aiod_lifetime)) {
1129c3869e4bSAlan Cox 			if (TAILQ_EMPTY(&aio_jobs)) {
1130b40ce416SJulian Elischer 				if ((aiop->aiothreadflags & AIOP_FREE) &&
113184af4da6SJohn Dyson 				    (num_aio_procs > target_aio_procs)) {
1132fd3bf775SJohn Dyson 					TAILQ_REMOVE(&aio_freeproc, aiop, list);
113384af4da6SJohn Dyson 					num_aio_procs--;
11341ce91824SDavid Xu 					mtx_unlock(&aio_job_mtx);
11351ce91824SDavid Xu 					uma_zfree(aiop_zone, aiop);
11361ce91824SDavid Xu 					free_unr(aiod_unr, id);
1137bfbbc4aaSJason Evans #ifdef DIAGNOSTIC
1138bfbbc4aaSJason Evans 					if (mycp->p_vmspace->vm_refcnt <= 1) {
1139bfbbc4aaSJason Evans 						printf("AIOD: bad vm refcnt for"
1140bfbbc4aaSJason Evans 						    " exiting daemon: %d\n",
1141fd3bf775SJohn Dyson 						    mycp->p_vmspace->vm_refcnt);
1142bfbbc4aaSJason Evans 					}
114311783b14SJohn Dyson #endif
11443745c395SJulian Elischer 					kproc_exit(0);
1145fd3bf775SJohn Dyson 				}
11462244ea07SJohn Dyson 			}
11472244ea07SJohn Dyson 		}
11482244ea07SJohn Dyson 	}
11491ce91824SDavid Xu 	mtx_unlock(&aio_job_mtx);
11501ce91824SDavid Xu 	panic("shouldn't be here\n");
11511ce91824SDavid Xu }
11522244ea07SJohn Dyson 
11532244ea07SJohn Dyson /*
1154bfbbc4aaSJason Evans  * Create a new AIO daemon. This is mostly a kernel-thread fork routine. The
1155bfbbc4aaSJason Evans  * AIO daemon modifies its environment itself.
11562244ea07SJohn Dyson  */
11572244ea07SJohn Dyson static int
11581ce91824SDavid Xu aio_newproc(int *start)
1159fd3bf775SJohn Dyson {
11602244ea07SJohn Dyson 	int error;
1161c9a970a7SAlan Cox 	struct proc *p;
11621ce91824SDavid Xu 	int id;
11632244ea07SJohn Dyson 
11641ce91824SDavid Xu 	id = alloc_unr(aiod_unr);
11653745c395SJulian Elischer 	error = kproc_create(aio_daemon, (void *)(intptr_t)id, &p,
11661ce91824SDavid Xu 		RFNOWAIT, 0, "aiod%d", id);
11671ce91824SDavid Xu 	if (error == 0) {
1168fd3bf775SJohn Dyson 		/*
11691ce91824SDavid Xu 		 * Wait until daemon is started.
1170fd3bf775SJohn Dyson 		 */
11711ce91824SDavid Xu 		sema_wait(&aio_newproc_sem);
11721ce91824SDavid Xu 		mtx_lock(&aio_job_mtx);
117384af4da6SJohn Dyson 		num_aio_procs++;
11741ce91824SDavid Xu 		if (start != NULL)
11757f34b521SDavid Xu 			(*start)--;
11761ce91824SDavid Xu 		mtx_unlock(&aio_job_mtx);
11771ce91824SDavid Xu 	} else {
11781ce91824SDavid Xu 		free_unr(aiod_unr, id);
11791ce91824SDavid Xu 	}
1180ac41f2efSAlfred Perlstein 	return (error);
11812244ea07SJohn Dyson }
11822244ea07SJohn Dyson 
11832244ea07SJohn Dyson /*
118488ed460eSAlan Cox  * Try the high-performance, low-overhead physio method for eligible
118588ed460eSAlan Cox  * VCHR devices.  This method doesn't use an aio helper thread, and
118688ed460eSAlan Cox  * thus has very low overhead.
118788ed460eSAlan Cox  *
1188a9bf5e37SDavid Xu  * Assumes that the caller, aio_aqueue(), has incremented the file
118988ed460eSAlan Cox  * structure's reference count, preventing its deallocation for the
119088ed460eSAlan Cox  * duration of this call.
1191fd3bf775SJohn Dyson  */
119288ed460eSAlan Cox static int
1193bfbbc4aaSJason Evans aio_qphysio(struct proc *p, struct aiocblist *aiocbe)
1194fd3bf775SJohn Dyson {
1195fd3bf775SJohn Dyson 	struct aiocb *cb;
1196fd3bf775SJohn Dyson 	struct file *fp;
1197fd3bf775SJohn Dyson 	struct buf *bp;
1198fd3bf775SJohn Dyson 	struct vnode *vp;
1199fd3bf775SJohn Dyson 	struct kaioinfo *ki;
12001ce91824SDavid Xu 	struct aioliojob *lj;
12011ce91824SDavid Xu 	int error;
1202fd3bf775SJohn Dyson 
120384af4da6SJohn Dyson 	cb = &aiocbe->uaiocb;
12049fbd7ccfSAlan Cox 	fp = aiocbe->fd_file;
1205fd3bf775SJohn Dyson 
1206008626c3SPoul-Henning Kamp 	if (fp->f_type != DTYPE_VNODE)
1207008626c3SPoul-Henning Kamp 		return (-1);
1208fd3bf775SJohn Dyson 
12093b6d9652SPoul-Henning Kamp 	vp = fp->f_vnode;
121011783b14SJohn Dyson 
1211f582ac06SBrian Feldman 	/*
1212f582ac06SBrian Feldman 	 * If its not a disk, we don't want to return a positive error.
1213f582ac06SBrian Feldman 	 * It causes the aio code to not fall through to try the thread
1214f582ac06SBrian Feldman 	 * way when you're talking to a regular file.
1215f582ac06SBrian Feldman 	 */
1216f582ac06SBrian Feldman 	if (!vn_isdisk(vp, &error)) {
1217f582ac06SBrian Feldman 		if (error == ENOTBLK)
1218f582ac06SBrian Feldman 			return (-1);
1219f582ac06SBrian Feldman 		else
1220ba4ad1fcSPoul-Henning Kamp 			return (error);
1221f582ac06SBrian Feldman 	}
1222fd3bf775SJohn Dyson 
1223ad8de0f2SDavid Xu 	if (vp->v_bufobj.bo_bsize == 0)
1224ad8de0f2SDavid Xu 		return (-1);
1225ad8de0f2SDavid Xu 
12265d9d81e7SPoul-Henning Kamp  	if (cb->aio_nbytes % vp->v_bufobj.bo_bsize)
1227008626c3SPoul-Henning Kamp 		return (-1);
1228fd3bf775SJohn Dyson 
122969cd28daSDoug Ambrisko 	if (cb->aio_nbytes > vp->v_rdev->si_iosize_max)
123069cd28daSDoug Ambrisko 		return (-1);
123169cd28daSDoug Ambrisko 
123213644654SAlan Cox 	if (cb->aio_nbytes >
123313644654SAlan Cox 	    MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK))
1234008626c3SPoul-Henning Kamp 		return (-1);
1235fd3bf775SJohn Dyson 
1236fd3bf775SJohn Dyson 	ki = p->p_aioinfo;
1237008626c3SPoul-Henning Kamp 	if (ki->kaio_buffer_count >= ki->kaio_ballowed_count)
1238008626c3SPoul-Henning Kamp 		return (-1);
1239fd3bf775SJohn Dyson 
1240bfbbc4aaSJason Evans 	/* Create and build a buffer header for a transfer. */
12411c7c3c6aSMatthew Dillon 	bp = (struct buf *)getpbuf(NULL);
124213644654SAlan Cox 	BUF_KERNPROC(bp);
1243fd3bf775SJohn Dyson 
1244759ccccaSDavid Xu 	AIO_LOCK(ki);
12451ce91824SDavid Xu 	ki->kaio_count++;
12461ce91824SDavid Xu 	ki->kaio_buffer_count++;
12471ce91824SDavid Xu 	lj = aiocbe->lio;
12481ce91824SDavid Xu 	if (lj)
12491ce91824SDavid Xu 		lj->lioj_count++;
1250759ccccaSDavid Xu 	AIO_UNLOCK(ki);
12511ce91824SDavid Xu 
1252fd3bf775SJohn Dyson 	/*
1253bfbbc4aaSJason Evans 	 * Get a copy of the kva from the physical buffer.
1254fd3bf775SJohn Dyson 	 */
1255ef38cda1SAlan Cox 	error = 0;
1256fd3bf775SJohn Dyson 
1257fd3bf775SJohn Dyson 	bp->b_bcount = cb->aio_nbytes;
1258fd3bf775SJohn Dyson 	bp->b_bufsize = cb->aio_nbytes;
1259fd3bf775SJohn Dyson 	bp->b_iodone = aio_physwakeup;
1260fd3bf775SJohn Dyson 	bp->b_saveaddr = bp->b_data;
126191369fc7SAlan Cox 	bp->b_data = (void *)(uintptr_t)cb->aio_buf;
1262a44ca4f0SHidetoshi Shimokawa 	bp->b_offset = cb->aio_offset;
1263a44ca4f0SHidetoshi Shimokawa 	bp->b_iooffset = cb->aio_offset;
1264fd3bf775SJohn Dyson 	bp->b_blkno = btodb(cb->aio_offset);
126506363906SAlan Cox 	bp->b_iocmd = cb->aio_lio_opcode == LIO_WRITE ? BIO_WRITE : BIO_READ;
1266fd3bf775SJohn Dyson 
12672d5c7e45SMatthew Dillon 	/*
12682d5c7e45SMatthew Dillon 	 * Bring buffer into kernel space.
12692d5c7e45SMatthew Dillon 	 */
12702d5c7e45SMatthew Dillon 	if (vmapbuf(bp) < 0) {
12712d5c7e45SMatthew Dillon 		error = EFAULT;
12722d5c7e45SMatthew Dillon 		goto doerror;
12732d5c7e45SMatthew Dillon 	}
1274fd3bf775SJohn Dyson 
1275759ccccaSDavid Xu 	AIO_LOCK(ki);
1276fd3bf775SJohn Dyson 	aiocbe->bp = bp;
12778edbaf85SHidetoshi Shimokawa 	bp->b_caller1 = (void *)aiocbe;
127884af4da6SJohn Dyson 	TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist);
12791ce91824SDavid Xu 	TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist);
1280fd3bf775SJohn Dyson 	aiocbe->jobstate = JOBST_JOBQBUF;
128184af4da6SJohn Dyson 	cb->_aiocb_private.status = cb->aio_nbytes;
1282759ccccaSDavid Xu 	AIO_UNLOCK(ki);
12831ce91824SDavid Xu 
12841ce91824SDavid Xu 	atomic_add_int(&num_queue_count, 1);
12851ce91824SDavid Xu 	atomic_add_int(&num_buf_aio, 1);
12861ce91824SDavid Xu 
1287fd3bf775SJohn Dyson 	bp->b_error = 0;
1288fd3bf775SJohn Dyson 
12891ce91824SDavid Xu 	TASK_INIT(&aiocbe->biotask, 0, biohelper, aiocbe);
1290bfbbc4aaSJason Evans 
1291bfbbc4aaSJason Evans 	/* Perform transfer. */
12926afb3b1cSPoul-Henning Kamp 	dev_strategy(vp->v_rdev, bp);
1293ac41f2efSAlfred Perlstein 	return (0);
1294fd3bf775SJohn Dyson 
1295fd3bf775SJohn Dyson doerror:
1296759ccccaSDavid Xu 	AIO_LOCK(ki);
12971ce91824SDavid Xu 	ki->kaio_count--;
1298fd3bf775SJohn Dyson 	ki->kaio_buffer_count--;
1299bfbbc4aaSJason Evans 	if (lj)
13001ce91824SDavid Xu 		lj->lioj_count--;
130184af4da6SJohn Dyson 	aiocbe->bp = NULL;
1302759ccccaSDavid Xu 	AIO_UNLOCK(ki);
13031c7c3c6aSMatthew Dillon 	relpbuf(bp, NULL);
1304fd3bf775SJohn Dyson 	return (error);
1305fd3bf775SJohn Dyson }
1306fd3bf775SJohn Dyson 
1307fd3bf775SJohn Dyson /*
1308bfbbc4aaSJason Evans  * Wake up aio requests that may be serviceable now.
1309bfbbc4aaSJason Evans  */
131048dac059SAlan Cox static void
131121d56e9cSAlfred Perlstein aio_swake_cb(struct socket *so, struct sockbuf *sb)
1312bfbbc4aaSJason Evans {
1313bfbbc4aaSJason Evans 	struct aiocblist *cb, *cbn;
131499eee864SDavid Xu 	int opcode;
1315bfbbc4aaSJason Evans 
13161aa4c324SDavid Xu 	if (sb == &so->so_snd)
1317bfbbc4aaSJason Evans 		opcode = LIO_WRITE;
13181aa4c324SDavid Xu 	else
1319bfbbc4aaSJason Evans 		opcode = LIO_READ;
1320bfbbc4aaSJason Evans 
13211aa4c324SDavid Xu 	SOCKBUF_LOCK(sb);
13221aa4c324SDavid Xu 	sb->sb_flags &= ~SB_AIO;
13231aa4c324SDavid Xu 	mtx_lock(&aio_job_mtx);
13242a522eb9SJohn Baldwin 	TAILQ_FOREACH_SAFE(cb, &so->so_aiojobq, list, cbn) {
1325bfbbc4aaSJason Evans 		if (opcode == cb->uaiocb.aio_lio_opcode) {
13268c0d9af5SDavid Xu 			if (cb->jobstate != JOBST_JOBQSOCK)
13271ce91824SDavid Xu 				panic("invalid queue value");
13281aa4c324SDavid Xu 			/* XXX
13291aa4c324SDavid Xu 			 * We don't have actual sockets backend yet,
13301aa4c324SDavid Xu 			 * so we simply move the requests to the generic
13311aa4c324SDavid Xu 			 * file I/O backend.
13321ce91824SDavid Xu 			 */
13331aa4c324SDavid Xu 			TAILQ_REMOVE(&so->so_aiojobq, cb, list);
13341ce91824SDavid Xu 			TAILQ_INSERT_TAIL(&aio_jobs, cb, list);
133599eee864SDavid Xu 			aio_kick_nowait(cb->userproc);
1336bfbbc4aaSJason Evans 		}
1337bfbbc4aaSJason Evans 	}
13381aa4c324SDavid Xu 	mtx_unlock(&aio_job_mtx);
13391aa4c324SDavid Xu 	SOCKBUF_UNLOCK(sb);
1340bfbbc4aaSJason Evans }
1341bfbbc4aaSJason Evans 
13423858a1f4SJohn Baldwin static int
13433858a1f4SJohn Baldwin convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig)
13443858a1f4SJohn Baldwin {
13453858a1f4SJohn Baldwin 
13463858a1f4SJohn Baldwin 	/*
13473858a1f4SJohn Baldwin 	 * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
13483858a1f4SJohn Baldwin 	 * supported by AIO with the old sigevent structure.
13493858a1f4SJohn Baldwin 	 */
13503858a1f4SJohn Baldwin 	nsig->sigev_notify = osig->sigev_notify;
13513858a1f4SJohn Baldwin 	switch (nsig->sigev_notify) {
13523858a1f4SJohn Baldwin 	case SIGEV_NONE:
13533858a1f4SJohn Baldwin 		break;
13543858a1f4SJohn Baldwin 	case SIGEV_SIGNAL:
13553858a1f4SJohn Baldwin 		nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
13563858a1f4SJohn Baldwin 		break;
13573858a1f4SJohn Baldwin 	case SIGEV_KEVENT:
13583858a1f4SJohn Baldwin 		nsig->sigev_notify_kqueue =
13593858a1f4SJohn Baldwin 		    osig->__sigev_u.__sigev_notify_kqueue;
13603858a1f4SJohn Baldwin 		nsig->sigev_value.sival_ptr = osig->sigev_value.sival_ptr;
13613858a1f4SJohn Baldwin 		break;
13623858a1f4SJohn Baldwin 	default:
13633858a1f4SJohn Baldwin 		return (EINVAL);
13643858a1f4SJohn Baldwin 	}
13653858a1f4SJohn Baldwin 	return (0);
13663858a1f4SJohn Baldwin }
13673858a1f4SJohn Baldwin 
13683858a1f4SJohn Baldwin static int
13693858a1f4SJohn Baldwin aiocb_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob)
13703858a1f4SJohn Baldwin {
13713858a1f4SJohn Baldwin 	struct oaiocb *ojob;
13723858a1f4SJohn Baldwin 	int error;
13733858a1f4SJohn Baldwin 
13743858a1f4SJohn Baldwin 	bzero(kjob, sizeof(struct aiocb));
13753858a1f4SJohn Baldwin 	error = copyin(ujob, kjob, sizeof(struct oaiocb));
13763858a1f4SJohn Baldwin 	if (error)
13773858a1f4SJohn Baldwin 		return (error);
13783858a1f4SJohn Baldwin 	ojob = (struct oaiocb *)kjob;
13793858a1f4SJohn Baldwin 	return (convert_old_sigevent(&ojob->aio_sigevent, &kjob->aio_sigevent));
13803858a1f4SJohn Baldwin }
13813858a1f4SJohn Baldwin 
13823858a1f4SJohn Baldwin static int
13833858a1f4SJohn Baldwin aiocb_copyin(struct aiocb *ujob, struct aiocb *kjob)
13843858a1f4SJohn Baldwin {
13853858a1f4SJohn Baldwin 
13863858a1f4SJohn Baldwin 	return (copyin(ujob, kjob, sizeof(struct aiocb)));
13873858a1f4SJohn Baldwin }
13883858a1f4SJohn Baldwin 
13893858a1f4SJohn Baldwin static long
13903858a1f4SJohn Baldwin aiocb_fetch_status(struct aiocb *ujob)
13913858a1f4SJohn Baldwin {
13923858a1f4SJohn Baldwin 
13933858a1f4SJohn Baldwin 	return (fuword(&ujob->_aiocb_private.status));
13943858a1f4SJohn Baldwin }
13953858a1f4SJohn Baldwin 
13963858a1f4SJohn Baldwin static long
13973858a1f4SJohn Baldwin aiocb_fetch_error(struct aiocb *ujob)
13983858a1f4SJohn Baldwin {
13993858a1f4SJohn Baldwin 
14003858a1f4SJohn Baldwin 	return (fuword(&ujob->_aiocb_private.error));
14013858a1f4SJohn Baldwin }
14023858a1f4SJohn Baldwin 
14033858a1f4SJohn Baldwin static int
14043858a1f4SJohn Baldwin aiocb_store_status(struct aiocb *ujob, long status)
14053858a1f4SJohn Baldwin {
14063858a1f4SJohn Baldwin 
14073858a1f4SJohn Baldwin 	return (suword(&ujob->_aiocb_private.status, status));
14083858a1f4SJohn Baldwin }
14093858a1f4SJohn Baldwin 
14103858a1f4SJohn Baldwin static int
14113858a1f4SJohn Baldwin aiocb_store_error(struct aiocb *ujob, long error)
14123858a1f4SJohn Baldwin {
14133858a1f4SJohn Baldwin 
14143858a1f4SJohn Baldwin 	return (suword(&ujob->_aiocb_private.error, error));
14153858a1f4SJohn Baldwin }
14163858a1f4SJohn Baldwin 
14173858a1f4SJohn Baldwin static int
14183858a1f4SJohn Baldwin aiocb_store_kernelinfo(struct aiocb *ujob, long jobref)
14193858a1f4SJohn Baldwin {
14203858a1f4SJohn Baldwin 
14213858a1f4SJohn Baldwin 	return (suword(&ujob->_aiocb_private.kernelinfo, jobref));
14223858a1f4SJohn Baldwin }
14233858a1f4SJohn Baldwin 
14243858a1f4SJohn Baldwin static int
14253858a1f4SJohn Baldwin aiocb_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
14263858a1f4SJohn Baldwin {
14273858a1f4SJohn Baldwin 
14283858a1f4SJohn Baldwin 	return (suword(ujobp, (long)ujob));
14293858a1f4SJohn Baldwin }
14303858a1f4SJohn Baldwin 
14313858a1f4SJohn Baldwin static struct aiocb_ops aiocb_ops = {
14323858a1f4SJohn Baldwin 	.copyin = aiocb_copyin,
14333858a1f4SJohn Baldwin 	.fetch_status = aiocb_fetch_status,
14343858a1f4SJohn Baldwin 	.fetch_error = aiocb_fetch_error,
14353858a1f4SJohn Baldwin 	.store_status = aiocb_store_status,
14363858a1f4SJohn Baldwin 	.store_error = aiocb_store_error,
14373858a1f4SJohn Baldwin 	.store_kernelinfo = aiocb_store_kernelinfo,
14383858a1f4SJohn Baldwin 	.store_aiocb = aiocb_store_aiocb,
14393858a1f4SJohn Baldwin };
14403858a1f4SJohn Baldwin 
14413858a1f4SJohn Baldwin static struct aiocb_ops aiocb_ops_osigevent = {
14423858a1f4SJohn Baldwin 	.copyin = aiocb_copyin_old_sigevent,
14433858a1f4SJohn Baldwin 	.fetch_status = aiocb_fetch_status,
14443858a1f4SJohn Baldwin 	.fetch_error = aiocb_fetch_error,
14453858a1f4SJohn Baldwin 	.store_status = aiocb_store_status,
14463858a1f4SJohn Baldwin 	.store_error = aiocb_store_error,
14473858a1f4SJohn Baldwin 	.store_kernelinfo = aiocb_store_kernelinfo,
14483858a1f4SJohn Baldwin 	.store_aiocb = aiocb_store_aiocb,
14493858a1f4SJohn Baldwin };
14503858a1f4SJohn Baldwin 
1451bfbbc4aaSJason Evans /*
1452bfbbc4aaSJason Evans  * Queue a new AIO request.  Choosing either the threaded or direct physio VCHR
1453bfbbc4aaSJason Evans  * technique is done in this code.
14542244ea07SJohn Dyson  */
14556a1162d4SAlexander Leidinger int
1456a9bf5e37SDavid Xu aio_aqueue(struct thread *td, struct aiocb *job, struct aioliojob *lj,
14573858a1f4SJohn Baldwin 	int type, struct aiocb_ops *ops)
1458fd3bf775SJohn Dyson {
1459b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
14602244ea07SJohn Dyson 	struct file *fp;
1461bfbbc4aaSJason Evans 	struct socket *so;
1462dbbccfe9SDavid Xu 	struct aiocblist *aiocbe, *cb;
14632244ea07SJohn Dyson 	struct kaioinfo *ki;
1464c6fa9f78SAlan Cox 	struct kevent kev;
1465576c004fSAlfred Perlstein 	struct sockbuf *sb;
14661ce91824SDavid Xu 	int opcode;
14671ce91824SDavid Xu 	int error;
14684db71d27SJohn-Mark Gurney 	int fd, kqfd;
14691ce91824SDavid Xu 	int jid;
14702244ea07SJohn Dyson 
1471a9bf5e37SDavid Xu 	if (p->p_aioinfo == NULL)
1472a9bf5e37SDavid Xu 		aio_init_aioinfo(p);
1473a9bf5e37SDavid Xu 
14741ce91824SDavid Xu 	ki = p->p_aioinfo;
14751ce91824SDavid Xu 
14763858a1f4SJohn Baldwin 	ops->store_status(job, -1);
14773858a1f4SJohn Baldwin 	ops->store_error(job, 0);
14783858a1f4SJohn Baldwin 	ops->store_kernelinfo(job, -1);
1479a9bf5e37SDavid Xu 
1480a9bf5e37SDavid Xu 	if (num_queue_count >= max_queue_count ||
1481a9bf5e37SDavid Xu 	    ki->kaio_count >= ki->kaio_qallowed_count) {
14823858a1f4SJohn Baldwin 		ops->store_error(job, EAGAIN);
1483a9bf5e37SDavid Xu 		return (EAGAIN);
1484a9bf5e37SDavid Xu 	}
1485a9bf5e37SDavid Xu 
14861ce91824SDavid Xu 	aiocbe = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO);
1487fd3bf775SJohn Dyson 	aiocbe->inputcharge = 0;
1488fd3bf775SJohn Dyson 	aiocbe->outputcharge = 0;
1489759ccccaSDavid Xu 	knlist_init(&aiocbe->klist, AIO_MTX(ki), NULL, NULL, NULL);
1490fd3bf775SJohn Dyson 
14913858a1f4SJohn Baldwin 	error = ops->copyin(job, &aiocbe->uaiocb);
14922244ea07SJohn Dyson 	if (error) {
14933858a1f4SJohn Baldwin 		ops->store_error(job, error);
1494c897b813SJeff Roberson 		uma_zfree(aiocb_zone, aiocbe);
1495ac41f2efSAlfred Perlstein 		return (error);
14962244ea07SJohn Dyson 	}
149768d71118SDavid Xu 
149868d71118SDavid Xu 	if (aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT &&
149968d71118SDavid Xu 	    aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL &&
150068d71118SDavid Xu 	    aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID &&
150168d71118SDavid Xu 	    aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) {
15023858a1f4SJohn Baldwin 		ops->store_error(job, EINVAL);
150368d71118SDavid Xu 		uma_zfree(aiocb_zone, aiocbe);
150468d71118SDavid Xu 		return (EINVAL);
150568d71118SDavid Xu 	}
150668d71118SDavid Xu 
15074c0fb2cfSDavid Xu 	if ((aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL ||
15084c0fb2cfSDavid Xu 	     aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) &&
15092f3cf918SAlfred Perlstein 		!_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) {
1510c897b813SJeff Roberson 		uma_zfree(aiocb_zone, aiocbe);
1511ac41f2efSAlfred Perlstein 		return (EINVAL);
15122f3cf918SAlfred Perlstein 	}
15132244ea07SJohn Dyson 
15144c0fb2cfSDavid Xu 	ksiginfo_init(&aiocbe->ksi);
15154c0fb2cfSDavid Xu 
1516bfbbc4aaSJason Evans 	/* Save userspace address of the job info. */
151711783b14SJohn Dyson 	aiocbe->uuaiocb = job;
151811783b14SJohn Dyson 
1519bfbbc4aaSJason Evans 	/* Get the opcode. */
1520bfbbc4aaSJason Evans 	if (type != LIO_NOP)
1521a624e84fSJohn Dyson 		aiocbe->uaiocb.aio_lio_opcode = type;
1522a624e84fSJohn Dyson 	opcode = aiocbe->uaiocb.aio_lio_opcode;
15232244ea07SJohn Dyson 
15242a522eb9SJohn Baldwin 	/* Fetch the file object for the specified file descriptor. */
15252244ea07SJohn Dyson 	fd = aiocbe->uaiocb.aio_fildes;
15262a522eb9SJohn Baldwin 	switch (opcode) {
15272a522eb9SJohn Baldwin 	case LIO_WRITE:
15282a522eb9SJohn Baldwin 		error = fget_write(td, fd, &fp);
15292a522eb9SJohn Baldwin 		break;
15302a522eb9SJohn Baldwin 	case LIO_READ:
15312a522eb9SJohn Baldwin 		error = fget_read(td, fd, &fp);
15322a522eb9SJohn Baldwin 		break;
15332a522eb9SJohn Baldwin 	default:
15342a522eb9SJohn Baldwin 		error = fget(td, fd, &fp);
15352a522eb9SJohn Baldwin 	}
15362a522eb9SJohn Baldwin 	if (error) {
1537c897b813SJeff Roberson 		uma_zfree(aiocb_zone, aiocbe);
15383858a1f4SJohn Baldwin 		ops->store_error(job, error);
1539af56abaaSJohn Baldwin 		return (error);
15402244ea07SJohn Dyson 	}
154199eee864SDavid Xu 
154299eee864SDavid Xu 	if (opcode == LIO_SYNC && fp->f_vnode == NULL) {
154399eee864SDavid Xu 		error = EINVAL;
154499eee864SDavid Xu 		goto aqueue_fail;
154599eee864SDavid Xu 	}
15462244ea07SJohn Dyson 
1547dbbccfe9SDavid Xu 	if (opcode != LIO_SYNC && aiocbe->uaiocb.aio_offset == -1LL) {
1548ae124fc4SAlan Cox 		error = EINVAL;
1549ae124fc4SAlan Cox 		goto aqueue_fail;
15502244ea07SJohn Dyson 	}
15511ce91824SDavid Xu 
155299eee864SDavid Xu 	aiocbe->fd_file = fp;
15531ce91824SDavid Xu 
155499eee864SDavid Xu 	mtx_lock(&aio_job_mtx);
155599eee864SDavid Xu 	jid = jobrefid++;
155699eee864SDavid Xu 	aiocbe->seqno = jobseqno++;
155799eee864SDavid Xu 	mtx_unlock(&aio_job_mtx);
15583858a1f4SJohn Baldwin 	error = ops->store_kernelinfo(job, jid);
15591ce91824SDavid Xu 	if (error) {
15601ce91824SDavid Xu 		error = EINVAL;
15611ce91824SDavid Xu 		goto aqueue_fail;
15621ce91824SDavid Xu 	}
15631ce91824SDavid Xu 	aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jid;
15642244ea07SJohn Dyson 
15652244ea07SJohn Dyson 	if (opcode == LIO_NOP) {
1566a5c0b1c0SAlan Cox 		fdrop(fp, td);
1567c897b813SJeff Roberson 		uma_zfree(aiocb_zone, aiocbe);
1568ac41f2efSAlfred Perlstein 		return (0);
15692244ea07SJohn Dyson 	}
157099eee864SDavid Xu 	if ((opcode != LIO_READ) && (opcode != LIO_WRITE) &&
157199eee864SDavid Xu 	    (opcode != LIO_SYNC)) {
1572ae124fc4SAlan Cox 		error = EINVAL;
1573ae124fc4SAlan Cox 		goto aqueue_fail;
15742244ea07SJohn Dyson 	}
15752244ea07SJohn Dyson 
15764db71d27SJohn-Mark Gurney 	if (aiocbe->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT)
1577cb679c38SJonathan Lemon 		goto no_kqueue;
15784db71d27SJohn-Mark Gurney 	kqfd = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue;
1579b46f1c55SAlan Cox 	kev.ident = (uintptr_t)aiocbe->uuaiocb;
1580cb679c38SJonathan Lemon 	kev.filter = EVFILT_AIO;
1581cb679c38SJonathan Lemon 	kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
1582b46f1c55SAlan Cox 	kev.data = (intptr_t)aiocbe;
15831ce91824SDavid Xu 	kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sival_ptr;
15844db71d27SJohn-Mark Gurney 	error = kqfd_register(kqfd, &kev, td, 1);
1585cb679c38SJonathan Lemon aqueue_fail:
1586cb679c38SJonathan Lemon 	if (error) {
1587a5c0b1c0SAlan Cox 		fdrop(fp, td);
1588c897b813SJeff Roberson 		uma_zfree(aiocb_zone, aiocbe);
15893858a1f4SJohn Baldwin 		ops->store_error(job, error);
1590279d7226SMatthew Dillon 		goto done;
1591cb679c38SJonathan Lemon 	}
1592cb679c38SJonathan Lemon no_kqueue:
1593cb679c38SJonathan Lemon 
15943858a1f4SJohn Baldwin 	ops->store_error(job, EINPROGRESS);
1595fd3bf775SJohn Dyson 	aiocbe->uaiocb._aiocb_private.error = EINPROGRESS;
15962244ea07SJohn Dyson 	aiocbe->userproc = p;
1597f8f750c5SRobert Watson 	aiocbe->cred = crhold(td->td_ucred);
15982244ea07SJohn Dyson 	aiocbe->jobflags = 0;
159984af4da6SJohn Dyson 	aiocbe->lio = lj;
16002244ea07SJohn Dyson 
160199eee864SDavid Xu 	if (opcode == LIO_SYNC)
160299eee864SDavid Xu 		goto queueit;
160399eee864SDavid Xu 
1604bfbbc4aaSJason Evans 	if (fp->f_type == DTYPE_SOCKET) {
1605bfbbc4aaSJason Evans 		/*
1606bfbbc4aaSJason Evans 		 * Alternate queueing for socket ops: Reach down into the
1607bfbbc4aaSJason Evans 		 * descriptor to get the socket data.  Then check to see if the
1608bfbbc4aaSJason Evans 		 * socket is ready to be read or written (based on the requested
1609bfbbc4aaSJason Evans 		 * operation).
1610bfbbc4aaSJason Evans 		 *
1611bfbbc4aaSJason Evans 		 * If it is not ready for io, then queue the aiocbe on the
1612bfbbc4aaSJason Evans 		 * socket, and set the flags so we get a call when sbnotify()
1613bfbbc4aaSJason Evans 		 * happens.
1614576c004fSAlfred Perlstein 		 *
1615576c004fSAlfred Perlstein 		 * Note if opcode is neither LIO_WRITE nor LIO_READ we lock
1616576c004fSAlfred Perlstein 		 * and unlock the snd sockbuf for no reason.
1617bfbbc4aaSJason Evans 		 */
161848e3128bSMatthew Dillon 		so = fp->f_data;
1619576c004fSAlfred Perlstein 		sb = (opcode == LIO_READ) ? &so->so_rcv : &so->so_snd;
1620576c004fSAlfred Perlstein 		SOCKBUF_LOCK(sb);
1621bfbbc4aaSJason Evans 		if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode ==
1622bfbbc4aaSJason Evans 		    LIO_WRITE) && (!sowriteable(so)))) {
1623576c004fSAlfred Perlstein 			sb->sb_flags |= SB_AIO;
16241aa4c324SDavid Xu 
16251aa4c324SDavid Xu 			mtx_lock(&aio_job_mtx);
16261aa4c324SDavid Xu 			TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list);
16271aa4c324SDavid Xu 			mtx_unlock(&aio_job_mtx);
16281aa4c324SDavid Xu 
1629759ccccaSDavid Xu 			AIO_LOCK(ki);
16301ce91824SDavid Xu 			TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist);
16311aa4c324SDavid Xu 			TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
16321ce91824SDavid Xu 			aiocbe->jobstate = JOBST_JOBQSOCK;
16331ce91824SDavid Xu 			ki->kaio_count++;
16341ce91824SDavid Xu 			if (lj)
16351ce91824SDavid Xu 				lj->lioj_count++;
1636759ccccaSDavid Xu 			AIO_UNLOCK(ki);
1637576c004fSAlfred Perlstein 			SOCKBUF_UNLOCK(sb);
16381ce91824SDavid Xu 			atomic_add_int(&num_queue_count, 1);
1639279d7226SMatthew Dillon 			error = 0;
1640279d7226SMatthew Dillon 			goto done;
1641bfbbc4aaSJason Evans 		}
1642576c004fSAlfred Perlstein 		SOCKBUF_UNLOCK(sb);
1643bfbbc4aaSJason Evans 	}
1644bfbbc4aaSJason Evans 
1645bfbbc4aaSJason Evans 	if ((error = aio_qphysio(p, aiocbe)) == 0)
1646279d7226SMatthew Dillon 		goto done;
16471ce91824SDavid Xu #if 0
1648279d7226SMatthew Dillon 	if (error > 0) {
1649fd3bf775SJohn Dyson 		aiocbe->uaiocb._aiocb_private.error = error;
16503858a1f4SJohn Baldwin 		ops->store_error(job, error);
1651279d7226SMatthew Dillon 		goto done;
1652fd3bf775SJohn Dyson 	}
16531ce91824SDavid Xu #endif
165499eee864SDavid Xu queueit:
1655bfbbc4aaSJason Evans 	/* No buffer for daemon I/O. */
165684af4da6SJohn Dyson 	aiocbe->bp = NULL;
165799eee864SDavid Xu 	atomic_add_int(&num_queue_count, 1);
165884af4da6SJohn Dyson 
1659759ccccaSDavid Xu 	AIO_LOCK(ki);
16601ce91824SDavid Xu 	ki->kaio_count++;
1661bfbbc4aaSJason Evans 	if (lj)
16621ce91824SDavid Xu 		lj->lioj_count++;
1663fd3bf775SJohn Dyson 	TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist);
16641ce91824SDavid Xu 	TAILQ_INSERT_TAIL(&ki->kaio_all, aiocbe, allist);
166599eee864SDavid Xu 	if (opcode == LIO_SYNC) {
1666dbbccfe9SDavid Xu 		TAILQ_FOREACH(cb, &ki->kaio_jobqueue, plist) {
1667dbbccfe9SDavid Xu 			if (cb->fd_file == aiocbe->fd_file &&
1668dbbccfe9SDavid Xu 			    cb->uaiocb.aio_lio_opcode != LIO_SYNC &&
1669dbbccfe9SDavid Xu 			    cb->seqno < aiocbe->seqno) {
1670dbbccfe9SDavid Xu 				cb->jobflags |= AIOCBLIST_CHECKSYNC;
1671dbbccfe9SDavid Xu 				aiocbe->pending++;
1672dbbccfe9SDavid Xu 			}
1673dbbccfe9SDavid Xu 		}
1674dbbccfe9SDavid Xu 		TAILQ_FOREACH(cb, &ki->kaio_bufqueue, plist) {
1675dbbccfe9SDavid Xu 			if (cb->fd_file == aiocbe->fd_file &&
1676dbbccfe9SDavid Xu 			    cb->uaiocb.aio_lio_opcode != LIO_SYNC &&
1677dbbccfe9SDavid Xu 			    cb->seqno < aiocbe->seqno) {
1678dbbccfe9SDavid Xu 				cb->jobflags |= AIOCBLIST_CHECKSYNC;
1679dbbccfe9SDavid Xu 				aiocbe->pending++;
1680dbbccfe9SDavid Xu 			}
1681dbbccfe9SDavid Xu 		}
1682dbbccfe9SDavid Xu 		if (aiocbe->pending != 0) {
168399eee864SDavid Xu 			TAILQ_INSERT_TAIL(&ki->kaio_syncqueue, aiocbe, list);
168499eee864SDavid Xu 			aiocbe->jobstate = JOBST_JOBQSYNC;
1685759ccccaSDavid Xu 			AIO_UNLOCK(ki);
1686dbbccfe9SDavid Xu 			goto done;
1687dbbccfe9SDavid Xu 		}
1688dbbccfe9SDavid Xu 	}
16891ce91824SDavid Xu 	mtx_lock(&aio_job_mtx);
16901ce91824SDavid Xu 	TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list);
16911ce91824SDavid Xu 	aiocbe->jobstate = JOBST_JOBQGLOBAL;
169299eee864SDavid Xu 	aio_kick_nowait(p);
169399eee864SDavid Xu 	mtx_unlock(&aio_job_mtx);
1694759ccccaSDavid Xu 	AIO_UNLOCK(ki);
16951ce91824SDavid Xu 	error = 0;
169699eee864SDavid Xu done:
169799eee864SDavid Xu 	return (error);
169899eee864SDavid Xu }
169999eee864SDavid Xu 
170099eee864SDavid Xu static void
170199eee864SDavid Xu aio_kick_nowait(struct proc *userp)
170299eee864SDavid Xu {
170399eee864SDavid Xu 	struct kaioinfo *ki = userp->p_aioinfo;
170499eee864SDavid Xu 	struct aiothreadlist *aiop;
170599eee864SDavid Xu 
170699eee864SDavid Xu 	mtx_assert(&aio_job_mtx, MA_OWNED);
170799eee864SDavid Xu 	if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
170899eee864SDavid Xu 		TAILQ_REMOVE(&aio_freeproc, aiop, list);
170999eee864SDavid Xu 		aiop->aiothreadflags &= ~AIOP_FREE;
171099eee864SDavid Xu 		wakeup(aiop->aiothread);
1711dbbccfe9SDavid Xu 	} else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
1712dbbccfe9SDavid Xu 	    ((ki->kaio_active_count + num_aio_resv_start) <
1713dbbccfe9SDavid Xu 	    ki->kaio_maxactive_count)) {
171499eee864SDavid Xu 		taskqueue_enqueue(taskqueue_aiod_bio, &ki->kaio_task);
171599eee864SDavid Xu 	}
171699eee864SDavid Xu }
171799eee864SDavid Xu 
1718dbbccfe9SDavid Xu static int
171999eee864SDavid Xu aio_kick(struct proc *userp)
172099eee864SDavid Xu {
172199eee864SDavid Xu 	struct kaioinfo *ki = userp->p_aioinfo;
172299eee864SDavid Xu 	struct aiothreadlist *aiop;
1723dbbccfe9SDavid Xu 	int error, ret = 0;
172499eee864SDavid Xu 
172599eee864SDavid Xu 	mtx_assert(&aio_job_mtx, MA_OWNED);
172699eee864SDavid Xu retryproc:
1727d254af07SMatthew Dillon 	if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
17282244ea07SJohn Dyson 		TAILQ_REMOVE(&aio_freeproc, aiop, list);
1729b40ce416SJulian Elischer 		aiop->aiothreadflags &= ~AIOP_FREE;
1730b40ce416SJulian Elischer 		wakeup(aiop->aiothread);
1731fd3bf775SJohn Dyson 	} else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) &&
1732fd3bf775SJohn Dyson 	    ((ki->kaio_active_count + num_aio_resv_start) <
1733fd3bf775SJohn Dyson 	    ki->kaio_maxactive_count)) {
1734fd3bf775SJohn Dyson 		num_aio_resv_start++;
17351ce91824SDavid Xu 		mtx_unlock(&aio_job_mtx);
17361ce91824SDavid Xu 		error = aio_newproc(&num_aio_resv_start);
17371ce91824SDavid Xu 		mtx_lock(&aio_job_mtx);
17381ce91824SDavid Xu 		if (error) {
173984af4da6SJohn Dyson 			num_aio_resv_start--;
17402244ea07SJohn Dyson 			goto retryproc;
1741fd3bf775SJohn Dyson 		}
1742dbbccfe9SDavid Xu 	} else {
1743dbbccfe9SDavid Xu 		ret = -1;
17441ce91824SDavid Xu 	}
1745dbbccfe9SDavid Xu 	return (ret);
174699eee864SDavid Xu }
17471ce91824SDavid Xu 
174899eee864SDavid Xu static void
174999eee864SDavid Xu aio_kick_helper(void *context, int pending)
175099eee864SDavid Xu {
175199eee864SDavid Xu 	struct proc *userp = context;
175299eee864SDavid Xu 
175399eee864SDavid Xu 	mtx_lock(&aio_job_mtx);
1754dbbccfe9SDavid Xu 	while (--pending >= 0) {
1755dbbccfe9SDavid Xu 		if (aio_kick(userp))
1756dbbccfe9SDavid Xu 			break;
1757dbbccfe9SDavid Xu 	}
175899eee864SDavid Xu 	mtx_unlock(&aio_job_mtx);
17592244ea07SJohn Dyson }
17602244ea07SJohn Dyson 
1761fd3bf775SJohn Dyson /*
1762bfbbc4aaSJason Evans  * Support the aio_return system call, as a side-effect, kernel resources are
1763bfbbc4aaSJason Evans  * released.
17642244ea07SJohn Dyson  */
17653858a1f4SJohn Baldwin static int
17663858a1f4SJohn Baldwin kern_aio_return(struct thread *td, struct aiocb *uaiocb, struct aiocb_ops *ops)
1767fd3bf775SJohn Dyson {
1768b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
17691ce91824SDavid Xu 	struct aiocblist *cb;
17702244ea07SJohn Dyson 	struct kaioinfo *ki;
17711ce91824SDavid Xu 	int status, error;
17722244ea07SJohn Dyson 
1773c0bf5caaSAlan Cox 	ki = p->p_aioinfo;
1774c0bf5caaSAlan Cox 	if (ki == NULL)
1775ac41f2efSAlfred Perlstein 		return (EINVAL);
1776759ccccaSDavid Xu 	AIO_LOCK(ki);
17771ce91824SDavid Xu 	TAILQ_FOREACH(cb, &ki->kaio_done, plist) {
17781ce91824SDavid Xu 		if (cb->uuaiocb == uaiocb)
1779c0bf5caaSAlan Cox 			break;
1780c0bf5caaSAlan Cox 	}
1781c0bf5caaSAlan Cox 	if (cb != NULL) {
17821ce91824SDavid Xu 		MPASS(cb->jobstate == JOBST_JOBFINISHED);
17831ce91824SDavid Xu 		status = cb->uaiocb._aiocb_private.status;
17841ce91824SDavid Xu 		error = cb->uaiocb._aiocb_private.error;
17851ce91824SDavid Xu 		td->td_retval[0] = status;
178669cd28daSDoug Ambrisko 		if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
17871c4bcd05SJeff Roberson 			td->td_ru.ru_oublock += cb->outputcharge;
178869cd28daSDoug Ambrisko 			cb->outputcharge = 0;
178969cd28daSDoug Ambrisko 		} else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
17901c4bcd05SJeff Roberson 			td->td_ru.ru_inblock += cb->inputcharge;
179169cd28daSDoug Ambrisko 			cb->inputcharge = 0;
179269cd28daSDoug Ambrisko 		}
179384af4da6SJohn Dyson 		aio_free_entry(cb);
1794759ccccaSDavid Xu 		AIO_UNLOCK(ki);
17953858a1f4SJohn Baldwin 		ops->store_error(uaiocb, error);
17963858a1f4SJohn Baldwin 		ops->store_status(uaiocb, status);
179755a122bfSDavid Xu 	} else {
17981ce91824SDavid Xu 		error = EINVAL;
1799759ccccaSDavid Xu 		AIO_UNLOCK(ki);
180055a122bfSDavid Xu 	}
18011ce91824SDavid Xu 	return (error);
18022244ea07SJohn Dyson }
18032244ea07SJohn Dyson 
18043858a1f4SJohn Baldwin int
18053858a1f4SJohn Baldwin aio_return(struct thread *td, struct aio_return_args *uap)
18063858a1f4SJohn Baldwin {
18073858a1f4SJohn Baldwin 
18083858a1f4SJohn Baldwin 	return (kern_aio_return(td, uap->aiocbp, &aiocb_ops));
18093858a1f4SJohn Baldwin }
18103858a1f4SJohn Baldwin 
18112244ea07SJohn Dyson /*
1812bfbbc4aaSJason Evans  * Allow a process to wakeup when any of the I/O requests are completed.
18132244ea07SJohn Dyson  */
18143858a1f4SJohn Baldwin static int
18153858a1f4SJohn Baldwin kern_aio_suspend(struct thread *td, int njoblist, struct aiocb **ujoblist,
18163858a1f4SJohn Baldwin     struct timespec *ts)
1817fd3bf775SJohn Dyson {
1818b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
18194a11ca4eSPoul-Henning Kamp 	struct timeval atv;
18202244ea07SJohn Dyson 	struct kaioinfo *ki;
18211ce91824SDavid Xu 	struct aiocblist *cb, *cbfirst;
18223858a1f4SJohn Baldwin 	int error, i, timo;
18232244ea07SJohn Dyson 
18242244ea07SJohn Dyson 	timo = 0;
18253858a1f4SJohn Baldwin 	if (ts) {
18263858a1f4SJohn Baldwin 		if (ts->tv_nsec < 0 || ts->tv_nsec >= 1000000000)
18272244ea07SJohn Dyson 			return (EINVAL);
18282244ea07SJohn Dyson 
18293858a1f4SJohn Baldwin 		TIMESPEC_TO_TIMEVAL(&atv, ts);
18302244ea07SJohn Dyson 		if (itimerfix(&atv))
18312244ea07SJohn Dyson 			return (EINVAL);
1832227ee8a1SPoul-Henning Kamp 		timo = tvtohz(&atv);
18332244ea07SJohn Dyson 	}
18342244ea07SJohn Dyson 
18352244ea07SJohn Dyson 	ki = p->p_aioinfo;
18362244ea07SJohn Dyson 	if (ki == NULL)
1837ac41f2efSAlfred Perlstein 		return (EAGAIN);
18382244ea07SJohn Dyson 
18393858a1f4SJohn Baldwin 	if (njoblist == 0)
1840ac41f2efSAlfred Perlstein 		return (0);
18412244ea07SJohn Dyson 
1842759ccccaSDavid Xu 	AIO_LOCK(ki);
18431ce91824SDavid Xu 	for (;;) {
18441ce91824SDavid Xu 		cbfirst = NULL;
18451ce91824SDavid Xu 		error = 0;
18461ce91824SDavid Xu 		TAILQ_FOREACH(cb, &ki->kaio_all, allist) {
184784af4da6SJohn Dyson 			for (i = 0; i < njoblist; i++) {
18481ce91824SDavid Xu 				if (cb->uuaiocb == ujoblist[i]) {
18491ce91824SDavid Xu 					if (cbfirst == NULL)
18501ce91824SDavid Xu 						cbfirst = cb;
18511ce91824SDavid Xu 					if (cb->jobstate == JOBST_JOBFINISHED)
18521ce91824SDavid Xu 						goto RETURN;
185384af4da6SJohn Dyson 				}
185484af4da6SJohn Dyson 			}
185584af4da6SJohn Dyson 		}
18561ce91824SDavid Xu 		/* All tasks were finished. */
18571ce91824SDavid Xu 		if (cbfirst == NULL)
18581ce91824SDavid Xu 			break;
18592244ea07SJohn Dyson 
1860fd3bf775SJohn Dyson 		ki->kaio_flags |= KAIO_WAKEUP;
1861759ccccaSDavid Xu 		error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH,
18621ce91824SDavid Xu 		    "aiospn", timo);
18631ce91824SDavid Xu 		if (error == ERESTART)
18641ce91824SDavid Xu 			error = EINTR;
18651ce91824SDavid Xu 		if (error)
18661ce91824SDavid Xu 			break;
18672244ea07SJohn Dyson 	}
18681ce91824SDavid Xu RETURN:
1869759ccccaSDavid Xu 	AIO_UNLOCK(ki);
18703858a1f4SJohn Baldwin 	return (error);
18713858a1f4SJohn Baldwin }
18723858a1f4SJohn Baldwin 
18733858a1f4SJohn Baldwin int
18743858a1f4SJohn Baldwin aio_suspend(struct thread *td, struct aio_suspend_args *uap)
18753858a1f4SJohn Baldwin {
18763858a1f4SJohn Baldwin 	struct timespec ts, *tsp;
18773858a1f4SJohn Baldwin 	struct aiocb **ujoblist;
18783858a1f4SJohn Baldwin 	int error;
18793858a1f4SJohn Baldwin 
18803858a1f4SJohn Baldwin 	if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
18813858a1f4SJohn Baldwin 		return (EINVAL);
18823858a1f4SJohn Baldwin 
18833858a1f4SJohn Baldwin 	if (uap->timeout) {
18843858a1f4SJohn Baldwin 		/* Get timespec struct. */
18853858a1f4SJohn Baldwin 		if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
18863858a1f4SJohn Baldwin 			return (error);
18873858a1f4SJohn Baldwin 		tsp = &ts;
18883858a1f4SJohn Baldwin 	} else
18893858a1f4SJohn Baldwin 		tsp = NULL;
18903858a1f4SJohn Baldwin 
18913858a1f4SJohn Baldwin 	ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
18923858a1f4SJohn Baldwin 	error = copyin(uap->aiocbp, ujoblist, uap->nent * sizeof(ujoblist[0]));
18933858a1f4SJohn Baldwin 	if (error == 0)
18943858a1f4SJohn Baldwin 		error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
18951ce91824SDavid Xu 	uma_zfree(aiol_zone, ujoblist);
18961ce91824SDavid Xu 	return (error);
18972244ea07SJohn Dyson }
1898ee877a35SJohn Dyson 
1899ee877a35SJohn Dyson /*
1900dd85920aSJason Evans  * aio_cancel cancels any non-physio aio operations not currently in
1901dd85920aSJason Evans  * progress.
1902ee877a35SJohn Dyson  */
1903ee877a35SJohn Dyson int
1904b40ce416SJulian Elischer aio_cancel(struct thread *td, struct aio_cancel_args *uap)
1905fd3bf775SJohn Dyson {
1906b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
1907dd85920aSJason Evans 	struct kaioinfo *ki;
1908dd85920aSJason Evans 	struct aiocblist *cbe, *cbn;
1909dd85920aSJason Evans 	struct file *fp;
1910dd85920aSJason Evans 	struct socket *so;
19111ce91824SDavid Xu 	int error;
19121aa4c324SDavid Xu 	int remove;
1913dd85920aSJason Evans 	int cancelled = 0;
1914dd85920aSJason Evans 	int notcancelled = 0;
1915dd85920aSJason Evans 	struct vnode *vp;
1916dd85920aSJason Evans 
19172a522eb9SJohn Baldwin 	/* Lookup file object. */
19181ce91824SDavid Xu 	error = fget(td, uap->fd, &fp);
19192a522eb9SJohn Baldwin 	if (error)
19202a522eb9SJohn Baldwin 		return (error);
1921dd85920aSJason Evans 
19221ce91824SDavid Xu 	ki = p->p_aioinfo;
19231ce91824SDavid Xu 	if (ki == NULL)
19241ce91824SDavid Xu 		goto done;
19251ce91824SDavid Xu 
1926dd85920aSJason Evans 	if (fp->f_type == DTYPE_VNODE) {
19273b6d9652SPoul-Henning Kamp 		vp = fp->f_vnode;
1928dd85920aSJason Evans 		if (vn_isdisk(vp, &error)) {
19292a522eb9SJohn Baldwin 			fdrop(fp, td);
1930b40ce416SJulian Elischer 			td->td_retval[0] = AIO_NOTCANCELED;
1931ac41f2efSAlfred Perlstein 			return (0);
1932dd85920aSJason Evans 		}
1933dd85920aSJason Evans 	}
1934dd85920aSJason Evans 
1935759ccccaSDavid Xu 	AIO_LOCK(ki);
19362a522eb9SJohn Baldwin 	TAILQ_FOREACH_SAFE(cbe, &ki->kaio_jobqueue, plist, cbn) {
1937dd85920aSJason Evans 		if ((uap->fd == cbe->uaiocb.aio_fildes) &&
1938dd85920aSJason Evans 		    ((uap->aiocbp == NULL) ||
1939dd85920aSJason Evans 		     (uap->aiocbp == cbe->uuaiocb))) {
19401aa4c324SDavid Xu 			remove = 0;
19411aa4c324SDavid Xu 
19421ce91824SDavid Xu 			mtx_lock(&aio_job_mtx);
1943dd85920aSJason Evans 			if (cbe->jobstate == JOBST_JOBQGLOBAL) {
1944dd85920aSJason Evans 				TAILQ_REMOVE(&aio_jobs, cbe, list);
19451aa4c324SDavid Xu 				remove = 1;
19461aa4c324SDavid Xu 			} else if (cbe->jobstate == JOBST_JOBQSOCK) {
19471aa4c324SDavid Xu 				MPASS(fp->f_type == DTYPE_SOCKET);
19481aa4c324SDavid Xu 				so = fp->f_data;
19491aa4c324SDavid Xu 				TAILQ_REMOVE(&so->so_aiojobq, cbe, list);
19501aa4c324SDavid Xu 				remove = 1;
195199eee864SDavid Xu 			} else if (cbe->jobstate == JOBST_JOBQSYNC) {
195299eee864SDavid Xu 				TAILQ_REMOVE(&ki->kaio_syncqueue, cbe, list);
195399eee864SDavid Xu 				remove = 1;
19541aa4c324SDavid Xu 			}
19551ce91824SDavid Xu 			mtx_unlock(&aio_job_mtx);
19561aa4c324SDavid Xu 
19571aa4c324SDavid Xu 			if (remove) {
19581ce91824SDavid Xu 				TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist);
1959dd85920aSJason Evans 				cbe->uaiocb._aiocb_private.status = -1;
1960dd85920aSJason Evans 				cbe->uaiocb._aiocb_private.error = ECANCELED;
19611ce91824SDavid Xu 				aio_bio_done_notify(p, cbe, DONE_QUEUE);
19621ce91824SDavid Xu 				cancelled++;
1963dd85920aSJason Evans 			} else {
1964dd85920aSJason Evans 				notcancelled++;
1965dd85920aSJason Evans 			}
19661aa4c324SDavid Xu 			if (uap->aiocbp != NULL)
19671aa4c324SDavid Xu 				break;
1968dd85920aSJason Evans 		}
1969dd85920aSJason Evans 	}
1970759ccccaSDavid Xu 	AIO_UNLOCK(ki);
19711ce91824SDavid Xu 
1972ad49abc0SAlan Cox done:
19732a522eb9SJohn Baldwin 	fdrop(fp, td);
19741aa4c324SDavid Xu 
19751aa4c324SDavid Xu 	if (uap->aiocbp != NULL) {
1976dd85920aSJason Evans 		if (cancelled) {
1977b40ce416SJulian Elischer 			td->td_retval[0] = AIO_CANCELED;
1978ac41f2efSAlfred Perlstein 			return (0);
1979dd85920aSJason Evans 		}
19801aa4c324SDavid Xu 	}
19811aa4c324SDavid Xu 
19821aa4c324SDavid Xu 	if (notcancelled) {
19831aa4c324SDavid Xu 		td->td_retval[0] = AIO_NOTCANCELED;
19841aa4c324SDavid Xu 		return (0);
19851aa4c324SDavid Xu 	}
19861aa4c324SDavid Xu 
19871aa4c324SDavid Xu 	if (cancelled) {
19881aa4c324SDavid Xu 		td->td_retval[0] = AIO_CANCELED;
19891aa4c324SDavid Xu 		return (0);
19901aa4c324SDavid Xu 	}
19911aa4c324SDavid Xu 
1992b40ce416SJulian Elischer 	td->td_retval[0] = AIO_ALLDONE;
1993dd85920aSJason Evans 
1994ac41f2efSAlfred Perlstein 	return (0);
1995ee877a35SJohn Dyson }
1996ee877a35SJohn Dyson 
1997ee877a35SJohn Dyson /*
1998873fbcd7SRobert Watson  * aio_error is implemented in the kernel level for compatibility purposes
1999873fbcd7SRobert Watson  * only.  For a user mode async implementation, it would be best to do it in
2000873fbcd7SRobert Watson  * a userland subroutine.
2001ee877a35SJohn Dyson  */
20023858a1f4SJohn Baldwin static int
20033858a1f4SJohn Baldwin kern_aio_error(struct thread *td, struct aiocb *aiocbp, struct aiocb_ops *ops)
2004fd3bf775SJohn Dyson {
2005b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
20062244ea07SJohn Dyson 	struct aiocblist *cb;
20072244ea07SJohn Dyson 	struct kaioinfo *ki;
20081ce91824SDavid Xu 	int status;
2009ee877a35SJohn Dyson 
20102244ea07SJohn Dyson 	ki = p->p_aioinfo;
20111ce91824SDavid Xu 	if (ki == NULL) {
20121ce91824SDavid Xu 		td->td_retval[0] = EINVAL;
20131ce91824SDavid Xu 		return (0);
20141ce91824SDavid Xu 	}
2015ee877a35SJohn Dyson 
2016759ccccaSDavid Xu 	AIO_LOCK(ki);
20171ce91824SDavid Xu 	TAILQ_FOREACH(cb, &ki->kaio_all, allist) {
20183858a1f4SJohn Baldwin 		if (cb->uuaiocb == aiocbp) {
20191ce91824SDavid Xu 			if (cb->jobstate == JOBST_JOBFINISHED)
20201ce91824SDavid Xu 				td->td_retval[0] =
20211ce91824SDavid Xu 					cb->uaiocb._aiocb_private.error;
20221ce91824SDavid Xu 			else
2023b40ce416SJulian Elischer 				td->td_retval[0] = EINPROGRESS;
2024759ccccaSDavid Xu 			AIO_UNLOCK(ki);
2025ac41f2efSAlfred Perlstein 			return (0);
20262244ea07SJohn Dyson 		}
20272244ea07SJohn Dyson 	}
2028759ccccaSDavid Xu 	AIO_UNLOCK(ki);
202984af4da6SJohn Dyson 
20302244ea07SJohn Dyson 	/*
2031a9bf5e37SDavid Xu 	 * Hack for failure of aio_aqueue.
20322244ea07SJohn Dyson 	 */
20333858a1f4SJohn Baldwin 	status = ops->fetch_status(aiocbp);
20341ce91824SDavid Xu 	if (status == -1) {
20353858a1f4SJohn Baldwin 		td->td_retval[0] = ops->fetch_error(aiocbp);
20361ce91824SDavid Xu 		return (0);
20371ce91824SDavid Xu 	}
20381ce91824SDavid Xu 
20391ce91824SDavid Xu 	td->td_retval[0] = EINVAL;
20401ce91824SDavid Xu 	return (0);
2041ee877a35SJohn Dyson }
2042ee877a35SJohn Dyson 
20433858a1f4SJohn Baldwin int
20443858a1f4SJohn Baldwin aio_error(struct thread *td, struct aio_error_args *uap)
20453858a1f4SJohn Baldwin {
20463858a1f4SJohn Baldwin 
20473858a1f4SJohn Baldwin 	return (kern_aio_error(td, uap->aiocbp, &aiocb_ops));
20483858a1f4SJohn Baldwin }
20493858a1f4SJohn Baldwin 
2050eb8e6d52SEivind Eklund /* syscall - asynchronous read from a file (REALTIME) */
2051ee877a35SJohn Dyson int
20520972628aSDavid Xu oaio_read(struct thread *td, struct oaio_read_args *uap)
20530972628aSDavid Xu {
20540972628aSDavid Xu 
20553858a1f4SJohn Baldwin 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
20563858a1f4SJohn Baldwin 	    &aiocb_ops_osigevent));
20570972628aSDavid Xu }
20580972628aSDavid Xu 
20590972628aSDavid Xu int
2060b40ce416SJulian Elischer aio_read(struct thread *td, struct aio_read_args *uap)
2061fd3bf775SJohn Dyson {
206221d56e9cSAlfred Perlstein 
20633858a1f4SJohn Baldwin 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, &aiocb_ops));
2064ee877a35SJohn Dyson }
2065ee877a35SJohn Dyson 
2066eb8e6d52SEivind Eklund /* syscall - asynchronous write to a file (REALTIME) */
2067ee877a35SJohn Dyson int
20680972628aSDavid Xu oaio_write(struct thread *td, struct oaio_write_args *uap)
20690972628aSDavid Xu {
20700972628aSDavid Xu 
20713858a1f4SJohn Baldwin 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
20723858a1f4SJohn Baldwin 	    &aiocb_ops_osigevent));
20730972628aSDavid Xu }
20740972628aSDavid Xu 
20750972628aSDavid Xu int
2076b40ce416SJulian Elischer aio_write(struct thread *td, struct aio_write_args *uap)
2077fd3bf775SJohn Dyson {
207821d56e9cSAlfred Perlstein 
20793858a1f4SJohn Baldwin 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops));
20800972628aSDavid Xu }
20810972628aSDavid Xu 
20820972628aSDavid Xu static int
20833858a1f4SJohn Baldwin kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list,
20843858a1f4SJohn Baldwin     struct aiocb **acb_list, int nent, struct sigevent *sig,
20853858a1f4SJohn Baldwin     struct aiocb_ops *ops)
20860972628aSDavid Xu {
2087b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
20883858a1f4SJohn Baldwin 	struct aiocb *iocb;
20892244ea07SJohn Dyson 	struct kaioinfo *ki;
20901ce91824SDavid Xu 	struct aioliojob *lj;
209169cd28daSDoug Ambrisko 	struct kevent kev;
20921ce91824SDavid Xu 	int error;
2093fd3bf775SJohn Dyson 	int nerror;
2094ee877a35SJohn Dyson 	int i;
2095ee877a35SJohn Dyson 
20963858a1f4SJohn Baldwin 	if ((mode != LIO_NOWAIT) && (mode != LIO_WAIT))
2097ac41f2efSAlfred Perlstein 		return (EINVAL);
20982244ea07SJohn Dyson 
2099ae3b195fSTim J. Robbins 	if (nent < 0 || nent > AIO_LISTIO_MAX)
2100ac41f2efSAlfred Perlstein 		return (EINVAL);
21012244ea07SJohn Dyson 
2102bfbbc4aaSJason Evans 	if (p->p_aioinfo == NULL)
21032244ea07SJohn Dyson 		aio_init_aioinfo(p);
21042244ea07SJohn Dyson 
21052244ea07SJohn Dyson 	ki = p->p_aioinfo;
21062244ea07SJohn Dyson 
2107a163d034SWarner Losh 	lj = uma_zalloc(aiolio_zone, M_WAITOK);
210884af4da6SJohn Dyson 	lj->lioj_flags = 0;
21091ce91824SDavid Xu 	lj->lioj_count = 0;
21101ce91824SDavid Xu 	lj->lioj_finished_count = 0;
2111759ccccaSDavid Xu 	knlist_init(&lj->klist, AIO_MTX(ki), NULL, NULL, NULL);
21124c0fb2cfSDavid Xu 	ksiginfo_init(&lj->lioj_ksi);
211369cd28daSDoug Ambrisko 
211484af4da6SJohn Dyson 	/*
2115bfbbc4aaSJason Evans 	 * Setup signal.
211684af4da6SJohn Dyson 	 */
21173858a1f4SJohn Baldwin 	if (sig && (mode == LIO_NOWAIT)) {
21183858a1f4SJohn Baldwin 		bcopy(sig, &lj->lioj_signal, sizeof(lj->lioj_signal));
211969cd28daSDoug Ambrisko 		if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
212069cd28daSDoug Ambrisko 			/* Assume only new style KEVENT */
212169cd28daSDoug Ambrisko 			kev.filter = EVFILT_LIO;
212269cd28daSDoug Ambrisko 			kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
21233858a1f4SJohn Baldwin 			kev.ident = (uintptr_t)uacb_list; /* something unique */
212469cd28daSDoug Ambrisko 			kev.data = (intptr_t)lj;
21251ce91824SDavid Xu 			/* pass user defined sigval data */
21261ce91824SDavid Xu 			kev.udata = lj->lioj_signal.sigev_value.sival_ptr;
21274db71d27SJohn-Mark Gurney 			error = kqfd_register(
21284db71d27SJohn-Mark Gurney 			    lj->lioj_signal.sigev_notify_kqueue, &kev, td, 1);
212969cd28daSDoug Ambrisko 			if (error) {
213069cd28daSDoug Ambrisko 				uma_zfree(aiolio_zone, lj);
213169cd28daSDoug Ambrisko 				return (error);
213269cd28daSDoug Ambrisko 			}
21331ce91824SDavid Xu 		} else if (lj->lioj_signal.sigev_notify == SIGEV_NONE) {
21341ce91824SDavid Xu 			;
213568d71118SDavid Xu 		} else if (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
213668d71118SDavid Xu 			   lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID) {
213768d71118SDavid Xu 				if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) {
213869cd28daSDoug Ambrisko 					uma_zfree(aiolio_zone, lj);
213969cd28daSDoug Ambrisko 					return EINVAL;
214068d71118SDavid Xu 				}
214184af4da6SJohn Dyson 				lj->lioj_flags |= LIOJ_SIGNAL;
214268d71118SDavid Xu 		} else {
214368d71118SDavid Xu 			uma_zfree(aiolio_zone, lj);
214468d71118SDavid Xu 			return EINVAL;
21454d752b01SAlan Cox 		}
21461ce91824SDavid Xu 	}
214769cd28daSDoug Ambrisko 
2148759ccccaSDavid Xu 	AIO_LOCK(ki);
21492f3cf918SAlfred Perlstein 	TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list);
21502244ea07SJohn Dyson 	/*
21511ce91824SDavid Xu 	 * Add extra aiocb count to avoid the lio to be freed
21521ce91824SDavid Xu 	 * by other threads doing aio_waitcomplete or aio_return,
21531ce91824SDavid Xu 	 * and prevent event from being sent until we have queued
21541ce91824SDavid Xu 	 * all tasks.
21551ce91824SDavid Xu 	 */
21561ce91824SDavid Xu 	lj->lioj_count = 1;
2157759ccccaSDavid Xu 	AIO_UNLOCK(ki);
21581ce91824SDavid Xu 
21591ce91824SDavid Xu 	/*
2160bfbbc4aaSJason Evans 	 * Get pointers to the list of I/O requests.
21612244ea07SJohn Dyson 	 */
2162fd3bf775SJohn Dyson 	nerror = 0;
21633858a1f4SJohn Baldwin 	for (i = 0; i < nent; i++) {
21643858a1f4SJohn Baldwin 		iocb = acb_list[i];
21653858a1f4SJohn Baldwin 		if (iocb != NULL) {
21663858a1f4SJohn Baldwin 			error = aio_aqueue(td, iocb, lj, LIO_NOP, ops);
21671ce91824SDavid Xu 			if (error != 0)
2168fd3bf775SJohn Dyson 				nerror++;
2169fd3bf775SJohn Dyson 		}
2170fd3bf775SJohn Dyson 	}
21712244ea07SJohn Dyson 
21721ce91824SDavid Xu 	error = 0;
2173759ccccaSDavid Xu 	AIO_LOCK(ki);
21743858a1f4SJohn Baldwin 	if (mode == LIO_WAIT) {
21751ce91824SDavid Xu 		while (lj->lioj_count - 1 != lj->lioj_finished_count) {
2176fd3bf775SJohn Dyson 			ki->kaio_flags |= KAIO_WAKEUP;
2177759ccccaSDavid Xu 			error = msleep(&p->p_aioinfo, AIO_MTX(ki),
21781ce91824SDavid Xu 			    PRIBIO | PCATCH, "aiospn", 0);
21791ce91824SDavid Xu 			if (error == ERESTART)
21801ce91824SDavid Xu 				error = EINTR;
21811ce91824SDavid Xu 			if (error)
21821ce91824SDavid Xu 				break;
21831ce91824SDavid Xu 		}
21841ce91824SDavid Xu 	} else {
21851ce91824SDavid Xu 		if (lj->lioj_count - 1 == lj->lioj_finished_count) {
21861ce91824SDavid Xu 			if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
21871ce91824SDavid Xu 				lj->lioj_flags |= LIOJ_KEVENT_POSTED;
21881ce91824SDavid Xu 				KNOTE_LOCKED(&lj->klist, 1);
21891ce91824SDavid Xu 			}
21901ce91824SDavid Xu 			if ((lj->lioj_flags & (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED))
21911ce91824SDavid Xu 			    == LIOJ_SIGNAL
21921ce91824SDavid Xu 			    && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
21931ce91824SDavid Xu 			    lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID)) {
21941ce91824SDavid Xu 				aio_sendsig(p, &lj->lioj_signal,
21951ce91824SDavid Xu 					    &lj->lioj_ksi);
21961ce91824SDavid Xu 				lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
21972244ea07SJohn Dyson 			}
21982244ea07SJohn Dyson 		}
21991ce91824SDavid Xu 	}
22001ce91824SDavid Xu 	lj->lioj_count--;
22011ce91824SDavid Xu 	if (lj->lioj_count == 0) {
22021ce91824SDavid Xu 		TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
22031ce91824SDavid Xu 		knlist_delete(&lj->klist, curthread, 1);
2204759ccccaSDavid Xu 		PROC_LOCK(p);
22051ce91824SDavid Xu 		sigqueue_take(&lj->lioj_ksi);
22061ce91824SDavid Xu 		PROC_UNLOCK(p);
2207759ccccaSDavid Xu 		AIO_UNLOCK(ki);
22081ce91824SDavid Xu 		uma_zfree(aiolio_zone, lj);
22091ce91824SDavid Xu 	} else
2210759ccccaSDavid Xu 		AIO_UNLOCK(ki);
22112244ea07SJohn Dyson 
22121ce91824SDavid Xu 	if (nerror)
22131ce91824SDavid Xu 		return (EIO);
22141ce91824SDavid Xu 	return (error);
2215ee877a35SJohn Dyson }
2216fd3bf775SJohn Dyson 
22173858a1f4SJohn Baldwin /* syscall - list directed I/O (REALTIME) */
22183858a1f4SJohn Baldwin int
22193858a1f4SJohn Baldwin olio_listio(struct thread *td, struct olio_listio_args *uap)
22203858a1f4SJohn Baldwin {
22213858a1f4SJohn Baldwin 	struct aiocb **acb_list;
22223858a1f4SJohn Baldwin 	struct sigevent *sigp, sig;
22233858a1f4SJohn Baldwin 	struct osigevent osig;
22243858a1f4SJohn Baldwin 	int error, nent;
22253858a1f4SJohn Baldwin 
22263858a1f4SJohn Baldwin 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
22273858a1f4SJohn Baldwin 		return (EINVAL);
22283858a1f4SJohn Baldwin 
22293858a1f4SJohn Baldwin 	nent = uap->nent;
22303858a1f4SJohn Baldwin 	if (nent < 0 || nent > AIO_LISTIO_MAX)
22313858a1f4SJohn Baldwin 		return (EINVAL);
22323858a1f4SJohn Baldwin 
22333858a1f4SJohn Baldwin 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
22343858a1f4SJohn Baldwin 		error = copyin(uap->sig, &osig, sizeof(osig));
22353858a1f4SJohn Baldwin 		if (error)
22363858a1f4SJohn Baldwin 			return (error);
22373858a1f4SJohn Baldwin 		error = convert_old_sigevent(&osig, &sig);
22383858a1f4SJohn Baldwin 		if (error)
22393858a1f4SJohn Baldwin 			return (error);
22403858a1f4SJohn Baldwin 		sigp = &sig;
22413858a1f4SJohn Baldwin 	} else
22423858a1f4SJohn Baldwin 		sigp = NULL;
22433858a1f4SJohn Baldwin 
22443858a1f4SJohn Baldwin 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
22453858a1f4SJohn Baldwin 	error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
22463858a1f4SJohn Baldwin 	if (error == 0)
22473858a1f4SJohn Baldwin 		error = kern_lio_listio(td, uap->mode,
22483858a1f4SJohn Baldwin 		    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
22493858a1f4SJohn Baldwin 		    &aiocb_ops_osigevent);
22503858a1f4SJohn Baldwin 	free(acb_list, M_LIO);
22513858a1f4SJohn Baldwin 	return (error);
22523858a1f4SJohn Baldwin }
22533858a1f4SJohn Baldwin 
22543858a1f4SJohn Baldwin /* syscall - list directed I/O (REALTIME) */
22553858a1f4SJohn Baldwin int
22563858a1f4SJohn Baldwin lio_listio(struct thread *td, struct lio_listio_args *uap)
22573858a1f4SJohn Baldwin {
22583858a1f4SJohn Baldwin 	struct aiocb **acb_list;
22593858a1f4SJohn Baldwin 	struct sigevent *sigp, sig;
22603858a1f4SJohn Baldwin 	int error, nent;
22613858a1f4SJohn Baldwin 
22623858a1f4SJohn Baldwin 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
22633858a1f4SJohn Baldwin 		return (EINVAL);
22643858a1f4SJohn Baldwin 
22653858a1f4SJohn Baldwin 	nent = uap->nent;
22663858a1f4SJohn Baldwin 	if (nent < 0 || nent > AIO_LISTIO_MAX)
22673858a1f4SJohn Baldwin 		return (EINVAL);
22683858a1f4SJohn Baldwin 
22693858a1f4SJohn Baldwin 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
22703858a1f4SJohn Baldwin 		error = copyin(uap->sig, &sig, sizeof(sig));
22713858a1f4SJohn Baldwin 		if (error)
22723858a1f4SJohn Baldwin 			return (error);
22733858a1f4SJohn Baldwin 		sigp = &sig;
22743858a1f4SJohn Baldwin 	} else
22753858a1f4SJohn Baldwin 		sigp = NULL;
22763858a1f4SJohn Baldwin 
22773858a1f4SJohn Baldwin 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
22783858a1f4SJohn Baldwin 	error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
22793858a1f4SJohn Baldwin 	if (error == 0)
22803858a1f4SJohn Baldwin 		error = kern_lio_listio(td, uap->mode, uap->acb_list, acb_list,
22813858a1f4SJohn Baldwin 		    nent, sigp, &aiocb_ops);
22823858a1f4SJohn Baldwin 	free(acb_list, M_LIO);
22833858a1f4SJohn Baldwin 	return (error);
22843858a1f4SJohn Baldwin }
22853858a1f4SJohn Baldwin 
228684af4da6SJohn Dyson /*
22871ce91824SDavid Xu  * Called from interrupt thread for physio, we should return as fast
22881ce91824SDavid Xu  * as possible, so we schedule a biohelper task.
228984af4da6SJohn Dyson  */
2290fd3bf775SJohn Dyson static void
2291bfbbc4aaSJason Evans aio_physwakeup(struct buf *bp)
2292fd3bf775SJohn Dyson {
229384af4da6SJohn Dyson 	struct aiocblist *aiocbe;
2294fd3bf775SJohn Dyson 
22958edbaf85SHidetoshi Shimokawa 	aiocbe = (struct aiocblist *)bp->b_caller1;
22961ce91824SDavid Xu 	taskqueue_enqueue(taskqueue_aiod_bio, &aiocbe->biotask);
22971ce91824SDavid Xu }
229884af4da6SJohn Dyson 
22991ce91824SDavid Xu /*
23001ce91824SDavid Xu  * Task routine to perform heavy tasks, process wakeup, and signals.
23011ce91824SDavid Xu  */
23021ce91824SDavid Xu static void
23031ce91824SDavid Xu biohelper(void *context, int pending)
23041ce91824SDavid Xu {
23051ce91824SDavid Xu 	struct aiocblist *aiocbe = context;
23061ce91824SDavid Xu 	struct buf *bp;
23071ce91824SDavid Xu 	struct proc *userp;
230827b8220dSDavid Xu 	struct kaioinfo *ki;
23091ce91824SDavid Xu 	int nblks;
23101ce91824SDavid Xu 
23111ce91824SDavid Xu 	bp = aiocbe->bp;
23121ce91824SDavid Xu 	userp = aiocbe->userproc;
231327b8220dSDavid Xu 	ki = userp->p_aioinfo;
2314759ccccaSDavid Xu 	AIO_LOCK(ki);
231584af4da6SJohn Dyson 	aiocbe->uaiocb._aiocb_private.status -= bp->b_resid;
231684af4da6SJohn Dyson 	aiocbe->uaiocb._aiocb_private.error = 0;
2317c244d2deSPoul-Henning Kamp 	if (bp->b_ioflags & BIO_ERROR)
231884af4da6SJohn Dyson 		aiocbe->uaiocb._aiocb_private.error = bp->b_error;
23191ce91824SDavid Xu 	nblks = btodb(aiocbe->uaiocb.aio_nbytes);
23201ce91824SDavid Xu 	if (aiocbe->uaiocb.aio_lio_opcode == LIO_WRITE)
23211ce91824SDavid Xu 		aiocbe->outputcharge += nblks;
23221ce91824SDavid Xu 	else
23231ce91824SDavid Xu 		aiocbe->inputcharge += nblks;
23241ce91824SDavid Xu 	aiocbe->bp = NULL;
23251ce91824SDavid Xu 	TAILQ_REMOVE(&userp->p_aioinfo->kaio_bufqueue, aiocbe, plist);
232627b8220dSDavid Xu 	ki->kaio_buffer_count--;
232769cd28daSDoug Ambrisko 	aio_bio_done_notify(userp, aiocbe, DONE_BUF);
2328759ccccaSDavid Xu 	AIO_UNLOCK(ki);
23291ce91824SDavid Xu 
23301ce91824SDavid Xu 	/* Release mapping into kernel space. */
23311ce91824SDavid Xu 	vunmapbuf(bp);
23321ce91824SDavid Xu 	relpbuf(bp, NULL);
23331ce91824SDavid Xu 	atomic_subtract_int(&num_buf_aio, 1);
233484af4da6SJohn Dyson }
2335bfbbc4aaSJason Evans 
2336eb8e6d52SEivind Eklund /* syscall - wait for the next completion of an aio request */
23373858a1f4SJohn Baldwin static int
23383858a1f4SJohn Baldwin kern_aio_waitcomplete(struct thread *td, struct aiocb **aiocbp,
23393858a1f4SJohn Baldwin     struct timespec *ts, struct aiocb_ops *ops)
2340bfbbc4aaSJason Evans {
2341b40ce416SJulian Elischer 	struct proc *p = td->td_proc;
2342bfbbc4aaSJason Evans 	struct timeval atv;
2343bfbbc4aaSJason Evans 	struct kaioinfo *ki;
23441ce91824SDavid Xu 	struct aiocblist *cb;
23451ce91824SDavid Xu 	struct aiocb *uuaiocb;
23461ce91824SDavid Xu 	int error, status, timo;
2347bfbbc4aaSJason Evans 
23483858a1f4SJohn Baldwin 	ops->store_aiocb(aiocbp, NULL);
2349dd85920aSJason Evans 
2350bfbbc4aaSJason Evans 	timo = 0;
23513858a1f4SJohn Baldwin 	if (ts) {
23523858a1f4SJohn Baldwin 		if ((ts->tv_nsec < 0) || (ts->tv_nsec >= 1000000000))
2353bfbbc4aaSJason Evans 			return (EINVAL);
2354bfbbc4aaSJason Evans 
23553858a1f4SJohn Baldwin 		TIMESPEC_TO_TIMEVAL(&atv, ts);
2356bfbbc4aaSJason Evans 		if (itimerfix(&atv))
2357bfbbc4aaSJason Evans 			return (EINVAL);
2358bfbbc4aaSJason Evans 		timo = tvtohz(&atv);
2359bfbbc4aaSJason Evans 	}
2360bfbbc4aaSJason Evans 
23618213baf0SChristian S.J. Peron 	if (p->p_aioinfo == NULL)
2362323fe565SDavid Xu 		aio_init_aioinfo(p);
23638213baf0SChristian S.J. Peron 	ki = p->p_aioinfo;
2364bfbbc4aaSJason Evans 
23651ce91824SDavid Xu 	error = 0;
23661ce91824SDavid Xu 	cb = NULL;
2367759ccccaSDavid Xu 	AIO_LOCK(ki);
23681ce91824SDavid Xu 	while ((cb = TAILQ_FIRST(&ki->kaio_done)) == NULL) {
23691ce91824SDavid Xu 		ki->kaio_flags |= KAIO_WAKEUP;
2370759ccccaSDavid Xu 		error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH,
23711ce91824SDavid Xu 		    "aiowc", timo);
237227b8220dSDavid Xu 		if (timo && error == ERESTART)
23731ce91824SDavid Xu 			error = EINTR;
23741ce91824SDavid Xu 		if (error)
23751ce91824SDavid Xu 			break;
23761ce91824SDavid Xu 	}
23771ce91824SDavid Xu 
23781ce91824SDavid Xu 	if (cb != NULL) {
23791ce91824SDavid Xu 		MPASS(cb->jobstate == JOBST_JOBFINISHED);
23801ce91824SDavid Xu 		uuaiocb = cb->uuaiocb;
23811ce91824SDavid Xu 		status = cb->uaiocb._aiocb_private.status;
23821ce91824SDavid Xu 		error = cb->uaiocb._aiocb_private.error;
23831ce91824SDavid Xu 		td->td_retval[0] = status;
2384bfbbc4aaSJason Evans 		if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) {
23851c4bcd05SJeff Roberson 			td->td_ru.ru_oublock += cb->outputcharge;
2386bfbbc4aaSJason Evans 			cb->outputcharge = 0;
2387bfbbc4aaSJason Evans 		} else if (cb->uaiocb.aio_lio_opcode == LIO_READ) {
23881c4bcd05SJeff Roberson 			td->td_ru.ru_inblock += cb->inputcharge;
2389bfbbc4aaSJason Evans 			cb->inputcharge = 0;
2390bfbbc4aaSJason Evans 		}
2391bfbbc4aaSJason Evans 		aio_free_entry(cb);
2392759ccccaSDavid Xu 		AIO_UNLOCK(ki);
23933858a1f4SJohn Baldwin 		ops->store_aiocb(aiocbp, uuaiocb);
23943858a1f4SJohn Baldwin 		ops->store_error(uuaiocb, error);
23953858a1f4SJohn Baldwin 		ops->store_status(uuaiocb, status);
23961ce91824SDavid Xu 	} else
2397759ccccaSDavid Xu 		AIO_UNLOCK(ki);
2398bfbbc4aaSJason Evans 
2399ac41f2efSAlfred Perlstein 	return (error);
2400bfbbc4aaSJason Evans }
2401cb679c38SJonathan Lemon 
240299eee864SDavid Xu int
24033858a1f4SJohn Baldwin aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
24043858a1f4SJohn Baldwin {
24053858a1f4SJohn Baldwin 	struct timespec ts, *tsp;
24063858a1f4SJohn Baldwin 	int error;
24073858a1f4SJohn Baldwin 
24083858a1f4SJohn Baldwin 	if (uap->timeout) {
24093858a1f4SJohn Baldwin 		/* Get timespec struct. */
24103858a1f4SJohn Baldwin 		error = copyin(uap->timeout, &ts, sizeof(ts));
24113858a1f4SJohn Baldwin 		if (error)
24123858a1f4SJohn Baldwin 			return (error);
24133858a1f4SJohn Baldwin 		tsp = &ts;
24143858a1f4SJohn Baldwin 	} else
24153858a1f4SJohn Baldwin 		tsp = NULL;
24163858a1f4SJohn Baldwin 
24173858a1f4SJohn Baldwin 	return (kern_aio_waitcomplete(td, uap->aiocbp, tsp, &aiocb_ops));
24183858a1f4SJohn Baldwin }
24193858a1f4SJohn Baldwin 
24203858a1f4SJohn Baldwin static int
24213858a1f4SJohn Baldwin kern_aio_fsync(struct thread *td, int op, struct aiocb *aiocbp,
24223858a1f4SJohn Baldwin     struct aiocb_ops *ops)
242399eee864SDavid Xu {
242499eee864SDavid Xu 	struct proc *p = td->td_proc;
242599eee864SDavid Xu 	struct kaioinfo *ki;
242699eee864SDavid Xu 
24273858a1f4SJohn Baldwin 	if (op != O_SYNC) /* XXX lack of O_DSYNC */
242899eee864SDavid Xu 		return (EINVAL);
242999eee864SDavid Xu 	ki = p->p_aioinfo;
243099eee864SDavid Xu 	if (ki == NULL)
243199eee864SDavid Xu 		aio_init_aioinfo(p);
24323858a1f4SJohn Baldwin 	return (aio_aqueue(td, aiocbp, NULL, LIO_SYNC, ops));
24333858a1f4SJohn Baldwin }
24343858a1f4SJohn Baldwin 
24353858a1f4SJohn Baldwin int
24363858a1f4SJohn Baldwin aio_fsync(struct thread *td, struct aio_fsync_args *uap)
24373858a1f4SJohn Baldwin {
24383858a1f4SJohn Baldwin 
24393858a1f4SJohn Baldwin 	return (kern_aio_fsync(td, uap->op, uap->aiocbp, &aiocb_ops));
244099eee864SDavid Xu }
244199eee864SDavid Xu 
2442eb8e6d52SEivind Eklund /* kqueue attach function */
2443cb679c38SJonathan Lemon static int
2444cb679c38SJonathan Lemon filt_aioattach(struct knote *kn)
2445cb679c38SJonathan Lemon {
2446b46f1c55SAlan Cox 	struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata;
2447cb679c38SJonathan Lemon 
2448cb679c38SJonathan Lemon 	/*
2449cb679c38SJonathan Lemon 	 * The aiocbe pointer must be validated before using it, so
2450cb679c38SJonathan Lemon 	 * registration is restricted to the kernel; the user cannot
2451cb679c38SJonathan Lemon 	 * set EV_FLAG1.
2452cb679c38SJonathan Lemon 	 */
2453cb679c38SJonathan Lemon 	if ((kn->kn_flags & EV_FLAG1) == 0)
2454cb679c38SJonathan Lemon 		return (EPERM);
2455a8afa221SJean-Sébastien Pédron 	kn->kn_ptr.p_aio = aiocbe;
2456cb679c38SJonathan Lemon 	kn->kn_flags &= ~EV_FLAG1;
2457cb679c38SJonathan Lemon 
2458ad3b9257SJohn-Mark Gurney 	knlist_add(&aiocbe->klist, kn, 0);
2459cb679c38SJonathan Lemon 
2460cb679c38SJonathan Lemon 	return (0);
2461cb679c38SJonathan Lemon }
2462cb679c38SJonathan Lemon 
2463eb8e6d52SEivind Eklund /* kqueue detach function */
2464cb679c38SJonathan Lemon static void
2465cb679c38SJonathan Lemon filt_aiodetach(struct knote *kn)
2466cb679c38SJonathan Lemon {
2467a8afa221SJean-Sébastien Pédron 	struct aiocblist *aiocbe = kn->kn_ptr.p_aio;
2468cb679c38SJonathan Lemon 
246969cd28daSDoug Ambrisko 	if (!knlist_empty(&aiocbe->klist))
2470ad3b9257SJohn-Mark Gurney 		knlist_remove(&aiocbe->klist, kn, 0);
2471cb679c38SJonathan Lemon }
2472cb679c38SJonathan Lemon 
2473eb8e6d52SEivind Eklund /* kqueue filter function */
2474cb679c38SJonathan Lemon /*ARGSUSED*/
2475cb679c38SJonathan Lemon static int
2476cb679c38SJonathan Lemon filt_aio(struct knote *kn, long hint)
2477cb679c38SJonathan Lemon {
2478a8afa221SJean-Sébastien Pédron 	struct aiocblist *aiocbe = kn->kn_ptr.p_aio;
2479cb679c38SJonathan Lemon 
248091369fc7SAlan Cox 	kn->kn_data = aiocbe->uaiocb._aiocb_private.error;
24811ce91824SDavid Xu 	if (aiocbe->jobstate != JOBST_JOBFINISHED)
2482cb679c38SJonathan Lemon 		return (0);
2483cb679c38SJonathan Lemon 	kn->kn_flags |= EV_EOF;
2484cb679c38SJonathan Lemon 	return (1);
2485cb679c38SJonathan Lemon }
248669cd28daSDoug Ambrisko 
248769cd28daSDoug Ambrisko /* kqueue attach function */
248869cd28daSDoug Ambrisko static int
248969cd28daSDoug Ambrisko filt_lioattach(struct knote *kn)
249069cd28daSDoug Ambrisko {
24911ce91824SDavid Xu 	struct aioliojob * lj = (struct aioliojob *)kn->kn_sdata;
249269cd28daSDoug Ambrisko 
249369cd28daSDoug Ambrisko 	/*
24941ce91824SDavid Xu 	 * The aioliojob pointer must be validated before using it, so
249569cd28daSDoug Ambrisko 	 * registration is restricted to the kernel; the user cannot
249669cd28daSDoug Ambrisko 	 * set EV_FLAG1.
249769cd28daSDoug Ambrisko 	 */
249869cd28daSDoug Ambrisko 	if ((kn->kn_flags & EV_FLAG1) == 0)
249969cd28daSDoug Ambrisko 		return (EPERM);
2500a8afa221SJean-Sébastien Pédron 	kn->kn_ptr.p_lio = lj;
250169cd28daSDoug Ambrisko 	kn->kn_flags &= ~EV_FLAG1;
250269cd28daSDoug Ambrisko 
250369cd28daSDoug Ambrisko 	knlist_add(&lj->klist, kn, 0);
250469cd28daSDoug Ambrisko 
250569cd28daSDoug Ambrisko 	return (0);
250669cd28daSDoug Ambrisko }
250769cd28daSDoug Ambrisko 
250869cd28daSDoug Ambrisko /* kqueue detach function */
250969cd28daSDoug Ambrisko static void
251069cd28daSDoug Ambrisko filt_liodetach(struct knote *kn)
251169cd28daSDoug Ambrisko {
2512a8afa221SJean-Sébastien Pédron 	struct aioliojob * lj = kn->kn_ptr.p_lio;
251369cd28daSDoug Ambrisko 
251469cd28daSDoug Ambrisko 	if (!knlist_empty(&lj->klist))
251569cd28daSDoug Ambrisko 		knlist_remove(&lj->klist, kn, 0);
251669cd28daSDoug Ambrisko }
251769cd28daSDoug Ambrisko 
251869cd28daSDoug Ambrisko /* kqueue filter function */
251969cd28daSDoug Ambrisko /*ARGSUSED*/
252069cd28daSDoug Ambrisko static int
252169cd28daSDoug Ambrisko filt_lio(struct knote *kn, long hint)
252269cd28daSDoug Ambrisko {
2523a8afa221SJean-Sébastien Pédron 	struct aioliojob * lj = kn->kn_ptr.p_lio;
25241ce91824SDavid Xu 
252569cd28daSDoug Ambrisko 	return (lj->lioj_flags & LIOJ_KEVENT_POSTED);
252669cd28daSDoug Ambrisko }
25273858a1f4SJohn Baldwin 
25283858a1f4SJohn Baldwin #ifdef COMPAT_IA32
25293858a1f4SJohn Baldwin #include <sys/mount.h>
25303858a1f4SJohn Baldwin #include <sys/socket.h>
25313858a1f4SJohn Baldwin #include <compat/freebsd32/freebsd32.h>
25323858a1f4SJohn Baldwin #include <compat/freebsd32/freebsd32_proto.h>
25333858a1f4SJohn Baldwin #include <compat/freebsd32/freebsd32_signal.h>
25343858a1f4SJohn Baldwin #include <compat/freebsd32/freebsd32_syscall.h>
25353858a1f4SJohn Baldwin #include <compat/freebsd32/freebsd32_util.h>
25363858a1f4SJohn Baldwin 
25373858a1f4SJohn Baldwin struct __aiocb_private32 {
25383858a1f4SJohn Baldwin 	int32_t	status;
25393858a1f4SJohn Baldwin 	int32_t	error;
25403858a1f4SJohn Baldwin 	uint32_t kernelinfo;
25413858a1f4SJohn Baldwin };
25423858a1f4SJohn Baldwin 
25433858a1f4SJohn Baldwin typedef struct oaiocb32 {
25443858a1f4SJohn Baldwin 	int	aio_fildes;		/* File descriptor */
25453858a1f4SJohn Baldwin 	uint64_t aio_offset __packed;	/* File offset for I/O */
25463858a1f4SJohn Baldwin 	uint32_t aio_buf;		/* I/O buffer in process space */
25473858a1f4SJohn Baldwin 	uint32_t aio_nbytes;		/* Number of bytes for I/O */
25483858a1f4SJohn Baldwin 	struct	osigevent32 aio_sigevent; /* Signal to deliver */
25493858a1f4SJohn Baldwin 	int	aio_lio_opcode;		/* LIO opcode */
25503858a1f4SJohn Baldwin 	int	aio_reqprio;		/* Request priority -- ignored */
25513858a1f4SJohn Baldwin 	struct	__aiocb_private32 _aiocb_private;
25523858a1f4SJohn Baldwin } oaiocb32_t;
25533858a1f4SJohn Baldwin 
25543858a1f4SJohn Baldwin typedef struct aiocb32 {
25553858a1f4SJohn Baldwin 	int32_t	aio_fildes;		/* File descriptor */
25563858a1f4SJohn Baldwin 	uint64_t aio_offset __packed;	/* File offset for I/O */
25573858a1f4SJohn Baldwin 	uint32_t aio_buf;		/* I/O buffer in process space */
25583858a1f4SJohn Baldwin 	uint32_t aio_nbytes;		/* Number of bytes for I/O */
25593858a1f4SJohn Baldwin 	int	__spare__[2];
25603858a1f4SJohn Baldwin 	uint32_t __spare2__;
25613858a1f4SJohn Baldwin 	int	aio_lio_opcode;		/* LIO opcode */
25623858a1f4SJohn Baldwin 	int	aio_reqprio;		/* Request priority -- ignored */
25633858a1f4SJohn Baldwin 	struct __aiocb_private32 _aiocb_private;
25643858a1f4SJohn Baldwin 	struct sigevent32 aio_sigevent;	/* Signal to deliver */
25653858a1f4SJohn Baldwin } aiocb32_t;
25663858a1f4SJohn Baldwin 
25673858a1f4SJohn Baldwin static int
25683858a1f4SJohn Baldwin convert_old_sigevent32(struct osigevent32 *osig, struct sigevent *nsig)
25693858a1f4SJohn Baldwin {
25703858a1f4SJohn Baldwin 
25713858a1f4SJohn Baldwin 	/*
25723858a1f4SJohn Baldwin 	 * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
25733858a1f4SJohn Baldwin 	 * supported by AIO with the old sigevent structure.
25743858a1f4SJohn Baldwin 	 */
25753858a1f4SJohn Baldwin 	CP(*osig, *nsig, sigev_notify);
25763858a1f4SJohn Baldwin 	switch (nsig->sigev_notify) {
25773858a1f4SJohn Baldwin 	case SIGEV_NONE:
25783858a1f4SJohn Baldwin 		break;
25793858a1f4SJohn Baldwin 	case SIGEV_SIGNAL:
25803858a1f4SJohn Baldwin 		nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
25813858a1f4SJohn Baldwin 		break;
25823858a1f4SJohn Baldwin 	case SIGEV_KEVENT:
25833858a1f4SJohn Baldwin 		nsig->sigev_notify_kqueue =
25843858a1f4SJohn Baldwin 		    osig->__sigev_u.__sigev_notify_kqueue;
25853858a1f4SJohn Baldwin 		PTRIN_CP(*osig, *nsig, sigev_value.sival_ptr);
25863858a1f4SJohn Baldwin 		break;
25873858a1f4SJohn Baldwin 	default:
25883858a1f4SJohn Baldwin 		return (EINVAL);
25893858a1f4SJohn Baldwin 	}
25903858a1f4SJohn Baldwin 	return (0);
25913858a1f4SJohn Baldwin }
25923858a1f4SJohn Baldwin 
25933858a1f4SJohn Baldwin static int
25943858a1f4SJohn Baldwin aiocb32_copyin_old_sigevent(struct aiocb *ujob, struct aiocb *kjob)
25953858a1f4SJohn Baldwin {
25963858a1f4SJohn Baldwin 	struct oaiocb32 job32;
25973858a1f4SJohn Baldwin 	int error;
25983858a1f4SJohn Baldwin 
25993858a1f4SJohn Baldwin 	bzero(kjob, sizeof(struct aiocb));
26003858a1f4SJohn Baldwin 	error = copyin(ujob, &job32, sizeof(job32));
26013858a1f4SJohn Baldwin 	if (error)
26023858a1f4SJohn Baldwin 		return (error);
26033858a1f4SJohn Baldwin 
26043858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_fildes);
26053858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_offset);
26063858a1f4SJohn Baldwin 	PTRIN_CP(job32, *kjob, aio_buf);
26073858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_nbytes);
26083858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_lio_opcode);
26093858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_reqprio);
26103858a1f4SJohn Baldwin 	CP(job32, *kjob, _aiocb_private.status);
26113858a1f4SJohn Baldwin 	CP(job32, *kjob, _aiocb_private.error);
26123858a1f4SJohn Baldwin 	PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo);
26133858a1f4SJohn Baldwin 	return (convert_old_sigevent32(&job32.aio_sigevent,
26143858a1f4SJohn Baldwin 	    &kjob->aio_sigevent));
26153858a1f4SJohn Baldwin }
26163858a1f4SJohn Baldwin 
26173858a1f4SJohn Baldwin static int
26183858a1f4SJohn Baldwin convert_sigevent32(struct sigevent32 *sig32, struct sigevent *sig)
26193858a1f4SJohn Baldwin {
26203858a1f4SJohn Baldwin 
26213858a1f4SJohn Baldwin 	CP(*sig32, *sig, sigev_notify);
26223858a1f4SJohn Baldwin 	switch (sig->sigev_notify) {
26233858a1f4SJohn Baldwin 	case SIGEV_NONE:
26243858a1f4SJohn Baldwin 		break;
26253858a1f4SJohn Baldwin 	case SIGEV_THREAD_ID:
26263858a1f4SJohn Baldwin 		CP(*sig32, *sig, sigev_notify_thread_id);
26273858a1f4SJohn Baldwin 		/* FALLTHROUGH */
26283858a1f4SJohn Baldwin 	case SIGEV_SIGNAL:
26293858a1f4SJohn Baldwin 		CP(*sig32, *sig, sigev_signo);
26303858a1f4SJohn Baldwin 		break;
26313858a1f4SJohn Baldwin 	case SIGEV_KEVENT:
26323858a1f4SJohn Baldwin 		CP(*sig32, *sig, sigev_notify_kqueue);
26333858a1f4SJohn Baldwin 		PTRIN_CP(*sig32, *sig, sigev_value.sival_ptr);
26343858a1f4SJohn Baldwin 		break;
26353858a1f4SJohn Baldwin 	default:
26363858a1f4SJohn Baldwin 		return (EINVAL);
26373858a1f4SJohn Baldwin 	}
26383858a1f4SJohn Baldwin 	return (0);
26393858a1f4SJohn Baldwin }
26403858a1f4SJohn Baldwin 
26413858a1f4SJohn Baldwin static int
26423858a1f4SJohn Baldwin aiocb32_copyin(struct aiocb *ujob, struct aiocb *kjob)
26433858a1f4SJohn Baldwin {
26443858a1f4SJohn Baldwin 	struct aiocb32 job32;
26453858a1f4SJohn Baldwin 	int error;
26463858a1f4SJohn Baldwin 
26473858a1f4SJohn Baldwin 	error = copyin(ujob, &job32, sizeof(job32));
26483858a1f4SJohn Baldwin 	if (error)
26493858a1f4SJohn Baldwin 		return (error);
26503858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_fildes);
26513858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_offset);
26523858a1f4SJohn Baldwin 	PTRIN_CP(job32, *kjob, aio_buf);
26533858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_nbytes);
26543858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_lio_opcode);
26553858a1f4SJohn Baldwin 	CP(job32, *kjob, aio_reqprio);
26563858a1f4SJohn Baldwin 	CP(job32, *kjob, _aiocb_private.status);
26573858a1f4SJohn Baldwin 	CP(job32, *kjob, _aiocb_private.error);
26583858a1f4SJohn Baldwin 	PTRIN_CP(job32, *kjob, _aiocb_private.kernelinfo);
26593858a1f4SJohn Baldwin 	return (convert_sigevent32(&job32.aio_sigevent, &kjob->aio_sigevent));
26603858a1f4SJohn Baldwin }
26613858a1f4SJohn Baldwin 
26623858a1f4SJohn Baldwin static long
26633858a1f4SJohn Baldwin aiocb32_fetch_status(struct aiocb *ujob)
26643858a1f4SJohn Baldwin {
26653858a1f4SJohn Baldwin 	struct aiocb32 *ujob32;
26663858a1f4SJohn Baldwin 
26673858a1f4SJohn Baldwin 	ujob32 = (struct aiocb32 *)ujob;
26683858a1f4SJohn Baldwin 	return (fuword32(&ujob32->_aiocb_private.status));
26693858a1f4SJohn Baldwin }
26703858a1f4SJohn Baldwin 
26713858a1f4SJohn Baldwin static long
26723858a1f4SJohn Baldwin aiocb32_fetch_error(struct aiocb *ujob)
26733858a1f4SJohn Baldwin {
26743858a1f4SJohn Baldwin 	struct aiocb32 *ujob32;
26753858a1f4SJohn Baldwin 
26763858a1f4SJohn Baldwin 	ujob32 = (struct aiocb32 *)ujob;
26773858a1f4SJohn Baldwin 	return (fuword32(&ujob32->_aiocb_private.error));
26783858a1f4SJohn Baldwin }
26793858a1f4SJohn Baldwin 
26803858a1f4SJohn Baldwin static int
26813858a1f4SJohn Baldwin aiocb32_store_status(struct aiocb *ujob, long status)
26823858a1f4SJohn Baldwin {
26833858a1f4SJohn Baldwin 	struct aiocb32 *ujob32;
26843858a1f4SJohn Baldwin 
26853858a1f4SJohn Baldwin 	ujob32 = (struct aiocb32 *)ujob;
26863858a1f4SJohn Baldwin 	return (suword32(&ujob32->_aiocb_private.status, status));
26873858a1f4SJohn Baldwin }
26883858a1f4SJohn Baldwin 
26893858a1f4SJohn Baldwin static int
26903858a1f4SJohn Baldwin aiocb32_store_error(struct aiocb *ujob, long error)
26913858a1f4SJohn Baldwin {
26923858a1f4SJohn Baldwin 	struct aiocb32 *ujob32;
26933858a1f4SJohn Baldwin 
26943858a1f4SJohn Baldwin 	ujob32 = (struct aiocb32 *)ujob;
26953858a1f4SJohn Baldwin 	return (suword32(&ujob32->_aiocb_private.error, error));
26963858a1f4SJohn Baldwin }
26973858a1f4SJohn Baldwin 
26983858a1f4SJohn Baldwin static int
26993858a1f4SJohn Baldwin aiocb32_store_kernelinfo(struct aiocb *ujob, long jobref)
27003858a1f4SJohn Baldwin {
27013858a1f4SJohn Baldwin 	struct aiocb32 *ujob32;
27023858a1f4SJohn Baldwin 
27033858a1f4SJohn Baldwin 	ujob32 = (struct aiocb32 *)ujob;
27043858a1f4SJohn Baldwin 	return (suword32(&ujob32->_aiocb_private.kernelinfo, jobref));
27053858a1f4SJohn Baldwin }
27063858a1f4SJohn Baldwin 
27073858a1f4SJohn Baldwin static int
27083858a1f4SJohn Baldwin aiocb32_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
27093858a1f4SJohn Baldwin {
27103858a1f4SJohn Baldwin 
27113858a1f4SJohn Baldwin 	return (suword32(ujobp, (long)ujob));
27123858a1f4SJohn Baldwin }
27133858a1f4SJohn Baldwin 
27143858a1f4SJohn Baldwin static struct aiocb_ops aiocb32_ops = {
27153858a1f4SJohn Baldwin 	.copyin = aiocb32_copyin,
27163858a1f4SJohn Baldwin 	.fetch_status = aiocb32_fetch_status,
27173858a1f4SJohn Baldwin 	.fetch_error = aiocb32_fetch_error,
27183858a1f4SJohn Baldwin 	.store_status = aiocb32_store_status,
27193858a1f4SJohn Baldwin 	.store_error = aiocb32_store_error,
27203858a1f4SJohn Baldwin 	.store_kernelinfo = aiocb32_store_kernelinfo,
27213858a1f4SJohn Baldwin 	.store_aiocb = aiocb32_store_aiocb,
27223858a1f4SJohn Baldwin };
27233858a1f4SJohn Baldwin 
27243858a1f4SJohn Baldwin static struct aiocb_ops aiocb32_ops_osigevent = {
27253858a1f4SJohn Baldwin 	.copyin = aiocb32_copyin_old_sigevent,
27263858a1f4SJohn Baldwin 	.fetch_status = aiocb32_fetch_status,
27273858a1f4SJohn Baldwin 	.fetch_error = aiocb32_fetch_error,
27283858a1f4SJohn Baldwin 	.store_status = aiocb32_store_status,
27293858a1f4SJohn Baldwin 	.store_error = aiocb32_store_error,
27303858a1f4SJohn Baldwin 	.store_kernelinfo = aiocb32_store_kernelinfo,
27313858a1f4SJohn Baldwin 	.store_aiocb = aiocb32_store_aiocb,
27323858a1f4SJohn Baldwin };
27333858a1f4SJohn Baldwin 
27343858a1f4SJohn Baldwin int
27353858a1f4SJohn Baldwin freebsd32_aio_return(struct thread *td, struct freebsd32_aio_return_args *uap)
27363858a1f4SJohn Baldwin {
27373858a1f4SJohn Baldwin 
27383858a1f4SJohn Baldwin 	return (kern_aio_return(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
27393858a1f4SJohn Baldwin }
27403858a1f4SJohn Baldwin 
27413858a1f4SJohn Baldwin int
27423858a1f4SJohn Baldwin freebsd32_aio_suspend(struct thread *td, struct freebsd32_aio_suspend_args *uap)
27433858a1f4SJohn Baldwin {
27443858a1f4SJohn Baldwin 	struct timespec32 ts32;
27453858a1f4SJohn Baldwin 	struct timespec ts, *tsp;
27463858a1f4SJohn Baldwin 	struct aiocb **ujoblist;
27473858a1f4SJohn Baldwin 	uint32_t *ujoblist32;
27483858a1f4SJohn Baldwin 	int error, i;
27493858a1f4SJohn Baldwin 
27503858a1f4SJohn Baldwin 	if (uap->nent < 0 || uap->nent > AIO_LISTIO_MAX)
27513858a1f4SJohn Baldwin 		return (EINVAL);
27523858a1f4SJohn Baldwin 
27533858a1f4SJohn Baldwin 	if (uap->timeout) {
27543858a1f4SJohn Baldwin 		/* Get timespec struct. */
27553858a1f4SJohn Baldwin 		if ((error = copyin(uap->timeout, &ts32, sizeof(ts32))) != 0)
27563858a1f4SJohn Baldwin 			return (error);
27573858a1f4SJohn Baldwin 		CP(ts32, ts, tv_sec);
27583858a1f4SJohn Baldwin 		CP(ts32, ts, tv_nsec);
27593858a1f4SJohn Baldwin 		tsp = &ts;
27603858a1f4SJohn Baldwin 	} else
27613858a1f4SJohn Baldwin 		tsp = NULL;
27623858a1f4SJohn Baldwin 
27633858a1f4SJohn Baldwin 	ujoblist = uma_zalloc(aiol_zone, M_WAITOK);
27643858a1f4SJohn Baldwin 	ujoblist32 = (uint32_t *)ujoblist;
27653858a1f4SJohn Baldwin 	error = copyin(uap->aiocbp, ujoblist32, uap->nent *
27663858a1f4SJohn Baldwin 	    sizeof(ujoblist32[0]));
27673858a1f4SJohn Baldwin 	if (error == 0) {
27683858a1f4SJohn Baldwin 		for (i = uap->nent; i > 0; i--)
27693858a1f4SJohn Baldwin 			ujoblist[i] = PTRIN(ujoblist32[i]);
27703858a1f4SJohn Baldwin 
27713858a1f4SJohn Baldwin 		error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
27723858a1f4SJohn Baldwin 	}
27733858a1f4SJohn Baldwin 	uma_zfree(aiol_zone, ujoblist);
27743858a1f4SJohn Baldwin 	return (error);
27753858a1f4SJohn Baldwin }
27763858a1f4SJohn Baldwin 
27773858a1f4SJohn Baldwin int
27783858a1f4SJohn Baldwin freebsd32_aio_cancel(struct thread *td, struct freebsd32_aio_cancel_args *uap)
27793858a1f4SJohn Baldwin {
27803858a1f4SJohn Baldwin 
27813858a1f4SJohn Baldwin 	return (aio_cancel(td, (struct aio_cancel_args *)uap));
27823858a1f4SJohn Baldwin }
27833858a1f4SJohn Baldwin 
27843858a1f4SJohn Baldwin int
27853858a1f4SJohn Baldwin freebsd32_aio_error(struct thread *td, struct freebsd32_aio_error_args *uap)
27863858a1f4SJohn Baldwin {
27873858a1f4SJohn Baldwin 
27883858a1f4SJohn Baldwin 	return (kern_aio_error(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
27893858a1f4SJohn Baldwin }
27903858a1f4SJohn Baldwin 
27913858a1f4SJohn Baldwin int
27923858a1f4SJohn Baldwin freebsd32_oaio_read(struct thread *td, struct freebsd32_oaio_read_args *uap)
27933858a1f4SJohn Baldwin {
27943858a1f4SJohn Baldwin 
27953858a1f4SJohn Baldwin 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
27963858a1f4SJohn Baldwin 	    &aiocb32_ops_osigevent));
27973858a1f4SJohn Baldwin }
27983858a1f4SJohn Baldwin 
27993858a1f4SJohn Baldwin int
28003858a1f4SJohn Baldwin freebsd32_aio_read(struct thread *td, struct freebsd32_aio_read_args *uap)
28013858a1f4SJohn Baldwin {
28023858a1f4SJohn Baldwin 
28033858a1f4SJohn Baldwin 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
28043858a1f4SJohn Baldwin 	    &aiocb32_ops));
28053858a1f4SJohn Baldwin }
28063858a1f4SJohn Baldwin 
28073858a1f4SJohn Baldwin int
28083858a1f4SJohn Baldwin freebsd32_oaio_write(struct thread *td, struct freebsd32_oaio_write_args *uap)
28093858a1f4SJohn Baldwin {
28103858a1f4SJohn Baldwin 
28113858a1f4SJohn Baldwin 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
28123858a1f4SJohn Baldwin 	    &aiocb32_ops_osigevent));
28133858a1f4SJohn Baldwin }
28143858a1f4SJohn Baldwin 
28153858a1f4SJohn Baldwin int
28163858a1f4SJohn Baldwin freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap)
28173858a1f4SJohn Baldwin {
28183858a1f4SJohn Baldwin 
28193858a1f4SJohn Baldwin 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
28203858a1f4SJohn Baldwin 	    &aiocb32_ops));
28213858a1f4SJohn Baldwin }
28223858a1f4SJohn Baldwin 
28233858a1f4SJohn Baldwin int
28243858a1f4SJohn Baldwin freebsd32_aio_waitcomplete(struct thread *td,
28253858a1f4SJohn Baldwin     struct freebsd32_aio_waitcomplete_args *uap)
28263858a1f4SJohn Baldwin {
2827e588eeb1SJohn Baldwin 	struct timespec32 ts32;
28283858a1f4SJohn Baldwin 	struct timespec ts, *tsp;
28293858a1f4SJohn Baldwin 	int error;
28303858a1f4SJohn Baldwin 
28313858a1f4SJohn Baldwin 	if (uap->timeout) {
28323858a1f4SJohn Baldwin 		/* Get timespec struct. */
28333858a1f4SJohn Baldwin 		error = copyin(uap->timeout, &ts32, sizeof(ts32));
28343858a1f4SJohn Baldwin 		if (error)
28353858a1f4SJohn Baldwin 			return (error);
28363858a1f4SJohn Baldwin 		CP(ts32, ts, tv_sec);
28373858a1f4SJohn Baldwin 		CP(ts32, ts, tv_nsec);
28383858a1f4SJohn Baldwin 		tsp = &ts;
28393858a1f4SJohn Baldwin 	} else
28403858a1f4SJohn Baldwin 		tsp = NULL;
28413858a1f4SJohn Baldwin 
28423858a1f4SJohn Baldwin 	return (kern_aio_waitcomplete(td, (struct aiocb **)uap->aiocbp, tsp,
28433858a1f4SJohn Baldwin 	    &aiocb32_ops));
28443858a1f4SJohn Baldwin }
28453858a1f4SJohn Baldwin 
28463858a1f4SJohn Baldwin int
28473858a1f4SJohn Baldwin freebsd32_aio_fsync(struct thread *td, struct freebsd32_aio_fsync_args *uap)
28483858a1f4SJohn Baldwin {
28493858a1f4SJohn Baldwin 
28503858a1f4SJohn Baldwin 	return (kern_aio_fsync(td, uap->op, (struct aiocb *)uap->aiocbp,
28513858a1f4SJohn Baldwin 	    &aiocb32_ops));
28523858a1f4SJohn Baldwin }
28533858a1f4SJohn Baldwin 
28543858a1f4SJohn Baldwin int
28553858a1f4SJohn Baldwin freebsd32_olio_listio(struct thread *td, struct freebsd32_olio_listio_args *uap)
28563858a1f4SJohn Baldwin {
28573858a1f4SJohn Baldwin 	struct aiocb **acb_list;
28583858a1f4SJohn Baldwin 	struct sigevent *sigp, sig;
28593858a1f4SJohn Baldwin 	struct osigevent32 osig;
28603858a1f4SJohn Baldwin 	uint32_t *acb_list32;
28613858a1f4SJohn Baldwin 	int error, i, nent;
28623858a1f4SJohn Baldwin 
28633858a1f4SJohn Baldwin 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
28643858a1f4SJohn Baldwin 		return (EINVAL);
28653858a1f4SJohn Baldwin 
28663858a1f4SJohn Baldwin 	nent = uap->nent;
28673858a1f4SJohn Baldwin 	if (nent < 0 || nent > AIO_LISTIO_MAX)
28683858a1f4SJohn Baldwin 		return (EINVAL);
28693858a1f4SJohn Baldwin 
28703858a1f4SJohn Baldwin 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
28713858a1f4SJohn Baldwin 		error = copyin(uap->sig, &osig, sizeof(osig));
28723858a1f4SJohn Baldwin 		if (error)
28733858a1f4SJohn Baldwin 			return (error);
28743858a1f4SJohn Baldwin 		error = convert_old_sigevent32(&osig, &sig);
28753858a1f4SJohn Baldwin 		if (error)
28763858a1f4SJohn Baldwin 			return (error);
28773858a1f4SJohn Baldwin 		sigp = &sig;
28783858a1f4SJohn Baldwin 	} else
28793858a1f4SJohn Baldwin 		sigp = NULL;
28803858a1f4SJohn Baldwin 
28813858a1f4SJohn Baldwin 	acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
28823858a1f4SJohn Baldwin 	error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
28833858a1f4SJohn Baldwin 	if (error) {
28843858a1f4SJohn Baldwin 		free(acb_list32, M_LIO);
28853858a1f4SJohn Baldwin 		return (error);
28863858a1f4SJohn Baldwin 	}
28873858a1f4SJohn Baldwin 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
28883858a1f4SJohn Baldwin 	for (i = 0; i < nent; i++)
28893858a1f4SJohn Baldwin 		acb_list[i] = PTRIN(acb_list32[i]);
28903858a1f4SJohn Baldwin 	free(acb_list32, M_LIO);
28913858a1f4SJohn Baldwin 
28923858a1f4SJohn Baldwin 	error = kern_lio_listio(td, uap->mode,
28933858a1f4SJohn Baldwin 	    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
28943858a1f4SJohn Baldwin 	    &aiocb32_ops_osigevent);
28953858a1f4SJohn Baldwin 	free(acb_list, M_LIO);
28963858a1f4SJohn Baldwin 	return (error);
28973858a1f4SJohn Baldwin }
28983858a1f4SJohn Baldwin 
28993858a1f4SJohn Baldwin int
29003858a1f4SJohn Baldwin freebsd32_lio_listio(struct thread *td, struct freebsd32_lio_listio_args *uap)
29013858a1f4SJohn Baldwin {
29023858a1f4SJohn Baldwin 	struct aiocb **acb_list;
29033858a1f4SJohn Baldwin 	struct sigevent *sigp, sig;
29043858a1f4SJohn Baldwin 	struct sigevent32 sig32;
29053858a1f4SJohn Baldwin 	uint32_t *acb_list32;
29063858a1f4SJohn Baldwin 	int error, i, nent;
29073858a1f4SJohn Baldwin 
29083858a1f4SJohn Baldwin 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
29093858a1f4SJohn Baldwin 		return (EINVAL);
29103858a1f4SJohn Baldwin 
29113858a1f4SJohn Baldwin 	nent = uap->nent;
29123858a1f4SJohn Baldwin 	if (nent < 0 || nent > AIO_LISTIO_MAX)
29133858a1f4SJohn Baldwin 		return (EINVAL);
29143858a1f4SJohn Baldwin 
29153858a1f4SJohn Baldwin 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
29163858a1f4SJohn Baldwin 		error = copyin(uap->sig, &sig32, sizeof(sig32));
29173858a1f4SJohn Baldwin 		if (error)
29183858a1f4SJohn Baldwin 			return (error);
29193858a1f4SJohn Baldwin 		error = convert_sigevent32(&sig32, &sig);
29203858a1f4SJohn Baldwin 		if (error)
29213858a1f4SJohn Baldwin 			return (error);
29223858a1f4SJohn Baldwin 		sigp = &sig;
29233858a1f4SJohn Baldwin 	} else
29243858a1f4SJohn Baldwin 		sigp = NULL;
29253858a1f4SJohn Baldwin 
29263858a1f4SJohn Baldwin 	acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
29273858a1f4SJohn Baldwin 	error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
29283858a1f4SJohn Baldwin 	if (error) {
29293858a1f4SJohn Baldwin 		free(acb_list32, M_LIO);
29303858a1f4SJohn Baldwin 		return (error);
29313858a1f4SJohn Baldwin 	}
29323858a1f4SJohn Baldwin 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
29333858a1f4SJohn Baldwin 	for (i = 0; i < nent; i++)
29343858a1f4SJohn Baldwin 		acb_list[i] = PTRIN(acb_list32[i]);
29353858a1f4SJohn Baldwin 	free(acb_list32, M_LIO);
29363858a1f4SJohn Baldwin 
29373858a1f4SJohn Baldwin 	error = kern_lio_listio(td, uap->mode,
29383858a1f4SJohn Baldwin 	    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
29393858a1f4SJohn Baldwin 	    &aiocb32_ops);
29403858a1f4SJohn Baldwin 	free(acb_list, M_LIO);
29413858a1f4SJohn Baldwin 	return (error);
29423858a1f4SJohn Baldwin }
29433858a1f4SJohn Baldwin 
29443858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_return);
29453858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_suspend);
29463858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_cancel);
29473858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_error);
29483858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_fsync);
29493858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_read);
29503858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_write);
29513858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_aio_waitcomplete);
29523858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_lio_listio);
29533858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_oaio_read);
29543858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_oaio_write);
29553858a1f4SJohn Baldwin SYSCALL32_MODULE_HELPER(freebsd32_olio_listio);
29563858a1f4SJohn Baldwin #endif
2957