xref: /freebsd/sys/kern/vfs_aio.c (revision 36138969847b231cd98f48272e2bdf88a8dc08dd)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 1997 John S. Dyson.  All rights reserved.
5  *
6  * Redistribution and use in source and binary forms, with or without
7  * modification, are permitted provided that the following conditions
8  * are met:
9  * 1. Redistributions of source code must retain the above copyright
10  *    notice, this list of conditions and the following disclaimer.
11  * 2. John S. Dyson's name may not be used to endorse or promote products
12  *    derived from this software without specific prior written permission.
13  *
14  * DISCLAIMER:  This code isn't warranted to do anything useful.  Anything
15  * bad that happens because of using this software isn't the responsibility
16  * of the author.  This software is distributed AS-IS.
17  */
18 
19 /*
20  * This file contains support for the POSIX 1003.1B AIO/LIO facility.
21  */
22 
23 #include <sys/param.h>
24 #include <sys/systm.h>
25 #include <sys/malloc.h>
26 #include <sys/bio.h>
27 #include <sys/buf.h>
28 #include <sys/capsicum.h>
29 #include <sys/eventhandler.h>
30 #include <sys/sysproto.h>
31 #include <sys/filedesc.h>
32 #include <sys/kernel.h>
33 #include <sys/module.h>
34 #include <sys/kthread.h>
35 #include <sys/fcntl.h>
36 #include <sys/file.h>
37 #include <sys/limits.h>
38 #include <sys/lock.h>
39 #include <sys/mutex.h>
40 #include <sys/unistd.h>
41 #include <sys/posix4.h>
42 #include <sys/proc.h>
43 #include <sys/resourcevar.h>
44 #include <sys/signalvar.h>
45 #include <sys/syscallsubr.h>
46 #include <sys/protosw.h>
47 #include <sys/rwlock.h>
48 #include <sys/sema.h>
49 #include <sys/socket.h>
50 #include <sys/socketvar.h>
51 #include <sys/syscall.h>
52 #include <sys/sysctl.h>
53 #include <sys/syslog.h>
54 #include <sys/sx.h>
55 #include <sys/taskqueue.h>
56 #include <sys/vnode.h>
57 #include <sys/conf.h>
58 #include <sys/event.h>
59 #include <sys/mount.h>
60 #include <geom/geom.h>
61 
62 #include <machine/atomic.h>
63 
64 #include <vm/vm.h>
65 #include <vm/vm_page.h>
66 #include <vm/vm_extern.h>
67 #include <vm/pmap.h>
68 #include <vm/vm_map.h>
69 #include <vm/vm_object.h>
70 #include <vm/vnode_pager.h>
71 #include <vm/uma.h>
72 #include <sys/aio.h>
73 
74 /*
75  * Counter for aio_fsync.
76  */
77 static uint64_t jobseqno;
78 
79 #ifndef MAX_AIO_PER_PROC
80 #define MAX_AIO_PER_PROC	32
81 #endif
82 
83 #ifndef MAX_AIO_QUEUE_PER_PROC
84 #define MAX_AIO_QUEUE_PER_PROC	256
85 #endif
86 
87 #ifndef MAX_AIO_QUEUE
88 #define MAX_AIO_QUEUE		1024 /* Bigger than MAX_AIO_QUEUE_PER_PROC */
89 #endif
90 
91 #ifndef MAX_BUF_AIO
92 #define MAX_BUF_AIO		16
93 #endif
94 
95 FEATURE(aio, "Asynchronous I/O");
96 SYSCTL_DECL(_p1003_1b);
97 
98 static MALLOC_DEFINE(M_LIO, "lio", "listio aio control block list");
99 static MALLOC_DEFINE(M_AIO, "aio", "structures for asynchronous I/O");
100 
101 static SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
102     "Async IO management");
103 
104 static int enable_aio_unsafe = 0;
105 SYSCTL_INT(_vfs_aio, OID_AUTO, enable_unsafe, CTLFLAG_RW, &enable_aio_unsafe, 0,
106     "Permit asynchronous IO on all file types, not just known-safe types");
107 
108 static unsigned int unsafe_warningcnt = 1;
109 SYSCTL_UINT(_vfs_aio, OID_AUTO, unsafe_warningcnt, CTLFLAG_RW,
110     &unsafe_warningcnt, 0,
111     "Warnings that will be triggered upon failed IO requests on unsafe files");
112 
113 static int max_aio_procs = MAX_AIO_PROCS;
114 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs, CTLFLAG_RW, &max_aio_procs, 0,
115     "Maximum number of kernel processes to use for handling async IO ");
116 
117 static int num_aio_procs = 0;
118 SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs, CTLFLAG_RD, &num_aio_procs, 0,
119     "Number of presently active kernel processes for async IO");
120 
121 /*
122  * The code will adjust the actual number of AIO processes towards this
123  * number when it gets a chance.
124  */
125 static int target_aio_procs = TARGET_AIO_PROCS;
126 SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW, &target_aio_procs,
127     0,
128     "Preferred number of ready kernel processes for async IO");
129 
130 static int max_queue_count = MAX_AIO_QUEUE;
131 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW, &max_queue_count, 0,
132     "Maximum number of aio requests to queue, globally");
133 
134 static int num_queue_count = 0;
135 SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD, &num_queue_count, 0,
136     "Number of queued aio requests");
137 
138 static int num_buf_aio = 0;
139 SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD, &num_buf_aio, 0,
140     "Number of aio requests presently handled by the buf subsystem");
141 
142 static int num_unmapped_aio = 0;
143 SYSCTL_INT(_vfs_aio, OID_AUTO, num_unmapped_aio, CTLFLAG_RD, &num_unmapped_aio,
144     0,
145     "Number of aio requests presently handled by unmapped I/O buffers");
146 
147 /* Number of async I/O processes in the process of being started */
148 /* XXX This should be local to aio_aqueue() */
149 static int num_aio_resv_start = 0;
150 
151 static int aiod_lifetime;
152 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW, &aiod_lifetime, 0,
153     "Maximum lifetime for idle aiod");
154 
155 static int max_aio_per_proc = MAX_AIO_PER_PROC;
156 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW, &max_aio_per_proc,
157     0,
158     "Maximum active aio requests per process");
159 
160 static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC;
161 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW,
162     &max_aio_queue_per_proc, 0,
163     "Maximum queued aio requests per process");
164 
165 static int max_buf_aio = MAX_BUF_AIO;
166 SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0,
167     "Maximum buf aio requests per process");
168 
169 /*
170  * Though redundant with vfs.aio.max_aio_queue_per_proc, POSIX requires
171  * sysconf(3) to support AIO_LISTIO_MAX, and we implement that with
172  * vfs.aio.aio_listio_max.
173  */
174 SYSCTL_INT(_p1003_1b, CTL_P1003_1B_AIO_LISTIO_MAX, aio_listio_max,
175     CTLFLAG_RD | CTLFLAG_CAPRD, &max_aio_queue_per_proc,
176     0, "Maximum aio requests for a single lio_listio call");
177 
178 #ifdef COMPAT_FREEBSD6
179 typedef struct oaiocb {
180 	int	aio_fildes;		/* File descriptor */
181 	off_t	aio_offset;		/* File offset for I/O */
182 	volatile void *aio_buf;         /* I/O buffer in process space */
183 	size_t	aio_nbytes;		/* Number of bytes for I/O */
184 	struct	osigevent aio_sigevent;	/* Signal to deliver */
185 	int	aio_lio_opcode;		/* LIO opcode */
186 	int	aio_reqprio;		/* Request priority -- ignored */
187 	struct	__aiocb_private	_aiocb_private;
188 } oaiocb_t;
189 #endif
190 
191 /*
192  * Below is a key of locks used to protect each member of struct kaiocb
193  * aioliojob and kaioinfo and any backends.
194  *
195  * * - need not protected
196  * a - locked by kaioinfo lock
197  * b - locked by backend lock, the backend lock can be null in some cases,
198  *     for example, BIO belongs to this type, in this case, proc lock is
199  *     reused.
200  * c - locked by aio_job_mtx, the lock for the generic file I/O backend.
201  */
202 
203 /*
204  * If the routine that services an AIO request blocks while running in an
205  * AIO kernel process it can starve other I/O requests.  BIO requests
206  * queued via aio_qbio() complete asynchronously and do not use AIO kernel
207  * processes at all.  Socket I/O requests use a separate pool of
208  * kprocs and also force non-blocking I/O.  Other file I/O requests
209  * use the generic fo_read/fo_write operations which can block.  The
210  * fsync and mlock operations can also block while executing.  Ideally
211  * none of these requests would block while executing.
212  *
213  * Note that the service routines cannot toggle O_NONBLOCK in the file
214  * structure directly while handling a request due to races with
215  * userland threads.
216  */
217 
218 /* jobflags */
219 #define	KAIOCB_QUEUEING		0x01
220 #define	KAIOCB_CANCELLED	0x02
221 #define	KAIOCB_CANCELLING	0x04
222 #define	KAIOCB_CHECKSYNC	0x08
223 #define	KAIOCB_CLEARED		0x10
224 #define	KAIOCB_FINISHED		0x20
225 #define	KAIOCB_MARKER		0x40
226 
227 /* ioflags */
228 #define	KAIOCB_IO_FOFFSET	0x01
229 
230 /*
231  * AIO process info
232  */
233 #define AIOP_FREE	0x1			/* proc on free queue */
234 
235 struct aioproc {
236 	int	aioprocflags;			/* (c) AIO proc flags */
237 	TAILQ_ENTRY(aioproc) list;		/* (c) list of processes */
238 	struct	proc *aioproc;			/* (*) the AIO proc */
239 };
240 
241 /*
242  * data-structure for lio signal management
243  */
244 struct aioliojob {
245 	int	lioj_flags;			/* (a) listio flags */
246 	int	lioj_count;			/* (a) count of jobs */
247 	int	lioj_finished_count;		/* (a) count of finished jobs */
248 	struct	sigevent lioj_signal;		/* (a) signal on all I/O done */
249 	TAILQ_ENTRY(aioliojob) lioj_list;	/* (a) lio list */
250 	struct	knlist klist;			/* (a) list of knotes */
251 	ksiginfo_t lioj_ksi;			/* (a) Realtime signal info */
252 };
253 
254 #define	LIOJ_SIGNAL		0x1	/* signal on all done (lio) */
255 #define	LIOJ_SIGNAL_POSTED	0x2	/* signal has been posted */
256 #define LIOJ_KEVENT_POSTED	0x4	/* kevent triggered */
257 
258 /*
259  * per process aio data structure
260  */
261 struct kaioinfo {
262 	struct	mtx kaio_mtx;		/* the lock to protect this struct */
263 	int	kaio_flags;		/* (a) per process kaio flags */
264 	int	kaio_active_count;	/* (c) number of currently used AIOs */
265 	int	kaio_count;		/* (a) size of AIO queue */
266 	int	kaio_buffer_count;	/* (a) number of bio buffers */
267 	TAILQ_HEAD(,kaiocb) kaio_all;	/* (a) all AIOs in a process */
268 	TAILQ_HEAD(,kaiocb) kaio_done;	/* (a) done queue for process */
269 	TAILQ_HEAD(,aioliojob) kaio_liojoblist; /* (a) list of lio jobs */
270 	TAILQ_HEAD(,kaiocb) kaio_jobqueue;	/* (a) job queue for process */
271 	TAILQ_HEAD(,kaiocb) kaio_syncqueue;	/* (a) queue for aio_fsync */
272 	TAILQ_HEAD(,kaiocb) kaio_syncready;  /* (a) second q for aio_fsync */
273 	struct	task kaio_task;		/* (*) task to kick aio processes */
274 	struct	task kaio_sync_task;	/* (*) task to schedule fsync jobs */
275 };
276 
277 #define AIO_LOCK(ki)		mtx_lock(&(ki)->kaio_mtx)
278 #define AIO_UNLOCK(ki)		mtx_unlock(&(ki)->kaio_mtx)
279 #define AIO_LOCK_ASSERT(ki, f)	mtx_assert(&(ki)->kaio_mtx, (f))
280 #define AIO_MTX(ki)		(&(ki)->kaio_mtx)
281 
282 #define KAIO_RUNDOWN	0x1	/* process is being run down */
283 #define KAIO_WAKEUP	0x2	/* wakeup process when AIO completes */
284 
285 /*
286  * Operations used to interact with userland aio control blocks.
287  * Different ABIs provide their own operations.
288  */
289 struct aiocb_ops {
290 	int	(*aio_copyin)(struct aiocb *ujob, struct kaiocb *kjob, int ty);
291 	long	(*fetch_status)(struct aiocb *ujob);
292 	long	(*fetch_error)(struct aiocb *ujob);
293 	int	(*store_status)(struct aiocb *ujob, long status);
294 	int	(*store_error)(struct aiocb *ujob, long error);
295 	int	(*store_aiocb)(struct aiocb **ujobp, struct aiocb *ujob);
296 };
297 
298 static TAILQ_HEAD(,aioproc) aio_freeproc;		/* (c) Idle daemons */
299 static struct sema aio_newproc_sem;
300 static struct mtx aio_job_mtx;
301 static TAILQ_HEAD(,kaiocb) aio_jobs;			/* (c) Async job list */
302 static struct unrhdr *aiod_unr;
303 
304 static void	aio_biocleanup(struct bio *bp);
305 static int	aio_init_aioinfo(struct proc *p);
306 static int	aio_onceonly(void);
307 static int	aio_free_entry(struct kaiocb *job);
308 static void	aio_process_rw(struct kaiocb *job);
309 static void	aio_process_sync(struct kaiocb *job);
310 static void	aio_process_mlock(struct kaiocb *job);
311 static void	aio_schedule_fsync(void *context, int pending);
312 static int	aio_newproc(int *);
313 static int	aio_aqueue(struct thread *td, struct aiocb *ujob,
314 		    struct aioliojob *lio, int type, struct aiocb_ops *ops);
315 static int	aio_queue_file(struct file *fp, struct kaiocb *job);
316 static void	aio_biowakeup(struct bio *bp);
317 static void	aio_proc_rundown(void *arg, struct proc *p);
318 static void	aio_proc_rundown_exec(void *arg, struct proc *p,
319 		    struct image_params *imgp);
320 static int	aio_qbio(struct proc *p, struct kaiocb *job);
321 static void	aio_daemon(void *param);
322 static void	aio_bio_done_notify(struct proc *userp, struct kaiocb *job);
323 static bool	aio_clear_cancel_function_locked(struct kaiocb *job);
324 static int	aio_kick(struct proc *userp);
325 static void	aio_kick_nowait(struct proc *userp);
326 static void	aio_kick_helper(void *context, int pending);
327 static int	filt_aioattach(struct knote *kn);
328 static void	filt_aiodetach(struct knote *kn);
329 static int	filt_aio(struct knote *kn, long hint);
330 static int	filt_lioattach(struct knote *kn);
331 static void	filt_liodetach(struct knote *kn);
332 static int	filt_lio(struct knote *kn, long hint);
333 
334 /*
335  * Zones for:
336  * 	kaio	Per process async io info
337  *	aiocb	async io jobs
338  *	aiolio	list io jobs
339  */
340 static uma_zone_t kaio_zone, aiocb_zone, aiolio_zone;
341 
342 /* kqueue filters for aio */
343 static const struct filterops aio_filtops = {
344 	.f_isfd = 0,
345 	.f_attach = filt_aioattach,
346 	.f_detach = filt_aiodetach,
347 	.f_event = filt_aio,
348 	.f_copy = knote_triv_copy,
349 };
350 static const struct filterops lio_filtops = {
351 	.f_isfd = 0,
352 	.f_attach = filt_lioattach,
353 	.f_detach = filt_liodetach,
354 	.f_event = filt_lio,
355 	.f_copy = knote_triv_copy,
356 };
357 
358 static eventhandler_tag exit_tag, exec_tag;
359 
360 TASKQUEUE_DEFINE_THREAD(aiod_kick);
361 
362 /*
363  * Main operations function for use as a kernel module.
364  */
365 static int
aio_modload(struct module * module,int cmd,void * arg)366 aio_modload(struct module *module, int cmd, void *arg)
367 {
368 	int error = 0;
369 
370 	switch (cmd) {
371 	case MOD_LOAD:
372 		aio_onceonly();
373 		break;
374 	case MOD_SHUTDOWN:
375 		break;
376 	default:
377 		error = EOPNOTSUPP;
378 		break;
379 	}
380 	return (error);
381 }
382 
383 static moduledata_t aio_mod = {
384 	"aio",
385 	&aio_modload,
386 	NULL
387 };
388 
389 DECLARE_MODULE(aio, aio_mod, SI_SUB_VFS, SI_ORDER_ANY);
390 MODULE_VERSION(aio, 1);
391 
392 /*
393  * Startup initialization
394  */
395 static int
aio_onceonly(void)396 aio_onceonly(void)
397 {
398 
399 	exit_tag = EVENTHANDLER_REGISTER(process_exit, aio_proc_rundown, NULL,
400 	    EVENTHANDLER_PRI_ANY);
401 	exec_tag = EVENTHANDLER_REGISTER(process_exec, aio_proc_rundown_exec,
402 	    NULL, EVENTHANDLER_PRI_ANY);
403 	kqueue_add_filteropts(EVFILT_AIO, &aio_filtops);
404 	kqueue_add_filteropts(EVFILT_LIO, &lio_filtops);
405 	TAILQ_INIT(&aio_freeproc);
406 	sema_init(&aio_newproc_sem, 0, "aio_new_proc");
407 	mtx_init(&aio_job_mtx, "aio_job", NULL, MTX_DEF);
408 	TAILQ_INIT(&aio_jobs);
409 	aiod_unr = new_unrhdr(1, INT_MAX, NULL);
410 	kaio_zone = uma_zcreate("AIO", sizeof(struct kaioinfo), NULL, NULL,
411 	    NULL, NULL, UMA_ALIGN_PTR, 0);
412 	aiocb_zone = uma_zcreate("AIOCB", sizeof(struct kaiocb), NULL, NULL,
413 	    NULL, NULL, UMA_ALIGN_PTR, 0);
414 	aiolio_zone = uma_zcreate("AIOLIO", sizeof(struct aioliojob), NULL,
415 	    NULL, NULL, NULL, UMA_ALIGN_PTR, 0);
416 	aiod_lifetime = AIOD_LIFETIME_DEFAULT;
417 	p31b_setcfg(CTL_P1003_1B_ASYNCHRONOUS_IO, _POSIX_ASYNCHRONOUS_IO);
418 	p31b_setcfg(CTL_P1003_1B_AIO_MAX, MAX_AIO_QUEUE);
419 	p31b_setcfg(CTL_P1003_1B_AIO_PRIO_DELTA_MAX, 0);
420 
421 	return (0);
422 }
423 
424 /*
425  * Init the per-process aioinfo structure.  The aioinfo limits are set
426  * per-process for user limit (resource) management.
427  */
428 static int
aio_init_aioinfo(struct proc * p)429 aio_init_aioinfo(struct proc *p)
430 {
431 	struct kaioinfo *ki;
432 	int error;
433 
434 	ki = uma_zalloc(kaio_zone, M_WAITOK);
435 	mtx_init(&ki->kaio_mtx, "aiomtx", NULL, MTX_DEF | MTX_NEW);
436 	ki->kaio_flags = 0;
437 	ki->kaio_active_count = 0;
438 	ki->kaio_count = 0;
439 	ki->kaio_buffer_count = 0;
440 	TAILQ_INIT(&ki->kaio_all);
441 	TAILQ_INIT(&ki->kaio_done);
442 	TAILQ_INIT(&ki->kaio_jobqueue);
443 	TAILQ_INIT(&ki->kaio_liojoblist);
444 	TAILQ_INIT(&ki->kaio_syncqueue);
445 	TAILQ_INIT(&ki->kaio_syncready);
446 	TASK_INIT(&ki->kaio_task, 0, aio_kick_helper, p);
447 	TASK_INIT(&ki->kaio_sync_task, 0, aio_schedule_fsync, ki);
448 	PROC_LOCK(p);
449 	if (p->p_aioinfo == NULL) {
450 		p->p_aioinfo = ki;
451 		PROC_UNLOCK(p);
452 	} else {
453 		PROC_UNLOCK(p);
454 		mtx_destroy(&ki->kaio_mtx);
455 		uma_zfree(kaio_zone, ki);
456 	}
457 
458 	error = 0;
459 	while (num_aio_procs < MIN(target_aio_procs, max_aio_procs)) {
460 		error = aio_newproc(NULL);
461 		if (error != 0) {
462 			/*
463 			 * At least one worker is enough to have AIO
464 			 * functional.  Clear error in that case.
465 			 */
466 			if (num_aio_procs > 0)
467 				error = 0;
468 			break;
469 		}
470 	}
471 	return (error);
472 }
473 
474 static int
aio_sendsig(struct proc * p,struct sigevent * sigev,ksiginfo_t * ksi,bool ext)475 aio_sendsig(struct proc *p, struct sigevent *sigev, ksiginfo_t *ksi, bool ext)
476 {
477 	struct thread *td;
478 	int error;
479 
480 	error = sigev_findtd(p, sigev, &td);
481 	if (error)
482 		return (error);
483 	if (!KSI_ONQ(ksi)) {
484 		ksiginfo_set_sigev(ksi, sigev);
485 		ksi->ksi_code = SI_ASYNCIO;
486 		ksi->ksi_flags |= ext ? (KSI_EXT | KSI_INS) : 0;
487 		tdsendsignal(p, td, ksi->ksi_signo, ksi);
488 	}
489 	PROC_UNLOCK(p);
490 	return (error);
491 }
492 
493 /*
494  * Free a job entry.  Wait for completion if it is currently active, but don't
495  * delay forever.  If we delay, we return a flag that says that we have to
496  * restart the queue scan.
497  */
498 static int
aio_free_entry(struct kaiocb * job)499 aio_free_entry(struct kaiocb *job)
500 {
501 	struct kaioinfo *ki;
502 	struct aioliojob *lj;
503 	struct proc *p;
504 
505 	p = job->userproc;
506 	MPASS(curproc == p);
507 	ki = p->p_aioinfo;
508 	MPASS(ki != NULL);
509 
510 	AIO_LOCK_ASSERT(ki, MA_OWNED);
511 	MPASS(job->jobflags & KAIOCB_FINISHED);
512 
513 	atomic_subtract_int(&num_queue_count, 1);
514 
515 	ki->kaio_count--;
516 	MPASS(ki->kaio_count >= 0);
517 
518 	TAILQ_REMOVE(&ki->kaio_done, job, plist);
519 	TAILQ_REMOVE(&ki->kaio_all, job, allist);
520 
521 	lj = job->lio;
522 	if (lj) {
523 		lj->lioj_count--;
524 		lj->lioj_finished_count--;
525 
526 		if (lj->lioj_count == 0) {
527 			TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
528 			/* lio is going away, we need to destroy any knotes */
529 			knlist_delete(&lj->klist, curthread, 1);
530 			PROC_LOCK(p);
531 			sigqueue_take(&lj->lioj_ksi);
532 			PROC_UNLOCK(p);
533 			uma_zfree(aiolio_zone, lj);
534 		}
535 	}
536 
537 	/* job is going away, we need to destroy any knotes */
538 	knlist_delete(&job->klist, curthread, 1);
539 	PROC_LOCK(p);
540 	sigqueue_take(&job->ksi);
541 	PROC_UNLOCK(p);
542 
543 	AIO_UNLOCK(ki);
544 
545 	/*
546 	 * The thread argument here is used to find the owning process
547 	 * and is also passed to fo_close() which may pass it to various
548 	 * places such as devsw close() routines.  Because of that, we
549 	 * need a thread pointer from the process owning the job that is
550 	 * persistent and won't disappear out from under us or move to
551 	 * another process.
552 	 *
553 	 * Currently, all the callers of this function call it to remove
554 	 * a kaiocb from the current process' job list either via a
555 	 * syscall or due to the current process calling exit() or
556 	 * execve().  Thus, we know that p == curproc.  We also know that
557 	 * curthread can't exit since we are curthread.
558 	 *
559 	 * Therefore, we use curthread as the thread to pass to
560 	 * knlist_delete().  This does mean that it is possible for the
561 	 * thread pointer at close time to differ from the thread pointer
562 	 * at open time, but this is already true of file descriptors in
563 	 * a multithreaded process.
564 	 */
565 	if (job->fd_file)
566 		fdrop(job->fd_file, curthread);
567 	crfree(job->cred);
568 	if (job->uiop != &job->uio)
569 		freeuio(job->uiop);
570 	uma_zfree(aiocb_zone, job);
571 	AIO_LOCK(ki);
572 
573 	return (0);
574 }
575 
576 static void
aio_proc_rundown_exec(void * arg,struct proc * p,struct image_params * imgp __unused)577 aio_proc_rundown_exec(void *arg, struct proc *p,
578     struct image_params *imgp __unused)
579 {
580    	aio_proc_rundown(arg, p);
581 }
582 
583 static int
aio_cancel_job(struct proc * p,struct kaioinfo * ki,struct kaiocb * job)584 aio_cancel_job(struct proc *p, struct kaioinfo *ki, struct kaiocb *job)
585 {
586 	aio_cancel_fn_t *func;
587 	int cancelled;
588 
589 	AIO_LOCK_ASSERT(ki, MA_OWNED);
590 
591 	/*
592 	 * If we're running down the queue, the process must be single-threaded,
593 	 * and so no markers should be present.
594 	 */
595 	MPASS((job->jobflags & KAIOCB_MARKER) == 0);
596 	if (job->jobflags & (KAIOCB_CANCELLED | KAIOCB_FINISHED))
597 		return (0);
598 	MPASS((job->jobflags & KAIOCB_CANCELLING) == 0);
599 	job->jobflags |= KAIOCB_CANCELLED;
600 
601 	func = job->cancel_fn;
602 
603 	/*
604 	 * If there is no cancel routine, just leave the job marked as
605 	 * cancelled.  The job should be in active use by a caller who
606 	 * should complete it normally or when it fails to install a
607 	 * cancel routine.
608 	 */
609 	if (func == NULL)
610 		return (0);
611 
612 	/*
613 	 * Set the CANCELLING flag so that aio_complete() will defer
614 	 * completions of this job.  This prevents the job from being
615 	 * freed out from under the cancel callback.  After the
616 	 * callback any deferred completion (whether from the callback
617 	 * or any other source) will be completed.
618 	 */
619 	job->jobflags |= KAIOCB_CANCELLING;
620 	AIO_UNLOCK(ki);
621 	func(job);
622 	AIO_LOCK(ki);
623 	job->jobflags &= ~KAIOCB_CANCELLING;
624 	if (job->jobflags & KAIOCB_FINISHED) {
625 		cancelled = job->uaiocb._aiocb_private.error == ECANCELED;
626 		TAILQ_REMOVE(&ki->kaio_jobqueue, job, plist);
627 		aio_bio_done_notify(p, job);
628 	} else {
629 		/*
630 		 * The cancel callback might have scheduled an
631 		 * operation to cancel this request, but it is
632 		 * only counted as cancelled if the request is
633 		 * cancelled when the callback returns.
634 		 */
635 		cancelled = 0;
636 	}
637 	return (cancelled);
638 }
639 
640 /*
641  * Rundown the jobs for a given process.
642  */
643 static void
aio_proc_rundown(void * arg,struct proc * p)644 aio_proc_rundown(void *arg, struct proc *p)
645 {
646 	struct kaioinfo *ki;
647 	struct aioliojob *lj;
648 	struct kaiocb *job, *jobn;
649 
650 	KASSERT(curthread->td_proc == p,
651 	    ("%s: called on non-curproc", __func__));
652 	ki = p->p_aioinfo;
653 	if (ki == NULL)
654 		return;
655 
656 	AIO_LOCK(ki);
657 	ki->kaio_flags |= KAIO_RUNDOWN;
658 
659 restart:
660 
661 	/*
662 	 * Try to cancel all pending requests. This code simulates
663 	 * aio_cancel on all pending I/O requests.
664 	 */
665 	TAILQ_FOREACH_SAFE(job, &ki->kaio_jobqueue, plist, jobn) {
666 		aio_cancel_job(p, ki, job);
667 	}
668 
669 	/* Wait for all running I/O to be finished */
670 	if (!TAILQ_EMPTY(&ki->kaio_jobqueue) || ki->kaio_active_count != 0) {
671 		ki->kaio_flags |= KAIO_WAKEUP;
672 		msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO, "aioprn", hz);
673 		goto restart;
674 	}
675 
676 	/* Free all completed I/O requests. */
677 	while ((job = TAILQ_FIRST(&ki->kaio_done)) != NULL)
678 		aio_free_entry(job);
679 
680 	while ((lj = TAILQ_FIRST(&ki->kaio_liojoblist)) != NULL) {
681 		if (lj->lioj_count == 0) {
682 			TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
683 			knlist_delete(&lj->klist, curthread, 1);
684 			PROC_LOCK(p);
685 			sigqueue_take(&lj->lioj_ksi);
686 			PROC_UNLOCK(p);
687 			uma_zfree(aiolio_zone, lj);
688 		} else {
689 			panic("LIO job not cleaned up: C:%d, FC:%d\n",
690 			    lj->lioj_count, lj->lioj_finished_count);
691 		}
692 	}
693 	AIO_UNLOCK(ki);
694 	taskqueue_drain(taskqueue_aiod_kick, &ki->kaio_task);
695 	taskqueue_drain(taskqueue_aiod_kick, &ki->kaio_sync_task);
696 	mtx_destroy(&ki->kaio_mtx);
697 	uma_zfree(kaio_zone, ki);
698 	p->p_aioinfo = NULL;
699 }
700 
701 /*
702  * Select a job to run (called by an AIO daemon).
703  */
704 static struct kaiocb *
aio_selectjob(struct aioproc * aiop)705 aio_selectjob(struct aioproc *aiop)
706 {
707 	struct kaiocb *job;
708 	struct kaioinfo *ki;
709 	struct proc *userp;
710 
711 	mtx_assert(&aio_job_mtx, MA_OWNED);
712 restart:
713 	TAILQ_FOREACH(job, &aio_jobs, list) {
714 		userp = job->userproc;
715 		ki = userp->p_aioinfo;
716 
717 		if (ki->kaio_active_count < max_aio_per_proc) {
718 			TAILQ_REMOVE(&aio_jobs, job, list);
719 			if (!aio_clear_cancel_function(job))
720 				goto restart;
721 
722 			/* Account for currently active jobs. */
723 			ki->kaio_active_count++;
724 			break;
725 		}
726 	}
727 	return (job);
728 }
729 
730 /*
731  * Move all data to a permanent storage device.  This code
732  * simulates the fsync and fdatasync syscalls.
733  */
734 static int
aio_fsync_vnode(struct thread * td,struct vnode * vp,int op)735 aio_fsync_vnode(struct thread *td, struct vnode *vp, int op)
736 {
737 	struct mount *mp;
738 	int error;
739 
740 	for (;;) {
741 		error = vn_start_write(vp, &mp, V_WAIT | V_PCATCH);
742 		if (error != 0)
743 			break;
744 		vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
745 		vnode_pager_clean_async(vp);
746 		if (op == LIO_DSYNC)
747 			error = VOP_FDATASYNC(vp, td);
748 		else
749 			error = VOP_FSYNC(vp, MNT_WAIT, td);
750 
751 		VOP_UNLOCK(vp);
752 		vn_finished_write(mp);
753 		if (error != ERELOOKUP)
754 			break;
755 	}
756 	return (error);
757 }
758 
759 /*
760  * The AIO processing activity for LIO_READ/LIO_WRITE.  This is the code that
761  * does the I/O request for the non-bio version of the operations.  The normal
762  * vn operations are used, and this code should work in all instances for every
763  * type of file, including pipes, sockets, fifos, and regular files.
764  *
765  * XXX I don't think it works well for socket, pipe, and fifo.
766  */
767 static void
aio_process_rw(struct kaiocb * job)768 aio_process_rw(struct kaiocb *job)
769 {
770 	struct ucred *td_savedcred;
771 	struct thread *td;
772 	struct file *fp;
773 	ssize_t cnt;
774 	long msgsnd_st, msgsnd_end;
775 	long msgrcv_st, msgrcv_end;
776 	long oublock_st, oublock_end;
777 	long inblock_st, inblock_end;
778 	int error, opcode;
779 
780 	opcode = job->uaiocb.aio_lio_opcode & ~LIO_FOFFSET;
781 	KASSERT(opcode == LIO_READ || opcode == LIO_READV ||
782 	    opcode == LIO_WRITE || opcode == LIO_WRITEV,
783 	    ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode));
784 
785 	aio_switch_vmspace(job);
786 	td = curthread;
787 	td_savedcred = td->td_ucred;
788 	td->td_ucred = job->cred;
789 	job->uiop->uio_td = td;
790 	fp = job->fd_file;
791 
792 	cnt = job->uiop->uio_resid;
793 
794 	msgrcv_st = td->td_ru.ru_msgrcv;
795 	msgsnd_st = td->td_ru.ru_msgsnd;
796 	inblock_st = td->td_ru.ru_inblock;
797 	oublock_st = td->td_ru.ru_oublock;
798 
799 	/*
800 	 * aio_aqueue() acquires a reference to the file that is
801 	 * released in aio_free_entry().
802 	 */
803 	if (opcode == LIO_READ || opcode == LIO_READV) {
804 		if (job->uiop->uio_resid == 0)
805 			error = 0;
806 		else
807 			error = fo_read(fp, job->uiop, fp->f_cred,
808 			    (job->ioflags & KAIOCB_IO_FOFFSET) != 0 ? 0 :
809 			    FOF_OFFSET, td);
810 	} else {
811 		if (fp->f_type == DTYPE_VNODE)
812 			bwillwrite();
813 		error = fo_write(fp, job->uiop, fp->f_cred, (job->ioflags &
814 		    KAIOCB_IO_FOFFSET) != 0 ? 0 : FOF_OFFSET, td);
815 	}
816 	msgrcv_end = td->td_ru.ru_msgrcv;
817 	msgsnd_end = td->td_ru.ru_msgsnd;
818 	inblock_end = td->td_ru.ru_inblock;
819 	oublock_end = td->td_ru.ru_oublock;
820 
821 	job->msgrcv = msgrcv_end - msgrcv_st;
822 	job->msgsnd = msgsnd_end - msgsnd_st;
823 	job->inblock = inblock_end - inblock_st;
824 	job->outblock = oublock_end - oublock_st;
825 
826 	if (error != 0 && job->uiop->uio_resid != cnt) {
827 		if (error == ERESTART || error == EINTR || error == EWOULDBLOCK)
828 			error = 0;
829 		if (error == EPIPE && (opcode & LIO_WRITE)) {
830 			PROC_LOCK(job->userproc);
831 			kern_psignal(job->userproc, SIGPIPE);
832 			PROC_UNLOCK(job->userproc);
833 		}
834 	}
835 
836 	cnt -= job->uiop->uio_resid;
837 	td->td_ucred = td_savedcred;
838 	if (error)
839 		aio_complete(job, -1, error);
840 	else
841 		aio_complete(job, cnt, 0);
842 }
843 
844 static void
aio_process_sync(struct kaiocb * job)845 aio_process_sync(struct kaiocb *job)
846 {
847 	struct thread *td = curthread;
848 	struct ucred *td_savedcred = td->td_ucred;
849 	struct file *fp = job->fd_file;
850 	int error = 0;
851 
852 	KASSERT(job->uaiocb.aio_lio_opcode & LIO_SYNC,
853 	    ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode));
854 
855 	td->td_ucred = job->cred;
856 	if (fp->f_vnode != NULL) {
857 		error = aio_fsync_vnode(td, fp->f_vnode,
858 		    job->uaiocb.aio_lio_opcode);
859 	}
860 	td->td_ucred = td_savedcred;
861 	if (error)
862 		aio_complete(job, -1, error);
863 	else
864 		aio_complete(job, 0, 0);
865 }
866 
867 static void
aio_process_mlock(struct kaiocb * job)868 aio_process_mlock(struct kaiocb *job)
869 {
870 	struct aiocb *cb = &job->uaiocb;
871 	int error;
872 
873 	KASSERT(job->uaiocb.aio_lio_opcode == LIO_MLOCK,
874 	    ("%s: opcode %d", __func__, job->uaiocb.aio_lio_opcode));
875 
876 	aio_switch_vmspace(job);
877 	error = kern_mlock(job->userproc, job->cred,
878 	    __DEVOLATILE(uintptr_t, cb->aio_buf), cb->aio_nbytes);
879 	aio_complete(job, error != 0 ? -1 : 0, error);
880 }
881 
882 static void
aio_bio_done_notify(struct proc * userp,struct kaiocb * job)883 aio_bio_done_notify(struct proc *userp, struct kaiocb *job)
884 {
885 	struct aioliojob *lj;
886 	struct kaioinfo *ki;
887 	struct kaiocb *sjob, *sjobn;
888 	int lj_done;
889 	bool schedule_fsync;
890 
891 	ki = userp->p_aioinfo;
892 	AIO_LOCK_ASSERT(ki, MA_OWNED);
893 	lj = job->lio;
894 	lj_done = 0;
895 	if (lj) {
896 		lj->lioj_finished_count++;
897 		if (lj->lioj_count == lj->lioj_finished_count)
898 			lj_done = 1;
899 	}
900 	TAILQ_INSERT_TAIL(&ki->kaio_done, job, plist);
901 	MPASS(job->jobflags & KAIOCB_FINISHED);
902 
903 	if (ki->kaio_flags & KAIO_RUNDOWN)
904 		goto notification_done;
905 
906 	if (job->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL ||
907 	    job->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID)
908 		aio_sendsig(userp, &job->uaiocb.aio_sigevent, &job->ksi, true);
909 
910 	KNOTE_LOCKED(&job->klist, 1);
911 
912 	if (lj_done) {
913 		if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
914 			lj->lioj_flags |= LIOJ_KEVENT_POSTED;
915 			KNOTE_LOCKED(&lj->klist, 1);
916 		}
917 		if ((lj->lioj_flags & (LIOJ_SIGNAL | LIOJ_SIGNAL_POSTED))
918 		    == LIOJ_SIGNAL &&
919 		    (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
920 		    lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID)) {
921 			aio_sendsig(userp, &lj->lioj_signal, &lj->lioj_ksi,
922 			    true);
923 			lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
924 		}
925 	}
926 
927 notification_done:
928 	if (job->jobflags & KAIOCB_CHECKSYNC) {
929 		schedule_fsync = false;
930 		TAILQ_FOREACH_SAFE(sjob, &ki->kaio_syncqueue, list, sjobn) {
931 			if (job->fd_file != sjob->fd_file ||
932 			    job->seqno >= sjob->seqno)
933 				continue;
934 			if (--sjob->pending > 0)
935 				continue;
936 			TAILQ_REMOVE(&ki->kaio_syncqueue, sjob, list);
937 			if (!aio_clear_cancel_function_locked(sjob))
938 				continue;
939 			TAILQ_INSERT_TAIL(&ki->kaio_syncready, sjob, list);
940 			schedule_fsync = true;
941 		}
942 		if (schedule_fsync)
943 			taskqueue_enqueue(taskqueue_aiod_kick,
944 			    &ki->kaio_sync_task);
945 	}
946 	if (ki->kaio_flags & KAIO_WAKEUP) {
947 		ki->kaio_flags &= ~KAIO_WAKEUP;
948 		wakeup(&userp->p_aioinfo);
949 	}
950 }
951 
952 static void
aio_schedule_fsync(void * context,int pending)953 aio_schedule_fsync(void *context, int pending)
954 {
955 	struct kaioinfo *ki;
956 	struct kaiocb *job;
957 
958 	ki = context;
959 	AIO_LOCK(ki);
960 	while (!TAILQ_EMPTY(&ki->kaio_syncready)) {
961 		job = TAILQ_FIRST(&ki->kaio_syncready);
962 		TAILQ_REMOVE(&ki->kaio_syncready, job, list);
963 		AIO_UNLOCK(ki);
964 		aio_schedule(job, aio_process_sync);
965 		AIO_LOCK(ki);
966 	}
967 	AIO_UNLOCK(ki);
968 }
969 
970 bool
aio_cancel_cleared(struct kaiocb * job)971 aio_cancel_cleared(struct kaiocb *job)
972 {
973 
974 	/*
975 	 * The caller should hold the same queue lock held when
976 	 * aio_clear_cancel_function() was called and set this flag
977 	 * ensuring this check sees an up-to-date value.  However,
978 	 * there is no way to assert that.
979 	 */
980 	return ((job->jobflags & KAIOCB_CLEARED) != 0);
981 }
982 
983 static bool
aio_clear_cancel_function_locked(struct kaiocb * job)984 aio_clear_cancel_function_locked(struct kaiocb *job)
985 {
986 
987 	AIO_LOCK_ASSERT(job->userproc->p_aioinfo, MA_OWNED);
988 	MPASS(job->cancel_fn != NULL);
989 	if (job->jobflags & KAIOCB_CANCELLING) {
990 		job->jobflags |= KAIOCB_CLEARED;
991 		return (false);
992 	}
993 	job->cancel_fn = NULL;
994 	return (true);
995 }
996 
997 bool
aio_clear_cancel_function(struct kaiocb * job)998 aio_clear_cancel_function(struct kaiocb *job)
999 {
1000 	struct kaioinfo *ki;
1001 	bool ret;
1002 
1003 	ki = job->userproc->p_aioinfo;
1004 	AIO_LOCK(ki);
1005 	ret = aio_clear_cancel_function_locked(job);
1006 	AIO_UNLOCK(ki);
1007 	return (ret);
1008 }
1009 
1010 static bool
aio_set_cancel_function_locked(struct kaiocb * job,aio_cancel_fn_t * func)1011 aio_set_cancel_function_locked(struct kaiocb *job, aio_cancel_fn_t *func)
1012 {
1013 
1014 	AIO_LOCK_ASSERT(job->userproc->p_aioinfo, MA_OWNED);
1015 	if (job->jobflags & KAIOCB_CANCELLED)
1016 		return (false);
1017 	job->cancel_fn = func;
1018 	return (true);
1019 }
1020 
1021 bool
aio_set_cancel_function(struct kaiocb * job,aio_cancel_fn_t * func)1022 aio_set_cancel_function(struct kaiocb *job, aio_cancel_fn_t *func)
1023 {
1024 	struct kaioinfo *ki;
1025 	bool ret;
1026 
1027 	ki = job->userproc->p_aioinfo;
1028 	AIO_LOCK(ki);
1029 	ret = aio_set_cancel_function_locked(job, func);
1030 	AIO_UNLOCK(ki);
1031 	return (ret);
1032 }
1033 
1034 void
aio_complete(struct kaiocb * job,long status,int error)1035 aio_complete(struct kaiocb *job, long status, int error)
1036 {
1037 	struct kaioinfo *ki;
1038 	struct proc *userp;
1039 
1040 	job->uaiocb._aiocb_private.error = error;
1041 	job->uaiocb._aiocb_private.status = status;
1042 
1043 	userp = job->userproc;
1044 	ki = userp->p_aioinfo;
1045 
1046 	AIO_LOCK(ki);
1047 	KASSERT(!(job->jobflags & KAIOCB_FINISHED),
1048 	    ("duplicate aio_complete"));
1049 	job->jobflags |= KAIOCB_FINISHED;
1050 	if ((job->jobflags & (KAIOCB_QUEUEING | KAIOCB_CANCELLING)) == 0) {
1051 		TAILQ_REMOVE(&ki->kaio_jobqueue, job, plist);
1052 		aio_bio_done_notify(userp, job);
1053 	}
1054 	AIO_UNLOCK(ki);
1055 }
1056 
1057 void
aio_cancel(struct kaiocb * job)1058 aio_cancel(struct kaiocb *job)
1059 {
1060 
1061 	aio_complete(job, -1, ECANCELED);
1062 }
1063 
1064 void
aio_switch_vmspace(struct kaiocb * job)1065 aio_switch_vmspace(struct kaiocb *job)
1066 {
1067 
1068 	vmspace_switch_aio(job->userproc->p_vmspace);
1069 }
1070 
1071 /*
1072  * The AIO daemon, most of the actual work is done in aio_process_*,
1073  * but the setup (and address space mgmt) is done in this routine.
1074  */
1075 static void
aio_daemon(void * _id)1076 aio_daemon(void *_id)
1077 {
1078 	struct kaiocb *job;
1079 	struct aioproc *aiop;
1080 	struct kaioinfo *ki;
1081 	struct proc *p;
1082 	struct vmspace *myvm;
1083 	struct thread *td = curthread;
1084 	int id = (intptr_t)_id;
1085 
1086 	/*
1087 	 * Grab an extra reference on the daemon's vmspace so that it
1088 	 * doesn't get freed by jobs that switch to a different
1089 	 * vmspace.
1090 	 */
1091 	p = td->td_proc;
1092 	myvm = vmspace_acquire_ref(p);
1093 
1094 	KASSERT(p->p_textvp == NULL, ("kthread has a textvp"));
1095 
1096 	/*
1097 	 * Allocate and ready the aio control info.  There is one aiop structure
1098 	 * per daemon.
1099 	 */
1100 	aiop = malloc(sizeof(*aiop), M_AIO, M_WAITOK);
1101 	aiop->aioproc = p;
1102 	aiop->aioprocflags = 0;
1103 
1104 	/*
1105 	 * Wakeup parent process.  (Parent sleeps to keep from blasting away
1106 	 * and creating too many daemons.)
1107 	 */
1108 	sema_post(&aio_newproc_sem);
1109 
1110 	mtx_lock(&aio_job_mtx);
1111 	for (;;) {
1112 		/*
1113 		 * Take daemon off of free queue
1114 		 */
1115 		if (aiop->aioprocflags & AIOP_FREE) {
1116 			TAILQ_REMOVE(&aio_freeproc, aiop, list);
1117 			aiop->aioprocflags &= ~AIOP_FREE;
1118 		}
1119 
1120 		/*
1121 		 * Check for jobs.
1122 		 */
1123 		while ((job = aio_selectjob(aiop)) != NULL) {
1124 			mtx_unlock(&aio_job_mtx);
1125 
1126 			ki = job->userproc->p_aioinfo;
1127 			job->handle_fn(job);
1128 
1129 			mtx_lock(&aio_job_mtx);
1130 			/* Decrement the active job count. */
1131 			ki->kaio_active_count--;
1132 		}
1133 
1134 		/*
1135 		 * Disconnect from user address space.
1136 		 */
1137 		if (p->p_vmspace != myvm) {
1138 			mtx_unlock(&aio_job_mtx);
1139 			vmspace_switch_aio(myvm);
1140 			mtx_lock(&aio_job_mtx);
1141 			/*
1142 			 * We have to restart to avoid race, we only sleep if
1143 			 * no job can be selected.
1144 			 */
1145 			continue;
1146 		}
1147 
1148 		mtx_assert(&aio_job_mtx, MA_OWNED);
1149 
1150 		TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list);
1151 		aiop->aioprocflags |= AIOP_FREE;
1152 
1153 		/*
1154 		 * If daemon is inactive for a long time, allow it to exit,
1155 		 * thereby freeing resources.
1156 		 */
1157 		if (msleep(p, &aio_job_mtx, PRIBIO, "aiordy",
1158 		    aiod_lifetime) == EWOULDBLOCK && TAILQ_EMPTY(&aio_jobs) &&
1159 		    (aiop->aioprocflags & AIOP_FREE) &&
1160 		    num_aio_procs > target_aio_procs)
1161 			break;
1162 	}
1163 	TAILQ_REMOVE(&aio_freeproc, aiop, list);
1164 	num_aio_procs--;
1165 	mtx_unlock(&aio_job_mtx);
1166 	free(aiop, M_AIO);
1167 	free_unr(aiod_unr, id);
1168 	vmspace_free(myvm);
1169 
1170 	KASSERT(p->p_vmspace == myvm,
1171 	    ("AIOD: bad vmspace for exiting daemon"));
1172 	KASSERT(refcount_load(&myvm->vm_refcnt) > 1,
1173 	    ("AIOD: bad vm refcnt for exiting daemon: %d",
1174 	    refcount_load(&myvm->vm_refcnt)));
1175 	kproc_exit(0);
1176 }
1177 
1178 /*
1179  * Create a new AIO daemon. This is mostly a kernel-thread fork routine. The
1180  * AIO daemon modifies its environment itself.
1181  */
1182 static int
aio_newproc(int * start)1183 aio_newproc(int *start)
1184 {
1185 	int error;
1186 	struct proc *p;
1187 	int id;
1188 
1189 	id = alloc_unr(aiod_unr);
1190 	error = kproc_create(aio_daemon, (void *)(intptr_t)id, &p,
1191 		RFNOWAIT, 0, "aiod%d", id);
1192 	if (error == 0) {
1193 		/*
1194 		 * Wait until daemon is started.
1195 		 */
1196 		sema_wait(&aio_newproc_sem);
1197 		mtx_lock(&aio_job_mtx);
1198 		num_aio_procs++;
1199 		if (start != NULL)
1200 			(*start)--;
1201 		mtx_unlock(&aio_job_mtx);
1202 	} else {
1203 		free_unr(aiod_unr, id);
1204 	}
1205 	return (error);
1206 }
1207 
1208 /*
1209  * Try the high-performance, low-overhead bio method for eligible
1210  * VCHR devices.  This method doesn't use an aio helper thread, and
1211  * thus has very low overhead.
1212  *
1213  * Assumes that the caller, aio_aqueue(), has incremented the file
1214  * structure's reference count, preventing its deallocation for the
1215  * duration of this call.
1216  */
1217 static int
aio_qbio(struct proc * p,struct kaiocb * job)1218 aio_qbio(struct proc *p, struct kaiocb *job)
1219 {
1220 	struct aiocb *cb;
1221 	struct file *fp;
1222 	struct buf *pbuf;
1223 	struct vnode *vp;
1224 	struct cdevsw *csw;
1225 	struct cdev *dev;
1226 	struct kaioinfo *ki;
1227 	struct bio **bios = NULL;
1228 	off_t offset;
1229 	int bio_cmd, error, i, iovcnt, opcode, poff, ref;
1230 	vm_prot_t prot;
1231 	bool use_unmapped;
1232 
1233 	cb = &job->uaiocb;
1234 	fp = job->fd_file;
1235 	opcode = cb->aio_lio_opcode;
1236 
1237 	if (!(opcode == LIO_WRITE || opcode == LIO_WRITEV ||
1238 	    opcode == LIO_READ || opcode == LIO_READV))
1239 		return (-1);
1240 	if (fp == NULL || fp->f_type != DTYPE_VNODE)
1241 		return (-1);
1242 
1243 	vp = fp->f_vnode;
1244 	if (vp->v_type != VCHR)
1245 		return (-1);
1246 	if (vp->v_bufobj.bo_bsize == 0)
1247 		return (-1);
1248 
1249 	bio_cmd = (opcode & LIO_WRITE) ? BIO_WRITE : BIO_READ;
1250 	iovcnt = job->uiop->uio_iovcnt;
1251 	if (iovcnt > max_buf_aio)
1252 		return (-1);
1253 	for (i = 0; i < iovcnt; i++) {
1254 		if (job->uiop->uio_iov[i].iov_len % vp->v_bufobj.bo_bsize != 0)
1255 			return (-1);
1256 		if (job->uiop->uio_iov[i].iov_len > maxphys) {
1257 			error = -1;
1258 			return (-1);
1259 		}
1260 	}
1261 	offset = cb->aio_offset;
1262 
1263 	ref = 0;
1264 	csw = devvn_refthread(vp, &dev, &ref);
1265 	if (csw == NULL)
1266 		return (ENXIO);
1267 
1268 	if ((csw->d_flags & D_DISK) == 0) {
1269 		error = -1;
1270 		goto unref;
1271 	}
1272 	if (job->uiop->uio_resid > dev->si_iosize_max) {
1273 		error = -1;
1274 		goto unref;
1275 	}
1276 
1277 	ki = p->p_aioinfo;
1278 	job->error = 0;
1279 
1280 	use_unmapped = (dev->si_flags & SI_UNMAPPED) && unmapped_buf_allowed;
1281 	if (!use_unmapped) {
1282 		AIO_LOCK(ki);
1283 		if (ki->kaio_buffer_count + iovcnt > max_buf_aio) {
1284 			AIO_UNLOCK(ki);
1285 			error = EAGAIN;
1286 			goto unref;
1287 		}
1288 		ki->kaio_buffer_count += iovcnt;
1289 		AIO_UNLOCK(ki);
1290 	}
1291 
1292 	bios = malloc(sizeof(struct bio *) * iovcnt, M_TEMP, M_WAITOK);
1293 	refcount_init(&job->nbio, iovcnt);
1294 	for (i = 0; i < iovcnt; i++) {
1295 		struct vm_page** pages;
1296 		struct bio *bp;
1297 		void *buf;
1298 		size_t nbytes;
1299 		int npages;
1300 
1301 		buf = job->uiop->uio_iov[i].iov_base;
1302 		nbytes = job->uiop->uio_iov[i].iov_len;
1303 
1304 		bios[i] = g_alloc_bio();
1305 		bp = bios[i];
1306 
1307 		poff = (vm_offset_t)buf & PAGE_MASK;
1308 		if (use_unmapped) {
1309 			pbuf = NULL;
1310 			pages = malloc(sizeof(vm_page_t) * (atop(round_page(
1311 			    nbytes)) + 1), M_TEMP, M_WAITOK | M_ZERO);
1312 		} else {
1313 			pbuf = uma_zalloc(pbuf_zone, M_WAITOK);
1314 			BUF_KERNPROC(pbuf);
1315 			pages = pbuf->b_pages;
1316 		}
1317 
1318 		bp->bio_length = nbytes;
1319 		bp->bio_bcount = nbytes;
1320 		bp->bio_done = aio_biowakeup;
1321 		bp->bio_offset = offset;
1322 		bp->bio_cmd = bio_cmd;
1323 		bp->bio_dev = dev;
1324 		bp->bio_caller1 = job;
1325 		bp->bio_caller2 = pbuf;
1326 
1327 		prot = VM_PROT_READ;
1328 		if (opcode == LIO_READ || opcode == LIO_READV)
1329 			prot |= VM_PROT_WRITE;	/* Less backwards than it looks */
1330 		npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
1331 		    (vm_offset_t)buf, bp->bio_length, prot, pages,
1332 		    atop(maxphys) + 1);
1333 		if (npages < 0) {
1334 			if (pbuf != NULL)
1335 				uma_zfree(pbuf_zone, pbuf);
1336 			else
1337 				free(pages, M_TEMP);
1338 			error = EFAULT;
1339 			g_destroy_bio(bp);
1340 			i--;
1341 			goto destroy_bios;
1342 		}
1343 		if (pbuf != NULL) {
1344 			pmap_qenter((vm_offset_t)pbuf->b_data, pages, npages);
1345 			bp->bio_data = pbuf->b_data + poff;
1346 			pbuf->b_npages = npages;
1347 			atomic_add_int(&num_buf_aio, 1);
1348 		} else {
1349 			bp->bio_ma = pages;
1350 			bp->bio_ma_n = npages;
1351 			bp->bio_ma_offset = poff;
1352 			bp->bio_data = unmapped_buf;
1353 			bp->bio_flags |= BIO_UNMAPPED;
1354 			atomic_add_int(&num_unmapped_aio, 1);
1355 		}
1356 
1357 		offset += nbytes;
1358 	}
1359 
1360 	/* Perform transfer. */
1361 	for (i = 0; i < iovcnt; i++)
1362 		csw->d_strategy(bios[i]);
1363 	free(bios, M_TEMP);
1364 
1365 	dev_relthread(dev, ref);
1366 	return (0);
1367 
1368 destroy_bios:
1369 	for (; i >= 0; i--)
1370 		aio_biocleanup(bios[i]);
1371 	free(bios, M_TEMP);
1372 unref:
1373 	dev_relthread(dev, ref);
1374 	return (error);
1375 }
1376 
1377 #ifdef COMPAT_FREEBSD6
1378 static int
convert_old_sigevent(struct osigevent * osig,struct sigevent * nsig)1379 convert_old_sigevent(struct osigevent *osig, struct sigevent *nsig)
1380 {
1381 
1382 	/*
1383 	 * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
1384 	 * supported by AIO with the old sigevent structure.
1385 	 */
1386 	nsig->sigev_notify = osig->sigev_notify;
1387 	switch (nsig->sigev_notify) {
1388 	case SIGEV_NONE:
1389 		break;
1390 	case SIGEV_SIGNAL:
1391 		nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
1392 		break;
1393 	case SIGEV_KEVENT:
1394 		nsig->sigev_notify_kqueue =
1395 		    osig->__sigev_u.__sigev_notify_kqueue;
1396 		nsig->sigev_value.sival_ptr = osig->sigev_value.sival_ptr;
1397 		break;
1398 	default:
1399 		return (EINVAL);
1400 	}
1401 	return (0);
1402 }
1403 
1404 static int
aiocb_copyin_old_sigevent(struct aiocb * ujob,struct kaiocb * kjob,int type __unused)1405 aiocb_copyin_old_sigevent(struct aiocb *ujob, struct kaiocb *kjob,
1406     int type __unused)
1407 {
1408 	struct oaiocb *ojob;
1409 	struct aiocb *kcb = &kjob->uaiocb;
1410 	int error;
1411 
1412 	bzero(kcb, sizeof(struct aiocb));
1413 	error = copyin(ujob, kcb, sizeof(struct oaiocb));
1414 	if (error)
1415 		return (error);
1416 	/* No need to copyin aio_iov, because it did not exist in FreeBSD 6 */
1417 	ojob = (struct oaiocb *)kcb;
1418 	return (convert_old_sigevent(&ojob->aio_sigevent, &kcb->aio_sigevent));
1419 }
1420 #endif
1421 
1422 static int
aiocb_copyin(struct aiocb * ujob,struct kaiocb * kjob,int type)1423 aiocb_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type)
1424 {
1425 	struct aiocb *kcb = &kjob->uaiocb;
1426 	int error;
1427 
1428 	error = copyin(ujob, kcb, sizeof(struct aiocb));
1429 	if (error)
1430 		return (error);
1431 	if (type == LIO_NOP)
1432 		type = kcb->aio_lio_opcode;
1433 	if (type & LIO_VECTORED) {
1434 		/* malloc a uio and copy in the iovec */
1435 		error = copyinuio(__DEVOLATILE(struct iovec*, kcb->aio_iov),
1436 		    kcb->aio_iovcnt, &kjob->uiop);
1437 	}
1438 
1439 	return (error);
1440 }
1441 
1442 static long
aiocb_fetch_status(struct aiocb * ujob)1443 aiocb_fetch_status(struct aiocb *ujob)
1444 {
1445 
1446 	return (fuword(&ujob->_aiocb_private.status));
1447 }
1448 
1449 static long
aiocb_fetch_error(struct aiocb * ujob)1450 aiocb_fetch_error(struct aiocb *ujob)
1451 {
1452 
1453 	return (fuword(&ujob->_aiocb_private.error));
1454 }
1455 
1456 static int
aiocb_store_status(struct aiocb * ujob,long status)1457 aiocb_store_status(struct aiocb *ujob, long status)
1458 {
1459 
1460 	return (suword(&ujob->_aiocb_private.status, status));
1461 }
1462 
1463 static int
aiocb_store_error(struct aiocb * ujob,long error)1464 aiocb_store_error(struct aiocb *ujob, long error)
1465 {
1466 
1467 	return (suword(&ujob->_aiocb_private.error, error));
1468 }
1469 
1470 static int
aiocb_store_aiocb(struct aiocb ** ujobp,struct aiocb * ujob)1471 aiocb_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
1472 {
1473 
1474 	return (suword(ujobp, (long)ujob));
1475 }
1476 
1477 static struct aiocb_ops aiocb_ops = {
1478 	.aio_copyin = aiocb_copyin,
1479 	.fetch_status = aiocb_fetch_status,
1480 	.fetch_error = aiocb_fetch_error,
1481 	.store_status = aiocb_store_status,
1482 	.store_error = aiocb_store_error,
1483 	.store_aiocb = aiocb_store_aiocb,
1484 };
1485 
1486 #ifdef COMPAT_FREEBSD6
1487 static struct aiocb_ops aiocb_ops_osigevent = {
1488 	.aio_copyin = aiocb_copyin_old_sigevent,
1489 	.fetch_status = aiocb_fetch_status,
1490 	.fetch_error = aiocb_fetch_error,
1491 	.store_status = aiocb_store_status,
1492 	.store_error = aiocb_store_error,
1493 	.store_aiocb = aiocb_store_aiocb,
1494 };
1495 #endif
1496 
1497 /*
1498  * Queue a new AIO request.  Choosing either the threaded or direct bio VCHR
1499  * technique is done in this code.
1500  */
1501 static int
aio_aqueue(struct thread * td,struct aiocb * ujob,struct aioliojob * lj,int type,struct aiocb_ops * ops)1502 aio_aqueue(struct thread *td, struct aiocb *ujob, struct aioliojob *lj,
1503     int type, struct aiocb_ops *ops)
1504 {
1505 	struct proc *p = td->td_proc;
1506 	struct file *fp = NULL;
1507 	struct kaiocb *job;
1508 	struct kaioinfo *ki;
1509 	struct kevent kev;
1510 	int opcode;
1511 	int error;
1512 	int fd, kqfd;
1513 	u_short evflags;
1514 
1515 	if (p->p_aioinfo == NULL) {
1516 		error = aio_init_aioinfo(p);
1517 		if (error != 0)
1518 			goto err1;
1519 	}
1520 
1521 	ki = p->p_aioinfo;
1522 
1523 	ops->store_status(ujob, -1);
1524 	ops->store_error(ujob, 0);
1525 
1526 	if (num_queue_count >= max_queue_count ||
1527 	    ki->kaio_count >= max_aio_queue_per_proc) {
1528 		error = EAGAIN;
1529 		goto err1;
1530 	}
1531 
1532 	job = uma_zalloc(aiocb_zone, M_WAITOK | M_ZERO);
1533 	knlist_init_mtx(&job->klist, AIO_MTX(ki));
1534 
1535 	error = ops->aio_copyin(ujob, job, type);
1536 	if (error)
1537 		goto err2;
1538 
1539 	if (job->uaiocb.aio_nbytes > IOSIZE_MAX) {
1540 		error = EINVAL;
1541 		goto err2;
1542 	}
1543 
1544 	if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT &&
1545 	    job->uaiocb.aio_sigevent.sigev_notify != SIGEV_SIGNAL &&
1546 	    job->uaiocb.aio_sigevent.sigev_notify != SIGEV_THREAD_ID &&
1547 	    job->uaiocb.aio_sigevent.sigev_notify != SIGEV_NONE) {
1548 		error = EINVAL;
1549 		goto err2;
1550 	}
1551 
1552 	if ((job->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL ||
1553 	     job->uaiocb.aio_sigevent.sigev_notify == SIGEV_THREAD_ID) &&
1554 		!_SIG_VALID(job->uaiocb.aio_sigevent.sigev_signo)) {
1555 		error = EINVAL;
1556 		goto err2;
1557 	}
1558 
1559 	/* Get the opcode. */
1560 	if (type == LIO_NOP) {
1561 		switch (job->uaiocb.aio_lio_opcode & ~LIO_FOFFSET) {
1562 		case LIO_WRITE:
1563 		case LIO_WRITEV:
1564 		case LIO_NOP:
1565 		case LIO_READ:
1566 		case LIO_READV:
1567 			opcode = job->uaiocb.aio_lio_opcode & ~LIO_FOFFSET;
1568 			if ((job->uaiocb.aio_lio_opcode & LIO_FOFFSET) != 0)
1569 				job->ioflags |= KAIOCB_IO_FOFFSET;
1570 			break;
1571 		default:
1572 			error = EINVAL;
1573 			goto err2;
1574 		}
1575 	} else
1576 		opcode = job->uaiocb.aio_lio_opcode = type;
1577 
1578 	ksiginfo_init(&job->ksi);
1579 
1580 	/* Save userspace address of the job info. */
1581 	job->ujob = ujob;
1582 
1583 	/*
1584 	 * Validate the opcode and fetch the file object for the specified
1585 	 * file descriptor.
1586 	 *
1587 	 * XXXRW: Moved the opcode validation up here so that we don't
1588 	 * retrieve a file descriptor without knowing what the capabiltity
1589 	 * should be.
1590 	 */
1591 	fd = job->uaiocb.aio_fildes;
1592 	switch (opcode) {
1593 	case LIO_WRITE:
1594 	case LIO_WRITEV:
1595 		error = fget_write(td, fd, &cap_pwrite_rights, &fp);
1596 		break;
1597 	case LIO_READ:
1598 	case LIO_READV:
1599 		error = fget_read(td, fd, &cap_pread_rights, &fp);
1600 		break;
1601 	case LIO_SYNC:
1602 	case LIO_DSYNC:
1603 		error = fget(td, fd, &cap_fsync_rights, &fp);
1604 		break;
1605 	case LIO_MLOCK:
1606 		break;
1607 	case LIO_NOP:
1608 		error = fget(td, fd, &cap_no_rights, &fp);
1609 		break;
1610 	default:
1611 		error = EINVAL;
1612 	}
1613 	if (error)
1614 		goto err3;
1615 
1616 	if ((opcode & LIO_SYNC) && fp->f_vnode == NULL) {
1617 		error = EINVAL;
1618 		goto err3;
1619 	}
1620 
1621 	if ((opcode == LIO_READ || opcode == LIO_READV ||
1622 	    opcode == LIO_WRITE || opcode == LIO_WRITEV) &&
1623 	    job->uaiocb.aio_offset < 0 &&
1624 	    (fp->f_vnode == NULL || fp->f_vnode->v_type != VCHR)) {
1625 		error = EINVAL;
1626 		goto err3;
1627 	}
1628 
1629 	if (fp != NULL && fp->f_ops == &path_fileops) {
1630 		error = EBADF;
1631 		goto err3;
1632 	}
1633 
1634 	job->fd_file = fp;
1635 
1636 	mtx_lock(&aio_job_mtx);
1637 	job->seqno = jobseqno++;
1638 	mtx_unlock(&aio_job_mtx);
1639 	if (opcode == LIO_NOP) {
1640 		fdrop(fp, td);
1641 		MPASS(job->uiop == &job->uio || job->uiop == NULL);
1642 		uma_zfree(aiocb_zone, job);
1643 		return (0);
1644 	}
1645 
1646 	if (job->uaiocb.aio_sigevent.sigev_notify != SIGEV_KEVENT)
1647 		goto no_kqueue;
1648 	evflags = job->uaiocb.aio_sigevent.sigev_notify_kevent_flags;
1649 	if ((evflags & ~(EV_CLEAR | EV_DISPATCH | EV_ONESHOT)) != 0) {
1650 		error = EINVAL;
1651 		goto err3;
1652 	}
1653 	kqfd = job->uaiocb.aio_sigevent.sigev_notify_kqueue;
1654 	memset(&kev, 0, sizeof(kev));
1655 	kev.ident = (uintptr_t)job->ujob;
1656 	kev.filter = EVFILT_AIO;
1657 	kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1 | evflags;
1658 	kev.data = (intptr_t)job;
1659 	kev.udata = job->uaiocb.aio_sigevent.sigev_value.sival_ptr;
1660 	error = kqfd_register(kqfd, &kev, td, M_WAITOK);
1661 	if (error)
1662 		goto err3;
1663 
1664 no_kqueue:
1665 
1666 	ops->store_error(ujob, EINPROGRESS);
1667 	job->uaiocb._aiocb_private.error = EINPROGRESS;
1668 	job->userproc = p;
1669 	job->cred = crhold(td->td_ucred);
1670 	job->jobflags = KAIOCB_QUEUEING;
1671 	job->lio = lj;
1672 
1673 	if (opcode & LIO_VECTORED) {
1674 		/* Use the uio copied in by aio_copyin */
1675 		MPASS(job->uiop != &job->uio && job->uiop != NULL);
1676 	} else {
1677 		/* Setup the inline uio */
1678 		job->iov[0].iov_base = (void *)(uintptr_t)job->uaiocb.aio_buf;
1679 		job->iov[0].iov_len = job->uaiocb.aio_nbytes;
1680 		job->uio.uio_iov = job->iov;
1681 		job->uio.uio_iovcnt = 1;
1682 		job->uio.uio_resid = job->uaiocb.aio_nbytes;
1683 		job->uio.uio_segflg = UIO_USERSPACE;
1684 		job->uiop = &job->uio;
1685 	}
1686 	switch (opcode & (LIO_READ | LIO_WRITE)) {
1687 	case LIO_READ:
1688 		job->uiop->uio_rw = UIO_READ;
1689 		break;
1690 	case LIO_WRITE:
1691 		job->uiop->uio_rw = UIO_WRITE;
1692 		break;
1693 	}
1694 	job->uiop->uio_offset = job->uaiocb.aio_offset;
1695 	job->uiop->uio_td = td;
1696 
1697 	if (opcode == LIO_MLOCK) {
1698 		aio_schedule(job, aio_process_mlock);
1699 		error = 0;
1700 	} else if (fp->f_ops->fo_aio_queue == NULL)
1701 		error = aio_queue_file(fp, job);
1702 	else
1703 		error = fo_aio_queue(fp, job);
1704 	if (error)
1705 		goto err4;
1706 
1707 	AIO_LOCK(ki);
1708 	job->jobflags &= ~KAIOCB_QUEUEING;
1709 	TAILQ_INSERT_TAIL(&ki->kaio_all, job, allist);
1710 	ki->kaio_count++;
1711 	if (lj)
1712 		lj->lioj_count++;
1713 	atomic_add_int(&num_queue_count, 1);
1714 	if (job->jobflags & KAIOCB_FINISHED) {
1715 		/*
1716 		 * The queue callback completed the request synchronously.
1717 		 * The bulk of the completion is deferred in that case
1718 		 * until this point.
1719 		 */
1720 		aio_bio_done_notify(p, job);
1721 	} else
1722 		TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, job, plist);
1723 	AIO_UNLOCK(ki);
1724 	return (0);
1725 
1726 err4:
1727 	crfree(job->cred);
1728 err3:
1729 	if (fp)
1730 		fdrop(fp, td);
1731 	knlist_delete(&job->klist, curthread, 0);
1732 err2:
1733 	if (job->uiop != &job->uio)
1734 		freeuio(job->uiop);
1735 	uma_zfree(aiocb_zone, job);
1736 err1:
1737 	ops->store_error(ujob, error);
1738 	return (error);
1739 }
1740 
1741 static void
aio_cancel_daemon_job(struct kaiocb * job)1742 aio_cancel_daemon_job(struct kaiocb *job)
1743 {
1744 
1745 	mtx_lock(&aio_job_mtx);
1746 	if (!aio_cancel_cleared(job))
1747 		TAILQ_REMOVE(&aio_jobs, job, list);
1748 	mtx_unlock(&aio_job_mtx);
1749 	aio_cancel(job);
1750 }
1751 
1752 void
aio_schedule(struct kaiocb * job,aio_handle_fn_t * func)1753 aio_schedule(struct kaiocb *job, aio_handle_fn_t *func)
1754 {
1755 
1756 	mtx_lock(&aio_job_mtx);
1757 	if (!aio_set_cancel_function(job, aio_cancel_daemon_job)) {
1758 		mtx_unlock(&aio_job_mtx);
1759 		aio_cancel(job);
1760 		return;
1761 	}
1762 	job->handle_fn = func;
1763 	TAILQ_INSERT_TAIL(&aio_jobs, job, list);
1764 	aio_kick_nowait(job->userproc);
1765 	mtx_unlock(&aio_job_mtx);
1766 }
1767 
1768 static void
aio_cancel_sync(struct kaiocb * job)1769 aio_cancel_sync(struct kaiocb *job)
1770 {
1771 	struct kaioinfo *ki;
1772 
1773 	ki = job->userproc->p_aioinfo;
1774 	AIO_LOCK(ki);
1775 	if (!aio_cancel_cleared(job))
1776 		TAILQ_REMOVE(&ki->kaio_syncqueue, job, list);
1777 	AIO_UNLOCK(ki);
1778 	aio_cancel(job);
1779 }
1780 
1781 int
aio_queue_file(struct file * fp,struct kaiocb * job)1782 aio_queue_file(struct file *fp, struct kaiocb *job)
1783 {
1784 	struct kaioinfo *ki;
1785 	struct kaiocb *job2;
1786 	struct vnode *vp;
1787 	struct mount *mp;
1788 	int error;
1789 	bool safe;
1790 
1791 	ki = job->userproc->p_aioinfo;
1792 	error = aio_qbio(job->userproc, job);
1793 	if (error >= 0)
1794 		return (error);
1795 	safe = false;
1796 	if (fp->f_type == DTYPE_VNODE) {
1797 		vp = fp->f_vnode;
1798 		if (vp->v_type == VREG || vp->v_type == VDIR) {
1799 			mp = fp->f_vnode->v_mount;
1800 			if (mp == NULL || (mp->mnt_flag & MNT_LOCAL) != 0)
1801 				safe = true;
1802 		}
1803 	}
1804 	if (!(safe || enable_aio_unsafe)) {
1805 		counted_warning(&unsafe_warningcnt,
1806 		    "is attempting to use unsafe AIO requests");
1807 		return (EOPNOTSUPP);
1808 	}
1809 
1810 	if (job->uaiocb.aio_lio_opcode & (LIO_WRITE | LIO_READ)) {
1811 		aio_schedule(job, aio_process_rw);
1812 		error = 0;
1813 	} else if (job->uaiocb.aio_lio_opcode & LIO_SYNC) {
1814 		AIO_LOCK(ki);
1815 		TAILQ_FOREACH(job2, &ki->kaio_jobqueue, plist) {
1816 			if ((job2->jobflags & KAIOCB_MARKER) != 0)
1817 				continue;
1818 			if (job2->fd_file == job->fd_file &&
1819 			    ((job2->uaiocb.aio_lio_opcode & LIO_SYNC) == 0) &&
1820 			    job2->seqno < job->seqno) {
1821 				job2->jobflags |= KAIOCB_CHECKSYNC;
1822 				job->pending++;
1823 			}
1824 		}
1825 		if (job->pending != 0) {
1826 			if (!aio_set_cancel_function_locked(job,
1827 				aio_cancel_sync)) {
1828 				AIO_UNLOCK(ki);
1829 				aio_cancel(job);
1830 				return (0);
1831 			}
1832 			TAILQ_INSERT_TAIL(&ki->kaio_syncqueue, job, list);
1833 			AIO_UNLOCK(ki);
1834 			return (0);
1835 		}
1836 		AIO_UNLOCK(ki);
1837 		aio_schedule(job, aio_process_sync);
1838 		error = 0;
1839 	} else {
1840 		error = EINVAL;
1841 	}
1842 	return (error);
1843 }
1844 
1845 static void
aio_kick_nowait(struct proc * userp)1846 aio_kick_nowait(struct proc *userp)
1847 {
1848 	struct kaioinfo *ki = userp->p_aioinfo;
1849 	struct aioproc *aiop;
1850 
1851 	mtx_assert(&aio_job_mtx, MA_OWNED);
1852 	if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
1853 		TAILQ_REMOVE(&aio_freeproc, aiop, list);
1854 		aiop->aioprocflags &= ~AIOP_FREE;
1855 		wakeup(aiop->aioproc);
1856 	} else if (num_aio_resv_start + num_aio_procs < max_aio_procs &&
1857 	    ki->kaio_active_count + num_aio_resv_start < max_aio_per_proc) {
1858 		taskqueue_enqueue(taskqueue_aiod_kick, &ki->kaio_task);
1859 	}
1860 }
1861 
1862 static int
aio_kick(struct proc * userp)1863 aio_kick(struct proc *userp)
1864 {
1865 	struct kaioinfo *ki = userp->p_aioinfo;
1866 	struct aioproc *aiop;
1867 	int error, ret = 0;
1868 
1869 	mtx_assert(&aio_job_mtx, MA_OWNED);
1870 retryproc:
1871 	if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) {
1872 		TAILQ_REMOVE(&aio_freeproc, aiop, list);
1873 		aiop->aioprocflags &= ~AIOP_FREE;
1874 		wakeup(aiop->aioproc);
1875 	} else if (num_aio_resv_start + num_aio_procs < max_aio_procs &&
1876 	    ki->kaio_active_count + num_aio_resv_start < max_aio_per_proc) {
1877 		num_aio_resv_start++;
1878 		mtx_unlock(&aio_job_mtx);
1879 		error = aio_newproc(&num_aio_resv_start);
1880 		mtx_lock(&aio_job_mtx);
1881 		if (error) {
1882 			num_aio_resv_start--;
1883 			goto retryproc;
1884 		}
1885 	} else {
1886 		ret = -1;
1887 	}
1888 	return (ret);
1889 }
1890 
1891 static void
aio_kick_helper(void * context,int pending)1892 aio_kick_helper(void *context, int pending)
1893 {
1894 	struct proc *userp = context;
1895 
1896 	mtx_lock(&aio_job_mtx);
1897 	while (--pending >= 0) {
1898 		if (aio_kick(userp))
1899 			break;
1900 	}
1901 	mtx_unlock(&aio_job_mtx);
1902 }
1903 
1904 /*
1905  * Support the aio_return system call, as a side-effect, kernel resources are
1906  * released.
1907  */
1908 static int
kern_aio_return(struct thread * td,struct aiocb * ujob,struct aiocb_ops * ops)1909 kern_aio_return(struct thread *td, struct aiocb *ujob, struct aiocb_ops *ops)
1910 {
1911 	struct proc *p = td->td_proc;
1912 	struct kaiocb *job;
1913 	struct kaioinfo *ki;
1914 	long status, error;
1915 
1916 	ki = p->p_aioinfo;
1917 	if (ki == NULL)
1918 		return (EINVAL);
1919 	AIO_LOCK(ki);
1920 	TAILQ_FOREACH(job, &ki->kaio_done, plist) {
1921 		if (job->ujob == ujob)
1922 			break;
1923 	}
1924 	if (job != NULL) {
1925 		MPASS(job->jobflags & KAIOCB_FINISHED);
1926 		status = job->uaiocb._aiocb_private.status;
1927 		error = job->uaiocb._aiocb_private.error;
1928 		td->td_retval[0] = status;
1929 		td->td_ru.ru_oublock += job->outblock;
1930 		td->td_ru.ru_inblock += job->inblock;
1931 		td->td_ru.ru_msgsnd += job->msgsnd;
1932 		td->td_ru.ru_msgrcv += job->msgrcv;
1933 		aio_free_entry(job);
1934 		AIO_UNLOCK(ki);
1935 		ops->store_error(ujob, error);
1936 		ops->store_status(ujob, status);
1937 	} else {
1938 		error = EINVAL;
1939 		AIO_UNLOCK(ki);
1940 	}
1941 	return (error);
1942 }
1943 
1944 int
sys_aio_return(struct thread * td,struct aio_return_args * uap)1945 sys_aio_return(struct thread *td, struct aio_return_args *uap)
1946 {
1947 
1948 	return (kern_aio_return(td, uap->aiocbp, &aiocb_ops));
1949 }
1950 
1951 /*
1952  * Allow a process to wakeup when any of the I/O requests are completed.
1953  */
1954 static int
kern_aio_suspend(struct thread * td,int njoblist,struct aiocb ** ujoblist,struct timespec * ts)1955 kern_aio_suspend(struct thread *td, int njoblist, struct aiocb **ujoblist,
1956     struct timespec *ts)
1957 {
1958 	struct proc *p = td->td_proc;
1959 	struct timeval atv;
1960 	struct kaioinfo *ki;
1961 	struct kaiocb *firstjob, *job;
1962 	int error, i, timo;
1963 
1964 	timo = 0;
1965 	if (ts) {
1966 		if (ts->tv_nsec < 0 || ts->tv_nsec >= 1000000000)
1967 			return (EINVAL);
1968 
1969 		TIMESPEC_TO_TIMEVAL(&atv, ts);
1970 		if (itimerfix(&atv))
1971 			return (EINVAL);
1972 		timo = tvtohz(&atv);
1973 	}
1974 
1975 	ki = p->p_aioinfo;
1976 	if (ki == NULL)
1977 		return (EAGAIN);
1978 
1979 	if (njoblist == 0)
1980 		return (0);
1981 
1982 	AIO_LOCK(ki);
1983 	for (;;) {
1984 		firstjob = NULL;
1985 		error = 0;
1986 		TAILQ_FOREACH(job, &ki->kaio_all, allist) {
1987 			for (i = 0; i < njoblist; i++) {
1988 				if (job->ujob == ujoblist[i]) {
1989 					if (firstjob == NULL)
1990 						firstjob = job;
1991 					if (job->jobflags & KAIOCB_FINISHED)
1992 						goto RETURN;
1993 				}
1994 			}
1995 		}
1996 		/* All tasks were finished. */
1997 		if (firstjob == NULL)
1998 			break;
1999 
2000 		ki->kaio_flags |= KAIO_WAKEUP;
2001 		error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH,
2002 		    "aiospn", timo);
2003 		if (error == ERESTART)
2004 			error = EINTR;
2005 		if (error)
2006 			break;
2007 	}
2008 RETURN:
2009 	AIO_UNLOCK(ki);
2010 	return (error);
2011 }
2012 
2013 int
sys_aio_suspend(struct thread * td,struct aio_suspend_args * uap)2014 sys_aio_suspend(struct thread *td, struct aio_suspend_args *uap)
2015 {
2016 	struct timespec ts, *tsp;
2017 	struct aiocb **ujoblist;
2018 	int error;
2019 
2020 	if (uap->nent < 0 || uap->nent > max_aio_queue_per_proc)
2021 		return (EINVAL);
2022 
2023 	if (uap->timeout) {
2024 		/* Get timespec struct. */
2025 		if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0)
2026 			return (error);
2027 		tsp = &ts;
2028 	} else
2029 		tsp = NULL;
2030 
2031 	ujoblist = malloc(uap->nent * sizeof(ujoblist[0]), M_AIO, M_WAITOK);
2032 	error = copyin(uap->aiocbp, ujoblist, uap->nent * sizeof(ujoblist[0]));
2033 	if (error == 0)
2034 		error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
2035 	free(ujoblist, M_AIO);
2036 	return (error);
2037 }
2038 
2039 /*
2040  * aio_cancel cancels any non-bio aio operations not currently in progress.
2041  */
2042 int
sys_aio_cancel(struct thread * td,struct aio_cancel_args * uap)2043 sys_aio_cancel(struct thread *td, struct aio_cancel_args *uap)
2044 {
2045 	struct proc *p = td->td_proc;
2046 	struct kaioinfo *ki;
2047 	struct kaiocb *job, *jobn, marker;
2048 	struct file *fp;
2049 	int error;
2050 	int cancelled = 0;
2051 	int notcancelled = 0;
2052 	struct vnode *vp;
2053 
2054 	/* Lookup file object. */
2055 	error = fget(td, uap->fd, &cap_no_rights, &fp);
2056 	if (error)
2057 		return (error);
2058 
2059 	ki = p->p_aioinfo;
2060 	if (ki == NULL)
2061 		goto done;
2062 
2063 	if (fp->f_type == DTYPE_VNODE) {
2064 		vp = fp->f_vnode;
2065 		if (vn_isdisk(vp)) {
2066 			fdrop(fp, td);
2067 			td->td_retval[0] = AIO_NOTCANCELED;
2068 			return (0);
2069 		}
2070 	}
2071 
2072 	/*
2073 	 * We may have to drop the list mutex in order to cancel a job.  After
2074 	 * that point it is unsafe to rely on the stability of the list.  We
2075 	 * could restart the search from the beginning after canceling a job,
2076 	 * but this may inefficient.  Instead, use a marker job to keep our
2077 	 * place in the list.
2078 	 */
2079 	memset(&marker, 0, sizeof(marker));
2080 	marker.jobflags = KAIOCB_MARKER;
2081 
2082 	AIO_LOCK(ki);
2083 	TAILQ_FOREACH_SAFE(job, &ki->kaio_jobqueue, plist, jobn) {
2084 		if (uap->fd == job->uaiocb.aio_fildes &&
2085 		    (uap->aiocbp == NULL || uap->aiocbp == job->ujob) &&
2086 		    (job->jobflags & KAIOCB_MARKER) == 0) {
2087 			TAILQ_INSERT_AFTER(&ki->kaio_jobqueue, job, &marker,
2088 			    plist);
2089 			if (aio_cancel_job(p, ki, job)) {
2090 				cancelled++;
2091 			} else {
2092 				notcancelled++;
2093 			}
2094 			jobn = TAILQ_NEXT(&marker, plist);
2095 			TAILQ_REMOVE(&ki->kaio_jobqueue, &marker, plist);
2096 			if (uap->aiocbp != NULL)
2097 				break;
2098 		}
2099 	}
2100 	AIO_UNLOCK(ki);
2101 
2102 done:
2103 	fdrop(fp, td);
2104 
2105 	if (uap->aiocbp != NULL) {
2106 		if (cancelled) {
2107 			td->td_retval[0] = AIO_CANCELED;
2108 			return (0);
2109 		}
2110 	}
2111 
2112 	if (notcancelled) {
2113 		td->td_retval[0] = AIO_NOTCANCELED;
2114 		return (0);
2115 	}
2116 
2117 	if (cancelled) {
2118 		td->td_retval[0] = AIO_CANCELED;
2119 		return (0);
2120 	}
2121 
2122 	td->td_retval[0] = AIO_ALLDONE;
2123 
2124 	return (0);
2125 }
2126 
2127 /*
2128  * aio_error is implemented in the kernel level for compatibility purposes
2129  * only.  For a user mode async implementation, it would be best to do it in
2130  * a userland subroutine.
2131  */
2132 static int
kern_aio_error(struct thread * td,struct aiocb * ujob,struct aiocb_ops * ops)2133 kern_aio_error(struct thread *td, struct aiocb *ujob, struct aiocb_ops *ops)
2134 {
2135 	struct proc *p = td->td_proc;
2136 	struct kaiocb *job;
2137 	struct kaioinfo *ki;
2138 	int status;
2139 
2140 	ki = p->p_aioinfo;
2141 	if (ki == NULL) {
2142 		td->td_retval[0] = EINVAL;
2143 		return (0);
2144 	}
2145 
2146 	AIO_LOCK(ki);
2147 	TAILQ_FOREACH(job, &ki->kaio_all, allist) {
2148 		if (job->ujob == ujob) {
2149 			if (job->jobflags & KAIOCB_FINISHED)
2150 				td->td_retval[0] =
2151 					job->uaiocb._aiocb_private.error;
2152 			else
2153 				td->td_retval[0] = EINPROGRESS;
2154 			AIO_UNLOCK(ki);
2155 			return (0);
2156 		}
2157 	}
2158 	AIO_UNLOCK(ki);
2159 
2160 	/*
2161 	 * Hack for failure of aio_aqueue.
2162 	 */
2163 	status = ops->fetch_status(ujob);
2164 	if (status == -1) {
2165 		td->td_retval[0] = ops->fetch_error(ujob);
2166 		return (0);
2167 	}
2168 
2169 	td->td_retval[0] = EINVAL;
2170 	return (0);
2171 }
2172 
2173 int
sys_aio_error(struct thread * td,struct aio_error_args * uap)2174 sys_aio_error(struct thread *td, struct aio_error_args *uap)
2175 {
2176 
2177 	return (kern_aio_error(td, uap->aiocbp, &aiocb_ops));
2178 }
2179 
2180 /* syscall - asynchronous read from a file (REALTIME) */
2181 #ifdef COMPAT_FREEBSD6
2182 int
freebsd6_aio_read(struct thread * td,struct freebsd6_aio_read_args * uap)2183 freebsd6_aio_read(struct thread *td, struct freebsd6_aio_read_args *uap)
2184 {
2185 
2186 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
2187 	    &aiocb_ops_osigevent));
2188 }
2189 #endif
2190 
2191 int
sys_aio_read(struct thread * td,struct aio_read_args * uap)2192 sys_aio_read(struct thread *td, struct aio_read_args *uap)
2193 {
2194 
2195 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READ, &aiocb_ops));
2196 }
2197 
2198 int
sys_aio_readv(struct thread * td,struct aio_readv_args * uap)2199 sys_aio_readv(struct thread *td, struct aio_readv_args *uap)
2200 {
2201 
2202 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_READV, &aiocb_ops));
2203 }
2204 
2205 /* syscall - asynchronous write to a file (REALTIME) */
2206 #ifdef COMPAT_FREEBSD6
2207 int
freebsd6_aio_write(struct thread * td,struct freebsd6_aio_write_args * uap)2208 freebsd6_aio_write(struct thread *td, struct freebsd6_aio_write_args *uap)
2209 {
2210 
2211 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
2212 	    &aiocb_ops_osigevent));
2213 }
2214 #endif
2215 
2216 int
sys_aio_write(struct thread * td,struct aio_write_args * uap)2217 sys_aio_write(struct thread *td, struct aio_write_args *uap)
2218 {
2219 
2220 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITE, &aiocb_ops));
2221 }
2222 
2223 int
sys_aio_writev(struct thread * td,struct aio_writev_args * uap)2224 sys_aio_writev(struct thread *td, struct aio_writev_args *uap)
2225 {
2226 
2227 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_WRITEV, &aiocb_ops));
2228 }
2229 
2230 int
sys_aio_mlock(struct thread * td,struct aio_mlock_args * uap)2231 sys_aio_mlock(struct thread *td, struct aio_mlock_args *uap)
2232 {
2233 
2234 	return (aio_aqueue(td, uap->aiocbp, NULL, LIO_MLOCK, &aiocb_ops));
2235 }
2236 
2237 static int
kern_lio_listio(struct thread * td,int mode,struct aiocb * const * uacb_list,struct aiocb ** acb_list,int nent,struct sigevent * sig,struct aiocb_ops * ops)2238 kern_lio_listio(struct thread *td, int mode, struct aiocb * const *uacb_list,
2239     struct aiocb **acb_list, int nent, struct sigevent *sig,
2240     struct aiocb_ops *ops)
2241 {
2242 	struct proc *p = td->td_proc;
2243 	struct aiocb *job;
2244 	struct kaioinfo *ki;
2245 	struct aioliojob *lj;
2246 	struct kevent kev;
2247 	int error;
2248 	int nagain, nerror;
2249 	int i;
2250 
2251 	if ((mode != LIO_NOWAIT) && (mode != LIO_WAIT))
2252 		return (EINVAL);
2253 
2254 	if (nent < 0 || nent > max_aio_queue_per_proc)
2255 		return (EINVAL);
2256 
2257 	if (p->p_aioinfo == NULL) {
2258 		error = aio_init_aioinfo(p);
2259 		if (error != 0)
2260 			return (error);
2261 	}
2262 
2263 	ki = p->p_aioinfo;
2264 
2265 	lj = uma_zalloc(aiolio_zone, M_WAITOK);
2266 	lj->lioj_flags = 0;
2267 	lj->lioj_count = 0;
2268 	lj->lioj_finished_count = 0;
2269 	lj->lioj_signal.sigev_notify = SIGEV_NONE;
2270 	knlist_init_mtx(&lj->klist, AIO_MTX(ki));
2271 	ksiginfo_init(&lj->lioj_ksi);
2272 
2273 	/*
2274 	 * Setup signal.
2275 	 */
2276 	if (sig && (mode == LIO_NOWAIT)) {
2277 		bcopy(sig, &lj->lioj_signal, sizeof(lj->lioj_signal));
2278 		if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
2279 			/* Assume only new style KEVENT */
2280 			memset(&kev, 0, sizeof(kev));
2281 			kev.filter = EVFILT_LIO;
2282 			kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1;
2283 			kev.ident = (uintptr_t)uacb_list; /* something unique */
2284 			kev.data = (intptr_t)lj;
2285 			/* pass user defined sigval data */
2286 			kev.udata = lj->lioj_signal.sigev_value.sival_ptr;
2287 			error = kqfd_register(
2288 			    lj->lioj_signal.sigev_notify_kqueue, &kev, td,
2289 			    M_WAITOK);
2290 			if (error) {
2291 				uma_zfree(aiolio_zone, lj);
2292 				return (error);
2293 			}
2294 		} else if (lj->lioj_signal.sigev_notify == SIGEV_NONE) {
2295 			;
2296 		} else if (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
2297 			   lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID) {
2298 				if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) {
2299 					uma_zfree(aiolio_zone, lj);
2300 					return EINVAL;
2301 				}
2302 				lj->lioj_flags |= LIOJ_SIGNAL;
2303 		} else {
2304 			uma_zfree(aiolio_zone, lj);
2305 			return EINVAL;
2306 		}
2307 	}
2308 
2309 	AIO_LOCK(ki);
2310 	TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list);
2311 	/*
2312 	 * Add extra aiocb count to avoid the lio to be freed
2313 	 * by other threads doing aio_waitcomplete or aio_return,
2314 	 * and prevent event from being sent until we have queued
2315 	 * all tasks.
2316 	 */
2317 	lj->lioj_count = 1;
2318 	AIO_UNLOCK(ki);
2319 
2320 	/*
2321 	 * Get pointers to the list of I/O requests.
2322 	 */
2323 	nagain = 0;
2324 	nerror = 0;
2325 	for (i = 0; i < nent; i++) {
2326 		job = acb_list[i];
2327 		if (job != NULL) {
2328 			error = aio_aqueue(td, job, lj, LIO_NOP, ops);
2329 			if (error == EAGAIN)
2330 				nagain++;
2331 			else if (error != 0)
2332 				nerror++;
2333 		}
2334 	}
2335 
2336 	error = 0;
2337 	AIO_LOCK(ki);
2338 	if (mode == LIO_WAIT) {
2339 		while (lj->lioj_count - 1 != lj->lioj_finished_count) {
2340 			ki->kaio_flags |= KAIO_WAKEUP;
2341 			error = msleep(&p->p_aioinfo, AIO_MTX(ki),
2342 			    PRIBIO | PCATCH, "aiospn", 0);
2343 			if (error == ERESTART)
2344 				error = EINTR;
2345 			if (error)
2346 				break;
2347 		}
2348 	} else {
2349 		if (lj->lioj_count - 1 == lj->lioj_finished_count) {
2350 			if (lj->lioj_signal.sigev_notify == SIGEV_KEVENT) {
2351 				lj->lioj_flags |= LIOJ_KEVENT_POSTED;
2352 				KNOTE_LOCKED(&lj->klist, 1);
2353 			}
2354 			if ((lj->lioj_flags & (LIOJ_SIGNAL |
2355 			    LIOJ_SIGNAL_POSTED)) == LIOJ_SIGNAL &&
2356 			    (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL ||
2357 			    lj->lioj_signal.sigev_notify == SIGEV_THREAD_ID)) {
2358 				aio_sendsig(p, &lj->lioj_signal, &lj->lioj_ksi,
2359 				    lj->lioj_count != 1);
2360 				lj->lioj_flags |= LIOJ_SIGNAL_POSTED;
2361 			}
2362 		}
2363 	}
2364 	lj->lioj_count--;
2365 	if (lj->lioj_count == 0) {
2366 		TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list);
2367 		knlist_delete(&lj->klist, curthread, 1);
2368 		PROC_LOCK(p);
2369 		sigqueue_take(&lj->lioj_ksi);
2370 		PROC_UNLOCK(p);
2371 		AIO_UNLOCK(ki);
2372 		uma_zfree(aiolio_zone, lj);
2373 	} else
2374 		AIO_UNLOCK(ki);
2375 
2376 	if (nerror)
2377 		return (EIO);
2378 	else if (nagain)
2379 		return (EAGAIN);
2380 	else
2381 		return (error);
2382 }
2383 
2384 /* syscall - list directed I/O (REALTIME) */
2385 #ifdef COMPAT_FREEBSD6
2386 int
freebsd6_lio_listio(struct thread * td,struct freebsd6_lio_listio_args * uap)2387 freebsd6_lio_listio(struct thread *td, struct freebsd6_lio_listio_args *uap)
2388 {
2389 	struct aiocb **acb_list;
2390 	struct sigevent *sigp, sig;
2391 	struct osigevent osig;
2392 	int error, nent;
2393 
2394 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
2395 		return (EINVAL);
2396 
2397 	nent = uap->nent;
2398 	if (nent < 0 || nent > max_aio_queue_per_proc)
2399 		return (EINVAL);
2400 
2401 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
2402 		error = copyin(uap->sig, &osig, sizeof(osig));
2403 		if (error)
2404 			return (error);
2405 		error = convert_old_sigevent(&osig, &sig);
2406 		if (error)
2407 			return (error);
2408 		sigp = &sig;
2409 	} else
2410 		sigp = NULL;
2411 
2412 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
2413 	error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
2414 	if (error == 0)
2415 		error = kern_lio_listio(td, uap->mode,
2416 		    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
2417 		    &aiocb_ops_osigevent);
2418 	free(acb_list, M_LIO);
2419 	return (error);
2420 }
2421 #endif
2422 
2423 /* syscall - list directed I/O (REALTIME) */
2424 int
sys_lio_listio(struct thread * td,struct lio_listio_args * uap)2425 sys_lio_listio(struct thread *td, struct lio_listio_args *uap)
2426 {
2427 	struct aiocb **acb_list;
2428 	struct sigevent *sigp, sig;
2429 	int error, nent;
2430 
2431 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
2432 		return (EINVAL);
2433 
2434 	nent = uap->nent;
2435 	if (nent < 0 || nent > max_aio_queue_per_proc)
2436 		return (EINVAL);
2437 
2438 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
2439 		error = copyin(uap->sig, &sig, sizeof(sig));
2440 		if (error)
2441 			return (error);
2442 		sigp = &sig;
2443 	} else
2444 		sigp = NULL;
2445 
2446 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
2447 	error = copyin(uap->acb_list, acb_list, nent * sizeof(acb_list[0]));
2448 	if (error == 0)
2449 		error = kern_lio_listio(td, uap->mode, uap->acb_list, acb_list,
2450 		    nent, sigp, &aiocb_ops);
2451 	free(acb_list, M_LIO);
2452 	return (error);
2453 }
2454 
2455 static void
aio_biocleanup(struct bio * bp)2456 aio_biocleanup(struct bio *bp)
2457 {
2458 	struct kaiocb *job = (struct kaiocb *)bp->bio_caller1;
2459 	struct kaioinfo *ki;
2460 	struct buf *pbuf = (struct buf *)bp->bio_caller2;
2461 
2462 	/* Release mapping into kernel space. */
2463 	if (pbuf != NULL) {
2464 		MPASS(pbuf->b_npages <= atop(maxphys) + 1);
2465 		pmap_qremove((vm_offset_t)pbuf->b_data, pbuf->b_npages);
2466 		vm_page_unhold_pages(pbuf->b_pages, pbuf->b_npages);
2467 		uma_zfree(pbuf_zone, pbuf);
2468 		atomic_subtract_int(&num_buf_aio, 1);
2469 		ki = job->userproc->p_aioinfo;
2470 		AIO_LOCK(ki);
2471 		ki->kaio_buffer_count--;
2472 		AIO_UNLOCK(ki);
2473 	} else {
2474 		MPASS(bp->bio_ma_n <= atop(maxphys) + 1);
2475 		vm_page_unhold_pages(bp->bio_ma, bp->bio_ma_n);
2476 		free(bp->bio_ma, M_TEMP);
2477 		atomic_subtract_int(&num_unmapped_aio, 1);
2478 	}
2479 	g_destroy_bio(bp);
2480 }
2481 
2482 static void
aio_biowakeup(struct bio * bp)2483 aio_biowakeup(struct bio *bp)
2484 {
2485 	struct kaiocb *job = (struct kaiocb *)bp->bio_caller1;
2486 	size_t nbytes;
2487 	long bcount = bp->bio_bcount;
2488 	long resid = bp->bio_resid;
2489 	int opcode, nblks;
2490 	int bio_error = bp->bio_error;
2491 	uint16_t flags = bp->bio_flags;
2492 
2493 	opcode = job->uaiocb.aio_lio_opcode;
2494 
2495 	aio_biocleanup(bp);
2496 
2497 	nbytes = bcount - resid;
2498 	atomic_add_acq_long(&job->nbytes, nbytes);
2499 	nblks = btodb(nbytes);
2500 
2501 	/*
2502 	 * If multiple bios experienced an error, the job will reflect the
2503 	 * error of whichever failed bio completed last.
2504 	 */
2505 	if (flags & BIO_ERROR)
2506 		atomic_store_int(&job->error, bio_error);
2507 	if (opcode & LIO_WRITE)
2508 		atomic_add_int(&job->outblock, nblks);
2509 	else
2510 		atomic_add_int(&job->inblock, nblks);
2511 
2512 	if (refcount_release(&job->nbio)) {
2513 		bio_error = atomic_load_int(&job->error);
2514 		if (bio_error != 0)
2515 			aio_complete(job, -1, bio_error);
2516 		else
2517 			aio_complete(job, atomic_load_long(&job->nbytes), 0);
2518 	}
2519 }
2520 
2521 /* syscall - wait for the next completion of an aio request */
2522 static int
kern_aio_waitcomplete(struct thread * td,struct aiocb ** ujobp,struct timespec * ts,struct aiocb_ops * ops)2523 kern_aio_waitcomplete(struct thread *td, struct aiocb **ujobp,
2524     struct timespec *ts, struct aiocb_ops *ops)
2525 {
2526 	struct proc *p = td->td_proc;
2527 	struct timeval atv;
2528 	struct kaioinfo *ki;
2529 	struct kaiocb *job;
2530 	struct aiocb *ujob;
2531 	long error, status;
2532 	int timo;
2533 
2534 	ops->store_aiocb(ujobp, NULL);
2535 
2536 	if (ts == NULL) {
2537 		timo = 0;
2538 	} else if (ts->tv_sec == 0 && ts->tv_nsec == 0) {
2539 		timo = -1;
2540 	} else {
2541 		if ((ts->tv_nsec < 0) || (ts->tv_nsec >= 1000000000))
2542 			return (EINVAL);
2543 
2544 		TIMESPEC_TO_TIMEVAL(&atv, ts);
2545 		if (itimerfix(&atv))
2546 			return (EINVAL);
2547 		timo = tvtohz(&atv);
2548 	}
2549 
2550 	if (p->p_aioinfo == NULL) {
2551 		error = aio_init_aioinfo(p);
2552 		if (error != 0)
2553 			return (error);
2554 	}
2555 	ki = p->p_aioinfo;
2556 
2557 	error = 0;
2558 	job = NULL;
2559 	AIO_LOCK(ki);
2560 	while ((job = TAILQ_FIRST(&ki->kaio_done)) == NULL) {
2561 		if (timo == -1) {
2562 			error = EWOULDBLOCK;
2563 			break;
2564 		}
2565 		ki->kaio_flags |= KAIO_WAKEUP;
2566 		error = msleep(&p->p_aioinfo, AIO_MTX(ki), PRIBIO | PCATCH,
2567 		    "aiowc", timo);
2568 		if (timo && error == ERESTART)
2569 			error = EINTR;
2570 		if (error)
2571 			break;
2572 	}
2573 
2574 	if (job != NULL) {
2575 		MPASS(job->jobflags & KAIOCB_FINISHED);
2576 		ujob = job->ujob;
2577 		status = job->uaiocb._aiocb_private.status;
2578 		error = job->uaiocb._aiocb_private.error;
2579 		td->td_retval[0] = status;
2580 		td->td_ru.ru_oublock += job->outblock;
2581 		td->td_ru.ru_inblock += job->inblock;
2582 		td->td_ru.ru_msgsnd += job->msgsnd;
2583 		td->td_ru.ru_msgrcv += job->msgrcv;
2584 		aio_free_entry(job);
2585 		AIO_UNLOCK(ki);
2586 		ops->store_aiocb(ujobp, ujob);
2587 		ops->store_error(ujob, error);
2588 		ops->store_status(ujob, status);
2589 	} else
2590 		AIO_UNLOCK(ki);
2591 
2592 	return (error);
2593 }
2594 
2595 int
sys_aio_waitcomplete(struct thread * td,struct aio_waitcomplete_args * uap)2596 sys_aio_waitcomplete(struct thread *td, struct aio_waitcomplete_args *uap)
2597 {
2598 	struct timespec ts, *tsp;
2599 	int error;
2600 
2601 	if (uap->timeout) {
2602 		/* Get timespec struct. */
2603 		error = copyin(uap->timeout, &ts, sizeof(ts));
2604 		if (error)
2605 			return (error);
2606 		tsp = &ts;
2607 	} else
2608 		tsp = NULL;
2609 
2610 	return (kern_aio_waitcomplete(td, uap->aiocbp, tsp, &aiocb_ops));
2611 }
2612 
2613 static int
kern_aio_fsync(struct thread * td,int op,struct aiocb * ujob,struct aiocb_ops * ops)2614 kern_aio_fsync(struct thread *td, int op, struct aiocb *ujob,
2615     struct aiocb_ops *ops)
2616 {
2617 	int listop;
2618 
2619 	switch (op) {
2620 	case O_SYNC:
2621 		listop = LIO_SYNC;
2622 		break;
2623 	case O_DSYNC:
2624 		listop = LIO_DSYNC;
2625 		break;
2626 	default:
2627 		return (EINVAL);
2628 	}
2629 
2630 	return (aio_aqueue(td, ujob, NULL, listop, ops));
2631 }
2632 
2633 int
sys_aio_fsync(struct thread * td,struct aio_fsync_args * uap)2634 sys_aio_fsync(struct thread *td, struct aio_fsync_args *uap)
2635 {
2636 
2637 	return (kern_aio_fsync(td, uap->op, uap->aiocbp, &aiocb_ops));
2638 }
2639 
2640 /* kqueue attach function */
2641 static int
filt_aioattach(struct knote * kn)2642 filt_aioattach(struct knote *kn)
2643 {
2644 	struct kaiocb *job;
2645 
2646 	job = (struct kaiocb *)(uintptr_t)kn->kn_sdata;
2647 
2648 	/*
2649 	 * The job pointer must be validated before using it, so
2650 	 * registration is restricted to the kernel; the user cannot
2651 	 * set EV_FLAG1.
2652 	 */
2653 	if ((kn->kn_flags & EV_FLAG1) == 0)
2654 		return (EPERM);
2655 	kn->kn_ptr.p_aio = job;
2656 	kn->kn_flags &= ~EV_FLAG1;
2657 
2658 	knlist_add(&job->klist, kn, 0);
2659 
2660 	return (0);
2661 }
2662 
2663 /* kqueue detach function */
2664 static void
filt_aiodetach(struct knote * kn)2665 filt_aiodetach(struct knote *kn)
2666 {
2667 	struct knlist *knl;
2668 
2669 	knl = &kn->kn_ptr.p_aio->klist;
2670 	knl->kl_lock(knl->kl_lockarg);
2671 	if (!knlist_empty(knl))
2672 		knlist_remove(knl, kn, 1);
2673 	knl->kl_unlock(knl->kl_lockarg);
2674 }
2675 
2676 /* kqueue filter function */
2677 /*ARGSUSED*/
2678 static int
filt_aio(struct knote * kn,long hint)2679 filt_aio(struct knote *kn, long hint)
2680 {
2681 	struct kaiocb *job = kn->kn_ptr.p_aio;
2682 
2683 	kn->kn_data = job->uaiocb._aiocb_private.error;
2684 	if (!(job->jobflags & KAIOCB_FINISHED))
2685 		return (0);
2686 	kn->kn_flags |= EV_EOF;
2687 	return (1);
2688 }
2689 
2690 /* kqueue attach function */
2691 static int
filt_lioattach(struct knote * kn)2692 filt_lioattach(struct knote *kn)
2693 {
2694 	struct aioliojob *lj;
2695 
2696 	lj = (struct aioliojob *)(uintptr_t)kn->kn_sdata;
2697 
2698 	/*
2699 	 * The aioliojob pointer must be validated before using it, so
2700 	 * registration is restricted to the kernel; the user cannot
2701 	 * set EV_FLAG1.
2702 	 */
2703 	if ((kn->kn_flags & EV_FLAG1) == 0)
2704 		return (EPERM);
2705 	kn->kn_ptr.p_lio = lj;
2706 	kn->kn_flags &= ~EV_FLAG1;
2707 
2708 	knlist_add(&lj->klist, kn, 0);
2709 
2710 	return (0);
2711 }
2712 
2713 /* kqueue detach function */
2714 static void
filt_liodetach(struct knote * kn)2715 filt_liodetach(struct knote *kn)
2716 {
2717 	struct knlist *knl;
2718 
2719 	knl = &kn->kn_ptr.p_lio->klist;
2720 	knl->kl_lock(knl->kl_lockarg);
2721 	if (!knlist_empty(knl))
2722 		knlist_remove(knl, kn, 1);
2723 	knl->kl_unlock(knl->kl_lockarg);
2724 }
2725 
2726 /* kqueue filter function */
2727 /*ARGSUSED*/
2728 static int
filt_lio(struct knote * kn,long hint)2729 filt_lio(struct knote *kn, long hint)
2730 {
2731 	struct aioliojob * lj = kn->kn_ptr.p_lio;
2732 
2733 	return (lj->lioj_flags & LIOJ_KEVENT_POSTED);
2734 }
2735 
2736 #ifdef COMPAT_FREEBSD32
2737 #include <sys/mount.h>
2738 #include <sys/socket.h>
2739 #include <sys/sysent.h>
2740 #include <compat/freebsd32/freebsd32.h>
2741 #include <compat/freebsd32/freebsd32_proto.h>
2742 #include <compat/freebsd32/freebsd32_signal.h>
2743 #include <compat/freebsd32/freebsd32_syscall.h>
2744 #include <compat/freebsd32/freebsd32_util.h>
2745 
2746 struct __aiocb_private32 {
2747 	int32_t	status;
2748 	int32_t	error;
2749 	uint32_t spare;
2750 };
2751 
2752 #ifdef COMPAT_FREEBSD6
2753 typedef struct oaiocb32 {
2754 	int	aio_fildes;		/* File descriptor */
2755 	uint64_t aio_offset __packed;	/* File offset for I/O */
2756 	uint32_t aio_buf;		/* I/O buffer in process space */
2757 	uint32_t aio_nbytes;		/* Number of bytes for I/O */
2758 	struct	osigevent32 aio_sigevent; /* Signal to deliver */
2759 	int	aio_lio_opcode;		/* LIO opcode */
2760 	int	aio_reqprio;		/* Request priority -- ignored */
2761 	struct	__aiocb_private32 _aiocb_private;
2762 } oaiocb32_t;
2763 #endif
2764 
2765 typedef struct aiocb32 {
2766 	int32_t	aio_fildes;		/* File descriptor */
2767 	uint64_t aio_offset __packed;	/* File offset for I/O */
2768 	uint32_t aio_buf;	/* I/O buffer in process space */
2769 	uint32_t aio_nbytes;	/* Number of bytes for I/O */
2770 	int	__spare__[2];
2771 	uint32_t __spare2__;
2772 	int	aio_lio_opcode;		/* LIO opcode */
2773 	int	aio_reqprio;		/* Request priority -- ignored */
2774 	struct	__aiocb_private32 _aiocb_private;
2775 	struct	sigevent32 aio_sigevent;	/* Signal to deliver */
2776 } aiocb32_t;
2777 
2778 #ifdef COMPAT_FREEBSD6
2779 static int
convert_old_sigevent32(struct osigevent32 * osig,struct sigevent * nsig)2780 convert_old_sigevent32(struct osigevent32 *osig, struct sigevent *nsig)
2781 {
2782 
2783 	/*
2784 	 * Only SIGEV_NONE, SIGEV_SIGNAL, and SIGEV_KEVENT are
2785 	 * supported by AIO with the old sigevent structure.
2786 	 */
2787 	CP(*osig, *nsig, sigev_notify);
2788 	switch (nsig->sigev_notify) {
2789 	case SIGEV_NONE:
2790 		break;
2791 	case SIGEV_SIGNAL:
2792 		nsig->sigev_signo = osig->__sigev_u.__sigev_signo;
2793 		break;
2794 	case SIGEV_KEVENT:
2795 		nsig->sigev_notify_kqueue =
2796 		    osig->__sigev_u.__sigev_notify_kqueue;
2797 		PTRIN_CP(*osig, *nsig, sigev_value.sival_ptr);
2798 		break;
2799 	default:
2800 		return (EINVAL);
2801 	}
2802 	return (0);
2803 }
2804 
2805 static int
aiocb32_copyin_old_sigevent(struct aiocb * ujob,struct kaiocb * kjob,int type __unused)2806 aiocb32_copyin_old_sigevent(struct aiocb *ujob, struct kaiocb *kjob,
2807     int type __unused)
2808 {
2809 	struct oaiocb32 job32;
2810 	struct aiocb *kcb = &kjob->uaiocb;
2811 	int error;
2812 
2813 	bzero(kcb, sizeof(struct aiocb));
2814 	error = copyin(ujob, &job32, sizeof(job32));
2815 	if (error)
2816 		return (error);
2817 
2818 	/* No need to copyin aio_iov, because it did not exist in FreeBSD 6 */
2819 
2820 	CP(job32, *kcb, aio_fildes);
2821 	CP(job32, *kcb, aio_offset);
2822 	PTRIN_CP(job32, *kcb, aio_buf);
2823 	CP(job32, *kcb, aio_nbytes);
2824 	CP(job32, *kcb, aio_lio_opcode);
2825 	CP(job32, *kcb, aio_reqprio);
2826 	CP(job32, *kcb, _aiocb_private.status);
2827 	CP(job32, *kcb, _aiocb_private.error);
2828 	return (convert_old_sigevent32(&job32.aio_sigevent,
2829 	    &kcb->aio_sigevent));
2830 }
2831 #endif
2832 
2833 static int
aiocb32_copyin(struct aiocb * ujob,struct kaiocb * kjob,int type)2834 aiocb32_copyin(struct aiocb *ujob, struct kaiocb *kjob, int type)
2835 {
2836 	struct aiocb32 job32;
2837 	struct aiocb *kcb = &kjob->uaiocb;
2838 	struct iovec32 *iov32;
2839 	int error;
2840 
2841 	error = copyin(ujob, &job32, sizeof(job32));
2842 	if (error)
2843 		return (error);
2844 	CP(job32, *kcb, aio_fildes);
2845 	CP(job32, *kcb, aio_offset);
2846 	CP(job32, *kcb, aio_lio_opcode);
2847 	if (type == LIO_NOP)
2848 		type = kcb->aio_lio_opcode;
2849 	if (type & LIO_VECTORED) {
2850 		iov32 = PTRIN(job32.aio_iov);
2851 		CP(job32, *kcb, aio_iovcnt);
2852 		/* malloc a uio and copy in the iovec */
2853 		error = freebsd32_copyinuio(iov32,
2854 		    kcb->aio_iovcnt, &kjob->uiop);
2855 		if (error)
2856 			return (error);
2857 	} else {
2858 		PTRIN_CP(job32, *kcb, aio_buf);
2859 		CP(job32, *kcb, aio_nbytes);
2860 	}
2861 	CP(job32, *kcb, aio_reqprio);
2862 	CP(job32, *kcb, _aiocb_private.status);
2863 	CP(job32, *kcb, _aiocb_private.error);
2864 	error = convert_sigevent32(&job32.aio_sigevent, &kcb->aio_sigevent);
2865 
2866 	return (error);
2867 }
2868 
2869 static long
aiocb32_fetch_status(struct aiocb * ujob)2870 aiocb32_fetch_status(struct aiocb *ujob)
2871 {
2872 	struct aiocb32 *ujob32;
2873 
2874 	ujob32 = (struct aiocb32 *)ujob;
2875 	return (fuword32(&ujob32->_aiocb_private.status));
2876 }
2877 
2878 static long
aiocb32_fetch_error(struct aiocb * ujob)2879 aiocb32_fetch_error(struct aiocb *ujob)
2880 {
2881 	struct aiocb32 *ujob32;
2882 
2883 	ujob32 = (struct aiocb32 *)ujob;
2884 	return (fuword32(&ujob32->_aiocb_private.error));
2885 }
2886 
2887 static int
aiocb32_store_status(struct aiocb * ujob,long status)2888 aiocb32_store_status(struct aiocb *ujob, long status)
2889 {
2890 	struct aiocb32 *ujob32;
2891 
2892 	ujob32 = (struct aiocb32 *)ujob;
2893 	return (suword32(&ujob32->_aiocb_private.status, status));
2894 }
2895 
2896 static int
aiocb32_store_error(struct aiocb * ujob,long error)2897 aiocb32_store_error(struct aiocb *ujob, long error)
2898 {
2899 	struct aiocb32 *ujob32;
2900 
2901 	ujob32 = (struct aiocb32 *)ujob;
2902 	return (suword32(&ujob32->_aiocb_private.error, error));
2903 }
2904 
2905 static int
aiocb32_store_aiocb(struct aiocb ** ujobp,struct aiocb * ujob)2906 aiocb32_store_aiocb(struct aiocb **ujobp, struct aiocb *ujob)
2907 {
2908 
2909 	return (suword32(ujobp, (long)ujob));
2910 }
2911 
2912 static struct aiocb_ops aiocb32_ops = {
2913 	.aio_copyin = aiocb32_copyin,
2914 	.fetch_status = aiocb32_fetch_status,
2915 	.fetch_error = aiocb32_fetch_error,
2916 	.store_status = aiocb32_store_status,
2917 	.store_error = aiocb32_store_error,
2918 	.store_aiocb = aiocb32_store_aiocb,
2919 };
2920 
2921 #ifdef COMPAT_FREEBSD6
2922 static struct aiocb_ops aiocb32_ops_osigevent = {
2923 	.aio_copyin = aiocb32_copyin_old_sigevent,
2924 	.fetch_status = aiocb32_fetch_status,
2925 	.fetch_error = aiocb32_fetch_error,
2926 	.store_status = aiocb32_store_status,
2927 	.store_error = aiocb32_store_error,
2928 	.store_aiocb = aiocb32_store_aiocb,
2929 };
2930 #endif
2931 
2932 int
freebsd32_aio_return(struct thread * td,struct freebsd32_aio_return_args * uap)2933 freebsd32_aio_return(struct thread *td, struct freebsd32_aio_return_args *uap)
2934 {
2935 
2936 	return (kern_aio_return(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
2937 }
2938 
2939 int
freebsd32_aio_suspend(struct thread * td,struct freebsd32_aio_suspend_args * uap)2940 freebsd32_aio_suspend(struct thread *td, struct freebsd32_aio_suspend_args *uap)
2941 {
2942 	struct timespec32 ts32;
2943 	struct timespec ts, *tsp;
2944 	struct aiocb **ujoblist;
2945 	uint32_t *ujoblist32;
2946 	int error, i;
2947 
2948 	if (uap->nent < 0 || uap->nent > max_aio_queue_per_proc)
2949 		return (EINVAL);
2950 
2951 	if (uap->timeout) {
2952 		/* Get timespec struct. */
2953 		if ((error = copyin(uap->timeout, &ts32, sizeof(ts32))) != 0)
2954 			return (error);
2955 		CP(ts32, ts, tv_sec);
2956 		CP(ts32, ts, tv_nsec);
2957 		tsp = &ts;
2958 	} else
2959 		tsp = NULL;
2960 
2961 	ujoblist = malloc(uap->nent * sizeof(ujoblist[0]), M_AIO, M_WAITOK);
2962 	ujoblist32 = (uint32_t *)ujoblist;
2963 	error = copyin(uap->aiocbp, ujoblist32, uap->nent *
2964 	    sizeof(ujoblist32[0]));
2965 	if (error == 0) {
2966 		for (i = uap->nent - 1; i >= 0; i--)
2967 			ujoblist[i] = PTRIN(ujoblist32[i]);
2968 
2969 		error = kern_aio_suspend(td, uap->nent, ujoblist, tsp);
2970 	}
2971 	free(ujoblist, M_AIO);
2972 	return (error);
2973 }
2974 
2975 int
freebsd32_aio_error(struct thread * td,struct freebsd32_aio_error_args * uap)2976 freebsd32_aio_error(struct thread *td, struct freebsd32_aio_error_args *uap)
2977 {
2978 
2979 	return (kern_aio_error(td, (struct aiocb *)uap->aiocbp, &aiocb32_ops));
2980 }
2981 
2982 #ifdef COMPAT_FREEBSD6
2983 int
freebsd6_freebsd32_aio_read(struct thread * td,struct freebsd6_freebsd32_aio_read_args * uap)2984 freebsd6_freebsd32_aio_read(struct thread *td,
2985     struct freebsd6_freebsd32_aio_read_args *uap)
2986 {
2987 
2988 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
2989 	    &aiocb32_ops_osigevent));
2990 }
2991 #endif
2992 
2993 int
freebsd32_aio_read(struct thread * td,struct freebsd32_aio_read_args * uap)2994 freebsd32_aio_read(struct thread *td, struct freebsd32_aio_read_args *uap)
2995 {
2996 
2997 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READ,
2998 	    &aiocb32_ops));
2999 }
3000 
3001 int
freebsd32_aio_readv(struct thread * td,struct freebsd32_aio_readv_args * uap)3002 freebsd32_aio_readv(struct thread *td, struct freebsd32_aio_readv_args *uap)
3003 {
3004 
3005 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_READV,
3006 	    &aiocb32_ops));
3007 }
3008 
3009 #ifdef COMPAT_FREEBSD6
3010 int
freebsd6_freebsd32_aio_write(struct thread * td,struct freebsd6_freebsd32_aio_write_args * uap)3011 freebsd6_freebsd32_aio_write(struct thread *td,
3012     struct freebsd6_freebsd32_aio_write_args *uap)
3013 {
3014 
3015 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
3016 	    &aiocb32_ops_osigevent));
3017 }
3018 #endif
3019 
3020 int
freebsd32_aio_write(struct thread * td,struct freebsd32_aio_write_args * uap)3021 freebsd32_aio_write(struct thread *td, struct freebsd32_aio_write_args *uap)
3022 {
3023 
3024 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITE,
3025 	    &aiocb32_ops));
3026 }
3027 
3028 int
freebsd32_aio_writev(struct thread * td,struct freebsd32_aio_writev_args * uap)3029 freebsd32_aio_writev(struct thread *td, struct freebsd32_aio_writev_args *uap)
3030 {
3031 
3032 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_WRITEV,
3033 	    &aiocb32_ops));
3034 }
3035 
3036 int
freebsd32_aio_mlock(struct thread * td,struct freebsd32_aio_mlock_args * uap)3037 freebsd32_aio_mlock(struct thread *td, struct freebsd32_aio_mlock_args *uap)
3038 {
3039 
3040 	return (aio_aqueue(td, (struct aiocb *)uap->aiocbp, NULL, LIO_MLOCK,
3041 	    &aiocb32_ops));
3042 }
3043 
3044 int
freebsd32_aio_waitcomplete(struct thread * td,struct freebsd32_aio_waitcomplete_args * uap)3045 freebsd32_aio_waitcomplete(struct thread *td,
3046     struct freebsd32_aio_waitcomplete_args *uap)
3047 {
3048 	struct timespec32 ts32;
3049 	struct timespec ts, *tsp;
3050 	int error;
3051 
3052 	if (uap->timeout) {
3053 		/* Get timespec struct. */
3054 		error = copyin(uap->timeout, &ts32, sizeof(ts32));
3055 		if (error)
3056 			return (error);
3057 		CP(ts32, ts, tv_sec);
3058 		CP(ts32, ts, tv_nsec);
3059 		tsp = &ts;
3060 	} else
3061 		tsp = NULL;
3062 
3063 	return (kern_aio_waitcomplete(td, (struct aiocb **)uap->aiocbp, tsp,
3064 	    &aiocb32_ops));
3065 }
3066 
3067 int
freebsd32_aio_fsync(struct thread * td,struct freebsd32_aio_fsync_args * uap)3068 freebsd32_aio_fsync(struct thread *td, struct freebsd32_aio_fsync_args *uap)
3069 {
3070 
3071 	return (kern_aio_fsync(td, uap->op, (struct aiocb *)uap->aiocbp,
3072 	    &aiocb32_ops));
3073 }
3074 
3075 #ifdef COMPAT_FREEBSD6
3076 int
freebsd6_freebsd32_lio_listio(struct thread * td,struct freebsd6_freebsd32_lio_listio_args * uap)3077 freebsd6_freebsd32_lio_listio(struct thread *td,
3078     struct freebsd6_freebsd32_lio_listio_args *uap)
3079 {
3080 	struct aiocb **acb_list;
3081 	struct sigevent *sigp, sig;
3082 	struct osigevent32 osig;
3083 	uint32_t *acb_list32;
3084 	int error, i, nent;
3085 
3086 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
3087 		return (EINVAL);
3088 
3089 	nent = uap->nent;
3090 	if (nent < 0 || nent > max_aio_queue_per_proc)
3091 		return (EINVAL);
3092 
3093 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
3094 		error = copyin(uap->sig, &osig, sizeof(osig));
3095 		if (error)
3096 			return (error);
3097 		error = convert_old_sigevent32(&osig, &sig);
3098 		if (error)
3099 			return (error);
3100 		sigp = &sig;
3101 	} else
3102 		sigp = NULL;
3103 
3104 	acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
3105 	error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
3106 	if (error) {
3107 		free(acb_list32, M_LIO);
3108 		return (error);
3109 	}
3110 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
3111 	for (i = 0; i < nent; i++)
3112 		acb_list[i] = PTRIN(acb_list32[i]);
3113 	free(acb_list32, M_LIO);
3114 
3115 	error = kern_lio_listio(td, uap->mode,
3116 	    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
3117 	    &aiocb32_ops_osigevent);
3118 	free(acb_list, M_LIO);
3119 	return (error);
3120 }
3121 #endif
3122 
3123 int
freebsd32_lio_listio(struct thread * td,struct freebsd32_lio_listio_args * uap)3124 freebsd32_lio_listio(struct thread *td, struct freebsd32_lio_listio_args *uap)
3125 {
3126 	struct aiocb **acb_list;
3127 	struct sigevent *sigp, sig;
3128 	struct sigevent32 sig32;
3129 	uint32_t *acb_list32;
3130 	int error, i, nent;
3131 
3132 	if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT))
3133 		return (EINVAL);
3134 
3135 	nent = uap->nent;
3136 	if (nent < 0 || nent > max_aio_queue_per_proc)
3137 		return (EINVAL);
3138 
3139 	if (uap->sig && (uap->mode == LIO_NOWAIT)) {
3140 		error = copyin(uap->sig, &sig32, sizeof(sig32));
3141 		if (error)
3142 			return (error);
3143 		error = convert_sigevent32(&sig32, &sig);
3144 		if (error)
3145 			return (error);
3146 		sigp = &sig;
3147 	} else
3148 		sigp = NULL;
3149 
3150 	acb_list32 = malloc(sizeof(uint32_t) * nent, M_LIO, M_WAITOK);
3151 	error = copyin(uap->acb_list, acb_list32, nent * sizeof(uint32_t));
3152 	if (error) {
3153 		free(acb_list32, M_LIO);
3154 		return (error);
3155 	}
3156 	acb_list = malloc(sizeof(struct aiocb *) * nent, M_LIO, M_WAITOK);
3157 	for (i = 0; i < nent; i++)
3158 		acb_list[i] = PTRIN(acb_list32[i]);
3159 	free(acb_list32, M_LIO);
3160 
3161 	error = kern_lio_listio(td, uap->mode,
3162 	    (struct aiocb * const *)uap->acb_list, acb_list, nent, sigp,
3163 	    &aiocb32_ops);
3164 	free(acb_list, M_LIO);
3165 	return (error);
3166 }
3167 
3168 #endif
3169