1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _ASYNCIO_H 28 #define _ASYNCIO_H 29 30 #ifdef __cplusplus 31 extern "C" { 32 #endif 33 34 #include <stdio.h> 35 #include <stdlib.h> 36 #include <unistd.h> 37 #include <string.h> 38 #include <errno.h> 39 #include <sys/types.h> 40 #include <sys/stat.h> 41 #include <thread.h> 42 #include <pthread.h> 43 #include <setjmp.h> 44 #include <signal.h> 45 #include <siginfo.h> 46 #include <aio.h> 47 #include <limits.h> 48 #include <ucontext.h> 49 #include <sys/asynch.h> 50 #include <sys/mman.h> 51 52 #if !defined(_LP64) 53 #define AIOSTKSIZE (64 * 1024) 54 #else 55 #define AIOSTKSIZE (128 * 1024) 56 #endif 57 58 #define SIGAIOCANCEL SIGLWP /* special aio cancelation signal */ 59 60 #define AIO_WAITN_MAXIOCBS 32768 /* max. iocbs per system call */ 61 62 /* 63 * Declare structure types. The structures themselves are defined below. 64 */ 65 typedef struct aio_args aio_args_t; 66 typedef struct aio_lio aio_lio_t; 67 typedef struct notif_param notif_param_t; 68 typedef struct aio_req aio_req_t; 69 typedef struct aio_worker aio_worker_t; 70 typedef struct aio_hash aio_hash_t; 71 72 struct aio_args { 73 int fd; 74 caddr_t buf; 75 size_t bufsz; 76 offset_t offset; 77 }; 78 79 /* 80 * list head for UFS list I/O 81 */ 82 struct aio_lio { 83 mutex_t lio_mutex; /* list mutex */ 84 cond_t lio_cond_cv; /* list notification for I/O done */ 85 aio_lio_t *lio_next; /* pointer to next on freelist */ 86 char lio_mode; /* LIO_WAIT/LIO_NOWAIT */ 87 char lio_canned; /* lio was canceled */ 88 char lio_largefile; /* largefile operation */ 89 char lio_waiting; /* waiting in __lio_listio() */ 90 int lio_nent; /* Number of list I/O's */ 91 int lio_refcnt; /* outstanding I/O's */ 92 int lio_event; /* Event number for notification */ 93 int lio_port; /* Port number for notification */ 94 int lio_signo; /* Signal number for notification */ 95 union sigval lio_sigval; /* Signal parameter */ 96 uintptr_t lio_object; /* for SIGEV_THREAD or SIGEV_PORT */ 97 struct sigevent *lio_sigevent; /* Notification function and attr. */ 98 }; 99 100 /* 101 * Notification parameters 102 */ 103 struct notif_param { 104 int np_signo; /* SIGEV_SIGNAL */ 105 int np_port; /* SIGEV_THREAD or SIGEV_PORT */ 106 void *np_user; 107 int np_event; 108 uintptr_t np_object; 109 int np_lio_signo; /* listio: SIGEV_SIGNAL */ 110 int np_lio_port; /* listio: SIGEV_THREAD or SIGEV_PORT */ 111 void *np_lio_user; 112 int np_lio_event; 113 uintptr_t np_lio_object; 114 }; 115 116 struct aio_req { 117 /* 118 * fields protected by _aio_mutex lock. 119 */ 120 aio_req_t *req_link; /* hash/freelist chain link */ 121 /* 122 * when req is on the doneq, then req_next is protected by 123 * the _aio_mutex lock. when the req is on a work q, then 124 * req_next is protected by a worker's work_qlock1 lock. 125 */ 126 aio_req_t *req_next; /* request/done queue link */ 127 aio_req_t *req_prev; /* double linked list */ 128 /* 129 * fields protected by a worker's work_qlock1 lock. 130 */ 131 char req_state; /* AIO_REQ_QUEUED, ... */ 132 /* 133 * fields require no locking. 134 */ 135 char req_type; /* AIO_POSIX_REQ or not */ 136 char req_largefile; /* largefile operation */ 137 char req_op; /* AIOREAD, etc. */ 138 aio_worker_t *req_worker; /* associate request with worker */ 139 aio_result_t *req_resultp; /* address of result buffer */ 140 aio_args_t req_args; /* arglist */ 141 aio_lio_t *req_head; /* list head for LIO */ 142 struct sigevent req_sigevent; 143 void *req_aiocbp; /* ptr to aiocb or aiocb64 */ 144 notif_param_t req_notify; /* notification parameters */ 145 }; 146 147 /* special lio type that destroys itself when lio refcnt becomes zero */ 148 #define LIO_FSYNC LIO_WAIT+1 149 #define LIO_DESTROY LIO_FSYNC+1 150 151 /* lio flags */ 152 #define LIO_FSYNC_CANCELED 0x1 153 154 /* values for aio_state */ 155 156 #define AIO_REQ_QUEUED 1 157 #define AIO_REQ_INPROGRESS 2 158 #define AIO_REQ_CANCELED 3 159 #define AIO_REQ_DONE 4 160 #define AIO_REQ_FREE 5 161 #define AIO_REQ_DONEQ 6 162 163 /* use KAIO in _aio_rw() */ 164 #define AIO_NO_KAIO 0x0 165 #define AIO_KAIO 0x1 166 #define AIO_NO_DUPS 0x2 167 168 #define AIO_POSIX_REQ 0x1 169 170 #define CHECK 1 171 #define NOCHECK 2 172 #define CHECKED 3 173 #define USERAIO 4 174 #define USERAIO_DONE 5 175 176 /* values for _aio_flags */ 177 178 /* if set, _aiodone() notifies aio_waitn about done requests */ 179 #define AIO_WAIT_INPROGRESS 0x1 180 /* if set, _aiodone() wakes up functions waiting for completed I/Os */ 181 #define AIO_IO_WAITING 0x2 182 #define AIO_LIB_WAITN 0x4 /* aio_waitn in progress */ 183 #define AIO_LIB_WAITN_PENDING 0x8 /* aio_waitn requests pending */ 184 185 /* 186 * Before a kaio() system call, the fd will be checked 187 * to ensure that kernel async. I/O is supported for this file. 188 * The only way to find out is if a kaio() call returns ENOTSUP, 189 * so the default will always be to try the kaio() call. Only in 190 * the specific instance of a kaio() call returning ENOTSUP 191 * will we stop submitting kaio() calls for that fd. 192 * If the fd is outside the array bounds, we will allow the kaio() 193 * call. 194 * 195 * The only way that an fd entry can go from ENOTSUP to supported 196 * is if that fd is freed up by a close(), and close will clear 197 * the entry for that fd. 198 * 199 * Each fd gets a bit in the array _kaio_supported[]. 200 * 201 * uint32_t _kaio_supported[MAX_KAIO_FDARRAY_SIZE]; 202 * 203 * Array is MAX_KAIO_ARRAY_SIZE of 32-bit elements, for 8kb. 204 * If more than (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE) 205 * files are open, this can be expanded. 206 */ 207 208 #define MAX_KAIO_FDARRAY_SIZE 2048 209 #define KAIO_FDARRAY_ELEM_SIZE WORD_BIT /* uint32_t */ 210 211 #define MAX_KAIO_FDS (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE) 212 213 #define VALID_FD(fdes) ((fdes) >= 0 && (fdes) < MAX_KAIO_FDS) 214 215 #define KAIO_SUPPORTED(fdes) \ 216 (!VALID_FD(fdes) || \ 217 ((_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] & \ 218 (uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))) == 0)) 219 220 #define SET_KAIO_NOT_SUPPORTED(fdes) \ 221 if (VALID_FD(fdes)) \ 222 _kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] |= \ 223 (uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE)) 224 225 #define CLEAR_KAIO_SUPPORTED(fdes) \ 226 if (VALID_FD(fdes)) \ 227 _kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] &= \ 228 ~(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE)) 229 230 struct aio_worker { 231 aio_worker_t *work_forw; /* forward link in list of workers */ 232 aio_worker_t *work_backw; /* backwards link in list of workers */ 233 mutex_t work_qlock1; /* lock for work queue 1 */ 234 cond_t work_idle_cv; /* place to sleep when idle */ 235 aio_req_t *work_head1; /* head of work request queue 1 */ 236 aio_req_t *work_tail1; /* tail of work request queue 1 */ 237 aio_req_t *work_next1; /* work queue one's next pointer */ 238 aio_req_t *work_prev1; /* last request done from queue 1 */ 239 aio_req_t *work_req; /* active work request */ 240 thread_t work_tid; /* worker's thread-id */ 241 int work_count1; /* length of work queue one */ 242 int work_done1; /* number of requests done */ 243 int work_minload1; /* min length of queue */ 244 int work_idleflg; /* when set, worker is idle */ 245 sigjmp_buf work_jmp_buf; /* cancellation point */ 246 }; 247 248 struct aio_hash { /* resultp hash table */ 249 mutex_t hash_lock; 250 aio_req_t *hash_ptr; 251 #if !defined(_LP64) 252 void *hash_pad; /* ensure sizeof (aio_hash_t) == 32 */ 253 #endif 254 }; 255 256 extern aio_hash_t *_aio_hash; 257 258 #define HASHSZ 2048 /* power of 2 */ 259 #define AIOHASH(resultp) ((((uintptr_t)(resultp) >> 17) ^ \ 260 ((uintptr_t)(resultp) >> 2)) & (HASHSZ - 1)) 261 #define POSIX_AIO(x) ((x)->req_type == AIO_POSIX_REQ) 262 263 extern int __uaio_init(void); 264 extern void _kaio_init(void); 265 extern intptr_t _kaio(int, ...); 266 extern int _aiorw(int, caddr_t, int, offset_t, int, aio_result_t *, int); 267 extern int _aio_rw(aiocb_t *, aio_lio_t *, aio_worker_t **, int, int); 268 #if !defined(_LP64) 269 extern int _aio_rw64(aiocb64_t *, aio_lio_t *, aio_worker_t **, int, int); 270 #endif 271 extern int _aio_create_worker(aio_req_t *, int); 272 extern int _aio_cancel_req(aio_worker_t *, aio_req_t *, int *, int *); 273 extern int aiocancel_all(int); 274 extern void aio_panic(const char *) __NORETURN; 275 extern aio_req_t *_aio_hash_find(aio_result_t *); 276 extern aio_req_t *_aio_hash_del(aio_result_t *); 277 extern void _aio_req_mark_done(aio_req_t *); 278 extern void _aio_waitn_wakeup(void); 279 extern aio_worker_t *_aio_worker_alloc(void); 280 extern void _aio_worker_free(void *); 281 extern aio_req_t *_aio_req_alloc(void); 282 extern void _aio_req_free(aio_req_t *); 283 extern aio_lio_t *_aio_lio_alloc(void); 284 extern void _aio_lio_free(aio_lio_t *); 285 extern int _aio_idle(aio_worker_t *); 286 extern void *_aio_do_request(void *); 287 extern void *_aio_do_notify(void *); 288 extern void _lio_remove(aio_req_t *); 289 extern aio_req_t *_aio_req_remove(aio_req_t *); 290 extern int _aio_get_timedelta(timespec_t *, timespec_t *); 291 extern aio_result_t *_aio_req_done(void); 292 extern void _aio_set_result(aio_req_t *, ssize_t, int); 293 extern int _aio_sigev_thread_init(struct sigevent *); 294 extern int _aio_sigev_thread(aiocb_t *); 295 #if !defined(_LP64) 296 extern int _aio_sigev_thread64(aiocb64_t *); 297 #endif 298 299 extern aio_worker_t *_kaiowp; /* points to kaio cleanup thread */ 300 extern aio_worker_t *__workers_rw; /* list of all rw workers */ 301 extern aio_worker_t *__nextworker_rw; /* worker chosen for next rw request */ 302 extern int __rw_workerscnt; /* number of rw workers */ 303 extern aio_worker_t *__workers_no; /* list of all notification workers */ 304 extern aio_worker_t *__nextworker_no; /* worker chosen, next notification */ 305 extern int __no_workerscnt; /* number of notification workers */ 306 extern mutex_t __aio_initlock; /* makes aio initialization atomic */ 307 extern cond_t __aio_initcv; 308 extern int __aio_initbusy; 309 extern mutex_t __aio_mutex; /* global aio lock */ 310 extern cond_t _aio_iowait_cv; /* wait for userland I/Os */ 311 extern cond_t _aio_waitn_cv; /* wait for end of aio_waitn */ 312 extern int _max_workers; /* max number of workers permitted */ 313 extern int _min_workers; /* min number of workers */ 314 extern sigset_t _worker_set; /* worker's signal mask */ 315 extern int _aio_worker_cnt; /* number of AIO workers */ 316 extern int _sigio_enabled; /* when set, send SIGIO signal */ 317 extern pid_t __pid; /* process's PID */ 318 extern int __uaio_ok; /* indicates if aio is initialized */ 319 extern int _kaio_ok; /* indicates if kaio is initialized */ 320 extern pthread_key_t _aio_key; /* for thread-specific data */ 321 extern aio_req_t *_aio_done_tail; /* list of done requests */ 322 extern aio_req_t *_aio_done_head; 323 extern aio_req_t *_aio_doneq; 324 extern int _aio_freelist_cnt; 325 extern int _aio_allocated_cnt; 326 extern int _aio_donecnt; 327 extern int _aio_doneq_cnt; 328 extern int _aio_waitncnt; /* # of requests for aio_waitn */ 329 extern int _aio_outstand_cnt; /* # of outstanding requests */ 330 extern int _kaio_outstand_cnt; /* # of outstanding kaio requests */ 331 extern int _aio_req_done_cnt; /* req. done but not in "done queue" */ 332 extern int _aio_kernel_suspend; /* active kernel kaio calls */ 333 extern int _aio_suscv_cnt; /* aio_suspend calls waiting on cv's */ 334 extern int _aiowait_flag; /* when set, aiowait() is inprogress */ 335 extern int _aio_flags; /* see defines, above */ 336 extern uint32_t *_kaio_supported; 337 338 extern const sigset_t maskset; /* all maskable signals */ 339 340 #ifdef __cplusplus 341 } 342 #endif 343 344 #endif /* _ASYNCIO_H */ 345