1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #ifndef _ASYNCIO_H 28 #define _ASYNCIO_H 29 30 #pragma ident "%Z%%M% %I% %E% SMI" 31 32 #ifdef __cplusplus 33 extern "C" { 34 #endif 35 36 #include <stdio.h> 37 #include <stdlib.h> 38 #include <unistd.h> 39 #include <string.h> 40 #include <errno.h> 41 #include <sys/types.h> 42 #include <sys/stat.h> 43 #include <thread.h> 44 #include <pthread.h> 45 #include <setjmp.h> 46 #include <signal.h> 47 #include <siginfo.h> 48 #include <aio.h> 49 #include <limits.h> 50 #include <ucontext.h> 51 #include <sys/asynch.h> 52 #include <sys/mman.h> 53 54 #if !defined(_LP64) 55 #define AIOSTKSIZE (64 * 1024) 56 #else 57 #define AIOSTKSIZE (128 * 1024) 58 #endif 59 60 #define SIGAIOCANCEL SIGLWP /* special aio cancelation signal */ 61 62 #define AIO_WAITN_MAXIOCBS 32768 /* max. iocbs per system call */ 63 64 /* 65 * Declare structure types. The structures themselves are defined below. 66 */ 67 typedef struct aio_args aio_args_t; 68 typedef struct aio_lio aio_lio_t; 69 typedef struct notif_param notif_param_t; 70 typedef struct aio_req aio_req_t; 71 typedef struct aio_worker aio_worker_t; 72 typedef struct aio_hash aio_hash_t; 73 74 struct aio_args { 75 int fd; 76 caddr_t buf; 77 size_t bufsz; 78 offset_t offset; 79 }; 80 81 /* 82 * list head for UFS list I/O 83 */ 84 struct aio_lio { 85 mutex_t lio_mutex; /* list mutex */ 86 cond_t lio_cond_cv; /* list notification for I/O done */ 87 aio_lio_t *lio_next; /* pointer to next on freelist */ 88 char lio_mode; /* LIO_WAIT/LIO_NOWAIT */ 89 char lio_canned; /* lio was canceled */ 90 char lio_largefile; /* largefile operation */ 91 char lio_waiting; /* waiting in __lio_listio() */ 92 int lio_nent; /* Number of list I/O's */ 93 int lio_refcnt; /* outstanding I/O's */ 94 int lio_event; /* Event number for notification */ 95 int lio_port; /* Port number for notification */ 96 int lio_signo; /* Signal number for notification */ 97 union sigval lio_sigval; /* Signal parameter */ 98 uintptr_t lio_object; /* for SIGEV_THREAD or SIGEV_PORT */ 99 struct sigevent *lio_sigevent; /* Notification function and attr. */ 100 }; 101 102 /* 103 * Notification parameters 104 */ 105 struct notif_param { 106 int np_signo; /* SIGEV_SIGNAL */ 107 int np_port; /* SIGEV_THREAD or SIGEV_PORT */ 108 void *np_user; 109 int np_event; 110 uintptr_t np_object; 111 int np_lio_signo; /* listio: SIGEV_SIGNAL */ 112 int np_lio_port; /* listio: SIGEV_THREAD or SIGEV_PORT */ 113 void *np_lio_user; 114 int np_lio_event; 115 uintptr_t np_lio_object; 116 }; 117 118 struct aio_req { 119 /* 120 * fields protected by _aio_mutex lock. 121 */ 122 aio_req_t *req_link; /* hash/freelist chain link */ 123 /* 124 * when req is on the doneq, then req_next is protected by 125 * the _aio_mutex lock. when the req is on a work q, then 126 * req_next is protected by a worker's work_qlock1 lock. 127 */ 128 aio_req_t *req_next; /* request/done queue link */ 129 aio_req_t *req_prev; /* double linked list */ 130 /* 131 * fields protected by a worker's work_qlock1 lock. 132 */ 133 char req_state; /* AIO_REQ_QUEUED, ... */ 134 /* 135 * fields require no locking. 136 */ 137 char req_type; /* AIO_POSIX_REQ or not */ 138 char req_largefile; /* largefile operation */ 139 char req_op; /* AIOREAD, etc. */ 140 aio_worker_t *req_worker; /* associate request with worker */ 141 aio_result_t *req_resultp; /* address of result buffer */ 142 aio_args_t req_args; /* arglist */ 143 aio_lio_t *req_head; /* list head for LIO */ 144 struct sigevent req_sigevent; 145 void *req_aiocbp; /* ptr to aiocb or aiocb64 */ 146 notif_param_t req_notify; /* notification parameters */ 147 }; 148 149 /* special lio type that destroys itself when lio refcnt becomes zero */ 150 #define LIO_FSYNC LIO_WAIT+1 151 #define LIO_DESTROY LIO_FSYNC+1 152 153 /* lio flags */ 154 #define LIO_FSYNC_CANCELED 0x1 155 156 /* values for aio_state */ 157 158 #define AIO_REQ_QUEUED 1 159 #define AIO_REQ_INPROGRESS 2 160 #define AIO_REQ_CANCELED 3 161 #define AIO_REQ_DONE 4 162 #define AIO_REQ_FREE 5 163 #define AIO_REQ_DONEQ 6 164 165 /* use KAIO in _aio_rw() */ 166 #define AIO_NO_KAIO 0x0 167 #define AIO_KAIO 0x1 168 #define AIO_NO_DUPS 0x2 169 170 #define AIO_POSIX_REQ 0x1 171 172 #define CHECK 1 173 #define NOCHECK 2 174 #define CHECKED 3 175 #define USERAIO 4 176 #define USERAIO_DONE 5 177 178 /* values for _aio_flags */ 179 180 /* if set, _aiodone() notifies aio_waitn about done requests */ 181 #define AIO_WAIT_INPROGRESS 0x1 182 /* if set, _aiodone() wakes up functions waiting for completed I/Os */ 183 #define AIO_IO_WAITING 0x2 184 #define AIO_LIB_WAITN 0x4 /* aio_waitn in progress */ 185 #define AIO_LIB_WAITN_PENDING 0x8 /* aio_waitn requests pending */ 186 187 /* 188 * Before a kaio() system call, the fd will be checked 189 * to ensure that kernel async. I/O is supported for this file. 190 * The only way to find out is if a kaio() call returns ENOTSUP, 191 * so the default will always be to try the kaio() call. Only in 192 * the specific instance of a kaio() call returning ENOTSUP 193 * will we stop submitting kaio() calls for that fd. 194 * If the fd is outside the array bounds, we will allow the kaio() 195 * call. 196 * 197 * The only way that an fd entry can go from ENOTSUP to supported 198 * is if that fd is freed up by a close(), and close will clear 199 * the entry for that fd. 200 * 201 * Each fd gets a bit in the array _kaio_supported[]. 202 * 203 * uint32_t _kaio_supported[MAX_KAIO_FDARRAY_SIZE]; 204 * 205 * Array is MAX_KAIO_ARRAY_SIZE of 32-bit elements, for 8kb. 206 * If more than (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE) 207 * files are open, this can be expanded. 208 */ 209 210 #define MAX_KAIO_FDARRAY_SIZE 2048 211 #define KAIO_FDARRAY_ELEM_SIZE WORD_BIT /* uint32_t */ 212 213 #define MAX_KAIO_FDS (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE) 214 215 #define VALID_FD(fdes) ((fdes) >= 0 && (fdes) < MAX_KAIO_FDS) 216 217 #define KAIO_SUPPORTED(fdes) \ 218 (!VALID_FD(fdes) || \ 219 ((_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] & \ 220 (uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))) == 0)) 221 222 #define SET_KAIO_NOT_SUPPORTED(fdes) \ 223 if (VALID_FD(fdes)) \ 224 _kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] |= \ 225 (uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE)) 226 227 #define CLEAR_KAIO_SUPPORTED(fdes) \ 228 if (VALID_FD(fdes)) \ 229 _kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] &= \ 230 ~(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE)) 231 232 struct aio_worker { 233 aio_worker_t *work_forw; /* forward link in list of workers */ 234 aio_worker_t *work_backw; /* backwards link in list of workers */ 235 mutex_t work_qlock1; /* lock for work queue 1 */ 236 cond_t work_idle_cv; /* place to sleep when idle */ 237 aio_req_t *work_head1; /* head of work request queue 1 */ 238 aio_req_t *work_tail1; /* tail of work request queue 1 */ 239 aio_req_t *work_next1; /* work queue one's next pointer */ 240 aio_req_t *work_prev1; /* last request done from queue 1 */ 241 aio_req_t *work_req; /* active work request */ 242 thread_t work_tid; /* worker's thread-id */ 243 int work_count1; /* length of work queue one */ 244 int work_done1; /* number of requests done */ 245 int work_minload1; /* min length of queue */ 246 int work_idleflg; /* when set, worker is idle */ 247 sigjmp_buf work_jmp_buf; /* cancellation point */ 248 }; 249 250 struct aio_hash { /* resultp hash table */ 251 mutex_t hash_lock; 252 aio_req_t *hash_ptr; 253 #if !defined(_LP64) 254 void *hash_pad; /* ensure sizeof (aio_hash_t) == 32 */ 255 #endif 256 }; 257 258 extern aio_hash_t *_aio_hash; 259 260 #define HASHSZ 2048 /* power of 2 */ 261 #define AIOHASH(resultp) ((((uintptr_t)(resultp) >> 17) ^ \ 262 ((uintptr_t)(resultp) >> 2)) & (HASHSZ - 1)) 263 #define POSIX_AIO(x) ((x)->req_type == AIO_POSIX_REQ) 264 265 extern int __uaio_init(void); 266 extern void _kaio_init(void); 267 extern intptr_t _kaio(int, ...); 268 extern int _aiorw(int, caddr_t, int, offset_t, int, aio_result_t *, int); 269 extern int _aio_rw(aiocb_t *, aio_lio_t *, aio_worker_t **, int, int); 270 #if !defined(_LP64) 271 extern int _aio_rw64(aiocb64_t *, aio_lio_t *, aio_worker_t **, int, int); 272 #endif 273 extern int _aio_create_worker(aio_req_t *, int); 274 extern int _aio_cancel_req(aio_worker_t *, aio_req_t *, int *, int *); 275 extern int aiocancel_all(int); 276 extern void aio_panic(const char *); 277 extern aio_req_t *_aio_hash_find(aio_result_t *); 278 extern aio_req_t *_aio_hash_del(aio_result_t *); 279 extern void _aio_req_mark_done(aio_req_t *); 280 extern void _aio_waitn_wakeup(void); 281 extern aio_worker_t *_aio_worker_alloc(void); 282 extern void _aio_worker_free(void *); 283 extern aio_req_t *_aio_req_alloc(void); 284 extern void _aio_req_free(aio_req_t *); 285 extern aio_lio_t *_aio_lio_alloc(void); 286 extern void _aio_lio_free(aio_lio_t *); 287 extern int _aio_idle(aio_worker_t *); 288 extern void *_aio_do_request(void *); 289 extern void *_aio_do_notify(void *); 290 extern void _lio_remove(aio_req_t *); 291 extern aio_req_t *_aio_req_remove(aio_req_t *); 292 extern int _aio_get_timedelta(timespec_t *, timespec_t *); 293 extern aio_result_t *_aio_req_done(void); 294 extern void _aio_set_result(aio_req_t *, ssize_t, int); 295 extern int _aio_sigev_thread_init(struct sigevent *); 296 extern int _aio_sigev_thread(aiocb_t *); 297 #if !defined(_LP64) 298 extern int _aio_sigev_thread64(aiocb64_t *); 299 #endif 300 301 extern aio_worker_t *_kaiowp; /* points to kaio cleanup thread */ 302 extern aio_worker_t *__workers_rw; /* list of all rw workers */ 303 extern aio_worker_t *__nextworker_rw; /* worker chosen for next rw request */ 304 extern int __rw_workerscnt; /* number of rw workers */ 305 extern aio_worker_t *__workers_no; /* list of all notification workers */ 306 extern aio_worker_t *__nextworker_no; /* worker chosen, next notification */ 307 extern int __no_workerscnt; /* number of notification workers */ 308 extern mutex_t __aio_initlock; /* makes aio initialization atomic */ 309 extern cond_t __aio_initcv; 310 extern int __aio_initbusy; 311 extern mutex_t __aio_mutex; /* global aio lock */ 312 extern cond_t _aio_iowait_cv; /* wait for userland I/Os */ 313 extern cond_t _aio_waitn_cv; /* wait for end of aio_waitn */ 314 extern int _max_workers; /* max number of workers permitted */ 315 extern int _min_workers; /* min number of workers */ 316 extern sigset_t _worker_set; /* worker's signal mask */ 317 extern int _aio_worker_cnt; /* number of AIO workers */ 318 extern int _sigio_enabled; /* when set, send SIGIO signal */ 319 extern pid_t __pid; /* process's PID */ 320 extern int __uaio_ok; /* indicates if aio is initialized */ 321 extern int _kaio_ok; /* indicates if kaio is initialized */ 322 extern pthread_key_t _aio_key; /* for thread-specific data */ 323 extern aio_req_t *_aio_done_tail; /* list of done requests */ 324 extern aio_req_t *_aio_done_head; 325 extern aio_req_t *_aio_doneq; 326 extern int _aio_freelist_cnt; 327 extern int _aio_allocated_cnt; 328 extern int _aio_donecnt; 329 extern int _aio_doneq_cnt; 330 extern int _aio_waitncnt; /* # of requests for aio_waitn */ 331 extern int _aio_outstand_cnt; /* # of outstanding requests */ 332 extern int _kaio_outstand_cnt; /* # of outstanding kaio requests */ 333 extern int _aio_req_done_cnt; /* req. done but not in "done queue" */ 334 extern int _aio_kernel_suspend; /* active kernel kaio calls */ 335 extern int _aio_suscv_cnt; /* aio_suspend calls waiting on cv's */ 336 extern int _aiowait_flag; /* when set, aiowait() is inprogress */ 337 extern int _aio_flags; /* see defines, above */ 338 extern uint32_t *_kaio_supported; 339 340 extern const sigset_t maskset; /* all maskable signals */ 341 342 #ifdef __cplusplus 343 } 344 #endif 345 346 #endif /* _ASYNCIO_H */ 347