1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Copyright 2026 Oxide Computer Company 29 */ 30 31 #ifndef _ASYNCIO_H 32 #define _ASYNCIO_H 33 34 #ifdef __cplusplus 35 extern "C" { 36 #endif 37 38 #include <stdio.h> 39 #include <stdlib.h> 40 #include <unistd.h> 41 #include <string.h> 42 #include <errno.h> 43 #include <sys/types.h> 44 #include <sys/stat.h> 45 #include <thread.h> 46 #include <pthread.h> 47 #include <setjmp.h> 48 #include <signal.h> 49 #include <siginfo.h> 50 #include <aio.h> 51 #include <limits.h> 52 #include <ucontext.h> 53 #include <sys/asynch.h> 54 #include <sys/mman.h> 55 56 #if !defined(_LP64) 57 #define AIOSTKSIZE (64 * 1024) 58 #else 59 #define AIOSTKSIZE (128 * 1024) 60 #endif 61 62 #define SIGAIOCANCEL SIGLWP /* special aio cancelation signal */ 63 64 #define AIO_WAITN_MAXIOCBS 32768 /* max. iocbs per system call */ 65 66 /* 67 * Declare structure types. The structures themselves are defined below. 68 */ 69 typedef struct aio_args aio_args_t; 70 typedef struct aio_lio aio_lio_t; 71 typedef struct notif_param notif_param_t; 72 typedef struct aio_req aio_req_t; 73 typedef struct aio_worker aio_worker_t; 74 typedef struct aio_hash aio_hash_t; 75 76 struct aio_args { 77 int fd; 78 caddr_t buf; 79 size_t bufsz; 80 offset_t offset; 81 }; 82 83 /* 84 * list head for UFS list I/O 85 */ 86 struct aio_lio { 87 mutex_t lio_mutex; /* list mutex */ 88 cond_t lio_cond_cv; /* list notification for I/O done */ 89 aio_lio_t *lio_next; /* pointer to next on freelist */ 90 char lio_mode; /* LIO_WAIT/LIO_NOWAIT */ 91 char lio_canned; /* lio was canceled */ 92 char lio_largefile; /* largefile operation */ 93 char lio_waiting; /* waiting in __lio_listio() */ 94 int lio_nent; /* Number of list I/O's */ 95 int lio_refcnt; /* outstanding I/O's */ 96 int lio_event; /* Event number for notification */ 97 int lio_port; /* Port number for notification */ 98 int lio_signo; /* Signal number for notification */ 99 union sigval lio_sigval; /* Signal parameter */ 100 uintptr_t lio_object; /* for SIGEV_THREAD or SIGEV_PORT */ 101 struct sigevent *lio_sigevent; /* Notification function and attr. */ 102 }; 103 104 /* 105 * Notification parameters 106 */ 107 struct notif_param { 108 int np_signo; /* SIGEV_SIGNAL */ 109 int np_port; /* SIGEV_THREAD or SIGEV_PORT */ 110 void *np_user; 111 int np_event; 112 uintptr_t np_object; 113 int np_lio_signo; /* listio: SIGEV_SIGNAL */ 114 int np_lio_port; /* listio: SIGEV_THREAD or SIGEV_PORT */ 115 void *np_lio_user; 116 int np_lio_event; 117 uintptr_t np_lio_object; 118 }; 119 120 struct aio_req { 121 /* 122 * fields protected by _aio_mutex lock. 123 */ 124 aio_req_t *req_link; /* hash/freelist chain link */ 125 /* 126 * when req is on the doneq, then req_next is protected by 127 * the _aio_mutex lock. when the req is on a work q, then 128 * req_next is protected by a worker's work_qlock1 lock. 129 */ 130 aio_req_t *req_next; /* request/done queue link */ 131 aio_req_t *req_prev; /* double linked list */ 132 /* 133 * fields protected by a worker's work_qlock1 lock. 134 */ 135 char req_state; /* AIO_REQ_QUEUED, ... */ 136 /* 137 * fields require no locking. 138 */ 139 char req_type; /* AIO_POSIX_REQ or not */ 140 char req_largefile; /* largefile operation */ 141 char req_op; /* AIOREAD, etc. */ 142 aio_worker_t *req_worker; /* associate request with worker */ 143 aio_result_t *req_resultp; /* address of result buffer */ 144 aio_args_t req_args; /* arglist */ 145 aio_lio_t *req_head; /* list head for LIO */ 146 struct sigevent req_sigevent; 147 void *req_aiocbp; /* ptr to aiocb or aiocb64 */ 148 notif_param_t req_notify; /* notification parameters */ 149 }; 150 151 /* special lio type that destroys itself when lio refcnt becomes zero */ 152 #define LIO_FSYNC LIO_WAIT+1 153 #define LIO_DESTROY LIO_FSYNC+1 154 155 /* lio flags */ 156 #define LIO_FSYNC_CANCELED 0x1 157 158 /* values for aio_state */ 159 160 #define AIO_REQ_QUEUED 1 161 #define AIO_REQ_INPROGRESS 2 162 #define AIO_REQ_CANCELED 3 163 #define AIO_REQ_DONE 4 164 #define AIO_REQ_FREE 5 165 #define AIO_REQ_DONEQ 6 166 167 /* use KAIO in _aio_rw() */ 168 #define AIO_NO_KAIO 0x0 169 #define AIO_KAIO 0x1 170 #define AIO_NO_DUPS 0x2 171 172 #define AIO_POSIX_REQ 0x1 173 174 #define CHECK 1 175 #define NOCHECK 2 176 #define CHECKED 3 177 #define USERAIO 4 178 #define USERAIO_DONE 5 179 180 /* values for _aio_flags */ 181 182 /* if set, _aiodone() notifies aio_waitn about done requests */ 183 #define AIO_WAIT_INPROGRESS 0x1 184 /* if set, _aiodone() wakes up functions waiting for completed I/Os */ 185 #define AIO_IO_WAITING 0x2 186 #define AIO_LIB_WAITN 0x4 /* aio_waitn in progress */ 187 #define AIO_LIB_WAITN_PENDING 0x8 /* aio_waitn requests pending */ 188 189 /* 190 * Before a kaio() system call, the fd will be checked 191 * to ensure that kernel async. I/O is supported for this file. 192 * The only way to find out is if a kaio() call returns ENOTSUP, 193 * so the default will always be to try the kaio() call. Only in 194 * the specific instance of a kaio() call returning ENOTSUP 195 * will we stop submitting kaio() calls for that fd. 196 * If the fd is outside the array bounds, we will allow the kaio() 197 * call. 198 * 199 * The only way that an fd entry can go from ENOTSUP to supported 200 * is if that fd is freed up by a close(), and close will clear 201 * the entry for that fd. 202 * 203 * Each fd gets a bit in the array _kaio_supported[]. 204 * 205 * uint32_t _kaio_supported[MAX_KAIO_FDARRAY_SIZE]; 206 * 207 * Array is MAX_KAIO_ARRAY_SIZE of 32-bit elements, for 8kb. 208 * If more than (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE) 209 * files are open, this can be expanded. 210 */ 211 212 #define MAX_KAIO_FDARRAY_SIZE 2048 213 #define KAIO_FDARRAY_ELEM_SIZE WORD_BIT /* uint32_t */ 214 215 #define MAX_KAIO_FDS (MAX_KAIO_FDARRAY_SIZE * KAIO_FDARRAY_ELEM_SIZE) 216 217 #define VALID_FD(fdes) ((fdes) >= 0 && (fdes) < MAX_KAIO_FDS) 218 219 #define KAIO_SUPPORTED(fdes) \ 220 (!VALID_FD(fdes) || \ 221 ((_kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] & \ 222 (uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE))) == 0)) 223 224 #define SET_KAIO_NOT_SUPPORTED(fdes) \ 225 if (VALID_FD(fdes)) \ 226 _kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] |= \ 227 (uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE)) 228 229 #define CLEAR_KAIO_SUPPORTED(fdes) \ 230 if (VALID_FD(fdes)) \ 231 _kaio_supported[(fdes) / KAIO_FDARRAY_ELEM_SIZE] &= \ 232 ~(uint32_t)(1 << ((fdes) % KAIO_FDARRAY_ELEM_SIZE)) 233 234 struct aio_worker { 235 aio_worker_t *work_forw; /* forward link in list of workers */ 236 aio_worker_t *work_backw; /* backwards link in list of workers */ 237 mutex_t work_qlock1; /* lock for work queue 1 */ 238 cond_t work_idle_cv; /* place to sleep when idle */ 239 aio_req_t *work_head1; /* head of work request queue 1 */ 240 aio_req_t *work_tail1; /* tail of work request queue 1 */ 241 aio_req_t *work_next1; /* work queue one's next pointer */ 242 aio_req_t *work_prev1; /* last request done from queue 1 */ 243 aio_req_t *work_req; /* active work request */ 244 thread_t work_tid; /* worker's thread-id */ 245 int work_count1; /* length of work queue one */ 246 int work_done1; /* number of requests done */ 247 int work_minload1; /* min length of queue */ 248 int work_idleflg; /* when set, worker is idle */ 249 sigjmp_buf work_jmp_buf; /* cancellation point */ 250 }; 251 252 struct aio_hash { /* resultp hash table */ 253 mutex_t hash_lock; 254 aio_req_t *hash_ptr; 255 #if !defined(_LP64) 256 void *hash_pad; /* ensure sizeof (aio_hash_t) == 32 */ 257 #endif 258 }; 259 260 extern aio_hash_t *_aio_hash; 261 262 #define HASHSZ 2048 /* power of 2 */ 263 #define AIOHASH(resultp) ((((uintptr_t)(resultp) >> 17) ^ \ 264 ((uintptr_t)(resultp) >> 2)) & (HASHSZ - 1)) 265 #define POSIX_AIO(x) ((x)->req_type == AIO_POSIX_REQ) 266 267 extern int __uaio_init(void); 268 extern void _kaio_init(void); 269 extern intptr_t _kaio(int, ...); 270 extern int _aiorw(int, caddr_t, int, offset_t, int, aio_result_t *, int); 271 extern int _aio_rw(aiocb_t *, aio_lio_t *, aio_worker_t **, int, int); 272 #if !defined(_LP64) 273 extern int _aio_rw64(aiocb64_t *, aio_lio_t *, aio_worker_t **, int, int); 274 #endif 275 extern int _aio_create_worker(aio_req_t *, int); 276 extern int _aio_cancel_req(aio_worker_t *, aio_req_t *, int *, int *); 277 extern int aiocancel_all(int); 278 extern void _aio_close(int); 279 extern void _aio_closerange(int, int); 280 extern void aio_panic(const char *) __NORETURN; 281 extern aio_req_t *_aio_hash_find(aio_result_t *); 282 extern aio_req_t *_aio_hash_del(aio_result_t *); 283 extern void _aio_req_mark_done(aio_req_t *); 284 extern void _aio_waitn_wakeup(void); 285 extern aio_worker_t *_aio_worker_alloc(void); 286 extern void _aio_worker_free(void *); 287 extern aio_req_t *_aio_req_alloc(void); 288 extern void _aio_req_free(aio_req_t *); 289 extern aio_lio_t *_aio_lio_alloc(void); 290 extern void _aio_lio_free(aio_lio_t *); 291 extern int _aio_idle(aio_worker_t *); 292 extern void *_aio_do_request(void *); 293 extern void *_aio_do_notify(void *); 294 extern void _lio_remove(aio_req_t *); 295 extern aio_req_t *_aio_req_remove(aio_req_t *); 296 extern int _aio_get_timedelta(timespec_t *, timespec_t *); 297 extern aio_result_t *_aio_req_done(void); 298 extern void _aio_set_result(aio_req_t *, ssize_t, int); 299 extern int _aio_sigev_thread_init(struct sigevent *); 300 extern int _aio_sigev_thread(aiocb_t *); 301 #if !defined(_LP64) 302 extern int _aio_sigev_thread64(aiocb64_t *); 303 #endif 304 305 extern aio_worker_t *_kaiowp; /* points to kaio cleanup thread */ 306 extern aio_worker_t *__workers_rw; /* list of all rw workers */ 307 extern aio_worker_t *__nextworker_rw; /* worker chosen for next rw request */ 308 extern int __rw_workerscnt; /* number of rw workers */ 309 extern aio_worker_t *__workers_no; /* list of all notification workers */ 310 extern aio_worker_t *__nextworker_no; /* worker chosen, next notification */ 311 extern int __no_workerscnt; /* number of notification workers */ 312 extern mutex_t __aio_initlock; /* makes aio initialization atomic */ 313 extern cond_t __aio_initcv; 314 extern int __aio_initbusy; 315 extern mutex_t __aio_mutex; /* global aio lock */ 316 extern cond_t _aio_iowait_cv; /* wait for userland I/Os */ 317 extern cond_t _aio_waitn_cv; /* wait for end of aio_waitn */ 318 extern int _max_workers; /* max number of workers permitted */ 319 extern int _min_workers; /* min number of workers */ 320 extern sigset_t _worker_set; /* worker's signal mask */ 321 extern int _aio_worker_cnt; /* number of AIO workers */ 322 extern int _sigio_enabled; /* when set, send SIGIO signal */ 323 extern pid_t __pid; /* process's PID */ 324 extern int __uaio_ok; /* indicates if aio is initialized */ 325 extern int _kaio_ok; /* indicates if kaio is initialized */ 326 extern pthread_key_t _aio_key; /* for thread-specific data */ 327 extern aio_req_t *_aio_done_tail; /* list of done requests */ 328 extern aio_req_t *_aio_done_head; 329 extern aio_req_t *_aio_doneq; 330 extern int _aio_freelist_cnt; 331 extern int _aio_allocated_cnt; 332 extern int _aio_donecnt; 333 extern int _aio_doneq_cnt; 334 extern int _aio_waitncnt; /* # of requests for aio_waitn */ 335 extern int _aio_outstand_cnt; /* # of outstanding requests */ 336 extern int _kaio_outstand_cnt; /* # of outstanding kaio requests */ 337 extern int _aio_req_done_cnt; /* req. done but not in "done queue" */ 338 extern int _aio_kernel_suspend; /* active kernel kaio calls */ 339 extern int _aio_suscv_cnt; /* aio_suspend calls waiting on cv's */ 340 extern int _aiowait_flag; /* when set, aiowait() is inprogress */ 341 extern int _aio_flags; /* see defines, above */ 342 extern uint32_t *_kaio_supported; 343 344 extern const sigset_t maskset; /* all maskable signals */ 345 346 #ifdef __cplusplus 347 } 348 #endif 349 350 #endif /* _ASYNCIO_H */ 351