50 static void fdb_zero_holes(fdbuffer_t *fdb);
56 	fdbuffer_t *fdb = buf;  in fdb_cache_constructor()  local
58 	mutex_init(&fdb->fd_mutex, NULL, MUTEX_DEFAULT, NULL);  in fdb_cache_constructor()
67 	fdbuffer_t *fdb = buf;  in fdb_cache_destructor()  local
69 	mutex_destroy(&fdb->fd_mutex);  in fdb_cache_destructor()
81 fdb_prepare(fdbuffer_t *fdb)  in fdb_prepare()  argument
83 	fdb->fd_holes = NULL;  in fdb_prepare()
84 	fdb->fd_iofunc = NULL;  in fdb_prepare()
85 	fdb->fd_iargp = NULL;  in fdb_prepare()
86 	fdb->fd_parentbp = NULL;  in fdb_prepare()
87 	fdb->fd_resid = 0;  in fdb_prepare()
88 	fdb->fd_iocount = 0;  in fdb_prepare()
89 	fdb->fd_iodispatch = 0;  in fdb_prepare()
90 	fdb->fd_err = 0;  in fdb_prepare()
96 	fdbuffer_t *fdb;  in fdb_page_create()  local
104 	fdb = kmem_cache_alloc(fdb_cache, KM_SLEEP);  in fdb_page_create()
106 	fdb_prepare(fdb);  in fdb_page_create()
108 	fdb->fd_type = FDB_PAGEIO;  in fdb_page_create()
109 	fdb->fd_len = len;  in fdb_page_create()
110 	fdb->fd_state = flags;  in fdb_page_create()
111 	fdb->fd_pages = pp;  in fdb_page_create()
113 	return (fdb);  in fdb_page_create()
124 	fdbuffer_t *fdb;  in fdb_addr_create()  local
132 	fdb = kmem_cache_alloc(fdb_cache, KM_SLEEP);  in fdb_addr_create()
134 	fdb_prepare(fdb);  in fdb_addr_create()
136 	fdb->fd_type = FDB_VADDR;  in fdb_addr_create()
137 	fdb->fd_len = len;  in fdb_addr_create()
138 	fdb->fd_state = flags;  in fdb_addr_create()
139 	fdb->fd_addr = addr;  in fdb_addr_create()
140 	fdb->fd_shadow = pplist;  in fdb_addr_create()
141 	fdb->fd_procp = procp;  in fdb_addr_create()
143 	return (fdb);  in fdb_addr_create()
147 fdb_set_iofunc(fdbuffer_t *fdb, fdb_iodone_t iofunc, void *ioargp, int flag)  in fdb_set_iofunc()  argument
149 	ASSERT(fdb);  in fdb_set_iofunc()
153 	fdb->fd_iofunc = iofunc;  in fdb_set_iofunc()
154 	fdb->fd_iargp = ioargp;  in fdb_set_iofunc()
156 	mutex_enter(&fdb->fd_mutex);  in fdb_set_iofunc()
159 		fdb->fd_state |= FDB_ICALLBACK;  in fdb_set_iofunc()
161 	fdb->fd_state |= FDB_ASYNC;  in fdb_set_iofunc()
163 	mutex_exit(&fdb->fd_mutex);  in fdb_set_iofunc()
167 fdb_get_error(fdbuffer_t *fdb)  in fdb_get_error()  argument
169 	return (fdb->fd_err);  in fdb_get_error()
173 fdb_free(fdbuffer_t *fdb)  in fdb_free()  argument
178 	    (void *)fdb, fdb->fd_state));  in fdb_free()
180 	ASSERT(fdb);  in fdb_free()
181 	ASSERT(fdb->fd_iodispatch == 0);  in fdb_free()
183 	if (fdb->fd_state & FDB_ZEROHOLE) {  in fdb_free()
184 		fdb_zero_holes(fdb);  in fdb_free()
187 	for (fdh = fdb->fd_holes; fdh; ) {  in fdb_free()
193 	if (fdb->fd_parentbp != NULL) {  in fdb_free()
194 		switch (fdb->fd_type) {  in fdb_free()
196 			pageio_done(fdb->fd_parentbp);  in fdb_free()
199 			kmem_free(fdb->fd_parentbp, sizeof (struct buf));  in fdb_free()
207 	kmem_cache_free(fdb_cache, fdb);  in fdb_free()
218 fdb_add_hole(fdbuffer_t *fdb, u_offset_t off, size_t len)  in fdb_add_hole()  argument
222 	ASSERT(fdb);  in fdb_add_hole()
223 	ASSERT(off < fdb->fd_len);  in fdb_add_hole()
232 	if (fdb->fd_holes == NULL || off < fdb->fd_holes->off) {  in fdb_add_hole()
233 		this_hole->next_hole = fdb->fd_holes;  in fdb_add_hole()
234 		fdb->fd_holes = this_hole;  in fdb_add_hole()
236 		fdb_holes_t *fdhp = fdb->fd_holes;  in fdb_add_hole()
245 	mutex_enter(&fdb->fd_mutex);  in fdb_add_hole()
247 	fdb->fd_iocount += len;  in fdb_add_hole()
249 	mutex_exit(&fdb->fd_mutex);  in fdb_add_hole()
253 fdb_get_holes(fdbuffer_t *fdb)  in fdb_get_holes()  argument
255 	ASSERT(fdb);  in fdb_get_holes()
257 	if (fdb->fd_state & FDB_ZEROHOLE) {  in fdb_get_holes()
258 		fdb_zero_holes(fdb);  in fdb_get_holes()
261 	return (fdb->fd_holes);  in fdb_get_holes()
270 fdb_zero_holes(fdbuffer_t *fdb)  in fdb_zero_holes()  argument
272 	fdb_holes_t *fdh = fdb->fd_holes;  in fdb_zero_holes()
275 	ASSERT(fdb);  in fdb_zero_holes()
280 	switch (fdb->fd_type) {  in fdb_zero_holes()
282 		pp = fdb->fd_pages;  in fdb_zero_holes()
330 			bzero(fdb->fd_addr + fdh->off, fdh->len);  in fdb_zero_holes()
343 fdb_iosetup(fdbuffer_t *fdb, u_offset_t off, size_t len, struct vnode *vp,  in fdb_iosetup()  argument
350 	    off, len, fdb->fd_len, fdb->fd_state));  in fdb_iosetup()
352 	ASSERT(fdb);  in fdb_iosetup()
354 	mutex_enter(&fdb->fd_mutex);  in fdb_iosetup()
356 	ASSERT(((b_flags & B_READ) && (fdb->fd_state & FDB_READ)) ||  in fdb_iosetup()
357 	    ((b_flags & B_WRITE) && (fdb->fd_state & FDB_WRITE)));  in fdb_iosetup()
366 	    ((fdb->fd_state & FDB_ASYNC) || !(fdb->fd_state & FDB_SYNC))) ||  in fdb_iosetup()
368 	    ((fdb->fd_state & FDB_SYNC) || !(fdb->fd_state & FDB_ASYNC))));  in fdb_iosetup()
371 	fdb->fd_state |= b_flags & B_ASYNC ? FDB_ASYNC : FDB_SYNC;  in fdb_iosetup()
373 	fdb->fd_iodispatch++;  in fdb_iosetup()
375 	ASSERT((fdb->fd_state & FDB_ASYNC && fdb->fd_iofunc != NULL) ||  in fdb_iosetup()
376 	    fdb->fd_state & FDB_SYNC);  in fdb_iosetup()
378 	mutex_exit(&fdb->fd_mutex);  in fdb_iosetup()
381 	ASSERT(off+len <= fdb->fd_len);  in fdb_iosetup()
383 	switch (fdb->fd_type) {  in fdb_iosetup()
385 		if (fdb->fd_parentbp == NULL) {  in fdb_iosetup()
386 			bp = pageio_setup(fdb->fd_pages, len, vp, b_flags);  in fdb_iosetup()
387 			fdb->fd_parentbp = bp;  in fdb_iosetup()
391 		if (fdb->fd_parentbp == NULL) {  in fdb_iosetup()
396 			bp->b_proc = fdb->fd_procp;  in fdb_iosetup()
399 			bp->b_un.b_addr = fdb->fd_addr;  in fdb_iosetup()
400 			bp->b_shadow = fdb->fd_shadow;  in fdb_iosetup()
401 			if (fdb->fd_shadow != NULL)  in fdb_iosetup()
403 			fdb->fd_parentbp = bp;  in fdb_iosetup()
411 	bp = bioclone(fdb->fd_parentbp, off, len, 0, 0,  in fdb_iosetup()
415 	bp->b_forw = (struct buf *)fdb;  in fdb_iosetup()
424 fdb_get_iolen(fdbuffer_t *fdb)  in fdb_get_iolen()  argument
426 	ASSERT(fdb);  in fdb_get_iolen()
427 	ASSERT(fdb->fd_iodispatch == 0);  in fdb_get_iolen()
429 	return (fdb->fd_iocount - fdb->fd_resid);  in fdb_get_iolen()
433 fdb_ioerrdone(fdbuffer_t *fdb, int error)  in fdb_ioerrdone()  argument
435 	ASSERT(fdb);  in fdb_ioerrdone()
436 	ASSERT(fdb->fd_state & FDB_ASYNC);  in fdb_ioerrdone()
440 	    fdb->fd_len, fdb->fd_state, error));  in fdb_ioerrdone()
442 	mutex_enter(&fdb->fd_mutex);  in fdb_ioerrdone()
444 	fdb->fd_err = error;  in fdb_ioerrdone()
447 		fdb->fd_state |= FDB_ERROR;  in fdb_ioerrdone()
449 		fdb->fd_state |= FDB_DONE;  in fdb_ioerrdone()
454 	if (fdb->fd_iodispatch > 0) {  in fdb_ioerrdone()
455 		mutex_exit(&fdb->fd_mutex);  in fdb_ioerrdone()
459 	mutex_exit(&fdb->fd_mutex);  in fdb_ioerrdone()
460 	fdb->fd_iofunc(fdb, fdb->fd_iargp, NULL);  in fdb_ioerrdone()
466 	fdbuffer_t *fdb = (fdbuffer_t *)bp->b_forw;  in fdb_iodone()  local
470 	ASSERT(fdb);  in fdb_iodone()
474 	    fdb->fd_len, fdb->fd_state, geterror(bp)));  in fdb_iodone()
479 	mutex_enter(&fdb->fd_mutex);  in fdb_iodone()
481 	icallback = fdb->fd_state & FDB_ICALLBACK;  in fdb_iodone()
482 	isasync = fdb->fd_state & FDB_ASYNC;  in fdb_iodone()
484 	ASSERT(fdb->fd_iodispatch > 0);  in fdb_iodone()
485 	fdb->fd_iodispatch--;  in fdb_iodone()
488 		fdb->fd_err = error;  in fdb_iodone()
490 			fdb->fd_resid += bp->b_resid;  in fdb_iodone()
492 			fdb->fd_resid += bp->b_bcount;  in fdb_iodone()
495 	fdb->fd_iocount += bp->b_bcount;  in fdb_iodone()
510 	if ((fdb->fd_iodispatch == 0) &&  in fdb_iodone()
511 	    (fdb->fd_state & (FDB_ERROR|FDB_DONE))) {  in fdb_iodone()
513 		mutex_exit(&fdb->fd_mutex);  in fdb_iodone()
516 			fdb->fd_iofunc(fdb, fdb->fd_iargp, bp);  in fdb_iodone()
521 		mutex_exit(&fdb->fd_mutex);  in fdb_iodone()
524 			fdb->fd_iofunc(fdb, fdb->fd_iargp, bp);  in fdb_iodone()