144 #define	BD_LOCKPTR(bd)		(&(bd)->bd_cleanq->bq_lock)  argument
145 #define	BD_LOCK(bd)		mtx_lock(BD_LOCKPTR((bd)))  argument
146 #define	BD_UNLOCK(bd)		mtx_unlock(BD_LOCKPTR((bd)))  argument
147 #define	BD_ASSERT_LOCKED(bd)	mtx_assert(BD_LOCKPTR((bd)), MA_OWNED)  argument
148 #define	BD_RUN_LOCKPTR(bd)	(&(bd)->bd_run_lock)  argument
149 #define	BD_RUN_LOCK(bd)		mtx_lock(BD_RUN_LOCKPTR((bd)))  argument
150 #define	BD_RUN_UNLOCK(bd)	mtx_unlock(BD_RUN_LOCKPTR((bd)))  argument
151 #define	BD_DOMAIN(bd)		(bd - bdomain)  argument
200 static void bd_init(struct bufdomain *bd);
201 static int bd_flushall(struct bufdomain *bd);
543 bd_clear(struct bufdomain *bd)  in bd_clear()  argument
547 	if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers)  in bd_clear()
548 		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);  in bd_clear()
549 	if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers)  in bd_clear()
550 		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);  in bd_clear()
561 bd_set(struct bufdomain *bd)  in bd_set()  argument
565 	if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers)  in bd_set()
566 		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);  in bd_set()
567 	if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers)  in bd_set()
568 		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);  in bd_set()
581 	struct bufdomain *bd;  in bdirtysub()  local
584 	bd = bufdomain(bp);  in bdirtysub()
585 	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1);  in bdirtysub()
586 	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)  in bdirtysub()
588 	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)  in bdirtysub()
589 		bd_clear(bd);  in bdirtysub()
601 	struct bufdomain *bd;  in bdirtyadd()  local
608 	bd = bufdomain(bp);  in bdirtyadd()
609 	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1);  in bdirtyadd()
610 	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)  in bdirtyadd()
612 	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)  in bdirtyadd()
613 		bd_set(bd);  in bdirtyadd()
622 bufspace_daemon_wakeup(struct bufdomain *bd)  in bufspace_daemon_wakeup()  argument
628 	if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) {  in bufspace_daemon_wakeup()
629 		BD_RUN_LOCK(bd);  in bufspace_daemon_wakeup()
630 		atomic_store_int(&bd->bd_running, 1);  in bufspace_daemon_wakeup()
631 		wakeup(&bd->bd_running);  in bufspace_daemon_wakeup()
632 		BD_RUN_UNLOCK(bd);  in bufspace_daemon_wakeup()
645 	struct bufdomain *bd;  in bufspace_adjust()  local
651 	bd = bufdomain(bp);  in bufspace_adjust()
654 		atomic_subtract_long(&bd->bd_bufspace, -diff);  in bufspace_adjust()
656 		space = atomic_fetchadd_long(&bd->bd_bufspace, diff);  in bufspace_adjust()
658 		if (space < bd->bd_bufspacethresh &&  in bufspace_adjust()
659 		    space + diff >= bd->bd_bufspacethresh)  in bufspace_adjust()
660 			bufspace_daemon_wakeup(bd);  in bufspace_adjust()
672 bufspace_reserve(struct bufdomain *bd, int size, bool metadata)  in bufspace_reserve()  argument
678 		limit = bd->bd_maxbufspace;  in bufspace_reserve()
680 		limit = bd->bd_hibufspace;  in bufspace_reserve()
681 	space = atomic_fetchadd_long(&bd->bd_bufspace, size);  in bufspace_reserve()
684 		atomic_subtract_long(&bd->bd_bufspace, size);  in bufspace_reserve()
689 	if (space < bd->bd_bufspacethresh && new >= bd->bd_bufspacethresh)  in bufspace_reserve()
690 		bufspace_daemon_wakeup(bd);  in bufspace_reserve()
701 bufspace_release(struct bufdomain *bd, int size)  in bufspace_release()  argument
704 	atomic_subtract_long(&bd->bd_bufspace, size);  in bufspace_release()
715 bufspace_wait(struct bufdomain *bd, struct vnode *vp, int gbflags,  in bufspace_wait()  argument
725 	BD_LOCK(bd);  in bufspace_wait()
726 	while (bd->bd_wanted) {  in bufspace_wait()
729 			BD_UNLOCK(bd);  in bufspace_wait()
750 			fl = buf_flush(vp, bd, flushbufqtarget);  in bufspace_wait()
752 			BD_LOCK(bd);  in bufspace_wait()
755 			if (bd->bd_wanted == 0)  in bufspace_wait()
758 		error = msleep(&bd->bd_wanted, BD_LOCKPTR(bd),  in bufspace_wait()
763 	BD_UNLOCK(bd);  in bufspace_wait()
769 	struct bufdomain *bd = arg;  in bufspace_daemon_shutdown()  local
775 	BD_RUN_LOCK(bd);  in bufspace_daemon_shutdown()
776 	bd->bd_shutdown = true;  in bufspace_daemon_shutdown()
777 	wakeup(&bd->bd_running);  in bufspace_daemon_shutdown()
778 	error = msleep(&bd->bd_shutdown, BD_RUN_LOCKPTR(bd), 0,  in bufspace_daemon_shutdown()
780 	BD_RUN_UNLOCK(bd);  in bufspace_daemon_shutdown()
795 	struct bufdomain *bd = arg;  in bufspace_daemon()  local
797 	EVENTHANDLER_REGISTER(shutdown_pre_sync, bufspace_daemon_shutdown, bd,  in bufspace_daemon()
800 	BD_RUN_LOCK(bd);  in bufspace_daemon()
801 	while (!bd->bd_shutdown) {  in bufspace_daemon()
802 		BD_RUN_UNLOCK(bd);  in bufspace_daemon()
829 		while (bd->bd_bufspace > bd->bd_lobufspace ||  in bufspace_daemon()
830 		    bd->bd_freebuffers < bd->bd_hifreebuffers) {  in bufspace_daemon()
831 			if (buf_recycle(bd, false) != 0) {  in bufspace_daemon()
832 				if (bd_flushall(bd))  in bufspace_daemon()
843 				BD_LOCK(bd);  in bufspace_daemon()
844 				if (bd->bd_wanted) {  in bufspace_daemon()
845 					msleep(&bd->bd_wanted, BD_LOCKPTR(bd),  in bufspace_daemon()
848 					BD_UNLOCK(bd);  in bufspace_daemon()
859 		BD_RUN_LOCK(bd);  in bufspace_daemon()
860 		if (bd->bd_shutdown)  in bufspace_daemon()
862 		atomic_store_int(&bd->bd_running, 0);  in bufspace_daemon()
863 		if (bd->bd_bufspace < bd->bd_bufspacethresh &&  in bufspace_daemon()
864 		    bd->bd_freebuffers > bd->bd_lofreebuffers) {  in bufspace_daemon()
865 			msleep(&bd->bd_running, BD_RUN_LOCKPTR(bd),  in bufspace_daemon()
869 			atomic_store_int(&bd->bd_running, 1);  in bufspace_daemon()
872 	wakeup(&bd->bd_shutdown);  in bufspace_daemon()
873 	BD_RUN_UNLOCK(bd);  in bufspace_daemon()
1330 		struct bufdomain *bd;  in bufinit()  local
1332 		bd = &bdomain[i];  in bufinit()
1333 		bd_init(bd);  in bufinit()
1334 		bd->bd_freebuffers = nbuf / buf_domains;  in bufinit()
1335 		bd->bd_hifreebuffers = hifreebuffers / buf_domains;  in bufinit()
1336 		bd->bd_lofreebuffers = lofreebuffers / buf_domains;  in bufinit()
1337 		bd->bd_bufspace = 0;  in bufinit()
1338 		bd->bd_maxbufspace = maxbufspace / buf_domains;  in bufinit()
1339 		bd->bd_hibufspace = hibufspace / buf_domains;  in bufinit()
1340 		bd->bd_lobufspace = lobufspace / buf_domains;  in bufinit()
1341 		bd->bd_bufspacethresh = bufspacethresh / buf_domains;  in bufinit()
1342 		bd->bd_numdirtybuffers = 0;  in bufinit()
1343 		bd->bd_hidirtybuffers = hidirtybuffers / buf_domains;  in bufinit()
1344 		bd->bd_lodirtybuffers = lodirtybuffers / buf_domains;  in bufinit()
1345 		bd->bd_dirtybufthresh = dirtybufthresh / buf_domains;  in bufinit()
1347 		bd->bd_lim = nbuf / buf_domains / 50 / mp_ncpus;  in bufinit()
1606 	struct bufdomain *bd;  in binsfree()  local
1627 	bd = bufdomain(bp);  in binsfree()
1629 		if (bd->bd_lim != 0)  in binsfree()
1630 			bq = &bd->bd_subq[PCPU_GET(cpuid)];  in binsfree()
1632 			bq = bd->bd_cleanq;  in binsfree()
1634 		bq = &bd->bd_dirtyq;  in binsfree()
1726 buf_alloc(struct bufdomain *bd)  in buf_alloc()  argument
1737 	freebufs = atomic_fetchadd_int(&bd->bd_freebuffers, -1);  in buf_alloc()
1741 		atomic_add_int(&bd->bd_freebuffers, 1);  in buf_alloc()
1742 		bufspace_daemon_wakeup(bd);  in buf_alloc()
1749 	if (freebufs == bd->bd_lofreebuffers)  in buf_alloc()
1750 		bufspace_daemon_wakeup(bd);  in buf_alloc()
1769 	bp->b_domain = BD_DOMAIN(bd);  in buf_alloc()
1801 buf_recycle(struct bufdomain *bd, bool kva)  in buf_recycle()  argument
1809 	bq = bd->bd_cleanq;  in buf_recycle()
1811 	KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd),  in buf_recycle()
1859 		KASSERT(bp->b_domain == BD_DOMAIN(bd),  in buf_recycle()
1861 		    bp->b_domain, (int)BD_DOMAIN(bd)));  in buf_recycle()
1883 	bd->bd_wanted = 1;  in buf_recycle()
1937 bd_init(struct bufdomain *bd)  in bd_init()  argument
1942 	bd->bd_subq = mallocarray(mp_maxid + 2, sizeof(struct bufqueue),  in bd_init()
1944 	bd->bd_cleanq = &bd->bd_subq[mp_maxid + 1];  in bd_init()
1945 	bq_init(bd->bd_cleanq, QUEUE_CLEAN, mp_maxid + 1, "bufq clean lock");  in bd_init()
1946 	bq_init(&bd->bd_dirtyq, QUEUE_DIRTY, -1, "bufq dirty lock");  in bd_init()
1948 		bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i,  in bd_init()
1950 	mtx_init(&bd->bd_run_lock, "bufspace daemon run lock", NULL, MTX_DEF);  in bd_init()
1983 bd_flush(struct bufdomain *bd, struct bufqueue *bq)  in bd_flush()  argument
1988 	if (bq != bd->bd_cleanq) {  in bd_flush()
1989 		BD_LOCK(bd);  in bd_flush()
1992 			TAILQ_INSERT_TAIL(&bd->bd_cleanq->bq_queue, bp,  in bd_flush()
1994 			bp->b_subqueue = bd->bd_cleanq->bq_subqueue;  in bd_flush()
1996 		bd->bd_cleanq->bq_len += bq->bq_len;  in bd_flush()
1999 	if (bd->bd_wanted) {  in bd_flush()
2000 		bd->bd_wanted = 0;  in bd_flush()
2001 		wakeup(&bd->bd_wanted);  in bd_flush()
2003 	if (bq != bd->bd_cleanq)  in bd_flush()
2004 		BD_UNLOCK(bd);  in bd_flush()
2008 bd_flushall(struct bufdomain *bd)  in bd_flushall()  argument
2014 	if (bd->bd_lim == 0)  in bd_flushall()
2018 		bq = &bd->bd_subq[i];  in bd_flushall()
2022 		bd_flush(bd, bq);  in bd_flushall()
2033 	struct bufdomain *bd;  in bq_insert()  local
2038 	bd = bufdomain(bp);  in bq_insert()
2042 			bq = bd->bd_cleanq;  in bq_insert()
2065 		if (bd->bd_wanted || (bq != bd->bd_cleanq &&  in bq_insert()
2066 		    bq->bq_len >= bd->bd_lim))  in bq_insert()
2067 			bd_flush(bd, bq);  in bq_insert()
2403 	struct bufdomain *bd;  in bufbdflush()  local
2405 	bd = &bdomain[bo->bo_domain];  in bufbdflush()
2406 	if (bo->bo_dirty.bv_cnt > bd->bd_dirtybufthresh + 10) {  in bufbdflush()
2409 	} else if (bo->bo_dirty.bv_cnt > bd->bd_dirtybufthresh) {  in bufbdflush()
3356 	struct bufdomain *bd;  in getnewbuf()  local
3372 		bd = &bdomain[0];  in getnewbuf()
3374 		bd = &bdomain[vp->v_bufobj.bo_domain];  in getnewbuf()
3380 		    bufspace_reserve(bd, maxsize, metadata) != 0) {  in getnewbuf()
3385 		if ((bp = buf_alloc(bd)) == NULL) {  in getnewbuf()
3392 	} while (buf_recycle(bd, false) == 0);  in getnewbuf()
3395 		bufspace_release(bd, maxsize);  in getnewbuf()
3400 	bufspace_wait(bd, vp, gbflags, slpflag, slptimeo);  in getnewbuf()
3420 buf_flush(struct vnode *vp, struct bufdomain *bd, int target)  in buf_flush()  argument
3424 	flushed = flushbufqueues(vp, bd, target, 0);  in buf_flush()
3433 		flushbufqueues(vp, bd, target, 1);  in buf_flush()
3459 	struct bufdomain *bd;  in buf_daemon()  local
3503 			bd = &bdomain[i];  in buf_daemon()
3505 				lodirty = bd->bd_numdirtybuffers / 2;  in buf_daemon()
3507 				lodirty = bd->bd_lodirtybuffers;  in buf_daemon()
3508 			while (bd->bd_numdirtybuffers > lodirty) {  in buf_daemon()
3509 				if (buf_flush(NULL, bd,  in buf_daemon()
3510 				    bd->bd_numdirtybuffers - lodirty) == 0)  in buf_daemon()
3572 flushbufqueues(struct vnode *lvp, struct bufdomain *bd, int target,  in flushbufqueues()  argument
3586 	bq = &bd->bd_dirtyq;  in flushbufqueues()
5543 	struct bufdomain *bd;  in DB_SHOW_COMMAND_FLAGS()  local
5551 		bd = &bdomain[i];  in DB_SHOW_COMMAND_FLAGS()
5553 		db_printf("\tfreebufs\t%d\n", bd->bd_freebuffers);  in DB_SHOW_COMMAND_FLAGS()
5554 		db_printf("\tlofreebufs\t%d\n", bd->bd_lofreebuffers);  in DB_SHOW_COMMAND_FLAGS()
5555 		db_printf("\thifreebufs\t%d\n", bd->bd_hifreebuffers);  in DB_SHOW_COMMAND_FLAGS()
5557 		db_printf("\tbufspace\t%ld\n", bd->bd_bufspace);  in DB_SHOW_COMMAND_FLAGS()
5558 		db_printf("\tmaxbufspace\t%ld\n", bd->bd_maxbufspace);  in DB_SHOW_COMMAND_FLAGS()
5559 		db_printf("\thibufspace\t%ld\n", bd->bd_hibufspace);  in DB_SHOW_COMMAND_FLAGS()
5560 		db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace);  in DB_SHOW_COMMAND_FLAGS()
5561 		db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh);  in DB_SHOW_COMMAND_FLAGS()
5563 		db_printf("\tnumdirtybuffers\t%d\n", bd->bd_numdirtybuffers);  in DB_SHOW_COMMAND_FLAGS()
5564 		db_printf("\tlodirtybuffers\t%d\n", bd->bd_lodirtybuffers);  in DB_SHOW_COMMAND_FLAGS()
5565 		db_printf("\thidirtybuffers\t%d\n", bd->bd_hidirtybuffers);  in DB_SHOW_COMMAND_FLAGS()
5566 		db_printf("\tdirtybufthresh\t%d\n", bd->bd_dirtybufthresh);  in DB_SHOW_COMMAND_FLAGS()
5569 		TAILQ_FOREACH(bp, &bd->bd_cleanq->bq_queue, b_freelist)  in DB_SHOW_COMMAND_FLAGS()
5572 		    bd->bd_cleanq->bq_len, total);  in DB_SHOW_COMMAND_FLAGS()
5574 		TAILQ_FOREACH(bp, &bd->bd_dirtyq.bq_queue, b_freelist)  in DB_SHOW_COMMAND_FLAGS()
5577 		    bd->bd_dirtyq.bq_len, total);  in DB_SHOW_COMMAND_FLAGS()
5578 		db_printf("\twakeup\t\t%d\n", bd->bd_wanted);  in DB_SHOW_COMMAND_FLAGS()
5579 		db_printf("\tlim\t\t%d\n", bd->bd_lim);  in DB_SHOW_COMMAND_FLAGS()
5582 			db_printf("%d, ", bd->bd_subq[j].bq_len);  in DB_SHOW_COMMAND_FLAGS()