77 #define	CRB_FREE(crb, me) \  argument
81 	(me)->me_crb = NULL;
83 #define	CRB_RELE(me) { \  argument
84 	crb_t *crb = (me)->me_crb; \
86 		CRB_FREE(crb, me) \
95 #define	HANDLE_DQUOT(me, melist) { \  argument
96 	if ((me->me_arg) && \
97 	    (me->me_func == ufs_trans_push_quota)) { \
98 		if (!((me->me_dt == melist->me_dt) && \
99 		    (me->me_arg == melist->me_arg) && \
100 		    (me->me_func == melist->me_func))) { \
101 			handle_dquot(me); \
113 	mapentry_t	*me;  in map_free_entries()  local
115 	while ((me = mtm->mtm_next) != (mapentry_t *)mtm) {  in map_free_entries()
116 		me->me_next->me_prev = me->me_prev;  in map_free_entries()
117 		me->me_prev->me_next = me->me_next;  in map_free_entries()
118 		CRB_RELE(me);  in map_free_entries()
119 		kmem_cache_free(mapentry_cache, me);  in map_free_entries()
227 	mapentry_t	*me;  in deltamap_add()  local
250 		for (me = *mep; me; me = me->me_hash) {  in deltamap_add()
251 			if (DATAwithinME(mof, hnb, me)) {  in deltamap_add()
258 				if (me->me_func == ufs_trans_push_quota) {  in deltamap_add()
264 			    (!DATAoverlapME(mof, hnb, me)) ||  in deltamap_add()
265 			    MEwithinDATA(me, mof, hnb));  in deltamap_add()
268 		if (me) {  in deltamap_add()
300 		me = kmem_cache_alloc(mapentry_cache, KM_NOSLEEP);  in deltamap_add()
301 		if (me == NULL) {  in deltamap_add()
303 			me = kmem_cache_alloc(mapentry_cache, KM_SLEEP);  in deltamap_add()
306 		bzero(me, sizeof (mapentry_t));  in deltamap_add()
311 		me->me_mof = mof;  in deltamap_add()
312 		me->me_nb = hnb;  in deltamap_add()
313 		me->me_func = func;  in deltamap_add()
314 		me->me_arg = arg;  in deltamap_add()
315 		me->me_dt = dtyp;  in deltamap_add()
316 		me->me_flags = ME_HASH;  in deltamap_add()
317 		me->me_tid = mtm->mtm_tid;  in deltamap_add()
319 		me->me_hash = *mep;  in deltamap_add()
320 		*mep = me;  in deltamap_add()
321 		me->me_next = (mapentry_t *)mtm;  in deltamap_add()
322 		me->me_prev = mtm->mtm_prev;  in deltamap_add()
323 		mtm->mtm_prev->me_next = me;  in deltamap_add()
324 		mtm->mtm_prev = me;  in deltamap_add()
340 	mapentry_t	*me;  in deltamap_remove()  local
359 		while ((me = *mep) != 0) {  in deltamap_remove()
360 			if (MEwithinDATA(me, mof, hnb)) {  in deltamap_remove()
361 				*mep = me->me_hash;  in deltamap_remove()
362 				me->me_next->me_prev = me->me_prev;  in deltamap_remove()
363 				me->me_prev->me_next = me->me_next;  in deltamap_remove()
364 				me->me_hash = mer;  in deltamap_remove()
365 				mer = me;  in deltamap_remove()
366 				me->me_flags |= ME_LIST;  in deltamap_remove()
367 				me->me_flags &= ~ME_HASH;  in deltamap_remove()
370 				mep = &me->me_hash;  in deltamap_remove()
387 	mapentry_t	*me;  in deltamap_del()  local
391 	while ((me = menext) != 0) {  in deltamap_del()
392 		menext = me->me_hash;  in deltamap_del()
393 		kmem_cache_free(mapentry_cache, me);  in deltamap_del()
410 	mapentry_t	*me;  in deltamap_push()  local
421 	while ((me = mtm->mtm_next) != (mapentry_t *)mtm) {  in deltamap_push()
422 		ASSERT(me->me_func);  in deltamap_push()
423 		func = me->me_func;  in deltamap_push()
424 		dtyp = me->me_dt;  in deltamap_push()
425 		arg = me->me_arg;  in deltamap_push()
426 		mof = me->me_mof;  in deltamap_push()
427 		nb = me->me_nb;  in deltamap_push()
543 	mapentry_t	*me;  in logmap_remove_roll()  local
563 		while ((me = *mep) != 0) {  in logmap_remove_roll()
564 			if ((me->me_flags & ME_ROLL) &&  in logmap_remove_roll()
565 			    (MEwithinDATA(me, mof, hnb))) {  in logmap_remove_roll()
566 				if (me->me_flags & ME_AGE) {  in logmap_remove_roll()
574 				*mep = me->me_hash;  in logmap_remove_roll()
575 				me->me_next->me_prev = me->me_prev;  in logmap_remove_roll()
576 				me->me_prev->me_next = me->me_next;  in logmap_remove_roll()
577 				me->me_flags &= ~(ME_HASH|ME_ROLL);  in logmap_remove_roll()
578 				ASSERT(!(me->me_flags & ME_USER));  in logmap_remove_roll()
583 				if ((me->me_flags & ME_CANCEL) == 0) {  in logmap_remove_roll()
584 					roll_stats[me->me_dt]++;  in logmap_remove_roll()
585 					CRB_RELE(me);  in logmap_remove_roll()
586 					kmem_cache_free(mapentry_cache, me);  in logmap_remove_roll()
589 				mep = &me->me_hash;  in logmap_remove_roll()
610 	mapentry_t *me;  in logmap_next_roll()  local
616 	for (me = logmap->mtm_next; me != (mapentry_t *)logmap;  in logmap_next_roll()
617 	    me = me->me_next) {  in logmap_next_roll()
619 		if (me->me_flags & ME_ROLL) {  in logmap_next_roll()
624 		if (me->me_tid == logmap->mtm_tid) {  in logmap_next_roll()
629 		if (me->me_tid == logmap->mtm_committid) {  in logmap_next_roll()
639 		ASSERT(!(me->me_dt == DT_CANCEL));  in logmap_next_roll()
641 		*mofp = me->me_mof;  in logmap_next_roll()
655 	mapentry_t	*me;  in logmap_list_age()  local
659 	for (me = *age; me; age = &me->me_agenext, me = *age) {  in logmap_list_age()
660 		if (me->me_age > meadd->me_age)  in logmap_list_age()
663 	meadd->me_agenext = me;  in logmap_list_age()
681 	mapentry_t	*me;  in logmap_list_get()  local
706 		for (me = *mep; me; me = me->me_hash) {  in logmap_list_get()
707 			if (me->me_dt == DT_CANCEL)  in logmap_list_get()
709 			if (!DATAoverlapME(mof, hnb, me))  in logmap_list_get()
715 			if (me->me_flags & ME_AGE) {  in logmap_list_get()
720 				for (me = *age; me; me = *age) {  in logmap_list_get()
721 					*age = me->me_agenext;  in logmap_list_get()
722 					me->me_flags &= ~ME_AGE;  in logmap_list_get()
733 				logmap_list_age(age, me);  in logmap_list_get()
734 				crb = me->me_crb;  in logmap_list_get()
740 					if (DATAwithinME(savmof, savnb, me)) {  in logmap_list_get()
760 	mapentry_t	*me, **mep, *age = NULL;  in logmap_list_get_roll()  local
775 	for (me = *mep; me; me = me->me_hash) {  in logmap_list_get_roll()
776 		if (!DATAoverlapME(mof, MAPBLOCKSIZE, me))  in logmap_list_get_roll()
778 		if (me->me_tid == logmap->mtm_tid)  in logmap_list_get_roll()
780 		if (me->me_tid == logmap->mtm_committid)  in logmap_list_get_roll()
782 		if (me->me_dt == DT_CANCEL)  in logmap_list_get_roll()
790 		if (me->me_flags & ME_AGE) {  in logmap_list_get_roll()
791 			for (me = age; me; me = age) {  in logmap_list_get_roll()
792 				age = me->me_agenext;  in logmap_list_get_roll()
793 				me->me_flags &= ~ME_AGE;  in logmap_list_get_roll()
799 			logmap_list_age(&age, me);  in logmap_list_get_roll()
809 	for (me = age; me; me = me->me_agenext) {  in logmap_list_get_roll()
810 		me->me_flags |= ME_ROLL;  in logmap_list_get_roll()
818 		for (me = age; me; me = me->me_agenext) {  in logmap_list_get_roll()
819 			if (me->me_crb != crb) {  in logmap_list_get_roll()
840 	mapentry_t	*me;  in logmap_list_put_roll()  local
844 	for (me = age; me; me = age) {  in logmap_list_put_roll()
845 		age = me->me_agenext;  in logmap_list_put_roll()
846 		me->me_flags &= ~ME_AGE;  in logmap_list_put_roll()
854 	mapentry_t	*me;  in logmap_list_put()  local
858 	for (me = age; me; me = age) {  in logmap_list_put()
859 		age = me->me_agenext;  in logmap_list_put()
860 		me->me_flags &= ~ME_AGE;  in logmap_list_put()
882 	mapentry_t *me;  in logmap_setup_read()  local
895 	for (me = age; me; me = me->me_agenext) {  in logmap_setup_read()
896 		crb = me->me_crb;  in logmap_setup_read()
901 			nb = me->me_nb;  in logmap_setup_read()
902 			mof = me->me_mof;  in logmap_setup_read()
920 		if (me->me_dt != DT_INODE) {  in logmap_setup_read()
1001 	mapentry_t	*me, **mep;  in logmap_abort()  local
1014 	while ((me = mtm->mtm_cancel) != NULL) {  in logmap_abort()
1015 		mtm->mtm_cancel = me->me_cancel;  in logmap_abort()
1016 		me->me_flags &= ~ME_CANCEL;  in logmap_abort()
1017 		me->me_cancel = NULL;  in logmap_abort()
1032 		while ((me = *mep) != NULL) {  in logmap_abort()
1033 			if (me->me_tid == tid ||  in logmap_abort()
1034 			    me->me_tid == mtm->mtm_committid) {  in logmap_abort()
1035 				*mep = me->me_hash;  in logmap_abort()
1036 				me->me_next->me_prev = me->me_prev;  in logmap_abort()
1037 				me->me_prev->me_next = me->me_next;  in logmap_abort()
1038 				if (!(me->me_flags & ME_USER)) {  in logmap_abort()
1041 				CRB_RELE(me);  in logmap_abort()
1042 				kmem_cache_free(mapentry_cache, me);  in logmap_abort()
1045 			mep = &me->me_hash;  in logmap_abort()
1061 logmap_wait_space(mt_map_t *mtm, ml_unit_t *ul, mapentry_t *me)  in logmap_wait_space()  argument
1065 	while (!ldl_has_space(ul, me)) {  in logmap_wait_space()
1090 	mapentry_t	*me;  in logmap_add()  local
1115 		while ((me = *mep) != 0) {  in logmap_add()
1127 			if (MEwithinDATA(me, mof, nb) &&  in logmap_add()
1128 			    ((me->me_flags & (ME_ROLL|ME_CANCEL)) == 0)) {  in logmap_add()
1129 				if (tid == me->me_tid &&  in logmap_add()
1130 				    ((me->me_flags & ME_AGE) == 0)) {  in logmap_add()
1131 					*mep = me->me_hash;  in logmap_add()
1132 					me->me_next->me_prev = me->me_prev;  in logmap_add()
1133 					me->me_prev->me_next = me->me_next;  in logmap_add()
1134 					ASSERT(!(me->me_flags & ME_USER));  in logmap_add()
1142 					if (me->me_dt == DT_QR)  in logmap_add()
1143 						HANDLE_DQUOT(me, melist);  in logmap_add()
1145 					kmem_cache_free(mapentry_cache, me);  in logmap_add()
1148 				me->me_cancel = mtm->mtm_cancel;  in logmap_add()
1149 				mtm->mtm_cancel = me;  in logmap_add()
1150 				me->me_flags |= ME_CANCEL;  in logmap_add()
1159 		me = melist;  in logmap_add()
1161 		me->me_flags &= ~ME_LIST;  in logmap_add()
1166 			ldl_write(ul, va, vamof, me);  in logmap_add()
1168 			kmem_cache_free(mapentry_cache, me);  in logmap_add()
1173 		    map_check_ldl_write(ul, va, vamof, me));  in logmap_add()
1179 		me->me_hash = *savmep;  in logmap_add()
1180 		*savmep = me;  in logmap_add()
1181 		me->me_next = (mapentry_t *)mtm;  in logmap_add()
1182 		me->me_prev = mtm->mtm_prev;  in logmap_add()
1183 		mtm->mtm_prev->me_next = me;  in logmap_add()
1184 		mtm->mtm_prev = me;  in logmap_add()
1185 		me->me_flags |= ME_HASH;  in logmap_add()
1186 		me->me_tid = tid;  in logmap_add()
1187 		me->me_age = mtm->mtm_age++;  in logmap_add()
1217 	mapentry_t	*me;  in logmap_add_buf()  local
1244 		while ((me = *mep) != 0) {  in logmap_add_buf()
1256 			crb = me->me_crb;  in logmap_add_buf()
1257 			if (MEwithinDATA(me, mof, nb) &&  in logmap_add_buf()
1258 			    ((me->me_flags & (ME_ROLL|ME_CANCEL)) == 0)) {  in logmap_add_buf()
1259 				if (tid == me->me_tid &&  in logmap_add_buf()
1260 				    ((me->me_flags & ME_AGE) == 0)) {  in logmap_add_buf()
1261 					*mep = me->me_hash;  in logmap_add_buf()
1262 					me->me_next->me_prev = me->me_prev;  in logmap_add_buf()
1263 					me->me_prev->me_next = me->me_next;  in logmap_add_buf()
1264 					ASSERT(!(me->me_flags & ME_USER));  in logmap_add_buf()
1272 					if (me->me_dt == DT_QR)  in logmap_add_buf()
1273 						HANDLE_DQUOT(me, melist);  in logmap_add_buf()
1283 							CRB_FREE(crb, me);  in logmap_add_buf()
1290 							me->me_crb = NULL;  in logmap_add_buf()
1293 					kmem_cache_free(mapentry_cache, me);  in logmap_add_buf()
1296 				me->me_cancel = mtm->mtm_cancel;  in logmap_add_buf()
1297 				mtm->mtm_cancel = me;  in logmap_add_buf()
1298 				me->me_flags |= ME_CANCEL;  in logmap_add_buf()
1307 			if ((me->me_dt == DT_INODE) && (tid == me->me_tid) &&  in logmap_add_buf()
1359 		me = melist;  in logmap_add_buf()
1361 		me->me_flags &= ~ME_LIST;  in logmap_add_buf()
1362 		me->me_crb = crbsav;  in logmap_add_buf()
1369 		ldl_write(ul, va, vamof, me); /* add to on-disk log */  in logmap_add_buf()
1371 			CRB_RELE(me);  in logmap_add_buf()
1372 			kmem_cache_free(mapentry_cache, me);  in logmap_add_buf()
1376 		    map_check_ldl_write(ul, va, vamof, me));  in logmap_add_buf()
1382 		me->me_hash = *savmep;  in logmap_add_buf()
1383 		*savmep = me;  in logmap_add_buf()
1384 		me->me_next = (mapentry_t *)mtm;  in logmap_add_buf()
1385 		me->me_prev = mtm->mtm_prev;  in logmap_add_buf()
1386 		mtm->mtm_prev->me_next = me;  in logmap_add_buf()
1387 		mtm->mtm_prev = me;  in logmap_add_buf()
1388 		me->me_flags |= ME_HASH;  in logmap_add_buf()
1389 		me->me_tid = tid;  in logmap_add_buf()
1390 		me->me_age = mtm->mtm_age++;  in logmap_add_buf()
1408 	mapentry_t	*me;  in logmap_free_cancel()  local
1429 	while ((me = *cancelhead) != NULL) {  in logmap_free_cancel()
1433 		if (me->me_flags & ME_AGE) {  in logmap_free_cancel()
1442 		*cancelhead = me->me_cancel;  in logmap_free_cancel()
1443 		me->me_cancel = NULL;  in logmap_free_cancel()
1444 		me->me_flags &= ~(ME_CANCEL);  in logmap_free_cancel()
1451 		if (me->me_flags & ME_ROLL)  in logmap_free_cancel()
1457 		if (me->me_flags & ME_HASH) {  in logmap_free_cancel()
1458 			mep = MAP_HASH(me->me_mof, mtm);  in logmap_free_cancel()
1460 				if (*mep == me) {  in logmap_free_cancel()
1461 					*mep = me->me_hash;  in logmap_free_cancel()
1462 					me->me_next->me_prev = me->me_prev;  in logmap_free_cancel()
1463 					me->me_prev->me_next = me->me_next;  in logmap_free_cancel()
1464 					me->me_flags &= ~(ME_HASH);  in logmap_free_cancel()
1465 					if (!(me->me_flags & ME_USER)) {  in logmap_free_cancel()
1476 		CRB_RELE(me);  in logmap_free_cancel()
1477 		kmem_cache_free(mapentry_cache, me);  in logmap_free_cancel()
1491 	mapentry_t	me;  in logmap_commit()  local
1504 		me.me_mof = mtm->mtm_tid;  in logmap_commit()
1505 		me.me_dt = DT_COMMIT;  in logmap_commit()
1506 		me.me_nb = 0;  in logmap_commit()
1507 		me.me_hash = NULL;  in logmap_commit()
1508 		logmap_wait_space(mtm, ul, &me);  in logmap_commit()
1509 		ldl_write(ul, NULL, (offset_t)0, &me);  in logmap_commit()
1532 	mapentry_t	*me;  in logmap_sethead()  local
1542 	me = mtm->mtm_next;  in logmap_sethead()
1543 	while (me != (mapentry_t *)mtm && me->me_lof == 0) {  in logmap_sethead()
1544 		me = me->me_next;  in logmap_sethead()
1547 	if (me == (mapentry_t *)mtm)  in logmap_sethead()
1550 		lof = me->me_lof;  in logmap_sethead()
1551 		tid = me->me_tid;  in logmap_sethead()
1593 	mapentry_t	*me;  in logmap_roll_dev()  local
1606 	for (me = mtm->mtm_next; me != (mapentry_t *)mtm; me = me->me_next) {  in logmap_roll_dev()
1607 		if (me->me_flags & ME_ROLL)  in logmap_roll_dev()
1609 		if (me->me_tid == mtm->mtm_tid)  in logmap_roll_dev()
1611 		if (me->me_tid == mtm->mtm_committid)  in logmap_roll_dev()
1620 	if (me != (mapentry_t *)mtm) {  in logmap_roll_dev()
1639 	mapentry_t	*me;  in logmap_cancel_delta()  local
1653 	me = kmem_cache_alloc(mapentry_cache, KM_SLEEP);  in logmap_cancel_delta()
1654 	bzero(me, sizeof (mapentry_t));  in logmap_cancel_delta()
1659 	me->me_mof = mof;  in logmap_cancel_delta()
1660 	me->me_nb = nb;  in logmap_cancel_delta()
1661 	me->me_dt = DT_CANCEL;  in logmap_cancel_delta()
1662 	me->me_tid = mtm->mtm_tid;  in logmap_cancel_delta()
1663 	me->me_hash = NULL;  in logmap_cancel_delta()
1676 		logmap_wait_space(mtm, ul, me);  in logmap_cancel_delta()
1677 		ldl_write(ul, NULL, (offset_t)0, me);  in logmap_cancel_delta()
1679 			kmem_cache_free(mapentry_cache, me);  in logmap_cancel_delta()
1690 	me->me_age = mtm->mtm_age++;  in logmap_cancel_delta()
1691 	me->me_hash = *mep;  in logmap_cancel_delta()
1692 	*mep = me;  in logmap_cancel_delta()
1693 	me->me_next = (mapentry_t *)mtm;  in logmap_cancel_delta()
1694 	me->me_prev = mtm->mtm_prev;  in logmap_cancel_delta()
1695 	mtm->mtm_prev->me_next = me;  in logmap_cancel_delta()
1696 	mtm->mtm_prev = me;  in logmap_cancel_delta()
1697 	me->me_cancel = mtm->mtm_cancel;  in logmap_cancel_delta()
1698 	mtm->mtm_cancel = me;  in logmap_cancel_delta()
1703 		me->me_flags = ME_USER;  in logmap_cancel_delta()
1705 	me->me_flags |= (ME_HASH|ME_CANCEL);  in logmap_cancel_delta()
1724 	mapentry_t	*me;  in logmap_cancel()  local
1745 			for (me = *mep; me; me = me->me_hash) {  in logmap_cancel()
1746 				if (!DATAoverlapME(mof, hnb, me))  in logmap_cancel()
1749 				ASSERT(MEwithinDATA(me, mof, hnb));  in logmap_cancel()
1751 				if ((me->me_flags & ME_CANCEL) == 0) {  in logmap_cancel()
1752 					me->me_cancel = mtm->mtm_cancel;  in logmap_cancel()
1753 					mtm->mtm_cancel = me;  in logmap_cancel()
1754 					me->me_flags |= ME_CANCEL;  in logmap_cancel()
1755 					crb = me->me_crb;  in logmap_cancel()
1781 	mapentry_t	*me;  in logmap_iscancel()  local
1793 		for (me = *mep; me; me = me->me_hash) {  in logmap_iscancel()
1794 			if (((me->me_flags & ME_ROLL) == 0) &&  in logmap_iscancel()
1795 			    (me->me_dt != DT_CANCEL))  in logmap_iscancel()
1797 			if (DATAoverlapME(mof, hnb, me))  in logmap_iscancel()
1804 		if (me) {  in logmap_iscancel()
1816 	mapentry_t	*me;  in logmap_logscan_add()  local
1872 	me = kmem_cache_alloc(mapentry_cache, KM_SLEEP);  in logmap_logscan_add()
1873 	bzero(me, sizeof (mapentry_t));  in logmap_logscan_add()
1874 	me->me_lof = lof;  in logmap_logscan_add()
1875 	me->me_mof = dp->d_mof;  in logmap_logscan_add()
1876 	me->me_nb = dp->d_nb;  in logmap_logscan_add()
1877 	me->me_tid = mtm->mtm_tid;  in logmap_logscan_add()
1878 	me->me_dt = dp->d_typ;  in logmap_logscan_add()
1879 	me->me_hash = NULL;  in logmap_logscan_add()
1880 	me->me_flags = (ME_LIST | ME_SCAN);  in logmap_logscan_add()
1881 	logmap_add(ul, NULL, 0, me);  in logmap_logscan_add()
1884 		me->me_flags |= ME_CANCEL;  in logmap_logscan_add()
1885 		me->me_cancel = mtm->mtm_cancel;  in logmap_logscan_add()
1886 		mtm->mtm_cancel = me;  in logmap_logscan_add()
1982 handle_dquot(mapentry_t *me)  in handle_dquot()  argument
1988 	dqp = (struct dquot *)me->me_arg;  in handle_dquot()