124 static void vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,
206 static int vm_pageout_launder(struct vm_domain *vmd, int launder,
692 vm_pageout_launder(struct vm_domain *vmd, int launder, bool in_shortfall)  in vm_pageout_launder()  argument
722 	marker = &vmd->vmd_markers[queue];  in vm_pageout_launder()
723 	pq = &vmd->vmd_pagequeues[queue];  in vm_pageout_launder()
959 	struct vm_domain *vmd;  in vm_pageout_laundry_worker()  local
966 	vmd = VM_DOMAIN(domain);  in vm_pageout_laundry_worker()
967 	pq = &vmd->vmd_pagequeues[PQ_LAUNDRY];  in vm_pageout_laundry_worker()
968 	KASSERT(vmd->vmd_segs != 0, ("domain without segments"));  in vm_pageout_laundry_worker()
979 	(void)EVENTHANDLER_REGISTER(swapon, vm_pageout_swapon, vmd,  in vm_pageout_laundry_worker()
981 	(void)EVENTHANDLER_REGISTER(swapoff, vm_pageout_swapoff, vmd,  in vm_pageout_laundry_worker()
1003 		else if (shortfall_cycle == 0 || vm_laundry_target(vmd) <= 0) {  in vm_pageout_laundry_worker()
1037 		nclean = vmd->vmd_free_count +  in vm_pageout_laundry_worker()
1038 		    vmd->vmd_pagequeues[PQ_INACTIVE].pq_cnt;  in vm_pageout_laundry_worker()
1039 		ndirty = vmd->vmd_pagequeues[PQ_LAUNDRY].pq_cnt;  in vm_pageout_laundry_worker()
1041 		    vmd->vmd_free_target - vmd->vmd_free_min)) >= nclean) {  in vm_pageout_laundry_worker()
1042 			target = vmd->vmd_background_launder_target;  in vm_pageout_laundry_worker()
1074 			target -= min(vm_pageout_launder(vmd, launder,  in vm_pageout_laundry_worker()
1085 		if (target == 0 && vmd->vmd_laundry_request == VM_LAUNDRY_IDLE)  in vm_pageout_laundry_worker()
1086 			(void)mtx_sleep(&vmd->vmd_laundry_request,  in vm_pageout_laundry_worker()
1094 		if (vmd->vmd_laundry_request == VM_LAUNDRY_SHORTFALL &&  in vm_pageout_laundry_worker()
1096 			shortfall = vm_laundry_target(vmd) +  in vm_pageout_laundry_worker()
1097 			    vmd->vmd_pageout_deficit;  in vm_pageout_laundry_worker()
1103 			vmd->vmd_laundry_request = VM_LAUNDRY_IDLE;  in vm_pageout_laundry_worker()
1104 		nfreed += vmd->vmd_clean_pages_freed;  in vm_pageout_laundry_worker()
1105 		vmd->vmd_clean_pages_freed = 0;  in vm_pageout_laundry_worker()
1123 vm_pageout_active_target(struct vm_domain *vmd)  in vm_pageout_active_target()  argument
1127 	shortage = vmd->vmd_inactive_target + vm_paging_target(vmd) -  in vm_pageout_active_target()
1128 	    (vmd->vmd_pagequeues[PQ_INACTIVE].pq_cnt +  in vm_pageout_active_target()
1129 	    vmd->vmd_pagequeues[PQ_LAUNDRY].pq_cnt / act_scan_laundry_weight);  in vm_pageout_active_target()
1139 vm_pageout_scan_active(struct vm_domain *vmd, int page_shortage)  in vm_pageout_scan_active()  argument
1150 	marker = &vmd->vmd_markers[PQ_ACTIVE];  in vm_pageout_scan_active()
1151 	pq = &vmd->vmd_pagequeues[PQ_ACTIVE];  in vm_pageout_scan_active()
1161 		min_scan *= scan_tick - vmd->vmd_last_active_scan;  in vm_pageout_scan_active()
1166 		vmd->vmd_last_active_scan = scan_tick;  in vm_pageout_scan_active()
1183 	vm_pageout_init_scan(&ss, pq, marker, &vmd->vmd_clock[0], max_scan);  in vm_pageout_scan_active()
1185 		if (__predict_false(m == &vmd->vmd_clock[1])) {  in vm_pageout_scan_active()
1187 			TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);  in vm_pageout_scan_active()
1188 			TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[1], plinks.q);  in vm_pageout_scan_active()
1189 			TAILQ_INSERT_HEAD(&pq->pq_pl, &vmd->vmd_clock[0],  in vm_pageout_scan_active()
1191 			TAILQ_INSERT_TAIL(&pq->pq_pl, &vmd->vmd_clock[1],  in vm_pageout_scan_active()
1338 	TAILQ_REMOVE(&pq->pq_pl, &vmd->vmd_clock[0], plinks.q);  in vm_pageout_scan_active()
1339 	TAILQ_INSERT_AFTER(&pq->pq_pl, marker, &vmd->vmd_clock[0], plinks.q);  in vm_pageout_scan_active()
1393 vm_pageout_scan_inactive(struct vm_domain *vmd, int page_shortage)  in vm_pageout_scan_inactive()  argument
1426 	pq = &vmd->vmd_pagequeues[PQ_INACTIVE];  in vm_pageout_scan_inactive()
1622 	atomic_add_int(&vmd->vmd_addl_shortage, addl_page_shortage);  in vm_pageout_scan_inactive()
1625 	atomic_add_int(&vmd->vmd_inactive_us,  in vm_pageout_scan_inactive()
1627 	atomic_add_int(&vmd->vmd_inactive_freed,  in vm_pageout_scan_inactive()
1636 vm_pageout_inactive_dispatch(struct vm_domain *vmd, int shortage)  in vm_pageout_inactive_dispatch()  argument
1640 	vmd->vmd_inactive_shortage = shortage;  in vm_pageout_inactive_dispatch()
1647 	if ((threads = vmd->vmd_inactive_threads) > 1 &&  in vm_pageout_inactive_dispatch()
1648 	    vmd->vmd_helper_threads_enabled &&  in vm_pageout_inactive_dispatch()
1649 	    vmd->vmd_inactive_pps != 0 &&  in vm_pageout_inactive_dispatch()
1650 	    shortage > vmd->vmd_inactive_pps / VM_INACT_SCAN_RATE / 4) {  in vm_pageout_inactive_dispatch()
1651 		vmd->vmd_inactive_shortage /= threads;  in vm_pageout_inactive_dispatch()
1653 		vm_domain_pageout_lock(vmd);  in vm_pageout_inactive_dispatch()
1654 		blockcount_acquire(&vmd->vmd_inactive_starting, threads - 1);  in vm_pageout_inactive_dispatch()
1655 		blockcount_acquire(&vmd->vmd_inactive_running, threads - 1);  in vm_pageout_inactive_dispatch()
1656 		wakeup(&vmd->vmd_inactive_shortage);  in vm_pageout_inactive_dispatch()
1657 		vm_domain_pageout_unlock(vmd);  in vm_pageout_inactive_dispatch()
1661 	vm_pageout_scan_inactive(vmd, vmd->vmd_inactive_shortage + slop);  in vm_pageout_inactive_dispatch()
1667 	blockcount_wait(&vmd->vmd_inactive_running, NULL, "vmpoid", PVM);  in vm_pageout_inactive_dispatch()
1668 	freed = atomic_readandclear_int(&vmd->vmd_inactive_freed);  in vm_pageout_inactive_dispatch()
1676 	us = max(atomic_readandclear_int(&vmd->vmd_inactive_us), 1);  in vm_pageout_inactive_dispatch()
1682 	vmd->vmd_inactive_pps = (vmd->vmd_inactive_pps / 2) + (pps / 2);  in vm_pageout_inactive_dispatch()
1692 vm_pageout_inactive(struct vm_domain *vmd, int shortage, int *addl_shortage)  in vm_pageout_inactive()  argument
1704 	deficit = atomic_readandclear_int(&vmd->vmd_pageout_deficit);  in vm_pageout_inactive()
1710 	page_shortage = vm_pageout_inactive_dispatch(vmd, starting_page_shortage);  in vm_pageout_inactive()
1711 	addl_page_shortage = atomic_readandclear_int(&vmd->vmd_addl_shortage);  in vm_pageout_inactive()
1725 		pq = &vmd->vmd_pagequeues[PQ_LAUNDRY];  in vm_pageout_inactive()
1727 		if (vmd->vmd_laundry_request == VM_LAUNDRY_IDLE &&  in vm_pageout_inactive()
1730 				vmd->vmd_laundry_request = VM_LAUNDRY_SHORTFALL;  in vm_pageout_inactive()
1732 			} else if (vmd->vmd_laundry_request !=  in vm_pageout_inactive()
1734 				vmd->vmd_laundry_request =  in vm_pageout_inactive()
1736 			wakeup(&vmd->vmd_laundry_request);  in vm_pageout_inactive()
1738 		vmd->vmd_clean_pages_freed +=  in vm_pageout_inactive()
1747 	vm_pageout_mightbe_oom(vmd, page_shortage, starting_page_shortage);  in vm_pageout_inactive()
1765 vm_pageout_mightbe_oom(struct vm_domain *vmd, int page_shortage,  in vm_pageout_mightbe_oom()  argument
1772 		vmd->vmd_oom_seq = 0;  in vm_pageout_mightbe_oom()
1774 		vmd->vmd_oom_seq++;  in vm_pageout_mightbe_oom()
1775 	if (vmd->vmd_oom_seq < vm_pageout_oom_seq) {  in vm_pageout_mightbe_oom()
1776 		if (vmd->vmd_oom) {  in vm_pageout_mightbe_oom()
1777 			vmd->vmd_oom = false;  in vm_pageout_mightbe_oom()
1787 	vmd->vmd_oom_seq = 0;  in vm_pageout_mightbe_oom()
1789 	if (vmd->vmd_oom)  in vm_pageout_mightbe_oom()
1792 	vmd->vmd_oom = true;  in vm_pageout_mightbe_oom()
1810 	vmd->vmd_oom = false;  in vm_pageout_mightbe_oom()
2067 	struct vm_domain *vmd;  in vm_pageout_worker()  local
2073 	vmd = VM_DOMAIN(domain);  in vm_pageout_worker()
2083 	KASSERT(vmd->vmd_segs != 0, ("domain without segments"));  in vm_pageout_worker()
2084 	vmd->vmd_last_active_scan = ticks;  in vm_pageout_worker()
2090 		vm_domain_pageout_lock(vmd);  in vm_pageout_worker()
2097 		atomic_store_int(&vmd->vmd_pageout_wanted, 0);  in vm_pageout_worker()
2102 		if (vm_paging_needed(vmd, vmd->vmd_free_count)) {  in vm_pageout_worker()
2108 			vm_domain_pageout_unlock(vmd);  in vm_pageout_worker()
2116 			if (mtx_sleep(&vmd->vmd_pageout_wanted,  in vm_pageout_worker()
2117 			    vm_domain_pageout_lockptr(vmd), PDROP | PVM,  in vm_pageout_worker()
2123 		atomic_store_int(&vmd->vmd_pageout_wanted, 1);  in vm_pageout_worker()
2131 		shortage = pidctrl_daemon(&vmd->vmd_pid, vmd->vmd_free_count);  in vm_pageout_worker()
2133 			ofree = vmd->vmd_free_count;  in vm_pageout_worker()
2134 			if (vm_pageout_lowmem() && vmd->vmd_free_count > ofree)  in vm_pageout_worker()
2135 				shortage -= min(vmd->vmd_free_count - ofree,  in vm_pageout_worker()
2137 			target_met = vm_pageout_inactive(vmd, shortage,  in vm_pageout_worker()
2147 		shortage = vm_pageout_active_target(vmd) + addl_shortage;  in vm_pageout_worker()
2148 		vm_pageout_scan_active(vmd, shortage);  in vm_pageout_worker()
2159 	struct vm_domain *vmd;  in vm_pageout_helper()  local
2163 	vmd = VM_DOMAIN(domain);  in vm_pageout_helper()
2165 	vm_domain_pageout_lock(vmd);  in vm_pageout_helper()
2167 		msleep(&vmd->vmd_inactive_shortage,  in vm_pageout_helper()
2168 		    vm_domain_pageout_lockptr(vmd), PVM, "psleep", 0);  in vm_pageout_helper()
2169 		blockcount_release(&vmd->vmd_inactive_starting, 1);  in vm_pageout_helper()
2171 		vm_domain_pageout_unlock(vmd);  in vm_pageout_helper()
2172 		vm_pageout_scan_inactive(vmd, vmd->vmd_inactive_shortage);  in vm_pageout_helper()
2173 		vm_domain_pageout_lock(vmd);  in vm_pageout_helper()
2179 		blockcount_release(&vmd->vmd_inactive_running, 1);  in vm_pageout_helper()
2184 get_pageout_threads_per_domain(const struct vm_domain *vmd)  in get_pageout_threads_per_domain()  argument
2188 	if (VM_DOMAIN_EMPTY(vmd->vmd_domain))  in get_pageout_threads_per_domain()
2201 	domain_cpus = CPU_COUNT(&cpuset_domain[vmd->vmd_domain]);  in get_pageout_threads_per_domain()
2226 	struct vm_domain *vmd;  in vm_pageout_init_domain()  local
2229 	vmd = VM_DOMAIN(domain);  in vm_pageout_init_domain()
2230 	vmd->vmd_interrupt_free_min = 2;  in vm_pageout_init_domain()
2237 	vmd->vmd_pageout_free_min = 2 * MAXBSIZE / PAGE_SIZE +  in vm_pageout_init_domain()
2238 	    vmd->vmd_interrupt_free_min;  in vm_pageout_init_domain()
2239 	vmd->vmd_free_reserved = vm_pageout_page_count +  in vm_pageout_init_domain()
2240 	    vmd->vmd_pageout_free_min + vmd->vmd_page_count / 768;  in vm_pageout_init_domain()
2241 	vmd->vmd_free_min = vmd->vmd_page_count / 200;  in vm_pageout_init_domain()
2242 	vmd->vmd_free_severe = vmd->vmd_free_min / 2;  in vm_pageout_init_domain()
2243 	vmd->vmd_free_target = 4 * vmd->vmd_free_min + vmd->vmd_free_reserved;  in vm_pageout_init_domain()
2244 	vmd->vmd_free_min += vmd->vmd_free_reserved;  in vm_pageout_init_domain()
2245 	vmd->vmd_free_severe += vmd->vmd_free_reserved;  in vm_pageout_init_domain()
2246 	vmd->vmd_inactive_target = (3 * vmd->vmd_free_target) / 2;  in vm_pageout_init_domain()
2247 	if (vmd->vmd_inactive_target > vmd->vmd_free_count / 3)  in vm_pageout_init_domain()
2248 		vmd->vmd_inactive_target = vmd->vmd_free_count / 3;  in vm_pageout_init_domain()
2254 	vmd->vmd_pageout_wakeup_thresh = (vmd->vmd_free_target / 10) * 9;  in vm_pageout_init_domain()
2261 	vmd->vmd_background_launder_target = (vmd->vmd_free_target -  in vm_pageout_init_domain()
2262 	    vmd->vmd_free_min) / 10;  in vm_pageout_init_domain()
2265 	pidctrl_init(&vmd->vmd_pid, hz / VM_INACT_SCAN_RATE,  in vm_pageout_init_domain()
2266 	    vmd->vmd_free_target, PIDCTRL_BOUND,  in vm_pageout_init_domain()
2268 	oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(vmd->vmd_oid), OID_AUTO,  in vm_pageout_init_domain()
2270 	pidctrl_init_sysctl(&vmd->vmd_pid, SYSCTL_CHILDREN(oid));  in vm_pageout_init_domain()
2272 	vmd->vmd_inactive_threads = get_pageout_threads_per_domain(vmd);  in vm_pageout_init_domain()
2273 	SYSCTL_ADD_BOOL(NULL, SYSCTL_CHILDREN(vmd->vmd_oid), OID_AUTO,  in vm_pageout_init_domain()
2275 	    &vmd->vmd_helper_threads_enabled, 0,  in vm_pageout_init_domain()
2290 		struct vm_domain *vmd;  in vm_pageout_init()  local
2293 		vmd = VM_DOMAIN(i);  in vm_pageout_init()
2294 		vm_cnt.v_free_reserved += vmd->vmd_free_reserved;  in vm_pageout_init()
2295 		vm_cnt.v_free_target += vmd->vmd_free_target;  in vm_pageout_init()
2296 		vm_cnt.v_free_min += vmd->vmd_free_min;  in vm_pageout_init()
2297 		vm_cnt.v_inactive_target += vmd->vmd_inactive_target;  in vm_pageout_init()
2298 		vm_cnt.v_pageout_free_min += vmd->vmd_pageout_free_min;  in vm_pageout_init()
2299 		vm_cnt.v_interrupt_free_min += vmd->vmd_interrupt_free_min;  in vm_pageout_init()
2300 		vm_cnt.v_free_severe += vmd->vmd_free_severe;  in vm_pageout_init()
2301 		freecount += vmd->vmd_free_count;  in vm_pageout_init()
2380 	struct vm_domain *vmd;  in pagedaemon_wakeup()  local
2382 	vmd = VM_DOMAIN(domain);  in pagedaemon_wakeup()
2383 	vm_domain_pageout_assert_unlocked(vmd);  in pagedaemon_wakeup()
2387 	if (atomic_fetchadd_int(&vmd->vmd_pageout_wanted, 1) == 0) {  in pagedaemon_wakeup()
2388 		vm_domain_pageout_lock(vmd);  in pagedaemon_wakeup()
2389 		atomic_store_int(&vmd->vmd_pageout_wanted, 1);  in pagedaemon_wakeup()
2390 		wakeup(&vmd->vmd_pageout_wanted);  in pagedaemon_wakeup()
2391 		vm_domain_pageout_unlock(vmd);  in pagedaemon_wakeup()