185 static int vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object,
214 	struct vm_domain *vmd;  in vm_page_init_cache_zones()  local
221 		vmd = VM_DOMAIN(domain);  in vm_page_init_cache_zones()
223 			pgcache = &vmd->vmd_pgcache[pool];  in vm_page_init_cache_zones()
236 			    vmd->vmd_page_count / 1000;  in vm_page_init_cache_zones()
331 	struct vm_domain *vmd;  in vm_page_blacklist_add()  local
339 	vmd = VM_DOMAIN(vm_phys_domain(pa));  in vm_page_blacklist_add()
340 	vm_domain_free_lock(vmd);  in vm_page_blacklist_add()
342 	vm_domain_free_unlock(vmd);  in vm_page_blacklist_add()
344 		vm_domain_freecnt_inc(vmd, -1);  in vm_page_blacklist_add()
444 	struct vm_domain *vmd;  in vm_page_domain_init()  local
448 	vmd = VM_DOMAIN(domain);  in vm_page_domain_init()
449 	bzero(vmd, sizeof(*vmd));  in vm_page_domain_init()
450 	*__DECONST(const char **, &vmd->vmd_pagequeues[PQ_INACTIVE].pq_name) =  in vm_page_domain_init()
452 	*__DECONST(const char **, &vmd->vmd_pagequeues[PQ_ACTIVE].pq_name) =  in vm_page_domain_init()
454 	*__DECONST(const char **, &vmd->vmd_pagequeues[PQ_LAUNDRY].pq_name) =  in vm_page_domain_init()
457 	    &vmd->vmd_pagequeues[PQ_UNSWAPPABLE].pq_name) =  in vm_page_domain_init()
459 	vmd->vmd_domain = domain;  in vm_page_domain_init()
460 	vmd->vmd_page_count = 0;  in vm_page_domain_init()
461 	vmd->vmd_free_count = 0;  in vm_page_domain_init()
462 	vmd->vmd_segs = 0;  in vm_page_domain_init()
463 	vmd->vmd_oom = FALSE;  in vm_page_domain_init()
465 		pq = &vmd->vmd_pagequeues[i];  in vm_page_domain_init()
470 		vm_page_init_marker(&vmd->vmd_markers[i], i, 0);  in vm_page_domain_init()
472 	mtx_init(&vmd->vmd_free_mtx, "vm page free queue", NULL, MTX_DEF);  in vm_page_domain_init()
473 	mtx_init(&vmd->vmd_pageout_mtx, "vm pageout lock", NULL, MTX_DEF);  in vm_page_domain_init()
474 	snprintf(vmd->vmd_name, sizeof(vmd->vmd_name), "%d", domain);  in vm_page_domain_init()
480 	vm_page_init_marker(&vmd->vmd_inacthead, PQ_INACTIVE, PGA_ENQUEUED);  in vm_page_domain_init()
481 	TAILQ_INSERT_HEAD(&vmd->vmd_pagequeues[PQ_INACTIVE].pq_pl,  in vm_page_domain_init()
482 	    &vmd->vmd_inacthead, plinks.q);  in vm_page_domain_init()
490 	vm_page_init_marker(&vmd->vmd_clock[0], PQ_ACTIVE, PGA_ENQUEUED);  in vm_page_domain_init()
491 	vm_page_init_marker(&vmd->vmd_clock[1], PQ_ACTIVE, PGA_ENQUEUED);  in vm_page_domain_init()
492 	TAILQ_INSERT_HEAD(&vmd->vmd_pagequeues[PQ_ACTIVE].pq_pl,  in vm_page_domain_init()
493 	    &vmd->vmd_clock[0], plinks.q);  in vm_page_domain_init()
494 	TAILQ_INSERT_TAIL(&vmd->vmd_pagequeues[PQ_ACTIVE].pq_pl,  in vm_page_domain_init()
495 	    &vmd->vmd_clock[1], plinks.q);  in vm_page_domain_init()
556 	struct vm_domain *vmd;  in vm_page_startup()  local
844 			vmd = VM_DOMAIN(seg->domain);  in vm_page_startup()
845 			vm_domain_free_lock(vmd);  in vm_page_startup()
847 			vm_domain_free_unlock(vmd);  in vm_page_startup()
848 			vm_domain_freecnt_inc(vmd, pagecount);  in vm_page_startup()
850 			vmd->vmd_page_count += (u_int)pagecount;  in vm_page_startup()
851 			vmd->vmd_segs |= 1UL << segind;  in vm_page_startup()
2201 _vm_domain_allocate(struct vm_domain *vmd, int req_class, int npages)  in _vm_domain_allocate()  argument
2208 		limit = vmd->vmd_interrupt_free_min;  in _vm_domain_allocate()
2210 		limit = vmd->vmd_free_reserved;  in _vm_domain_allocate()
2216 	old = atomic_load_int(&vmd->vmd_free_count);  in _vm_domain_allocate()
2221 	} while (atomic_fcmpset_int(&vmd->vmd_free_count, &old, new) == 0);  in _vm_domain_allocate()
2224 	if (vm_paging_needed(vmd, new) && !vm_paging_needed(vmd, old))  in _vm_domain_allocate()
2225 		pagedaemon_wakeup(vmd->vmd_domain);  in _vm_domain_allocate()
2228 	if ((old >= vmd->vmd_free_min && new < vmd->vmd_free_min) ||  in _vm_domain_allocate()
2229 	    (old >= vmd->vmd_free_severe && new < vmd->vmd_free_severe))  in _vm_domain_allocate()
2230 		vm_domain_set(vmd);  in _vm_domain_allocate()
2236 vm_domain_allocate(struct vm_domain *vmd, int req, int npages)  in vm_domain_allocate()  argument
2246 	return (_vm_domain_allocate(vmd, req_class, npages));  in vm_domain_allocate()
2253 	struct vm_domain *vmd;  in vm_page_alloc_domain_after()  local
2292 	vmd = VM_DOMAIN(domain);  in vm_page_alloc_domain_after()
2293 	if (vmd->vmd_pgcache[VM_FREEPOOL_DEFAULT].zone != NULL) {  in vm_page_alloc_domain_after()
2294 		m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DEFAULT].zone,  in vm_page_alloc_domain_after()
2301 	if (vm_domain_allocate(vmd, req, 1)) {  in vm_page_alloc_domain_after()
2305 		vm_domain_free_lock(vmd);  in vm_page_alloc_domain_after()
2307 		vm_domain_free_unlock(vmd);  in vm_page_alloc_domain_after()
2309 			vm_domain_freecnt_inc(vmd, 1);  in vm_page_alloc_domain_after()
2320 		if (vm_domain_alloc_fail(vmd, object, req))  in vm_page_alloc_domain_after()
2452 	struct vm_domain *vmd;  in vm_page_find_contig_domain()  local
2459 	vmd = VM_DOMAIN(domain);  in vm_page_find_contig_domain()
2460 	if (!vm_domain_allocate(vmd, req, npages))  in vm_page_find_contig_domain()
2465 	vm_domain_free_lock(vmd);  in vm_page_find_contig_domain()
2468 	vm_domain_free_unlock(vmd);  in vm_page_find_contig_domain()
2482 	vm_domain_freecnt_inc(vmd, npages);  in vm_page_find_contig_domain()
2597 	struct vm_domain *vmd;  in vm_page_alloc_noobj_domain()  local
2611 	vmd = VM_DOMAIN(domain);  in vm_page_alloc_noobj_domain()
2619 	if (vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone != NULL) {  in vm_page_alloc_noobj_domain()
2620 		m = uma_zalloc(vmd->vmd_pgcache[VM_FREEPOOL_DIRECT].zone,  in vm_page_alloc_noobj_domain()
2628 	if (vm_domain_allocate(vmd, req, 1)) {  in vm_page_alloc_noobj_domain()
2629 		vm_domain_free_lock(vmd);  in vm_page_alloc_noobj_domain()
2631 		vm_domain_free_unlock(vmd);  in vm_page_alloc_noobj_domain()
2633 			vm_domain_freecnt_inc(vmd, 1);  in vm_page_alloc_noobj_domain()
2641 		if (vm_domain_alloc_fail(vmd, NULL, req))  in vm_page_alloc_noobj_domain()
2690 	struct vm_domain *vmd;  in vm_page_alloc_nofree_domain()  local
2695 	vmd = VM_DOMAIN(domain);  in vm_page_alloc_nofree_domain()
2696 	nqp = &vmd->vmd_nofreeq;  in vm_page_alloc_nofree_domain()
2697 	vm_domain_free_lock(vmd);  in vm_page_alloc_nofree_domain()
2699 		if (!vm_domain_allocate(vmd, req,  in vm_page_alloc_nofree_domain()
2701 			vm_domain_free_unlock(vmd);  in vm_page_alloc_nofree_domain()
2707 			vm_domain_freecnt_inc(vmd, 1 << VM_NOFREE_IMPORT_ORDER);  in vm_page_alloc_nofree_domain()
2708 			vm_domain_free_unlock(vmd);  in vm_page_alloc_nofree_domain()
2714 	vm_domain_free_unlock(vmd);  in vm_page_alloc_nofree_domain()
2845 	struct vm_domain *vmd;  in vm_page_zone_import()  local
2850 	vmd = VM_DOMAIN(pgcache->domain);  in vm_page_zone_import()
2856 	if (vmd->vmd_severeset || curproc == pageproc ||  in vm_page_zone_import()
2857 	    !_vm_domain_allocate(vmd, VM_ALLOC_NORMAL, cnt))  in vm_page_zone_import()
2859 	domain = vmd->vmd_domain;  in vm_page_zone_import()
2860 	vm_domain_free_lock(vmd);  in vm_page_zone_import()
2863 	vm_domain_free_unlock(vmd);  in vm_page_zone_import()
2865 		vm_domain_freecnt_inc(vmd, cnt - i);  in vm_page_zone_import()
2873 	struct vm_domain *vmd;  in vm_page_zone_release()  local
2879 	vmd = VM_DOMAIN(pgcache->domain);  in vm_page_zone_release()
2880 	vm_domain_free_lock(vmd);  in vm_page_zone_release()
2885 	vm_domain_free_unlock(vmd);  in vm_page_zone_release()
2886 	vm_domain_freecnt_inc(vmd, cnt);  in vm_page_zone_release()
3089 	struct vm_domain *vmd;  in vm_page_reclaim_run()  local
3236 			vmd = VM_DOMAIN(domain);  in vm_page_reclaim_run()
3237 			vm_domain_free_lock(vmd);  in vm_page_reclaim_run()
3254 			vm_domain_free_unlock(vmd);  in vm_page_reclaim_run()
3262 		vmd = VM_DOMAIN(domain);  in vm_page_reclaim_run()
3264 		vm_domain_free_lock(vmd);  in vm_page_reclaim_run()
3271 		vm_domain_free_unlock(vmd);  in vm_page_reclaim_run()
3272 		vm_domain_freecnt_inc(vmd, cnt);  in vm_page_reclaim_run()
3313 	struct vm_domain *vmd;  in vm_page_reclaim_contig_domain_ext()  local
3370 	vmd = VM_DOMAIN(domain);  in vm_page_reclaim_contig_domain_ext()
3371 	count = vmd->vmd_free_count;  in vm_page_reclaim_contig_domain_ext()
3372 	if (count < npages + vmd->vmd_free_reserved || (count < npages +  in vm_page_reclaim_contig_domain_ext()
3373 	    vmd->vmd_interrupt_free_min && req_class == VM_ALLOC_SYSTEM) ||  in vm_page_reclaim_contig_domain_ext()
3488 vm_domain_set(struct vm_domain *vmd)  in vm_domain_set()  argument
3492 	if (!vmd->vmd_minset && vm_paging_min(vmd)) {  in vm_domain_set()
3493 		vmd->vmd_minset = 1;  in vm_domain_set()
3494 		DOMAINSET_SET(vmd->vmd_domain, &vm_min_domains);  in vm_domain_set()
3496 	if (!vmd->vmd_severeset && vm_paging_severe(vmd)) {  in vm_domain_set()
3497 		vmd->vmd_severeset = 1;  in vm_domain_set()
3498 		DOMAINSET_SET(vmd->vmd_domain, &vm_severe_domains);  in vm_domain_set()
3507 vm_domain_clear(struct vm_domain *vmd)  in vm_domain_clear()  argument
3511 	if (vmd->vmd_minset && !vm_paging_min(vmd)) {  in vm_domain_clear()
3512 		vmd->vmd_minset = 0;  in vm_domain_clear()
3513 		DOMAINSET_CLR(vmd->vmd_domain, &vm_min_domains);  in vm_domain_clear()
3519 	if (vmd->vmd_severeset && !vm_paging_severe(vmd)) {  in vm_domain_clear()
3520 		vmd->vmd_severeset = 0;  in vm_domain_clear()
3521 		DOMAINSET_CLR(vmd->vmd_domain, &vm_severe_domains);  in vm_domain_clear()
3532 	if (vmd->vmd_pageout_pages_needed &&  in vm_domain_clear()
3533 	    vmd->vmd_free_count >= vmd->vmd_pageout_free_min) {  in vm_domain_clear()
3534 		wakeup(&vmd->vmd_pageout_pages_needed);  in vm_domain_clear()
3535 		vmd->vmd_pageout_pages_needed = 0;  in vm_domain_clear()
3632 	struct vm_domain *vmd;  in vm_wait_domain()  local
3635 	vmd = VM_DOMAIN(domain);  in vm_wait_domain()
3636 	vm_domain_free_assert_unlocked(vmd);  in vm_wait_domain()
3640 		if (vmd->vmd_free_count < vmd->vmd_pageout_free_min) {  in vm_wait_domain()
3641 			vmd->vmd_pageout_pages_needed = 1;  in vm_wait_domain()
3642 			msleep(&vmd->vmd_pageout_pages_needed,  in vm_wait_domain()
3648 		DOMAINSET_SET(vmd->vmd_domain, &wdom);  in vm_wait_domain()
3703 vm_domain_alloc_fail(struct vm_domain *vmd, vm_object_t object, int req)  in vm_domain_alloc_fail()  argument
3706 	vm_domain_free_assert_unlocked(vmd);  in vm_domain_alloc_fail()
3708 	atomic_add_int(&vmd->vmd_pageout_deficit,  in vm_domain_alloc_fail()
3713 		vm_wait_domain(vmd->vmd_domain);  in vm_domain_alloc_fail()
3860 	struct vm_domain *vmd;  in _vm_page_pqstate_commit_requeue()  local
3882 		vmd = vm_pagequeue_domain(m);  in _vm_page_pqstate_commit_requeue()
3883 		KASSERT(pq == &vmd->vmd_pagequeues[PQ_INACTIVE],  in _vm_page_pqstate_commit_requeue()
3885 		TAILQ_INSERT_BEFORE(&vmd->vmd_inacthead, m, plinks.q);  in _vm_page_pqstate_commit_requeue()
4052 	struct vm_domain *vmd;  in vm_page_pqbatch_drain()  local
4063 			vmd = VM_DOMAIN(domain);  in vm_page_pqbatch_drain()
4065 				pq = &vmd->vmd_pagequeues[queue];  in vm_page_pqbatch_drain()
4260 	struct vm_domain *vmd;  in vm_page_free_toq()  local
4266 	vmd = vm_pagequeue_domain(m);  in vm_page_free_toq()
4267 	zone = vmd->vmd_pgcache[m->pool].zone;  in vm_page_free_toq()
4272 	vm_domain_free_lock(vmd);  in vm_page_free_toq()
4274 	vm_domain_free_unlock(vmd);  in vm_page_free_toq()
4275 	vm_domain_freecnt_inc(vmd, 1);  in vm_page_free_toq()