66 static void dev_pager_free_page(vm_object_t object, vm_page_t m);
67 static int dev_pager_populate(vm_object_t object, vm_pindex_t pidx,
98 static int old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset,
120 	vm_object_t object;  in cdev_pager_lookup()  local
124 	object = vm_pager_object_lookup(&dev_pager_object_list, handle);  in cdev_pager_lookup()
125 	if (object != NULL && object->un_pager.devp.handle == NULL) {  in cdev_pager_lookup()
126 		msleep(&object->un_pager.devp.handle, &dev_pager_mtx,  in cdev_pager_lookup()
128 		vm_object_deallocate(object);  in cdev_pager_lookup()
132 	return (object);  in cdev_pager_lookup()
140 	vm_object_t object;  in cdev_pager_allocate()  local
175 	object = vm_pager_object_lookup(&dev_pager_object_list, handle);  in cdev_pager_allocate()
176 	if (object == NULL) {  in cdev_pager_allocate()
187 		object = vm_pager_object_lookup(&dev_pager_object_list, handle);  in cdev_pager_allocate()
188 		if (object != NULL) {  in cdev_pager_allocate()
192 			if (object->un_pager.devp.handle == NULL) {  in cdev_pager_allocate()
193 				msleep(&object->un_pager.devp.handle,  in cdev_pager_allocate()
195 				vm_object_deallocate(object);  in cdev_pager_allocate()
202 			if (pindex > object->size)  in cdev_pager_allocate()
203 				object->size = pindex;  in cdev_pager_allocate()
204 			KASSERT(object->type == tp,  in cdev_pager_allocate()
206 			    object, tp));  in cdev_pager_allocate()
207 			KASSERT(object->un_pager.devp.ops == ops,  in cdev_pager_allocate()
208 			    ("Inconsistent devops %p %p", object, ops));  in cdev_pager_allocate()
212 			object = object1;  in cdev_pager_allocate()
214 			object->handle = handle;  in cdev_pager_allocate()
215 			object->un_pager.devp.ops = ops;  in cdev_pager_allocate()
216 			TAILQ_INIT(&object->un_pager.devp.devp_pglist);  in cdev_pager_allocate()
217 			TAILQ_INSERT_TAIL(&dev_pager_object_list, object,  in cdev_pager_allocate()
221 				vm_object_set_flag(object, OBJ_POPULATE);  in cdev_pager_allocate()
225 				TAILQ_REMOVE(&dev_pager_object_list, object,  in cdev_pager_allocate()
227 				wakeup(&object->un_pager.devp.handle);  in cdev_pager_allocate()
229 				object->type = OBJT_DEAD;  in cdev_pager_allocate()
230 				vm_object_deallocate(object);  in cdev_pager_allocate()
231 				object = NULL;  in cdev_pager_allocate()
235 				object->flags |= OBJ_COLORED;  in cdev_pager_allocate()
236 				object->pg_color = color;  in cdev_pager_allocate()
237 				object->un_pager.devp.handle = handle;  in cdev_pager_allocate()
238 				wakeup(&object->un_pager.devp.handle);  in cdev_pager_allocate()
243 		if (object->un_pager.devp.handle == NULL) {  in cdev_pager_allocate()
244 			msleep(&object->un_pager.devp.handle,  in cdev_pager_allocate()
246 			vm_object_deallocate(object);  in cdev_pager_allocate()
249 		if (pindex > object->size)  in cdev_pager_allocate()
250 			object->size = pindex;  in cdev_pager_allocate()
251 		KASSERT(object->type == tp,  in cdev_pager_allocate()
252 		    ("Inconsistent device pager type %p %d", object, tp));  in cdev_pager_allocate()
255 	return (object);  in cdev_pager_allocate()
268 cdev_pager_get_path(vm_object_t object, char *path, size_t sz)  in cdev_pager_get_path()  argument
270 	if (object->un_pager.devp.ops->cdev_pg_path != NULL)  in cdev_pager_get_path()
271 		object->un_pager.devp.ops->cdev_pg_path(  in cdev_pager_get_path()
272 		    object->un_pager.devp.handle, path, sz);  in cdev_pager_get_path()
276 cdev_pager_free_page(vm_object_t object, vm_page_t m)  in cdev_pager_free_page()  argument
279 	if (object->type == OBJT_MGTDEVICE) {  in cdev_pager_free_page()
282 		vm_page_iter_init(&pages, object);  in cdev_pager_free_page()
285 	} else if (object->type == OBJT_DEVICE)  in cdev_pager_free_page()
286 		dev_pager_free_page(object, m);  in cdev_pager_free_page()
289 		    ("Invalid device type obj %p m %p", object, m));  in cdev_pager_free_page()
300 cdev_mgtdev_pager_free_pages(vm_object_t object)  in cdev_mgtdev_pager_free_pages()  argument
305 	vm_page_iter_init(&pages, object);  in cdev_mgtdev_pager_free_pages()
306 	VM_OBJECT_WLOCK(object);  in cdev_mgtdev_pager_free_pages()
317 	VM_OBJECT_WUNLOCK(object);  in cdev_mgtdev_pager_free_pages()
321 dev_pager_free_page(vm_object_t object, vm_page_t m)  in dev_pager_free_page()  argument
324 	VM_OBJECT_ASSERT_WLOCKED(object);  in dev_pager_free_page()
325 	KASSERT((object->type == OBJT_DEVICE &&  in dev_pager_free_page()
327 	    ("Managed device or page obj %p m %p", object, m));  in dev_pager_free_page()
328 	TAILQ_REMOVE(&object->un_pager.devp.devp_pglist, m, plinks.q);  in dev_pager_free_page()
333 dev_pager_dealloc(vm_object_t object)  in dev_pager_dealloc()  argument
337 	VM_OBJECT_WUNLOCK(object);  in dev_pager_dealloc()
338 	object->un_pager.devp.ops->cdev_pg_dtor(object->un_pager.devp.handle);  in dev_pager_dealloc()
341 	TAILQ_REMOVE(&dev_pager_object_list, object, pager_object_list);  in dev_pager_dealloc()
343 	VM_OBJECT_WLOCK(object);  in dev_pager_dealloc()
345 	if (object->type == OBJT_DEVICE) {  in dev_pager_dealloc()
349 		while ((m = TAILQ_FIRST(&object->un_pager.devp.devp_pglist))  in dev_pager_dealloc()
354 			dev_pager_free_page(object, m);  in dev_pager_dealloc()
357 	object->handle = NULL;  in dev_pager_dealloc()
358 	object->type = OBJT_DEAD;  in dev_pager_dealloc()
362 dev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int *rbehind,  in dev_pager_getpages()  argument
369 	if (object->un_pager.devp.ops->cdev_pg_fault == NULL)  in dev_pager_getpages()
371 	VM_OBJECT_WLOCK(object);  in dev_pager_getpages()
372 	error = object->un_pager.devp.ops->cdev_pg_fault(object,  in dev_pager_getpages()
375 	VM_OBJECT_ASSERT_WLOCKED(object);  in dev_pager_getpages()
378 		KASSERT((object->type == OBJT_DEVICE &&  in dev_pager_getpages()
380 		    (object->type == OBJT_MGTDEVICE &&  in dev_pager_getpages()
382 		    ("Wrong page type %p %p", ma[0], object));  in dev_pager_getpages()
383 		if (object->type == OBJT_DEVICE) {  in dev_pager_getpages()
384 			TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist,  in dev_pager_getpages()
392 	VM_OBJECT_WUNLOCK(object);  in dev_pager_getpages()
398 dev_pager_populate(vm_object_t object, vm_pindex_t pidx, int fault_type,  in dev_pager_populate()  argument
402 	VM_OBJECT_ASSERT_WLOCKED(object);  in dev_pager_populate()
403 	if (object->un_pager.devp.ops->cdev_pg_populate == NULL)  in dev_pager_populate()
405 	return (object->un_pager.devp.ops->cdev_pg_populate(object, pidx,  in dev_pager_populate()
410 old_dev_pager_fault(vm_object_t object, vm_ooffset_t offset, int prot,  in old_dev_pager_fault()  argument
422 	memattr = object->memattr;  in old_dev_pager_fault()
424 	VM_OBJECT_WUNLOCK(object);  in old_dev_pager_fault()
426 	dev = object->handle;  in old_dev_pager_fault()
429 		VM_OBJECT_WLOCK(object);  in old_dev_pager_fault()
441 		VM_OBJECT_WLOCK(object);  in old_dev_pager_fault()
466 		VM_OBJECT_WLOCK(object);  in old_dev_pager_fault()
474 		VM_OBJECT_WLOCK(object);  in old_dev_pager_fault()
475 		vm_page_replace(page, object, (*mres)->pindex, *mres);  in old_dev_pager_fault()
483 dev_pager_putpages(vm_object_t object, vm_page_t *m, int count, int flags,  in dev_pager_putpages()  argument
491 dev_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before,  in dev_pager_haspage()  argument