xref: /freebsd/sys/vm/vm_pager.c (revision 7f3dea244c40159a41ab22da77a434d7c5b5e85a)
1 /*
2  * Copyright (c) 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * This code is derived from software contributed to Berkeley by
6  * The Mach Operating System project at Carnegie-Mellon University.
7  *
8  * Redistribution and use in source and binary forms, with or without
9  * modification, are permitted provided that the following conditions
10  * are met:
11  * 1. Redistributions of source code must retain the above copyright
12  *    notice, this list of conditions and the following disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  * 3. All advertising materials mentioning features or use of this software
17  *    must display the following acknowledgement:
18  *	This product includes software developed by the University of
19  *	California, Berkeley and its contributors.
20  * 4. Neither the name of the University nor the names of its contributors
21  *    may be used to endorse or promote products derived from this software
22  *    without specific prior written permission.
23  *
24  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
25  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
26  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
27  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
28  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
29  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
30  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
31  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
32  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
33  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
34  * SUCH DAMAGE.
35  *
36  *	from: @(#)vm_pager.c	8.6 (Berkeley) 1/12/94
37  *
38  *
39  * Copyright (c) 1987, 1990 Carnegie-Mellon University.
40  * All rights reserved.
41  *
42  * Authors: Avadis Tevanian, Jr., Michael Wayne Young
43  *
44  * Permission to use, copy, modify and distribute this software and
45  * its documentation is hereby granted, provided that both the copyright
46  * notice and this permission notice appear in all copies of the
47  * software, derivative works or modified versions, and any portions
48  * thereof, and that both notices appear in supporting documentation.
49  *
50  * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS"
51  * CONDITION.  CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND
52  * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE.
53  *
54  * Carnegie Mellon requests users of this software to return to
55  *
56  *  Software Distribution Coordinator  or  Software.Distribution@CS.CMU.EDU
57  *  School of Computer Science
58  *  Carnegie Mellon University
59  *  Pittsburgh PA 15213-3890
60  *
61  * any improvements or extensions that they make and grant Carnegie the
62  * rights to redistribute these changes.
63  *
64  * $Id: vm_pager.c,v 1.50 1999/07/04 00:25:38 mckusick Exp $
65  */
66 
67 /*
68  *	Paging space routine stubs.  Emulates a matchmaker-like interface
69  *	for builtin pagers.
70  */
71 
72 #include <sys/param.h>
73 #include <sys/systm.h>
74 #include <sys/kernel.h>
75 #include <sys/vnode.h>
76 #include <sys/buf.h>
77 #include <sys/ucred.h>
78 #include <sys/malloc.h>
79 #include <sys/proc.h>
80 
81 #include <vm/vm.h>
82 #include <vm/vm_param.h>
83 #include <vm/vm_prot.h>
84 #include <vm/vm_object.h>
85 #include <vm/vm_page.h>
86 #include <vm/vm_pager.h>
87 #include <vm/vm_extern.h>
88 
89 MALLOC_DEFINE(M_VMPGDATA, "VM pgdata", "XXX: VM pager private data");
90 
91 extern struct pagerops defaultpagerops;
92 extern struct pagerops swappagerops;
93 extern struct pagerops vnodepagerops;
94 extern struct pagerops devicepagerops;
95 
96 int cluster_pbuf_freecnt = -1;	/* unlimited to begin with */
97 
98 static int dead_pager_getpages __P((vm_object_t, vm_page_t *, int, int));
99 static vm_object_t dead_pager_alloc __P((void *, vm_ooffset_t, vm_prot_t,
100 	vm_ooffset_t));
101 static void dead_pager_putpages __P((vm_object_t, vm_page_t *, int, int, int *));
102 static boolean_t dead_pager_haspage __P((vm_object_t, vm_pindex_t, int *, int *));
103 static void dead_pager_dealloc __P((vm_object_t));
104 
105 static int
106 dead_pager_getpages(obj, ma, count, req)
107 	vm_object_t obj;
108 	vm_page_t *ma;
109 	int count;
110 	int req;
111 {
112 	return VM_PAGER_FAIL;
113 }
114 
115 static vm_object_t
116 dead_pager_alloc(handle, size, prot, off)
117 	void *handle;
118 	vm_ooffset_t size;
119 	vm_prot_t prot;
120 	vm_ooffset_t off;
121 {
122 	return NULL;
123 }
124 
125 static void
126 dead_pager_putpages(object, m, count, flags, rtvals)
127 	vm_object_t object;
128 	vm_page_t *m;
129 	int count;
130 	int flags;
131 	int *rtvals;
132 {
133 	int i;
134 
135 	for (i = 0; i < count; i++) {
136 		rtvals[i] = VM_PAGER_AGAIN;
137 	}
138 }
139 
140 static int
141 dead_pager_haspage(object, pindex, prev, next)
142 	vm_object_t object;
143 	vm_pindex_t pindex;
144 	int *prev;
145 	int *next;
146 {
147 	if (prev)
148 		*prev = 0;
149 	if (next)
150 		*next = 0;
151 	return FALSE;
152 }
153 
154 static void
155 dead_pager_dealloc(object)
156 	vm_object_t object;
157 {
158 	return;
159 }
160 
161 static struct pagerops deadpagerops = {
162 	NULL,
163 	dead_pager_alloc,
164 	dead_pager_dealloc,
165 	dead_pager_getpages,
166 	dead_pager_putpages,
167 	dead_pager_haspage,
168 	NULL
169 };
170 
171 struct pagerops *pagertab[] = {
172 	&defaultpagerops,	/* OBJT_DEFAULT */
173 	&swappagerops,		/* OBJT_SWAP */
174 	&vnodepagerops,		/* OBJT_VNODE */
175 	&devicepagerops,	/* OBJT_DEVICE */
176 	&deadpagerops		/* OBJT_DEAD */
177 };
178 
179 int npagers = sizeof(pagertab) / sizeof(pagertab[0]);
180 
181 /*
182  * Kernel address space for mapping pages.
183  * Used by pagers where KVAs are needed for IO.
184  *
185  * XXX needs to be large enough to support the number of pending async
186  * cleaning requests (NPENDINGIO == 64) * the maximum swap cluster size
187  * (MAXPHYS == 64k) if you want to get the most efficiency.
188  */
189 #define PAGER_MAP_SIZE	(8 * 1024 * 1024)
190 
191 int pager_map_size = PAGER_MAP_SIZE;
192 vm_map_t pager_map;
193 static int bswneeded;
194 static vm_offset_t swapbkva;		/* swap buffers kva */
195 
196 void
197 vm_pager_init()
198 {
199 	struct pagerops **pgops;
200 
201 	/*
202 	 * Initialize known pagers
203 	 */
204 	for (pgops = pagertab; pgops < &pagertab[npagers]; pgops++)
205 		if (pgops && ((*pgops)->pgo_init != NULL))
206 			(*(*pgops)->pgo_init) ();
207 }
208 
209 void
210 vm_pager_bufferinit()
211 {
212 	struct buf *bp;
213 	int i;
214 
215 	bp = swbuf;
216 	/*
217 	 * Now set up swap and physical I/O buffer headers.
218 	 */
219 	for (i = 0; i < nswbuf; i++, bp++) {
220 		TAILQ_INSERT_HEAD(&bswlist, bp, b_freelist);
221 		BUF_LOCKINIT(bp);
222 		LIST_INIT(&bp->b_dep);
223 		bp->b_rcred = bp->b_wcred = NOCRED;
224 		bp->b_xflags = 0;
225 	}
226 
227 	cluster_pbuf_freecnt = nswbuf / 2;
228 
229 	swapbkva = kmem_alloc_pageable(pager_map, nswbuf * MAXPHYS);
230 	if (!swapbkva)
231 		panic("Not enough pager_map VM space for physical buffers");
232 }
233 
234 /*
235  * Allocate an instance of a pager of the given type.
236  * Size, protection and offset parameters are passed in for pagers that
237  * need to perform page-level validation (e.g. the device pager).
238  */
239 vm_object_t
240 vm_pager_allocate(objtype_t type, void *handle, vm_ooffset_t size, vm_prot_t prot,
241 		  vm_ooffset_t off)
242 {
243 	struct pagerops *ops;
244 
245 	ops = pagertab[type];
246 	if (ops)
247 		return ((*ops->pgo_alloc) (handle, size, prot, off));
248 	return (NULL);
249 }
250 
251 void
252 vm_pager_deallocate(object)
253 	vm_object_t object;
254 {
255 	(*pagertab[object->type]->pgo_dealloc) (object);
256 }
257 
258 /*
259  *      vm_pager_strategy:
260  *
261  *      called with no specific spl
262  *      Execute strategy routine directly to pager.
263  */
264 
265 void
266 vm_pager_strategy(vm_object_t object, struct buf *bp)
267 {
268 	if (pagertab[object->type]->pgo_strategy) {
269 	    (*pagertab[object->type]->pgo_strategy)(object, bp);
270 	} else {
271 		bp->b_flags |= B_ERROR;
272 		bp->b_error = ENXIO;
273 		biodone(bp);
274 	}
275 }
276 
277 /*
278  * vm_pager_get_pages() - inline, see vm/vm_pager.h
279  * vm_pager_put_pages() - inline, see vm/vm_pager.h
280  * vm_pager_has_page() - inline, see vm/vm_pager.h
281  * vm_pager_page_inserted() - inline, see vm/vm_pager.h
282  * vm_pager_page_removed() - inline, see vm/vm_pager.h
283  */
284 
285 #if 0
286 /*
287  *	vm_pager_sync:
288  *
289  *	Called by pageout daemon before going back to sleep.
290  *	Gives pagers a chance to clean up any completed async pageing
291  *	operations.
292  */
293 void
294 vm_pager_sync()
295 {
296 	struct pagerops **pgops;
297 
298 	for (pgops = pagertab; pgops < &pagertab[npagers]; pgops++)
299 		if (pgops && ((*pgops)->pgo_sync != NULL))
300 			(*(*pgops)->pgo_sync) ();
301 }
302 
303 #endif
304 
305 vm_offset_t
306 vm_pager_map_page(m)
307 	vm_page_t m;
308 {
309 	vm_offset_t kva;
310 
311 	kva = kmem_alloc_wait(pager_map, PAGE_SIZE);
312 	pmap_kenter(kva, VM_PAGE_TO_PHYS(m));
313 	return (kva);
314 }
315 
316 void
317 vm_pager_unmap_page(kva)
318 	vm_offset_t kva;
319 {
320 	pmap_kremove(kva);
321 	kmem_free_wakeup(pager_map, kva, PAGE_SIZE);
322 }
323 
324 vm_object_t
325 vm_pager_object_lookup(pg_list, handle)
326 	register struct pagerlst *pg_list;
327 	void *handle;
328 {
329 	register vm_object_t object;
330 
331 	for (object = TAILQ_FIRST(pg_list); object != NULL; object = TAILQ_NEXT(object,pager_object_list))
332 		if (object->handle == handle)
333 			return (object);
334 	return (NULL);
335 }
336 
337 /*
338  * initialize a physical buffer
339  */
340 
341 static void
342 initpbuf(struct buf *bp)
343 {
344 	bp->b_rcred = NOCRED;
345 	bp->b_wcred = NOCRED;
346 	bp->b_qindex = QUEUE_NONE;
347 	bp->b_data = (caddr_t) (MAXPHYS * (bp - swbuf)) + swapbkva;
348 	bp->b_kvabase = bp->b_data;
349 	bp->b_kvasize = MAXPHYS;
350 	bp->b_xflags = 0;
351 	bp->b_flags = 0;
352 	bp->b_error = 0;
353 	BUF_LOCK(bp, LK_EXCLUSIVE);
354 }
355 
356 /*
357  * allocate a physical buffer
358  *
359  *	There are a limited number (nswbuf) of physical buffers.  We need
360  *	to make sure that no single subsystem is able to hog all of them,
361  *	so each subsystem implements a counter which is typically initialized
362  *	to 1/2 nswbuf.  getpbuf() decrements this counter in allocation and
363  *	increments it on release, and blocks if the counter hits zero.  A
364  *	subsystem may initialize the counter to -1 to disable the feature,
365  *	but it must still be sure to match up all uses of getpbuf() with
366  *	relpbuf() using the same variable.
367  *
368  *	NOTE: pfreecnt can be NULL, but this 'feature' will be removed
369  *	relatively soon when the rest of the subsystems get smart about it. XXX
370  */
371 struct buf *
372 getpbuf(pfreecnt)
373 	int *pfreecnt;
374 {
375 	int s;
376 	struct buf *bp;
377 
378 	s = splvm();
379 
380 	for (;;) {
381 		if (pfreecnt) {
382 			while (*pfreecnt == 0) {
383 				tsleep(pfreecnt, PVM, "wswbuf0", 0);
384 			}
385 		}
386 
387 		/* get a bp from the swap buffer header pool */
388 		if ((bp = TAILQ_FIRST(&bswlist)) != NULL)
389 			break;
390 
391 		bswneeded = 1;
392 		tsleep(&bswneeded, PVM, "wswbuf1", 0);
393 		/* loop in case someone else grabbed one */
394 	}
395 	TAILQ_REMOVE(&bswlist, bp, b_freelist);
396 	if (pfreecnt)
397 		--*pfreecnt;
398 	splx(s);
399 
400 	initpbuf(bp);
401 	return bp;
402 }
403 
404 /*
405  * allocate a physical buffer, if one is available.
406  *
407  *	Note that there is no NULL hack here - all subsystems using this
408  *	call understand how to use pfreecnt.
409  */
410 struct buf *
411 trypbuf(pfreecnt)
412 	int *pfreecnt;
413 {
414 	int s;
415 	struct buf *bp;
416 
417 	s = splvm();
418 	if (*pfreecnt == 0 || (bp = TAILQ_FIRST(&bswlist)) == NULL) {
419 		splx(s);
420 		return NULL;
421 	}
422 	TAILQ_REMOVE(&bswlist, bp, b_freelist);
423 
424 	--*pfreecnt;
425 
426 	splx(s);
427 
428 	initpbuf(bp);
429 
430 	return bp;
431 }
432 
433 /*
434  * release a physical buffer
435  *
436  *	NOTE: pfreecnt can be NULL, but this 'feature' will be removed
437  *	relatively soon when the rest of the subsystems get smart about it. XXX
438  */
439 void
440 relpbuf(bp, pfreecnt)
441 	struct buf *bp;
442 	int *pfreecnt;
443 {
444 	int s;
445 
446 	s = splvm();
447 
448 	if (bp->b_rcred != NOCRED) {
449 		crfree(bp->b_rcred);
450 		bp->b_rcred = NOCRED;
451 	}
452 	if (bp->b_wcred != NOCRED) {
453 		crfree(bp->b_wcred);
454 		bp->b_wcred = NOCRED;
455 	}
456 
457 	if (bp->b_vp)
458 		pbrelvp(bp);
459 
460 	BUF_UNLOCK(bp);
461 
462 	TAILQ_INSERT_HEAD(&bswlist, bp, b_freelist);
463 
464 	if (bswneeded) {
465 		bswneeded = 0;
466 		wakeup(&bswneeded);
467 	}
468 	if (pfreecnt) {
469 		if (++*pfreecnt == 1)
470 			wakeup(pfreecnt);
471 	}
472 	splx(s);
473 }
474 
475 /********************************************************
476  *		CHAINING FUNCTIONS			*
477  ********************************************************
478  *
479  *	These functions support recursion of I/O operations
480  *	on bp's, typically by chaining one or more 'child' bp's
481  *	to the parent.  Synchronous, asynchronous, and semi-synchronous
482  *	chaining is possible.
483  */
484 
485 /*
486  *	vm_pager_chain_iodone:
487  *
488  *	io completion routine for child bp.  Currently we fudge a bit
489  *	on dealing with b_resid.   Since users of these routines may issue
490  *	multiple children simultaniously, sequencing of the error can be lost.
491  */
492 
493 static void
494 vm_pager_chain_iodone(struct buf *nbp)
495 {
496 	struct buf *bp;
497 
498 	if ((bp = nbp->b_chain.parent) != NULL) {
499 		if (nbp->b_flags & B_ERROR) {
500 			bp->b_flags |= B_ERROR;
501 			bp->b_error = nbp->b_error;
502 		} else if (nbp->b_resid != 0) {
503 			bp->b_flags |= B_ERROR;
504 			bp->b_error = EINVAL;
505 		} else {
506 			bp->b_resid -= nbp->b_bcount;
507 		}
508 		nbp->b_chain.parent = NULL;
509 		--bp->b_chain.count;
510 		if (bp->b_flags & B_WANT) {
511 			bp->b_flags &= ~B_WANT;
512 			wakeup(bp);
513 		}
514 		if (!bp->b_chain.count && (bp->b_flags & B_AUTOCHAINDONE)) {
515 			bp->b_flags &= ~B_AUTOCHAINDONE;
516 			if (bp->b_resid != 0 && !(bp->b_flags & B_ERROR)) {
517 				bp->b_flags |= B_ERROR;
518 				bp->b_error = EINVAL;
519 			}
520 			biodone(bp);
521 		}
522 	}
523 	nbp->b_flags |= B_DONE;
524 	nbp->b_flags &= ~B_ASYNC;
525 	relpbuf(nbp, NULL);
526 }
527 
528 /*
529  *	getchainbuf:
530  *
531  *	Obtain a physical buffer and chain it to its parent buffer.  When
532  *	I/O completes, the parent buffer will be B_SIGNAL'd.  Errors are
533  *	automatically propogated to the parent
534  *
535  *	Since these are brand new buffers, we do not have to clear B_INVAL
536  *	and B_ERROR because they are already clear.
537  */
538 
539 struct buf *
540 getchainbuf(struct buf *bp, struct vnode *vp, int flags)
541 {
542 	struct buf *nbp = getpbuf(NULL);
543 
544 	nbp->b_chain.parent = bp;
545 	++bp->b_chain.count;
546 
547 	if (bp->b_chain.count > 4)
548 		waitchainbuf(bp, 4, 0);
549 
550 	nbp->b_flags = B_CALL | (bp->b_flags & B_ORDERED) | flags;
551 	nbp->b_rcred = nbp->b_wcred = proc0.p_ucred;
552 	nbp->b_iodone = vm_pager_chain_iodone;
553 
554 	crhold(nbp->b_rcred);
555 	crhold(nbp->b_wcred);
556 
557 	if (vp)
558 		pbgetvp(vp, nbp);
559 	return(nbp);
560 }
561 
562 void
563 flushchainbuf(struct buf *nbp)
564 {
565 	if (nbp->b_bcount) {
566 		nbp->b_bufsize = nbp->b_bcount;
567 		if ((nbp->b_flags & B_READ) == 0)
568 			nbp->b_dirtyend = nbp->b_bcount;
569 		VOP_STRATEGY(nbp->b_vp, nbp);
570 	} else {
571 		biodone(nbp);
572 	}
573 }
574 
575 void
576 waitchainbuf(struct buf *bp, int count, int done)
577 {
578  	int s;
579 
580 	s = splbio();
581 	while (bp->b_chain.count > count) {
582 		bp->b_flags |= B_WANT;
583 		tsleep(bp, PRIBIO + 4, "bpchain", 0);
584 	}
585 	if (done) {
586 		if (bp->b_resid != 0 && !(bp->b_flags & B_ERROR)) {
587 			bp->b_flags |= B_ERROR;
588 			bp->b_error = EINVAL;
589 		}
590 		biodone(bp);
591 	}
592 	splx(s);
593 }
594 
595 void
596 autochaindone(struct buf *bp)
597 {
598  	int s;
599 
600 	s = splbio();
601 	if (bp->b_chain.count == 0)
602 		biodone(bp);
603 	else
604 		bp->b_flags |= B_AUTOCHAINDONE;
605 	splx(s);
606 }
607 
608