xref: /titanic_41/usr/src/uts/common/fs/swapfs/swap_vnops.c (revision 9c8b6a95c9422a18be4946d4fca84a58fa1385ff)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/buf.h>
32 #include <sys/cred.h>
33 #include <sys/errno.h>
34 #include <sys/vnode.h>
35 #include <sys/vfs_opreg.h>
36 #include <sys/cmn_err.h>
37 #include <sys/swap.h>
38 #include <sys/mman.h>
39 #include <sys/vmsystm.h>
40 #include <sys/vtrace.h>
41 #include <sys/debug.h>
42 #include <sys/sysmacros.h>
43 #include <sys/vm.h>
44 
45 #include <sys/fs/swapnode.h>
46 
47 #include <vm/seg.h>
48 #include <vm/page.h>
49 #include <vm/pvn.h>
50 #include <fs/fs_subr.h>
51 
52 #include <vm/seg_kp.h>
53 
54 /*
55  * Define the routines within this file.
56  */
57 static int	swap_getpage(struct vnode *vp, offset_t off, size_t len,
58     uint_t *protp, struct page **plarr, size_t plsz, struct seg *seg,
59     caddr_t addr, enum seg_rw rw, struct cred *cr, caller_context_t *ct);
60 static int	swap_putpage(struct vnode *vp, offset_t off, size_t len,
61     int flags, struct cred *cr, caller_context_t *ct);
62 static void	swap_inactive(struct vnode *vp, struct cred *cr,
63     caller_context_t *ct);
64 static void	swap_dispose(vnode_t *vp, page_t *pp, int fl, int dn,
65     cred_t *cr, caller_context_t *ct);
66 
67 static int	swap_getapage(struct vnode *vp, u_offset_t off, size_t len,
68     uint_t *protp, page_t **plarr, size_t plsz,
69     struct seg *seg, caddr_t addr, enum seg_rw rw, struct cred *cr);
70 
71 int	swap_getconpage(struct vnode *vp, u_offset_t off, size_t len,
72     uint_t *protp, page_t **plarr, size_t plsz, page_t *conpp,
73     uint_t *pszc, spgcnt_t *nreloc, struct seg *seg, caddr_t addr,
74     enum seg_rw rw, struct cred *cr);
75 
76 static int 	swap_putapage(struct vnode *vp, page_t *pp, u_offset_t *off,
77     size_t *lenp, int flags, struct cred *cr);
78 
79 const fs_operation_def_t swap_vnodeops_template[] = {
80 	VOPNAME_INACTIVE,	{ .vop_inactive = swap_inactive },
81 	VOPNAME_GETPAGE,	{ .vop_getpage = swap_getpage },
82 	VOPNAME_PUTPAGE,	{ .vop_putpage = swap_putpage },
83 	VOPNAME_DISPOSE,	{ .vop_dispose = swap_dispose },
84 	VOPNAME_SETFL,		{ .error = fs_error },
85 	VOPNAME_POLL,		{ .error = fs_error },
86 	VOPNAME_PATHCONF,	{ .error = fs_error },
87 	VOPNAME_GETSECATTR,	{ .error = fs_error },
88 	VOPNAME_SHRLOCK,	{ .error = fs_error },
89 	NULL,			NULL
90 };
91 
92 vnodeops_t *swap_vnodeops;
93 
94 /* ARGSUSED */
95 static void
96 swap_inactive(
97 	struct vnode *vp,
98 	struct cred *cr,
99 	caller_context_t *ct)
100 {
101 	SWAPFS_PRINT(SWAP_VOPS, "swap_inactive: vp %x\n", vp, 0, 0, 0, 0);
102 }
103 
104 /*
105  * Return all the pages from [off..off+len] in given file
106  */
107 /*ARGSUSED*/
108 static int
109 swap_getpage(
110 	struct vnode *vp,
111 	offset_t off,
112 	size_t len,
113 	uint_t *protp,
114 	page_t *pl[],
115 	size_t plsz,
116 	struct seg *seg,
117 	caddr_t addr,
118 	enum seg_rw rw,
119 	struct cred *cr,
120 	caller_context_t *ct)
121 {
122 	int err;
123 
124 	SWAPFS_PRINT(SWAP_VOPS, "swap_getpage: vp %p, off %llx, len %lx\n",
125 	    (void *)vp, off, len, 0, 0);
126 
127 	TRACE_3(TR_FAC_SWAPFS, TR_SWAPFS_GETPAGE,
128 	    "swapfs getpage:vp %p off %llx len %ld",
129 	    (void *)vp, off, len);
130 
131 	if (len <= PAGESIZE) {
132 		err = swap_getapage(vp, (u_offset_t)off, len, protp, pl, plsz,
133 		    seg, addr, rw, cr);
134 	} else {
135 		err = pvn_getpages(swap_getapage, vp, (u_offset_t)off, len,
136 		    protp, pl, plsz, seg, addr, rw, cr);
137 	}
138 
139 	return (err);
140 }
141 
142 /*
143  * Called from pvn_getpages or swap_getpage to get a particular page.
144  */
145 /*ARGSUSED*/
146 static int
147 swap_getapage(
148 	struct vnode *vp,
149 	u_offset_t off,
150 	size_t len,
151 	uint_t *protp,
152 	page_t *pl[],
153 	size_t plsz,
154 	struct seg *seg,
155 	caddr_t addr,
156 	enum seg_rw rw,
157 	struct cred *cr)
158 {
159 	struct page *pp, *rpp;
160 	int flags;
161 	int err = 0;
162 	struct vnode *pvp = NULL;
163 	u_offset_t poff;
164 	int flag_noreloc;
165 	se_t lock;
166 	extern int kcage_on;
167 	int upgrade = 0;
168 
169 	SWAPFS_PRINT(SWAP_VOPS, "swap_getapage: vp %p, off %llx, len %lx\n",
170 	    vp, off, len, 0, 0);
171 
172 	/*
173 	 * Until there is a call-back mechanism to cause SEGKP
174 	 * pages to be unlocked, make them non-relocatable.
175 	 */
176 	if (SEG_IS_SEGKP(seg))
177 		flag_noreloc = PG_NORELOC;
178 	else
179 		flag_noreloc = 0;
180 
181 	if (protp != NULL)
182 		*protp = PROT_ALL;
183 
184 	lock = (rw == S_CREATE ? SE_EXCL : SE_SHARED);
185 
186 again:
187 	if (pp = page_lookup(vp, off, lock)) {
188 		/*
189 		 * In very rare instances, a segkp page may have been
190 		 * relocated outside of the kernel by the kernel cage
191 		 * due to the window between page_unlock() and
192 		 * VOP_PUTPAGE() in segkp_unlock().  Due to the
193 		 * rareness of these occurances, the solution is to
194 		 * relocate the page to a P_NORELOC page.
195 		 */
196 		if (flag_noreloc != 0) {
197 			if (!PP_ISNORELOC(pp) && kcage_on) {
198 				if (lock != SE_EXCL) {
199 					upgrade = 1;
200 					if (!page_tryupgrade(pp)) {
201 						page_unlock(pp);
202 						lock = SE_EXCL;
203 						goto again;
204 					}
205 				}
206 
207 				if (page_relocate_cage(&pp, &rpp) != 0)
208 					panic("swap_getapage: "
209 					    "page_relocate_cage failed");
210 
211 				pp = rpp;
212 			}
213 		}
214 
215 		if (pl) {
216 			if (upgrade)
217 				page_downgrade(pp);
218 
219 			pl[0] = pp;
220 			pl[1] = NULL;
221 		} else {
222 			page_unlock(pp);
223 		}
224 	} else {
225 		pp = page_create_va(vp, off, PAGESIZE,
226 		    PG_WAIT | PG_EXCL | flag_noreloc,
227 		    seg, addr);
228 		/*
229 		 * Someone raced in and created the page after we did the
230 		 * lookup but before we did the create, so go back and
231 		 * try to look it up again.
232 		 */
233 		if (pp == NULL)
234 			goto again;
235 		if (rw != S_CREATE) {
236 			err = swap_getphysname(vp, off, &pvp, &poff);
237 			if (pvp) {
238 				struct anon *ap;
239 				kmutex_t *ahm;
240 
241 				flags = (pl == NULL ? B_ASYNC|B_READ : B_READ);
242 				err = VOP_PAGEIO(pvp, pp, poff,
243 				    PAGESIZE, flags, cr, NULL);
244 
245 				if (!err) {
246 					ahm = &anonhash_lock[AH_LOCK(vp, off)];
247 					mutex_enter(ahm);
248 
249 					ap = swap_anon(vp, off);
250 					if (ap == NULL) {
251 						panic("swap_getapage:"
252 						    " null anon");
253 					}
254 
255 					if (ap->an_pvp == pvp &&
256 					    ap->an_poff == poff) {
257 						swap_phys_free(pvp, poff,
258 						    PAGESIZE);
259 						ap->an_pvp = NULL;
260 						ap->an_poff = NULL;
261 						hat_setmod(pp);
262 					}
263 
264 					mutex_exit(ahm);
265 				}
266 			} else {
267 				if (!err)
268 					pagezero(pp, 0, PAGESIZE);
269 
270 				/*
271 				 * If it's a fault ahead, release page_io_lock
272 				 * and SE_EXCL we grabbed in page_create_va
273 				 *
274 				 * If we are here, we haven't called VOP_PAGEIO
275 				 * and thus calling pvn_read_done(pp, B_READ)
276 				 * below may mislead that we tried i/o. Besides,
277 				 * in case of async, pvn_read_done() should
278 				 * not be called by *getpage()
279 				 */
280 				if (pl == NULL) {
281 					/*
282 					 * swap_getphysname can return error
283 					 * only when we are getting called from
284 					 * swapslot_free which passes non-NULL
285 					 * pl to VOP_GETPAGE.
286 					 */
287 					ASSERT(err == 0);
288 					page_io_unlock(pp);
289 					page_unlock(pp);
290 				}
291 			}
292 		}
293 
294 		ASSERT(pp != NULL);
295 
296 		if (err && pl)
297 			pvn_read_done(pp, B_ERROR);
298 
299 		if (!err && pl)
300 			pvn_plist_init(pp, pl, plsz, off, PAGESIZE, rw);
301 	}
302 	TRACE_3(TR_FAC_SWAPFS, TR_SWAPFS_GETAPAGE,
303 	    "swapfs getapage:pp %p vp %p off %llx", pp, vp, off);
304 	return (err);
305 }
306 
307 /*
308  * Called from large page anon routines only! This is an ugly hack where
309  * the anon layer directly calls into swapfs with a preallocated large page.
310  * Another method would have been to change to VOP and add an extra arg for
311  * the preallocated large page. This all could be cleaned up later when we
312  * solve the anonymous naming problem and no longer need to loop across of
313  * the VOP in PAGESIZE increments to fill in or initialize a large page as
314  * is done today. I think the latter is better since it avoid a change to
315  * the VOP interface that could later be avoided.
316  */
317 int
318 swap_getconpage(
319 	struct vnode *vp,
320 	u_offset_t off,
321 	size_t len,
322 	uint_t *protp,
323 	page_t *pl[],
324 	size_t plsz,
325 	page_t	*conpp,
326 	uint_t	*pszc,
327 	spgcnt_t *nreloc,
328 	struct seg *seg,
329 	caddr_t addr,
330 	enum seg_rw rw,
331 	struct cred *cr)
332 {
333 	struct page	*pp;
334 	int 		err = 0;
335 	struct vnode	*pvp = NULL;
336 	u_offset_t	poff;
337 
338 	ASSERT(len == PAGESIZE);
339 	ASSERT(pl != NULL);
340 	ASSERT(plsz == PAGESIZE);
341 	ASSERT(protp == NULL);
342 	ASSERT(nreloc != NULL);
343 	ASSERT(!SEG_IS_SEGKP(seg)); /* XXX for now not supported */
344 	SWAPFS_PRINT(SWAP_VOPS, "swap_getconpage: vp %p, off %llx, len %lx\n",
345 	    vp, off, len, 0, 0);
346 
347 	/*
348 	 * If we are not using a preallocated page then we know one already
349 	 * exists. So just let the old code handle it.
350 	 */
351 	if (conpp == NULL) {
352 		err = swap_getapage(vp, (u_offset_t)off, len, protp, pl, plsz,
353 		    seg, addr, rw, cr);
354 		return (err);
355 	}
356 	ASSERT(conpp->p_szc != 0);
357 	ASSERT(PAGE_EXCL(conpp));
358 
359 
360 	ASSERT(conpp->p_next == conpp);
361 	ASSERT(conpp->p_prev == conpp);
362 	ASSERT(!PP_ISAGED(conpp));
363 	ASSERT(!PP_ISFREE(conpp));
364 
365 	*nreloc = 0;
366 	pp = page_lookup_create(vp, off, SE_SHARED, conpp, nreloc, 0);
367 
368 	/*
369 	 * If existing page is found we may need to relocate.
370 	 */
371 	if (pp != conpp) {
372 		ASSERT(rw != S_CREATE);
373 		ASSERT(pszc != NULL);
374 		ASSERT(PAGE_SHARED(pp));
375 		if (pp->p_szc < conpp->p_szc) {
376 			*pszc = pp->p_szc;
377 			page_unlock(pp);
378 			err = -1;
379 		} else if (pp->p_szc > conpp->p_szc &&
380 		    seg->s_szc > conpp->p_szc) {
381 			*pszc = MIN(pp->p_szc, seg->s_szc);
382 			page_unlock(pp);
383 			err = -2;
384 		} else {
385 			pl[0] = pp;
386 			pl[1] = NULL;
387 			if (page_pptonum(pp) &
388 			    (page_get_pagecnt(conpp->p_szc) - 1))
389 				cmn_err(CE_PANIC, "swap_getconpage: no root");
390 		}
391 		return (err);
392 	}
393 
394 	ASSERT(PAGE_EXCL(pp));
395 
396 	if (*nreloc != 0) {
397 		ASSERT(rw != S_CREATE);
398 		pl[0] = pp;
399 		pl[1] = NULL;
400 		return (0);
401 	}
402 
403 	*nreloc = 1;
404 
405 	/*
406 	 * If necessary do the page io.
407 	 */
408 	if (rw != S_CREATE) {
409 		/*
410 		 * Since we are only called now on behalf of an
411 		 * address space operation it's impossible for
412 		 * us to fail unlike swap_getapge() which
413 		 * also gets called from swapslot_free().
414 		 */
415 		if (swap_getphysname(vp, off, &pvp, &poff)) {
416 			cmn_err(CE_PANIC,
417 			    "swap_getconpage: swap_getphysname failed!");
418 		}
419 
420 		if (pvp != NULL) {
421 			err = VOP_PAGEIO(pvp, pp, poff, PAGESIZE, B_READ,
422 			    cr, NULL);
423 			if (err == 0) {
424 				struct anon *ap;
425 				kmutex_t *ahm;
426 
427 				ahm = &anonhash_lock[AH_LOCK(vp, off)];
428 				mutex_enter(ahm);
429 				ap = swap_anon(vp, off);
430 				if (ap == NULL)
431 					panic("swap_getconpage: null anon");
432 				if (ap->an_pvp != pvp || ap->an_poff != poff)
433 					panic("swap_getconpage: bad anon");
434 
435 				swap_phys_free(pvp, poff, PAGESIZE);
436 				ap->an_pvp = NULL;
437 				ap->an_poff = NULL;
438 				hat_setmod(pp);
439 				mutex_exit(ahm);
440 			}
441 		} else {
442 			pagezero(pp, 0, PAGESIZE);
443 		}
444 	}
445 
446 	/*
447 	 * Normally we would let pvn_read_done() destroy
448 	 * the page on IO error. But since this is a preallocated
449 	 * page we'll let the anon layer handle it.
450 	 */
451 	page_io_unlock(pp);
452 	if (err != 0)
453 		page_hashout(pp, NULL);
454 	ASSERT(pp->p_next == pp);
455 	ASSERT(pp->p_prev == pp);
456 
457 	TRACE_3(TR_FAC_SWAPFS, TR_SWAPFS_GETAPAGE,
458 	    "swapfs getconpage:pp %p vp %p off %llx", pp, vp, off);
459 
460 	pl[0] = pp;
461 	pl[1] = NULL;
462 	return (err);
463 }
464 
465 /* Async putpage klustering stuff */
466 int sw_pending_size;
467 extern int klustsize;
468 extern struct async_reqs *sw_getreq();
469 extern void sw_putreq(struct async_reqs *);
470 extern void sw_putbackreq(struct async_reqs *);
471 extern struct async_reqs *sw_getfree();
472 extern void sw_putfree(struct async_reqs *);
473 
474 static size_t swap_putpagecnt, swap_pagespushed;
475 static size_t swap_otherfail, swap_otherpages;
476 static size_t swap_klustfail, swap_klustpages;
477 static size_t swap_getiofail, swap_getiopages;
478 
479 /*
480  * Flags are composed of {B_INVAL, B_DIRTY B_FREE, B_DONTNEED}.
481  * If len == 0, do from off to EOF.
482  */
483 static int swap_nopage = 0;	/* Don't do swap_putpage's if set */
484 
485 /* ARGSUSED */
486 static int
487 swap_putpage(
488 	struct vnode *vp,
489 	offset_t off,
490 	size_t len,
491 	int flags,
492 	struct cred *cr,
493 	caller_context_t *ct)
494 {
495 	page_t *pp;
496 	u_offset_t io_off;
497 	size_t io_len = 0;
498 	int err = 0;
499 	struct async_reqs *arg;
500 
501 	if (swap_nopage)
502 		return (0);
503 
504 	ASSERT(vp->v_count != 0);
505 
506 	/*
507 	 * Clear force flag so that p_lckcnt pages are not invalidated.
508 	 */
509 	flags &= ~B_FORCE;
510 
511 	SWAPFS_PRINT(SWAP_VOPS,
512 	    "swap_putpage: vp %p, off %llx len %lx, flags %x\n",
513 	    (void *)vp, off, len, flags, 0);
514 	TRACE_3(TR_FAC_SWAPFS, TR_SWAPFS_PUTPAGE,
515 	    "swapfs putpage:vp %p off %llx len %ld", (void *)vp, off, len);
516 
517 	if (vp->v_flag & VNOMAP)
518 		return (ENOSYS);
519 
520 	if (!vn_has_cached_data(vp))
521 		return (0);
522 
523 	if (len == 0) {
524 		if (curproc == proc_pageout)
525 			cmn_err(CE_PANIC, "swapfs: pageout can't block");
526 
527 		/* Search the entire vp list for pages >= off. */
528 		err = pvn_vplist_dirty(vp, (u_offset_t)off, swap_putapage,
529 		    flags, cr);
530 	} else {
531 		u_offset_t eoff;
532 
533 		/*
534 		 * Loop over all offsets in the range [off...off + len]
535 		 * looking for pages to deal with.
536 		 */
537 		eoff = off + len;
538 		for (io_off = (u_offset_t)off; io_off < eoff;
539 		    io_off += io_len) {
540 			/*
541 			 * If we run out of the async req slot, put the page
542 			 * now instead of queuing.
543 			 */
544 			if (flags == (B_ASYNC | B_FREE) &&
545 			    sw_pending_size < klustsize &&
546 			    (arg = sw_getfree())) {
547 				/*
548 				 * If we are clustering, we should allow
549 				 * pageout to feed us more pages because # of
550 				 * pushes is limited by # of I/Os, and one
551 				 * cluster is considered to be one I/O.
552 				 */
553 				if (pushes)
554 					pushes--;
555 
556 				arg->a_vp = vp;
557 				arg->a_off = io_off;
558 				arg->a_len = PAGESIZE;
559 				arg->a_flags = B_ASYNC | B_FREE;
560 				arg->a_cred = kcred;
561 				sw_putreq(arg);
562 				io_len = PAGESIZE;
563 				continue;
564 			}
565 			/*
566 			 * If we are not invalidating pages, use the
567 			 * routine page_lookup_nowait() to prevent
568 			 * reclaiming them from the free list.
569 			 */
570 			if ((flags & B_INVAL) ||
571 			    (flags & (B_ASYNC | B_FREE)) == B_FREE)
572 				pp = page_lookup(vp, io_off, SE_EXCL);
573 			else
574 				pp = page_lookup_nowait(vp, io_off,
575 				    (flags & B_FREE) ? SE_EXCL : SE_SHARED);
576 
577 			if (pp == NULL || pvn_getdirty(pp, flags) == 0)
578 				io_len = PAGESIZE;
579 			else {
580 				err = swap_putapage(vp, pp, &io_off, &io_len,
581 				    flags, cr);
582 				if (err != 0)
583 					break;
584 			}
585 		}
586 	}
587 	/* If invalidating, verify all pages on vnode list are gone. */
588 	if (err == 0 && off == 0 && len == 0 &&
589 	    (flags & B_INVAL) && vn_has_cached_data(vp)) {
590 		cmn_err(CE_WARN,
591 		    "swap_putpage: B_INVAL, pages not gone");
592 	}
593 	return (err);
594 }
595 
596 /*
597  * Write out a single page.
598  * For swapfs this means choose a physical swap slot and write the page
599  * out using VOP_PAGEIO.
600  * In the (B_ASYNC | B_FREE) case we try to find a bunch of other dirty
601  * swapfs pages, a bunch of contiguous swap slots and then write them
602  * all out in one clustered i/o.
603  */
604 /*ARGSUSED*/
605 static int
606 swap_putapage(
607 	struct vnode *vp,
608 	page_t *pp,
609 	u_offset_t *offp,
610 	size_t *lenp,
611 	int flags,
612 	struct cred *cr)
613 {
614 	int err;
615 	struct vnode *pvp;
616 	u_offset_t poff, off;
617 	u_offset_t doff;
618 	size_t dlen;
619 	size_t klsz = 0;
620 	u_offset_t klstart = 0;
621 	struct vnode *klvp = NULL;
622 	page_t *pplist;
623 	se_t se;
624 	struct async_reqs *arg;
625 	size_t swap_klustsize;
626 
627 	/*
628 	 * This check is added for callers who access swap_putpage with len = 0.
629 	 * swap_putpage calls swap_putapage page-by-page via pvn_vplist_dirty.
630 	 * And it's necessary to do the same queuing if users have the same
631 	 * B_ASYNC|B_FREE flags on.
632 	 */
633 	if (flags == (B_ASYNC | B_FREE) &&
634 	    sw_pending_size < klustsize && (arg = sw_getfree())) {
635 
636 		hat_setmod(pp);
637 		page_io_unlock(pp);
638 		page_unlock(pp);
639 
640 		arg->a_vp = vp;
641 		arg->a_off = pp->p_offset;
642 		arg->a_len = PAGESIZE;
643 		arg->a_flags = B_ASYNC | B_FREE;
644 		arg->a_cred = kcred;
645 		sw_putreq(arg);
646 
647 		return (0);
648 	}
649 
650 	SWAPFS_PRINT(SWAP_PUTP,
651 	    "swap_putapage: pp %p, vp %p, off %llx, flags %x\n",
652 	    pp, vp, pp->p_offset, flags, 0);
653 
654 	ASSERT(PAGE_LOCKED(pp));
655 
656 	off = pp->p_offset;
657 
658 	doff = off;
659 	dlen = PAGESIZE;
660 
661 	if (err = swap_newphysname(vp, off, &doff, &dlen, &pvp, &poff)) {
662 		err = (flags == (B_ASYNC | B_FREE) ? ENOMEM : 0);
663 		hat_setmod(pp);
664 		page_io_unlock(pp);
665 		page_unlock(pp);
666 		goto out;
667 	}
668 
669 	klvp = pvp;
670 	klstart = poff;
671 	pplist = pp;
672 	/*
673 	 * If this is ASYNC | FREE and we've accumulated a bunch of such
674 	 * pending requests, kluster.
675 	 */
676 	if (flags == (B_ASYNC | B_FREE))
677 		swap_klustsize = klustsize;
678 	else
679 		swap_klustsize = PAGESIZE;
680 	se = (flags & B_FREE ? SE_EXCL : SE_SHARED);
681 	klsz = PAGESIZE;
682 	while (klsz < swap_klustsize) {
683 		if ((arg = sw_getreq()) == NULL) {
684 			swap_getiofail++;
685 			swap_getiopages += btop(klsz);
686 			break;
687 		}
688 		ASSERT(vn_matchops(arg->a_vp, swap_vnodeops));
689 		vp = arg->a_vp;
690 		off = arg->a_off;
691 
692 		if ((pp = page_lookup_nowait(vp, off, se)) == NULL) {
693 			swap_otherfail++;
694 			swap_otherpages += btop(klsz);
695 			sw_putfree(arg);
696 			break;
697 		}
698 		if (pvn_getdirty(pp, flags | B_DELWRI) == 0) {
699 			sw_putfree(arg);
700 			continue;
701 		}
702 		/* Get new physical backing store for the page */
703 		doff = off;
704 		dlen = PAGESIZE;
705 		if (err = swap_newphysname(vp, off, &doff, &dlen,
706 		    &pvp, &poff)) {
707 			swap_otherfail++;
708 			swap_otherpages += btop(klsz);
709 			hat_setmod(pp);
710 			page_io_unlock(pp);
711 			page_unlock(pp);
712 			sw_putbackreq(arg);
713 			break;
714 		}
715 		/* Try to cluster new physical name with previous ones */
716 		if (klvp == pvp && poff == klstart + klsz) {
717 			klsz += PAGESIZE;
718 			page_add(&pplist, pp);
719 			pplist = pplist->p_next;
720 			sw_putfree(arg);
721 		} else if (klvp == pvp && poff == klstart - PAGESIZE) {
722 			klsz += PAGESIZE;
723 			klstart -= PAGESIZE;
724 			page_add(&pplist, pp);
725 			sw_putfree(arg);
726 		} else {
727 			swap_klustfail++;
728 			swap_klustpages += btop(klsz);
729 			hat_setmod(pp);
730 			page_io_unlock(pp);
731 			page_unlock(pp);
732 			sw_putbackreq(arg);
733 			break;
734 		}
735 	}
736 
737 	err = VOP_PAGEIO(klvp, pplist, klstart, klsz,
738 	    B_WRITE | flags, cr, NULL);
739 
740 	if ((flags & B_ASYNC) == 0)
741 		pvn_write_done(pp, ((err) ? B_ERROR : 0) | B_WRITE | flags);
742 
743 	/* Statistics */
744 	if (!err) {
745 		swap_putpagecnt++;
746 		swap_pagespushed += btop(klsz);
747 	}
748 out:
749 	TRACE_4(TR_FAC_SWAPFS, TR_SWAPFS_PUTAPAGE,
750 	    "swapfs putapage:vp %p klvp %p, klstart %lx, klsz %lx",
751 	    vp, klvp, klstart, klsz);
752 	if (err && err != ENOMEM)
753 		cmn_err(CE_WARN, "swapfs_putapage: err %d\n", err);
754 	if (lenp)
755 		*lenp = PAGESIZE;
756 	return (err);
757 }
758 
759 static void
760 swap_dispose(
761 	vnode_t *vp,
762 	page_t *pp,
763 	int fl,
764 	int dn,
765 	cred_t *cr,
766 	caller_context_t *ct)
767 {
768 	int err;
769 	u_offset_t off = pp->p_offset;
770 	vnode_t *pvp;
771 	u_offset_t poff;
772 
773 	ASSERT(PAGE_EXCL(pp));
774 
775 	/*
776 	 * The caller will free/invalidate large page in one shot instead of
777 	 * one small page at a time.
778 	 */
779 	if (pp->p_szc != 0) {
780 		page_unlock(pp);
781 		return;
782 	}
783 
784 	err = swap_getphysname(vp, off, &pvp, &poff);
785 	if (!err && pvp != NULL)
786 		VOP_DISPOSE(pvp, pp, fl, dn, cr, ct);
787 	else
788 		fs_dispose(vp, pp, fl, dn, cr, ct);
789 }
790