xref: /freebsd/sys/kern/vfs_cluster.c (revision f6a3b357e9be4c6423c85eff9a847163a0d307c8)
1 /*-
2  * SPDX-License-Identifier: BSD-3-Clause
3  *
4  * Copyright (c) 1993
5  *	The Regents of the University of California.  All rights reserved.
6  * Modifications/enhancements:
7  * 	Copyright (c) 1995 John S. Dyson.  All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  * 3. Neither the name of the University nor the names of its contributors
18  *    may be used to endorse or promote products derived from this software
19  *    without specific prior written permission.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  *
33  *	@(#)vfs_cluster.c	8.7 (Berkeley) 2/13/94
34  */
35 
36 #include <sys/cdefs.h>
37 __FBSDID("$FreeBSD$");
38 
39 #include "opt_debug_cluster.h"
40 
41 #include <sys/param.h>
42 #include <sys/systm.h>
43 #include <sys/kernel.h>
44 #include <sys/proc.h>
45 #include <sys/bio.h>
46 #include <sys/buf.h>
47 #include <sys/vnode.h>
48 #include <sys/malloc.h>
49 #include <sys/mount.h>
50 #include <sys/racct.h>
51 #include <sys/resourcevar.h>
52 #include <sys/rwlock.h>
53 #include <sys/vmmeter.h>
54 #include <vm/vm.h>
55 #include <vm/vm_object.h>
56 #include <vm/vm_page.h>
57 #include <sys/sysctl.h>
58 
59 #if defined(CLUSTERDEBUG)
60 static int	rcluster= 0;
61 SYSCTL_INT(_debug, OID_AUTO, rcluster, CTLFLAG_RW, &rcluster, 0,
62     "Debug VFS clustering code");
63 #endif
64 
65 static MALLOC_DEFINE(M_SEGMENT, "cl_savebuf", "cluster_save buffer");
66 static uma_zone_t cluster_pbuf_zone;
67 
68 static void cluster_init(void *);
69 static struct cluster_save *cluster_collectbufs(struct vnode *vp,
70 	    struct buf *last_bp, int gbflags);
71 static struct buf *cluster_rbuild(struct vnode *vp, u_quad_t filesize,
72 	    daddr_t lbn, daddr_t blkno, long size, int run, int gbflags,
73 	    struct buf *fbp);
74 static void cluster_callback(struct buf *);
75 
76 static int write_behind = 1;
77 SYSCTL_INT(_vfs, OID_AUTO, write_behind, CTLFLAG_RW, &write_behind, 0,
78     "Cluster write-behind; 0: disable, 1: enable, 2: backed off");
79 
80 static int read_max = 64;
81 SYSCTL_INT(_vfs, OID_AUTO, read_max, CTLFLAG_RW, &read_max, 0,
82     "Cluster read-ahead max block count");
83 
84 static int read_min = 1;
85 SYSCTL_INT(_vfs, OID_AUTO, read_min, CTLFLAG_RW, &read_min, 0,
86     "Cluster read min block count");
87 
88 SYSINIT(cluster, SI_SUB_CPU, SI_ORDER_ANY, cluster_init, NULL);
89 
90 static void
91 cluster_init(void *dummy)
92 {
93 
94 	cluster_pbuf_zone = pbuf_zsecond_create("clpbuf", nswbuf / 2);
95 }
96 
97 /*
98  * Read data to a buf, including read-ahead if we find this to be beneficial.
99  * cluster_read replaces bread.
100  */
101 int
102 cluster_read(struct vnode *vp, u_quad_t filesize, daddr_t lblkno, long size,
103     struct ucred *cred, long totread, int seqcount, int gbflags,
104     struct buf **bpp)
105 {
106 	struct buf *bp, *rbp, *reqbp;
107 	struct bufobj *bo;
108 	struct thread *td;
109 	daddr_t blkno, origblkno;
110 	int maxra, racluster;
111 	int error, ncontig;
112 	int i;
113 
114 	error = 0;
115 	td = curthread;
116 	bo = &vp->v_bufobj;
117 	if (!unmapped_buf_allowed)
118 		gbflags &= ~GB_UNMAPPED;
119 
120 	/*
121 	 * Try to limit the amount of read-ahead by a few
122 	 * ad-hoc parameters.  This needs work!!!
123 	 */
124 	racluster = vp->v_mount->mnt_iosize_max / size;
125 	maxra = seqcount;
126 	maxra = min(read_max, maxra);
127 	maxra = min(nbuf/8, maxra);
128 	if (((u_quad_t)(lblkno + maxra + 1) * size) > filesize)
129 		maxra = (filesize / size) - lblkno;
130 
131 	/*
132 	 * get the requested block
133 	 */
134 	error = getblkx(vp, lblkno, size, 0, 0, gbflags, &bp);
135 	if (error != 0) {
136 		*bpp = NULL;
137 		return (error);
138 	}
139 	gbflags &= ~GB_NOSPARSE;
140 	origblkno = lblkno;
141 	*bpp = reqbp = bp;
142 
143 	/*
144 	 * if it is in the cache, then check to see if the reads have been
145 	 * sequential.  If they have, then try some read-ahead, otherwise
146 	 * back-off on prospective read-aheads.
147 	 */
148 	if (bp->b_flags & B_CACHE) {
149 		if (!seqcount) {
150 			return 0;
151 		} else if ((bp->b_flags & B_RAM) == 0) {
152 			return 0;
153 		} else {
154 			bp->b_flags &= ~B_RAM;
155 			BO_RLOCK(bo);
156 			for (i = 1; i < maxra; i++) {
157 				/*
158 				 * Stop if the buffer does not exist or it
159 				 * is invalid (about to go away?)
160 				 */
161 				rbp = gbincore(&vp->v_bufobj, lblkno+i);
162 				if (rbp == NULL || (rbp->b_flags & B_INVAL))
163 					break;
164 
165 				/*
166 				 * Set another read-ahead mark so we know
167 				 * to check again. (If we can lock the
168 				 * buffer without waiting)
169 				 */
170 				if ((((i % racluster) == (racluster - 1)) ||
171 				    (i == (maxra - 1)))
172 				    && (0 == BUF_LOCK(rbp,
173 					LK_EXCLUSIVE | LK_NOWAIT, NULL))) {
174 					rbp->b_flags |= B_RAM;
175 					BUF_UNLOCK(rbp);
176 				}
177 			}
178 			BO_RUNLOCK(bo);
179 			if (i >= maxra) {
180 				return 0;
181 			}
182 			lblkno += i;
183 		}
184 		reqbp = bp = NULL;
185 	/*
186 	 * If it isn't in the cache, then get a chunk from
187 	 * disk if sequential, otherwise just get the block.
188 	 */
189 	} else {
190 		off_t firstread = bp->b_offset;
191 		int nblks;
192 		long minread;
193 
194 		KASSERT(bp->b_offset != NOOFFSET,
195 		    ("cluster_read: no buffer offset"));
196 
197 		ncontig = 0;
198 
199 		/*
200 		 * Adjust totread if needed
201 		 */
202 		minread = read_min * size;
203 		if (minread > totread)
204 			totread = minread;
205 
206 		/*
207 		 * Compute the total number of blocks that we should read
208 		 * synchronously.
209 		 */
210 		if (firstread + totread > filesize)
211 			totread = filesize - firstread;
212 		nblks = howmany(totread, size);
213 		if (nblks > racluster)
214 			nblks = racluster;
215 
216 		/*
217 		 * Now compute the number of contiguous blocks.
218 		 */
219 		if (nblks > 1) {
220 	    		error = VOP_BMAP(vp, lblkno, NULL,
221 				&blkno, &ncontig, NULL);
222 			/*
223 			 * If this failed to map just do the original block.
224 			 */
225 			if (error || blkno == -1)
226 				ncontig = 0;
227 		}
228 
229 		/*
230 		 * If we have contiguous data available do a cluster
231 		 * otherwise just read the requested block.
232 		 */
233 		if (ncontig) {
234 			/* Account for our first block. */
235 			ncontig = min(ncontig + 1, nblks);
236 			if (ncontig < nblks)
237 				nblks = ncontig;
238 			bp = cluster_rbuild(vp, filesize, lblkno,
239 			    blkno, size, nblks, gbflags, bp);
240 			lblkno += (bp->b_bufsize / size);
241 		} else {
242 			bp->b_flags |= B_RAM;
243 			bp->b_iocmd = BIO_READ;
244 			lblkno += 1;
245 		}
246 	}
247 
248 	/*
249 	 * handle the synchronous read so that it is available ASAP.
250 	 */
251 	if (bp) {
252 		if ((bp->b_flags & B_CLUSTER) == 0) {
253 			vfs_busy_pages(bp, 0);
254 		}
255 		bp->b_flags &= ~B_INVAL;
256 		bp->b_ioflags &= ~BIO_ERROR;
257 		if ((bp->b_flags & B_ASYNC) || bp->b_iodone != NULL)
258 			BUF_KERNPROC(bp);
259 		bp->b_iooffset = dbtob(bp->b_blkno);
260 		bstrategy(bp);
261 #ifdef RACCT
262 		if (racct_enable) {
263 			PROC_LOCK(td->td_proc);
264 			racct_add_buf(td->td_proc, bp, 0);
265 			PROC_UNLOCK(td->td_proc);
266 		}
267 #endif /* RACCT */
268 		td->td_ru.ru_inblock++;
269 	}
270 
271 	/*
272 	 * If we have been doing sequential I/O, then do some read-ahead.
273 	 */
274 	while (lblkno < (origblkno + maxra)) {
275 		error = VOP_BMAP(vp, lblkno, NULL, &blkno, &ncontig, NULL);
276 		if (error)
277 			break;
278 
279 		if (blkno == -1)
280 			break;
281 
282 		/*
283 		 * We could throttle ncontig here by maxra but we might as
284 		 * well read the data if it is contiguous.  We're throttled
285 		 * by racluster anyway.
286 		 */
287 		if (ncontig) {
288 			ncontig = min(ncontig + 1, racluster);
289 			rbp = cluster_rbuild(vp, filesize, lblkno, blkno,
290 			    size, ncontig, gbflags, NULL);
291 			lblkno += (rbp->b_bufsize / size);
292 			if (rbp->b_flags & B_DELWRI) {
293 				bqrelse(rbp);
294 				continue;
295 			}
296 		} else {
297 			rbp = getblk(vp, lblkno, size, 0, 0, gbflags);
298 			lblkno += 1;
299 			if (rbp->b_flags & B_DELWRI) {
300 				bqrelse(rbp);
301 				continue;
302 			}
303 			rbp->b_flags |= B_ASYNC | B_RAM;
304 			rbp->b_iocmd = BIO_READ;
305 			rbp->b_blkno = blkno;
306 		}
307 		if (rbp->b_flags & B_CACHE) {
308 			rbp->b_flags &= ~B_ASYNC;
309 			bqrelse(rbp);
310 			continue;
311 		}
312 		if ((rbp->b_flags & B_CLUSTER) == 0) {
313 			vfs_busy_pages(rbp, 0);
314 		}
315 		rbp->b_flags &= ~B_INVAL;
316 		rbp->b_ioflags &= ~BIO_ERROR;
317 		if ((rbp->b_flags & B_ASYNC) || rbp->b_iodone != NULL)
318 			BUF_KERNPROC(rbp);
319 		rbp->b_iooffset = dbtob(rbp->b_blkno);
320 		bstrategy(rbp);
321 #ifdef RACCT
322 		if (racct_enable) {
323 			PROC_LOCK(td->td_proc);
324 			racct_add_buf(td->td_proc, rbp, 0);
325 			PROC_UNLOCK(td->td_proc);
326 		}
327 #endif /* RACCT */
328 		td->td_ru.ru_inblock++;
329 	}
330 
331 	if (reqbp) {
332 		/*
333 		 * Like bread, always brelse() the buffer when
334 		 * returning an error.
335 		 */
336 		error = bufwait(reqbp);
337 		if (error != 0) {
338 			brelse(reqbp);
339 			*bpp = NULL;
340 		}
341 	}
342 	return (error);
343 }
344 
345 /*
346  * If blocks are contiguous on disk, use this to provide clustered
347  * read ahead.  We will read as many blocks as possible sequentially
348  * and then parcel them up into logical blocks in the buffer hash table.
349  */
350 static struct buf *
351 cluster_rbuild(struct vnode *vp, u_quad_t filesize, daddr_t lbn,
352     daddr_t blkno, long size, int run, int gbflags, struct buf *fbp)
353 {
354 	struct buf *bp, *tbp;
355 	daddr_t bn;
356 	off_t off;
357 	long tinc, tsize;
358 	int i, inc, j, k, toff;
359 
360 	KASSERT(size == vp->v_mount->mnt_stat.f_iosize,
361 	    ("cluster_rbuild: size %ld != f_iosize %jd\n",
362 	    size, (intmax_t)vp->v_mount->mnt_stat.f_iosize));
363 
364 	/*
365 	 * avoid a division
366 	 */
367 	while ((u_quad_t) size * (lbn + run) > filesize) {
368 		--run;
369 	}
370 
371 	if (fbp) {
372 		tbp = fbp;
373 		tbp->b_iocmd = BIO_READ;
374 	} else {
375 		tbp = getblk(vp, lbn, size, 0, 0, gbflags);
376 		if (tbp->b_flags & B_CACHE)
377 			return tbp;
378 		tbp->b_flags |= B_ASYNC | B_RAM;
379 		tbp->b_iocmd = BIO_READ;
380 	}
381 	tbp->b_blkno = blkno;
382 	if( (tbp->b_flags & B_MALLOC) ||
383 		((tbp->b_flags & B_VMIO) == 0) || (run <= 1) )
384 		return tbp;
385 
386 	bp = uma_zalloc(cluster_pbuf_zone, M_NOWAIT);
387 	if (bp == NULL)
388 		return tbp;
389 
390 	/*
391 	 * We are synthesizing a buffer out of vm_page_t's, but
392 	 * if the block size is not page aligned then the starting
393 	 * address may not be either.  Inherit the b_data offset
394 	 * from the original buffer.
395 	 */
396 	bp->b_flags = B_ASYNC | B_CLUSTER | B_VMIO;
397 	if ((gbflags & GB_UNMAPPED) != 0) {
398 		bp->b_data = unmapped_buf;
399 	} else {
400 		bp->b_data = (char *)((vm_offset_t)bp->b_data |
401 		    ((vm_offset_t)tbp->b_data & PAGE_MASK));
402 	}
403 	bp->b_iocmd = BIO_READ;
404 	bp->b_iodone = cluster_callback;
405 	bp->b_blkno = blkno;
406 	bp->b_lblkno = lbn;
407 	bp->b_offset = tbp->b_offset;
408 	KASSERT(bp->b_offset != NOOFFSET, ("cluster_rbuild: no buffer offset"));
409 	pbgetvp(vp, bp);
410 
411 	TAILQ_INIT(&bp->b_cluster.cluster_head);
412 
413 	bp->b_bcount = 0;
414 	bp->b_bufsize = 0;
415 	bp->b_npages = 0;
416 
417 	inc = btodb(size);
418 	for (bn = blkno, i = 0; i < run; ++i, bn += inc) {
419 		if (i == 0) {
420 			VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
421 			vfs_drain_busy_pages(tbp);
422 			vm_object_pip_add(tbp->b_bufobj->bo_object,
423 			    tbp->b_npages);
424 			for (k = 0; k < tbp->b_npages; k++)
425 				vm_page_sbusy(tbp->b_pages[k]);
426 			VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
427 		} else {
428 			if ((bp->b_npages * PAGE_SIZE) +
429 			    round_page(size) > vp->v_mount->mnt_iosize_max) {
430 				break;
431 			}
432 
433 			tbp = getblk(vp, lbn + i, size, 0, 0, GB_LOCK_NOWAIT |
434 			    (gbflags & GB_UNMAPPED));
435 
436 			/* Don't wait around for locked bufs. */
437 			if (tbp == NULL)
438 				break;
439 
440 			/*
441 			 * Stop scanning if the buffer is fully valid
442 			 * (marked B_CACHE), or locked (may be doing a
443 			 * background write), or if the buffer is not
444 			 * VMIO backed.  The clustering code can only deal
445 			 * with VMIO-backed buffers.  The bo lock is not
446 			 * required for the BKGRDINPROG check since it
447 			 * can not be set without the buf lock.
448 			 */
449 			if ((tbp->b_vflags & BV_BKGRDINPROG) ||
450 			    (tbp->b_flags & B_CACHE) ||
451 			    (tbp->b_flags & B_VMIO) == 0) {
452 				bqrelse(tbp);
453 				break;
454 			}
455 
456 			/*
457 			 * The buffer must be completely invalid in order to
458 			 * take part in the cluster.  If it is partially valid
459 			 * then we stop.
460 			 */
461 			off = tbp->b_offset;
462 			tsize = size;
463 			VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
464 			for (j = 0; tsize > 0; j++) {
465 				toff = off & PAGE_MASK;
466 				tinc = tsize;
467 				if (toff + tinc > PAGE_SIZE)
468 					tinc = PAGE_SIZE - toff;
469 				VM_OBJECT_ASSERT_WLOCKED(tbp->b_pages[j]->object);
470 				if ((tbp->b_pages[j]->valid &
471 				    vm_page_bits(toff, tinc)) != 0)
472 					break;
473 				if (vm_page_xbusied(tbp->b_pages[j]))
474 					break;
475 				vm_object_pip_add(tbp->b_bufobj->bo_object, 1);
476 				vm_page_sbusy(tbp->b_pages[j]);
477 				off += tinc;
478 				tsize -= tinc;
479 			}
480 			if (tsize > 0) {
481 clean_sbusy:
482 				vm_object_pip_wakeupn(tbp->b_bufobj->bo_object,
483 				    j);
484 				for (k = 0; k < j; k++)
485 					vm_page_sunbusy(tbp->b_pages[k]);
486 				VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
487 				bqrelse(tbp);
488 				break;
489 			}
490 			VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
491 
492 			/*
493 			 * Set a read-ahead mark as appropriate
494 			 */
495 			if ((fbp && (i == 1)) || (i == (run - 1)))
496 				tbp->b_flags |= B_RAM;
497 
498 			/*
499 			 * Set the buffer up for an async read (XXX should
500 			 * we do this only if we do not wind up brelse()ing?).
501 			 * Set the block number if it isn't set, otherwise
502 			 * if it is make sure it matches the block number we
503 			 * expect.
504 			 */
505 			tbp->b_flags |= B_ASYNC;
506 			tbp->b_iocmd = BIO_READ;
507 			if (tbp->b_blkno == tbp->b_lblkno) {
508 				tbp->b_blkno = bn;
509 			} else if (tbp->b_blkno != bn) {
510 				VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
511 				goto clean_sbusy;
512 			}
513 		}
514 		/*
515 		 * XXX fbp from caller may not be B_ASYNC, but we are going
516 		 * to biodone() it in cluster_callback() anyway
517 		 */
518 		BUF_KERNPROC(tbp);
519 		TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
520 			tbp, b_cluster.cluster_entry);
521 		VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
522 		for (j = 0; j < tbp->b_npages; j += 1) {
523 			vm_page_t m;
524 			m = tbp->b_pages[j];
525 			if ((bp->b_npages == 0) ||
526 			    (bp->b_pages[bp->b_npages-1] != m)) {
527 				bp->b_pages[bp->b_npages] = m;
528 				bp->b_npages++;
529 			}
530 			if (m->valid == VM_PAGE_BITS_ALL)
531 				tbp->b_pages[j] = bogus_page;
532 		}
533 		VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
534 		/*
535 		 * Don't inherit tbp->b_bufsize as it may be larger due to
536 		 * a non-page-aligned size.  Instead just aggregate using
537 		 * 'size'.
538 		 */
539 		if (tbp->b_bcount != size)
540 			printf("warning: tbp->b_bcount wrong %ld vs %ld\n", tbp->b_bcount, size);
541 		if (tbp->b_bufsize != size)
542 			printf("warning: tbp->b_bufsize wrong %ld vs %ld\n", tbp->b_bufsize, size);
543 		bp->b_bcount += size;
544 		bp->b_bufsize += size;
545 	}
546 
547 	/*
548 	 * Fully valid pages in the cluster are already good and do not need
549 	 * to be re-read from disk.  Replace the page with bogus_page
550 	 */
551 	VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
552 	for (j = 0; j < bp->b_npages; j++) {
553 		VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[j]->object);
554 		if (bp->b_pages[j]->valid == VM_PAGE_BITS_ALL)
555 			bp->b_pages[j] = bogus_page;
556 	}
557 	VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
558 	if (bp->b_bufsize > bp->b_kvasize)
559 		panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
560 		    bp->b_bufsize, bp->b_kvasize);
561 
562 	if (buf_mapped(bp)) {
563 		pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
564 		    (vm_page_t *)bp->b_pages, bp->b_npages);
565 	}
566 	return (bp);
567 }
568 
569 /*
570  * Cleanup after a clustered read or write.
571  * This is complicated by the fact that any of the buffers might have
572  * extra memory (if there were no empty buffer headers at allocbuf time)
573  * that we will need to shift around.
574  */
575 static void
576 cluster_callback(struct buf *bp)
577 {
578 	struct buf *nbp, *tbp;
579 	int error = 0;
580 
581 	/*
582 	 * Must propagate errors to all the components.
583 	 */
584 	if (bp->b_ioflags & BIO_ERROR)
585 		error = bp->b_error;
586 
587 	if (buf_mapped(bp)) {
588 		pmap_qremove(trunc_page((vm_offset_t) bp->b_data),
589 		    bp->b_npages);
590 	}
591 	/*
592 	 * Move memory from the large cluster buffer into the component
593 	 * buffers and mark IO as done on these.
594 	 */
595 	for (tbp = TAILQ_FIRST(&bp->b_cluster.cluster_head);
596 		tbp; tbp = nbp) {
597 		nbp = TAILQ_NEXT(&tbp->b_cluster, cluster_entry);
598 		if (error) {
599 			tbp->b_ioflags |= BIO_ERROR;
600 			tbp->b_error = error;
601 		} else {
602 			tbp->b_dirtyoff = tbp->b_dirtyend = 0;
603 			tbp->b_flags &= ~B_INVAL;
604 			tbp->b_ioflags &= ~BIO_ERROR;
605 			/*
606 			 * XXX the bdwrite()/bqrelse() issued during
607 			 * cluster building clears B_RELBUF (see bqrelse()
608 			 * comment).  If direct I/O was specified, we have
609 			 * to restore it here to allow the buffer and VM
610 			 * to be freed.
611 			 */
612 			if (tbp->b_flags & B_DIRECT)
613 				tbp->b_flags |= B_RELBUF;
614 		}
615 		bufdone(tbp);
616 	}
617 	pbrelvp(bp);
618 	uma_zfree(cluster_pbuf_zone, bp);
619 }
620 
621 /*
622  *	cluster_wbuild_wb:
623  *
624  *	Implement modified write build for cluster.
625  *
626  *		write_behind = 0	write behind disabled
627  *		write_behind = 1	write behind normal (default)
628  *		write_behind = 2	write behind backed-off
629  */
630 
631 static __inline int
632 cluster_wbuild_wb(struct vnode *vp, long size, daddr_t start_lbn, int len,
633     int gbflags)
634 {
635 	int r = 0;
636 
637 	switch (write_behind) {
638 	case 2:
639 		if (start_lbn < len)
640 			break;
641 		start_lbn -= len;
642 		/* FALLTHROUGH */
643 	case 1:
644 		r = cluster_wbuild(vp, size, start_lbn, len, gbflags);
645 		/* FALLTHROUGH */
646 	default:
647 		/* FALLTHROUGH */
648 		break;
649 	}
650 	return(r);
651 }
652 
653 /*
654  * Do clustered write for FFS.
655  *
656  * Three cases:
657  *	1. Write is not sequential (write asynchronously)
658  *	Write is sequential:
659  *	2.	beginning of cluster - begin cluster
660  *	3.	middle of a cluster - add to cluster
661  *	4.	end of a cluster - asynchronously write cluster
662  */
663 void
664 cluster_write(struct vnode *vp, struct buf *bp, u_quad_t filesize, int seqcount,
665     int gbflags)
666 {
667 	daddr_t lbn;
668 	int maxclen, cursize;
669 	int lblocksize;
670 	int async;
671 
672 	if (!unmapped_buf_allowed)
673 		gbflags &= ~GB_UNMAPPED;
674 
675 	if (vp->v_type == VREG) {
676 		async = DOINGASYNC(vp);
677 		lblocksize = vp->v_mount->mnt_stat.f_iosize;
678 	} else {
679 		async = 0;
680 		lblocksize = bp->b_bufsize;
681 	}
682 	lbn = bp->b_lblkno;
683 	KASSERT(bp->b_offset != NOOFFSET, ("cluster_write: no buffer offset"));
684 
685 	/* Initialize vnode to beginning of file. */
686 	if (lbn == 0)
687 		vp->v_lasta = vp->v_clen = vp->v_cstart = vp->v_lastw = 0;
688 
689 	if (vp->v_clen == 0 || lbn != vp->v_lastw + 1 ||
690 	    (bp->b_blkno != vp->v_lasta + btodb(lblocksize))) {
691 		maxclen = vp->v_mount->mnt_iosize_max / lblocksize - 1;
692 		if (vp->v_clen != 0) {
693 			/*
694 			 * Next block is not sequential.
695 			 *
696 			 * If we are not writing at end of file, the process
697 			 * seeked to another point in the file since its last
698 			 * write, or we have reached our maximum cluster size,
699 			 * then push the previous cluster. Otherwise try
700 			 * reallocating to make it sequential.
701 			 *
702 			 * Change to algorithm: only push previous cluster if
703 			 * it was sequential from the point of view of the
704 			 * seqcount heuristic, otherwise leave the buffer
705 			 * intact so we can potentially optimize the I/O
706 			 * later on in the buf_daemon or update daemon
707 			 * flush.
708 			 */
709 			cursize = vp->v_lastw - vp->v_cstart + 1;
710 			if (((u_quad_t) bp->b_offset + lblocksize) != filesize ||
711 			    lbn != vp->v_lastw + 1 || vp->v_clen <= cursize) {
712 				if (!async && seqcount > 0) {
713 					cluster_wbuild_wb(vp, lblocksize,
714 					    vp->v_cstart, cursize, gbflags);
715 				}
716 			} else {
717 				struct buf **bpp, **endbp;
718 				struct cluster_save *buflist;
719 
720 				buflist = cluster_collectbufs(vp, bp, gbflags);
721 				endbp = &buflist->bs_children
722 				    [buflist->bs_nchildren - 1];
723 				if (VOP_REALLOCBLKS(vp, buflist)) {
724 					/*
725 					 * Failed, push the previous cluster
726 					 * if *really* writing sequentially
727 					 * in the logical file (seqcount > 1),
728 					 * otherwise delay it in the hopes that
729 					 * the low level disk driver can
730 					 * optimize the write ordering.
731 					 */
732 					for (bpp = buflist->bs_children;
733 					     bpp < endbp; bpp++)
734 						brelse(*bpp);
735 					free(buflist, M_SEGMENT);
736 					if (seqcount > 1) {
737 						cluster_wbuild_wb(vp,
738 						    lblocksize, vp->v_cstart,
739 						    cursize, gbflags);
740 					}
741 				} else {
742 					/*
743 					 * Succeeded, keep building cluster.
744 					 */
745 					for (bpp = buflist->bs_children;
746 					     bpp <= endbp; bpp++)
747 						bdwrite(*bpp);
748 					free(buflist, M_SEGMENT);
749 					vp->v_lastw = lbn;
750 					vp->v_lasta = bp->b_blkno;
751 					return;
752 				}
753 			}
754 		}
755 		/*
756 		 * Consider beginning a cluster. If at end of file, make
757 		 * cluster as large as possible, otherwise find size of
758 		 * existing cluster.
759 		 */
760 		if ((vp->v_type == VREG) &&
761 			((u_quad_t) bp->b_offset + lblocksize) != filesize &&
762 		    (bp->b_blkno == bp->b_lblkno) &&
763 		    (VOP_BMAP(vp, lbn, NULL, &bp->b_blkno, &maxclen, NULL) ||
764 		     bp->b_blkno == -1)) {
765 			bawrite(bp);
766 			vp->v_clen = 0;
767 			vp->v_lasta = bp->b_blkno;
768 			vp->v_cstart = lbn + 1;
769 			vp->v_lastw = lbn;
770 			return;
771 		}
772 		vp->v_clen = maxclen;
773 		if (!async && maxclen == 0) {	/* I/O not contiguous */
774 			vp->v_cstart = lbn + 1;
775 			bawrite(bp);
776 		} else {	/* Wait for rest of cluster */
777 			vp->v_cstart = lbn;
778 			bdwrite(bp);
779 		}
780 	} else if (lbn == vp->v_cstart + vp->v_clen) {
781 		/*
782 		 * At end of cluster, write it out if seqcount tells us we
783 		 * are operating sequentially, otherwise let the buf or
784 		 * update daemon handle it.
785 		 */
786 		bdwrite(bp);
787 		if (seqcount > 1) {
788 			cluster_wbuild_wb(vp, lblocksize, vp->v_cstart,
789 			    vp->v_clen + 1, gbflags);
790 		}
791 		vp->v_clen = 0;
792 		vp->v_cstart = lbn + 1;
793 	} else if (vm_page_count_severe()) {
794 		/*
795 		 * We are low on memory, get it going NOW
796 		 */
797 		bawrite(bp);
798 	} else {
799 		/*
800 		 * In the middle of a cluster, so just delay the I/O for now.
801 		 */
802 		bdwrite(bp);
803 	}
804 	vp->v_lastw = lbn;
805 	vp->v_lasta = bp->b_blkno;
806 }
807 
808 
809 /*
810  * This is an awful lot like cluster_rbuild...wish they could be combined.
811  * The last lbn argument is the current block on which I/O is being
812  * performed.  Check to see that it doesn't fall in the middle of
813  * the current block (if last_bp == NULL).
814  */
815 int
816 cluster_wbuild(struct vnode *vp, long size, daddr_t start_lbn, int len,
817     int gbflags)
818 {
819 	struct buf *bp, *tbp;
820 	struct bufobj *bo;
821 	int i, j;
822 	int totalwritten = 0;
823 	int dbsize = btodb(size);
824 
825 	if (!unmapped_buf_allowed)
826 		gbflags &= ~GB_UNMAPPED;
827 
828 	bo = &vp->v_bufobj;
829 	while (len > 0) {
830 		/*
831 		 * If the buffer is not delayed-write (i.e. dirty), or it
832 		 * is delayed-write but either locked or inval, it cannot
833 		 * partake in the clustered write.
834 		 */
835 		BO_LOCK(bo);
836 		if ((tbp = gbincore(&vp->v_bufobj, start_lbn)) == NULL ||
837 		    (tbp->b_vflags & BV_BKGRDINPROG)) {
838 			BO_UNLOCK(bo);
839 			++start_lbn;
840 			--len;
841 			continue;
842 		}
843 		if (BUF_LOCK(tbp,
844 		    LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK, BO_LOCKPTR(bo))) {
845 			++start_lbn;
846 			--len;
847 			continue;
848 		}
849 		if ((tbp->b_flags & (B_INVAL | B_DELWRI)) != B_DELWRI) {
850 			BUF_UNLOCK(tbp);
851 			++start_lbn;
852 			--len;
853 			continue;
854 		}
855 		bremfree(tbp);
856 		tbp->b_flags &= ~B_DONE;
857 
858 		/*
859 		 * Extra memory in the buffer, punt on this buffer.
860 		 * XXX we could handle this in most cases, but we would
861 		 * have to push the extra memory down to after our max
862 		 * possible cluster size and then potentially pull it back
863 		 * up if the cluster was terminated prematurely--too much
864 		 * hassle.
865 		 */
866 		if (((tbp->b_flags & (B_CLUSTEROK | B_MALLOC | B_VMIO)) !=
867 		     (B_CLUSTEROK | B_VMIO)) ||
868 		  (tbp->b_bcount != tbp->b_bufsize) ||
869 		  (tbp->b_bcount != size) ||
870 		  (len == 1) ||
871 		  ((bp = uma_zalloc(cluster_pbuf_zone,
872 		  (vp->v_vflag & VV_MD) != 0 ? M_NOWAIT : M_WAITOK)) == NULL)) {
873 			totalwritten += tbp->b_bufsize;
874 			bawrite(tbp);
875 			++start_lbn;
876 			--len;
877 			continue;
878 		}
879 
880 		/*
881 		 * We got a pbuf to make the cluster in.
882 		 * so initialise it.
883 		 */
884 		TAILQ_INIT(&bp->b_cluster.cluster_head);
885 		bp->b_bcount = 0;
886 		bp->b_bufsize = 0;
887 		bp->b_npages = 0;
888 		if (tbp->b_wcred != NOCRED)
889 			bp->b_wcred = crhold(tbp->b_wcred);
890 
891 		bp->b_blkno = tbp->b_blkno;
892 		bp->b_lblkno = tbp->b_lblkno;
893 		bp->b_offset = tbp->b_offset;
894 
895 		/*
896 		 * We are synthesizing a buffer out of vm_page_t's, but
897 		 * if the block size is not page aligned then the starting
898 		 * address may not be either.  Inherit the b_data offset
899 		 * from the original buffer.
900 		 */
901 		if ((gbflags & GB_UNMAPPED) == 0 ||
902 		    (tbp->b_flags & B_VMIO) == 0) {
903 			bp->b_data = (char *)((vm_offset_t)bp->b_data |
904 			    ((vm_offset_t)tbp->b_data & PAGE_MASK));
905 		} else {
906 			bp->b_data = unmapped_buf;
907 		}
908 		bp->b_flags |= B_CLUSTER | (tbp->b_flags & (B_VMIO |
909 		    B_NEEDCOMMIT));
910 		bp->b_iodone = cluster_callback;
911 		pbgetvp(vp, bp);
912 		/*
913 		 * From this location in the file, scan forward to see
914 		 * if there are buffers with adjacent data that need to
915 		 * be written as well.
916 		 */
917 		for (i = 0; i < len; ++i, ++start_lbn) {
918 			if (i != 0) { /* If not the first buffer */
919 				/*
920 				 * If the adjacent data is not even in core it
921 				 * can't need to be written.
922 				 */
923 				BO_LOCK(bo);
924 				if ((tbp = gbincore(bo, start_lbn)) == NULL ||
925 				    (tbp->b_vflags & BV_BKGRDINPROG)) {
926 					BO_UNLOCK(bo);
927 					break;
928 				}
929 
930 				/*
931 				 * If it IS in core, but has different
932 				 * characteristics, or is locked (which
933 				 * means it could be undergoing a background
934 				 * I/O or be in a weird state), then don't
935 				 * cluster with it.
936 				 */
937 				if (BUF_LOCK(tbp,
938 				    LK_EXCLUSIVE | LK_NOWAIT | LK_INTERLOCK,
939 				    BO_LOCKPTR(bo)))
940 					break;
941 
942 				if ((tbp->b_flags & (B_VMIO | B_CLUSTEROK |
943 				    B_INVAL | B_DELWRI | B_NEEDCOMMIT))
944 				    != (B_DELWRI | B_CLUSTEROK |
945 				    (bp->b_flags & (B_VMIO | B_NEEDCOMMIT))) ||
946 				    tbp->b_wcred != bp->b_wcred) {
947 					BUF_UNLOCK(tbp);
948 					break;
949 				}
950 
951 				/*
952 				 * Check that the combined cluster
953 				 * would make sense with regard to pages
954 				 * and would not be too large
955 				 */
956 				if ((tbp->b_bcount != size) ||
957 				  ((bp->b_blkno + (dbsize * i)) !=
958 				    tbp->b_blkno) ||
959 				  ((tbp->b_npages + bp->b_npages) >
960 				    (vp->v_mount->mnt_iosize_max / PAGE_SIZE))) {
961 					BUF_UNLOCK(tbp);
962 					break;
963 				}
964 
965 				/*
966 				 * Ok, it's passed all the tests,
967 				 * so remove it from the free list
968 				 * and mark it busy. We will use it.
969 				 */
970 				bremfree(tbp);
971 				tbp->b_flags &= ~B_DONE;
972 			} /* end of code for non-first buffers only */
973 			/*
974 			 * If the IO is via the VM then we do some
975 			 * special VM hackery (yuck).  Since the buffer's
976 			 * block size may not be page-aligned it is possible
977 			 * for a page to be shared between two buffers.  We
978 			 * have to get rid of the duplication when building
979 			 * the cluster.
980 			 */
981 			if (tbp->b_flags & B_VMIO) {
982 				vm_page_t m;
983 
984 				VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
985 				if (i == 0) {
986 					vfs_drain_busy_pages(tbp);
987 				} else { /* if not first buffer */
988 					for (j = 0; j < tbp->b_npages; j += 1) {
989 						m = tbp->b_pages[j];
990 						if (vm_page_xbusied(m)) {
991 							VM_OBJECT_WUNLOCK(
992 							    tbp->b_object);
993 							bqrelse(tbp);
994 							goto finishcluster;
995 						}
996 					}
997 				}
998 				for (j = 0; j < tbp->b_npages; j += 1) {
999 					m = tbp->b_pages[j];
1000 					vm_page_sbusy(m);
1001 					vm_object_pip_add(m->object, 1);
1002 					if ((bp->b_npages == 0) ||
1003 					  (bp->b_pages[bp->b_npages - 1] != m)) {
1004 						bp->b_pages[bp->b_npages] = m;
1005 						bp->b_npages++;
1006 					}
1007 				}
1008 				VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
1009 			}
1010 			bp->b_bcount += size;
1011 			bp->b_bufsize += size;
1012 			/*
1013 			 * If any of the clustered buffers have their
1014 			 * B_BARRIER flag set, transfer that request to
1015 			 * the cluster.
1016 			 */
1017 			bp->b_flags |= (tbp->b_flags & B_BARRIER);
1018 			tbp->b_flags &= ~(B_DONE | B_BARRIER);
1019 			tbp->b_flags |= B_ASYNC;
1020 			tbp->b_ioflags &= ~BIO_ERROR;
1021 			tbp->b_iocmd = BIO_WRITE;
1022 			bundirty(tbp);
1023 			reassignbuf(tbp);		/* put on clean list */
1024 			bufobj_wref(tbp->b_bufobj);
1025 			BUF_KERNPROC(tbp);
1026 			buf_track(tbp, __func__);
1027 			TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
1028 				tbp, b_cluster.cluster_entry);
1029 		}
1030 	finishcluster:
1031 		if (buf_mapped(bp)) {
1032 			pmap_qenter(trunc_page((vm_offset_t) bp->b_data),
1033 			    (vm_page_t *)bp->b_pages, bp->b_npages);
1034 		}
1035 		if (bp->b_bufsize > bp->b_kvasize)
1036 			panic(
1037 			    "cluster_wbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
1038 			    bp->b_bufsize, bp->b_kvasize);
1039 		totalwritten += bp->b_bufsize;
1040 		bp->b_dirtyoff = 0;
1041 		bp->b_dirtyend = bp->b_bufsize;
1042 		bawrite(bp);
1043 
1044 		len -= i;
1045 	}
1046 	return totalwritten;
1047 }
1048 
1049 /*
1050  * Collect together all the buffers in a cluster.
1051  * Plus add one additional buffer.
1052  */
1053 static struct cluster_save *
1054 cluster_collectbufs(struct vnode *vp, struct buf *last_bp, int gbflags)
1055 {
1056 	struct cluster_save *buflist;
1057 	struct buf *bp;
1058 	daddr_t lbn;
1059 	int i, len;
1060 
1061 	len = vp->v_lastw - vp->v_cstart + 1;
1062 	buflist = malloc(sizeof(struct buf *) * (len + 1) + sizeof(*buflist),
1063 	    M_SEGMENT, M_WAITOK);
1064 	buflist->bs_nchildren = 0;
1065 	buflist->bs_children = (struct buf **) (buflist + 1);
1066 	for (lbn = vp->v_cstart, i = 0; i < len; lbn++, i++) {
1067 		(void)bread_gb(vp, lbn, last_bp->b_bcount, NOCRED,
1068 		    gbflags, &bp);
1069 		buflist->bs_children[i] = bp;
1070 		if (bp->b_blkno == bp->b_lblkno)
1071 			VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno,
1072 				NULL, NULL);
1073 	}
1074 	buflist->bs_children[i] = bp = last_bp;
1075 	if (bp->b_blkno == bp->b_lblkno)
1076 		VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
1077 	buflist->bs_nchildren = i + 1;
1078 	return (buflist);
1079 }
1080