xref: /freebsd/sys/kern/vfs_bio.c (revision 656f49f8e2b0656824a5f10aeb760a00fdd3753f)
1 /*-
2  * Copyright (c) 2004 Poul-Henning Kamp
3  * Copyright (c) 1994,1997 John S. Dyson
4  * Copyright (c) 2013 The FreeBSD Foundation
5  * All rights reserved.
6  *
7  * Portions of this software were developed by Konstantin Belousov
8  * under sponsorship from the FreeBSD Foundation.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
20  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
21  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
22  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
23  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
24  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
25  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
26  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
28  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
29  * SUCH DAMAGE.
30  */
31 
32 /*
33  * this file contains a new buffer I/O scheme implementing a coherent
34  * VM object and buffer cache scheme.  Pains have been taken to make
35  * sure that the performance degradation associated with schemes such
36  * as this is not realized.
37  *
38  * Author:  John S. Dyson
39  * Significant help during the development and debugging phases
40  * had been provided by David Greenman, also of the FreeBSD core team.
41  *
42  * see man buf(9) for more info.
43  */
44 
45 #include <sys/cdefs.h>
46 __FBSDID("$FreeBSD$");
47 
48 #include <sys/param.h>
49 #include <sys/systm.h>
50 #include <sys/bio.h>
51 #include <sys/conf.h>
52 #include <sys/buf.h>
53 #include <sys/devicestat.h>
54 #include <sys/eventhandler.h>
55 #include <sys/fail.h>
56 #include <sys/limits.h>
57 #include <sys/lock.h>
58 #include <sys/malloc.h>
59 #include <sys/mount.h>
60 #include <sys/mutex.h>
61 #include <sys/kernel.h>
62 #include <sys/kthread.h>
63 #include <sys/proc.h>
64 #include <sys/resourcevar.h>
65 #include <sys/rwlock.h>
66 #include <sys/sysctl.h>
67 #include <sys/sysproto.h>
68 #include <sys/vmem.h>
69 #include <sys/vmmeter.h>
70 #include <sys/vnode.h>
71 #include <sys/watchdog.h>
72 #include <geom/geom.h>
73 #include <vm/vm.h>
74 #include <vm/vm_param.h>
75 #include <vm/vm_kern.h>
76 #include <vm/vm_pageout.h>
77 #include <vm/vm_page.h>
78 #include <vm/vm_object.h>
79 #include <vm/vm_extern.h>
80 #include <vm/vm_map.h>
81 #include <vm/swap_pager.h>
82 #include "opt_compat.h"
83 #include "opt_swap.h"
84 
85 static MALLOC_DEFINE(M_BIOBUF, "biobuf", "BIO buffer");
86 
87 struct	bio_ops bioops;		/* I/O operation notification */
88 
89 struct	buf_ops buf_ops_bio = {
90 	.bop_name	=	"buf_ops_bio",
91 	.bop_write	=	bufwrite,
92 	.bop_strategy	=	bufstrategy,
93 	.bop_sync	=	bufsync,
94 	.bop_bdflush	=	bufbdflush,
95 };
96 
97 static struct buf *buf;		/* buffer header pool */
98 extern struct buf *swbuf;	/* Swap buffer header pool. */
99 caddr_t unmapped_buf;
100 
101 /* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
102 struct proc *bufdaemonproc;
103 
104 static int inmem(struct vnode *vp, daddr_t blkno);
105 static void vm_hold_free_pages(struct buf *bp, int newbsize);
106 static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
107 		vm_offset_t to);
108 static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
109 static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
110 		vm_page_t m);
111 static void vfs_clean_pages_dirty_buf(struct buf *bp);
112 static void vfs_setdirty_locked_object(struct buf *bp);
113 static void vfs_vmio_invalidate(struct buf *bp);
114 static void vfs_vmio_truncate(struct buf *bp, int npages);
115 static void vfs_vmio_extend(struct buf *bp, int npages, int size);
116 static int vfs_bio_clcheck(struct vnode *vp, int size,
117 		daddr_t lblkno, daddr_t blkno);
118 static int buf_flush(struct vnode *vp, int);
119 static int flushbufqueues(struct vnode *, int, int);
120 static void buf_daemon(void);
121 static void bremfreel(struct buf *bp);
122 static __inline void bd_wakeup(void);
123 static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
124 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
125     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
126 static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
127 #endif
128 
129 int vmiodirenable = TRUE;
130 SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW, &vmiodirenable, 0,
131     "Use the VM system for directory writes");
132 long runningbufspace;
133 SYSCTL_LONG(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, &runningbufspace, 0,
134     "Amount of presently outstanding async buffer io");
135 static long bufspace;
136 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
137     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
138 SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RD,
139     &bufspace, 0, sysctl_bufspace, "L", "Virtual memory used for buffers");
140 #else
141 SYSCTL_LONG(_vfs, OID_AUTO, bufspace, CTLFLAG_RD, &bufspace, 0,
142     "Physical memory used for buffers");
143 #endif
144 static long bufkvaspace;
145 SYSCTL_LONG(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace, 0,
146     "Kernel virtual memory used for buffers");
147 static long maxbufspace;
148 SYSCTL_LONG(_vfs, OID_AUTO, maxbufspace, CTLFLAG_RD, &maxbufspace, 0,
149     "Maximum allowed value of bufspace (including buf_daemon)");
150 static long bufmallocspace;
151 SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
152     "Amount of malloced memory for buffers");
153 static long maxbufmallocspace;
154 SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace, 0,
155     "Maximum amount of malloced memory for buffers");
156 static long lobufspace;
157 SYSCTL_LONG(_vfs, OID_AUTO, lobufspace, CTLFLAG_RD, &lobufspace, 0,
158     "Minimum amount of buffers we want to have");
159 long hibufspace;
160 SYSCTL_LONG(_vfs, OID_AUTO, hibufspace, CTLFLAG_RD, &hibufspace, 0,
161     "Maximum allowed value of bufspace (excluding buf_daemon)");
162 static int bufreusecnt;
163 SYSCTL_INT(_vfs, OID_AUTO, bufreusecnt, CTLFLAG_RW, &bufreusecnt, 0,
164     "Number of times we have reused a buffer");
165 static int buffreekvacnt;
166 SYSCTL_INT(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt, 0,
167     "Number of times we have freed the KVA space from some buffer");
168 static int bufdefragcnt;
169 SYSCTL_INT(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW, &bufdefragcnt, 0,
170     "Number of times we have had to repeat buffer allocation to defragment");
171 static long lorunningspace;
172 SYSCTL_PROC(_vfs, OID_AUTO, lorunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
173     CTLFLAG_RW, &lorunningspace, 0, sysctl_runningspace, "L",
174     "Minimum preferred space used for in-progress I/O");
175 static long hirunningspace;
176 SYSCTL_PROC(_vfs, OID_AUTO, hirunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
177     CTLFLAG_RW, &hirunningspace, 0, sysctl_runningspace, "L",
178     "Maximum amount of space to use for in-progress I/O");
179 int dirtybufferflushes;
180 SYSCTL_INT(_vfs, OID_AUTO, dirtybufferflushes, CTLFLAG_RW, &dirtybufferflushes,
181     0, "Number of bdwrite to bawrite conversions to limit dirty buffers");
182 int bdwriteskip;
183 SYSCTL_INT(_vfs, OID_AUTO, bdwriteskip, CTLFLAG_RW, &bdwriteskip,
184     0, "Number of buffers supplied to bdwrite with snapshot deadlock risk");
185 int altbufferflushes;
186 SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFLAG_RW, &altbufferflushes,
187     0, "Number of fsync flushes to limit dirty buffers");
188 static int recursiveflushes;
189 SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW, &recursiveflushes,
190     0, "Number of flushes skipped due to being recursive");
191 static int numdirtybuffers;
192 SYSCTL_INT(_vfs, OID_AUTO, numdirtybuffers, CTLFLAG_RD, &numdirtybuffers, 0,
193     "Number of buffers that are dirty (has unwritten changes) at the moment");
194 static int lodirtybuffers;
195 SYSCTL_INT(_vfs, OID_AUTO, lodirtybuffers, CTLFLAG_RW, &lodirtybuffers, 0,
196     "How many buffers we want to have free before bufdaemon can sleep");
197 static int hidirtybuffers;
198 SYSCTL_INT(_vfs, OID_AUTO, hidirtybuffers, CTLFLAG_RW, &hidirtybuffers, 0,
199     "When the number of dirty buffers is considered severe");
200 int dirtybufthresh;
201 SYSCTL_INT(_vfs, OID_AUTO, dirtybufthresh, CTLFLAG_RW, &dirtybufthresh,
202     0, "Number of bdwrite to bawrite conversions to clear dirty buffers");
203 static int numfreebuffers;
204 SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
205     "Number of free buffers");
206 static int lofreebuffers;
207 SYSCTL_INT(_vfs, OID_AUTO, lofreebuffers, CTLFLAG_RW, &lofreebuffers, 0,
208    "XXX Unused");
209 static int hifreebuffers;
210 SYSCTL_INT(_vfs, OID_AUTO, hifreebuffers, CTLFLAG_RW, &hifreebuffers, 0,
211    "XXX Complicatedly unused");
212 static int getnewbufcalls;
213 SYSCTL_INT(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RW, &getnewbufcalls, 0,
214    "Number of calls to getnewbuf");
215 static int getnewbufrestarts;
216 SYSCTL_INT(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RW, &getnewbufrestarts, 0,
217     "Number of times getnewbuf has had to restart a buffer aquisition");
218 static int mappingrestarts;
219 SYSCTL_INT(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RW, &mappingrestarts, 0,
220     "Number of times getblk has had to restart a buffer mapping for "
221     "unmapped buffer");
222 static int flushbufqtarget = 100;
223 SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
224     "Amount of work to do in flushbufqueues when helping bufdaemon");
225 static long notbufdflushes;
226 SYSCTL_LONG(_vfs, OID_AUTO, notbufdflushes, CTLFLAG_RD, &notbufdflushes, 0,
227     "Number of dirty buffer flushes done by the bufdaemon helpers");
228 static long barrierwrites;
229 SYSCTL_LONG(_vfs, OID_AUTO, barrierwrites, CTLFLAG_RW, &barrierwrites, 0,
230     "Number of barrier writes");
231 SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed, CTLFLAG_RD,
232     &unmapped_buf_allowed, 0,
233     "Permit the use of the unmapped i/o");
234 
235 /*
236  * Lock for the non-dirty bufqueues
237  */
238 static struct mtx_padalign bqclean;
239 
240 /*
241  * Lock for the dirty queue.
242  */
243 static struct mtx_padalign bqdirty;
244 
245 /*
246  * This lock synchronizes access to bd_request.
247  */
248 static struct mtx_padalign bdlock;
249 
250 /*
251  * This lock protects the runningbufreq and synchronizes runningbufwakeup and
252  * waitrunningbufspace().
253  */
254 static struct mtx_padalign rbreqlock;
255 
256 /*
257  * Lock that protects needsbuffer and the sleeps/wakeups surrounding it.
258  */
259 static struct rwlock_padalign nblock;
260 
261 /*
262  * Lock that protects bdirtywait.
263  */
264 static struct mtx_padalign bdirtylock;
265 
266 /*
267  * Wakeup point for bufdaemon, as well as indicator of whether it is already
268  * active.  Set to 1 when the bufdaemon is already "on" the queue, 0 when it
269  * is idling.
270  */
271 static int bd_request;
272 
273 /*
274  * Request for the buf daemon to write more buffers than is indicated by
275  * lodirtybuf.  This may be necessary to push out excess dependencies or
276  * defragment the address space where a simple count of the number of dirty
277  * buffers is insufficient to characterize the demand for flushing them.
278  */
279 static int bd_speedupreq;
280 
281 /*
282  * bogus page -- for I/O to/from partially complete buffers
283  * this is a temporary solution to the problem, but it is not
284  * really that bad.  it would be better to split the buffer
285  * for input in the case of buffers partially already in memory,
286  * but the code is intricate enough already.
287  */
288 vm_page_t bogus_page;
289 
290 /*
291  * Synchronization (sleep/wakeup) variable for active buffer space requests.
292  * Set when wait starts, cleared prior to wakeup().
293  * Used in runningbufwakeup() and waitrunningbufspace().
294  */
295 static int runningbufreq;
296 
297 /*
298  * Synchronization (sleep/wakeup) variable for buffer requests.
299  * Can contain the VFS_BIO_NEED flags defined below; setting/clearing is done
300  * by and/or.
301  * Used in numdirtywakeup(), bufspacewakeup(), bufcountadd(), bwillwrite(),
302  * getnewbuf(), and getblk().
303  */
304 static volatile int needsbuffer;
305 
306 /*
307  * Synchronization for bwillwrite() waiters.
308  */
309 static int bdirtywait;
310 
311 /*
312  * Definitions for the buffer free lists.
313  */
314 #define BUFFER_QUEUES	4	/* number of free buffer queues */
315 
316 #define QUEUE_NONE	0	/* on no queue */
317 #define QUEUE_CLEAN	1	/* non-B_DELWRI buffers */
318 #define QUEUE_DIRTY	2	/* B_DELWRI buffers */
319 #define QUEUE_EMPTY	3	/* empty buffer headers */
320 #define QUEUE_SENTINEL	1024	/* not an queue index, but mark for sentinel */
321 
322 /* Queues for free buffers with various properties */
323 static TAILQ_HEAD(bqueues, buf) bufqueues[BUFFER_QUEUES] = { { 0 } };
324 #ifdef INVARIANTS
325 static int bq_len[BUFFER_QUEUES];
326 #endif
327 
328 /*
329  * Single global constant for BUF_WMESG, to avoid getting multiple references.
330  * buf_wmesg is referred from macros.
331  */
332 const char *buf_wmesg = BUF_WMESG;
333 
334 #define VFS_BIO_NEED_ANY	0x01	/* any freeable buffer */
335 #define VFS_BIO_NEED_FREE	0x04	/* wait for free bufs, hi hysteresis */
336 #define VFS_BIO_NEED_BUFSPACE	0x08	/* wait for buf space, lo hysteresis */
337 
338 static int
339 sysctl_runningspace(SYSCTL_HANDLER_ARGS)
340 {
341 	long value;
342 	int error;
343 
344 	value = *(long *)arg1;
345 	error = sysctl_handle_long(oidp, &value, 0, req);
346 	if (error != 0 || req->newptr == NULL)
347 		return (error);
348 	mtx_lock(&rbreqlock);
349 	if (arg1 == &hirunningspace) {
350 		if (value < lorunningspace)
351 			error = EINVAL;
352 		else
353 			hirunningspace = value;
354 	} else {
355 		KASSERT(arg1 == &lorunningspace,
356 		    ("%s: unknown arg1", __func__));
357 		if (value > hirunningspace)
358 			error = EINVAL;
359 		else
360 			lorunningspace = value;
361 	}
362 	mtx_unlock(&rbreqlock);
363 	return (error);
364 }
365 
366 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
367     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
368 static int
369 sysctl_bufspace(SYSCTL_HANDLER_ARGS)
370 {
371 	long lvalue;
372 	int ivalue;
373 
374 	if (sizeof(int) == sizeof(long) || req->oldlen >= sizeof(long))
375 		return (sysctl_handle_long(oidp, arg1, arg2, req));
376 	lvalue = *(long *)arg1;
377 	if (lvalue > INT_MAX)
378 		/* On overflow, still write out a long to trigger ENOMEM. */
379 		return (sysctl_handle_long(oidp, &lvalue, 0, req));
380 	ivalue = lvalue;
381 	return (sysctl_handle_int(oidp, &ivalue, 0, req));
382 }
383 #endif
384 
385 /*
386  *	bqlock:
387  *
388  *	Return the appropriate queue lock based on the index.
389  */
390 static inline struct mtx *
391 bqlock(int qindex)
392 {
393 
394 	if (qindex == QUEUE_DIRTY)
395 		return (struct mtx *)(&bqdirty);
396 	return (struct mtx *)(&bqclean);
397 }
398 
399 /*
400  *	bdirtywakeup:
401  *
402  *	Wakeup any bwillwrite() waiters.
403  */
404 static void
405 bdirtywakeup(void)
406 {
407 	mtx_lock(&bdirtylock);
408 	if (bdirtywait) {
409 		bdirtywait = 0;
410 		wakeup(&bdirtywait);
411 	}
412 	mtx_unlock(&bdirtylock);
413 }
414 
415 /*
416  *	bdirtysub:
417  *
418  *	Decrement the numdirtybuffers count by one and wakeup any
419  *	threads blocked in bwillwrite().
420  */
421 static void
422 bdirtysub(void)
423 {
424 
425 	if (atomic_fetchadd_int(&numdirtybuffers, -1) ==
426 	    (lodirtybuffers + hidirtybuffers) / 2)
427 		bdirtywakeup();
428 }
429 
430 /*
431  *	bdirtyadd:
432  *
433  *	Increment the numdirtybuffers count by one and wakeup the buf
434  *	daemon if needed.
435  */
436 static void
437 bdirtyadd(void)
438 {
439 
440 	/*
441 	 * Only do the wakeup once as we cross the boundary.  The
442 	 * buf daemon will keep running until the condition clears.
443 	 */
444 	if (atomic_fetchadd_int(&numdirtybuffers, 1) ==
445 	    (lodirtybuffers + hidirtybuffers) / 2)
446 		bd_wakeup();
447 }
448 
449 /*
450  *	bufspacewakeup:
451  *
452  *	Called when buffer space is potentially available for recovery.
453  *	getnewbuf() will block on this flag when it is unable to free
454  *	sufficient buffer space.  Buffer space becomes recoverable when
455  *	bp's get placed back in the queues.
456  */
457 static __inline void
458 bufspacewakeup(void)
459 {
460 	int need_wakeup, on;
461 
462 	/*
463 	 * If someone is waiting for bufspace, wake them up.  Even
464 	 * though we may not have freed the kva space yet, the waiting
465 	 * process will be able to now.
466 	 */
467 	rw_rlock(&nblock);
468 	for (;;) {
469 		need_wakeup = 0;
470 		on = needsbuffer;
471 		if ((on & VFS_BIO_NEED_BUFSPACE) == 0)
472 			break;
473 		need_wakeup = 1;
474 		if (atomic_cmpset_rel_int(&needsbuffer, on,
475 		    on & ~VFS_BIO_NEED_BUFSPACE))
476 			break;
477 	}
478 	if (need_wakeup)
479 		wakeup(__DEVOLATILE(void *, &needsbuffer));
480 	rw_runlock(&nblock);
481 }
482 
483 /*
484  *	bufspaceadjust:
485  *
486  *	Adjust the reported bufspace for a KVA managed buffer, possibly
487  * 	waking any waiters.
488  */
489 static void
490 bufspaceadjust(struct buf *bp, int bufsize)
491 {
492 	int diff;
493 
494 	KASSERT((bp->b_flags & B_MALLOC) == 0,
495 	    ("bufspaceadjust: malloc buf %p", bp));
496 	diff = bufsize - bp->b_bufsize;
497 	if (diff < 0) {
498 		atomic_subtract_long(&bufspace, -diff);
499 		bufspacewakeup();
500 	} else
501 		atomic_add_long(&bufspace, diff);
502 	bp->b_bufsize = bufsize;
503 }
504 
505 /*
506  *	bufmallocadjust:
507  *
508  *	Adjust the reported bufspace for a malloc managed buffer, possibly
509  *	waking any waiters.
510  */
511 static void
512 bufmallocadjust(struct buf *bp, int bufsize)
513 {
514 	int diff;
515 
516 	KASSERT((bp->b_flags & B_MALLOC) != 0,
517 	    ("bufmallocadjust: non-malloc buf %p", bp));
518 	diff = bufsize - bp->b_bufsize;
519 	if (diff < 0) {
520 		atomic_subtract_long(&bufmallocspace, -diff);
521 		bufspacewakeup();
522 	} else
523 		atomic_add_long(&bufmallocspace, diff);
524 	bp->b_bufsize = bufsize;
525 }
526 
527 /*
528  *	runningwakeup:
529  *
530  *	Wake up processes that are waiting on asynchronous writes to fall
531  *	below lorunningspace.
532  */
533 static void
534 runningwakeup(void)
535 {
536 
537 	mtx_lock(&rbreqlock);
538 	if (runningbufreq) {
539 		runningbufreq = 0;
540 		wakeup(&runningbufreq);
541 	}
542 	mtx_unlock(&rbreqlock);
543 }
544 
545 /*
546  *	runningbufwakeup:
547  *
548  *	Decrement the outstanding write count according.
549  */
550 void
551 runningbufwakeup(struct buf *bp)
552 {
553 	long space, bspace;
554 
555 	bspace = bp->b_runningbufspace;
556 	if (bspace == 0)
557 		return;
558 	space = atomic_fetchadd_long(&runningbufspace, -bspace);
559 	KASSERT(space >= bspace, ("runningbufspace underflow %ld %ld",
560 	    space, bspace));
561 	bp->b_runningbufspace = 0;
562 	/*
563 	 * Only acquire the lock and wakeup on the transition from exceeding
564 	 * the threshold to falling below it.
565 	 */
566 	if (space < lorunningspace)
567 		return;
568 	if (space - bspace > lorunningspace)
569 		return;
570 	runningwakeup();
571 }
572 
573 /*
574  *	bufcountadd:
575  *
576  *	Called when a buffer has been added to one of the free queues to
577  *	account for the buffer and to wakeup anyone waiting for free buffers.
578  *	This typically occurs when large amounts of metadata are being handled
579  *	by the buffer cache ( else buffer space runs out first, usually ).
580  */
581 static __inline void
582 bufcountadd(struct buf *bp)
583 {
584 	int mask, need_wakeup, old, on;
585 
586 	KASSERT((bp->b_flags & B_INFREECNT) == 0,
587 	    ("buf %p already counted as free", bp));
588 	bp->b_flags |= B_INFREECNT;
589 	old = atomic_fetchadd_int(&numfreebuffers, 1);
590 	KASSERT(old >= 0 && old < nbuf,
591 	    ("numfreebuffers climbed to %d", old + 1));
592 	mask = VFS_BIO_NEED_ANY;
593 	if (numfreebuffers >= hifreebuffers)
594 		mask |= VFS_BIO_NEED_FREE;
595 	rw_rlock(&nblock);
596 	for (;;) {
597 		need_wakeup = 0;
598 		on = needsbuffer;
599 		if (on == 0)
600 			break;
601 		need_wakeup = 1;
602 		if (atomic_cmpset_rel_int(&needsbuffer, on, on & ~mask))
603 			break;
604 	}
605 	if (need_wakeup)
606 		wakeup(__DEVOLATILE(void *, &needsbuffer));
607 	rw_runlock(&nblock);
608 }
609 
610 /*
611  *	bufcountsub:
612  *
613  *	Decrement the numfreebuffers count as needed.
614  */
615 static void
616 bufcountsub(struct buf *bp)
617 {
618 	int old;
619 
620 	/*
621 	 * Fixup numfreebuffers count.  If the buffer is invalid or not
622 	 * delayed-write, the buffer was free and we must decrement
623 	 * numfreebuffers.
624 	 */
625 	if ((bp->b_flags & B_INVAL) || (bp->b_flags & B_DELWRI) == 0) {
626 		KASSERT((bp->b_flags & B_INFREECNT) != 0,
627 		    ("buf %p not counted in numfreebuffers", bp));
628 		bp->b_flags &= ~B_INFREECNT;
629 		old = atomic_fetchadd_int(&numfreebuffers, -1);
630 		KASSERT(old > 0, ("numfreebuffers dropped to %d", old - 1));
631 	}
632 }
633 
634 /*
635  *	waitrunningbufspace()
636  *
637  *	runningbufspace is a measure of the amount of I/O currently
638  *	running.  This routine is used in async-write situations to
639  *	prevent creating huge backups of pending writes to a device.
640  *	Only asynchronous writes are governed by this function.
641  *
642  *	This does NOT turn an async write into a sync write.  It waits
643  *	for earlier writes to complete and generally returns before the
644  *	caller's write has reached the device.
645  */
646 void
647 waitrunningbufspace(void)
648 {
649 
650 	mtx_lock(&rbreqlock);
651 	while (runningbufspace > hirunningspace) {
652 		runningbufreq = 1;
653 		msleep(&runningbufreq, &rbreqlock, PVM, "wdrain", 0);
654 	}
655 	mtx_unlock(&rbreqlock);
656 }
657 
658 
659 /*
660  *	vfs_buf_test_cache:
661  *
662  *	Called when a buffer is extended.  This function clears the B_CACHE
663  *	bit if the newly extended portion of the buffer does not contain
664  *	valid data.
665  */
666 static __inline void
667 vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
668     vm_offset_t size, vm_page_t m)
669 {
670 
671 	VM_OBJECT_ASSERT_LOCKED(m->object);
672 	if (bp->b_flags & B_CACHE) {
673 		int base = (foff + off) & PAGE_MASK;
674 		if (vm_page_is_valid(m, base, size) == 0)
675 			bp->b_flags &= ~B_CACHE;
676 	}
677 }
678 
679 /* Wake up the buffer daemon if necessary */
680 static __inline void
681 bd_wakeup(void)
682 {
683 
684 	mtx_lock(&bdlock);
685 	if (bd_request == 0) {
686 		bd_request = 1;
687 		wakeup(&bd_request);
688 	}
689 	mtx_unlock(&bdlock);
690 }
691 
692 /*
693  * bd_speedup - speedup the buffer cache flushing code
694  */
695 void
696 bd_speedup(void)
697 {
698 	int needwake;
699 
700 	mtx_lock(&bdlock);
701 	needwake = 0;
702 	if (bd_speedupreq == 0 || bd_request == 0)
703 		needwake = 1;
704 	bd_speedupreq = 1;
705 	bd_request = 1;
706 	if (needwake)
707 		wakeup(&bd_request);
708 	mtx_unlock(&bdlock);
709 }
710 
711 #ifndef NSWBUF_MIN
712 #define	NSWBUF_MIN	16
713 #endif
714 
715 #ifdef __i386__
716 #define	TRANSIENT_DENOM	5
717 #else
718 #define	TRANSIENT_DENOM 10
719 #endif
720 
721 /*
722  * Calculating buffer cache scaling values and reserve space for buffer
723  * headers.  This is called during low level kernel initialization and
724  * may be called more then once.  We CANNOT write to the memory area
725  * being reserved at this time.
726  */
727 caddr_t
728 kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est)
729 {
730 	int tuned_nbuf;
731 	long maxbuf, maxbuf_sz, buf_sz,	biotmap_sz;
732 
733 	/*
734 	 * physmem_est is in pages.  Convert it to kilobytes (assumes
735 	 * PAGE_SIZE is >= 1K)
736 	 */
737 	physmem_est = physmem_est * (PAGE_SIZE / 1024);
738 
739 	/*
740 	 * The nominal buffer size (and minimum KVA allocation) is BKVASIZE.
741 	 * For the first 64MB of ram nominally allocate sufficient buffers to
742 	 * cover 1/4 of our ram.  Beyond the first 64MB allocate additional
743 	 * buffers to cover 1/10 of our ram over 64MB.  When auto-sizing
744 	 * the buffer cache we limit the eventual kva reservation to
745 	 * maxbcache bytes.
746 	 *
747 	 * factor represents the 1/4 x ram conversion.
748 	 */
749 	if (nbuf == 0) {
750 		int factor = 4 * BKVASIZE / 1024;
751 
752 		nbuf = 50;
753 		if (physmem_est > 4096)
754 			nbuf += min((physmem_est - 4096) / factor,
755 			    65536 / factor);
756 		if (physmem_est > 65536)
757 			nbuf += min((physmem_est - 65536) * 2 / (factor * 5),
758 			    32 * 1024 * 1024 / (factor * 5));
759 
760 		if (maxbcache && nbuf > maxbcache / BKVASIZE)
761 			nbuf = maxbcache / BKVASIZE;
762 		tuned_nbuf = 1;
763 	} else
764 		tuned_nbuf = 0;
765 
766 	/* XXX Avoid unsigned long overflows later on with maxbufspace. */
767 	maxbuf = (LONG_MAX / 3) / BKVASIZE;
768 	if (nbuf > maxbuf) {
769 		if (!tuned_nbuf)
770 			printf("Warning: nbufs lowered from %d to %ld\n", nbuf,
771 			    maxbuf);
772 		nbuf = maxbuf;
773 	}
774 
775 	/*
776 	 * Ideal allocation size for the transient bio submap is 10%
777 	 * of the maximal space buffer map.  This roughly corresponds
778 	 * to the amount of the buffer mapped for typical UFS load.
779 	 *
780 	 * Clip the buffer map to reserve space for the transient
781 	 * BIOs, if its extent is bigger than 90% (80% on i386) of the
782 	 * maximum buffer map extent on the platform.
783 	 *
784 	 * The fall-back to the maxbuf in case of maxbcache unset,
785 	 * allows to not trim the buffer KVA for the architectures
786 	 * with ample KVA space.
787 	 */
788 	if (bio_transient_maxcnt == 0 && unmapped_buf_allowed) {
789 		maxbuf_sz = maxbcache != 0 ? maxbcache : maxbuf * BKVASIZE;
790 		buf_sz = (long)nbuf * BKVASIZE;
791 		if (buf_sz < maxbuf_sz / TRANSIENT_DENOM *
792 		    (TRANSIENT_DENOM - 1)) {
793 			/*
794 			 * There is more KVA than memory.  Do not
795 			 * adjust buffer map size, and assign the rest
796 			 * of maxbuf to transient map.
797 			 */
798 			biotmap_sz = maxbuf_sz - buf_sz;
799 		} else {
800 			/*
801 			 * Buffer map spans all KVA we could afford on
802 			 * this platform.  Give 10% (20% on i386) of
803 			 * the buffer map to the transient bio map.
804 			 */
805 			biotmap_sz = buf_sz / TRANSIENT_DENOM;
806 			buf_sz -= biotmap_sz;
807 		}
808 		if (biotmap_sz / INT_MAX > MAXPHYS)
809 			bio_transient_maxcnt = INT_MAX;
810 		else
811 			bio_transient_maxcnt = biotmap_sz / MAXPHYS;
812 		/*
813 		 * Artifically limit to 1024 simultaneous in-flight I/Os
814 		 * using the transient mapping.
815 		 */
816 		if (bio_transient_maxcnt > 1024)
817 			bio_transient_maxcnt = 1024;
818 		if (tuned_nbuf)
819 			nbuf = buf_sz / BKVASIZE;
820 	}
821 
822 	/*
823 	 * swbufs are used as temporary holders for I/O, such as paging I/O.
824 	 * We have no less then 16 and no more then 256.
825 	 */
826 	nswbuf = min(nbuf / 4, 256);
827 	TUNABLE_INT_FETCH("kern.nswbuf", &nswbuf);
828 	if (nswbuf < NSWBUF_MIN)
829 		nswbuf = NSWBUF_MIN;
830 
831 	/*
832 	 * Reserve space for the buffer cache buffers
833 	 */
834 	swbuf = (void *)v;
835 	v = (caddr_t)(swbuf + nswbuf);
836 	buf = (void *)v;
837 	v = (caddr_t)(buf + nbuf);
838 
839 	return(v);
840 }
841 
842 /* Initialize the buffer subsystem.  Called before use of any buffers. */
843 void
844 bufinit(void)
845 {
846 	struct buf *bp;
847 	int i;
848 
849 	CTASSERT(MAXBCACHEBUF >= MAXBSIZE);
850 	mtx_init(&bqclean, "bufq clean lock", NULL, MTX_DEF);
851 	mtx_init(&bqdirty, "bufq dirty lock", NULL, MTX_DEF);
852 	mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
853 	rw_init(&nblock, "needsbuffer lock");
854 	mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
855 	mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF);
856 
857 	/* next, make a null set of free lists */
858 	for (i = 0; i < BUFFER_QUEUES; i++)
859 		TAILQ_INIT(&bufqueues[i]);
860 
861 	unmapped_buf = (caddr_t)kva_alloc(MAXPHYS);
862 
863 	/* finally, initialize each buffer header and stick on empty q */
864 	for (i = 0; i < nbuf; i++) {
865 		bp = &buf[i];
866 		bzero(bp, sizeof *bp);
867 		bp->b_flags = B_INVAL | B_INFREECNT;
868 		bp->b_rcred = NOCRED;
869 		bp->b_wcred = NOCRED;
870 		bp->b_qindex = QUEUE_EMPTY;
871 		bp->b_xflags = 0;
872 		bp->b_data = bp->b_kvabase = unmapped_buf;
873 		LIST_INIT(&bp->b_dep);
874 		BUF_LOCKINIT(bp);
875 		TAILQ_INSERT_TAIL(&bufqueues[QUEUE_EMPTY], bp, b_freelist);
876 #ifdef INVARIANTS
877 		bq_len[QUEUE_EMPTY]++;
878 #endif
879 	}
880 
881 	/*
882 	 * maxbufspace is the absolute maximum amount of buffer space we are
883 	 * allowed to reserve in KVM and in real terms.  The absolute maximum
884 	 * is nominally used by buf_daemon.  hibufspace is the nominal maximum
885 	 * used by most other processes.  The differential is required to
886 	 * ensure that buf_daemon is able to run when other processes might
887 	 * be blocked waiting for buffer space.
888 	 *
889 	 * maxbufspace is based on BKVASIZE.  Allocating buffers larger then
890 	 * this may result in KVM fragmentation which is not handled optimally
891 	 * by the system.
892 	 */
893 	maxbufspace = (long)nbuf * BKVASIZE;
894 	hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - MAXBCACHEBUF * 10);
895 	lobufspace = hibufspace - MAXBCACHEBUF;
896 
897 	/*
898 	 * Note: The 16 MiB upper limit for hirunningspace was chosen
899 	 * arbitrarily and may need further tuning. It corresponds to
900 	 * 128 outstanding write IO requests (if IO size is 128 KiB),
901 	 * which fits with many RAID controllers' tagged queuing limits.
902 	 * The lower 1 MiB limit is the historical upper limit for
903 	 * hirunningspace.
904 	 */
905 	hirunningspace = lmax(lmin(roundup(hibufspace / 64, MAXBCACHEBUF),
906 	    16 * 1024 * 1024), 1024 * 1024);
907 	lorunningspace = roundup((hirunningspace * 2) / 3, MAXBCACHEBUF);
908 
909 /*
910  * Limit the amount of malloc memory since it is wired permanently into
911  * the kernel space.  Even though this is accounted for in the buffer
912  * allocation, we don't want the malloced region to grow uncontrolled.
913  * The malloc scheme improves memory utilization significantly on average
914  * (small) directories.
915  */
916 	maxbufmallocspace = hibufspace / 20;
917 
918 /*
919  * Reduce the chance of a deadlock occuring by limiting the number
920  * of delayed-write dirty buffers we allow to stack up.
921  */
922 	hidirtybuffers = nbuf / 4 + 20;
923 	dirtybufthresh = hidirtybuffers * 9 / 10;
924 	numdirtybuffers = 0;
925 /*
926  * To support extreme low-memory systems, make sure hidirtybuffers cannot
927  * eat up all available buffer space.  This occurs when our minimum cannot
928  * be met.  We try to size hidirtybuffers to 3/4 our buffer space assuming
929  * BKVASIZE'd buffers.
930  */
931 	while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
932 		hidirtybuffers >>= 1;
933 	}
934 	lodirtybuffers = hidirtybuffers / 2;
935 
936 /*
937  * Try to keep the number of free buffers in the specified range,
938  * and give special processes (e.g. like buf_daemon) access to an
939  * emergency reserve.
940  */
941 	lofreebuffers = nbuf / 18 + 5;
942 	hifreebuffers = 2 * lofreebuffers;
943 	numfreebuffers = nbuf;
944 
945 	bogus_page = vm_page_alloc(NULL, 0, VM_ALLOC_NOOBJ |
946 	    VM_ALLOC_NORMAL | VM_ALLOC_WIRED);
947 }
948 
949 #ifdef INVARIANTS
950 static inline void
951 vfs_buf_check_mapped(struct buf *bp)
952 {
953 
954 	KASSERT(bp->b_kvabase != unmapped_buf,
955 	    ("mapped buf: b_kvabase was not updated %p", bp));
956 	KASSERT(bp->b_data != unmapped_buf,
957 	    ("mapped buf: b_data was not updated %p", bp));
958 	KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf +
959 	    MAXPHYS, ("b_data + b_offset unmapped %p", bp));
960 }
961 
962 static inline void
963 vfs_buf_check_unmapped(struct buf *bp)
964 {
965 
966 	KASSERT(bp->b_data == unmapped_buf,
967 	    ("unmapped buf: corrupted b_data %p", bp));
968 }
969 
970 #define	BUF_CHECK_MAPPED(bp) vfs_buf_check_mapped(bp)
971 #define	BUF_CHECK_UNMAPPED(bp) vfs_buf_check_unmapped(bp)
972 #else
973 #define	BUF_CHECK_MAPPED(bp) do {} while (0)
974 #define	BUF_CHECK_UNMAPPED(bp) do {} while (0)
975 #endif
976 
977 static int
978 isbufbusy(struct buf *bp)
979 {
980 	if (((bp->b_flags & (B_INVAL | B_PERSISTENT)) == 0 &&
981 	    BUF_ISLOCKED(bp)) ||
982 	    ((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI))
983 		return (1);
984 	return (0);
985 }
986 
987 /*
988  * Shutdown the system cleanly to prepare for reboot, halt, or power off.
989  */
990 void
991 bufshutdown(int show_busybufs)
992 {
993 	static int first_buf_printf = 1;
994 	struct buf *bp;
995 	int iter, nbusy, pbusy;
996 #ifndef PREEMPTION
997 	int subiter;
998 #endif
999 
1000 	/*
1001 	 * Sync filesystems for shutdown
1002 	 */
1003 	wdog_kern_pat(WD_LASTVAL);
1004 	sys_sync(curthread, NULL);
1005 
1006 	/*
1007 	 * With soft updates, some buffers that are
1008 	 * written will be remarked as dirty until other
1009 	 * buffers are written.
1010 	 */
1011 	for (iter = pbusy = 0; iter < 20; iter++) {
1012 		nbusy = 0;
1013 		for (bp = &buf[nbuf]; --bp >= buf; )
1014 			if (isbufbusy(bp))
1015 				nbusy++;
1016 		if (nbusy == 0) {
1017 			if (first_buf_printf)
1018 				printf("All buffers synced.");
1019 			break;
1020 		}
1021 		if (first_buf_printf) {
1022 			printf("Syncing disks, buffers remaining... ");
1023 			first_buf_printf = 0;
1024 		}
1025 		printf("%d ", nbusy);
1026 		if (nbusy < pbusy)
1027 			iter = 0;
1028 		pbusy = nbusy;
1029 
1030 		wdog_kern_pat(WD_LASTVAL);
1031 		sys_sync(curthread, NULL);
1032 
1033 #ifdef PREEMPTION
1034 		/*
1035 		 * Drop Giant and spin for a while to allow
1036 		 * interrupt threads to run.
1037 		 */
1038 		DROP_GIANT();
1039 		DELAY(50000 * iter);
1040 		PICKUP_GIANT();
1041 #else
1042 		/*
1043 		 * Drop Giant and context switch several times to
1044 		 * allow interrupt threads to run.
1045 		 */
1046 		DROP_GIANT();
1047 		for (subiter = 0; subiter < 50 * iter; subiter++) {
1048 			thread_lock(curthread);
1049 			mi_switch(SW_VOL, NULL);
1050 			thread_unlock(curthread);
1051 			DELAY(1000);
1052 		}
1053 		PICKUP_GIANT();
1054 #endif
1055 	}
1056 	printf("\n");
1057 	/*
1058 	 * Count only busy local buffers to prevent forcing
1059 	 * a fsck if we're just a client of a wedged NFS server
1060 	 */
1061 	nbusy = 0;
1062 	for (bp = &buf[nbuf]; --bp >= buf; ) {
1063 		if (isbufbusy(bp)) {
1064 #if 0
1065 /* XXX: This is bogus.  We should probably have a BO_REMOTE flag instead */
1066 			if (bp->b_dev == NULL) {
1067 				TAILQ_REMOVE(&mountlist,
1068 				    bp->b_vp->v_mount, mnt_list);
1069 				continue;
1070 			}
1071 #endif
1072 			nbusy++;
1073 			if (show_busybufs > 0) {
1074 				printf(
1075 	    "%d: buf:%p, vnode:%p, flags:%0x, blkno:%jd, lblkno:%jd, buflock:",
1076 				    nbusy, bp, bp->b_vp, bp->b_flags,
1077 				    (intmax_t)bp->b_blkno,
1078 				    (intmax_t)bp->b_lblkno);
1079 				BUF_LOCKPRINTINFO(bp);
1080 				if (show_busybufs > 1)
1081 					vn_printf(bp->b_vp,
1082 					    "vnode content: ");
1083 			}
1084 		}
1085 	}
1086 	if (nbusy) {
1087 		/*
1088 		 * Failed to sync all blocks. Indicate this and don't
1089 		 * unmount filesystems (thus forcing an fsck on reboot).
1090 		 */
1091 		printf("Giving up on %d buffers\n", nbusy);
1092 		DELAY(5000000);	/* 5 seconds */
1093 	} else {
1094 		if (!first_buf_printf)
1095 			printf("Final sync complete\n");
1096 		/*
1097 		 * Unmount filesystems
1098 		 */
1099 		if (panicstr == 0)
1100 			vfs_unmountall();
1101 	}
1102 	swapoff_all();
1103 	DELAY(100000);		/* wait for console output to finish */
1104 }
1105 
1106 static void
1107 bpmap_qenter(struct buf *bp)
1108 {
1109 
1110 	BUF_CHECK_MAPPED(bp);
1111 
1112 	/*
1113 	 * bp->b_data is relative to bp->b_offset, but
1114 	 * bp->b_offset may be offset into the first page.
1115 	 */
1116 	bp->b_data = (caddr_t)trunc_page((vm_offset_t)bp->b_data);
1117 	pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
1118 	bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
1119 	    (vm_offset_t)(bp->b_offset & PAGE_MASK));
1120 }
1121 
1122 /*
1123  *	binsfree:
1124  *
1125  *	Insert the buffer into the appropriate free list.
1126  */
1127 static void
1128 binsfree(struct buf *bp, int qindex)
1129 {
1130 	struct mtx *olock, *nlock;
1131 
1132 	BUF_ASSERT_XLOCKED(bp);
1133 
1134 	nlock = bqlock(qindex);
1135 	/* Handle delayed bremfree() processing. */
1136 	if (bp->b_flags & B_REMFREE) {
1137 		olock = bqlock(bp->b_qindex);
1138 		mtx_lock(olock);
1139 		bremfreel(bp);
1140 		if (olock != nlock) {
1141 			mtx_unlock(olock);
1142 			mtx_lock(nlock);
1143 		}
1144 	} else
1145 		mtx_lock(nlock);
1146 
1147 	if (bp->b_qindex != QUEUE_NONE)
1148 		panic("binsfree: free buffer onto another queue???");
1149 
1150 	bp->b_qindex = qindex;
1151 	if (bp->b_flags & B_AGE)
1152 		TAILQ_INSERT_HEAD(&bufqueues[bp->b_qindex], bp, b_freelist);
1153 	else
1154 		TAILQ_INSERT_TAIL(&bufqueues[bp->b_qindex], bp, b_freelist);
1155 #ifdef INVARIANTS
1156 	bq_len[bp->b_qindex]++;
1157 #endif
1158 	mtx_unlock(nlock);
1159 
1160 	/*
1161 	 * Something we can maybe free or reuse.
1162 	 */
1163 	if (bp->b_bufsize && !(bp->b_flags & B_DELWRI))
1164 		bufspacewakeup();
1165 
1166 	if ((bp->b_flags & B_INVAL) || !(bp->b_flags & B_DELWRI))
1167 		bufcountadd(bp);
1168 }
1169 
1170 /*
1171  *	bremfree:
1172  *
1173  *	Mark the buffer for removal from the appropriate free list.
1174  *
1175  */
1176 void
1177 bremfree(struct buf *bp)
1178 {
1179 
1180 	CTR3(KTR_BUF, "bremfree(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1181 	KASSERT((bp->b_flags & B_REMFREE) == 0,
1182 	    ("bremfree: buffer %p already marked for delayed removal.", bp));
1183 	KASSERT(bp->b_qindex != QUEUE_NONE,
1184 	    ("bremfree: buffer %p not on a queue.", bp));
1185 	BUF_ASSERT_XLOCKED(bp);
1186 
1187 	bp->b_flags |= B_REMFREE;
1188 	bufcountsub(bp);
1189 }
1190 
1191 /*
1192  *	bremfreef:
1193  *
1194  *	Force an immediate removal from a free list.  Used only in nfs when
1195  *	it abuses the b_freelist pointer.
1196  */
1197 void
1198 bremfreef(struct buf *bp)
1199 {
1200 	struct mtx *qlock;
1201 
1202 	qlock = bqlock(bp->b_qindex);
1203 	mtx_lock(qlock);
1204 	bremfreel(bp);
1205 	mtx_unlock(qlock);
1206 }
1207 
1208 /*
1209  *	bremfreel:
1210  *
1211  *	Removes a buffer from the free list, must be called with the
1212  *	correct qlock held.
1213  */
1214 static void
1215 bremfreel(struct buf *bp)
1216 {
1217 
1218 	CTR3(KTR_BUF, "bremfreel(%p) vp %p flags %X",
1219 	    bp, bp->b_vp, bp->b_flags);
1220 	KASSERT(bp->b_qindex != QUEUE_NONE,
1221 	    ("bremfreel: buffer %p not on a queue.", bp));
1222 	BUF_ASSERT_XLOCKED(bp);
1223 	mtx_assert(bqlock(bp->b_qindex), MA_OWNED);
1224 
1225 	TAILQ_REMOVE(&bufqueues[bp->b_qindex], bp, b_freelist);
1226 #ifdef INVARIANTS
1227 	KASSERT(bq_len[bp->b_qindex] >= 1, ("queue %d underflow",
1228 	    bp->b_qindex));
1229 	bq_len[bp->b_qindex]--;
1230 #endif
1231 	bp->b_qindex = QUEUE_NONE;
1232 	/*
1233 	 * If this was a delayed bremfree() we only need to remove the buffer
1234 	 * from the queue and return the stats are already done.
1235 	 */
1236 	if (bp->b_flags & B_REMFREE) {
1237 		bp->b_flags &= ~B_REMFREE;
1238 		return;
1239 	}
1240 	bufcountsub(bp);
1241 }
1242 
1243 /*
1244  *	bufkvafree:
1245  *
1246  *	Free the kva allocation for a buffer.
1247  *
1248  */
1249 static void
1250 bufkvafree(struct buf *bp)
1251 {
1252 
1253 #ifdef INVARIANTS
1254 	if (bp->b_kvasize == 0) {
1255 		KASSERT(bp->b_kvabase == unmapped_buf &&
1256 		    bp->b_data == unmapped_buf,
1257 		    ("Leaked KVA space on %p", bp));
1258 	} else if (buf_mapped(bp))
1259 		BUF_CHECK_MAPPED(bp);
1260 	else
1261 		BUF_CHECK_UNMAPPED(bp);
1262 #endif
1263 	if (bp->b_kvasize == 0)
1264 		return;
1265 
1266 	vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
1267 	atomic_subtract_long(&bufkvaspace, bp->b_kvasize);
1268 	atomic_add_int(&buffreekvacnt, 1);
1269 	bp->b_data = bp->b_kvabase = unmapped_buf;
1270 	bp->b_kvasize = 0;
1271 }
1272 
1273 /*
1274  *	bufkvaalloc:
1275  *
1276  *	Allocate the buffer KVA and set b_kvasize and b_kvabase.
1277  */
1278 static int
1279 bufkvaalloc(struct buf *bp, int maxsize, int gbflags)
1280 {
1281 	vm_offset_t addr;
1282 	int error;
1283 
1284 	KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0,
1285 	    ("Invalid gbflags 0x%x in %s", gbflags, __func__));
1286 
1287 	bufkvafree(bp);
1288 
1289 	addr = 0;
1290 	error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr);
1291 	if (error != 0) {
1292 		/*
1293 		 * Buffer map is too fragmented.  Request the caller
1294 		 * to defragment the map.
1295 		 */
1296 		atomic_add_int(&bufdefragcnt, 1);
1297 		return (error);
1298 	}
1299 	bp->b_kvabase = (caddr_t)addr;
1300 	bp->b_kvasize = maxsize;
1301 	atomic_add_long(&bufkvaspace, bp->b_kvasize);
1302 	if ((gbflags & GB_UNMAPPED) != 0) {
1303 		bp->b_data = unmapped_buf;
1304 		BUF_CHECK_UNMAPPED(bp);
1305 	} else {
1306 		bp->b_data = bp->b_kvabase;
1307 		BUF_CHECK_MAPPED(bp);
1308 	}
1309 	return (0);
1310 }
1311 
1312 /*
1313  * Attempt to initiate asynchronous I/O on read-ahead blocks.  We must
1314  * clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
1315  * the buffer is valid and we do not have to do anything.
1316  */
1317 void
1318 breada(struct vnode * vp, daddr_t * rablkno, int * rabsize,
1319     int cnt, struct ucred * cred)
1320 {
1321 	struct buf *rabp;
1322 	int i;
1323 
1324 	for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
1325 		if (inmem(vp, *rablkno))
1326 			continue;
1327 		rabp = getblk(vp, *rablkno, *rabsize, 0, 0, 0);
1328 
1329 		if ((rabp->b_flags & B_CACHE) == 0) {
1330 			if (!TD_IS_IDLETHREAD(curthread))
1331 				curthread->td_ru.ru_inblock++;
1332 			rabp->b_flags |= B_ASYNC;
1333 			rabp->b_flags &= ~B_INVAL;
1334 			rabp->b_ioflags &= ~BIO_ERROR;
1335 			rabp->b_iocmd = BIO_READ;
1336 			if (rabp->b_rcred == NOCRED && cred != NOCRED)
1337 				rabp->b_rcred = crhold(cred);
1338 			vfs_busy_pages(rabp, 0);
1339 			BUF_KERNPROC(rabp);
1340 			rabp->b_iooffset = dbtob(rabp->b_blkno);
1341 			bstrategy(rabp);
1342 		} else {
1343 			brelse(rabp);
1344 		}
1345 	}
1346 }
1347 
1348 /*
1349  * Entry point for bread() and breadn() via #defines in sys/buf.h.
1350  *
1351  * Get a buffer with the specified data.  Look in the cache first.  We
1352  * must clear BIO_ERROR and B_INVAL prior to initiating I/O.  If B_CACHE
1353  * is set, the buffer is valid and we do not have to do anything, see
1354  * getblk(). Also starts asynchronous I/O on read-ahead blocks.
1355  */
1356 int
1357 breadn_flags(struct vnode *vp, daddr_t blkno, int size, daddr_t *rablkno,
1358     int *rabsize, int cnt, struct ucred *cred, int flags, struct buf **bpp)
1359 {
1360 	struct buf *bp;
1361 	int rv = 0, readwait = 0;
1362 
1363 	CTR3(KTR_BUF, "breadn(%p, %jd, %d)", vp, blkno, size);
1364 	/*
1365 	 * Can only return NULL if GB_LOCK_NOWAIT flag is specified.
1366 	 */
1367 	*bpp = bp = getblk(vp, blkno, size, 0, 0, flags);
1368 	if (bp == NULL)
1369 		return (EBUSY);
1370 
1371 	/* if not found in cache, do some I/O */
1372 	if ((bp->b_flags & B_CACHE) == 0) {
1373 		if (!TD_IS_IDLETHREAD(curthread))
1374 			curthread->td_ru.ru_inblock++;
1375 		bp->b_iocmd = BIO_READ;
1376 		bp->b_flags &= ~B_INVAL;
1377 		bp->b_ioflags &= ~BIO_ERROR;
1378 		if (bp->b_rcred == NOCRED && cred != NOCRED)
1379 			bp->b_rcred = crhold(cred);
1380 		vfs_busy_pages(bp, 0);
1381 		bp->b_iooffset = dbtob(bp->b_blkno);
1382 		bstrategy(bp);
1383 		++readwait;
1384 	}
1385 
1386 	breada(vp, rablkno, rabsize, cnt, cred);
1387 
1388 	if (readwait) {
1389 		rv = bufwait(bp);
1390 	}
1391 	return (rv);
1392 }
1393 
1394 /*
1395  * Write, release buffer on completion.  (Done by iodone
1396  * if async).  Do not bother writing anything if the buffer
1397  * is invalid.
1398  *
1399  * Note that we set B_CACHE here, indicating that buffer is
1400  * fully valid and thus cacheable.  This is true even of NFS
1401  * now so we set it generally.  This could be set either here
1402  * or in biodone() since the I/O is synchronous.  We put it
1403  * here.
1404  */
1405 int
1406 bufwrite(struct buf *bp)
1407 {
1408 	int oldflags;
1409 	struct vnode *vp;
1410 	long space;
1411 	int vp_md;
1412 
1413 	CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1414 	if ((bp->b_bufobj->bo_flag & BO_DEAD) != 0) {
1415 		bp->b_flags |= B_INVAL | B_RELBUF;
1416 		bp->b_flags &= ~B_CACHE;
1417 		brelse(bp);
1418 		return (ENXIO);
1419 	}
1420 	if (bp->b_flags & B_INVAL) {
1421 		brelse(bp);
1422 		return (0);
1423 	}
1424 
1425 	if (bp->b_flags & B_BARRIER)
1426 		barrierwrites++;
1427 
1428 	oldflags = bp->b_flags;
1429 
1430 	BUF_ASSERT_HELD(bp);
1431 
1432 	if (bp->b_pin_count > 0)
1433 		bunpin_wait(bp);
1434 
1435 	KASSERT(!(bp->b_vflags & BV_BKGRDINPROG),
1436 	    ("FFS background buffer should not get here %p", bp));
1437 
1438 	vp = bp->b_vp;
1439 	if (vp)
1440 		vp_md = vp->v_vflag & VV_MD;
1441 	else
1442 		vp_md = 0;
1443 
1444 	/*
1445 	 * Mark the buffer clean.  Increment the bufobj write count
1446 	 * before bundirty() call, to prevent other thread from seeing
1447 	 * empty dirty list and zero counter for writes in progress,
1448 	 * falsely indicating that the bufobj is clean.
1449 	 */
1450 	bufobj_wref(bp->b_bufobj);
1451 	bundirty(bp);
1452 
1453 	bp->b_flags &= ~B_DONE;
1454 	bp->b_ioflags &= ~BIO_ERROR;
1455 	bp->b_flags |= B_CACHE;
1456 	bp->b_iocmd = BIO_WRITE;
1457 
1458 	vfs_busy_pages(bp, 1);
1459 
1460 	/*
1461 	 * Normal bwrites pipeline writes
1462 	 */
1463 	bp->b_runningbufspace = bp->b_bufsize;
1464 	space = atomic_fetchadd_long(&runningbufspace, bp->b_runningbufspace);
1465 
1466 	if (!TD_IS_IDLETHREAD(curthread))
1467 		curthread->td_ru.ru_oublock++;
1468 	if (oldflags & B_ASYNC)
1469 		BUF_KERNPROC(bp);
1470 	bp->b_iooffset = dbtob(bp->b_blkno);
1471 	bstrategy(bp);
1472 
1473 	if ((oldflags & B_ASYNC) == 0) {
1474 		int rtval = bufwait(bp);
1475 		brelse(bp);
1476 		return (rtval);
1477 	} else if (space > hirunningspace) {
1478 		/*
1479 		 * don't allow the async write to saturate the I/O
1480 		 * system.  We will not deadlock here because
1481 		 * we are blocking waiting for I/O that is already in-progress
1482 		 * to complete. We do not block here if it is the update
1483 		 * or syncer daemon trying to clean up as that can lead
1484 		 * to deadlock.
1485 		 */
1486 		if ((curthread->td_pflags & TDP_NORUNNINGBUF) == 0 && !vp_md)
1487 			waitrunningbufspace();
1488 	}
1489 
1490 	return (0);
1491 }
1492 
1493 void
1494 bufbdflush(struct bufobj *bo, struct buf *bp)
1495 {
1496 	struct buf *nbp;
1497 
1498 	if (bo->bo_dirty.bv_cnt > dirtybufthresh + 10) {
1499 		(void) VOP_FSYNC(bp->b_vp, MNT_NOWAIT, curthread);
1500 		altbufferflushes++;
1501 	} else if (bo->bo_dirty.bv_cnt > dirtybufthresh) {
1502 		BO_LOCK(bo);
1503 		/*
1504 		 * Try to find a buffer to flush.
1505 		 */
1506 		TAILQ_FOREACH(nbp, &bo->bo_dirty.bv_hd, b_bobufs) {
1507 			if ((nbp->b_vflags & BV_BKGRDINPROG) ||
1508 			    BUF_LOCK(nbp,
1509 				     LK_EXCLUSIVE | LK_NOWAIT, NULL))
1510 				continue;
1511 			if (bp == nbp)
1512 				panic("bdwrite: found ourselves");
1513 			BO_UNLOCK(bo);
1514 			/* Don't countdeps with the bo lock held. */
1515 			if (buf_countdeps(nbp, 0)) {
1516 				BO_LOCK(bo);
1517 				BUF_UNLOCK(nbp);
1518 				continue;
1519 			}
1520 			if (nbp->b_flags & B_CLUSTEROK) {
1521 				vfs_bio_awrite(nbp);
1522 			} else {
1523 				bremfree(nbp);
1524 				bawrite(nbp);
1525 			}
1526 			dirtybufferflushes++;
1527 			break;
1528 		}
1529 		if (nbp == NULL)
1530 			BO_UNLOCK(bo);
1531 	}
1532 }
1533 
1534 /*
1535  * Delayed write. (Buffer is marked dirty).  Do not bother writing
1536  * anything if the buffer is marked invalid.
1537  *
1538  * Note that since the buffer must be completely valid, we can safely
1539  * set B_CACHE.  In fact, we have to set B_CACHE here rather then in
1540  * biodone() in order to prevent getblk from writing the buffer
1541  * out synchronously.
1542  */
1543 void
1544 bdwrite(struct buf *bp)
1545 {
1546 	struct thread *td = curthread;
1547 	struct vnode *vp;
1548 	struct bufobj *bo;
1549 
1550 	CTR3(KTR_BUF, "bdwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1551 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
1552 	KASSERT((bp->b_flags & B_BARRIER) == 0,
1553 	    ("Barrier request in delayed write %p", bp));
1554 	BUF_ASSERT_HELD(bp);
1555 
1556 	if (bp->b_flags & B_INVAL) {
1557 		brelse(bp);
1558 		return;
1559 	}
1560 
1561 	/*
1562 	 * If we have too many dirty buffers, don't create any more.
1563 	 * If we are wildly over our limit, then force a complete
1564 	 * cleanup. Otherwise, just keep the situation from getting
1565 	 * out of control. Note that we have to avoid a recursive
1566 	 * disaster and not try to clean up after our own cleanup!
1567 	 */
1568 	vp = bp->b_vp;
1569 	bo = bp->b_bufobj;
1570 	if ((td->td_pflags & (TDP_COWINPROGRESS|TDP_INBDFLUSH)) == 0) {
1571 		td->td_pflags |= TDP_INBDFLUSH;
1572 		BO_BDFLUSH(bo, bp);
1573 		td->td_pflags &= ~TDP_INBDFLUSH;
1574 	} else
1575 		recursiveflushes++;
1576 
1577 	bdirty(bp);
1578 	/*
1579 	 * Set B_CACHE, indicating that the buffer is fully valid.  This is
1580 	 * true even of NFS now.
1581 	 */
1582 	bp->b_flags |= B_CACHE;
1583 
1584 	/*
1585 	 * This bmap keeps the system from needing to do the bmap later,
1586 	 * perhaps when the system is attempting to do a sync.  Since it
1587 	 * is likely that the indirect block -- or whatever other datastructure
1588 	 * that the filesystem needs is still in memory now, it is a good
1589 	 * thing to do this.  Note also, that if the pageout daemon is
1590 	 * requesting a sync -- there might not be enough memory to do
1591 	 * the bmap then...  So, this is important to do.
1592 	 */
1593 	if (vp->v_type != VCHR && bp->b_lblkno == bp->b_blkno) {
1594 		VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
1595 	}
1596 
1597 	/*
1598 	 * Set the *dirty* buffer range based upon the VM system dirty
1599 	 * pages.
1600 	 *
1601 	 * Mark the buffer pages as clean.  We need to do this here to
1602 	 * satisfy the vnode_pager and the pageout daemon, so that it
1603 	 * thinks that the pages have been "cleaned".  Note that since
1604 	 * the pages are in a delayed write buffer -- the VFS layer
1605 	 * "will" see that the pages get written out on the next sync,
1606 	 * or perhaps the cluster will be completed.
1607 	 */
1608 	vfs_clean_pages_dirty_buf(bp);
1609 	bqrelse(bp);
1610 
1611 	/*
1612 	 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
1613 	 * due to the softdep code.
1614 	 */
1615 }
1616 
1617 /*
1618  *	bdirty:
1619  *
1620  *	Turn buffer into delayed write request.  We must clear BIO_READ and
1621  *	B_RELBUF, and we must set B_DELWRI.  We reassign the buffer to
1622  *	itself to properly update it in the dirty/clean lists.  We mark it
1623  *	B_DONE to ensure that any asynchronization of the buffer properly
1624  *	clears B_DONE ( else a panic will occur later ).
1625  *
1626  *	bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
1627  *	might have been set pre-getblk().  Unlike bwrite/bdwrite, bdirty()
1628  *	should only be called if the buffer is known-good.
1629  *
1630  *	Since the buffer is not on a queue, we do not update the numfreebuffers
1631  *	count.
1632  *
1633  *	The buffer must be on QUEUE_NONE.
1634  */
1635 void
1636 bdirty(struct buf *bp)
1637 {
1638 
1639 	CTR3(KTR_BUF, "bdirty(%p) vp %p flags %X",
1640 	    bp, bp->b_vp, bp->b_flags);
1641 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
1642 	KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
1643 	    ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
1644 	BUF_ASSERT_HELD(bp);
1645 	bp->b_flags &= ~(B_RELBUF);
1646 	bp->b_iocmd = BIO_WRITE;
1647 
1648 	if ((bp->b_flags & B_DELWRI) == 0) {
1649 		bp->b_flags |= /* XXX B_DONE | */ B_DELWRI;
1650 		reassignbuf(bp);
1651 		bdirtyadd();
1652 	}
1653 }
1654 
1655 /*
1656  *	bundirty:
1657  *
1658  *	Clear B_DELWRI for buffer.
1659  *
1660  *	Since the buffer is not on a queue, we do not update the numfreebuffers
1661  *	count.
1662  *
1663  *	The buffer must be on QUEUE_NONE.
1664  */
1665 
1666 void
1667 bundirty(struct buf *bp)
1668 {
1669 
1670 	CTR3(KTR_BUF, "bundirty(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1671 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
1672 	KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
1673 	    ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
1674 	BUF_ASSERT_HELD(bp);
1675 
1676 	if (bp->b_flags & B_DELWRI) {
1677 		bp->b_flags &= ~B_DELWRI;
1678 		reassignbuf(bp);
1679 		bdirtysub();
1680 	}
1681 	/*
1682 	 * Since it is now being written, we can clear its deferred write flag.
1683 	 */
1684 	bp->b_flags &= ~B_DEFERRED;
1685 }
1686 
1687 /*
1688  *	bawrite:
1689  *
1690  *	Asynchronous write.  Start output on a buffer, but do not wait for
1691  *	it to complete.  The buffer is released when the output completes.
1692  *
1693  *	bwrite() ( or the VOP routine anyway ) is responsible for handling
1694  *	B_INVAL buffers.  Not us.
1695  */
1696 void
1697 bawrite(struct buf *bp)
1698 {
1699 
1700 	bp->b_flags |= B_ASYNC;
1701 	(void) bwrite(bp);
1702 }
1703 
1704 /*
1705  *	babarrierwrite:
1706  *
1707  *	Asynchronous barrier write.  Start output on a buffer, but do not
1708  *	wait for it to complete.  Place a write barrier after this write so
1709  *	that this buffer and all buffers written before it are committed to
1710  *	the disk before any buffers written after this write are committed
1711  *	to the disk.  The buffer is released when the output completes.
1712  */
1713 void
1714 babarrierwrite(struct buf *bp)
1715 {
1716 
1717 	bp->b_flags |= B_ASYNC | B_BARRIER;
1718 	(void) bwrite(bp);
1719 }
1720 
1721 /*
1722  *	bbarrierwrite:
1723  *
1724  *	Synchronous barrier write.  Start output on a buffer and wait for
1725  *	it to complete.  Place a write barrier after this write so that
1726  *	this buffer and all buffers written before it are committed to
1727  *	the disk before any buffers written after this write are committed
1728  *	to the disk.  The buffer is released when the output completes.
1729  */
1730 int
1731 bbarrierwrite(struct buf *bp)
1732 {
1733 
1734 	bp->b_flags |= B_BARRIER;
1735 	return (bwrite(bp));
1736 }
1737 
1738 /*
1739  *	bwillwrite:
1740  *
1741  *	Called prior to the locking of any vnodes when we are expecting to
1742  *	write.  We do not want to starve the buffer cache with too many
1743  *	dirty buffers so we block here.  By blocking prior to the locking
1744  *	of any vnodes we attempt to avoid the situation where a locked vnode
1745  *	prevents the various system daemons from flushing related buffers.
1746  */
1747 void
1748 bwillwrite(void)
1749 {
1750 
1751 	if (numdirtybuffers >= hidirtybuffers) {
1752 		mtx_lock(&bdirtylock);
1753 		while (numdirtybuffers >= hidirtybuffers) {
1754 			bdirtywait = 1;
1755 			msleep(&bdirtywait, &bdirtylock, (PRIBIO + 4),
1756 			    "flswai", 0);
1757 		}
1758 		mtx_unlock(&bdirtylock);
1759 	}
1760 }
1761 
1762 /*
1763  * Return true if we have too many dirty buffers.
1764  */
1765 int
1766 buf_dirty_count_severe(void)
1767 {
1768 
1769 	return(numdirtybuffers >= hidirtybuffers);
1770 }
1771 
1772 /*
1773  *	brelse:
1774  *
1775  *	Release a busy buffer and, if requested, free its resources.  The
1776  *	buffer will be stashed in the appropriate bufqueue[] allowing it
1777  *	to be accessed later as a cache entity or reused for other purposes.
1778  */
1779 void
1780 brelse(struct buf *bp)
1781 {
1782 	int qindex;
1783 
1784 	CTR3(KTR_BUF, "brelse(%p) vp %p flags %X",
1785 	    bp, bp->b_vp, bp->b_flags);
1786 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
1787 	    ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
1788 	KASSERT((bp->b_flags & B_VMIO) != 0 || (bp->b_flags & B_NOREUSE) == 0,
1789 	    ("brelse: non-VMIO buffer marked NOREUSE"));
1790 
1791 	if (BUF_LOCKRECURSED(bp)) {
1792 		/*
1793 		 * Do not process, in particular, do not handle the
1794 		 * B_INVAL/B_RELBUF and do not release to free list.
1795 		 */
1796 		BUF_UNLOCK(bp);
1797 		return;
1798 	}
1799 
1800 	if (bp->b_flags & B_MANAGED) {
1801 		bqrelse(bp);
1802 		return;
1803 	}
1804 
1805 	if ((bp->b_vflags & (BV_BKGRDINPROG | BV_BKGRDERR)) == BV_BKGRDERR) {
1806 		BO_LOCK(bp->b_bufobj);
1807 		bp->b_vflags &= ~BV_BKGRDERR;
1808 		BO_UNLOCK(bp->b_bufobj);
1809 		bdirty(bp);
1810 	}
1811 	if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
1812 	    bp->b_error == EIO && !(bp->b_flags & B_INVAL)) {
1813 		/*
1814 		 * Failed write, redirty.  Must clear BIO_ERROR to prevent
1815 		 * pages from being scrapped.  If the error is anything
1816 		 * other than an I/O error (EIO), assume that retrying
1817 		 * is futile.
1818 		 */
1819 		bp->b_ioflags &= ~BIO_ERROR;
1820 		bdirty(bp);
1821 	} else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
1822 	    (bp->b_ioflags & BIO_ERROR) || (bp->b_bufsize <= 0)) {
1823 		/*
1824 		 * Either a failed I/O or we were asked to free or not
1825 		 * cache the buffer.
1826 		 */
1827 		bp->b_flags |= B_INVAL;
1828 		if (!LIST_EMPTY(&bp->b_dep))
1829 			buf_deallocate(bp);
1830 		if (bp->b_flags & B_DELWRI)
1831 			bdirtysub();
1832 		bp->b_flags &= ~(B_DELWRI | B_CACHE);
1833 		if ((bp->b_flags & B_VMIO) == 0) {
1834 			allocbuf(bp, 0);
1835 			if (bp->b_vp)
1836 				brelvp(bp);
1837 		}
1838 	}
1839 
1840 	/*
1841 	 * We must clear B_RELBUF if B_DELWRI is set.  If vfs_vmio_truncate()
1842 	 * is called with B_DELWRI set, the underlying pages may wind up
1843 	 * getting freed causing a previous write (bdwrite()) to get 'lost'
1844 	 * because pages associated with a B_DELWRI bp are marked clean.
1845 	 *
1846 	 * We still allow the B_INVAL case to call vfs_vmio_truncate(), even
1847 	 * if B_DELWRI is set.
1848 	 */
1849 	if (bp->b_flags & B_DELWRI)
1850 		bp->b_flags &= ~B_RELBUF;
1851 
1852 	/*
1853 	 * VMIO buffer rundown.  It is not very necessary to keep a VMIO buffer
1854 	 * constituted, not even NFS buffers now.  Two flags effect this.  If
1855 	 * B_INVAL, the struct buf is invalidated but the VM object is kept
1856 	 * around ( i.e. so it is trivial to reconstitute the buffer later ).
1857 	 *
1858 	 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
1859 	 * invalidated.  BIO_ERROR cannot be set for a failed write unless the
1860 	 * buffer is also B_INVAL because it hits the re-dirtying code above.
1861 	 *
1862 	 * Normally we can do this whether a buffer is B_DELWRI or not.  If
1863 	 * the buffer is an NFS buffer, it is tracking piecemeal writes or
1864 	 * the commit state and we cannot afford to lose the buffer. If the
1865 	 * buffer has a background write in progress, we need to keep it
1866 	 * around to prevent it from being reconstituted and starting a second
1867 	 * background write.
1868 	 */
1869 	if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE ||
1870 	    (bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
1871 	    !(bp->b_vp->v_mount != NULL &&
1872 	    (bp->b_vp->v_mount->mnt_vfc->vfc_flags & VFCF_NETWORK) != 0 &&
1873 	    !vn_isdisk(bp->b_vp, NULL) && (bp->b_flags & B_DELWRI))) {
1874 		vfs_vmio_invalidate(bp);
1875 		allocbuf(bp, 0);
1876 	}
1877 
1878 	if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0 ||
1879 	    (bp->b_flags & (B_DELWRI | B_NOREUSE)) == B_NOREUSE) {
1880 		allocbuf(bp, 0);
1881 		bp->b_flags &= ~B_NOREUSE;
1882 		if (bp->b_vp != NULL)
1883 			brelvp(bp);
1884 	}
1885 
1886 	/*
1887 	 * If the buffer has junk contents signal it and eventually
1888 	 * clean up B_DELWRI and diassociate the vnode so that gbincore()
1889 	 * doesn't find it.
1890 	 */
1891 	if (bp->b_bufsize == 0 || (bp->b_ioflags & BIO_ERROR) != 0 ||
1892 	    (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF)) != 0)
1893 		bp->b_flags |= B_INVAL;
1894 	if (bp->b_flags & B_INVAL) {
1895 		if (bp->b_flags & B_DELWRI)
1896 			bundirty(bp);
1897 		if (bp->b_vp)
1898 			brelvp(bp);
1899 	}
1900 
1901 	/* buffers with no memory */
1902 	if (bp->b_bufsize == 0) {
1903 		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
1904 		if (bp->b_vflags & BV_BKGRDINPROG)
1905 			panic("losing buffer 1");
1906 		bufkvafree(bp);
1907 		qindex = QUEUE_EMPTY;
1908 		bp->b_flags |= B_AGE;
1909 	/* buffers with junk contents */
1910 	} else if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
1911 	    (bp->b_ioflags & BIO_ERROR)) {
1912 		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
1913 		if (bp->b_vflags & BV_BKGRDINPROG)
1914 			panic("losing buffer 2");
1915 		qindex = QUEUE_CLEAN;
1916 		bp->b_flags |= B_AGE;
1917 	/* remaining buffers */
1918 	} else if (bp->b_flags & B_DELWRI)
1919 		qindex = QUEUE_DIRTY;
1920 	else
1921 		qindex = QUEUE_CLEAN;
1922 
1923 	binsfree(bp, qindex);
1924 
1925 	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF | B_DIRECT);
1926 	if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
1927 		panic("brelse: not dirty");
1928 	/* unlock */
1929 	BUF_UNLOCK(bp);
1930 }
1931 
1932 /*
1933  * Release a buffer back to the appropriate queue but do not try to free
1934  * it.  The buffer is expected to be used again soon.
1935  *
1936  * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
1937  * biodone() to requeue an async I/O on completion.  It is also used when
1938  * known good buffers need to be requeued but we think we may need the data
1939  * again soon.
1940  *
1941  * XXX we should be able to leave the B_RELBUF hint set on completion.
1942  */
1943 void
1944 bqrelse(struct buf *bp)
1945 {
1946 	int qindex;
1947 
1948 	CTR3(KTR_BUF, "bqrelse(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1949 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
1950 	    ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
1951 
1952 	if (BUF_LOCKRECURSED(bp)) {
1953 		/* do not release to free list */
1954 		BUF_UNLOCK(bp);
1955 		return;
1956 	}
1957 	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
1958 
1959 	if (bp->b_flags & B_MANAGED) {
1960 		if (bp->b_flags & B_REMFREE)
1961 			bremfreef(bp);
1962 		goto out;
1963 	}
1964 
1965 	/* buffers with stale but valid contents */
1966 	if ((bp->b_flags & B_DELWRI) != 0 || (bp->b_vflags & (BV_BKGRDINPROG |
1967 	    BV_BKGRDERR)) == BV_BKGRDERR) {
1968 		BO_LOCK(bp->b_bufobj);
1969 		bp->b_vflags &= ~BV_BKGRDERR;
1970 		BO_UNLOCK(bp->b_bufobj);
1971 		qindex = QUEUE_DIRTY;
1972 	} else {
1973 		if ((bp->b_flags & B_DELWRI) == 0 &&
1974 		    (bp->b_xflags & BX_VNDIRTY))
1975 			panic("bqrelse: not dirty");
1976 		if ((bp->b_flags & B_NOREUSE) != 0) {
1977 			brelse(bp);
1978 			return;
1979 		}
1980 		qindex = QUEUE_CLEAN;
1981 	}
1982 	binsfree(bp, qindex);
1983 
1984 out:
1985 	/* unlock */
1986 	BUF_UNLOCK(bp);
1987 }
1988 
1989 /*
1990  * Complete I/O to a VMIO backed page.  Validate the pages as appropriate,
1991  * restore bogus pages.
1992  */
1993 static void
1994 vfs_vmio_iodone(struct buf *bp)
1995 {
1996 	vm_ooffset_t foff;
1997 	vm_page_t m;
1998 	vm_object_t obj;
1999 	struct vnode *vp;
2000 	int bogus, i, iosize;
2001 
2002 	obj = bp->b_bufobj->bo_object;
2003 	KASSERT(obj->paging_in_progress >= bp->b_npages,
2004 	    ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
2005 	    obj->paging_in_progress, bp->b_npages));
2006 
2007 	vp = bp->b_vp;
2008 	KASSERT(vp->v_holdcnt > 0,
2009 	    ("vfs_vmio_iodone: vnode %p has zero hold count", vp));
2010 	KASSERT(vp->v_object != NULL,
2011 	    ("vfs_vmio_iodone: vnode %p has no vm_object", vp));
2012 
2013 	foff = bp->b_offset;
2014 	KASSERT(bp->b_offset != NOOFFSET,
2015 	    ("vfs_vmio_iodone: bp %p has no buffer offset", bp));
2016 
2017 	bogus = 0;
2018 	iosize = bp->b_bcount - bp->b_resid;
2019 	VM_OBJECT_WLOCK(obj);
2020 	for (i = 0; i < bp->b_npages; i++) {
2021 		int resid;
2022 
2023 		resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
2024 		if (resid > iosize)
2025 			resid = iosize;
2026 
2027 		/*
2028 		 * cleanup bogus pages, restoring the originals
2029 		 */
2030 		m = bp->b_pages[i];
2031 		if (m == bogus_page) {
2032 			bogus = 1;
2033 			m = vm_page_lookup(obj, OFF_TO_IDX(foff));
2034 			if (m == NULL)
2035 				panic("biodone: page disappeared!");
2036 			bp->b_pages[i] = m;
2037 		} else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
2038 			/*
2039 			 * In the write case, the valid and clean bits are
2040 			 * already changed correctly ( see bdwrite() ), so we
2041 			 * only need to do this here in the read case.
2042 			 */
2043 			KASSERT((m->dirty & vm_page_bits(foff & PAGE_MASK,
2044 			    resid)) == 0, ("vfs_vmio_iodone: page %p "
2045 			    "has unexpected dirty bits", m));
2046 			vfs_page_set_valid(bp, foff, m);
2047 		}
2048 		KASSERT(OFF_TO_IDX(foff) == m->pindex,
2049 		    ("vfs_vmio_iodone: foff(%jd)/pindex(%ju) mismatch",
2050 		    (intmax_t)foff, (uintmax_t)m->pindex));
2051 
2052 		vm_page_sunbusy(m);
2053 		foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
2054 		iosize -= resid;
2055 	}
2056 	vm_object_pip_wakeupn(obj, bp->b_npages);
2057 	VM_OBJECT_WUNLOCK(obj);
2058 	if (bogus && buf_mapped(bp)) {
2059 		BUF_CHECK_MAPPED(bp);
2060 		pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
2061 		    bp->b_pages, bp->b_npages);
2062 	}
2063 }
2064 
2065 /*
2066  * Unwire a page held by a buf and place it on the appropriate vm queue.
2067  */
2068 static void
2069 vfs_vmio_unwire(struct buf *bp, vm_page_t m)
2070 {
2071 	bool freed;
2072 
2073 	vm_page_lock(m);
2074 	if (vm_page_unwire(m, PQ_NONE)) {
2075 		/*
2076 		 * Determine if the page should be freed before adding
2077 		 * it to the inactive queue.
2078 		 */
2079 		if (m->valid == 0) {
2080 			freed = !vm_page_busied(m);
2081 			if (freed)
2082 				vm_page_free(m);
2083 		} else if ((bp->b_flags & B_DIRECT) != 0)
2084 			freed = vm_page_try_to_free(m);
2085 		else
2086 			freed = false;
2087 		if (!freed) {
2088 			/*
2089 			 * If the page is unlikely to be reused, let the
2090 			 * VM know.  Otherwise, maintain LRU page
2091 			 * ordering and put the page at the tail of the
2092 			 * inactive queue.
2093 			 */
2094 			if ((bp->b_flags & B_NOREUSE) != 0)
2095 				vm_page_deactivate_noreuse(m);
2096 			else
2097 				vm_page_deactivate(m);
2098 		}
2099 	}
2100 	vm_page_unlock(m);
2101 }
2102 
2103 /*
2104  * Perform page invalidation when a buffer is released.  The fully invalid
2105  * pages will be reclaimed later in vfs_vmio_truncate().
2106  */
2107 static void
2108 vfs_vmio_invalidate(struct buf *bp)
2109 {
2110 	vm_object_t obj;
2111 	vm_page_t m;
2112 	int i, resid, poffset, presid;
2113 
2114 	if (buf_mapped(bp)) {
2115 		BUF_CHECK_MAPPED(bp);
2116 		pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
2117 	} else
2118 		BUF_CHECK_UNMAPPED(bp);
2119 	/*
2120 	 * Get the base offset and length of the buffer.  Note that
2121 	 * in the VMIO case if the buffer block size is not
2122 	 * page-aligned then b_data pointer may not be page-aligned.
2123 	 * But our b_pages[] array *IS* page aligned.
2124 	 *
2125 	 * block sizes less then DEV_BSIZE (usually 512) are not
2126 	 * supported due to the page granularity bits (m->valid,
2127 	 * m->dirty, etc...).
2128 	 *
2129 	 * See man buf(9) for more information
2130 	 */
2131 	obj = bp->b_bufobj->bo_object;
2132 	resid = bp->b_bufsize;
2133 	poffset = bp->b_offset & PAGE_MASK;
2134 	VM_OBJECT_WLOCK(obj);
2135 	for (i = 0; i < bp->b_npages; i++) {
2136 		m = bp->b_pages[i];
2137 		if (m == bogus_page)
2138 			panic("vfs_vmio_invalidate: Unexpected bogus page.");
2139 		bp->b_pages[i] = NULL;
2140 
2141 		presid = resid > (PAGE_SIZE - poffset) ?
2142 		    (PAGE_SIZE - poffset) : resid;
2143 		KASSERT(presid >= 0, ("brelse: extra page"));
2144 		while (vm_page_xbusied(m)) {
2145 			vm_page_lock(m);
2146 			VM_OBJECT_WUNLOCK(obj);
2147 			vm_page_busy_sleep(m, "mbncsh");
2148 			VM_OBJECT_WLOCK(obj);
2149 		}
2150 		if (pmap_page_wired_mappings(m) == 0)
2151 			vm_page_set_invalid(m, poffset, presid);
2152 		vfs_vmio_unwire(bp, m);
2153 		resid -= presid;
2154 		poffset = 0;
2155 	}
2156 	VM_OBJECT_WUNLOCK(obj);
2157 	bp->b_npages = 0;
2158 }
2159 
2160 /*
2161  * Page-granular truncation of an existing VMIO buffer.
2162  */
2163 static void
2164 vfs_vmio_truncate(struct buf *bp, int desiredpages)
2165 {
2166 	vm_object_t obj;
2167 	vm_page_t m;
2168 	int i;
2169 
2170 	if (bp->b_npages == desiredpages)
2171 		return;
2172 
2173 	if (buf_mapped(bp)) {
2174 		BUF_CHECK_MAPPED(bp);
2175 		pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
2176 		    (desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
2177 	} else
2178 		BUF_CHECK_UNMAPPED(bp);
2179 	obj = bp->b_bufobj->bo_object;
2180 	if (obj != NULL)
2181 		VM_OBJECT_WLOCK(obj);
2182 	for (i = desiredpages; i < bp->b_npages; i++) {
2183 		m = bp->b_pages[i];
2184 		KASSERT(m != bogus_page, ("allocbuf: bogus page found"));
2185 		bp->b_pages[i] = NULL;
2186 		vfs_vmio_unwire(bp, m);
2187 	}
2188 	if (obj != NULL)
2189 		VM_OBJECT_WUNLOCK(obj);
2190 	bp->b_npages = desiredpages;
2191 }
2192 
2193 /*
2194  * Byte granular extension of VMIO buffers.
2195  */
2196 static void
2197 vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
2198 {
2199 	/*
2200 	 * We are growing the buffer, possibly in a
2201 	 * byte-granular fashion.
2202 	 */
2203 	vm_object_t obj;
2204 	vm_offset_t toff;
2205 	vm_offset_t tinc;
2206 	vm_page_t m;
2207 
2208 	/*
2209 	 * Step 1, bring in the VM pages from the object, allocating
2210 	 * them if necessary.  We must clear B_CACHE if these pages
2211 	 * are not valid for the range covered by the buffer.
2212 	 */
2213 	obj = bp->b_bufobj->bo_object;
2214 	VM_OBJECT_WLOCK(obj);
2215 	while (bp->b_npages < desiredpages) {
2216 		/*
2217 		 * We must allocate system pages since blocking
2218 		 * here could interfere with paging I/O, no
2219 		 * matter which process we are.
2220 		 *
2221 		 * Only exclusive busy can be tested here.
2222 		 * Blocking on shared busy might lead to
2223 		 * deadlocks once allocbuf() is called after
2224 		 * pages are vfs_busy_pages().
2225 		 */
2226 		m = vm_page_grab(obj, OFF_TO_IDX(bp->b_offset) + bp->b_npages,
2227 		    VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM |
2228 		    VM_ALLOC_WIRED | VM_ALLOC_IGN_SBUSY |
2229 		    VM_ALLOC_COUNT(desiredpages - bp->b_npages));
2230 		if (m->valid == 0)
2231 			bp->b_flags &= ~B_CACHE;
2232 		bp->b_pages[bp->b_npages] = m;
2233 		++bp->b_npages;
2234 	}
2235 
2236 	/*
2237 	 * Step 2.  We've loaded the pages into the buffer,
2238 	 * we have to figure out if we can still have B_CACHE
2239 	 * set.  Note that B_CACHE is set according to the
2240 	 * byte-granular range ( bcount and size ), not the
2241 	 * aligned range ( newbsize ).
2242 	 *
2243 	 * The VM test is against m->valid, which is DEV_BSIZE
2244 	 * aligned.  Needless to say, the validity of the data
2245 	 * needs to also be DEV_BSIZE aligned.  Note that this
2246 	 * fails with NFS if the server or some other client
2247 	 * extends the file's EOF.  If our buffer is resized,
2248 	 * B_CACHE may remain set! XXX
2249 	 */
2250 	toff = bp->b_bcount;
2251 	tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
2252 	while ((bp->b_flags & B_CACHE) && toff < size) {
2253 		vm_pindex_t pi;
2254 
2255 		if (tinc > (size - toff))
2256 			tinc = size - toff;
2257 		pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
2258 		m = bp->b_pages[pi];
2259 		vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
2260 		toff += tinc;
2261 		tinc = PAGE_SIZE;
2262 	}
2263 	VM_OBJECT_WUNLOCK(obj);
2264 
2265 	/*
2266 	 * Step 3, fixup the KVA pmap.
2267 	 */
2268 	if (buf_mapped(bp))
2269 		bpmap_qenter(bp);
2270 	else
2271 		BUF_CHECK_UNMAPPED(bp);
2272 }
2273 
2274 /*
2275  * Check to see if a block at a particular lbn is available for a clustered
2276  * write.
2277  */
2278 static int
2279 vfs_bio_clcheck(struct vnode *vp, int size, daddr_t lblkno, daddr_t blkno)
2280 {
2281 	struct buf *bpa;
2282 	int match;
2283 
2284 	match = 0;
2285 
2286 	/* If the buf isn't in core skip it */
2287 	if ((bpa = gbincore(&vp->v_bufobj, lblkno)) == NULL)
2288 		return (0);
2289 
2290 	/* If the buf is busy we don't want to wait for it */
2291 	if (BUF_LOCK(bpa, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
2292 		return (0);
2293 
2294 	/* Only cluster with valid clusterable delayed write buffers */
2295 	if ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) !=
2296 	    (B_DELWRI | B_CLUSTEROK))
2297 		goto done;
2298 
2299 	if (bpa->b_bufsize != size)
2300 		goto done;
2301 
2302 	/*
2303 	 * Check to see if it is in the expected place on disk and that the
2304 	 * block has been mapped.
2305 	 */
2306 	if ((bpa->b_blkno != bpa->b_lblkno) && (bpa->b_blkno == blkno))
2307 		match = 1;
2308 done:
2309 	BUF_UNLOCK(bpa);
2310 	return (match);
2311 }
2312 
2313 /*
2314  *	vfs_bio_awrite:
2315  *
2316  *	Implement clustered async writes for clearing out B_DELWRI buffers.
2317  *	This is much better then the old way of writing only one buffer at
2318  *	a time.  Note that we may not be presented with the buffers in the
2319  *	correct order, so we search for the cluster in both directions.
2320  */
2321 int
2322 vfs_bio_awrite(struct buf *bp)
2323 {
2324 	struct bufobj *bo;
2325 	int i;
2326 	int j;
2327 	daddr_t lblkno = bp->b_lblkno;
2328 	struct vnode *vp = bp->b_vp;
2329 	int ncl;
2330 	int nwritten;
2331 	int size;
2332 	int maxcl;
2333 	int gbflags;
2334 
2335 	bo = &vp->v_bufobj;
2336 	gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
2337 	/*
2338 	 * right now we support clustered writing only to regular files.  If
2339 	 * we find a clusterable block we could be in the middle of a cluster
2340 	 * rather then at the beginning.
2341 	 */
2342 	if ((vp->v_type == VREG) &&
2343 	    (vp->v_mount != 0) && /* Only on nodes that have the size info */
2344 	    (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
2345 
2346 		size = vp->v_mount->mnt_stat.f_iosize;
2347 		maxcl = MAXPHYS / size;
2348 
2349 		BO_RLOCK(bo);
2350 		for (i = 1; i < maxcl; i++)
2351 			if (vfs_bio_clcheck(vp, size, lblkno + i,
2352 			    bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0)
2353 				break;
2354 
2355 		for (j = 1; i + j <= maxcl && j <= lblkno; j++)
2356 			if (vfs_bio_clcheck(vp, size, lblkno - j,
2357 			    bp->b_blkno - ((j * size) >> DEV_BSHIFT)) == 0)
2358 				break;
2359 		BO_RUNLOCK(bo);
2360 		--j;
2361 		ncl = i + j;
2362 		/*
2363 		 * this is a possible cluster write
2364 		 */
2365 		if (ncl != 1) {
2366 			BUF_UNLOCK(bp);
2367 			nwritten = cluster_wbuild(vp, size, lblkno - j, ncl,
2368 			    gbflags);
2369 			return (nwritten);
2370 		}
2371 	}
2372 	bremfree(bp);
2373 	bp->b_flags |= B_ASYNC;
2374 	/*
2375 	 * default (old) behavior, writing out only one block
2376 	 *
2377 	 * XXX returns b_bufsize instead of b_bcount for nwritten?
2378 	 */
2379 	nwritten = bp->b_bufsize;
2380 	(void) bwrite(bp);
2381 
2382 	return (nwritten);
2383 }
2384 
2385 /*
2386  * Ask the bufdaemon for help, or act as bufdaemon itself, when a
2387  * locked vnode is supplied.
2388  */
2389 static void
2390 getnewbuf_bufd_help(struct vnode *vp, int gbflags, int slpflag, int slptimeo,
2391     int defrag)
2392 {
2393 	struct thread *td;
2394 	char *waitmsg;
2395 	int error, fl, flags, norunbuf;
2396 
2397 	mtx_assert(&bqclean, MA_OWNED);
2398 
2399 	if (defrag) {
2400 		flags = VFS_BIO_NEED_BUFSPACE;
2401 		waitmsg = "nbufkv";
2402 	} else if (bufspace >= hibufspace) {
2403 		waitmsg = "nbufbs";
2404 		flags = VFS_BIO_NEED_BUFSPACE;
2405 	} else {
2406 		waitmsg = "newbuf";
2407 		flags = VFS_BIO_NEED_ANY;
2408 	}
2409 	atomic_set_int(&needsbuffer, flags);
2410 	mtx_unlock(&bqclean);
2411 
2412 	bd_speedup();	/* heeeelp */
2413 	if ((gbflags & GB_NOWAIT_BD) != 0)
2414 		return;
2415 
2416 	td = curthread;
2417 	rw_wlock(&nblock);
2418 	while ((needsbuffer & flags) != 0) {
2419 		if (vp != NULL && vp->v_type != VCHR &&
2420 		    (td->td_pflags & TDP_BUFNEED) == 0) {
2421 			rw_wunlock(&nblock);
2422 			/*
2423 			 * getblk() is called with a vnode locked, and
2424 			 * some majority of the dirty buffers may as
2425 			 * well belong to the vnode.  Flushing the
2426 			 * buffers there would make a progress that
2427 			 * cannot be achieved by the buf_daemon, that
2428 			 * cannot lock the vnode.
2429 			 */
2430 			norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
2431 			    (td->td_pflags & TDP_NORUNNINGBUF);
2432 
2433 			/*
2434 			 * Play bufdaemon.  The getnewbuf() function
2435 			 * may be called while the thread owns lock
2436 			 * for another dirty buffer for the same
2437 			 * vnode, which makes it impossible to use
2438 			 * VOP_FSYNC() there, due to the buffer lock
2439 			 * recursion.
2440 			 */
2441 			td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
2442 			fl = buf_flush(vp, flushbufqtarget);
2443 			td->td_pflags &= norunbuf;
2444 			rw_wlock(&nblock);
2445 			if (fl != 0)
2446 				continue;
2447 			if ((needsbuffer & flags) == 0)
2448 				break;
2449 		}
2450 		error = rw_sleep(__DEVOLATILE(void *, &needsbuffer), &nblock,
2451 		    (PRIBIO + 4) | slpflag, waitmsg, slptimeo);
2452 		if (error != 0)
2453 			break;
2454 	}
2455 	rw_wunlock(&nblock);
2456 }
2457 
2458 static void
2459 getnewbuf_reuse_bp(struct buf *bp, int qindex)
2460 {
2461 
2462 	CTR6(KTR_BUF, "getnewbuf(%p) vp %p flags %X kvasize %d bufsize %d "
2463 	    "queue %d (recycling)", bp, bp->b_vp, bp->b_flags,
2464 	     bp->b_kvasize, bp->b_bufsize, qindex);
2465 	mtx_assert(&bqclean, MA_NOTOWNED);
2466 
2467 	/*
2468 	 * Note: we no longer distinguish between VMIO and non-VMIO
2469 	 * buffers.
2470 	 */
2471 	KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
2472 	    ("invalid buffer %p flags %#x found in queue %d", bp, bp->b_flags,
2473 	    qindex));
2474 
2475 	/*
2476 	 * When recycling a clean buffer we have to truncate it and
2477 	 * release the vnode.
2478 	 */
2479 	if (qindex == QUEUE_CLEAN) {
2480 		allocbuf(bp, 0);
2481 		if (bp->b_vp != NULL)
2482 			brelvp(bp);
2483 	}
2484 
2485 	/*
2486 	 * Get the rest of the buffer freed up.  b_kva* is still valid
2487 	 * after this operation.
2488 	 */
2489 	if (bp->b_rcred != NOCRED) {
2490 		crfree(bp->b_rcred);
2491 		bp->b_rcred = NOCRED;
2492 	}
2493 	if (bp->b_wcred != NOCRED) {
2494 		crfree(bp->b_wcred);
2495 		bp->b_wcred = NOCRED;
2496 	}
2497 	if (!LIST_EMPTY(&bp->b_dep))
2498 		buf_deallocate(bp);
2499 	if (bp->b_vflags & BV_BKGRDINPROG)
2500 		panic("losing buffer 3");
2501 	KASSERT(bp->b_vp == NULL, ("bp: %p still has vnode %p.  qindex: %d",
2502 	    bp, bp->b_vp, qindex));
2503 	KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
2504 	    ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
2505 	KASSERT(bp->b_npages == 0,
2506 	    ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
2507 
2508 	bp->b_flags = 0;
2509 	bp->b_ioflags = 0;
2510 	bp->b_xflags = 0;
2511 	KASSERT((bp->b_flags & B_INFREECNT) == 0,
2512 	    ("buf %p still counted as free?", bp));
2513 	bp->b_vflags = 0;
2514 	bp->b_vp = NULL;
2515 	bp->b_blkno = bp->b_lblkno = 0;
2516 	bp->b_offset = NOOFFSET;
2517 	bp->b_iodone = 0;
2518 	bp->b_error = 0;
2519 	bp->b_resid = 0;
2520 	bp->b_bcount = 0;
2521 	bp->b_npages = 0;
2522 	bp->b_dirtyoff = bp->b_dirtyend = 0;
2523 	bp->b_bufobj = NULL;
2524 	bp->b_pin_count = 0;
2525 	bp->b_data = bp->b_kvabase;
2526 	bp->b_fsprivate1 = NULL;
2527 	bp->b_fsprivate2 = NULL;
2528 	bp->b_fsprivate3 = NULL;
2529 
2530 	LIST_INIT(&bp->b_dep);
2531 }
2532 
2533 static struct buf *
2534 getnewbuf_scan(int maxsize, int defrag, int unmapped, int metadata)
2535 {
2536 	struct buf *bp, *nbp;
2537 	int nqindex, qindex, pass;
2538 
2539 	KASSERT(!unmapped || !defrag, ("both unmapped and defrag"));
2540 
2541 	pass = 0;
2542 restart:
2543 	if (pass != 0)
2544 		atomic_add_int(&getnewbufrestarts, 1);
2545 
2546 	nbp = NULL;
2547 	mtx_lock(&bqclean);
2548 	/*
2549 	 * If we're not defragging or low on bufspace attempt to make a new
2550 	 * buf from a header.
2551 	 */
2552 	if (defrag == 0 && bufspace + maxsize < hibufspace) {
2553 		nqindex = QUEUE_EMPTY;
2554 		nbp = TAILQ_FIRST(&bufqueues[nqindex]);
2555 	}
2556 	/*
2557 	 * All available buffers might be clean or we need to start recycling.
2558 	 */
2559 	if (nbp == NULL) {
2560 		nqindex = QUEUE_CLEAN;
2561 		nbp = TAILQ_FIRST(&bufqueues[QUEUE_CLEAN]);
2562 	}
2563 
2564 	/*
2565 	 * Run scan, possibly freeing data and/or kva mappings on the fly
2566 	 * depending.
2567 	 */
2568 	while ((bp = nbp) != NULL) {
2569 		qindex = nqindex;
2570 
2571 		/*
2572 		 * Calculate next bp (we can only use it if we do not
2573 		 * release the bqlock)
2574 		 */
2575 		if ((nbp = TAILQ_NEXT(bp, b_freelist)) == NULL) {
2576 			switch (qindex) {
2577 			case QUEUE_EMPTY:
2578 				nqindex = QUEUE_CLEAN;
2579 				nbp = TAILQ_FIRST(&bufqueues[nqindex]);
2580 				if (nbp != NULL)
2581 					break;
2582 				/* FALLTHROUGH */
2583 			case QUEUE_CLEAN:
2584 				if (metadata && pass == 0) {
2585 					pass = 1;
2586 					nqindex = QUEUE_EMPTY;
2587 					nbp = TAILQ_FIRST(&bufqueues[nqindex]);
2588 				}
2589 				/*
2590 				 * nbp is NULL.
2591 				 */
2592 				break;
2593 			}
2594 		}
2595 		/*
2596 		 * If we are defragging then we need a buffer with
2597 		 * b_kvasize != 0.  This situation occurs when we
2598 		 * have many unmapped bufs.
2599 		 */
2600 		if (defrag && bp->b_kvasize == 0)
2601 			continue;
2602 
2603 		/*
2604 		 * Start freeing the bp.  This is somewhat involved.  nbp
2605 		 * remains valid only for QUEUE_EMPTY[KVA] bp's.
2606 		 */
2607 		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
2608 			continue;
2609 		/*
2610 		 * BKGRDINPROG can only be set with the buf and bufobj
2611 		 * locks both held.  We tolerate a race to clear it here.
2612 		 */
2613 		if (bp->b_vflags & BV_BKGRDINPROG) {
2614 			BUF_UNLOCK(bp);
2615 			continue;
2616 		}
2617 
2618 		/*
2619 		 * Requeue the background write buffer with error.
2620 		 */
2621 		if ((bp->b_vflags & BV_BKGRDERR) != 0) {
2622 			bremfreel(bp);
2623 			mtx_unlock(&bqclean);
2624 			bqrelse(bp);
2625 			continue;
2626 		}
2627 
2628 		KASSERT(bp->b_qindex == qindex,
2629 		    ("getnewbuf: inconsistent queue %d bp %p", qindex, bp));
2630 
2631 		bremfreel(bp);
2632 		mtx_unlock(&bqclean);
2633 
2634 		/*
2635 		 * NOTE:  nbp is now entirely invalid.  We can only restart
2636 		 * the scan from this point on.
2637 		 */
2638 		getnewbuf_reuse_bp(bp, qindex);
2639 		mtx_assert(&bqclean, MA_NOTOWNED);
2640 
2641 		/*
2642 		 * If we are defragging then free the buffer.
2643 		 */
2644 		if (defrag) {
2645 			bp->b_flags |= B_INVAL;
2646 			brelse(bp);
2647 			defrag = 0;
2648 			goto restart;
2649 		}
2650 
2651 		/*
2652 		 * Notify any waiters for the buffer lock about
2653 		 * identity change by freeing the buffer.
2654 		 */
2655 		if (qindex == QUEUE_CLEAN && BUF_LOCKWAITERS(bp)) {
2656 			bp->b_flags |= B_INVAL;
2657 			brelse(bp);
2658 			goto restart;
2659 		}
2660 
2661 		if (metadata)
2662 			break;
2663 
2664 		/*
2665 		 * If we are overcomitted then recover the buffer and its
2666 		 * KVM space.  This occurs in rare situations when multiple
2667 		 * processes are blocked in getnewbuf() or allocbuf().
2668 		 */
2669 		if (bufspace >= hibufspace && bp->b_kvasize != 0) {
2670 			bp->b_flags |= B_INVAL;
2671 			brelse(bp);
2672 			goto restart;
2673 		}
2674 		break;
2675 	}
2676 	return (bp);
2677 }
2678 
2679 /*
2680  *	getnewbuf:
2681  *
2682  *	Find and initialize a new buffer header, freeing up existing buffers
2683  *	in the bufqueues as necessary.  The new buffer is returned locked.
2684  *
2685  *	Important:  B_INVAL is not set.  If the caller wishes to throw the
2686  *	buffer away, the caller must set B_INVAL prior to calling brelse().
2687  *
2688  *	We block if:
2689  *		We have insufficient buffer headers
2690  *		We have insufficient buffer space
2691  *		buffer_arena is too fragmented ( space reservation fails )
2692  *		If we have to flush dirty buffers ( but we try to avoid this )
2693  */
2694 static struct buf *
2695 getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int size, int maxsize,
2696     int gbflags)
2697 {
2698 	struct buf *bp;
2699 	int defrag, metadata;
2700 
2701 	KASSERT((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
2702 	    ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
2703 	if (!unmapped_buf_allowed)
2704 		gbflags &= ~(GB_UNMAPPED | GB_KVAALLOC);
2705 
2706 	defrag = 0;
2707 	if (vp == NULL || (vp->v_vflag & (VV_MD | VV_SYSTEM)) != 0 ||
2708 	    vp->v_type == VCHR)
2709 		metadata = 1;
2710 	else
2711 		metadata = 0;
2712 	/*
2713 	 * We can't afford to block since we might be holding a vnode lock,
2714 	 * which may prevent system daemons from running.  We deal with
2715 	 * low-memory situations by proactively returning memory and running
2716 	 * async I/O rather then sync I/O.
2717 	 */
2718 	atomic_add_int(&getnewbufcalls, 1);
2719 restart:
2720 	bp = getnewbuf_scan(maxsize, defrag, (gbflags & (GB_UNMAPPED |
2721 	    GB_KVAALLOC)) == GB_UNMAPPED, metadata);
2722 	if (bp != NULL)
2723 		defrag = 0;
2724 
2725 	/*
2726 	 * If we exhausted our list, sleep as appropriate.  We may have to
2727 	 * wakeup various daemons and write out some dirty buffers.
2728 	 *
2729 	 * Generally we are sleeping due to insufficient buffer space.
2730 	 */
2731 	if (bp == NULL) {
2732 		mtx_assert(&bqclean, MA_OWNED);
2733 		getnewbuf_bufd_help(vp, gbflags, slpflag, slptimeo, defrag);
2734 		mtx_assert(&bqclean, MA_NOTOWNED);
2735 	} else if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) == GB_UNMAPPED) {
2736 		mtx_assert(&bqclean, MA_NOTOWNED);
2737 
2738 		bufkvafree(bp);
2739 		atomic_add_int(&bufreusecnt, 1);
2740 	} else {
2741 		mtx_assert(&bqclean, MA_NOTOWNED);
2742 
2743 		/*
2744 		 * We finally have a valid bp.  We aren't quite out of the
2745 		 * woods, we still have to reserve kva space. In order to
2746 		 * keep fragmentation sane we only allocate kva in BKVASIZE
2747 		 * chunks.
2748 		 */
2749 		maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
2750 
2751 		if (maxsize != bp->b_kvasize &&
2752 		    bufkvaalloc(bp, maxsize, gbflags)) {
2753 			defrag = 1;
2754 			bp->b_flags |= B_INVAL;
2755 			brelse(bp);
2756 			goto restart;
2757 		} else if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) ==
2758 		    (GB_UNMAPPED | GB_KVAALLOC)) {
2759 			bp->b_data = unmapped_buf;
2760 			BUF_CHECK_UNMAPPED(bp);
2761 		}
2762 		atomic_add_int(&bufreusecnt, 1);
2763 	}
2764 	return (bp);
2765 }
2766 
2767 /*
2768  *	buf_daemon:
2769  *
2770  *	buffer flushing daemon.  Buffers are normally flushed by the
2771  *	update daemon but if it cannot keep up this process starts to
2772  *	take the load in an attempt to prevent getnewbuf() from blocking.
2773  */
2774 
2775 static struct kproc_desc buf_kp = {
2776 	"bufdaemon",
2777 	buf_daemon,
2778 	&bufdaemonproc
2779 };
2780 SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp);
2781 
2782 static int
2783 buf_flush(struct vnode *vp, int target)
2784 {
2785 	int flushed;
2786 
2787 	flushed = flushbufqueues(vp, target, 0);
2788 	if (flushed == 0) {
2789 		/*
2790 		 * Could not find any buffers without rollback
2791 		 * dependencies, so just write the first one
2792 		 * in the hopes of eventually making progress.
2793 		 */
2794 		if (vp != NULL && target > 2)
2795 			target /= 2;
2796 		flushbufqueues(vp, target, 1);
2797 	}
2798 	return (flushed);
2799 }
2800 
2801 static void
2802 buf_daemon()
2803 {
2804 	int lodirty;
2805 
2806 	/*
2807 	 * This process needs to be suspended prior to shutdown sync.
2808 	 */
2809 	EVENTHANDLER_REGISTER(shutdown_pre_sync, kproc_shutdown, bufdaemonproc,
2810 	    SHUTDOWN_PRI_LAST);
2811 
2812 	/*
2813 	 * This process is allowed to take the buffer cache to the limit
2814 	 */
2815 	curthread->td_pflags |= TDP_NORUNNINGBUF | TDP_BUFNEED;
2816 	mtx_lock(&bdlock);
2817 	for (;;) {
2818 		bd_request = 0;
2819 		mtx_unlock(&bdlock);
2820 
2821 		kproc_suspend_check(bufdaemonproc);
2822 		lodirty = lodirtybuffers;
2823 		if (bd_speedupreq) {
2824 			lodirty = numdirtybuffers / 2;
2825 			bd_speedupreq = 0;
2826 		}
2827 		/*
2828 		 * Do the flush.  Limit the amount of in-transit I/O we
2829 		 * allow to build up, otherwise we would completely saturate
2830 		 * the I/O system.
2831 		 */
2832 		while (numdirtybuffers > lodirty) {
2833 			if (buf_flush(NULL, numdirtybuffers - lodirty) == 0)
2834 				break;
2835 			kern_yield(PRI_USER);
2836 		}
2837 
2838 		/*
2839 		 * Only clear bd_request if we have reached our low water
2840 		 * mark.  The buf_daemon normally waits 1 second and
2841 		 * then incrementally flushes any dirty buffers that have
2842 		 * built up, within reason.
2843 		 *
2844 		 * If we were unable to hit our low water mark and couldn't
2845 		 * find any flushable buffers, we sleep for a short period
2846 		 * to avoid endless loops on unlockable buffers.
2847 		 */
2848 		mtx_lock(&bdlock);
2849 		if (numdirtybuffers <= lodirtybuffers) {
2850 			/*
2851 			 * We reached our low water mark, reset the
2852 			 * request and sleep until we are needed again.
2853 			 * The sleep is just so the suspend code works.
2854 			 */
2855 			bd_request = 0;
2856 			/*
2857 			 * Do an extra wakeup in case dirty threshold
2858 			 * changed via sysctl and the explicit transition
2859 			 * out of shortfall was missed.
2860 			 */
2861 			bdirtywakeup();
2862 			if (runningbufspace <= lorunningspace)
2863 				runningwakeup();
2864 			msleep(&bd_request, &bdlock, PVM, "psleep", hz);
2865 		} else {
2866 			/*
2867 			 * We couldn't find any flushable dirty buffers but
2868 			 * still have too many dirty buffers, we
2869 			 * have to sleep and try again.  (rare)
2870 			 */
2871 			msleep(&bd_request, &bdlock, PVM, "qsleep", hz / 10);
2872 		}
2873 	}
2874 }
2875 
2876 /*
2877  *	flushbufqueues:
2878  *
2879  *	Try to flush a buffer in the dirty queue.  We must be careful to
2880  *	free up B_INVAL buffers instead of write them, which NFS is
2881  *	particularly sensitive to.
2882  */
2883 static int flushwithdeps = 0;
2884 SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_RW, &flushwithdeps,
2885     0, "Number of buffers flushed with dependecies that require rollbacks");
2886 
2887 static int
2888 flushbufqueues(struct vnode *lvp, int target, int flushdeps)
2889 {
2890 	struct buf *sentinel;
2891 	struct vnode *vp;
2892 	struct mount *mp;
2893 	struct buf *bp;
2894 	int hasdeps;
2895 	int flushed;
2896 	int queue;
2897 	int error;
2898 	bool unlock;
2899 
2900 	flushed = 0;
2901 	queue = QUEUE_DIRTY;
2902 	bp = NULL;
2903 	sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK | M_ZERO);
2904 	sentinel->b_qindex = QUEUE_SENTINEL;
2905 	mtx_lock(&bqdirty);
2906 	TAILQ_INSERT_HEAD(&bufqueues[queue], sentinel, b_freelist);
2907 	mtx_unlock(&bqdirty);
2908 	while (flushed != target) {
2909 		maybe_yield();
2910 		mtx_lock(&bqdirty);
2911 		bp = TAILQ_NEXT(sentinel, b_freelist);
2912 		if (bp != NULL) {
2913 			TAILQ_REMOVE(&bufqueues[queue], sentinel, b_freelist);
2914 			TAILQ_INSERT_AFTER(&bufqueues[queue], bp, sentinel,
2915 			    b_freelist);
2916 		} else {
2917 			mtx_unlock(&bqdirty);
2918 			break;
2919 		}
2920 		/*
2921 		 * Skip sentinels inserted by other invocations of the
2922 		 * flushbufqueues(), taking care to not reorder them.
2923 		 *
2924 		 * Only flush the buffers that belong to the
2925 		 * vnode locked by the curthread.
2926 		 */
2927 		if (bp->b_qindex == QUEUE_SENTINEL || (lvp != NULL &&
2928 		    bp->b_vp != lvp)) {
2929 			mtx_unlock(&bqdirty);
2930  			continue;
2931 		}
2932 		error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL);
2933 		mtx_unlock(&bqdirty);
2934 		if (error != 0)
2935 			continue;
2936 		if (bp->b_pin_count > 0) {
2937 			BUF_UNLOCK(bp);
2938 			continue;
2939 		}
2940 		/*
2941 		 * BKGRDINPROG can only be set with the buf and bufobj
2942 		 * locks both held.  We tolerate a race to clear it here.
2943 		 */
2944 		if ((bp->b_vflags & BV_BKGRDINPROG) != 0 ||
2945 		    (bp->b_flags & B_DELWRI) == 0) {
2946 			BUF_UNLOCK(bp);
2947 			continue;
2948 		}
2949 		if (bp->b_flags & B_INVAL) {
2950 			bremfreef(bp);
2951 			brelse(bp);
2952 			flushed++;
2953 			continue;
2954 		}
2955 
2956 		if (!LIST_EMPTY(&bp->b_dep) && buf_countdeps(bp, 0)) {
2957 			if (flushdeps == 0) {
2958 				BUF_UNLOCK(bp);
2959 				continue;
2960 			}
2961 			hasdeps = 1;
2962 		} else
2963 			hasdeps = 0;
2964 		/*
2965 		 * We must hold the lock on a vnode before writing
2966 		 * one of its buffers. Otherwise we may confuse, or
2967 		 * in the case of a snapshot vnode, deadlock the
2968 		 * system.
2969 		 *
2970 		 * The lock order here is the reverse of the normal
2971 		 * of vnode followed by buf lock.  This is ok because
2972 		 * the NOWAIT will prevent deadlock.
2973 		 */
2974 		vp = bp->b_vp;
2975 		if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
2976 			BUF_UNLOCK(bp);
2977 			continue;
2978 		}
2979 		if (lvp == NULL) {
2980 			unlock = true;
2981 			error = vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT);
2982 		} else {
2983 			ASSERT_VOP_LOCKED(vp, "getbuf");
2984 			unlock = false;
2985 			error = VOP_ISLOCKED(vp) == LK_EXCLUSIVE ? 0 :
2986 			    vn_lock(vp, LK_TRYUPGRADE);
2987 		}
2988 		if (error == 0) {
2989 			CTR3(KTR_BUF, "flushbufqueue(%p) vp %p flags %X",
2990 			    bp, bp->b_vp, bp->b_flags);
2991 			if (curproc == bufdaemonproc) {
2992 				vfs_bio_awrite(bp);
2993 			} else {
2994 				bremfree(bp);
2995 				bwrite(bp);
2996 				notbufdflushes++;
2997 			}
2998 			vn_finished_write(mp);
2999 			if (unlock)
3000 				VOP_UNLOCK(vp, 0);
3001 			flushwithdeps += hasdeps;
3002 			flushed++;
3003 
3004 			/*
3005 			 * Sleeping on runningbufspace while holding
3006 			 * vnode lock leads to deadlock.
3007 			 */
3008 			if (curproc == bufdaemonproc &&
3009 			    runningbufspace > hirunningspace)
3010 				waitrunningbufspace();
3011 			continue;
3012 		}
3013 		vn_finished_write(mp);
3014 		BUF_UNLOCK(bp);
3015 	}
3016 	mtx_lock(&bqdirty);
3017 	TAILQ_REMOVE(&bufqueues[queue], sentinel, b_freelist);
3018 	mtx_unlock(&bqdirty);
3019 	free(sentinel, M_TEMP);
3020 	return (flushed);
3021 }
3022 
3023 /*
3024  * Check to see if a block is currently memory resident.
3025  */
3026 struct buf *
3027 incore(struct bufobj *bo, daddr_t blkno)
3028 {
3029 	struct buf *bp;
3030 
3031 	BO_RLOCK(bo);
3032 	bp = gbincore(bo, blkno);
3033 	BO_RUNLOCK(bo);
3034 	return (bp);
3035 }
3036 
3037 /*
3038  * Returns true if no I/O is needed to access the
3039  * associated VM object.  This is like incore except
3040  * it also hunts around in the VM system for the data.
3041  */
3042 
3043 static int
3044 inmem(struct vnode * vp, daddr_t blkno)
3045 {
3046 	vm_object_t obj;
3047 	vm_offset_t toff, tinc, size;
3048 	vm_page_t m;
3049 	vm_ooffset_t off;
3050 
3051 	ASSERT_VOP_LOCKED(vp, "inmem");
3052 
3053 	if (incore(&vp->v_bufobj, blkno))
3054 		return 1;
3055 	if (vp->v_mount == NULL)
3056 		return 0;
3057 	obj = vp->v_object;
3058 	if (obj == NULL)
3059 		return (0);
3060 
3061 	size = PAGE_SIZE;
3062 	if (size > vp->v_mount->mnt_stat.f_iosize)
3063 		size = vp->v_mount->mnt_stat.f_iosize;
3064 	off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
3065 
3066 	VM_OBJECT_RLOCK(obj);
3067 	for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
3068 		m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
3069 		if (!m)
3070 			goto notinmem;
3071 		tinc = size;
3072 		if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
3073 			tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
3074 		if (vm_page_is_valid(m,
3075 		    (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
3076 			goto notinmem;
3077 	}
3078 	VM_OBJECT_RUNLOCK(obj);
3079 	return 1;
3080 
3081 notinmem:
3082 	VM_OBJECT_RUNLOCK(obj);
3083 	return (0);
3084 }
3085 
3086 /*
3087  * Set the dirty range for a buffer based on the status of the dirty
3088  * bits in the pages comprising the buffer.  The range is limited
3089  * to the size of the buffer.
3090  *
3091  * Tell the VM system that the pages associated with this buffer
3092  * are clean.  This is used for delayed writes where the data is
3093  * going to go to disk eventually without additional VM intevention.
3094  *
3095  * Note that while we only really need to clean through to b_bcount, we
3096  * just go ahead and clean through to b_bufsize.
3097  */
3098 static void
3099 vfs_clean_pages_dirty_buf(struct buf *bp)
3100 {
3101 	vm_ooffset_t foff, noff, eoff;
3102 	vm_page_t m;
3103 	int i;
3104 
3105 	if ((bp->b_flags & B_VMIO) == 0 || bp->b_bufsize == 0)
3106 		return;
3107 
3108 	foff = bp->b_offset;
3109 	KASSERT(bp->b_offset != NOOFFSET,
3110 	    ("vfs_clean_pages_dirty_buf: no buffer offset"));
3111 
3112 	VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
3113 	vfs_drain_busy_pages(bp);
3114 	vfs_setdirty_locked_object(bp);
3115 	for (i = 0; i < bp->b_npages; i++) {
3116 		noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
3117 		eoff = noff;
3118 		if (eoff > bp->b_offset + bp->b_bufsize)
3119 			eoff = bp->b_offset + bp->b_bufsize;
3120 		m = bp->b_pages[i];
3121 		vfs_page_set_validclean(bp, foff, m);
3122 		/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
3123 		foff = noff;
3124 	}
3125 	VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
3126 }
3127 
3128 static void
3129 vfs_setdirty_locked_object(struct buf *bp)
3130 {
3131 	vm_object_t object;
3132 	int i;
3133 
3134 	object = bp->b_bufobj->bo_object;
3135 	VM_OBJECT_ASSERT_WLOCKED(object);
3136 
3137 	/*
3138 	 * We qualify the scan for modified pages on whether the
3139 	 * object has been flushed yet.
3140 	 */
3141 	if ((object->flags & OBJ_MIGHTBEDIRTY) != 0) {
3142 		vm_offset_t boffset;
3143 		vm_offset_t eoffset;
3144 
3145 		/*
3146 		 * test the pages to see if they have been modified directly
3147 		 * by users through the VM system.
3148 		 */
3149 		for (i = 0; i < bp->b_npages; i++)
3150 			vm_page_test_dirty(bp->b_pages[i]);
3151 
3152 		/*
3153 		 * Calculate the encompassing dirty range, boffset and eoffset,
3154 		 * (eoffset - boffset) bytes.
3155 		 */
3156 
3157 		for (i = 0; i < bp->b_npages; i++) {
3158 			if (bp->b_pages[i]->dirty)
3159 				break;
3160 		}
3161 		boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3162 
3163 		for (i = bp->b_npages - 1; i >= 0; --i) {
3164 			if (bp->b_pages[i]->dirty) {
3165 				break;
3166 			}
3167 		}
3168 		eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3169 
3170 		/*
3171 		 * Fit it to the buffer.
3172 		 */
3173 
3174 		if (eoffset > bp->b_bcount)
3175 			eoffset = bp->b_bcount;
3176 
3177 		/*
3178 		 * If we have a good dirty range, merge with the existing
3179 		 * dirty range.
3180 		 */
3181 
3182 		if (boffset < eoffset) {
3183 			if (bp->b_dirtyoff > boffset)
3184 				bp->b_dirtyoff = boffset;
3185 			if (bp->b_dirtyend < eoffset)
3186 				bp->b_dirtyend = eoffset;
3187 		}
3188 	}
3189 }
3190 
3191 /*
3192  * Allocate the KVA mapping for an existing buffer.
3193  * If an unmapped buffer is provided but a mapped buffer is requested, take
3194  * also care to properly setup mappings between pages and KVA.
3195  */
3196 static void
3197 bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
3198 {
3199 	struct buf *scratch_bp;
3200 	int bsize, maxsize, need_mapping, need_kva;
3201 	off_t offset;
3202 
3203 	need_mapping = bp->b_data == unmapped_buf &&
3204 	    (gbflags & GB_UNMAPPED) == 0;
3205 	need_kva = bp->b_kvabase == unmapped_buf &&
3206 	    bp->b_data == unmapped_buf &&
3207 	    (gbflags & GB_KVAALLOC) != 0;
3208 	if (!need_mapping && !need_kva)
3209 		return;
3210 
3211 	BUF_CHECK_UNMAPPED(bp);
3212 
3213 	if (need_mapping && bp->b_kvabase != unmapped_buf) {
3214 		/*
3215 		 * Buffer is not mapped, but the KVA was already
3216 		 * reserved at the time of the instantiation.  Use the
3217 		 * allocated space.
3218 		 */
3219 		goto has_addr;
3220 	}
3221 
3222 	/*
3223 	 * Calculate the amount of the address space we would reserve
3224 	 * if the buffer was mapped.
3225 	 */
3226 	bsize = vn_isdisk(bp->b_vp, NULL) ? DEV_BSIZE : bp->b_bufobj->bo_bsize;
3227 	KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
3228 	offset = blkno * bsize;
3229 	maxsize = size + (offset & PAGE_MASK);
3230 	maxsize = imax(maxsize, bsize);
3231 
3232 mapping_loop:
3233 	if (bufkvaalloc(bp, maxsize, gbflags)) {
3234 		/*
3235 		 * Request defragmentation. getnewbuf() returns us the
3236 		 * allocated space by the scratch buffer KVA.
3237 		 */
3238 		scratch_bp = getnewbuf(bp->b_vp, 0, 0, size, maxsize, gbflags |
3239 		    (GB_UNMAPPED | GB_KVAALLOC));
3240 		if (scratch_bp == NULL) {
3241 			if ((gbflags & GB_NOWAIT_BD) != 0) {
3242 				/*
3243 				 * XXXKIB: defragmentation cannot
3244 				 * succeed, not sure what else to do.
3245 				 */
3246 				panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
3247 			}
3248 			atomic_add_int(&mappingrestarts, 1);
3249 			goto mapping_loop;
3250 		}
3251 		KASSERT(scratch_bp->b_kvabase != unmapped_buf,
3252 		    ("scratch bp has no KVA %p", scratch_bp));
3253 		/* Grab pointers. */
3254 		bp->b_kvabase = scratch_bp->b_kvabase;
3255 		bp->b_kvasize = scratch_bp->b_kvasize;
3256 		bp->b_data = scratch_bp->b_data;
3257 
3258 		/* Get rid of the scratch buffer. */
3259 		scratch_bp->b_kvasize = 0;
3260 		scratch_bp->b_flags |= B_INVAL;
3261 		scratch_bp->b_data = scratch_bp->b_kvabase = unmapped_buf;
3262 		brelse(scratch_bp);
3263 	}
3264 has_addr:
3265 	if (need_mapping) {
3266 		/* b_offset is handled by bpmap_qenter. */
3267 		bp->b_data = bp->b_kvabase;
3268 		BUF_CHECK_MAPPED(bp);
3269 		bpmap_qenter(bp);
3270 	}
3271 }
3272 
3273 /*
3274  *	getblk:
3275  *
3276  *	Get a block given a specified block and offset into a file/device.
3277  *	The buffers B_DONE bit will be cleared on return, making it almost
3278  * 	ready for an I/O initiation.  B_INVAL may or may not be set on
3279  *	return.  The caller should clear B_INVAL prior to initiating a
3280  *	READ.
3281  *
3282  *	For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
3283  *	an existing buffer.
3284  *
3285  *	For a VMIO buffer, B_CACHE is modified according to the backing VM.
3286  *	If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
3287  *	and then cleared based on the backing VM.  If the previous buffer is
3288  *	non-0-sized but invalid, B_CACHE will be cleared.
3289  *
3290  *	If getblk() must create a new buffer, the new buffer is returned with
3291  *	both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
3292  *	case it is returned with B_INVAL clear and B_CACHE set based on the
3293  *	backing VM.
3294  *
3295  *	getblk() also forces a bwrite() for any B_DELWRI buffer whos
3296  *	B_CACHE bit is clear.
3297  *
3298  *	What this means, basically, is that the caller should use B_CACHE to
3299  *	determine whether the buffer is fully valid or not and should clear
3300  *	B_INVAL prior to issuing a read.  If the caller intends to validate
3301  *	the buffer by loading its data area with something, the caller needs
3302  *	to clear B_INVAL.  If the caller does this without issuing an I/O,
3303  *	the caller should set B_CACHE ( as an optimization ), else the caller
3304  *	should issue the I/O and biodone() will set B_CACHE if the I/O was
3305  *	a write attempt or if it was a successfull read.  If the caller
3306  *	intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
3307  *	prior to issuing the READ.  biodone() will *not* clear B_INVAL.
3308  */
3309 struct buf *
3310 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo,
3311     int flags)
3312 {
3313 	struct buf *bp;
3314 	struct bufobj *bo;
3315 	int bsize, error, maxsize, vmio;
3316 	off_t offset;
3317 
3318 	CTR3(KTR_BUF, "getblk(%p, %ld, %d)", vp, (long)blkno, size);
3319 	KASSERT((flags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
3320 	    ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
3321 	ASSERT_VOP_LOCKED(vp, "getblk");
3322 	if (size > MAXBCACHEBUF)
3323 		panic("getblk: size(%d) > MAXBCACHEBUF(%d)\n", size,
3324 		    MAXBCACHEBUF);
3325 	if (!unmapped_buf_allowed)
3326 		flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
3327 
3328 	bo = &vp->v_bufobj;
3329 loop:
3330 	BO_RLOCK(bo);
3331 	bp = gbincore(bo, blkno);
3332 	if (bp != NULL) {
3333 		int lockflags;
3334 		/*
3335 		 * Buffer is in-core.  If the buffer is not busy nor managed,
3336 		 * it must be on a queue.
3337 		 */
3338 		lockflags = LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK;
3339 
3340 		if (flags & GB_LOCK_NOWAIT)
3341 			lockflags |= LK_NOWAIT;
3342 
3343 		error = BUF_TIMELOCK(bp, lockflags,
3344 		    BO_LOCKPTR(bo), "getblk", slpflag, slptimeo);
3345 
3346 		/*
3347 		 * If we slept and got the lock we have to restart in case
3348 		 * the buffer changed identities.
3349 		 */
3350 		if (error == ENOLCK)
3351 			goto loop;
3352 		/* We timed out or were interrupted. */
3353 		else if (error)
3354 			return (NULL);
3355 		/* If recursed, assume caller knows the rules. */
3356 		else if (BUF_LOCKRECURSED(bp))
3357 			goto end;
3358 
3359 		/*
3360 		 * The buffer is locked.  B_CACHE is cleared if the buffer is
3361 		 * invalid.  Otherwise, for a non-VMIO buffer, B_CACHE is set
3362 		 * and for a VMIO buffer B_CACHE is adjusted according to the
3363 		 * backing VM cache.
3364 		 */
3365 		if (bp->b_flags & B_INVAL)
3366 			bp->b_flags &= ~B_CACHE;
3367 		else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
3368 			bp->b_flags |= B_CACHE;
3369 		if (bp->b_flags & B_MANAGED)
3370 			MPASS(bp->b_qindex == QUEUE_NONE);
3371 		else
3372 			bremfree(bp);
3373 
3374 		/*
3375 		 * check for size inconsistencies for non-VMIO case.
3376 		 */
3377 		if (bp->b_bcount != size) {
3378 			if ((bp->b_flags & B_VMIO) == 0 ||
3379 			    (size > bp->b_kvasize)) {
3380 				if (bp->b_flags & B_DELWRI) {
3381 					/*
3382 					 * If buffer is pinned and caller does
3383 					 * not want sleep  waiting for it to be
3384 					 * unpinned, bail out
3385 					 * */
3386 					if (bp->b_pin_count > 0) {
3387 						if (flags & GB_LOCK_NOWAIT) {
3388 							bqrelse(bp);
3389 							return (NULL);
3390 						} else {
3391 							bunpin_wait(bp);
3392 						}
3393 					}
3394 					bp->b_flags |= B_NOCACHE;
3395 					bwrite(bp);
3396 				} else {
3397 					if (LIST_EMPTY(&bp->b_dep)) {
3398 						bp->b_flags |= B_RELBUF;
3399 						brelse(bp);
3400 					} else {
3401 						bp->b_flags |= B_NOCACHE;
3402 						bwrite(bp);
3403 					}
3404 				}
3405 				goto loop;
3406 			}
3407 		}
3408 
3409 		/*
3410 		 * Handle the case of unmapped buffer which should
3411 		 * become mapped, or the buffer for which KVA
3412 		 * reservation is requested.
3413 		 */
3414 		bp_unmapped_get_kva(bp, blkno, size, flags);
3415 
3416 		/*
3417 		 * If the size is inconsistant in the VMIO case, we can resize
3418 		 * the buffer.  This might lead to B_CACHE getting set or
3419 		 * cleared.  If the size has not changed, B_CACHE remains
3420 		 * unchanged from its previous state.
3421 		 */
3422 		allocbuf(bp, size);
3423 
3424 		KASSERT(bp->b_offset != NOOFFSET,
3425 		    ("getblk: no buffer offset"));
3426 
3427 		/*
3428 		 * A buffer with B_DELWRI set and B_CACHE clear must
3429 		 * be committed before we can return the buffer in
3430 		 * order to prevent the caller from issuing a read
3431 		 * ( due to B_CACHE not being set ) and overwriting
3432 		 * it.
3433 		 *
3434 		 * Most callers, including NFS and FFS, need this to
3435 		 * operate properly either because they assume they
3436 		 * can issue a read if B_CACHE is not set, or because
3437 		 * ( for example ) an uncached B_DELWRI might loop due
3438 		 * to softupdates re-dirtying the buffer.  In the latter
3439 		 * case, B_CACHE is set after the first write completes,
3440 		 * preventing further loops.
3441 		 * NOTE!  b*write() sets B_CACHE.  If we cleared B_CACHE
3442 		 * above while extending the buffer, we cannot allow the
3443 		 * buffer to remain with B_CACHE set after the write
3444 		 * completes or it will represent a corrupt state.  To
3445 		 * deal with this we set B_NOCACHE to scrap the buffer
3446 		 * after the write.
3447 		 *
3448 		 * We might be able to do something fancy, like setting
3449 		 * B_CACHE in bwrite() except if B_DELWRI is already set,
3450 		 * so the below call doesn't set B_CACHE, but that gets real
3451 		 * confusing.  This is much easier.
3452 		 */
3453 
3454 		if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
3455 			bp->b_flags |= B_NOCACHE;
3456 			bwrite(bp);
3457 			goto loop;
3458 		}
3459 		bp->b_flags &= ~B_DONE;
3460 	} else {
3461 		/*
3462 		 * Buffer is not in-core, create new buffer.  The buffer
3463 		 * returned by getnewbuf() is locked.  Note that the returned
3464 		 * buffer is also considered valid (not marked B_INVAL).
3465 		 */
3466 		BO_RUNLOCK(bo);
3467 		/*
3468 		 * If the user does not want us to create the buffer, bail out
3469 		 * here.
3470 		 */
3471 		if (flags & GB_NOCREAT)
3472 			return NULL;
3473 		if (numfreebuffers == 0 && TD_IS_IDLETHREAD(curthread))
3474 			return NULL;
3475 
3476 		bsize = vn_isdisk(vp, NULL) ? DEV_BSIZE : bo->bo_bsize;
3477 		KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
3478 		offset = blkno * bsize;
3479 		vmio = vp->v_object != NULL;
3480 		if (vmio) {
3481 			maxsize = size + (offset & PAGE_MASK);
3482 		} else {
3483 			maxsize = size;
3484 			/* Do not allow non-VMIO notmapped buffers. */
3485 			flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
3486 		}
3487 		maxsize = imax(maxsize, bsize);
3488 
3489 		bp = getnewbuf(vp, slpflag, slptimeo, size, maxsize, flags);
3490 		if (bp == NULL) {
3491 			if (slpflag || slptimeo)
3492 				return NULL;
3493 			goto loop;
3494 		}
3495 
3496 		/*
3497 		 * This code is used to make sure that a buffer is not
3498 		 * created while the getnewbuf routine is blocked.
3499 		 * This can be a problem whether the vnode is locked or not.
3500 		 * If the buffer is created out from under us, we have to
3501 		 * throw away the one we just created.
3502 		 *
3503 		 * Note: this must occur before we associate the buffer
3504 		 * with the vp especially considering limitations in
3505 		 * the splay tree implementation when dealing with duplicate
3506 		 * lblkno's.
3507 		 */
3508 		BO_LOCK(bo);
3509 		if (gbincore(bo, blkno)) {
3510 			BO_UNLOCK(bo);
3511 			bp->b_flags |= B_INVAL;
3512 			brelse(bp);
3513 			goto loop;
3514 		}
3515 
3516 		/*
3517 		 * Insert the buffer into the hash, so that it can
3518 		 * be found by incore.
3519 		 */
3520 		bp->b_blkno = bp->b_lblkno = blkno;
3521 		bp->b_offset = offset;
3522 		bgetvp(vp, bp);
3523 		BO_UNLOCK(bo);
3524 
3525 		/*
3526 		 * set B_VMIO bit.  allocbuf() the buffer bigger.  Since the
3527 		 * buffer size starts out as 0, B_CACHE will be set by
3528 		 * allocbuf() for the VMIO case prior to it testing the
3529 		 * backing store for validity.
3530 		 */
3531 
3532 		if (vmio) {
3533 			bp->b_flags |= B_VMIO;
3534 			KASSERT(vp->v_object == bp->b_bufobj->bo_object,
3535 			    ("ARGH! different b_bufobj->bo_object %p %p %p\n",
3536 			    bp, vp->v_object, bp->b_bufobj->bo_object));
3537 		} else {
3538 			bp->b_flags &= ~B_VMIO;
3539 			KASSERT(bp->b_bufobj->bo_object == NULL,
3540 			    ("ARGH! has b_bufobj->bo_object %p %p\n",
3541 			    bp, bp->b_bufobj->bo_object));
3542 			BUF_CHECK_MAPPED(bp);
3543 		}
3544 
3545 		allocbuf(bp, size);
3546 		bp->b_flags &= ~B_DONE;
3547 	}
3548 	CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
3549 	BUF_ASSERT_HELD(bp);
3550 end:
3551 	KASSERT(bp->b_bufobj == bo,
3552 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
3553 	return (bp);
3554 }
3555 
3556 /*
3557  * Get an empty, disassociated buffer of given size.  The buffer is initially
3558  * set to B_INVAL.
3559  */
3560 struct buf *
3561 geteblk(int size, int flags)
3562 {
3563 	struct buf *bp;
3564 	int maxsize;
3565 
3566 	maxsize = (size + BKVAMASK) & ~BKVAMASK;
3567 	while ((bp = getnewbuf(NULL, 0, 0, size, maxsize, flags)) == NULL) {
3568 		if ((flags & GB_NOWAIT_BD) &&
3569 		    (curthread->td_pflags & TDP_BUFNEED) != 0)
3570 			return (NULL);
3571 	}
3572 	allocbuf(bp, size);
3573 	bp->b_flags |= B_INVAL;	/* b_dep cleared by getnewbuf() */
3574 	BUF_ASSERT_HELD(bp);
3575 	return (bp);
3576 }
3577 
3578 /*
3579  * Truncate the backing store for a non-vmio buffer.
3580  */
3581 static void
3582 vfs_nonvmio_truncate(struct buf *bp, int newbsize)
3583 {
3584 
3585 	if (bp->b_flags & B_MALLOC) {
3586 		/*
3587 		 * malloced buffers are not shrunk
3588 		 */
3589 		if (newbsize == 0) {
3590 			bufmallocadjust(bp, 0);
3591 			free(bp->b_data, M_BIOBUF);
3592 			bp->b_data = bp->b_kvabase;
3593 			bp->b_flags &= ~B_MALLOC;
3594 		}
3595 		return;
3596 	}
3597 	vm_hold_free_pages(bp, newbsize);
3598 	bufspaceadjust(bp, newbsize);
3599 }
3600 
3601 /*
3602  * Extend the backing for a non-VMIO buffer.
3603  */
3604 static void
3605 vfs_nonvmio_extend(struct buf *bp, int newbsize)
3606 {
3607 	caddr_t origbuf;
3608 	int origbufsize;
3609 
3610 	/*
3611 	 * We only use malloced memory on the first allocation.
3612 	 * and revert to page-allocated memory when the buffer
3613 	 * grows.
3614 	 *
3615 	 * There is a potential smp race here that could lead
3616 	 * to bufmallocspace slightly passing the max.  It
3617 	 * is probably extremely rare and not worth worrying
3618 	 * over.
3619 	 */
3620 	if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
3621 	    bufmallocspace < maxbufmallocspace) {
3622 		bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
3623 		bp->b_flags |= B_MALLOC;
3624 		bufmallocadjust(bp, newbsize);
3625 		return;
3626 	}
3627 
3628 	/*
3629 	 * If the buffer is growing on its other-than-first
3630 	 * allocation then we revert to the page-allocation
3631 	 * scheme.
3632 	 */
3633 	origbuf = NULL;
3634 	origbufsize = 0;
3635 	if (bp->b_flags & B_MALLOC) {
3636 		origbuf = bp->b_data;
3637 		origbufsize = bp->b_bufsize;
3638 		bp->b_data = bp->b_kvabase;
3639 		bufmallocadjust(bp, 0);
3640 		bp->b_flags &= ~B_MALLOC;
3641 		newbsize = round_page(newbsize);
3642 	}
3643 	vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
3644 	    (vm_offset_t) bp->b_data + newbsize);
3645 	if (origbuf != NULL) {
3646 		bcopy(origbuf, bp->b_data, origbufsize);
3647 		free(origbuf, M_BIOBUF);
3648 	}
3649 	bufspaceadjust(bp, newbsize);
3650 }
3651 
3652 /*
3653  * This code constitutes the buffer memory from either anonymous system
3654  * memory (in the case of non-VMIO operations) or from an associated
3655  * VM object (in the case of VMIO operations).  This code is able to
3656  * resize a buffer up or down.
3657  *
3658  * Note that this code is tricky, and has many complications to resolve
3659  * deadlock or inconsistant data situations.  Tread lightly!!!
3660  * There are B_CACHE and B_DELWRI interactions that must be dealt with by
3661  * the caller.  Calling this code willy nilly can result in the loss of data.
3662  *
3663  * allocbuf() only adjusts B_CACHE for VMIO buffers.  getblk() deals with
3664  * B_CACHE for the non-VMIO case.
3665  */
3666 int
3667 allocbuf(struct buf *bp, int size)
3668 {
3669 	int newbsize;
3670 
3671 	BUF_ASSERT_HELD(bp);
3672 
3673 	if (bp->b_bcount == size)
3674 		return (1);
3675 
3676 	if (bp->b_kvasize != 0 && bp->b_kvasize < size)
3677 		panic("allocbuf: buffer too small");
3678 
3679 	newbsize = (size + DEV_BSIZE - 1) & ~(DEV_BSIZE - 1);
3680 	if ((bp->b_flags & B_VMIO) == 0) {
3681 		if ((bp->b_flags & B_MALLOC) == 0)
3682 			newbsize = round_page(newbsize);
3683 		/*
3684 		 * Just get anonymous memory from the kernel.  Don't
3685 		 * mess with B_CACHE.
3686 		 */
3687 		if (newbsize < bp->b_bufsize)
3688 			vfs_nonvmio_truncate(bp, newbsize);
3689 		else if (newbsize > bp->b_bufsize)
3690 			vfs_nonvmio_extend(bp, newbsize);
3691 	} else {
3692 		int desiredpages;
3693 
3694 		desiredpages = (size == 0) ? 0 :
3695 		    num_pages((bp->b_offset & PAGE_MASK) + newbsize);
3696 
3697 		if (bp->b_flags & B_MALLOC)
3698 			panic("allocbuf: VMIO buffer can't be malloced");
3699 		/*
3700 		 * Set B_CACHE initially if buffer is 0 length or will become
3701 		 * 0-length.
3702 		 */
3703 		if (size == 0 || bp->b_bufsize == 0)
3704 			bp->b_flags |= B_CACHE;
3705 
3706 		if (newbsize < bp->b_bufsize)
3707 			vfs_vmio_truncate(bp, desiredpages);
3708 		/* XXX This looks as if it should be newbsize > b_bufsize */
3709 		else if (size > bp->b_bcount)
3710 			vfs_vmio_extend(bp, desiredpages, size);
3711 		bufspaceadjust(bp, newbsize);
3712 	}
3713 	bp->b_bcount = size;		/* requested buffer size. */
3714 	return (1);
3715 }
3716 
3717 extern int inflight_transient_maps;
3718 
3719 void
3720 biodone(struct bio *bp)
3721 {
3722 	struct mtx *mtxp;
3723 	void (*done)(struct bio *);
3724 	vm_offset_t start, end;
3725 
3726 	if ((bp->bio_flags & BIO_TRANSIENT_MAPPING) != 0) {
3727 		bp->bio_flags &= ~BIO_TRANSIENT_MAPPING;
3728 		bp->bio_flags |= BIO_UNMAPPED;
3729 		start = trunc_page((vm_offset_t)bp->bio_data);
3730 		end = round_page((vm_offset_t)bp->bio_data + bp->bio_length);
3731 		bp->bio_data = unmapped_buf;
3732 		pmap_qremove(start, OFF_TO_IDX(end - start));
3733 		vmem_free(transient_arena, start, end - start);
3734 		atomic_add_int(&inflight_transient_maps, -1);
3735 	}
3736 	done = bp->bio_done;
3737 	if (done == NULL) {
3738 		mtxp = mtx_pool_find(mtxpool_sleep, bp);
3739 		mtx_lock(mtxp);
3740 		bp->bio_flags |= BIO_DONE;
3741 		wakeup(bp);
3742 		mtx_unlock(mtxp);
3743 	} else {
3744 		bp->bio_flags |= BIO_DONE;
3745 		done(bp);
3746 	}
3747 }
3748 
3749 /*
3750  * Wait for a BIO to finish.
3751  */
3752 int
3753 biowait(struct bio *bp, const char *wchan)
3754 {
3755 	struct mtx *mtxp;
3756 
3757 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
3758 	mtx_lock(mtxp);
3759 	while ((bp->bio_flags & BIO_DONE) == 0)
3760 		msleep(bp, mtxp, PRIBIO, wchan, 0);
3761 	mtx_unlock(mtxp);
3762 	if (bp->bio_error != 0)
3763 		return (bp->bio_error);
3764 	if (!(bp->bio_flags & BIO_ERROR))
3765 		return (0);
3766 	return (EIO);
3767 }
3768 
3769 void
3770 biofinish(struct bio *bp, struct devstat *stat, int error)
3771 {
3772 
3773 	if (error) {
3774 		bp->bio_error = error;
3775 		bp->bio_flags |= BIO_ERROR;
3776 	}
3777 	if (stat != NULL)
3778 		devstat_end_transaction_bio(stat, bp);
3779 	biodone(bp);
3780 }
3781 
3782 /*
3783  *	bufwait:
3784  *
3785  *	Wait for buffer I/O completion, returning error status.  The buffer
3786  *	is left locked and B_DONE on return.  B_EINTR is converted into an EINTR
3787  *	error and cleared.
3788  */
3789 int
3790 bufwait(struct buf *bp)
3791 {
3792 	if (bp->b_iocmd == BIO_READ)
3793 		bwait(bp, PRIBIO, "biord");
3794 	else
3795 		bwait(bp, PRIBIO, "biowr");
3796 	if (bp->b_flags & B_EINTR) {
3797 		bp->b_flags &= ~B_EINTR;
3798 		return (EINTR);
3799 	}
3800 	if (bp->b_ioflags & BIO_ERROR) {
3801 		return (bp->b_error ? bp->b_error : EIO);
3802 	} else {
3803 		return (0);
3804 	}
3805 }
3806 
3807 /*
3808  *	bufdone:
3809  *
3810  *	Finish I/O on a buffer, optionally calling a completion function.
3811  *	This is usually called from an interrupt so process blocking is
3812  *	not allowed.
3813  *
3814  *	biodone is also responsible for setting B_CACHE in a B_VMIO bp.
3815  *	In a non-VMIO bp, B_CACHE will be set on the next getblk()
3816  *	assuming B_INVAL is clear.
3817  *
3818  *	For the VMIO case, we set B_CACHE if the op was a read and no
3819  *	read error occured, or if the op was a write.  B_CACHE is never
3820  *	set if the buffer is invalid or otherwise uncacheable.
3821  *
3822  *	biodone does not mess with B_INVAL, allowing the I/O routine or the
3823  *	initiator to leave B_INVAL set to brelse the buffer out of existance
3824  *	in the biodone routine.
3825  */
3826 void
3827 bufdone(struct buf *bp)
3828 {
3829 	struct bufobj *dropobj;
3830 	void    (*biodone)(struct buf *);
3831 
3832 	CTR3(KTR_BUF, "bufdone(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
3833 	dropobj = NULL;
3834 
3835 	KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
3836 	BUF_ASSERT_HELD(bp);
3837 
3838 	runningbufwakeup(bp);
3839 	if (bp->b_iocmd == BIO_WRITE)
3840 		dropobj = bp->b_bufobj;
3841 	/* call optional completion function if requested */
3842 	if (bp->b_iodone != NULL) {
3843 		biodone = bp->b_iodone;
3844 		bp->b_iodone = NULL;
3845 		(*biodone) (bp);
3846 		if (dropobj)
3847 			bufobj_wdrop(dropobj);
3848 		return;
3849 	}
3850 
3851 	bufdone_finish(bp);
3852 
3853 	if (dropobj)
3854 		bufobj_wdrop(dropobj);
3855 }
3856 
3857 void
3858 bufdone_finish(struct buf *bp)
3859 {
3860 	BUF_ASSERT_HELD(bp);
3861 
3862 	if (!LIST_EMPTY(&bp->b_dep))
3863 		buf_complete(bp);
3864 
3865 	if (bp->b_flags & B_VMIO) {
3866 		/*
3867 		 * Set B_CACHE if the op was a normal read and no error
3868 		 * occured.  B_CACHE is set for writes in the b*write()
3869 		 * routines.
3870 		 */
3871 		if (bp->b_iocmd == BIO_READ &&
3872 		    !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
3873 		    !(bp->b_ioflags & BIO_ERROR))
3874 			bp->b_flags |= B_CACHE;
3875 		vfs_vmio_iodone(bp);
3876 	}
3877 
3878 	/*
3879 	 * For asynchronous completions, release the buffer now. The brelse
3880 	 * will do a wakeup there if necessary - so no need to do a wakeup
3881 	 * here in the async case. The sync case always needs to do a wakeup.
3882 	 */
3883 	if (bp->b_flags & B_ASYNC) {
3884 		if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) ||
3885 		    (bp->b_ioflags & BIO_ERROR))
3886 			brelse(bp);
3887 		else
3888 			bqrelse(bp);
3889 	} else
3890 		bdone(bp);
3891 }
3892 
3893 /*
3894  * This routine is called in lieu of iodone in the case of
3895  * incomplete I/O.  This keeps the busy status for pages
3896  * consistant.
3897  */
3898 void
3899 vfs_unbusy_pages(struct buf *bp)
3900 {
3901 	int i;
3902 	vm_object_t obj;
3903 	vm_page_t m;
3904 
3905 	runningbufwakeup(bp);
3906 	if (!(bp->b_flags & B_VMIO))
3907 		return;
3908 
3909 	obj = bp->b_bufobj->bo_object;
3910 	VM_OBJECT_WLOCK(obj);
3911 	for (i = 0; i < bp->b_npages; i++) {
3912 		m = bp->b_pages[i];
3913 		if (m == bogus_page) {
3914 			m = vm_page_lookup(obj, OFF_TO_IDX(bp->b_offset) + i);
3915 			if (!m)
3916 				panic("vfs_unbusy_pages: page missing\n");
3917 			bp->b_pages[i] = m;
3918 			if (buf_mapped(bp)) {
3919 				BUF_CHECK_MAPPED(bp);
3920 				pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
3921 				    bp->b_pages, bp->b_npages);
3922 			} else
3923 				BUF_CHECK_UNMAPPED(bp);
3924 		}
3925 		vm_page_sunbusy(m);
3926 	}
3927 	vm_object_pip_wakeupn(obj, bp->b_npages);
3928 	VM_OBJECT_WUNLOCK(obj);
3929 }
3930 
3931 /*
3932  * vfs_page_set_valid:
3933  *
3934  *	Set the valid bits in a page based on the supplied offset.   The
3935  *	range is restricted to the buffer's size.
3936  *
3937  *	This routine is typically called after a read completes.
3938  */
3939 static void
3940 vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m)
3941 {
3942 	vm_ooffset_t eoff;
3943 
3944 	/*
3945 	 * Compute the end offset, eoff, such that [off, eoff) does not span a
3946 	 * page boundary and eoff is not greater than the end of the buffer.
3947 	 * The end of the buffer, in this case, is our file EOF, not the
3948 	 * allocation size of the buffer.
3949 	 */
3950 	eoff = (off + PAGE_SIZE) & ~(vm_ooffset_t)PAGE_MASK;
3951 	if (eoff > bp->b_offset + bp->b_bcount)
3952 		eoff = bp->b_offset + bp->b_bcount;
3953 
3954 	/*
3955 	 * Set valid range.  This is typically the entire buffer and thus the
3956 	 * entire page.
3957 	 */
3958 	if (eoff > off)
3959 		vm_page_set_valid_range(m, off & PAGE_MASK, eoff - off);
3960 }
3961 
3962 /*
3963  * vfs_page_set_validclean:
3964  *
3965  *	Set the valid bits and clear the dirty bits in a page based on the
3966  *	supplied offset.   The range is restricted to the buffer's size.
3967  */
3968 static void
3969 vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m)
3970 {
3971 	vm_ooffset_t soff, eoff;
3972 
3973 	/*
3974 	 * Start and end offsets in buffer.  eoff - soff may not cross a
3975 	 * page boundry or cross the end of the buffer.  The end of the
3976 	 * buffer, in this case, is our file EOF, not the allocation size
3977 	 * of the buffer.
3978 	 */
3979 	soff = off;
3980 	eoff = (off + PAGE_SIZE) & ~(off_t)PAGE_MASK;
3981 	if (eoff > bp->b_offset + bp->b_bcount)
3982 		eoff = bp->b_offset + bp->b_bcount;
3983 
3984 	/*
3985 	 * Set valid range.  This is typically the entire buffer and thus the
3986 	 * entire page.
3987 	 */
3988 	if (eoff > soff) {
3989 		vm_page_set_validclean(
3990 		    m,
3991 		   (vm_offset_t) (soff & PAGE_MASK),
3992 		   (vm_offset_t) (eoff - soff)
3993 		);
3994 	}
3995 }
3996 
3997 /*
3998  * Ensure that all buffer pages are not exclusive busied.  If any page is
3999  * exclusive busy, drain it.
4000  */
4001 void
4002 vfs_drain_busy_pages(struct buf *bp)
4003 {
4004 	vm_page_t m;
4005 	int i, last_busied;
4006 
4007 	VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object);
4008 	last_busied = 0;
4009 	for (i = 0; i < bp->b_npages; i++) {
4010 		m = bp->b_pages[i];
4011 		if (vm_page_xbusied(m)) {
4012 			for (; last_busied < i; last_busied++)
4013 				vm_page_sbusy(bp->b_pages[last_busied]);
4014 			while (vm_page_xbusied(m)) {
4015 				vm_page_lock(m);
4016 				VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
4017 				vm_page_busy_sleep(m, "vbpage");
4018 				VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
4019 			}
4020 		}
4021 	}
4022 	for (i = 0; i < last_busied; i++)
4023 		vm_page_sunbusy(bp->b_pages[i]);
4024 }
4025 
4026 /*
4027  * This routine is called before a device strategy routine.
4028  * It is used to tell the VM system that paging I/O is in
4029  * progress, and treat the pages associated with the buffer
4030  * almost as being exclusive busy.  Also the object paging_in_progress
4031  * flag is handled to make sure that the object doesn't become
4032  * inconsistant.
4033  *
4034  * Since I/O has not been initiated yet, certain buffer flags
4035  * such as BIO_ERROR or B_INVAL may be in an inconsistant state
4036  * and should be ignored.
4037  */
4038 void
4039 vfs_busy_pages(struct buf *bp, int clear_modify)
4040 {
4041 	int i, bogus;
4042 	vm_object_t obj;
4043 	vm_ooffset_t foff;
4044 	vm_page_t m;
4045 
4046 	if (!(bp->b_flags & B_VMIO))
4047 		return;
4048 
4049 	obj = bp->b_bufobj->bo_object;
4050 	foff = bp->b_offset;
4051 	KASSERT(bp->b_offset != NOOFFSET,
4052 	    ("vfs_busy_pages: no buffer offset"));
4053 	VM_OBJECT_WLOCK(obj);
4054 	vfs_drain_busy_pages(bp);
4055 	if (bp->b_bufsize != 0)
4056 		vfs_setdirty_locked_object(bp);
4057 	bogus = 0;
4058 	for (i = 0; i < bp->b_npages; i++) {
4059 		m = bp->b_pages[i];
4060 
4061 		if ((bp->b_flags & B_CLUSTER) == 0) {
4062 			vm_object_pip_add(obj, 1);
4063 			vm_page_sbusy(m);
4064 		}
4065 		/*
4066 		 * When readying a buffer for a read ( i.e
4067 		 * clear_modify == 0 ), it is important to do
4068 		 * bogus_page replacement for valid pages in
4069 		 * partially instantiated buffers.  Partially
4070 		 * instantiated buffers can, in turn, occur when
4071 		 * reconstituting a buffer from its VM backing store
4072 		 * base.  We only have to do this if B_CACHE is
4073 		 * clear ( which causes the I/O to occur in the
4074 		 * first place ).  The replacement prevents the read
4075 		 * I/O from overwriting potentially dirty VM-backed
4076 		 * pages.  XXX bogus page replacement is, uh, bogus.
4077 		 * It may not work properly with small-block devices.
4078 		 * We need to find a better way.
4079 		 */
4080 		if (clear_modify) {
4081 			pmap_remove_write(m);
4082 			vfs_page_set_validclean(bp, foff, m);
4083 		} else if (m->valid == VM_PAGE_BITS_ALL &&
4084 		    (bp->b_flags & B_CACHE) == 0) {
4085 			bp->b_pages[i] = bogus_page;
4086 			bogus++;
4087 		}
4088 		foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
4089 	}
4090 	VM_OBJECT_WUNLOCK(obj);
4091 	if (bogus && buf_mapped(bp)) {
4092 		BUF_CHECK_MAPPED(bp);
4093 		pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
4094 		    bp->b_pages, bp->b_npages);
4095 	}
4096 }
4097 
4098 /*
4099  *	vfs_bio_set_valid:
4100  *
4101  *	Set the range within the buffer to valid.  The range is
4102  *	relative to the beginning of the buffer, b_offset.  Note that
4103  *	b_offset itself may be offset from the beginning of the first
4104  *	page.
4105  */
4106 void
4107 vfs_bio_set_valid(struct buf *bp, int base, int size)
4108 {
4109 	int i, n;
4110 	vm_page_t m;
4111 
4112 	if (!(bp->b_flags & B_VMIO))
4113 		return;
4114 
4115 	/*
4116 	 * Fixup base to be relative to beginning of first page.
4117 	 * Set initial n to be the maximum number of bytes in the
4118 	 * first page that can be validated.
4119 	 */
4120 	base += (bp->b_offset & PAGE_MASK);
4121 	n = PAGE_SIZE - (base & PAGE_MASK);
4122 
4123 	VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
4124 	for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4125 		m = bp->b_pages[i];
4126 		if (n > size)
4127 			n = size;
4128 		vm_page_set_valid_range(m, base & PAGE_MASK, n);
4129 		base += n;
4130 		size -= n;
4131 		n = PAGE_SIZE;
4132 	}
4133 	VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
4134 }
4135 
4136 /*
4137  *	vfs_bio_clrbuf:
4138  *
4139  *	If the specified buffer is a non-VMIO buffer, clear the entire
4140  *	buffer.  If the specified buffer is a VMIO buffer, clear and
4141  *	validate only the previously invalid portions of the buffer.
4142  *	This routine essentially fakes an I/O, so we need to clear
4143  *	BIO_ERROR and B_INVAL.
4144  *
4145  *	Note that while we only theoretically need to clear through b_bcount,
4146  *	we go ahead and clear through b_bufsize.
4147  */
4148 void
4149 vfs_bio_clrbuf(struct buf *bp)
4150 {
4151 	int i, j, mask, sa, ea, slide;
4152 
4153 	if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) {
4154 		clrbuf(bp);
4155 		return;
4156 	}
4157 	bp->b_flags &= ~B_INVAL;
4158 	bp->b_ioflags &= ~BIO_ERROR;
4159 	VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
4160 	if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
4161 	    (bp->b_offset & PAGE_MASK) == 0) {
4162 		if (bp->b_pages[0] == bogus_page)
4163 			goto unlock;
4164 		mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
4165 		VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object);
4166 		if ((bp->b_pages[0]->valid & mask) == mask)
4167 			goto unlock;
4168 		if ((bp->b_pages[0]->valid & mask) == 0) {
4169 			pmap_zero_page_area(bp->b_pages[0], 0, bp->b_bufsize);
4170 			bp->b_pages[0]->valid |= mask;
4171 			goto unlock;
4172 		}
4173 	}
4174 	sa = bp->b_offset & PAGE_MASK;
4175 	slide = 0;
4176 	for (i = 0; i < bp->b_npages; i++, sa = 0) {
4177 		slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize);
4178 		ea = slide & PAGE_MASK;
4179 		if (ea == 0)
4180 			ea = PAGE_SIZE;
4181 		if (bp->b_pages[i] == bogus_page)
4182 			continue;
4183 		j = sa / DEV_BSIZE;
4184 		mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
4185 		VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[i]->object);
4186 		if ((bp->b_pages[i]->valid & mask) == mask)
4187 			continue;
4188 		if ((bp->b_pages[i]->valid & mask) == 0)
4189 			pmap_zero_page_area(bp->b_pages[i], sa, ea - sa);
4190 		else {
4191 			for (; sa < ea; sa += DEV_BSIZE, j++) {
4192 				if ((bp->b_pages[i]->valid & (1 << j)) == 0) {
4193 					pmap_zero_page_area(bp->b_pages[i],
4194 					    sa, DEV_BSIZE);
4195 				}
4196 			}
4197 		}
4198 		bp->b_pages[i]->valid |= mask;
4199 	}
4200 unlock:
4201 	VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
4202 	bp->b_resid = 0;
4203 }
4204 
4205 void
4206 vfs_bio_bzero_buf(struct buf *bp, int base, int size)
4207 {
4208 	vm_page_t m;
4209 	int i, n;
4210 
4211 	if (buf_mapped(bp)) {
4212 		BUF_CHECK_MAPPED(bp);
4213 		bzero(bp->b_data + base, size);
4214 	} else {
4215 		BUF_CHECK_UNMAPPED(bp);
4216 		n = PAGE_SIZE - (base & PAGE_MASK);
4217 		for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4218 			m = bp->b_pages[i];
4219 			if (n > size)
4220 				n = size;
4221 			pmap_zero_page_area(m, base & PAGE_MASK, n);
4222 			base += n;
4223 			size -= n;
4224 			n = PAGE_SIZE;
4225 		}
4226 	}
4227 }
4228 
4229 /*
4230  * vm_hold_load_pages and vm_hold_free_pages get pages into
4231  * a buffers address space.  The pages are anonymous and are
4232  * not associated with a file object.
4233  */
4234 static void
4235 vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
4236 {
4237 	vm_offset_t pg;
4238 	vm_page_t p;
4239 	int index;
4240 
4241 	BUF_CHECK_MAPPED(bp);
4242 
4243 	to = round_page(to);
4244 	from = round_page(from);
4245 	index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
4246 
4247 	for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
4248 tryagain:
4249 		/*
4250 		 * note: must allocate system pages since blocking here
4251 		 * could interfere with paging I/O, no matter which
4252 		 * process we are.
4253 		 */
4254 		p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
4255 		    VM_ALLOC_WIRED | VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT));
4256 		if (p == NULL) {
4257 			VM_WAIT;
4258 			goto tryagain;
4259 		}
4260 		pmap_qenter(pg, &p, 1);
4261 		bp->b_pages[index] = p;
4262 	}
4263 	bp->b_npages = index;
4264 }
4265 
4266 /* Return pages associated with this buf to the vm system */
4267 static void
4268 vm_hold_free_pages(struct buf *bp, int newbsize)
4269 {
4270 	vm_offset_t from;
4271 	vm_page_t p;
4272 	int index, newnpages;
4273 
4274 	BUF_CHECK_MAPPED(bp);
4275 
4276 	from = round_page((vm_offset_t)bp->b_data + newbsize);
4277 	newnpages = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
4278 	if (bp->b_npages > newnpages)
4279 		pmap_qremove(from, bp->b_npages - newnpages);
4280 	for (index = newnpages; index < bp->b_npages; index++) {
4281 		p = bp->b_pages[index];
4282 		bp->b_pages[index] = NULL;
4283 		if (vm_page_sbusied(p))
4284 			printf("vm_hold_free_pages: blkno: %jd, lblkno: %jd\n",
4285 			    (intmax_t)bp->b_blkno, (intmax_t)bp->b_lblkno);
4286 		p->wire_count--;
4287 		vm_page_free(p);
4288 		atomic_subtract_int(&vm_cnt.v_wire_count, 1);
4289 	}
4290 	bp->b_npages = newnpages;
4291 }
4292 
4293 /*
4294  * Map an IO request into kernel virtual address space.
4295  *
4296  * All requests are (re)mapped into kernel VA space.
4297  * Notice that we use b_bufsize for the size of the buffer
4298  * to be mapped.  b_bcount might be modified by the driver.
4299  *
4300  * Note that even if the caller determines that the address space should
4301  * be valid, a race or a smaller-file mapped into a larger space may
4302  * actually cause vmapbuf() to fail, so all callers of vmapbuf() MUST
4303  * check the return value.
4304  *
4305  * This function only works with pager buffers.
4306  */
4307 int
4308 vmapbuf(struct buf *bp, int mapbuf)
4309 {
4310 	vm_prot_t prot;
4311 	int pidx;
4312 
4313 	if (bp->b_bufsize < 0)
4314 		return (-1);
4315 	prot = VM_PROT_READ;
4316 	if (bp->b_iocmd == BIO_READ)
4317 		prot |= VM_PROT_WRITE;	/* Less backwards than it looks */
4318 	if ((pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
4319 	    (vm_offset_t)bp->b_data, bp->b_bufsize, prot, bp->b_pages,
4320 	    btoc(MAXPHYS))) < 0)
4321 		return (-1);
4322 	bp->b_npages = pidx;
4323 	bp->b_offset = ((vm_offset_t)bp->b_data) & PAGE_MASK;
4324 	if (mapbuf || !unmapped_buf_allowed) {
4325 		pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
4326 		bp->b_data = bp->b_kvabase + bp->b_offset;
4327 	} else
4328 		bp->b_data = unmapped_buf;
4329 	return(0);
4330 }
4331 
4332 /*
4333  * Free the io map PTEs associated with this IO operation.
4334  * We also invalidate the TLB entries and restore the original b_addr.
4335  *
4336  * This function only works with pager buffers.
4337  */
4338 void
4339 vunmapbuf(struct buf *bp)
4340 {
4341 	int npages;
4342 
4343 	npages = bp->b_npages;
4344 	if (buf_mapped(bp))
4345 		pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
4346 	vm_page_unhold_pages(bp->b_pages, npages);
4347 
4348 	bp->b_data = unmapped_buf;
4349 }
4350 
4351 void
4352 bdone(struct buf *bp)
4353 {
4354 	struct mtx *mtxp;
4355 
4356 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
4357 	mtx_lock(mtxp);
4358 	bp->b_flags |= B_DONE;
4359 	wakeup(bp);
4360 	mtx_unlock(mtxp);
4361 }
4362 
4363 void
4364 bwait(struct buf *bp, u_char pri, const char *wchan)
4365 {
4366 	struct mtx *mtxp;
4367 
4368 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
4369 	mtx_lock(mtxp);
4370 	while ((bp->b_flags & B_DONE) == 0)
4371 		msleep(bp, mtxp, pri, wchan, 0);
4372 	mtx_unlock(mtxp);
4373 }
4374 
4375 int
4376 bufsync(struct bufobj *bo, int waitfor)
4377 {
4378 
4379 	return (VOP_FSYNC(bo->__bo_vnode, waitfor, curthread));
4380 }
4381 
4382 void
4383 bufstrategy(struct bufobj *bo, struct buf *bp)
4384 {
4385 	int i = 0;
4386 	struct vnode *vp;
4387 
4388 	vp = bp->b_vp;
4389 	KASSERT(vp == bo->bo_private, ("Inconsistent vnode bufstrategy"));
4390 	KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
4391 	    ("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp));
4392 	i = VOP_STRATEGY(vp, bp);
4393 	KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp));
4394 }
4395 
4396 void
4397 bufobj_wrefl(struct bufobj *bo)
4398 {
4399 
4400 	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
4401 	ASSERT_BO_WLOCKED(bo);
4402 	bo->bo_numoutput++;
4403 }
4404 
4405 void
4406 bufobj_wref(struct bufobj *bo)
4407 {
4408 
4409 	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
4410 	BO_LOCK(bo);
4411 	bo->bo_numoutput++;
4412 	BO_UNLOCK(bo);
4413 }
4414 
4415 void
4416 bufobj_wdrop(struct bufobj *bo)
4417 {
4418 
4419 	KASSERT(bo != NULL, ("NULL bo in bufobj_wdrop"));
4420 	BO_LOCK(bo);
4421 	KASSERT(bo->bo_numoutput > 0, ("bufobj_wdrop non-positive count"));
4422 	if ((--bo->bo_numoutput == 0) && (bo->bo_flag & BO_WWAIT)) {
4423 		bo->bo_flag &= ~BO_WWAIT;
4424 		wakeup(&bo->bo_numoutput);
4425 	}
4426 	BO_UNLOCK(bo);
4427 }
4428 
4429 int
4430 bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
4431 {
4432 	int error;
4433 
4434 	KASSERT(bo != NULL, ("NULL bo in bufobj_wwait"));
4435 	ASSERT_BO_WLOCKED(bo);
4436 	error = 0;
4437 	while (bo->bo_numoutput) {
4438 		bo->bo_flag |= BO_WWAIT;
4439 		error = msleep(&bo->bo_numoutput, BO_LOCKPTR(bo),
4440 		    slpflag | (PRIBIO + 1), "bo_wwait", timeo);
4441 		if (error)
4442 			break;
4443 	}
4444 	return (error);
4445 }
4446 
4447 void
4448 bpin(struct buf *bp)
4449 {
4450 	struct mtx *mtxp;
4451 
4452 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
4453 	mtx_lock(mtxp);
4454 	bp->b_pin_count++;
4455 	mtx_unlock(mtxp);
4456 }
4457 
4458 void
4459 bunpin(struct buf *bp)
4460 {
4461 	struct mtx *mtxp;
4462 
4463 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
4464 	mtx_lock(mtxp);
4465 	if (--bp->b_pin_count == 0)
4466 		wakeup(bp);
4467 	mtx_unlock(mtxp);
4468 }
4469 
4470 void
4471 bunpin_wait(struct buf *bp)
4472 {
4473 	struct mtx *mtxp;
4474 
4475 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
4476 	mtx_lock(mtxp);
4477 	while (bp->b_pin_count > 0)
4478 		msleep(bp, mtxp, PRIBIO, "bwunpin", 0);
4479 	mtx_unlock(mtxp);
4480 }
4481 
4482 /*
4483  * Set bio_data or bio_ma for struct bio from the struct buf.
4484  */
4485 void
4486 bdata2bio(struct buf *bp, struct bio *bip)
4487 {
4488 
4489 	if (!buf_mapped(bp)) {
4490 		KASSERT(unmapped_buf_allowed, ("unmapped"));
4491 		bip->bio_ma = bp->b_pages;
4492 		bip->bio_ma_n = bp->b_npages;
4493 		bip->bio_data = unmapped_buf;
4494 		bip->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
4495 		bip->bio_flags |= BIO_UNMAPPED;
4496 		KASSERT(round_page(bip->bio_ma_offset + bip->bio_length) /
4497 		    PAGE_SIZE == bp->b_npages,
4498 		    ("Buffer %p too short: %d %lld %d", bp, bip->bio_ma_offset,
4499 		    (long long)bip->bio_length, bip->bio_ma_n));
4500 	} else {
4501 		bip->bio_data = bp->b_data;
4502 		bip->bio_ma = NULL;
4503 	}
4504 }
4505 
4506 #include "opt_ddb.h"
4507 #ifdef DDB
4508 #include <ddb/ddb.h>
4509 
4510 /* DDB command to show buffer data */
4511 DB_SHOW_COMMAND(buffer, db_show_buffer)
4512 {
4513 	/* get args */
4514 	struct buf *bp = (struct buf *)addr;
4515 
4516 	if (!have_addr) {
4517 		db_printf("usage: show buffer <addr>\n");
4518 		return;
4519 	}
4520 
4521 	db_printf("buf at %p\n", bp);
4522 	db_printf("b_flags = 0x%b, b_xflags=0x%b, b_vflags=0x%b\n",
4523 	    (u_int)bp->b_flags, PRINT_BUF_FLAGS, (u_int)bp->b_xflags,
4524 	    PRINT_BUF_XFLAGS, (u_int)bp->b_vflags, PRINT_BUF_VFLAGS);
4525 	db_printf(
4526 	    "b_error = %d, b_bufsize = %ld, b_bcount = %ld, b_resid = %ld\n"
4527 	    "b_bufobj = (%p), b_data = %p, b_blkno = %jd, b_lblkno = %jd, "
4528 	    "b_dep = %p\n",
4529 	    bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
4530 	    bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
4531 	    (intmax_t)bp->b_lblkno, bp->b_dep.lh_first);
4532 	db_printf("b_kvabase = %p, b_kvasize = %d\n",
4533 	    bp->b_kvabase, bp->b_kvasize);
4534 	if (bp->b_npages) {
4535 		int i;
4536 		db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
4537 		for (i = 0; i < bp->b_npages; i++) {
4538 			vm_page_t m;
4539 			m = bp->b_pages[i];
4540 			db_printf("(%p, 0x%lx, 0x%lx)", (void *)m->object,
4541 			    (u_long)m->pindex, (u_long)VM_PAGE_TO_PHYS(m));
4542 			if ((i + 1) < bp->b_npages)
4543 				db_printf(",");
4544 		}
4545 		db_printf("\n");
4546 	}
4547 	db_printf(" ");
4548 	BUF_LOCKPRINTINFO(bp);
4549 }
4550 
4551 DB_SHOW_COMMAND(lockedbufs, lockedbufs)
4552 {
4553 	struct buf *bp;
4554 	int i;
4555 
4556 	for (i = 0; i < nbuf; i++) {
4557 		bp = &buf[i];
4558 		if (BUF_ISLOCKED(bp)) {
4559 			db_show_buffer((uintptr_t)bp, 1, 0, NULL);
4560 			db_printf("\n");
4561 		}
4562 	}
4563 }
4564 
4565 DB_SHOW_COMMAND(vnodebufs, db_show_vnodebufs)
4566 {
4567 	struct vnode *vp;
4568 	struct buf *bp;
4569 
4570 	if (!have_addr) {
4571 		db_printf("usage: show vnodebufs <addr>\n");
4572 		return;
4573 	}
4574 	vp = (struct vnode *)addr;
4575 	db_printf("Clean buffers:\n");
4576 	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_clean.bv_hd, b_bobufs) {
4577 		db_show_buffer((uintptr_t)bp, 1, 0, NULL);
4578 		db_printf("\n");
4579 	}
4580 	db_printf("Dirty buffers:\n");
4581 	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
4582 		db_show_buffer((uintptr_t)bp, 1, 0, NULL);
4583 		db_printf("\n");
4584 	}
4585 }
4586 
4587 DB_COMMAND(countfreebufs, db_coundfreebufs)
4588 {
4589 	struct buf *bp;
4590 	int i, used = 0, nfree = 0;
4591 
4592 	if (have_addr) {
4593 		db_printf("usage: countfreebufs\n");
4594 		return;
4595 	}
4596 
4597 	for (i = 0; i < nbuf; i++) {
4598 		bp = &buf[i];
4599 		if ((bp->b_flags & B_INFREECNT) != 0)
4600 			nfree++;
4601 		else
4602 			used++;
4603 	}
4604 
4605 	db_printf("Counted %d free, %d used (%d tot)\n", nfree, used,
4606 	    nfree + used);
4607 	db_printf("numfreebuffers is %d\n", numfreebuffers);
4608 }
4609 #endif /* DDB */
4610