xref: /freebsd/sys/kern/vfs_bio.c (revision 5e3190f700637fcfc1a52daeaa4a031fdd2557c7)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2004 Poul-Henning Kamp
5  * Copyright (c) 1994,1997 John S. Dyson
6  * Copyright (c) 2013 The FreeBSD Foundation
7  * All rights reserved.
8  *
9  * Portions of this software were developed by Konstantin Belousov
10  * under sponsorship from the FreeBSD Foundation.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 /*
35  * this file contains a new buffer I/O scheme implementing a coherent
36  * VM object and buffer cache scheme.  Pains have been taken to make
37  * sure that the performance degradation associated with schemes such
38  * as this is not realized.
39  *
40  * Author:  John S. Dyson
41  * Significant help during the development and debugging phases
42  * had been provided by David Greenman, also of the FreeBSD core team.
43  *
44  * see man buf(9) for more info.
45  */
46 
47 #include <sys/cdefs.h>
48 #include <sys/param.h>
49 #include <sys/systm.h>
50 #include <sys/asan.h>
51 #include <sys/bio.h>
52 #include <sys/bitset.h>
53 #include <sys/boottrace.h>
54 #include <sys/buf.h>
55 #include <sys/conf.h>
56 #include <sys/counter.h>
57 #include <sys/devicestat.h>
58 #include <sys/eventhandler.h>
59 #include <sys/fail.h>
60 #include <sys/ktr.h>
61 #include <sys/limits.h>
62 #include <sys/lock.h>
63 #include <sys/malloc.h>
64 #include <sys/memdesc.h>
65 #include <sys/mount.h>
66 #include <sys/mutex.h>
67 #include <sys/kernel.h>
68 #include <sys/kthread.h>
69 #include <sys/pctrie.h>
70 #include <sys/proc.h>
71 #include <sys/racct.h>
72 #include <sys/refcount.h>
73 #include <sys/resourcevar.h>
74 #include <sys/rwlock.h>
75 #include <sys/sched.h>
76 #include <sys/smp.h>
77 #include <sys/sysctl.h>
78 #include <sys/syscallsubr.h>
79 #include <sys/vmem.h>
80 #include <sys/vmmeter.h>
81 #include <sys/vnode.h>
82 #include <sys/watchdog.h>
83 #include <geom/geom.h>
84 #include <vm/vm.h>
85 #include <vm/vm_param.h>
86 #include <vm/vm_kern.h>
87 #include <vm/vm_object.h>
88 #include <vm/vm_page.h>
89 #include <vm/vm_pageout.h>
90 #include <vm/vm_pager.h>
91 #include <vm/vm_extern.h>
92 #include <vm/vm_map.h>
93 #include <vm/swap_pager.h>
94 
95 static MALLOC_DEFINE(M_BIOBUF, "biobuf", "BIO buffer");
96 
97 struct	bio_ops bioops;		/* I/O operation notification */
98 
99 struct	buf_ops buf_ops_bio = {
100 	.bop_name	=	"buf_ops_bio",
101 	.bop_write	=	bufwrite,
102 	.bop_strategy	=	bufstrategy,
103 	.bop_sync	=	bufsync,
104 	.bop_bdflush	=	bufbdflush,
105 };
106 
107 struct bufqueue {
108 	struct mtx_padalign	bq_lock;
109 	TAILQ_HEAD(, buf)	bq_queue;
110 	uint8_t			bq_index;
111 	uint16_t		bq_subqueue;
112 	int			bq_len;
113 } __aligned(CACHE_LINE_SIZE);
114 
115 #define	BQ_LOCKPTR(bq)		(&(bq)->bq_lock)
116 #define	BQ_LOCK(bq)		mtx_lock(BQ_LOCKPTR((bq)))
117 #define	BQ_UNLOCK(bq)		mtx_unlock(BQ_LOCKPTR((bq)))
118 #define	BQ_ASSERT_LOCKED(bq)	mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED)
119 
120 struct bufdomain {
121 	struct bufqueue	*bd_subq;
122 	struct bufqueue bd_dirtyq;
123 	struct bufqueue	*bd_cleanq;
124 	struct mtx_padalign bd_run_lock;
125 	/* Constants */
126 	long		bd_maxbufspace;
127 	long		bd_hibufspace;
128 	long 		bd_lobufspace;
129 	long 		bd_bufspacethresh;
130 	int		bd_hifreebuffers;
131 	int		bd_lofreebuffers;
132 	int		bd_hidirtybuffers;
133 	int		bd_lodirtybuffers;
134 	int		bd_dirtybufthresh;
135 	int		bd_lim;
136 	/* atomics */
137 	int		bd_wanted;
138 	bool		bd_shutdown;
139 	int __aligned(CACHE_LINE_SIZE)	bd_numdirtybuffers;
140 	int __aligned(CACHE_LINE_SIZE)	bd_running;
141 	long __aligned(CACHE_LINE_SIZE) bd_bufspace;
142 	int __aligned(CACHE_LINE_SIZE)	bd_freebuffers;
143 } __aligned(CACHE_LINE_SIZE);
144 
145 #define	BD_LOCKPTR(bd)		(&(bd)->bd_cleanq->bq_lock)
146 #define	BD_LOCK(bd)		mtx_lock(BD_LOCKPTR((bd)))
147 #define	BD_UNLOCK(bd)		mtx_unlock(BD_LOCKPTR((bd)))
148 #define	BD_ASSERT_LOCKED(bd)	mtx_assert(BD_LOCKPTR((bd)), MA_OWNED)
149 #define	BD_RUN_LOCKPTR(bd)	(&(bd)->bd_run_lock)
150 #define	BD_RUN_LOCK(bd)		mtx_lock(BD_RUN_LOCKPTR((bd)))
151 #define	BD_RUN_UNLOCK(bd)	mtx_unlock(BD_RUN_LOCKPTR((bd)))
152 #define	BD_DOMAIN(bd)		(bd - bdomain)
153 
154 static char *buf;		/* buffer header pool */
155 static struct buf *
156 nbufp(unsigned i)
157 {
158 	return ((struct buf *)(buf + (sizeof(struct buf) +
159 	    sizeof(vm_page_t) * atop(maxbcachebuf)) * i));
160 }
161 
162 caddr_t __read_mostly unmapped_buf;
163 
164 /* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
165 struct proc *bufdaemonproc;
166 
167 static void vm_hold_free_pages(struct buf *bp, int newbsize);
168 static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
169 		vm_offset_t to);
170 static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
171 static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
172 		vm_page_t m);
173 static void vfs_clean_pages_dirty_buf(struct buf *bp);
174 static void vfs_setdirty_range(struct buf *bp);
175 static void vfs_vmio_invalidate(struct buf *bp);
176 static void vfs_vmio_truncate(struct buf *bp, int npages);
177 static void vfs_vmio_extend(struct buf *bp, int npages, int size);
178 static int vfs_bio_clcheck(struct vnode *vp, int size,
179 		daddr_t lblkno, daddr_t blkno);
180 static void breada(struct vnode *, daddr_t *, int *, int, struct ucred *, int,
181 		void (*)(struct buf *));
182 static int buf_flush(struct vnode *vp, struct bufdomain *, int);
183 static int flushbufqueues(struct vnode *, struct bufdomain *, int, int);
184 static void buf_daemon(void);
185 static __inline void bd_wakeup(void);
186 static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
187 static void bufkva_reclaim(vmem_t *, int);
188 static void bufkva_free(struct buf *);
189 static int buf_import(void *, void **, int, int, int);
190 static void buf_release(void *, void **, int);
191 static void maxbcachebuf_adjust(void);
192 static inline struct bufdomain *bufdomain(struct buf *);
193 static void bq_remove(struct bufqueue *bq, struct buf *bp);
194 static void bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock);
195 static int buf_recycle(struct bufdomain *, bool kva);
196 static void bq_init(struct bufqueue *bq, int qindex, int cpu,
197 	    const char *lockname);
198 static void bd_init(struct bufdomain *bd);
199 static int bd_flushall(struct bufdomain *bd);
200 static int sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS);
201 static int sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS);
202 
203 static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
204 int vmiodirenable = TRUE;
205 SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW, &vmiodirenable, 0,
206     "Use the VM system for directory writes");
207 long runningbufspace;
208 SYSCTL_LONG(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, &runningbufspace, 0,
209     "Amount of presently outstanding async buffer io");
210 SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RD,
211     NULL, 0, sysctl_bufspace, "L", "Physical memory used for buffers");
212 static counter_u64_t bufkvaspace;
213 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace,
214     "Kernel virtual memory used for buffers");
215 static long maxbufspace;
216 SYSCTL_PROC(_vfs, OID_AUTO, maxbufspace,
217     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &maxbufspace,
218     __offsetof(struct bufdomain, bd_maxbufspace), sysctl_bufdomain_long, "L",
219     "Maximum allowed value of bufspace (including metadata)");
220 static long bufmallocspace;
221 SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
222     "Amount of malloced memory for buffers");
223 static long maxbufmallocspace;
224 SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
225     0, "Maximum amount of malloced memory for buffers");
226 static long lobufspace;
227 SYSCTL_PROC(_vfs, OID_AUTO, lobufspace,
228     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &lobufspace,
229     __offsetof(struct bufdomain, bd_lobufspace), sysctl_bufdomain_long, "L",
230     "Minimum amount of buffers we want to have");
231 long hibufspace;
232 SYSCTL_PROC(_vfs, OID_AUTO, hibufspace,
233     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &hibufspace,
234     __offsetof(struct bufdomain, bd_hibufspace), sysctl_bufdomain_long, "L",
235     "Maximum allowed value of bufspace (excluding metadata)");
236 long bufspacethresh;
237 SYSCTL_PROC(_vfs, OID_AUTO, bufspacethresh,
238     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &bufspacethresh,
239     __offsetof(struct bufdomain, bd_bufspacethresh), sysctl_bufdomain_long, "L",
240     "Bufspace consumed before waking the daemon to free some");
241 static counter_u64_t buffreekvacnt;
242 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt,
243     "Number of times we have freed the KVA space from some buffer");
244 static counter_u64_t bufdefragcnt;
245 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW, &bufdefragcnt,
246     "Number of times we have had to repeat buffer allocation to defragment");
247 static long lorunningspace;
248 SYSCTL_PROC(_vfs, OID_AUTO, lorunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
249     CTLFLAG_RW, &lorunningspace, 0, sysctl_runningspace, "L",
250     "Minimum preferred space used for in-progress I/O");
251 static long hirunningspace;
252 SYSCTL_PROC(_vfs, OID_AUTO, hirunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
253     CTLFLAG_RW, &hirunningspace, 0, sysctl_runningspace, "L",
254     "Maximum amount of space to use for in-progress I/O");
255 int dirtybufferflushes;
256 SYSCTL_INT(_vfs, OID_AUTO, dirtybufferflushes, CTLFLAG_RW, &dirtybufferflushes,
257     0, "Number of bdwrite to bawrite conversions to limit dirty buffers");
258 int bdwriteskip;
259 SYSCTL_INT(_vfs, OID_AUTO, bdwriteskip, CTLFLAG_RW, &bdwriteskip,
260     0, "Number of buffers supplied to bdwrite with snapshot deadlock risk");
261 int altbufferflushes;
262 SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFLAG_RW | CTLFLAG_STATS,
263     &altbufferflushes, 0, "Number of fsync flushes to limit dirty buffers");
264 static int recursiveflushes;
265 SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW | CTLFLAG_STATS,
266     &recursiveflushes, 0, "Number of flushes skipped due to being recursive");
267 static int sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS);
268 SYSCTL_PROC(_vfs, OID_AUTO, numdirtybuffers,
269     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RD, NULL, 0, sysctl_numdirtybuffers, "I",
270     "Number of buffers that are dirty (has unwritten changes) at the moment");
271 static int lodirtybuffers;
272 SYSCTL_PROC(_vfs, OID_AUTO, lodirtybuffers,
273     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lodirtybuffers,
274     __offsetof(struct bufdomain, bd_lodirtybuffers), sysctl_bufdomain_int, "I",
275     "How many buffers we want to have free before bufdaemon can sleep");
276 static int hidirtybuffers;
277 SYSCTL_PROC(_vfs, OID_AUTO, hidirtybuffers,
278     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hidirtybuffers,
279     __offsetof(struct bufdomain, bd_hidirtybuffers), sysctl_bufdomain_int, "I",
280     "When the number of dirty buffers is considered severe");
281 int dirtybufthresh;
282 SYSCTL_PROC(_vfs, OID_AUTO, dirtybufthresh,
283     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &dirtybufthresh,
284     __offsetof(struct bufdomain, bd_dirtybufthresh), sysctl_bufdomain_int, "I",
285     "Number of bdwrite to bawrite conversions to clear dirty buffers");
286 static int numfreebuffers;
287 SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
288     "Number of free buffers");
289 static int lofreebuffers;
290 SYSCTL_PROC(_vfs, OID_AUTO, lofreebuffers,
291     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lofreebuffers,
292     __offsetof(struct bufdomain, bd_lofreebuffers), sysctl_bufdomain_int, "I",
293    "Target number of free buffers");
294 static int hifreebuffers;
295 SYSCTL_PROC(_vfs, OID_AUTO, hifreebuffers,
296     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hifreebuffers,
297     __offsetof(struct bufdomain, bd_hifreebuffers), sysctl_bufdomain_int, "I",
298    "Threshold for clean buffer recycling");
299 static counter_u64_t getnewbufcalls;
300 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RD,
301    &getnewbufcalls, "Number of calls to getnewbuf");
302 static counter_u64_t getnewbufrestarts;
303 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RD,
304     &getnewbufrestarts,
305     "Number of times getnewbuf has had to restart a buffer acquisition");
306 static counter_u64_t mappingrestarts;
307 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RD,
308     &mappingrestarts,
309     "Number of times getblk has had to restart a buffer mapping for "
310     "unmapped buffer");
311 static counter_u64_t numbufallocfails;
312 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW,
313     &numbufallocfails, "Number of times buffer allocations failed");
314 static int flushbufqtarget = 100;
315 SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
316     "Amount of work to do in flushbufqueues when helping bufdaemon");
317 static counter_u64_t notbufdflushes;
318 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, notbufdflushes, CTLFLAG_RD, &notbufdflushes,
319     "Number of dirty buffer flushes done by the bufdaemon helpers");
320 static long barrierwrites;
321 SYSCTL_LONG(_vfs, OID_AUTO, barrierwrites, CTLFLAG_RW | CTLFLAG_STATS,
322     &barrierwrites, 0, "Number of barrier writes");
323 SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed,
324     CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
325     &unmapped_buf_allowed, 0,
326     "Permit the use of the unmapped i/o");
327 int maxbcachebuf = MAXBCACHEBUF;
328 SYSCTL_INT(_vfs, OID_AUTO, maxbcachebuf, CTLFLAG_RDTUN, &maxbcachebuf, 0,
329     "Maximum size of a buffer cache block");
330 
331 /*
332  * This lock synchronizes access to bd_request.
333  */
334 static struct mtx_padalign __exclusive_cache_line bdlock;
335 
336 /*
337  * This lock protects the runningbufreq and synchronizes runningbufwakeup and
338  * waitrunningbufspace().
339  */
340 static struct mtx_padalign __exclusive_cache_line rbreqlock;
341 
342 /*
343  * Lock that protects bdirtywait.
344  */
345 static struct mtx_padalign __exclusive_cache_line bdirtylock;
346 
347 /*
348  * bufdaemon shutdown request and sleep channel.
349  */
350 static bool bd_shutdown;
351 
352 /*
353  * Wakeup point for bufdaemon, as well as indicator of whether it is already
354  * active.  Set to 1 when the bufdaemon is already "on" the queue, 0 when it
355  * is idling.
356  */
357 static int bd_request;
358 
359 /*
360  * Request for the buf daemon to write more buffers than is indicated by
361  * lodirtybuf.  This may be necessary to push out excess dependencies or
362  * defragment the address space where a simple count of the number of dirty
363  * buffers is insufficient to characterize the demand for flushing them.
364  */
365 static int bd_speedupreq;
366 
367 /*
368  * Synchronization (sleep/wakeup) variable for active buffer space requests.
369  * Set when wait starts, cleared prior to wakeup().
370  * Used in runningbufwakeup() and waitrunningbufspace().
371  */
372 static int runningbufreq;
373 
374 /*
375  * Synchronization for bwillwrite() waiters.
376  */
377 static int bdirtywait;
378 
379 /*
380  * Definitions for the buffer free lists.
381  */
382 #define QUEUE_NONE	0	/* on no queue */
383 #define QUEUE_EMPTY	1	/* empty buffer headers */
384 #define QUEUE_DIRTY	2	/* B_DELWRI buffers */
385 #define QUEUE_CLEAN	3	/* non-B_DELWRI buffers */
386 #define QUEUE_SENTINEL	4	/* not an queue index, but mark for sentinel */
387 
388 /* Maximum number of buffer domains. */
389 #define	BUF_DOMAINS	8
390 
391 struct bufdomainset bdlodirty;		/* Domains > lodirty */
392 struct bufdomainset bdhidirty;		/* Domains > hidirty */
393 
394 /* Configured number of clean queues. */
395 static int __read_mostly buf_domains;
396 
397 BITSET_DEFINE(bufdomainset, BUF_DOMAINS);
398 struct bufdomain __exclusive_cache_line bdomain[BUF_DOMAINS];
399 struct bufqueue __exclusive_cache_line bqempty;
400 
401 /*
402  * per-cpu empty buffer cache.
403  */
404 uma_zone_t buf_zone;
405 
406 static int
407 sysctl_runningspace(SYSCTL_HANDLER_ARGS)
408 {
409 	long value;
410 	int error;
411 
412 	value = *(long *)arg1;
413 	error = sysctl_handle_long(oidp, &value, 0, req);
414 	if (error != 0 || req->newptr == NULL)
415 		return (error);
416 	mtx_lock(&rbreqlock);
417 	if (arg1 == &hirunningspace) {
418 		if (value < lorunningspace)
419 			error = EINVAL;
420 		else
421 			hirunningspace = value;
422 	} else {
423 		KASSERT(arg1 == &lorunningspace,
424 		    ("%s: unknown arg1", __func__));
425 		if (value > hirunningspace)
426 			error = EINVAL;
427 		else
428 			lorunningspace = value;
429 	}
430 	mtx_unlock(&rbreqlock);
431 	return (error);
432 }
433 
434 static int
435 sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS)
436 {
437 	int error;
438 	int value;
439 	int i;
440 
441 	value = *(int *)arg1;
442 	error = sysctl_handle_int(oidp, &value, 0, req);
443 	if (error != 0 || req->newptr == NULL)
444 		return (error);
445 	*(int *)arg1 = value;
446 	for (i = 0; i < buf_domains; i++)
447 		*(int *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
448 		    value / buf_domains;
449 
450 	return (error);
451 }
452 
453 static int
454 sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS)
455 {
456 	long value;
457 	int error;
458 	int i;
459 
460 	value = *(long *)arg1;
461 	error = sysctl_handle_long(oidp, &value, 0, req);
462 	if (error != 0 || req->newptr == NULL)
463 		return (error);
464 	*(long *)arg1 = value;
465 	for (i = 0; i < buf_domains; i++)
466 		*(long *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
467 		    value / buf_domains;
468 
469 	return (error);
470 }
471 
472 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
473     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
474 static int
475 sysctl_bufspace(SYSCTL_HANDLER_ARGS)
476 {
477 	long lvalue;
478 	int ivalue;
479 	int i;
480 
481 	lvalue = 0;
482 	for (i = 0; i < buf_domains; i++)
483 		lvalue += bdomain[i].bd_bufspace;
484 	if (sizeof(int) == sizeof(long) || req->oldlen >= sizeof(long))
485 		return (sysctl_handle_long(oidp, &lvalue, 0, req));
486 	if (lvalue > INT_MAX)
487 		/* On overflow, still write out a long to trigger ENOMEM. */
488 		return (sysctl_handle_long(oidp, &lvalue, 0, req));
489 	ivalue = lvalue;
490 	return (sysctl_handle_int(oidp, &ivalue, 0, req));
491 }
492 #else
493 static int
494 sysctl_bufspace(SYSCTL_HANDLER_ARGS)
495 {
496 	long lvalue;
497 	int i;
498 
499 	lvalue = 0;
500 	for (i = 0; i < buf_domains; i++)
501 		lvalue += bdomain[i].bd_bufspace;
502 	return (sysctl_handle_long(oidp, &lvalue, 0, req));
503 }
504 #endif
505 
506 static int
507 sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS)
508 {
509 	int value;
510 	int i;
511 
512 	value = 0;
513 	for (i = 0; i < buf_domains; i++)
514 		value += bdomain[i].bd_numdirtybuffers;
515 	return (sysctl_handle_int(oidp, &value, 0, req));
516 }
517 
518 /*
519  *	bdirtywakeup:
520  *
521  *	Wakeup any bwillwrite() waiters.
522  */
523 static void
524 bdirtywakeup(void)
525 {
526 	mtx_lock(&bdirtylock);
527 	if (bdirtywait) {
528 		bdirtywait = 0;
529 		wakeup(&bdirtywait);
530 	}
531 	mtx_unlock(&bdirtylock);
532 }
533 
534 /*
535  *	bd_clear:
536  *
537  *	Clear a domain from the appropriate bitsets when dirtybuffers
538  *	is decremented.
539  */
540 static void
541 bd_clear(struct bufdomain *bd)
542 {
543 
544 	mtx_lock(&bdirtylock);
545 	if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers)
546 		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
547 	if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers)
548 		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
549 	mtx_unlock(&bdirtylock);
550 }
551 
552 /*
553  *	bd_set:
554  *
555  *	Set a domain in the appropriate bitsets when dirtybuffers
556  *	is incremented.
557  */
558 static void
559 bd_set(struct bufdomain *bd)
560 {
561 
562 	mtx_lock(&bdirtylock);
563 	if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers)
564 		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
565 	if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers)
566 		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
567 	mtx_unlock(&bdirtylock);
568 }
569 
570 /*
571  *	bdirtysub:
572  *
573  *	Decrement the numdirtybuffers count by one and wakeup any
574  *	threads blocked in bwillwrite().
575  */
576 static void
577 bdirtysub(struct buf *bp)
578 {
579 	struct bufdomain *bd;
580 	int num;
581 
582 	bd = bufdomain(bp);
583 	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1);
584 	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
585 		bdirtywakeup();
586 	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
587 		bd_clear(bd);
588 }
589 
590 /*
591  *	bdirtyadd:
592  *
593  *	Increment the numdirtybuffers count by one and wakeup the buf
594  *	daemon if needed.
595  */
596 static void
597 bdirtyadd(struct buf *bp)
598 {
599 	struct bufdomain *bd;
600 	int num;
601 
602 	/*
603 	 * Only do the wakeup once as we cross the boundary.  The
604 	 * buf daemon will keep running until the condition clears.
605 	 */
606 	bd = bufdomain(bp);
607 	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1);
608 	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
609 		bd_wakeup();
610 	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
611 		bd_set(bd);
612 }
613 
614 /*
615  *	bufspace_daemon_wakeup:
616  *
617  *	Wakeup the daemons responsible for freeing clean bufs.
618  */
619 static void
620 bufspace_daemon_wakeup(struct bufdomain *bd)
621 {
622 
623 	/*
624 	 * avoid the lock if the daemon is running.
625 	 */
626 	if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) {
627 		BD_RUN_LOCK(bd);
628 		atomic_store_int(&bd->bd_running, 1);
629 		wakeup(&bd->bd_running);
630 		BD_RUN_UNLOCK(bd);
631 	}
632 }
633 
634 /*
635  *	bufspace_adjust:
636  *
637  *	Adjust the reported bufspace for a KVA managed buffer, possibly
638  * 	waking any waiters.
639  */
640 static void
641 bufspace_adjust(struct buf *bp, int bufsize)
642 {
643 	struct bufdomain *bd;
644 	long space;
645 	int diff;
646 
647 	KASSERT((bp->b_flags & B_MALLOC) == 0,
648 	    ("bufspace_adjust: malloc buf %p", bp));
649 	bd = bufdomain(bp);
650 	diff = bufsize - bp->b_bufsize;
651 	if (diff < 0) {
652 		atomic_subtract_long(&bd->bd_bufspace, -diff);
653 	} else if (diff > 0) {
654 		space = atomic_fetchadd_long(&bd->bd_bufspace, diff);
655 		/* Wake up the daemon on the transition. */
656 		if (space < bd->bd_bufspacethresh &&
657 		    space + diff >= bd->bd_bufspacethresh)
658 			bufspace_daemon_wakeup(bd);
659 	}
660 	bp->b_bufsize = bufsize;
661 }
662 
663 /*
664  *	bufspace_reserve:
665  *
666  *	Reserve bufspace before calling allocbuf().  metadata has a
667  *	different space limit than data.
668  */
669 static int
670 bufspace_reserve(struct bufdomain *bd, int size, bool metadata)
671 {
672 	long limit, new;
673 	long space;
674 
675 	if (metadata)
676 		limit = bd->bd_maxbufspace;
677 	else
678 		limit = bd->bd_hibufspace;
679 	space = atomic_fetchadd_long(&bd->bd_bufspace, size);
680 	new = space + size;
681 	if (new > limit) {
682 		atomic_subtract_long(&bd->bd_bufspace, size);
683 		return (ENOSPC);
684 	}
685 
686 	/* Wake up the daemon on the transition. */
687 	if (space < bd->bd_bufspacethresh && new >= bd->bd_bufspacethresh)
688 		bufspace_daemon_wakeup(bd);
689 
690 	return (0);
691 }
692 
693 /*
694  *	bufspace_release:
695  *
696  *	Release reserved bufspace after bufspace_adjust() has consumed it.
697  */
698 static void
699 bufspace_release(struct bufdomain *bd, int size)
700 {
701 
702 	atomic_subtract_long(&bd->bd_bufspace, size);
703 }
704 
705 /*
706  *	bufspace_wait:
707  *
708  *	Wait for bufspace, acting as the buf daemon if a locked vnode is
709  *	supplied.  bd_wanted must be set prior to polling for space.  The
710  *	operation must be re-tried on return.
711  */
712 static void
713 bufspace_wait(struct bufdomain *bd, struct vnode *vp, int gbflags,
714     int slpflag, int slptimeo)
715 {
716 	struct thread *td;
717 	int error, fl, norunbuf;
718 
719 	if ((gbflags & GB_NOWAIT_BD) != 0)
720 		return;
721 
722 	td = curthread;
723 	BD_LOCK(bd);
724 	while (bd->bd_wanted) {
725 		if (vp != NULL && vp->v_type != VCHR &&
726 		    (td->td_pflags & TDP_BUFNEED) == 0) {
727 			BD_UNLOCK(bd);
728 			/*
729 			 * getblk() is called with a vnode locked, and
730 			 * some majority of the dirty buffers may as
731 			 * well belong to the vnode.  Flushing the
732 			 * buffers there would make a progress that
733 			 * cannot be achieved by the buf_daemon, that
734 			 * cannot lock the vnode.
735 			 */
736 			norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
737 			    (td->td_pflags & TDP_NORUNNINGBUF);
738 
739 			/*
740 			 * Play bufdaemon.  The getnewbuf() function
741 			 * may be called while the thread owns lock
742 			 * for another dirty buffer for the same
743 			 * vnode, which makes it impossible to use
744 			 * VOP_FSYNC() there, due to the buffer lock
745 			 * recursion.
746 			 */
747 			td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
748 			fl = buf_flush(vp, bd, flushbufqtarget);
749 			td->td_pflags &= norunbuf;
750 			BD_LOCK(bd);
751 			if (fl != 0)
752 				continue;
753 			if (bd->bd_wanted == 0)
754 				break;
755 		}
756 		error = msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
757 		    (PRIBIO + 4) | slpflag, "newbuf", slptimeo);
758 		if (error != 0)
759 			break;
760 	}
761 	BD_UNLOCK(bd);
762 }
763 
764 static void
765 bufspace_daemon_shutdown(void *arg, int howto __unused)
766 {
767 	struct bufdomain *bd = arg;
768 	int error;
769 
770 	if (KERNEL_PANICKED())
771 		return;
772 
773 	BD_RUN_LOCK(bd);
774 	bd->bd_shutdown = true;
775 	wakeup(&bd->bd_running);
776 	error = msleep(&bd->bd_shutdown, BD_RUN_LOCKPTR(bd), 0,
777 	    "bufspace_shutdown", 60 * hz);
778 	BD_RUN_UNLOCK(bd);
779 	if (error != 0)
780 		printf("bufspacedaemon wait error: %d\n", error);
781 }
782 
783 /*
784  *	bufspace_daemon:
785  *
786  *	buffer space management daemon.  Tries to maintain some marginal
787  *	amount of free buffer space so that requesting processes neither
788  *	block nor work to reclaim buffers.
789  */
790 static void
791 bufspace_daemon(void *arg)
792 {
793 	struct bufdomain *bd = arg;
794 
795 	EVENTHANDLER_REGISTER(shutdown_pre_sync, bufspace_daemon_shutdown, bd,
796 	    SHUTDOWN_PRI_LAST + 100);
797 
798 	BD_RUN_LOCK(bd);
799 	while (!bd->bd_shutdown) {
800 		BD_RUN_UNLOCK(bd);
801 
802 		/*
803 		 * Free buffers from the clean queue until we meet our
804 		 * targets.
805 		 *
806 		 * Theory of operation:  The buffer cache is most efficient
807 		 * when some free buffer headers and space are always
808 		 * available to getnewbuf().  This daemon attempts to prevent
809 		 * the excessive blocking and synchronization associated
810 		 * with shortfall.  It goes through three phases according
811 		 * demand:
812 		 *
813 		 * 1)	The daemon wakes up voluntarily once per-second
814 		 *	during idle periods when the counters are below
815 		 *	the wakeup thresholds (bufspacethresh, lofreebuffers).
816 		 *
817 		 * 2)	The daemon wakes up as we cross the thresholds
818 		 *	ahead of any potential blocking.  This may bounce
819 		 *	slightly according to the rate of consumption and
820 		 *	release.
821 		 *
822 		 * 3)	The daemon and consumers are starved for working
823 		 *	clean buffers.  This is the 'bufspace' sleep below
824 		 *	which will inefficiently trade bufs with bqrelse
825 		 *	until we return to condition 2.
826 		 */
827 		while (bd->bd_bufspace > bd->bd_lobufspace ||
828 		    bd->bd_freebuffers < bd->bd_hifreebuffers) {
829 			if (buf_recycle(bd, false) != 0) {
830 				if (bd_flushall(bd))
831 					continue;
832 				/*
833 				 * Speedup dirty if we've run out of clean
834 				 * buffers.  This is possible in particular
835 				 * because softdep may held many bufs locked
836 				 * pending writes to other bufs which are
837 				 * marked for delayed write, exhausting
838 				 * clean space until they are written.
839 				 */
840 				bd_speedup();
841 				BD_LOCK(bd);
842 				if (bd->bd_wanted) {
843 					msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
844 					    PRIBIO|PDROP, "bufspace", hz/10);
845 				} else
846 					BD_UNLOCK(bd);
847 			}
848 			maybe_yield();
849 		}
850 
851 		/*
852 		 * Re-check our limits and sleep.  bd_running must be
853 		 * cleared prior to checking the limits to avoid missed
854 		 * wakeups.  The waker will adjust one of bufspace or
855 		 * freebuffers prior to checking bd_running.
856 		 */
857 		BD_RUN_LOCK(bd);
858 		if (bd->bd_shutdown)
859 			break;
860 		atomic_store_int(&bd->bd_running, 0);
861 		if (bd->bd_bufspace < bd->bd_bufspacethresh &&
862 		    bd->bd_freebuffers > bd->bd_lofreebuffers) {
863 			msleep(&bd->bd_running, BD_RUN_LOCKPTR(bd),
864 			    PRIBIO, "-", hz);
865 		} else {
866 			/* Avoid spurious wakeups while running. */
867 			atomic_store_int(&bd->bd_running, 1);
868 		}
869 	}
870 	wakeup(&bd->bd_shutdown);
871 	BD_RUN_UNLOCK(bd);
872 	kthread_exit();
873 }
874 
875 /*
876  *	bufmallocadjust:
877  *
878  *	Adjust the reported bufspace for a malloc managed buffer, possibly
879  *	waking any waiters.
880  */
881 static void
882 bufmallocadjust(struct buf *bp, int bufsize)
883 {
884 	int diff;
885 
886 	KASSERT((bp->b_flags & B_MALLOC) != 0,
887 	    ("bufmallocadjust: non-malloc buf %p", bp));
888 	diff = bufsize - bp->b_bufsize;
889 	if (diff < 0)
890 		atomic_subtract_long(&bufmallocspace, -diff);
891 	else
892 		atomic_add_long(&bufmallocspace, diff);
893 	bp->b_bufsize = bufsize;
894 }
895 
896 /*
897  *	runningwakeup:
898  *
899  *	Wake up processes that are waiting on asynchronous writes to fall
900  *	below lorunningspace.
901  */
902 static void
903 runningwakeup(void)
904 {
905 
906 	mtx_lock(&rbreqlock);
907 	if (runningbufreq) {
908 		runningbufreq = 0;
909 		wakeup(&runningbufreq);
910 	}
911 	mtx_unlock(&rbreqlock);
912 }
913 
914 /*
915  *	runningbufwakeup:
916  *
917  *	Decrement the outstanding write count according.
918  */
919 void
920 runningbufwakeup(struct buf *bp)
921 {
922 	long space, bspace;
923 
924 	bspace = bp->b_runningbufspace;
925 	if (bspace == 0)
926 		return;
927 	space = atomic_fetchadd_long(&runningbufspace, -bspace);
928 	KASSERT(space >= bspace, ("runningbufspace underflow %ld %ld",
929 	    space, bspace));
930 	bp->b_runningbufspace = 0;
931 	/*
932 	 * Only acquire the lock and wakeup on the transition from exceeding
933 	 * the threshold to falling below it.
934 	 */
935 	if (space < lorunningspace)
936 		return;
937 	if (space - bspace > lorunningspace)
938 		return;
939 	runningwakeup();
940 }
941 
942 /*
943  *	waitrunningbufspace()
944  *
945  *	runningbufspace is a measure of the amount of I/O currently
946  *	running.  This routine is used in async-write situations to
947  *	prevent creating huge backups of pending writes to a device.
948  *	Only asynchronous writes are governed by this function.
949  *
950  *	This does NOT turn an async write into a sync write.  It waits
951  *	for earlier writes to complete and generally returns before the
952  *	caller's write has reached the device.
953  */
954 void
955 waitrunningbufspace(void)
956 {
957 
958 	mtx_lock(&rbreqlock);
959 	while (runningbufspace > hirunningspace) {
960 		runningbufreq = 1;
961 		msleep(&runningbufreq, &rbreqlock, PVM, "wdrain", 0);
962 	}
963 	mtx_unlock(&rbreqlock);
964 }
965 
966 /*
967  *	vfs_buf_test_cache:
968  *
969  *	Called when a buffer is extended.  This function clears the B_CACHE
970  *	bit if the newly extended portion of the buffer does not contain
971  *	valid data.
972  */
973 static __inline void
974 vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
975     vm_offset_t size, vm_page_t m)
976 {
977 
978 	/*
979 	 * This function and its results are protected by higher level
980 	 * synchronization requiring vnode and buf locks to page in and
981 	 * validate pages.
982 	 */
983 	if (bp->b_flags & B_CACHE) {
984 		int base = (foff + off) & PAGE_MASK;
985 		if (vm_page_is_valid(m, base, size) == 0)
986 			bp->b_flags &= ~B_CACHE;
987 	}
988 }
989 
990 /* Wake up the buffer daemon if necessary */
991 static void
992 bd_wakeup(void)
993 {
994 
995 	mtx_lock(&bdlock);
996 	if (bd_request == 0) {
997 		bd_request = 1;
998 		wakeup(&bd_request);
999 	}
1000 	mtx_unlock(&bdlock);
1001 }
1002 
1003 /*
1004  * Adjust the maxbcachbuf tunable.
1005  */
1006 static void
1007 maxbcachebuf_adjust(void)
1008 {
1009 	int i;
1010 
1011 	/*
1012 	 * maxbcachebuf must be a power of 2 >= MAXBSIZE.
1013 	 */
1014 	i = 2;
1015 	while (i * 2 <= maxbcachebuf)
1016 		i *= 2;
1017 	maxbcachebuf = i;
1018 	if (maxbcachebuf < MAXBSIZE)
1019 		maxbcachebuf = MAXBSIZE;
1020 	if (maxbcachebuf > maxphys)
1021 		maxbcachebuf = maxphys;
1022 	if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF)
1023 		printf("maxbcachebuf=%d\n", maxbcachebuf);
1024 }
1025 
1026 /*
1027  * bd_speedup - speedup the buffer cache flushing code
1028  */
1029 void
1030 bd_speedup(void)
1031 {
1032 	int needwake;
1033 
1034 	mtx_lock(&bdlock);
1035 	needwake = 0;
1036 	if (bd_speedupreq == 0 || bd_request == 0)
1037 		needwake = 1;
1038 	bd_speedupreq = 1;
1039 	bd_request = 1;
1040 	if (needwake)
1041 		wakeup(&bd_request);
1042 	mtx_unlock(&bdlock);
1043 }
1044 
1045 #ifdef __i386__
1046 #define	TRANSIENT_DENOM	5
1047 #else
1048 #define	TRANSIENT_DENOM 10
1049 #endif
1050 
1051 /*
1052  * Calculating buffer cache scaling values and reserve space for buffer
1053  * headers.  This is called during low level kernel initialization and
1054  * may be called more then once.  We CANNOT write to the memory area
1055  * being reserved at this time.
1056  */
1057 caddr_t
1058 kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est)
1059 {
1060 	int tuned_nbuf;
1061 	long maxbuf, maxbuf_sz, buf_sz,	biotmap_sz;
1062 
1063 	/*
1064 	 * With KASAN or KMSAN enabled, the kernel map is shadowed.  Account for
1065 	 * this when sizing maps based on the amount of physical memory
1066 	 * available.
1067 	 */
1068 #if defined(KASAN)
1069 	physmem_est = (physmem_est * KASAN_SHADOW_SCALE) /
1070 	    (KASAN_SHADOW_SCALE + 1);
1071 #elif defined(KMSAN)
1072 	physmem_est /= 3;
1073 
1074 	/*
1075 	 * KMSAN cannot reliably determine whether buffer data is initialized
1076 	 * unless it is updated through a KVA mapping.
1077 	 */
1078 	unmapped_buf_allowed = 0;
1079 #endif
1080 
1081 	/*
1082 	 * physmem_est is in pages.  Convert it to kilobytes (assumes
1083 	 * PAGE_SIZE is >= 1K)
1084 	 */
1085 	physmem_est = physmem_est * (PAGE_SIZE / 1024);
1086 
1087 	maxbcachebuf_adjust();
1088 	/*
1089 	 * The nominal buffer size (and minimum KVA allocation) is BKVASIZE.
1090 	 * For the first 64MB of ram nominally allocate sufficient buffers to
1091 	 * cover 1/4 of our ram.  Beyond the first 64MB allocate additional
1092 	 * buffers to cover 1/10 of our ram over 64MB.  When auto-sizing
1093 	 * the buffer cache we limit the eventual kva reservation to
1094 	 * maxbcache bytes.
1095 	 *
1096 	 * factor represents the 1/4 x ram conversion.
1097 	 */
1098 	if (nbuf == 0) {
1099 		int factor = 4 * BKVASIZE / 1024;
1100 
1101 		nbuf = 50;
1102 		if (physmem_est > 4096)
1103 			nbuf += min((physmem_est - 4096) / factor,
1104 			    65536 / factor);
1105 		if (physmem_est > 65536)
1106 			nbuf += min((physmem_est - 65536) * 2 / (factor * 5),
1107 			    32 * 1024 * 1024 / (factor * 5));
1108 
1109 		if (maxbcache && nbuf > maxbcache / BKVASIZE)
1110 			nbuf = maxbcache / BKVASIZE;
1111 		tuned_nbuf = 1;
1112 	} else
1113 		tuned_nbuf = 0;
1114 
1115 	/* XXX Avoid unsigned long overflows later on with maxbufspace. */
1116 	maxbuf = (LONG_MAX / 3) / BKVASIZE;
1117 	if (nbuf > maxbuf) {
1118 		if (!tuned_nbuf)
1119 			printf("Warning: nbufs lowered from %d to %ld\n", nbuf,
1120 			    maxbuf);
1121 		nbuf = maxbuf;
1122 	}
1123 
1124 	/*
1125 	 * Ideal allocation size for the transient bio submap is 10%
1126 	 * of the maximal space buffer map.  This roughly corresponds
1127 	 * to the amount of the buffer mapped for typical UFS load.
1128 	 *
1129 	 * Clip the buffer map to reserve space for the transient
1130 	 * BIOs, if its extent is bigger than 90% (80% on i386) of the
1131 	 * maximum buffer map extent on the platform.
1132 	 *
1133 	 * The fall-back to the maxbuf in case of maxbcache unset,
1134 	 * allows to not trim the buffer KVA for the architectures
1135 	 * with ample KVA space.
1136 	 */
1137 	if (bio_transient_maxcnt == 0 && unmapped_buf_allowed) {
1138 		maxbuf_sz = maxbcache != 0 ? maxbcache : maxbuf * BKVASIZE;
1139 		buf_sz = (long)nbuf * BKVASIZE;
1140 		if (buf_sz < maxbuf_sz / TRANSIENT_DENOM *
1141 		    (TRANSIENT_DENOM - 1)) {
1142 			/*
1143 			 * There is more KVA than memory.  Do not
1144 			 * adjust buffer map size, and assign the rest
1145 			 * of maxbuf to transient map.
1146 			 */
1147 			biotmap_sz = maxbuf_sz - buf_sz;
1148 		} else {
1149 			/*
1150 			 * Buffer map spans all KVA we could afford on
1151 			 * this platform.  Give 10% (20% on i386) of
1152 			 * the buffer map to the transient bio map.
1153 			 */
1154 			biotmap_sz = buf_sz / TRANSIENT_DENOM;
1155 			buf_sz -= biotmap_sz;
1156 		}
1157 		if (biotmap_sz / INT_MAX > maxphys)
1158 			bio_transient_maxcnt = INT_MAX;
1159 		else
1160 			bio_transient_maxcnt = biotmap_sz / maxphys;
1161 		/*
1162 		 * Artificially limit to 1024 simultaneous in-flight I/Os
1163 		 * using the transient mapping.
1164 		 */
1165 		if (bio_transient_maxcnt > 1024)
1166 			bio_transient_maxcnt = 1024;
1167 		if (tuned_nbuf)
1168 			nbuf = buf_sz / BKVASIZE;
1169 	}
1170 
1171 	if (nswbuf == 0) {
1172 		/*
1173 		 * Pager buffers are allocated for short periods, so scale the
1174 		 * number of reserved buffers based on the number of CPUs rather
1175 		 * than amount of memory.
1176 		 */
1177 		nswbuf = min(nbuf / 4, 32 * mp_ncpus);
1178 		if (nswbuf < NSWBUF_MIN)
1179 			nswbuf = NSWBUF_MIN;
1180 	}
1181 
1182 	/*
1183 	 * Reserve space for the buffer cache buffers
1184 	 */
1185 	buf = (char *)v;
1186 	v = (caddr_t)buf + (sizeof(struct buf) + sizeof(vm_page_t) *
1187 	    atop(maxbcachebuf)) * nbuf;
1188 
1189 	return (v);
1190 }
1191 
1192 /*
1193  * Single global constant for BUF_WMESG, to avoid getting multiple
1194  * references.
1195  */
1196 static const char buf_wmesg[] = "bufwait";
1197 
1198 /* Initialize the buffer subsystem.  Called before use of any buffers. */
1199 void
1200 bufinit(void)
1201 {
1202 	struct buf *bp;
1203 	int i;
1204 
1205 	TSENTER();
1206 	KASSERT(maxbcachebuf >= MAXBSIZE,
1207 	    ("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf,
1208 	    MAXBSIZE));
1209 	bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock");
1210 	mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
1211 	mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
1212 	mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF);
1213 
1214 	unmapped_buf = (caddr_t)kva_alloc(maxphys);
1215 
1216 	/* finally, initialize each buffer header and stick on empty q */
1217 	for (i = 0; i < nbuf; i++) {
1218 		bp = nbufp(i);
1219 		bzero(bp, sizeof(*bp) + sizeof(vm_page_t) * atop(maxbcachebuf));
1220 		bp->b_flags = B_INVAL;
1221 		bp->b_rcred = NOCRED;
1222 		bp->b_wcred = NOCRED;
1223 		bp->b_qindex = QUEUE_NONE;
1224 		bp->b_domain = -1;
1225 		bp->b_subqueue = mp_maxid + 1;
1226 		bp->b_xflags = 0;
1227 		bp->b_data = bp->b_kvabase = unmapped_buf;
1228 		LIST_INIT(&bp->b_dep);
1229 		BUF_LOCKINIT(bp, buf_wmesg);
1230 		bq_insert(&bqempty, bp, false);
1231 	}
1232 
1233 	/*
1234 	 * maxbufspace is the absolute maximum amount of buffer space we are
1235 	 * allowed to reserve in KVM and in real terms.  The absolute maximum
1236 	 * is nominally used by metadata.  hibufspace is the nominal maximum
1237 	 * used by most other requests.  The differential is required to
1238 	 * ensure that metadata deadlocks don't occur.
1239 	 *
1240 	 * maxbufspace is based on BKVASIZE.  Allocating buffers larger then
1241 	 * this may result in KVM fragmentation which is not handled optimally
1242 	 * by the system. XXX This is less true with vmem.  We could use
1243 	 * PAGE_SIZE.
1244 	 */
1245 	maxbufspace = (long)nbuf * BKVASIZE;
1246 	hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - maxbcachebuf * 10);
1247 	lobufspace = (hibufspace / 20) * 19; /* 95% */
1248 	bufspacethresh = lobufspace + (hibufspace - lobufspace) / 2;
1249 
1250 	/*
1251 	 * Note: The 16 MiB upper limit for hirunningspace was chosen
1252 	 * arbitrarily and may need further tuning. It corresponds to
1253 	 * 128 outstanding write IO requests (if IO size is 128 KiB),
1254 	 * which fits with many RAID controllers' tagged queuing limits.
1255 	 * The lower 1 MiB limit is the historical upper limit for
1256 	 * hirunningspace.
1257 	 */
1258 	hirunningspace = lmax(lmin(roundup(hibufspace / 64, maxbcachebuf),
1259 	    16 * 1024 * 1024), 1024 * 1024);
1260 	lorunningspace = roundup((hirunningspace * 2) / 3, maxbcachebuf);
1261 
1262 	/*
1263 	 * Limit the amount of malloc memory since it is wired permanently into
1264 	 * the kernel space.  Even though this is accounted for in the buffer
1265 	 * allocation, we don't want the malloced region to grow uncontrolled.
1266 	 * The malloc scheme improves memory utilization significantly on
1267 	 * average (small) directories.
1268 	 */
1269 	maxbufmallocspace = hibufspace / 20;
1270 
1271 	/*
1272 	 * Reduce the chance of a deadlock occurring by limiting the number
1273 	 * of delayed-write dirty buffers we allow to stack up.
1274 	 */
1275 	hidirtybuffers = nbuf / 4 + 20;
1276 	dirtybufthresh = hidirtybuffers * 9 / 10;
1277 	/*
1278 	 * To support extreme low-memory systems, make sure hidirtybuffers
1279 	 * cannot eat up all available buffer space.  This occurs when our
1280 	 * minimum cannot be met.  We try to size hidirtybuffers to 3/4 our
1281 	 * buffer space assuming BKVASIZE'd buffers.
1282 	 */
1283 	while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
1284 		hidirtybuffers >>= 1;
1285 	}
1286 	lodirtybuffers = hidirtybuffers / 2;
1287 
1288 	/*
1289 	 * lofreebuffers should be sufficient to avoid stalling waiting on
1290 	 * buf headers under heavy utilization.  The bufs in per-cpu caches
1291 	 * are counted as free but will be unavailable to threads executing
1292 	 * on other cpus.
1293 	 *
1294 	 * hifreebuffers is the free target for the bufspace daemon.  This
1295 	 * should be set appropriately to limit work per-iteration.
1296 	 */
1297 	lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus);
1298 	hifreebuffers = (3 * lofreebuffers) / 2;
1299 	numfreebuffers = nbuf;
1300 
1301 	/* Setup the kva and free list allocators. */
1302 	vmem_set_reclaim(buffer_arena, bufkva_reclaim);
1303 	buf_zone = uma_zcache_create("buf free cache",
1304 	    sizeof(struct buf) + sizeof(vm_page_t) * atop(maxbcachebuf),
1305 	    NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0);
1306 
1307 	/*
1308 	 * Size the clean queue according to the amount of buffer space.
1309 	 * One queue per-256mb up to the max.  More queues gives better
1310 	 * concurrency but less accurate LRU.
1311 	 */
1312 	buf_domains = MIN(howmany(maxbufspace, 256*1024*1024), BUF_DOMAINS);
1313 	for (i = 0 ; i < buf_domains; i++) {
1314 		struct bufdomain *bd;
1315 
1316 		bd = &bdomain[i];
1317 		bd_init(bd);
1318 		bd->bd_freebuffers = nbuf / buf_domains;
1319 		bd->bd_hifreebuffers = hifreebuffers / buf_domains;
1320 		bd->bd_lofreebuffers = lofreebuffers / buf_domains;
1321 		bd->bd_bufspace = 0;
1322 		bd->bd_maxbufspace = maxbufspace / buf_domains;
1323 		bd->bd_hibufspace = hibufspace / buf_domains;
1324 		bd->bd_lobufspace = lobufspace / buf_domains;
1325 		bd->bd_bufspacethresh = bufspacethresh / buf_domains;
1326 		bd->bd_numdirtybuffers = 0;
1327 		bd->bd_hidirtybuffers = hidirtybuffers / buf_domains;
1328 		bd->bd_lodirtybuffers = lodirtybuffers / buf_domains;
1329 		bd->bd_dirtybufthresh = dirtybufthresh / buf_domains;
1330 		/* Don't allow more than 2% of bufs in the per-cpu caches. */
1331 		bd->bd_lim = nbuf / buf_domains / 50 / mp_ncpus;
1332 	}
1333 	getnewbufcalls = counter_u64_alloc(M_WAITOK);
1334 	getnewbufrestarts = counter_u64_alloc(M_WAITOK);
1335 	mappingrestarts = counter_u64_alloc(M_WAITOK);
1336 	numbufallocfails = counter_u64_alloc(M_WAITOK);
1337 	notbufdflushes = counter_u64_alloc(M_WAITOK);
1338 	buffreekvacnt = counter_u64_alloc(M_WAITOK);
1339 	bufdefragcnt = counter_u64_alloc(M_WAITOK);
1340 	bufkvaspace = counter_u64_alloc(M_WAITOK);
1341 	TSEXIT();
1342 }
1343 
1344 #ifdef INVARIANTS
1345 static inline void
1346 vfs_buf_check_mapped(struct buf *bp)
1347 {
1348 
1349 	KASSERT(bp->b_kvabase != unmapped_buf,
1350 	    ("mapped buf: b_kvabase was not updated %p", bp));
1351 	KASSERT(bp->b_data != unmapped_buf,
1352 	    ("mapped buf: b_data was not updated %p", bp));
1353 	KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf +
1354 	    maxphys, ("b_data + b_offset unmapped %p", bp));
1355 }
1356 
1357 static inline void
1358 vfs_buf_check_unmapped(struct buf *bp)
1359 {
1360 
1361 	KASSERT(bp->b_data == unmapped_buf,
1362 	    ("unmapped buf: corrupted b_data %p", bp));
1363 }
1364 
1365 #define	BUF_CHECK_MAPPED(bp) vfs_buf_check_mapped(bp)
1366 #define	BUF_CHECK_UNMAPPED(bp) vfs_buf_check_unmapped(bp)
1367 #else
1368 #define	BUF_CHECK_MAPPED(bp) do {} while (0)
1369 #define	BUF_CHECK_UNMAPPED(bp) do {} while (0)
1370 #endif
1371 
1372 static int
1373 isbufbusy(struct buf *bp)
1374 {
1375 	if (((bp->b_flags & B_INVAL) == 0 && BUF_ISLOCKED(bp)) ||
1376 	    ((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI))
1377 		return (1);
1378 	return (0);
1379 }
1380 
1381 /*
1382  * Shutdown the system cleanly to prepare for reboot, halt, or power off.
1383  */
1384 void
1385 bufshutdown(int show_busybufs)
1386 {
1387 	static int first_buf_printf = 1;
1388 	struct buf *bp;
1389 	int i, iter, nbusy, pbusy;
1390 #ifndef PREEMPTION
1391 	int subiter;
1392 #endif
1393 
1394 	/*
1395 	 * Sync filesystems for shutdown
1396 	 */
1397 	wdog_kern_pat(WD_LASTVAL);
1398 	kern_sync(curthread);
1399 
1400 	/*
1401 	 * With soft updates, some buffers that are
1402 	 * written will be remarked as dirty until other
1403 	 * buffers are written.
1404 	 */
1405 	for (iter = pbusy = 0; iter < 20; iter++) {
1406 		nbusy = 0;
1407 		for (i = nbuf - 1; i >= 0; i--) {
1408 			bp = nbufp(i);
1409 			if (isbufbusy(bp))
1410 				nbusy++;
1411 		}
1412 		if (nbusy == 0) {
1413 			if (first_buf_printf)
1414 				printf("All buffers synced.");
1415 			break;
1416 		}
1417 		if (first_buf_printf) {
1418 			printf("Syncing disks, buffers remaining... ");
1419 			first_buf_printf = 0;
1420 		}
1421 		printf("%d ", nbusy);
1422 		if (nbusy < pbusy)
1423 			iter = 0;
1424 		pbusy = nbusy;
1425 
1426 		wdog_kern_pat(WD_LASTVAL);
1427 		kern_sync(curthread);
1428 
1429 #ifdef PREEMPTION
1430 		/*
1431 		 * Spin for a while to allow interrupt threads to run.
1432 		 */
1433 		DELAY(50000 * iter);
1434 #else
1435 		/*
1436 		 * Context switch several times to allow interrupt
1437 		 * threads to run.
1438 		 */
1439 		for (subiter = 0; subiter < 50 * iter; subiter++) {
1440 			sched_relinquish(curthread);
1441 			DELAY(1000);
1442 		}
1443 #endif
1444 	}
1445 	printf("\n");
1446 	/*
1447 	 * Count only busy local buffers to prevent forcing
1448 	 * a fsck if we're just a client of a wedged NFS server
1449 	 */
1450 	nbusy = 0;
1451 	for (i = nbuf - 1; i >= 0; i--) {
1452 		bp = nbufp(i);
1453 		if (isbufbusy(bp)) {
1454 #if 0
1455 /* XXX: This is bogus.  We should probably have a BO_REMOTE flag instead */
1456 			if (bp->b_dev == NULL) {
1457 				TAILQ_REMOVE(&mountlist,
1458 				    bp->b_vp->v_mount, mnt_list);
1459 				continue;
1460 			}
1461 #endif
1462 			nbusy++;
1463 			if (show_busybufs > 0) {
1464 				printf(
1465 	    "%d: buf:%p, vnode:%p, flags:%0x, blkno:%jd, lblkno:%jd, buflock:",
1466 				    nbusy, bp, bp->b_vp, bp->b_flags,
1467 				    (intmax_t)bp->b_blkno,
1468 				    (intmax_t)bp->b_lblkno);
1469 				BUF_LOCKPRINTINFO(bp);
1470 				if (show_busybufs > 1)
1471 					vn_printf(bp->b_vp,
1472 					    "vnode content: ");
1473 			}
1474 		}
1475 	}
1476 	if (nbusy) {
1477 		/*
1478 		 * Failed to sync all blocks. Indicate this and don't
1479 		 * unmount filesystems (thus forcing an fsck on reboot).
1480 		 */
1481 		BOOTTRACE("shutdown failed to sync buffers");
1482 		printf("Giving up on %d buffers\n", nbusy);
1483 		DELAY(5000000);	/* 5 seconds */
1484 		swapoff_all();
1485 	} else {
1486 		BOOTTRACE("shutdown sync complete");
1487 		if (!first_buf_printf)
1488 			printf("Final sync complete\n");
1489 
1490 		/*
1491 		 * Unmount filesystems and perform swapoff, to quiesce
1492 		 * the system as much as possible.  In particular, no
1493 		 * I/O should be initiated from top levels since it
1494 		 * might be abruptly terminated by reset, or otherwise
1495 		 * erronously handled because other parts of the
1496 		 * system are disabled.
1497 		 *
1498 		 * Swapoff before unmount, because file-backed swap is
1499 		 * non-operational after unmount of the underlying
1500 		 * filesystem.
1501 		 */
1502 		if (!KERNEL_PANICKED()) {
1503 			swapoff_all();
1504 			vfs_unmountall();
1505 		}
1506 		BOOTTRACE("shutdown unmounted all filesystems");
1507 	}
1508 	DELAY(100000);		/* wait for console output to finish */
1509 }
1510 
1511 static void
1512 bpmap_qenter(struct buf *bp)
1513 {
1514 
1515 	BUF_CHECK_MAPPED(bp);
1516 
1517 	/*
1518 	 * bp->b_data is relative to bp->b_offset, but
1519 	 * bp->b_offset may be offset into the first page.
1520 	 */
1521 	bp->b_data = (caddr_t)trunc_page((vm_offset_t)bp->b_data);
1522 	pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
1523 	bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
1524 	    (vm_offset_t)(bp->b_offset & PAGE_MASK));
1525 }
1526 
1527 static inline struct bufdomain *
1528 bufdomain(struct buf *bp)
1529 {
1530 
1531 	return (&bdomain[bp->b_domain]);
1532 }
1533 
1534 static struct bufqueue *
1535 bufqueue(struct buf *bp)
1536 {
1537 
1538 	switch (bp->b_qindex) {
1539 	case QUEUE_NONE:
1540 		/* FALLTHROUGH */
1541 	case QUEUE_SENTINEL:
1542 		return (NULL);
1543 	case QUEUE_EMPTY:
1544 		return (&bqempty);
1545 	case QUEUE_DIRTY:
1546 		return (&bufdomain(bp)->bd_dirtyq);
1547 	case QUEUE_CLEAN:
1548 		return (&bufdomain(bp)->bd_subq[bp->b_subqueue]);
1549 	default:
1550 		break;
1551 	}
1552 	panic("bufqueue(%p): Unhandled type %d\n", bp, bp->b_qindex);
1553 }
1554 
1555 /*
1556  * Return the locked bufqueue that bp is a member of.
1557  */
1558 static struct bufqueue *
1559 bufqueue_acquire(struct buf *bp)
1560 {
1561 	struct bufqueue *bq, *nbq;
1562 
1563 	/*
1564 	 * bp can be pushed from a per-cpu queue to the
1565 	 * cleanq while we're waiting on the lock.  Retry
1566 	 * if the queues don't match.
1567 	 */
1568 	bq = bufqueue(bp);
1569 	BQ_LOCK(bq);
1570 	for (;;) {
1571 		nbq = bufqueue(bp);
1572 		if (bq == nbq)
1573 			break;
1574 		BQ_UNLOCK(bq);
1575 		BQ_LOCK(nbq);
1576 		bq = nbq;
1577 	}
1578 	return (bq);
1579 }
1580 
1581 /*
1582  *	binsfree:
1583  *
1584  *	Insert the buffer into the appropriate free list.  Requires a
1585  *	locked buffer on entry and buffer is unlocked before return.
1586  */
1587 static void
1588 binsfree(struct buf *bp, int qindex)
1589 {
1590 	struct bufdomain *bd;
1591 	struct bufqueue *bq;
1592 
1593 	KASSERT(qindex == QUEUE_CLEAN || qindex == QUEUE_DIRTY,
1594 	    ("binsfree: Invalid qindex %d", qindex));
1595 	BUF_ASSERT_XLOCKED(bp);
1596 
1597 	/*
1598 	 * Handle delayed bremfree() processing.
1599 	 */
1600 	if (bp->b_flags & B_REMFREE) {
1601 		if (bp->b_qindex == qindex) {
1602 			bp->b_flags |= B_REUSE;
1603 			bp->b_flags &= ~B_REMFREE;
1604 			BUF_UNLOCK(bp);
1605 			return;
1606 		}
1607 		bq = bufqueue_acquire(bp);
1608 		bq_remove(bq, bp);
1609 		BQ_UNLOCK(bq);
1610 	}
1611 	bd = bufdomain(bp);
1612 	if (qindex == QUEUE_CLEAN) {
1613 		if (bd->bd_lim != 0)
1614 			bq = &bd->bd_subq[PCPU_GET(cpuid)];
1615 		else
1616 			bq = bd->bd_cleanq;
1617 	} else
1618 		bq = &bd->bd_dirtyq;
1619 	bq_insert(bq, bp, true);
1620 }
1621 
1622 /*
1623  * buf_free:
1624  *
1625  *	Free a buffer to the buf zone once it no longer has valid contents.
1626  */
1627 static void
1628 buf_free(struct buf *bp)
1629 {
1630 
1631 	if (bp->b_flags & B_REMFREE)
1632 		bremfreef(bp);
1633 	if (bp->b_vflags & BV_BKGRDINPROG)
1634 		panic("losing buffer 1");
1635 	if (bp->b_rcred != NOCRED) {
1636 		crfree(bp->b_rcred);
1637 		bp->b_rcred = NOCRED;
1638 	}
1639 	if (bp->b_wcred != NOCRED) {
1640 		crfree(bp->b_wcred);
1641 		bp->b_wcred = NOCRED;
1642 	}
1643 	if (!LIST_EMPTY(&bp->b_dep))
1644 		buf_deallocate(bp);
1645 	bufkva_free(bp);
1646 	atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1);
1647 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
1648 	BUF_UNLOCK(bp);
1649 	uma_zfree(buf_zone, bp);
1650 }
1651 
1652 /*
1653  * buf_import:
1654  *
1655  *	Import bufs into the uma cache from the buf list.  The system still
1656  *	expects a static array of bufs and much of the synchronization
1657  *	around bufs assumes type stable storage.  As a result, UMA is used
1658  *	only as a per-cpu cache of bufs still maintained on a global list.
1659  */
1660 static int
1661 buf_import(void *arg, void **store, int cnt, int domain, int flags)
1662 {
1663 	struct buf *bp;
1664 	int i;
1665 
1666 	BQ_LOCK(&bqempty);
1667 	for (i = 0; i < cnt; i++) {
1668 		bp = TAILQ_FIRST(&bqempty.bq_queue);
1669 		if (bp == NULL)
1670 			break;
1671 		bq_remove(&bqempty, bp);
1672 		store[i] = bp;
1673 	}
1674 	BQ_UNLOCK(&bqempty);
1675 
1676 	return (i);
1677 }
1678 
1679 /*
1680  * buf_release:
1681  *
1682  *	Release bufs from the uma cache back to the buffer queues.
1683  */
1684 static void
1685 buf_release(void *arg, void **store, int cnt)
1686 {
1687 	struct bufqueue *bq;
1688 	struct buf *bp;
1689         int i;
1690 
1691 	bq = &bqempty;
1692 	BQ_LOCK(bq);
1693         for (i = 0; i < cnt; i++) {
1694 		bp = store[i];
1695 		/* Inline bq_insert() to batch locking. */
1696 		TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
1697 		bp->b_flags &= ~(B_AGE | B_REUSE);
1698 		bq->bq_len++;
1699 		bp->b_qindex = bq->bq_index;
1700 	}
1701 	BQ_UNLOCK(bq);
1702 }
1703 
1704 /*
1705  * buf_alloc:
1706  *
1707  *	Allocate an empty buffer header.
1708  */
1709 static struct buf *
1710 buf_alloc(struct bufdomain *bd)
1711 {
1712 	struct buf *bp;
1713 	int freebufs, error;
1714 
1715 	/*
1716 	 * We can only run out of bufs in the buf zone if the average buf
1717 	 * is less than BKVASIZE.  In this case the actual wait/block will
1718 	 * come from buf_reycle() failing to flush one of these small bufs.
1719 	 */
1720 	bp = NULL;
1721 	freebufs = atomic_fetchadd_int(&bd->bd_freebuffers, -1);
1722 	if (freebufs > 0)
1723 		bp = uma_zalloc(buf_zone, M_NOWAIT);
1724 	if (bp == NULL) {
1725 		atomic_add_int(&bd->bd_freebuffers, 1);
1726 		bufspace_daemon_wakeup(bd);
1727 		counter_u64_add(numbufallocfails, 1);
1728 		return (NULL);
1729 	}
1730 	/*
1731 	 * Wake-up the bufspace daemon on transition below threshold.
1732 	 */
1733 	if (freebufs == bd->bd_lofreebuffers)
1734 		bufspace_daemon_wakeup(bd);
1735 
1736 	error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWITNESS, NULL);
1737 	KASSERT(error == 0, ("%s: BUF_LOCK on free buf %p: %d.", __func__, bp,
1738 	    error));
1739 	(void)error;
1740 
1741 	KASSERT(bp->b_vp == NULL,
1742 	    ("bp: %p still has vnode %p.", bp, bp->b_vp));
1743 	KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
1744 	    ("invalid buffer %p flags %#x", bp, bp->b_flags));
1745 	KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
1746 	    ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
1747 	KASSERT(bp->b_npages == 0,
1748 	    ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
1749 	KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
1750 	KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));
1751 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
1752 
1753 	bp->b_domain = BD_DOMAIN(bd);
1754 	bp->b_flags = 0;
1755 	bp->b_ioflags = 0;
1756 	bp->b_xflags = 0;
1757 	bp->b_vflags = 0;
1758 	bp->b_vp = NULL;
1759 	bp->b_blkno = bp->b_lblkno = 0;
1760 	bp->b_offset = NOOFFSET;
1761 	bp->b_iodone = 0;
1762 	bp->b_error = 0;
1763 	bp->b_resid = 0;
1764 	bp->b_bcount = 0;
1765 	bp->b_npages = 0;
1766 	bp->b_dirtyoff = bp->b_dirtyend = 0;
1767 	bp->b_bufobj = NULL;
1768 	bp->b_data = bp->b_kvabase = unmapped_buf;
1769 	bp->b_fsprivate1 = NULL;
1770 	bp->b_fsprivate2 = NULL;
1771 	bp->b_fsprivate3 = NULL;
1772 	LIST_INIT(&bp->b_dep);
1773 
1774 	return (bp);
1775 }
1776 
1777 /*
1778  *	buf_recycle:
1779  *
1780  *	Free a buffer from the given bufqueue.  kva controls whether the
1781  *	freed buf must own some kva resources.  This is used for
1782  *	defragmenting.
1783  */
1784 static int
1785 buf_recycle(struct bufdomain *bd, bool kva)
1786 {
1787 	struct bufqueue *bq;
1788 	struct buf *bp, *nbp;
1789 
1790 	if (kva)
1791 		counter_u64_add(bufdefragcnt, 1);
1792 	nbp = NULL;
1793 	bq = bd->bd_cleanq;
1794 	BQ_LOCK(bq);
1795 	KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd),
1796 	    ("buf_recycle: Locks don't match"));
1797 	nbp = TAILQ_FIRST(&bq->bq_queue);
1798 
1799 	/*
1800 	 * Run scan, possibly freeing data and/or kva mappings on the fly
1801 	 * depending.
1802 	 */
1803 	while ((bp = nbp) != NULL) {
1804 		/*
1805 		 * Calculate next bp (we can only use it if we do not
1806 		 * release the bqlock).
1807 		 */
1808 		nbp = TAILQ_NEXT(bp, b_freelist);
1809 
1810 		/*
1811 		 * If we are defragging then we need a buffer with
1812 		 * some kva to reclaim.
1813 		 */
1814 		if (kva && bp->b_kvasize == 0)
1815 			continue;
1816 
1817 		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
1818 			continue;
1819 
1820 		/*
1821 		 * Implement a second chance algorithm for frequently
1822 		 * accessed buffers.
1823 		 */
1824 		if ((bp->b_flags & B_REUSE) != 0) {
1825 			TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
1826 			TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
1827 			bp->b_flags &= ~B_REUSE;
1828 			BUF_UNLOCK(bp);
1829 			continue;
1830 		}
1831 
1832 		/*
1833 		 * Skip buffers with background writes in progress.
1834 		 */
1835 		if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
1836 			BUF_UNLOCK(bp);
1837 			continue;
1838 		}
1839 
1840 		KASSERT(bp->b_qindex == QUEUE_CLEAN,
1841 		    ("buf_recycle: inconsistent queue %d bp %p",
1842 		    bp->b_qindex, bp));
1843 		KASSERT(bp->b_domain == BD_DOMAIN(bd),
1844 		    ("getnewbuf: queue domain %d doesn't match request %d",
1845 		    bp->b_domain, (int)BD_DOMAIN(bd)));
1846 		/*
1847 		 * NOTE:  nbp is now entirely invalid.  We can only restart
1848 		 * the scan from this point on.
1849 		 */
1850 		bq_remove(bq, bp);
1851 		BQ_UNLOCK(bq);
1852 
1853 		/*
1854 		 * Requeue the background write buffer with error and
1855 		 * restart the scan.
1856 		 */
1857 		if ((bp->b_vflags & BV_BKGRDERR) != 0) {
1858 			bqrelse(bp);
1859 			BQ_LOCK(bq);
1860 			nbp = TAILQ_FIRST(&bq->bq_queue);
1861 			continue;
1862 		}
1863 		bp->b_flags |= B_INVAL;
1864 		brelse(bp);
1865 		return (0);
1866 	}
1867 	bd->bd_wanted = 1;
1868 	BQ_UNLOCK(bq);
1869 
1870 	return (ENOBUFS);
1871 }
1872 
1873 /*
1874  *	bremfree:
1875  *
1876  *	Mark the buffer for removal from the appropriate free list.
1877  *
1878  */
1879 void
1880 bremfree(struct buf *bp)
1881 {
1882 
1883 	CTR3(KTR_BUF, "bremfree(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1884 	KASSERT((bp->b_flags & B_REMFREE) == 0,
1885 	    ("bremfree: buffer %p already marked for delayed removal.", bp));
1886 	KASSERT(bp->b_qindex != QUEUE_NONE,
1887 	    ("bremfree: buffer %p not on a queue.", bp));
1888 	BUF_ASSERT_XLOCKED(bp);
1889 
1890 	bp->b_flags |= B_REMFREE;
1891 }
1892 
1893 /*
1894  *	bremfreef:
1895  *
1896  *	Force an immediate removal from a free list.  Used only in nfs when
1897  *	it abuses the b_freelist pointer.
1898  */
1899 void
1900 bremfreef(struct buf *bp)
1901 {
1902 	struct bufqueue *bq;
1903 
1904 	bq = bufqueue_acquire(bp);
1905 	bq_remove(bq, bp);
1906 	BQ_UNLOCK(bq);
1907 }
1908 
1909 static void
1910 bq_init(struct bufqueue *bq, int qindex, int subqueue, const char *lockname)
1911 {
1912 
1913 	mtx_init(&bq->bq_lock, lockname, NULL, MTX_DEF);
1914 	TAILQ_INIT(&bq->bq_queue);
1915 	bq->bq_len = 0;
1916 	bq->bq_index = qindex;
1917 	bq->bq_subqueue = subqueue;
1918 }
1919 
1920 static void
1921 bd_init(struct bufdomain *bd)
1922 {
1923 	int i;
1924 
1925 	/* Per-CPU clean buf queues, plus one global queue. */
1926 	bd->bd_subq = mallocarray(mp_maxid + 2, sizeof(struct bufqueue),
1927 	    M_BIOBUF, M_WAITOK | M_ZERO);
1928 	bd->bd_cleanq = &bd->bd_subq[mp_maxid + 1];
1929 	bq_init(bd->bd_cleanq, QUEUE_CLEAN, mp_maxid + 1, "bufq clean lock");
1930 	bq_init(&bd->bd_dirtyq, QUEUE_DIRTY, -1, "bufq dirty lock");
1931 	for (i = 0; i <= mp_maxid; i++)
1932 		bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i,
1933 		    "bufq clean subqueue lock");
1934 	mtx_init(&bd->bd_run_lock, "bufspace daemon run lock", NULL, MTX_DEF);
1935 }
1936 
1937 /*
1938  *	bq_remove:
1939  *
1940  *	Removes a buffer from the free list, must be called with the
1941  *	correct qlock held.
1942  */
1943 static void
1944 bq_remove(struct bufqueue *bq, struct buf *bp)
1945 {
1946 
1947 	CTR3(KTR_BUF, "bq_remove(%p) vp %p flags %X",
1948 	    bp, bp->b_vp, bp->b_flags);
1949 	KASSERT(bp->b_qindex != QUEUE_NONE,
1950 	    ("bq_remove: buffer %p not on a queue.", bp));
1951 	KASSERT(bufqueue(bp) == bq,
1952 	    ("bq_remove: Remove buffer %p from wrong queue.", bp));
1953 
1954 	BQ_ASSERT_LOCKED(bq);
1955 	if (bp->b_qindex != QUEUE_EMPTY) {
1956 		BUF_ASSERT_XLOCKED(bp);
1957 	}
1958 	KASSERT(bq->bq_len >= 1,
1959 	    ("queue %d underflow", bp->b_qindex));
1960 	TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
1961 	bq->bq_len--;
1962 	bp->b_qindex = QUEUE_NONE;
1963 	bp->b_flags &= ~(B_REMFREE | B_REUSE);
1964 }
1965 
1966 static void
1967 bd_flush(struct bufdomain *bd, struct bufqueue *bq)
1968 {
1969 	struct buf *bp;
1970 
1971 	BQ_ASSERT_LOCKED(bq);
1972 	if (bq != bd->bd_cleanq) {
1973 		BD_LOCK(bd);
1974 		while ((bp = TAILQ_FIRST(&bq->bq_queue)) != NULL) {
1975 			TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
1976 			TAILQ_INSERT_TAIL(&bd->bd_cleanq->bq_queue, bp,
1977 			    b_freelist);
1978 			bp->b_subqueue = bd->bd_cleanq->bq_subqueue;
1979 		}
1980 		bd->bd_cleanq->bq_len += bq->bq_len;
1981 		bq->bq_len = 0;
1982 	}
1983 	if (bd->bd_wanted) {
1984 		bd->bd_wanted = 0;
1985 		wakeup(&bd->bd_wanted);
1986 	}
1987 	if (bq != bd->bd_cleanq)
1988 		BD_UNLOCK(bd);
1989 }
1990 
1991 static int
1992 bd_flushall(struct bufdomain *bd)
1993 {
1994 	struct bufqueue *bq;
1995 	int flushed;
1996 	int i;
1997 
1998 	if (bd->bd_lim == 0)
1999 		return (0);
2000 	flushed = 0;
2001 	for (i = 0; i <= mp_maxid; i++) {
2002 		bq = &bd->bd_subq[i];
2003 		if (bq->bq_len == 0)
2004 			continue;
2005 		BQ_LOCK(bq);
2006 		bd_flush(bd, bq);
2007 		BQ_UNLOCK(bq);
2008 		flushed++;
2009 	}
2010 
2011 	return (flushed);
2012 }
2013 
2014 static void
2015 bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock)
2016 {
2017 	struct bufdomain *bd;
2018 
2019 	if (bp->b_qindex != QUEUE_NONE)
2020 		panic("bq_insert: free buffer %p onto another queue?", bp);
2021 
2022 	bd = bufdomain(bp);
2023 	if (bp->b_flags & B_AGE) {
2024 		/* Place this buf directly on the real queue. */
2025 		if (bq->bq_index == QUEUE_CLEAN)
2026 			bq = bd->bd_cleanq;
2027 		BQ_LOCK(bq);
2028 		TAILQ_INSERT_HEAD(&bq->bq_queue, bp, b_freelist);
2029 	} else {
2030 		BQ_LOCK(bq);
2031 		TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
2032 	}
2033 	bp->b_flags &= ~(B_AGE | B_REUSE);
2034 	bq->bq_len++;
2035 	bp->b_qindex = bq->bq_index;
2036 	bp->b_subqueue = bq->bq_subqueue;
2037 
2038 	/*
2039 	 * Unlock before we notify so that we don't wakeup a waiter that
2040 	 * fails a trylock on the buf and sleeps again.
2041 	 */
2042 	if (unlock)
2043 		BUF_UNLOCK(bp);
2044 
2045 	if (bp->b_qindex == QUEUE_CLEAN) {
2046 		/*
2047 		 * Flush the per-cpu queue and notify any waiters.
2048 		 */
2049 		if (bd->bd_wanted || (bq != bd->bd_cleanq &&
2050 		    bq->bq_len >= bd->bd_lim))
2051 			bd_flush(bd, bq);
2052 	}
2053 	BQ_UNLOCK(bq);
2054 }
2055 
2056 /*
2057  *	bufkva_free:
2058  *
2059  *	Free the kva allocation for a buffer.
2060  *
2061  */
2062 static void
2063 bufkva_free(struct buf *bp)
2064 {
2065 
2066 #ifdef INVARIANTS
2067 	if (bp->b_kvasize == 0) {
2068 		KASSERT(bp->b_kvabase == unmapped_buf &&
2069 		    bp->b_data == unmapped_buf,
2070 		    ("Leaked KVA space on %p", bp));
2071 	} else if (buf_mapped(bp))
2072 		BUF_CHECK_MAPPED(bp);
2073 	else
2074 		BUF_CHECK_UNMAPPED(bp);
2075 #endif
2076 	if (bp->b_kvasize == 0)
2077 		return;
2078 
2079 	vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
2080 	counter_u64_add(bufkvaspace, -bp->b_kvasize);
2081 	counter_u64_add(buffreekvacnt, 1);
2082 	bp->b_data = bp->b_kvabase = unmapped_buf;
2083 	bp->b_kvasize = 0;
2084 }
2085 
2086 /*
2087  *	bufkva_alloc:
2088  *
2089  *	Allocate the buffer KVA and set b_kvasize and b_kvabase.
2090  */
2091 static int
2092 bufkva_alloc(struct buf *bp, int maxsize, int gbflags)
2093 {
2094 	vm_offset_t addr;
2095 	int error;
2096 
2097 	KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0,
2098 	    ("Invalid gbflags 0x%x in %s", gbflags, __func__));
2099 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
2100 	KASSERT(maxsize <= maxbcachebuf,
2101 	    ("bufkva_alloc kva too large %d %u", maxsize, maxbcachebuf));
2102 
2103 	bufkva_free(bp);
2104 
2105 	addr = 0;
2106 	error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr);
2107 	if (error != 0) {
2108 		/*
2109 		 * Buffer map is too fragmented.  Request the caller
2110 		 * to defragment the map.
2111 		 */
2112 		return (error);
2113 	}
2114 	bp->b_kvabase = (caddr_t)addr;
2115 	bp->b_kvasize = maxsize;
2116 	counter_u64_add(bufkvaspace, bp->b_kvasize);
2117 	if ((gbflags & GB_UNMAPPED) != 0) {
2118 		bp->b_data = unmapped_buf;
2119 		BUF_CHECK_UNMAPPED(bp);
2120 	} else {
2121 		bp->b_data = bp->b_kvabase;
2122 		BUF_CHECK_MAPPED(bp);
2123 	}
2124 	return (0);
2125 }
2126 
2127 /*
2128  *	bufkva_reclaim:
2129  *
2130  *	Reclaim buffer kva by freeing buffers holding kva.  This is a vmem
2131  *	callback that fires to avoid returning failure.
2132  */
2133 static void
2134 bufkva_reclaim(vmem_t *vmem, int flags)
2135 {
2136 	bool done;
2137 	int q;
2138 	int i;
2139 
2140 	done = false;
2141 	for (i = 0; i < 5; i++) {
2142 		for (q = 0; q < buf_domains; q++)
2143 			if (buf_recycle(&bdomain[q], true) != 0)
2144 				done = true;
2145 		if (done)
2146 			break;
2147 	}
2148 	return;
2149 }
2150 
2151 /*
2152  * Attempt to initiate asynchronous I/O on read-ahead blocks.  We must
2153  * clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
2154  * the buffer is valid and we do not have to do anything.
2155  */
2156 static void
2157 breada(struct vnode * vp, daddr_t * rablkno, int * rabsize, int cnt,
2158     struct ucred * cred, int flags, void (*ckhashfunc)(struct buf *))
2159 {
2160 	struct buf *rabp;
2161 	struct thread *td;
2162 	int i;
2163 
2164 	td = curthread;
2165 
2166 	for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
2167 		if (inmem(vp, *rablkno))
2168 			continue;
2169 		rabp = getblk(vp, *rablkno, *rabsize, 0, 0, 0);
2170 		if ((rabp->b_flags & B_CACHE) != 0) {
2171 			brelse(rabp);
2172 			continue;
2173 		}
2174 #ifdef RACCT
2175 		if (racct_enable) {
2176 			PROC_LOCK(curproc);
2177 			racct_add_buf(curproc, rabp, 0);
2178 			PROC_UNLOCK(curproc);
2179 		}
2180 #endif /* RACCT */
2181 		td->td_ru.ru_inblock++;
2182 		rabp->b_flags |= B_ASYNC;
2183 		rabp->b_flags &= ~B_INVAL;
2184 		if ((flags & GB_CKHASH) != 0) {
2185 			rabp->b_flags |= B_CKHASH;
2186 			rabp->b_ckhashcalc = ckhashfunc;
2187 		}
2188 		rabp->b_ioflags &= ~BIO_ERROR;
2189 		rabp->b_iocmd = BIO_READ;
2190 		if (rabp->b_rcred == NOCRED && cred != NOCRED)
2191 			rabp->b_rcred = crhold(cred);
2192 		vfs_busy_pages(rabp, 0);
2193 		BUF_KERNPROC(rabp);
2194 		rabp->b_iooffset = dbtob(rabp->b_blkno);
2195 		bstrategy(rabp);
2196 	}
2197 }
2198 
2199 /*
2200  * Entry point for bread() and breadn() via #defines in sys/buf.h.
2201  *
2202  * Get a buffer with the specified data.  Look in the cache first.  We
2203  * must clear BIO_ERROR and B_INVAL prior to initiating I/O.  If B_CACHE
2204  * is set, the buffer is valid and we do not have to do anything, see
2205  * getblk(). Also starts asynchronous I/O on read-ahead blocks.
2206  *
2207  * Always return a NULL buffer pointer (in bpp) when returning an error.
2208  *
2209  * The blkno parameter is the logical block being requested. Normally
2210  * the mapping of logical block number to disk block address is done
2211  * by calling VOP_BMAP(). However, if the mapping is already known, the
2212  * disk block address can be passed using the dblkno parameter. If the
2213  * disk block address is not known, then the same value should be passed
2214  * for blkno and dblkno.
2215  */
2216 int
2217 breadn_flags(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size,
2218     daddr_t *rablkno, int *rabsize, int cnt, struct ucred *cred, int flags,
2219     void (*ckhashfunc)(struct buf *), struct buf **bpp)
2220 {
2221 	struct buf *bp;
2222 	struct thread *td;
2223 	int error, readwait, rv;
2224 
2225 	CTR3(KTR_BUF, "breadn(%p, %jd, %d)", vp, blkno, size);
2226 	td = curthread;
2227 	/*
2228 	 * Can only return NULL if GB_LOCK_NOWAIT or GB_SPARSE flags
2229 	 * are specified.
2230 	 */
2231 	error = getblkx(vp, blkno, dblkno, size, 0, 0, flags, &bp);
2232 	if (error != 0) {
2233 		*bpp = NULL;
2234 		return (error);
2235 	}
2236 	KASSERT(blkno == bp->b_lblkno,
2237 	    ("getblkx returned buffer for blkno %jd instead of blkno %jd",
2238 	    (intmax_t)bp->b_lblkno, (intmax_t)blkno));
2239 	flags &= ~GB_NOSPARSE;
2240 	*bpp = bp;
2241 
2242 	/*
2243 	 * If not found in cache, do some I/O
2244 	 */
2245 	readwait = 0;
2246 	if ((bp->b_flags & B_CACHE) == 0) {
2247 #ifdef RACCT
2248 		if (racct_enable) {
2249 			PROC_LOCK(td->td_proc);
2250 			racct_add_buf(td->td_proc, bp, 0);
2251 			PROC_UNLOCK(td->td_proc);
2252 		}
2253 #endif /* RACCT */
2254 		td->td_ru.ru_inblock++;
2255 		bp->b_iocmd = BIO_READ;
2256 		bp->b_flags &= ~B_INVAL;
2257 		if ((flags & GB_CKHASH) != 0) {
2258 			bp->b_flags |= B_CKHASH;
2259 			bp->b_ckhashcalc = ckhashfunc;
2260 		}
2261 		if ((flags & GB_CVTENXIO) != 0)
2262 			bp->b_xflags |= BX_CVTENXIO;
2263 		bp->b_ioflags &= ~BIO_ERROR;
2264 		if (bp->b_rcred == NOCRED && cred != NOCRED)
2265 			bp->b_rcred = crhold(cred);
2266 		vfs_busy_pages(bp, 0);
2267 		bp->b_iooffset = dbtob(bp->b_blkno);
2268 		bstrategy(bp);
2269 		++readwait;
2270 	}
2271 
2272 	/*
2273 	 * Attempt to initiate asynchronous I/O on read-ahead blocks.
2274 	 */
2275 	breada(vp, rablkno, rabsize, cnt, cred, flags, ckhashfunc);
2276 
2277 	rv = 0;
2278 	if (readwait) {
2279 		rv = bufwait(bp);
2280 		if (rv != 0) {
2281 			brelse(bp);
2282 			*bpp = NULL;
2283 		}
2284 	}
2285 	return (rv);
2286 }
2287 
2288 /*
2289  * Write, release buffer on completion.  (Done by iodone
2290  * if async).  Do not bother writing anything if the buffer
2291  * is invalid.
2292  *
2293  * Note that we set B_CACHE here, indicating that buffer is
2294  * fully valid and thus cacheable.  This is true even of NFS
2295  * now so we set it generally.  This could be set either here
2296  * or in biodone() since the I/O is synchronous.  We put it
2297  * here.
2298  */
2299 int
2300 bufwrite(struct buf *bp)
2301 {
2302 	int oldflags;
2303 	struct vnode *vp;
2304 	long space;
2305 	int vp_md;
2306 
2307 	CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2308 	if ((bp->b_bufobj->bo_flag & BO_DEAD) != 0) {
2309 		bp->b_flags |= B_INVAL | B_RELBUF;
2310 		bp->b_flags &= ~B_CACHE;
2311 		brelse(bp);
2312 		return (ENXIO);
2313 	}
2314 	if (bp->b_flags & B_INVAL) {
2315 		brelse(bp);
2316 		return (0);
2317 	}
2318 
2319 	if (bp->b_flags & B_BARRIER)
2320 		atomic_add_long(&barrierwrites, 1);
2321 
2322 	oldflags = bp->b_flags;
2323 
2324 	KASSERT(!(bp->b_vflags & BV_BKGRDINPROG),
2325 	    ("FFS background buffer should not get here %p", bp));
2326 
2327 	vp = bp->b_vp;
2328 	if (vp)
2329 		vp_md = vp->v_vflag & VV_MD;
2330 	else
2331 		vp_md = 0;
2332 
2333 	/*
2334 	 * Mark the buffer clean.  Increment the bufobj write count
2335 	 * before bundirty() call, to prevent other thread from seeing
2336 	 * empty dirty list and zero counter for writes in progress,
2337 	 * falsely indicating that the bufobj is clean.
2338 	 */
2339 	bufobj_wref(bp->b_bufobj);
2340 	bundirty(bp);
2341 
2342 	bp->b_flags &= ~B_DONE;
2343 	bp->b_ioflags &= ~BIO_ERROR;
2344 	bp->b_flags |= B_CACHE;
2345 	bp->b_iocmd = BIO_WRITE;
2346 
2347 	vfs_busy_pages(bp, 1);
2348 
2349 	/*
2350 	 * Normal bwrites pipeline writes
2351 	 */
2352 	bp->b_runningbufspace = bp->b_bufsize;
2353 	space = atomic_fetchadd_long(&runningbufspace, bp->b_runningbufspace);
2354 
2355 #ifdef RACCT
2356 	if (racct_enable) {
2357 		PROC_LOCK(curproc);
2358 		racct_add_buf(curproc, bp, 1);
2359 		PROC_UNLOCK(curproc);
2360 	}
2361 #endif /* RACCT */
2362 	curthread->td_ru.ru_oublock++;
2363 	if (oldflags & B_ASYNC)
2364 		BUF_KERNPROC(bp);
2365 	bp->b_iooffset = dbtob(bp->b_blkno);
2366 	buf_track(bp, __func__);
2367 	bstrategy(bp);
2368 
2369 	if ((oldflags & B_ASYNC) == 0) {
2370 		int rtval = bufwait(bp);
2371 		brelse(bp);
2372 		return (rtval);
2373 	} else if (space > hirunningspace) {
2374 		/*
2375 		 * don't allow the async write to saturate the I/O
2376 		 * system.  We will not deadlock here because
2377 		 * we are blocking waiting for I/O that is already in-progress
2378 		 * to complete. We do not block here if it is the update
2379 		 * or syncer daemon trying to clean up as that can lead
2380 		 * to deadlock.
2381 		 */
2382 		if ((curthread->td_pflags & TDP_NORUNNINGBUF) == 0 && !vp_md)
2383 			waitrunningbufspace();
2384 	}
2385 
2386 	return (0);
2387 }
2388 
2389 void
2390 bufbdflush(struct bufobj *bo, struct buf *bp)
2391 {
2392 	struct buf *nbp;
2393 	struct bufdomain *bd;
2394 
2395 	bd = &bdomain[bo->bo_domain];
2396 	if (bo->bo_dirty.bv_cnt > bd->bd_dirtybufthresh + 10) {
2397 		(void) VOP_FSYNC(bp->b_vp, MNT_NOWAIT, curthread);
2398 		altbufferflushes++;
2399 	} else if (bo->bo_dirty.bv_cnt > bd->bd_dirtybufthresh) {
2400 		BO_LOCK(bo);
2401 		/*
2402 		 * Try to find a buffer to flush.
2403 		 */
2404 		TAILQ_FOREACH(nbp, &bo->bo_dirty.bv_hd, b_bobufs) {
2405 			if ((nbp->b_vflags & BV_BKGRDINPROG) ||
2406 			    BUF_LOCK(nbp,
2407 				     LK_EXCLUSIVE | LK_NOWAIT, NULL))
2408 				continue;
2409 			if (bp == nbp)
2410 				panic("bdwrite: found ourselves");
2411 			BO_UNLOCK(bo);
2412 			/* Don't countdeps with the bo lock held. */
2413 			if (buf_countdeps(nbp, 0)) {
2414 				BO_LOCK(bo);
2415 				BUF_UNLOCK(nbp);
2416 				continue;
2417 			}
2418 			if (nbp->b_flags & B_CLUSTEROK) {
2419 				vfs_bio_awrite(nbp);
2420 			} else {
2421 				bremfree(nbp);
2422 				bawrite(nbp);
2423 			}
2424 			dirtybufferflushes++;
2425 			break;
2426 		}
2427 		if (nbp == NULL)
2428 			BO_UNLOCK(bo);
2429 	}
2430 }
2431 
2432 /*
2433  * Delayed write. (Buffer is marked dirty).  Do not bother writing
2434  * anything if the buffer is marked invalid.
2435  *
2436  * Note that since the buffer must be completely valid, we can safely
2437  * set B_CACHE.  In fact, we have to set B_CACHE here rather then in
2438  * biodone() in order to prevent getblk from writing the buffer
2439  * out synchronously.
2440  */
2441 void
2442 bdwrite(struct buf *bp)
2443 {
2444 	struct thread *td = curthread;
2445 	struct vnode *vp;
2446 	struct bufobj *bo;
2447 
2448 	CTR3(KTR_BUF, "bdwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2449 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2450 	KASSERT((bp->b_flags & B_BARRIER) == 0,
2451 	    ("Barrier request in delayed write %p", bp));
2452 
2453 	if (bp->b_flags & B_INVAL) {
2454 		brelse(bp);
2455 		return;
2456 	}
2457 
2458 	/*
2459 	 * If we have too many dirty buffers, don't create any more.
2460 	 * If we are wildly over our limit, then force a complete
2461 	 * cleanup. Otherwise, just keep the situation from getting
2462 	 * out of control. Note that we have to avoid a recursive
2463 	 * disaster and not try to clean up after our own cleanup!
2464 	 */
2465 	vp = bp->b_vp;
2466 	bo = bp->b_bufobj;
2467 	if ((td->td_pflags & (TDP_COWINPROGRESS|TDP_INBDFLUSH)) == 0) {
2468 		td->td_pflags |= TDP_INBDFLUSH;
2469 		BO_BDFLUSH(bo, bp);
2470 		td->td_pflags &= ~TDP_INBDFLUSH;
2471 	} else
2472 		recursiveflushes++;
2473 
2474 	bdirty(bp);
2475 	/*
2476 	 * Set B_CACHE, indicating that the buffer is fully valid.  This is
2477 	 * true even of NFS now.
2478 	 */
2479 	bp->b_flags |= B_CACHE;
2480 
2481 	/*
2482 	 * This bmap keeps the system from needing to do the bmap later,
2483 	 * perhaps when the system is attempting to do a sync.  Since it
2484 	 * is likely that the indirect block -- or whatever other datastructure
2485 	 * that the filesystem needs is still in memory now, it is a good
2486 	 * thing to do this.  Note also, that if the pageout daemon is
2487 	 * requesting a sync -- there might not be enough memory to do
2488 	 * the bmap then...  So, this is important to do.
2489 	 */
2490 	if (vp->v_type != VCHR && bp->b_lblkno == bp->b_blkno) {
2491 		VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
2492 	}
2493 
2494 	buf_track(bp, __func__);
2495 
2496 	/*
2497 	 * Set the *dirty* buffer range based upon the VM system dirty
2498 	 * pages.
2499 	 *
2500 	 * Mark the buffer pages as clean.  We need to do this here to
2501 	 * satisfy the vnode_pager and the pageout daemon, so that it
2502 	 * thinks that the pages have been "cleaned".  Note that since
2503 	 * the pages are in a delayed write buffer -- the VFS layer
2504 	 * "will" see that the pages get written out on the next sync,
2505 	 * or perhaps the cluster will be completed.
2506 	 */
2507 	vfs_clean_pages_dirty_buf(bp);
2508 	bqrelse(bp);
2509 
2510 	/*
2511 	 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
2512 	 * due to the softdep code.
2513 	 */
2514 }
2515 
2516 /*
2517  *	bdirty:
2518  *
2519  *	Turn buffer into delayed write request.  We must clear BIO_READ and
2520  *	B_RELBUF, and we must set B_DELWRI.  We reassign the buffer to
2521  *	itself to properly update it in the dirty/clean lists.  We mark it
2522  *	B_DONE to ensure that any asynchronization of the buffer properly
2523  *	clears B_DONE ( else a panic will occur later ).
2524  *
2525  *	bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
2526  *	might have been set pre-getblk().  Unlike bwrite/bdwrite, bdirty()
2527  *	should only be called if the buffer is known-good.
2528  *
2529  *	Since the buffer is not on a queue, we do not update the numfreebuffers
2530  *	count.
2531  *
2532  *	The buffer must be on QUEUE_NONE.
2533  */
2534 void
2535 bdirty(struct buf *bp)
2536 {
2537 
2538 	CTR3(KTR_BUF, "bdirty(%p) vp %p flags %X",
2539 	    bp, bp->b_vp, bp->b_flags);
2540 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2541 	KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
2542 	    ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
2543 	bp->b_flags &= ~(B_RELBUF);
2544 	bp->b_iocmd = BIO_WRITE;
2545 
2546 	if ((bp->b_flags & B_DELWRI) == 0) {
2547 		bp->b_flags |= /* XXX B_DONE | */ B_DELWRI;
2548 		reassignbuf(bp);
2549 		bdirtyadd(bp);
2550 	}
2551 }
2552 
2553 /*
2554  *	bundirty:
2555  *
2556  *	Clear B_DELWRI for buffer.
2557  *
2558  *	Since the buffer is not on a queue, we do not update the numfreebuffers
2559  *	count.
2560  *
2561  *	The buffer must be on QUEUE_NONE.
2562  */
2563 
2564 void
2565 bundirty(struct buf *bp)
2566 {
2567 
2568 	CTR3(KTR_BUF, "bundirty(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2569 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2570 	KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
2571 	    ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
2572 
2573 	if (bp->b_flags & B_DELWRI) {
2574 		bp->b_flags &= ~B_DELWRI;
2575 		reassignbuf(bp);
2576 		bdirtysub(bp);
2577 	}
2578 	/*
2579 	 * Since it is now being written, we can clear its deferred write flag.
2580 	 */
2581 	bp->b_flags &= ~B_DEFERRED;
2582 }
2583 
2584 /*
2585  *	bawrite:
2586  *
2587  *	Asynchronous write.  Start output on a buffer, but do not wait for
2588  *	it to complete.  The buffer is released when the output completes.
2589  *
2590  *	bwrite() ( or the VOP routine anyway ) is responsible for handling
2591  *	B_INVAL buffers.  Not us.
2592  */
2593 void
2594 bawrite(struct buf *bp)
2595 {
2596 
2597 	bp->b_flags |= B_ASYNC;
2598 	(void) bwrite(bp);
2599 }
2600 
2601 /*
2602  *	babarrierwrite:
2603  *
2604  *	Asynchronous barrier write.  Start output on a buffer, but do not
2605  *	wait for it to complete.  Place a write barrier after this write so
2606  *	that this buffer and all buffers written before it are committed to
2607  *	the disk before any buffers written after this write are committed
2608  *	to the disk.  The buffer is released when the output completes.
2609  */
2610 void
2611 babarrierwrite(struct buf *bp)
2612 {
2613 
2614 	bp->b_flags |= B_ASYNC | B_BARRIER;
2615 	(void) bwrite(bp);
2616 }
2617 
2618 /*
2619  *	bbarrierwrite:
2620  *
2621  *	Synchronous barrier write.  Start output on a buffer and wait for
2622  *	it to complete.  Place a write barrier after this write so that
2623  *	this buffer and all buffers written before it are committed to
2624  *	the disk before any buffers written after this write are committed
2625  *	to the disk.  The buffer is released when the output completes.
2626  */
2627 int
2628 bbarrierwrite(struct buf *bp)
2629 {
2630 
2631 	bp->b_flags |= B_BARRIER;
2632 	return (bwrite(bp));
2633 }
2634 
2635 /*
2636  *	bwillwrite:
2637  *
2638  *	Called prior to the locking of any vnodes when we are expecting to
2639  *	write.  We do not want to starve the buffer cache with too many
2640  *	dirty buffers so we block here.  By blocking prior to the locking
2641  *	of any vnodes we attempt to avoid the situation where a locked vnode
2642  *	prevents the various system daemons from flushing related buffers.
2643  */
2644 void
2645 bwillwrite(void)
2646 {
2647 
2648 	if (buf_dirty_count_severe()) {
2649 		mtx_lock(&bdirtylock);
2650 		while (buf_dirty_count_severe()) {
2651 			bdirtywait = 1;
2652 			msleep(&bdirtywait, &bdirtylock, (PRIBIO + 4),
2653 			    "flswai", 0);
2654 		}
2655 		mtx_unlock(&bdirtylock);
2656 	}
2657 }
2658 
2659 /*
2660  * Return true if we have too many dirty buffers.
2661  */
2662 int
2663 buf_dirty_count_severe(void)
2664 {
2665 
2666 	return (!BIT_EMPTY(BUF_DOMAINS, &bdhidirty));
2667 }
2668 
2669 /*
2670  *	brelse:
2671  *
2672  *	Release a busy buffer and, if requested, free its resources.  The
2673  *	buffer will be stashed in the appropriate bufqueue[] allowing it
2674  *	to be accessed later as a cache entity or reused for other purposes.
2675  */
2676 void
2677 brelse(struct buf *bp)
2678 {
2679 	struct mount *v_mnt;
2680 	int qindex;
2681 
2682 	/*
2683 	 * Many functions erroneously call brelse with a NULL bp under rare
2684 	 * error conditions. Simply return when called with a NULL bp.
2685 	 */
2686 	if (bp == NULL)
2687 		return;
2688 	CTR3(KTR_BUF, "brelse(%p) vp %p flags %X",
2689 	    bp, bp->b_vp, bp->b_flags);
2690 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
2691 	    ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
2692 	KASSERT((bp->b_flags & B_VMIO) != 0 || (bp->b_flags & B_NOREUSE) == 0,
2693 	    ("brelse: non-VMIO buffer marked NOREUSE"));
2694 
2695 	if (BUF_LOCKRECURSED(bp)) {
2696 		/*
2697 		 * Do not process, in particular, do not handle the
2698 		 * B_INVAL/B_RELBUF and do not release to free list.
2699 		 */
2700 		BUF_UNLOCK(bp);
2701 		return;
2702 	}
2703 
2704 	if (bp->b_flags & B_MANAGED) {
2705 		bqrelse(bp);
2706 		return;
2707 	}
2708 
2709 	if (LIST_EMPTY(&bp->b_dep)) {
2710 		bp->b_flags &= ~B_IOSTARTED;
2711 	} else {
2712 		KASSERT((bp->b_flags & B_IOSTARTED) == 0,
2713 		    ("brelse: SU io not finished bp %p", bp));
2714 	}
2715 
2716 	if ((bp->b_vflags & (BV_BKGRDINPROG | BV_BKGRDERR)) == BV_BKGRDERR) {
2717 		BO_LOCK(bp->b_bufobj);
2718 		bp->b_vflags &= ~BV_BKGRDERR;
2719 		BO_UNLOCK(bp->b_bufobj);
2720 		bdirty(bp);
2721 	}
2722 
2723 	if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
2724 	    (bp->b_flags & B_INVALONERR)) {
2725 		/*
2726 		 * Forced invalidation of dirty buffer contents, to be used
2727 		 * after a failed write in the rare case that the loss of the
2728 		 * contents is acceptable.  The buffer is invalidated and
2729 		 * freed.
2730 		 */
2731 		bp->b_flags |= B_INVAL | B_RELBUF | B_NOCACHE;
2732 		bp->b_flags &= ~(B_ASYNC | B_CACHE);
2733 	}
2734 
2735 	if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
2736 	    (bp->b_error != ENXIO || !LIST_EMPTY(&bp->b_dep)) &&
2737 	    !(bp->b_flags & B_INVAL)) {
2738 		/*
2739 		 * Failed write, redirty.  All errors except ENXIO (which
2740 		 * means the device is gone) are treated as being
2741 		 * transient.
2742 		 *
2743 		 * XXX Treating EIO as transient is not correct; the
2744 		 * contract with the local storage device drivers is that
2745 		 * they will only return EIO once the I/O is no longer
2746 		 * retriable.  Network I/O also respects this through the
2747 		 * guarantees of TCP and/or the internal retries of NFS.
2748 		 * ENOMEM might be transient, but we also have no way of
2749 		 * knowing when its ok to retry/reschedule.  In general,
2750 		 * this entire case should be made obsolete through better
2751 		 * error handling/recovery and resource scheduling.
2752 		 *
2753 		 * Do this also for buffers that failed with ENXIO, but have
2754 		 * non-empty dependencies - the soft updates code might need
2755 		 * to access the buffer to untangle them.
2756 		 *
2757 		 * Must clear BIO_ERROR to prevent pages from being scrapped.
2758 		 */
2759 		bp->b_ioflags &= ~BIO_ERROR;
2760 		bdirty(bp);
2761 	} else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
2762 	    (bp->b_ioflags & BIO_ERROR) || (bp->b_bufsize <= 0)) {
2763 		/*
2764 		 * Either a failed read I/O, or we were asked to free or not
2765 		 * cache the buffer, or we failed to write to a device that's
2766 		 * no longer present.
2767 		 */
2768 		bp->b_flags |= B_INVAL;
2769 		if (!LIST_EMPTY(&bp->b_dep))
2770 			buf_deallocate(bp);
2771 		if (bp->b_flags & B_DELWRI)
2772 			bdirtysub(bp);
2773 		bp->b_flags &= ~(B_DELWRI | B_CACHE);
2774 		if ((bp->b_flags & B_VMIO) == 0) {
2775 			allocbuf(bp, 0);
2776 			if (bp->b_vp)
2777 				brelvp(bp);
2778 		}
2779 	}
2780 
2781 	/*
2782 	 * We must clear B_RELBUF if B_DELWRI is set.  If vfs_vmio_truncate()
2783 	 * is called with B_DELWRI set, the underlying pages may wind up
2784 	 * getting freed causing a previous write (bdwrite()) to get 'lost'
2785 	 * because pages associated with a B_DELWRI bp are marked clean.
2786 	 *
2787 	 * We still allow the B_INVAL case to call vfs_vmio_truncate(), even
2788 	 * if B_DELWRI is set.
2789 	 */
2790 	if (bp->b_flags & B_DELWRI)
2791 		bp->b_flags &= ~B_RELBUF;
2792 
2793 	/*
2794 	 * VMIO buffer rundown.  It is not very necessary to keep a VMIO buffer
2795 	 * constituted, not even NFS buffers now.  Two flags effect this.  If
2796 	 * B_INVAL, the struct buf is invalidated but the VM object is kept
2797 	 * around ( i.e. so it is trivial to reconstitute the buffer later ).
2798 	 *
2799 	 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
2800 	 * invalidated.  BIO_ERROR cannot be set for a failed write unless the
2801 	 * buffer is also B_INVAL because it hits the re-dirtying code above.
2802 	 *
2803 	 * Normally we can do this whether a buffer is B_DELWRI or not.  If
2804 	 * the buffer is an NFS buffer, it is tracking piecemeal writes or
2805 	 * the commit state and we cannot afford to lose the buffer. If the
2806 	 * buffer has a background write in progress, we need to keep it
2807 	 * around to prevent it from being reconstituted and starting a second
2808 	 * background write.
2809 	 */
2810 
2811 	v_mnt = bp->b_vp != NULL ? bp->b_vp->v_mount : NULL;
2812 
2813 	if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE ||
2814 	    (bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
2815 	    (v_mnt == NULL || (v_mnt->mnt_vfc->vfc_flags & VFCF_NETWORK) == 0 ||
2816 	    vn_isdisk(bp->b_vp) || (bp->b_flags & B_DELWRI) == 0)) {
2817 		vfs_vmio_invalidate(bp);
2818 		allocbuf(bp, 0);
2819 	}
2820 
2821 	if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0 ||
2822 	    (bp->b_flags & (B_DELWRI | B_NOREUSE)) == B_NOREUSE) {
2823 		allocbuf(bp, 0);
2824 		bp->b_flags &= ~B_NOREUSE;
2825 		if (bp->b_vp != NULL)
2826 			brelvp(bp);
2827 	}
2828 
2829 	/*
2830 	 * If the buffer has junk contents signal it and eventually
2831 	 * clean up B_DELWRI and diassociate the vnode so that gbincore()
2832 	 * doesn't find it.
2833 	 */
2834 	if (bp->b_bufsize == 0 || (bp->b_ioflags & BIO_ERROR) != 0 ||
2835 	    (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF)) != 0)
2836 		bp->b_flags |= B_INVAL;
2837 	if (bp->b_flags & B_INVAL) {
2838 		if (bp->b_flags & B_DELWRI)
2839 			bundirty(bp);
2840 		if (bp->b_vp)
2841 			brelvp(bp);
2842 	}
2843 
2844 	buf_track(bp, __func__);
2845 
2846 	/* buffers with no memory */
2847 	if (bp->b_bufsize == 0) {
2848 		buf_free(bp);
2849 		return;
2850 	}
2851 	/* buffers with junk contents */
2852 	if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
2853 	    (bp->b_ioflags & BIO_ERROR)) {
2854 		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
2855 		if (bp->b_vflags & BV_BKGRDINPROG)
2856 			panic("losing buffer 2");
2857 		qindex = QUEUE_CLEAN;
2858 		bp->b_flags |= B_AGE;
2859 	/* remaining buffers */
2860 	} else if (bp->b_flags & B_DELWRI)
2861 		qindex = QUEUE_DIRTY;
2862 	else
2863 		qindex = QUEUE_CLEAN;
2864 
2865 	if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
2866 		panic("brelse: not dirty");
2867 
2868 	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_RELBUF | B_DIRECT);
2869 	bp->b_xflags &= ~(BX_CVTENXIO);
2870 	/* binsfree unlocks bp. */
2871 	binsfree(bp, qindex);
2872 }
2873 
2874 /*
2875  * Release a buffer back to the appropriate queue but do not try to free
2876  * it.  The buffer is expected to be used again soon.
2877  *
2878  * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
2879  * biodone() to requeue an async I/O on completion.  It is also used when
2880  * known good buffers need to be requeued but we think we may need the data
2881  * again soon.
2882  *
2883  * XXX we should be able to leave the B_RELBUF hint set on completion.
2884  */
2885 void
2886 bqrelse(struct buf *bp)
2887 {
2888 	int qindex;
2889 
2890 	CTR3(KTR_BUF, "bqrelse(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2891 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
2892 	    ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
2893 
2894 	qindex = QUEUE_NONE;
2895 	if (BUF_LOCKRECURSED(bp)) {
2896 		/* do not release to free list */
2897 		BUF_UNLOCK(bp);
2898 		return;
2899 	}
2900 	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
2901 	bp->b_xflags &= ~(BX_CVTENXIO);
2902 
2903 	if (LIST_EMPTY(&bp->b_dep)) {
2904 		bp->b_flags &= ~B_IOSTARTED;
2905 	} else {
2906 		KASSERT((bp->b_flags & B_IOSTARTED) == 0,
2907 		    ("bqrelse: SU io not finished bp %p", bp));
2908 	}
2909 
2910 	if (bp->b_flags & B_MANAGED) {
2911 		if (bp->b_flags & B_REMFREE)
2912 			bremfreef(bp);
2913 		goto out;
2914 	}
2915 
2916 	/* buffers with stale but valid contents */
2917 	if ((bp->b_flags & B_DELWRI) != 0 || (bp->b_vflags & (BV_BKGRDINPROG |
2918 	    BV_BKGRDERR)) == BV_BKGRDERR) {
2919 		BO_LOCK(bp->b_bufobj);
2920 		bp->b_vflags &= ~BV_BKGRDERR;
2921 		BO_UNLOCK(bp->b_bufobj);
2922 		qindex = QUEUE_DIRTY;
2923 	} else {
2924 		if ((bp->b_flags & B_DELWRI) == 0 &&
2925 		    (bp->b_xflags & BX_VNDIRTY))
2926 			panic("bqrelse: not dirty");
2927 		if ((bp->b_flags & B_NOREUSE) != 0) {
2928 			brelse(bp);
2929 			return;
2930 		}
2931 		qindex = QUEUE_CLEAN;
2932 	}
2933 	buf_track(bp, __func__);
2934 	/* binsfree unlocks bp. */
2935 	binsfree(bp, qindex);
2936 	return;
2937 
2938 out:
2939 	buf_track(bp, __func__);
2940 	/* unlock */
2941 	BUF_UNLOCK(bp);
2942 }
2943 
2944 /*
2945  * Complete I/O to a VMIO backed page.  Validate the pages as appropriate,
2946  * restore bogus pages.
2947  */
2948 static void
2949 vfs_vmio_iodone(struct buf *bp)
2950 {
2951 	vm_ooffset_t foff;
2952 	vm_page_t m;
2953 	vm_object_t obj;
2954 	struct vnode *vp __unused;
2955 	int i, iosize, resid;
2956 	bool bogus;
2957 
2958 	obj = bp->b_bufobj->bo_object;
2959 	KASSERT(blockcount_read(&obj->paging_in_progress) >= bp->b_npages,
2960 	    ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
2961 	    blockcount_read(&obj->paging_in_progress), bp->b_npages));
2962 
2963 	vp = bp->b_vp;
2964 	VNPASS(vp->v_holdcnt > 0, vp);
2965 	VNPASS(vp->v_object != NULL, vp);
2966 
2967 	foff = bp->b_offset;
2968 	KASSERT(bp->b_offset != NOOFFSET,
2969 	    ("vfs_vmio_iodone: bp %p has no buffer offset", bp));
2970 
2971 	bogus = false;
2972 	iosize = bp->b_bcount - bp->b_resid;
2973 	for (i = 0; i < bp->b_npages; i++) {
2974 		resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
2975 		if (resid > iosize)
2976 			resid = iosize;
2977 
2978 		/*
2979 		 * cleanup bogus pages, restoring the originals
2980 		 */
2981 		m = bp->b_pages[i];
2982 		if (m == bogus_page) {
2983 			bogus = true;
2984 			m = vm_page_relookup(obj, OFF_TO_IDX(foff));
2985 			if (m == NULL)
2986 				panic("biodone: page disappeared!");
2987 			bp->b_pages[i] = m;
2988 		} else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
2989 			/*
2990 			 * In the write case, the valid and clean bits are
2991 			 * already changed correctly ( see bdwrite() ), so we
2992 			 * only need to do this here in the read case.
2993 			 */
2994 			KASSERT((m->dirty & vm_page_bits(foff & PAGE_MASK,
2995 			    resid)) == 0, ("vfs_vmio_iodone: page %p "
2996 			    "has unexpected dirty bits", m));
2997 			vfs_page_set_valid(bp, foff, m);
2998 		}
2999 		KASSERT(OFF_TO_IDX(foff) == m->pindex,
3000 		    ("vfs_vmio_iodone: foff(%jd)/pindex(%ju) mismatch",
3001 		    (intmax_t)foff, (uintmax_t)m->pindex));
3002 
3003 		vm_page_sunbusy(m);
3004 		foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
3005 		iosize -= resid;
3006 	}
3007 	vm_object_pip_wakeupn(obj, bp->b_npages);
3008 	if (bogus && buf_mapped(bp)) {
3009 		BUF_CHECK_MAPPED(bp);
3010 		pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
3011 		    bp->b_pages, bp->b_npages);
3012 	}
3013 }
3014 
3015 /*
3016  * Perform page invalidation when a buffer is released.  The fully invalid
3017  * pages will be reclaimed later in vfs_vmio_truncate().
3018  */
3019 static void
3020 vfs_vmio_invalidate(struct buf *bp)
3021 {
3022 	vm_object_t obj;
3023 	vm_page_t m;
3024 	int flags, i, resid, poffset, presid;
3025 
3026 	if (buf_mapped(bp)) {
3027 		BUF_CHECK_MAPPED(bp);
3028 		pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
3029 	} else
3030 		BUF_CHECK_UNMAPPED(bp);
3031 	/*
3032 	 * Get the base offset and length of the buffer.  Note that
3033 	 * in the VMIO case if the buffer block size is not
3034 	 * page-aligned then b_data pointer may not be page-aligned.
3035 	 * But our b_pages[] array *IS* page aligned.
3036 	 *
3037 	 * block sizes less then DEV_BSIZE (usually 512) are not
3038 	 * supported due to the page granularity bits (m->valid,
3039 	 * m->dirty, etc...).
3040 	 *
3041 	 * See man buf(9) for more information
3042 	 */
3043 	flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
3044 	obj = bp->b_bufobj->bo_object;
3045 	resid = bp->b_bufsize;
3046 	poffset = bp->b_offset & PAGE_MASK;
3047 	VM_OBJECT_WLOCK(obj);
3048 	for (i = 0; i < bp->b_npages; i++) {
3049 		m = bp->b_pages[i];
3050 		if (m == bogus_page)
3051 			panic("vfs_vmio_invalidate: Unexpected bogus page.");
3052 		bp->b_pages[i] = NULL;
3053 
3054 		presid = resid > (PAGE_SIZE - poffset) ?
3055 		    (PAGE_SIZE - poffset) : resid;
3056 		KASSERT(presid >= 0, ("brelse: extra page"));
3057 		vm_page_busy_acquire(m, VM_ALLOC_SBUSY);
3058 		if (pmap_page_wired_mappings(m) == 0)
3059 			vm_page_set_invalid(m, poffset, presid);
3060 		vm_page_sunbusy(m);
3061 		vm_page_release_locked(m, flags);
3062 		resid -= presid;
3063 		poffset = 0;
3064 	}
3065 	VM_OBJECT_WUNLOCK(obj);
3066 	bp->b_npages = 0;
3067 }
3068 
3069 /*
3070  * Page-granular truncation of an existing VMIO buffer.
3071  */
3072 static void
3073 vfs_vmio_truncate(struct buf *bp, int desiredpages)
3074 {
3075 	vm_object_t obj;
3076 	vm_page_t m;
3077 	int flags, i;
3078 
3079 	if (bp->b_npages == desiredpages)
3080 		return;
3081 
3082 	if (buf_mapped(bp)) {
3083 		BUF_CHECK_MAPPED(bp);
3084 		pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
3085 		    (desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
3086 	} else
3087 		BUF_CHECK_UNMAPPED(bp);
3088 
3089 	/*
3090 	 * The object lock is needed only if we will attempt to free pages.
3091 	 */
3092 	flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
3093 	if ((bp->b_flags & B_DIRECT) != 0) {
3094 		flags |= VPR_TRYFREE;
3095 		obj = bp->b_bufobj->bo_object;
3096 		VM_OBJECT_WLOCK(obj);
3097 	} else {
3098 		obj = NULL;
3099 	}
3100 	for (i = desiredpages; i < bp->b_npages; i++) {
3101 		m = bp->b_pages[i];
3102 		KASSERT(m != bogus_page, ("allocbuf: bogus page found"));
3103 		bp->b_pages[i] = NULL;
3104 		if (obj != NULL)
3105 			vm_page_release_locked(m, flags);
3106 		else
3107 			vm_page_release(m, flags);
3108 	}
3109 	if (obj != NULL)
3110 		VM_OBJECT_WUNLOCK(obj);
3111 	bp->b_npages = desiredpages;
3112 }
3113 
3114 /*
3115  * Byte granular extension of VMIO buffers.
3116  */
3117 static void
3118 vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
3119 {
3120 	/*
3121 	 * We are growing the buffer, possibly in a
3122 	 * byte-granular fashion.
3123 	 */
3124 	vm_object_t obj;
3125 	vm_offset_t toff;
3126 	vm_offset_t tinc;
3127 	vm_page_t m;
3128 
3129 	/*
3130 	 * Step 1, bring in the VM pages from the object, allocating
3131 	 * them if necessary.  We must clear B_CACHE if these pages
3132 	 * are not valid for the range covered by the buffer.
3133 	 */
3134 	obj = bp->b_bufobj->bo_object;
3135 	if (bp->b_npages < desiredpages) {
3136 		KASSERT(desiredpages <= atop(maxbcachebuf),
3137 		    ("vfs_vmio_extend past maxbcachebuf %p %d %u",
3138 		    bp, desiredpages, maxbcachebuf));
3139 
3140 		/*
3141 		 * We must allocate system pages since blocking
3142 		 * here could interfere with paging I/O, no
3143 		 * matter which process we are.
3144 		 *
3145 		 * Only exclusive busy can be tested here.
3146 		 * Blocking on shared busy might lead to
3147 		 * deadlocks once allocbuf() is called after
3148 		 * pages are vfs_busy_pages().
3149 		 */
3150 		(void)vm_page_grab_pages_unlocked(obj,
3151 		    OFF_TO_IDX(bp->b_offset) + bp->b_npages,
3152 		    VM_ALLOC_SYSTEM | VM_ALLOC_IGN_SBUSY |
3153 		    VM_ALLOC_NOBUSY | VM_ALLOC_WIRED,
3154 		    &bp->b_pages[bp->b_npages], desiredpages - bp->b_npages);
3155 		bp->b_npages = desiredpages;
3156 	}
3157 
3158 	/*
3159 	 * Step 2.  We've loaded the pages into the buffer,
3160 	 * we have to figure out if we can still have B_CACHE
3161 	 * set.  Note that B_CACHE is set according to the
3162 	 * byte-granular range ( bcount and size ), not the
3163 	 * aligned range ( newbsize ).
3164 	 *
3165 	 * The VM test is against m->valid, which is DEV_BSIZE
3166 	 * aligned.  Needless to say, the validity of the data
3167 	 * needs to also be DEV_BSIZE aligned.  Note that this
3168 	 * fails with NFS if the server or some other client
3169 	 * extends the file's EOF.  If our buffer is resized,
3170 	 * B_CACHE may remain set! XXX
3171 	 */
3172 	toff = bp->b_bcount;
3173 	tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
3174 	while ((bp->b_flags & B_CACHE) && toff < size) {
3175 		vm_pindex_t pi;
3176 
3177 		if (tinc > (size - toff))
3178 			tinc = size - toff;
3179 		pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
3180 		m = bp->b_pages[pi];
3181 		vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
3182 		toff += tinc;
3183 		tinc = PAGE_SIZE;
3184 	}
3185 
3186 	/*
3187 	 * Step 3, fixup the KVA pmap.
3188 	 */
3189 	if (buf_mapped(bp))
3190 		bpmap_qenter(bp);
3191 	else
3192 		BUF_CHECK_UNMAPPED(bp);
3193 }
3194 
3195 /*
3196  * Check to see if a block at a particular lbn is available for a clustered
3197  * write.
3198  */
3199 static int
3200 vfs_bio_clcheck(struct vnode *vp, int size, daddr_t lblkno, daddr_t blkno)
3201 {
3202 	struct buf *bpa;
3203 	int match;
3204 
3205 	match = 0;
3206 
3207 	/* If the buf isn't in core skip it */
3208 	if ((bpa = gbincore(&vp->v_bufobj, lblkno)) == NULL)
3209 		return (0);
3210 
3211 	/* If the buf is busy we don't want to wait for it */
3212 	if (BUF_LOCK(bpa, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
3213 		return (0);
3214 
3215 	/* Only cluster with valid clusterable delayed write buffers */
3216 	if ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) !=
3217 	    (B_DELWRI | B_CLUSTEROK))
3218 		goto done;
3219 
3220 	if (bpa->b_bufsize != size)
3221 		goto done;
3222 
3223 	/*
3224 	 * Check to see if it is in the expected place on disk and that the
3225 	 * block has been mapped.
3226 	 */
3227 	if ((bpa->b_blkno != bpa->b_lblkno) && (bpa->b_blkno == blkno))
3228 		match = 1;
3229 done:
3230 	BUF_UNLOCK(bpa);
3231 	return (match);
3232 }
3233 
3234 /*
3235  *	vfs_bio_awrite:
3236  *
3237  *	Implement clustered async writes for clearing out B_DELWRI buffers.
3238  *	This is much better then the old way of writing only one buffer at
3239  *	a time.  Note that we may not be presented with the buffers in the
3240  *	correct order, so we search for the cluster in both directions.
3241  */
3242 int
3243 vfs_bio_awrite(struct buf *bp)
3244 {
3245 	struct bufobj *bo;
3246 	int i;
3247 	int j;
3248 	daddr_t lblkno = bp->b_lblkno;
3249 	struct vnode *vp = bp->b_vp;
3250 	int ncl;
3251 	int nwritten;
3252 	int size;
3253 	int maxcl;
3254 	int gbflags;
3255 
3256 	bo = &vp->v_bufobj;
3257 	gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
3258 	/*
3259 	 * right now we support clustered writing only to regular files.  If
3260 	 * we find a clusterable block we could be in the middle of a cluster
3261 	 * rather then at the beginning.
3262 	 */
3263 	if ((vp->v_type == VREG) &&
3264 	    (vp->v_mount != 0) && /* Only on nodes that have the size info */
3265 	    (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
3266 		size = vp->v_mount->mnt_stat.f_iosize;
3267 		maxcl = maxphys / size;
3268 
3269 		BO_RLOCK(bo);
3270 		for (i = 1; i < maxcl; i++)
3271 			if (vfs_bio_clcheck(vp, size, lblkno + i,
3272 			    bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0)
3273 				break;
3274 
3275 		for (j = 1; i + j <= maxcl && j <= lblkno; j++)
3276 			if (vfs_bio_clcheck(vp, size, lblkno - j,
3277 			    bp->b_blkno - ((j * size) >> DEV_BSHIFT)) == 0)
3278 				break;
3279 		BO_RUNLOCK(bo);
3280 		--j;
3281 		ncl = i + j;
3282 		/*
3283 		 * this is a possible cluster write
3284 		 */
3285 		if (ncl != 1) {
3286 			BUF_UNLOCK(bp);
3287 			nwritten = cluster_wbuild(vp, size, lblkno - j, ncl,
3288 			    gbflags);
3289 			return (nwritten);
3290 		}
3291 	}
3292 	bremfree(bp);
3293 	bp->b_flags |= B_ASYNC;
3294 	/*
3295 	 * default (old) behavior, writing out only one block
3296 	 *
3297 	 * XXX returns b_bufsize instead of b_bcount for nwritten?
3298 	 */
3299 	nwritten = bp->b_bufsize;
3300 	(void) bwrite(bp);
3301 
3302 	return (nwritten);
3303 }
3304 
3305 /*
3306  *	getnewbuf_kva:
3307  *
3308  *	Allocate KVA for an empty buf header according to gbflags.
3309  */
3310 static int
3311 getnewbuf_kva(struct buf *bp, int gbflags, int maxsize)
3312 {
3313 
3314 	if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_UNMAPPED) {
3315 		/*
3316 		 * In order to keep fragmentation sane we only allocate kva
3317 		 * in BKVASIZE chunks.  XXX with vmem we can do page size.
3318 		 */
3319 		maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
3320 
3321 		if (maxsize != bp->b_kvasize &&
3322 		    bufkva_alloc(bp, maxsize, gbflags))
3323 			return (ENOSPC);
3324 	}
3325 	return (0);
3326 }
3327 
3328 /*
3329  *	getnewbuf:
3330  *
3331  *	Find and initialize a new buffer header, freeing up existing buffers
3332  *	in the bufqueues as necessary.  The new buffer is returned locked.
3333  *
3334  *	We block if:
3335  *		We have insufficient buffer headers
3336  *		We have insufficient buffer space
3337  *		buffer_arena is too fragmented ( space reservation fails )
3338  *		If we have to flush dirty buffers ( but we try to avoid this )
3339  *
3340  *	The caller is responsible for releasing the reserved bufspace after
3341  *	allocbuf() is called.
3342  */
3343 static struct buf *
3344 getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int maxsize, int gbflags)
3345 {
3346 	struct bufdomain *bd;
3347 	struct buf *bp;
3348 	bool metadata, reserved;
3349 
3350 	bp = NULL;
3351 	KASSERT((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
3352 	    ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
3353 	if (!unmapped_buf_allowed)
3354 		gbflags &= ~(GB_UNMAPPED | GB_KVAALLOC);
3355 
3356 	if (vp == NULL || (vp->v_vflag & (VV_MD | VV_SYSTEM)) != 0 ||
3357 	    vp->v_type == VCHR)
3358 		metadata = true;
3359 	else
3360 		metadata = false;
3361 	if (vp == NULL)
3362 		bd = &bdomain[0];
3363 	else
3364 		bd = &bdomain[vp->v_bufobj.bo_domain];
3365 
3366 	counter_u64_add(getnewbufcalls, 1);
3367 	reserved = false;
3368 	do {
3369 		if (reserved == false &&
3370 		    bufspace_reserve(bd, maxsize, metadata) != 0) {
3371 			counter_u64_add(getnewbufrestarts, 1);
3372 			continue;
3373 		}
3374 		reserved = true;
3375 		if ((bp = buf_alloc(bd)) == NULL) {
3376 			counter_u64_add(getnewbufrestarts, 1);
3377 			continue;
3378 		}
3379 		if (getnewbuf_kva(bp, gbflags, maxsize) == 0)
3380 			return (bp);
3381 		break;
3382 	} while (buf_recycle(bd, false) == 0);
3383 
3384 	if (reserved)
3385 		bufspace_release(bd, maxsize);
3386 	if (bp != NULL) {
3387 		bp->b_flags |= B_INVAL;
3388 		brelse(bp);
3389 	}
3390 	bufspace_wait(bd, vp, gbflags, slpflag, slptimeo);
3391 
3392 	return (NULL);
3393 }
3394 
3395 /*
3396  *	buf_daemon:
3397  *
3398  *	buffer flushing daemon.  Buffers are normally flushed by the
3399  *	update daemon but if it cannot keep up this process starts to
3400  *	take the load in an attempt to prevent getnewbuf() from blocking.
3401  */
3402 static struct kproc_desc buf_kp = {
3403 	"bufdaemon",
3404 	buf_daemon,
3405 	&bufdaemonproc
3406 };
3407 SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp);
3408 
3409 static int
3410 buf_flush(struct vnode *vp, struct bufdomain *bd, int target)
3411 {
3412 	int flushed;
3413 
3414 	flushed = flushbufqueues(vp, bd, target, 0);
3415 	if (flushed == 0) {
3416 		/*
3417 		 * Could not find any buffers without rollback
3418 		 * dependencies, so just write the first one
3419 		 * in the hopes of eventually making progress.
3420 		 */
3421 		if (vp != NULL && target > 2)
3422 			target /= 2;
3423 		flushbufqueues(vp, bd, target, 1);
3424 	}
3425 	return (flushed);
3426 }
3427 
3428 static void
3429 buf_daemon_shutdown(void *arg __unused, int howto __unused)
3430 {
3431 	int error;
3432 
3433 	if (KERNEL_PANICKED())
3434 		return;
3435 
3436 	mtx_lock(&bdlock);
3437 	bd_shutdown = true;
3438 	wakeup(&bd_request);
3439 	error = msleep(&bd_shutdown, &bdlock, 0, "buf_daemon_shutdown",
3440 	    60 * hz);
3441 	mtx_unlock(&bdlock);
3442 	if (error != 0)
3443 		printf("bufdaemon wait error: %d\n", error);
3444 }
3445 
3446 static void
3447 buf_daemon(void)
3448 {
3449 	struct bufdomain *bd;
3450 	int speedupreq;
3451 	int lodirty;
3452 	int i;
3453 
3454 	/*
3455 	 * This process needs to be suspended prior to shutdown sync.
3456 	 */
3457 	EVENTHANDLER_REGISTER(shutdown_pre_sync, buf_daemon_shutdown, NULL,
3458 	    SHUTDOWN_PRI_LAST + 100);
3459 
3460 	/*
3461 	 * Start the buf clean daemons as children threads.
3462 	 */
3463 	for (i = 0 ; i < buf_domains; i++) {
3464 		int error;
3465 
3466 		error = kthread_add((void (*)(void *))bufspace_daemon,
3467 		    &bdomain[i], curproc, NULL, 0, 0, "bufspacedaemon-%d", i);
3468 		if (error)
3469 			panic("error %d spawning bufspace daemon", error);
3470 	}
3471 
3472 	/*
3473 	 * This process is allowed to take the buffer cache to the limit
3474 	 */
3475 	curthread->td_pflags |= TDP_NORUNNINGBUF | TDP_BUFNEED;
3476 	mtx_lock(&bdlock);
3477 	while (!bd_shutdown) {
3478 		bd_request = 0;
3479 		mtx_unlock(&bdlock);
3480 
3481 		/*
3482 		 * Save speedupreq for this pass and reset to capture new
3483 		 * requests.
3484 		 */
3485 		speedupreq = bd_speedupreq;
3486 		bd_speedupreq = 0;
3487 
3488 		/*
3489 		 * Flush each domain sequentially according to its level and
3490 		 * the speedup request.
3491 		 */
3492 		for (i = 0; i < buf_domains; i++) {
3493 			bd = &bdomain[i];
3494 			if (speedupreq)
3495 				lodirty = bd->bd_numdirtybuffers / 2;
3496 			else
3497 				lodirty = bd->bd_lodirtybuffers;
3498 			while (bd->bd_numdirtybuffers > lodirty) {
3499 				if (buf_flush(NULL, bd,
3500 				    bd->bd_numdirtybuffers - lodirty) == 0)
3501 					break;
3502 				kern_yield(PRI_USER);
3503 			}
3504 		}
3505 
3506 		/*
3507 		 * Only clear bd_request if we have reached our low water
3508 		 * mark.  The buf_daemon normally waits 1 second and
3509 		 * then incrementally flushes any dirty buffers that have
3510 		 * built up, within reason.
3511 		 *
3512 		 * If we were unable to hit our low water mark and couldn't
3513 		 * find any flushable buffers, we sleep for a short period
3514 		 * to avoid endless loops on unlockable buffers.
3515 		 */
3516 		mtx_lock(&bdlock);
3517 		if (bd_shutdown)
3518 			break;
3519 		if (BIT_EMPTY(BUF_DOMAINS, &bdlodirty)) {
3520 			/*
3521 			 * We reached our low water mark, reset the
3522 			 * request and sleep until we are needed again.
3523 			 * The sleep is just so the suspend code works.
3524 			 */
3525 			bd_request = 0;
3526 			/*
3527 			 * Do an extra wakeup in case dirty threshold
3528 			 * changed via sysctl and the explicit transition
3529 			 * out of shortfall was missed.
3530 			 */
3531 			bdirtywakeup();
3532 			if (runningbufspace <= lorunningspace)
3533 				runningwakeup();
3534 			msleep(&bd_request, &bdlock, PVM, "psleep", hz);
3535 		} else {
3536 			/*
3537 			 * We couldn't find any flushable dirty buffers but
3538 			 * still have too many dirty buffers, we
3539 			 * have to sleep and try again.  (rare)
3540 			 */
3541 			msleep(&bd_request, &bdlock, PVM, "qsleep", hz / 10);
3542 		}
3543 	}
3544 	wakeup(&bd_shutdown);
3545 	mtx_unlock(&bdlock);
3546 	kthread_exit();
3547 }
3548 
3549 /*
3550  *	flushbufqueues:
3551  *
3552  *	Try to flush a buffer in the dirty queue.  We must be careful to
3553  *	free up B_INVAL buffers instead of write them, which NFS is
3554  *	particularly sensitive to.
3555  */
3556 static int flushwithdeps = 0;
3557 SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_RW | CTLFLAG_STATS,
3558     &flushwithdeps, 0,
3559     "Number of buffers flushed with dependencies that require rollbacks");
3560 
3561 static int
3562 flushbufqueues(struct vnode *lvp, struct bufdomain *bd, int target,
3563     int flushdeps)
3564 {
3565 	struct bufqueue *bq;
3566 	struct buf *sentinel;
3567 	struct vnode *vp;
3568 	struct mount *mp;
3569 	struct buf *bp;
3570 	int hasdeps;
3571 	int flushed;
3572 	int error;
3573 	bool unlock;
3574 
3575 	flushed = 0;
3576 	bq = &bd->bd_dirtyq;
3577 	bp = NULL;
3578 	sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK | M_ZERO);
3579 	sentinel->b_qindex = QUEUE_SENTINEL;
3580 	BQ_LOCK(bq);
3581 	TAILQ_INSERT_HEAD(&bq->bq_queue, sentinel, b_freelist);
3582 	BQ_UNLOCK(bq);
3583 	while (flushed != target) {
3584 		maybe_yield();
3585 		BQ_LOCK(bq);
3586 		bp = TAILQ_NEXT(sentinel, b_freelist);
3587 		if (bp != NULL) {
3588 			TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
3589 			TAILQ_INSERT_AFTER(&bq->bq_queue, bp, sentinel,
3590 			    b_freelist);
3591 		} else {
3592 			BQ_UNLOCK(bq);
3593 			break;
3594 		}
3595 		/*
3596 		 * Skip sentinels inserted by other invocations of the
3597 		 * flushbufqueues(), taking care to not reorder them.
3598 		 *
3599 		 * Only flush the buffers that belong to the
3600 		 * vnode locked by the curthread.
3601 		 */
3602 		if (bp->b_qindex == QUEUE_SENTINEL || (lvp != NULL &&
3603 		    bp->b_vp != lvp)) {
3604 			BQ_UNLOCK(bq);
3605 			continue;
3606 		}
3607 		error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL);
3608 		BQ_UNLOCK(bq);
3609 		if (error != 0)
3610 			continue;
3611 
3612 		/*
3613 		 * BKGRDINPROG can only be set with the buf and bufobj
3614 		 * locks both held.  We tolerate a race to clear it here.
3615 		 */
3616 		if ((bp->b_vflags & BV_BKGRDINPROG) != 0 ||
3617 		    (bp->b_flags & B_DELWRI) == 0) {
3618 			BUF_UNLOCK(bp);
3619 			continue;
3620 		}
3621 		if (bp->b_flags & B_INVAL) {
3622 			bremfreef(bp);
3623 			brelse(bp);
3624 			flushed++;
3625 			continue;
3626 		}
3627 
3628 		if (!LIST_EMPTY(&bp->b_dep) && buf_countdeps(bp, 0)) {
3629 			if (flushdeps == 0) {
3630 				BUF_UNLOCK(bp);
3631 				continue;
3632 			}
3633 			hasdeps = 1;
3634 		} else
3635 			hasdeps = 0;
3636 		/*
3637 		 * We must hold the lock on a vnode before writing
3638 		 * one of its buffers. Otherwise we may confuse, or
3639 		 * in the case of a snapshot vnode, deadlock the
3640 		 * system.
3641 		 *
3642 		 * The lock order here is the reverse of the normal
3643 		 * of vnode followed by buf lock.  This is ok because
3644 		 * the NOWAIT will prevent deadlock.
3645 		 */
3646 		vp = bp->b_vp;
3647 		if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
3648 			BUF_UNLOCK(bp);
3649 			continue;
3650 		}
3651 		if (lvp == NULL) {
3652 			unlock = true;
3653 			error = vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT);
3654 		} else {
3655 			ASSERT_VOP_LOCKED(vp, "getbuf");
3656 			unlock = false;
3657 			error = VOP_ISLOCKED(vp) == LK_EXCLUSIVE ? 0 :
3658 			    vn_lock(vp, LK_TRYUPGRADE);
3659 		}
3660 		if (error == 0) {
3661 			CTR3(KTR_BUF, "flushbufqueue(%p) vp %p flags %X",
3662 			    bp, bp->b_vp, bp->b_flags);
3663 			if (curproc == bufdaemonproc) {
3664 				vfs_bio_awrite(bp);
3665 			} else {
3666 				bremfree(bp);
3667 				bwrite(bp);
3668 				counter_u64_add(notbufdflushes, 1);
3669 			}
3670 			vn_finished_write(mp);
3671 			if (unlock)
3672 				VOP_UNLOCK(vp);
3673 			flushwithdeps += hasdeps;
3674 			flushed++;
3675 
3676 			/*
3677 			 * Sleeping on runningbufspace while holding
3678 			 * vnode lock leads to deadlock.
3679 			 */
3680 			if (curproc == bufdaemonproc &&
3681 			    runningbufspace > hirunningspace)
3682 				waitrunningbufspace();
3683 			continue;
3684 		}
3685 		vn_finished_write(mp);
3686 		BUF_UNLOCK(bp);
3687 	}
3688 	BQ_LOCK(bq);
3689 	TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
3690 	BQ_UNLOCK(bq);
3691 	free(sentinel, M_TEMP);
3692 	return (flushed);
3693 }
3694 
3695 /*
3696  * Check to see if a block is currently memory resident.
3697  */
3698 struct buf *
3699 incore(struct bufobj *bo, daddr_t blkno)
3700 {
3701 	return (gbincore_unlocked(bo, blkno));
3702 }
3703 
3704 /*
3705  * Returns true if no I/O is needed to access the
3706  * associated VM object.  This is like incore except
3707  * it also hunts around in the VM system for the data.
3708  */
3709 bool
3710 inmem(struct vnode * vp, daddr_t blkno)
3711 {
3712 	vm_object_t obj;
3713 	vm_offset_t toff, tinc, size;
3714 	vm_page_t m, n;
3715 	vm_ooffset_t off;
3716 	int valid;
3717 
3718 	ASSERT_VOP_LOCKED(vp, "inmem");
3719 
3720 	if (incore(&vp->v_bufobj, blkno))
3721 		return (true);
3722 	if (vp->v_mount == NULL)
3723 		return (false);
3724 	obj = vp->v_object;
3725 	if (obj == NULL)
3726 		return (false);
3727 
3728 	size = PAGE_SIZE;
3729 	if (size > vp->v_mount->mnt_stat.f_iosize)
3730 		size = vp->v_mount->mnt_stat.f_iosize;
3731 	off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
3732 
3733 	for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
3734 		m = vm_page_lookup_unlocked(obj, OFF_TO_IDX(off + toff));
3735 recheck:
3736 		if (m == NULL)
3737 			return (false);
3738 
3739 		tinc = size;
3740 		if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
3741 			tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
3742 		/*
3743 		 * Consider page validity only if page mapping didn't change
3744 		 * during the check.
3745 		 */
3746 		valid = vm_page_is_valid(m,
3747 		    (vm_offset_t)((toff + off) & PAGE_MASK), tinc);
3748 		n = vm_page_lookup_unlocked(obj, OFF_TO_IDX(off + toff));
3749 		if (m != n) {
3750 			m = n;
3751 			goto recheck;
3752 		}
3753 		if (!valid)
3754 			return (false);
3755 	}
3756 	return (true);
3757 }
3758 
3759 /*
3760  * Set the dirty range for a buffer based on the status of the dirty
3761  * bits in the pages comprising the buffer.  The range is limited
3762  * to the size of the buffer.
3763  *
3764  * Tell the VM system that the pages associated with this buffer
3765  * are clean.  This is used for delayed writes where the data is
3766  * going to go to disk eventually without additional VM intevention.
3767  *
3768  * Note that while we only really need to clean through to b_bcount, we
3769  * just go ahead and clean through to b_bufsize.
3770  */
3771 static void
3772 vfs_clean_pages_dirty_buf(struct buf *bp)
3773 {
3774 	vm_ooffset_t foff, noff, eoff;
3775 	vm_page_t m;
3776 	int i;
3777 
3778 	if ((bp->b_flags & B_VMIO) == 0 || bp->b_bufsize == 0)
3779 		return;
3780 
3781 	foff = bp->b_offset;
3782 	KASSERT(bp->b_offset != NOOFFSET,
3783 	    ("vfs_clean_pages_dirty_buf: no buffer offset"));
3784 
3785 	vfs_busy_pages_acquire(bp);
3786 	vfs_setdirty_range(bp);
3787 	for (i = 0; i < bp->b_npages; i++) {
3788 		noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
3789 		eoff = noff;
3790 		if (eoff > bp->b_offset + bp->b_bufsize)
3791 			eoff = bp->b_offset + bp->b_bufsize;
3792 		m = bp->b_pages[i];
3793 		vfs_page_set_validclean(bp, foff, m);
3794 		/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
3795 		foff = noff;
3796 	}
3797 	vfs_busy_pages_release(bp);
3798 }
3799 
3800 static void
3801 vfs_setdirty_range(struct buf *bp)
3802 {
3803 	vm_offset_t boffset;
3804 	vm_offset_t eoffset;
3805 	int i;
3806 
3807 	/*
3808 	 * test the pages to see if they have been modified directly
3809 	 * by users through the VM system.
3810 	 */
3811 	for (i = 0; i < bp->b_npages; i++)
3812 		vm_page_test_dirty(bp->b_pages[i]);
3813 
3814 	/*
3815 	 * Calculate the encompassing dirty range, boffset and eoffset,
3816 	 * (eoffset - boffset) bytes.
3817 	 */
3818 
3819 	for (i = 0; i < bp->b_npages; i++) {
3820 		if (bp->b_pages[i]->dirty)
3821 			break;
3822 	}
3823 	boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3824 
3825 	for (i = bp->b_npages - 1; i >= 0; --i) {
3826 		if (bp->b_pages[i]->dirty) {
3827 			break;
3828 		}
3829 	}
3830 	eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3831 
3832 	/*
3833 	 * Fit it to the buffer.
3834 	 */
3835 
3836 	if (eoffset > bp->b_bcount)
3837 		eoffset = bp->b_bcount;
3838 
3839 	/*
3840 	 * If we have a good dirty range, merge with the existing
3841 	 * dirty range.
3842 	 */
3843 
3844 	if (boffset < eoffset) {
3845 		if (bp->b_dirtyoff > boffset)
3846 			bp->b_dirtyoff = boffset;
3847 		if (bp->b_dirtyend < eoffset)
3848 			bp->b_dirtyend = eoffset;
3849 	}
3850 }
3851 
3852 /*
3853  * Allocate the KVA mapping for an existing buffer.
3854  * If an unmapped buffer is provided but a mapped buffer is requested, take
3855  * also care to properly setup mappings between pages and KVA.
3856  */
3857 static void
3858 bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
3859 {
3860 	int bsize, maxsize, need_mapping, need_kva;
3861 	off_t offset;
3862 
3863 	need_mapping = bp->b_data == unmapped_buf &&
3864 	    (gbflags & GB_UNMAPPED) == 0;
3865 	need_kva = bp->b_kvabase == unmapped_buf &&
3866 	    bp->b_data == unmapped_buf &&
3867 	    (gbflags & GB_KVAALLOC) != 0;
3868 	if (!need_mapping && !need_kva)
3869 		return;
3870 
3871 	BUF_CHECK_UNMAPPED(bp);
3872 
3873 	if (need_mapping && bp->b_kvabase != unmapped_buf) {
3874 		/*
3875 		 * Buffer is not mapped, but the KVA was already
3876 		 * reserved at the time of the instantiation.  Use the
3877 		 * allocated space.
3878 		 */
3879 		goto has_addr;
3880 	}
3881 
3882 	/*
3883 	 * Calculate the amount of the address space we would reserve
3884 	 * if the buffer was mapped.
3885 	 */
3886 	bsize = vn_isdisk(bp->b_vp) ? DEV_BSIZE : bp->b_bufobj->bo_bsize;
3887 	KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
3888 	offset = blkno * bsize;
3889 	maxsize = size + (offset & PAGE_MASK);
3890 	maxsize = imax(maxsize, bsize);
3891 
3892 	while (bufkva_alloc(bp, maxsize, gbflags) != 0) {
3893 		if ((gbflags & GB_NOWAIT_BD) != 0) {
3894 			/*
3895 			 * XXXKIB: defragmentation cannot
3896 			 * succeed, not sure what else to do.
3897 			 */
3898 			panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
3899 		}
3900 		counter_u64_add(mappingrestarts, 1);
3901 		bufspace_wait(bufdomain(bp), bp->b_vp, gbflags, 0, 0);
3902 	}
3903 has_addr:
3904 	if (need_mapping) {
3905 		/* b_offset is handled by bpmap_qenter. */
3906 		bp->b_data = bp->b_kvabase;
3907 		BUF_CHECK_MAPPED(bp);
3908 		bpmap_qenter(bp);
3909 	}
3910 }
3911 
3912 struct buf *
3913 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo,
3914     int flags)
3915 {
3916 	struct buf *bp;
3917 	int error;
3918 
3919 	error = getblkx(vp, blkno, blkno, size, slpflag, slptimeo, flags, &bp);
3920 	if (error != 0)
3921 		return (NULL);
3922 	return (bp);
3923 }
3924 
3925 /*
3926  *	getblkx:
3927  *
3928  *	Get a block given a specified block and offset into a file/device.
3929  *	The buffers B_DONE bit will be cleared on return, making it almost
3930  * 	ready for an I/O initiation.  B_INVAL may or may not be set on
3931  *	return.  The caller should clear B_INVAL prior to initiating a
3932  *	READ.
3933  *
3934  *	For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
3935  *	an existing buffer.
3936  *
3937  *	For a VMIO buffer, B_CACHE is modified according to the backing VM.
3938  *	If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
3939  *	and then cleared based on the backing VM.  If the previous buffer is
3940  *	non-0-sized but invalid, B_CACHE will be cleared.
3941  *
3942  *	If getblk() must create a new buffer, the new buffer is returned with
3943  *	both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
3944  *	case it is returned with B_INVAL clear and B_CACHE set based on the
3945  *	backing VM.
3946  *
3947  *	getblk() also forces a bwrite() for any B_DELWRI buffer whose
3948  *	B_CACHE bit is clear.
3949  *
3950  *	What this means, basically, is that the caller should use B_CACHE to
3951  *	determine whether the buffer is fully valid or not and should clear
3952  *	B_INVAL prior to issuing a read.  If the caller intends to validate
3953  *	the buffer by loading its data area with something, the caller needs
3954  *	to clear B_INVAL.  If the caller does this without issuing an I/O,
3955  *	the caller should set B_CACHE ( as an optimization ), else the caller
3956  *	should issue the I/O and biodone() will set B_CACHE if the I/O was
3957  *	a write attempt or if it was a successful read.  If the caller
3958  *	intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
3959  *	prior to issuing the READ.  biodone() will *not* clear B_INVAL.
3960  *
3961  *	The blkno parameter is the logical block being requested. Normally
3962  *	the mapping of logical block number to disk block address is done
3963  *	by calling VOP_BMAP(). However, if the mapping is already known, the
3964  *	disk block address can be passed using the dblkno parameter. If the
3965  *	disk block address is not known, then the same value should be passed
3966  *	for blkno and dblkno.
3967  */
3968 int
3969 getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
3970     int slptimeo, int flags, struct buf **bpp)
3971 {
3972 	struct buf *bp;
3973 	struct bufobj *bo;
3974 	daddr_t d_blkno;
3975 	int bsize, error, maxsize, vmio;
3976 	off_t offset;
3977 
3978 	CTR3(KTR_BUF, "getblk(%p, %ld, %d)", vp, (long)blkno, size);
3979 	KASSERT((flags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
3980 	    ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
3981 	if (vp->v_type != VCHR)
3982 		ASSERT_VOP_LOCKED(vp, "getblk");
3983 	if (size > maxbcachebuf)
3984 		panic("getblk: size(%d) > maxbcachebuf(%d)\n", size,
3985 		    maxbcachebuf);
3986 	if (!unmapped_buf_allowed)
3987 		flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
3988 
3989 	bo = &vp->v_bufobj;
3990 	d_blkno = dblkno;
3991 
3992 	/* Attempt lockless lookup first. */
3993 	bp = gbincore_unlocked(bo, blkno);
3994 	if (bp == NULL) {
3995 		/*
3996 		 * With GB_NOCREAT we must be sure about not finding the buffer
3997 		 * as it may have been reassigned during unlocked lookup.
3998 		 */
3999 		if ((flags & GB_NOCREAT) != 0)
4000 			goto loop;
4001 		goto newbuf_unlocked;
4002 	}
4003 
4004 	error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL, "getblku", 0,
4005 	    0);
4006 	if (error != 0)
4007 		goto loop;
4008 
4009 	/* Verify buf identify has not changed since lookup. */
4010 	if (bp->b_bufobj == bo && bp->b_lblkno == blkno)
4011 		goto foundbuf_fastpath;
4012 
4013 	/* It changed, fallback to locked lookup. */
4014 	BUF_UNLOCK_RAW(bp);
4015 
4016 loop:
4017 	BO_RLOCK(bo);
4018 	bp = gbincore(bo, blkno);
4019 	if (bp != NULL) {
4020 		int lockflags;
4021 
4022 		/*
4023 		 * Buffer is in-core.  If the buffer is not busy nor managed,
4024 		 * it must be on a queue.
4025 		 */
4026 		lockflags = LK_EXCLUSIVE | LK_INTERLOCK |
4027 		    ((flags & GB_LOCK_NOWAIT) != 0 ? LK_NOWAIT : LK_SLEEPFAIL);
4028 #ifdef WITNESS
4029 		lockflags |= (flags & GB_NOWITNESS) != 0 ? LK_NOWITNESS : 0;
4030 #endif
4031 
4032 		error = BUF_TIMELOCK(bp, lockflags,
4033 		    BO_LOCKPTR(bo), "getblk", slpflag, slptimeo);
4034 
4035 		/*
4036 		 * If we slept and got the lock we have to restart in case
4037 		 * the buffer changed identities.
4038 		 */
4039 		if (error == ENOLCK)
4040 			goto loop;
4041 		/* We timed out or were interrupted. */
4042 		else if (error != 0)
4043 			return (error);
4044 
4045 foundbuf_fastpath:
4046 		/* If recursed, assume caller knows the rules. */
4047 		if (BUF_LOCKRECURSED(bp))
4048 			goto end;
4049 
4050 		/*
4051 		 * The buffer is locked.  B_CACHE is cleared if the buffer is
4052 		 * invalid.  Otherwise, for a non-VMIO buffer, B_CACHE is set
4053 		 * and for a VMIO buffer B_CACHE is adjusted according to the
4054 		 * backing VM cache.
4055 		 */
4056 		if (bp->b_flags & B_INVAL)
4057 			bp->b_flags &= ~B_CACHE;
4058 		else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
4059 			bp->b_flags |= B_CACHE;
4060 		if (bp->b_flags & B_MANAGED)
4061 			MPASS(bp->b_qindex == QUEUE_NONE);
4062 		else
4063 			bremfree(bp);
4064 
4065 		/*
4066 		 * check for size inconsistencies for non-VMIO case.
4067 		 */
4068 		if (bp->b_bcount != size) {
4069 			if ((bp->b_flags & B_VMIO) == 0 ||
4070 			    (size > bp->b_kvasize)) {
4071 				if (bp->b_flags & B_DELWRI) {
4072 					bp->b_flags |= B_NOCACHE;
4073 					bwrite(bp);
4074 				} else {
4075 					if (LIST_EMPTY(&bp->b_dep)) {
4076 						bp->b_flags |= B_RELBUF;
4077 						brelse(bp);
4078 					} else {
4079 						bp->b_flags |= B_NOCACHE;
4080 						bwrite(bp);
4081 					}
4082 				}
4083 				goto loop;
4084 			}
4085 		}
4086 
4087 		/*
4088 		 * Handle the case of unmapped buffer which should
4089 		 * become mapped, or the buffer for which KVA
4090 		 * reservation is requested.
4091 		 */
4092 		bp_unmapped_get_kva(bp, blkno, size, flags);
4093 
4094 		/*
4095 		 * If the size is inconsistent in the VMIO case, we can resize
4096 		 * the buffer.  This might lead to B_CACHE getting set or
4097 		 * cleared.  If the size has not changed, B_CACHE remains
4098 		 * unchanged from its previous state.
4099 		 */
4100 		allocbuf(bp, size);
4101 
4102 		KASSERT(bp->b_offset != NOOFFSET,
4103 		    ("getblk: no buffer offset"));
4104 
4105 		/*
4106 		 * A buffer with B_DELWRI set and B_CACHE clear must
4107 		 * be committed before we can return the buffer in
4108 		 * order to prevent the caller from issuing a read
4109 		 * ( due to B_CACHE not being set ) and overwriting
4110 		 * it.
4111 		 *
4112 		 * Most callers, including NFS and FFS, need this to
4113 		 * operate properly either because they assume they
4114 		 * can issue a read if B_CACHE is not set, or because
4115 		 * ( for example ) an uncached B_DELWRI might loop due
4116 		 * to softupdates re-dirtying the buffer.  In the latter
4117 		 * case, B_CACHE is set after the first write completes,
4118 		 * preventing further loops.
4119 		 * NOTE!  b*write() sets B_CACHE.  If we cleared B_CACHE
4120 		 * above while extending the buffer, we cannot allow the
4121 		 * buffer to remain with B_CACHE set after the write
4122 		 * completes or it will represent a corrupt state.  To
4123 		 * deal with this we set B_NOCACHE to scrap the buffer
4124 		 * after the write.
4125 		 *
4126 		 * We might be able to do something fancy, like setting
4127 		 * B_CACHE in bwrite() except if B_DELWRI is already set,
4128 		 * so the below call doesn't set B_CACHE, but that gets real
4129 		 * confusing.  This is much easier.
4130 		 */
4131 
4132 		if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
4133 			bp->b_flags |= B_NOCACHE;
4134 			bwrite(bp);
4135 			goto loop;
4136 		}
4137 		bp->b_flags &= ~B_DONE;
4138 	} else {
4139 		/*
4140 		 * Buffer is not in-core, create new buffer.  The buffer
4141 		 * returned by getnewbuf() is locked.  Note that the returned
4142 		 * buffer is also considered valid (not marked B_INVAL).
4143 		 */
4144 		BO_RUNLOCK(bo);
4145 newbuf_unlocked:
4146 		/*
4147 		 * If the user does not want us to create the buffer, bail out
4148 		 * here.
4149 		 */
4150 		if (flags & GB_NOCREAT)
4151 			return (EEXIST);
4152 
4153 		bsize = vn_isdisk(vp) ? DEV_BSIZE : bo->bo_bsize;
4154 		KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
4155 		offset = blkno * bsize;
4156 		vmio = vp->v_object != NULL;
4157 		if (vmio) {
4158 			maxsize = size + (offset & PAGE_MASK);
4159 		} else {
4160 			maxsize = size;
4161 			/* Do not allow non-VMIO notmapped buffers. */
4162 			flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
4163 		}
4164 		maxsize = imax(maxsize, bsize);
4165 		if ((flags & GB_NOSPARSE) != 0 && vmio &&
4166 		    !vn_isdisk(vp)) {
4167 			error = VOP_BMAP(vp, blkno, NULL, &d_blkno, 0, 0);
4168 			KASSERT(error != EOPNOTSUPP,
4169 			    ("GB_NOSPARSE from fs not supporting bmap, vp %p",
4170 			    vp));
4171 			if (error != 0)
4172 				return (error);
4173 			if (d_blkno == -1)
4174 				return (EJUSTRETURN);
4175 		}
4176 
4177 		bp = getnewbuf(vp, slpflag, slptimeo, maxsize, flags);
4178 		if (bp == NULL) {
4179 			if (slpflag || slptimeo)
4180 				return (ETIMEDOUT);
4181 			/*
4182 			 * XXX This is here until the sleep path is diagnosed
4183 			 * enough to work under very low memory conditions.
4184 			 *
4185 			 * There's an issue on low memory, 4BSD+non-preempt
4186 			 * systems (eg MIPS routers with 32MB RAM) where buffer
4187 			 * exhaustion occurs without sleeping for buffer
4188 			 * reclaimation.  This just sticks in a loop and
4189 			 * constantly attempts to allocate a buffer, which
4190 			 * hits exhaustion and tries to wakeup bufdaemon.
4191 			 * This never happens because we never yield.
4192 			 *
4193 			 * The real solution is to identify and fix these cases
4194 			 * so we aren't effectively busy-waiting in a loop
4195 			 * until the reclaimation path has cycles to run.
4196 			 */
4197 			kern_yield(PRI_USER);
4198 			goto loop;
4199 		}
4200 
4201 		/*
4202 		 * This code is used to make sure that a buffer is not
4203 		 * created while the getnewbuf routine is blocked.
4204 		 * This can be a problem whether the vnode is locked or not.
4205 		 * If the buffer is created out from under us, we have to
4206 		 * throw away the one we just created.
4207 		 *
4208 		 * Note: this must occur before we associate the buffer
4209 		 * with the vp especially considering limitations in
4210 		 * the splay tree implementation when dealing with duplicate
4211 		 * lblkno's.
4212 		 */
4213 		BO_LOCK(bo);
4214 		if (gbincore(bo, blkno)) {
4215 			BO_UNLOCK(bo);
4216 			bp->b_flags |= B_INVAL;
4217 			bufspace_release(bufdomain(bp), maxsize);
4218 			brelse(bp);
4219 			goto loop;
4220 		}
4221 
4222 		/*
4223 		 * Insert the buffer into the hash, so that it can
4224 		 * be found by incore.
4225 		 */
4226 		bp->b_lblkno = blkno;
4227 		bp->b_blkno = d_blkno;
4228 		bp->b_offset = offset;
4229 		bgetvp(vp, bp);
4230 		BO_UNLOCK(bo);
4231 
4232 		/*
4233 		 * set B_VMIO bit.  allocbuf() the buffer bigger.  Since the
4234 		 * buffer size starts out as 0, B_CACHE will be set by
4235 		 * allocbuf() for the VMIO case prior to it testing the
4236 		 * backing store for validity.
4237 		 */
4238 
4239 		if (vmio) {
4240 			bp->b_flags |= B_VMIO;
4241 			KASSERT(vp->v_object == bp->b_bufobj->bo_object,
4242 			    ("ARGH! different b_bufobj->bo_object %p %p %p\n",
4243 			    bp, vp->v_object, bp->b_bufobj->bo_object));
4244 		} else {
4245 			bp->b_flags &= ~B_VMIO;
4246 			KASSERT(bp->b_bufobj->bo_object == NULL,
4247 			    ("ARGH! has b_bufobj->bo_object %p %p\n",
4248 			    bp, bp->b_bufobj->bo_object));
4249 			BUF_CHECK_MAPPED(bp);
4250 		}
4251 
4252 		allocbuf(bp, size);
4253 		bufspace_release(bufdomain(bp), maxsize);
4254 		bp->b_flags &= ~B_DONE;
4255 	}
4256 	CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
4257 end:
4258 	buf_track(bp, __func__);
4259 	KASSERT(bp->b_bufobj == bo,
4260 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
4261 	*bpp = bp;
4262 	return (0);
4263 }
4264 
4265 /*
4266  * Get an empty, disassociated buffer of given size.  The buffer is initially
4267  * set to B_INVAL.
4268  */
4269 struct buf *
4270 geteblk(int size, int flags)
4271 {
4272 	struct buf *bp;
4273 	int maxsize;
4274 
4275 	maxsize = (size + BKVAMASK) & ~BKVAMASK;
4276 	while ((bp = getnewbuf(NULL, 0, 0, maxsize, flags)) == NULL) {
4277 		if ((flags & GB_NOWAIT_BD) &&
4278 		    (curthread->td_pflags & TDP_BUFNEED) != 0)
4279 			return (NULL);
4280 	}
4281 	allocbuf(bp, size);
4282 	bufspace_release(bufdomain(bp), maxsize);
4283 	bp->b_flags |= B_INVAL;	/* b_dep cleared by getnewbuf() */
4284 	return (bp);
4285 }
4286 
4287 /*
4288  * Truncate the backing store for a non-vmio buffer.
4289  */
4290 static void
4291 vfs_nonvmio_truncate(struct buf *bp, int newbsize)
4292 {
4293 
4294 	if (bp->b_flags & B_MALLOC) {
4295 		/*
4296 		 * malloced buffers are not shrunk
4297 		 */
4298 		if (newbsize == 0) {
4299 			bufmallocadjust(bp, 0);
4300 			free(bp->b_data, M_BIOBUF);
4301 			bp->b_data = bp->b_kvabase;
4302 			bp->b_flags &= ~B_MALLOC;
4303 		}
4304 		return;
4305 	}
4306 	vm_hold_free_pages(bp, newbsize);
4307 	bufspace_adjust(bp, newbsize);
4308 }
4309 
4310 /*
4311  * Extend the backing for a non-VMIO buffer.
4312  */
4313 static void
4314 vfs_nonvmio_extend(struct buf *bp, int newbsize)
4315 {
4316 	caddr_t origbuf;
4317 	int origbufsize;
4318 
4319 	/*
4320 	 * We only use malloced memory on the first allocation.
4321 	 * and revert to page-allocated memory when the buffer
4322 	 * grows.
4323 	 *
4324 	 * There is a potential smp race here that could lead
4325 	 * to bufmallocspace slightly passing the max.  It
4326 	 * is probably extremely rare and not worth worrying
4327 	 * over.
4328 	 */
4329 	if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
4330 	    bufmallocspace < maxbufmallocspace) {
4331 		bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
4332 		bp->b_flags |= B_MALLOC;
4333 		bufmallocadjust(bp, newbsize);
4334 		return;
4335 	}
4336 
4337 	/*
4338 	 * If the buffer is growing on its other-than-first
4339 	 * allocation then we revert to the page-allocation
4340 	 * scheme.
4341 	 */
4342 	origbuf = NULL;
4343 	origbufsize = 0;
4344 	if (bp->b_flags & B_MALLOC) {
4345 		origbuf = bp->b_data;
4346 		origbufsize = bp->b_bufsize;
4347 		bp->b_data = bp->b_kvabase;
4348 		bufmallocadjust(bp, 0);
4349 		bp->b_flags &= ~B_MALLOC;
4350 		newbsize = round_page(newbsize);
4351 	}
4352 	vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
4353 	    (vm_offset_t) bp->b_data + newbsize);
4354 	if (origbuf != NULL) {
4355 		bcopy(origbuf, bp->b_data, origbufsize);
4356 		free(origbuf, M_BIOBUF);
4357 	}
4358 	bufspace_adjust(bp, newbsize);
4359 }
4360 
4361 /*
4362  * This code constitutes the buffer memory from either anonymous system
4363  * memory (in the case of non-VMIO operations) or from an associated
4364  * VM object (in the case of VMIO operations).  This code is able to
4365  * resize a buffer up or down.
4366  *
4367  * Note that this code is tricky, and has many complications to resolve
4368  * deadlock or inconsistent data situations.  Tread lightly!!!
4369  * There are B_CACHE and B_DELWRI interactions that must be dealt with by
4370  * the caller.  Calling this code willy nilly can result in the loss of data.
4371  *
4372  * allocbuf() only adjusts B_CACHE for VMIO buffers.  getblk() deals with
4373  * B_CACHE for the non-VMIO case.
4374  */
4375 int
4376 allocbuf(struct buf *bp, int size)
4377 {
4378 	int newbsize;
4379 
4380 	if (bp->b_bcount == size)
4381 		return (1);
4382 
4383 	KASSERT(bp->b_kvasize == 0 || bp->b_kvasize >= size,
4384 	    ("allocbuf: buffer too small %p %#x %#x",
4385 	    bp, bp->b_kvasize, size));
4386 
4387 	newbsize = roundup2(size, DEV_BSIZE);
4388 	if ((bp->b_flags & B_VMIO) == 0) {
4389 		if ((bp->b_flags & B_MALLOC) == 0)
4390 			newbsize = round_page(newbsize);
4391 		/*
4392 		 * Just get anonymous memory from the kernel.  Don't
4393 		 * mess with B_CACHE.
4394 		 */
4395 		if (newbsize < bp->b_bufsize)
4396 			vfs_nonvmio_truncate(bp, newbsize);
4397 		else if (newbsize > bp->b_bufsize)
4398 			vfs_nonvmio_extend(bp, newbsize);
4399 	} else {
4400 		int desiredpages;
4401 
4402 		desiredpages = size == 0 ? 0 :
4403 		    num_pages((bp->b_offset & PAGE_MASK) + newbsize);
4404 
4405 		KASSERT((bp->b_flags & B_MALLOC) == 0,
4406 		    ("allocbuf: VMIO buffer can't be malloced %p", bp));
4407 
4408 		/*
4409 		 * Set B_CACHE initially if buffer is 0 length or will become
4410 		 * 0-length.
4411 		 */
4412 		if (size == 0 || bp->b_bufsize == 0)
4413 			bp->b_flags |= B_CACHE;
4414 
4415 		if (newbsize < bp->b_bufsize)
4416 			vfs_vmio_truncate(bp, desiredpages);
4417 		/* XXX This looks as if it should be newbsize > b_bufsize */
4418 		else if (size > bp->b_bcount)
4419 			vfs_vmio_extend(bp, desiredpages, size);
4420 		bufspace_adjust(bp, newbsize);
4421 	}
4422 	bp->b_bcount = size;		/* requested buffer size. */
4423 	return (1);
4424 }
4425 
4426 extern int inflight_transient_maps;
4427 
4428 static struct bio_queue nondump_bios;
4429 
4430 void
4431 biodone(struct bio *bp)
4432 {
4433 	struct mtx *mtxp;
4434 	void (*done)(struct bio *);
4435 	vm_offset_t start, end;
4436 
4437 	biotrack(bp, __func__);
4438 
4439 	/*
4440 	 * Avoid completing I/O when dumping after a panic since that may
4441 	 * result in a deadlock in the filesystem or pager code.  Note that
4442 	 * this doesn't affect dumps that were started manually since we aim
4443 	 * to keep the system usable after it has been resumed.
4444 	 */
4445 	if (__predict_false(dumping && SCHEDULER_STOPPED())) {
4446 		TAILQ_INSERT_HEAD(&nondump_bios, bp, bio_queue);
4447 		return;
4448 	}
4449 	if ((bp->bio_flags & BIO_TRANSIENT_MAPPING) != 0) {
4450 		bp->bio_flags &= ~BIO_TRANSIENT_MAPPING;
4451 		bp->bio_flags |= BIO_UNMAPPED;
4452 		start = trunc_page((vm_offset_t)bp->bio_data);
4453 		end = round_page((vm_offset_t)bp->bio_data + bp->bio_length);
4454 		bp->bio_data = unmapped_buf;
4455 		pmap_qremove(start, atop(end - start));
4456 		vmem_free(transient_arena, start, end - start);
4457 		atomic_add_int(&inflight_transient_maps, -1);
4458 	}
4459 	done = bp->bio_done;
4460 	/*
4461 	 * The check for done == biodone is to allow biodone to be
4462 	 * used as a bio_done routine.
4463 	 */
4464 	if (done == NULL || done == biodone) {
4465 		mtxp = mtx_pool_find(mtxpool_sleep, bp);
4466 		mtx_lock(mtxp);
4467 		bp->bio_flags |= BIO_DONE;
4468 		wakeup(bp);
4469 		mtx_unlock(mtxp);
4470 	} else
4471 		done(bp);
4472 }
4473 
4474 /*
4475  * Wait for a BIO to finish.
4476  */
4477 int
4478 biowait(struct bio *bp, const char *wmesg)
4479 {
4480 	struct mtx *mtxp;
4481 
4482 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
4483 	mtx_lock(mtxp);
4484 	while ((bp->bio_flags & BIO_DONE) == 0)
4485 		msleep(bp, mtxp, PRIBIO, wmesg, 0);
4486 	mtx_unlock(mtxp);
4487 	if (bp->bio_error != 0)
4488 		return (bp->bio_error);
4489 	if (!(bp->bio_flags & BIO_ERROR))
4490 		return (0);
4491 	return (EIO);
4492 }
4493 
4494 void
4495 biofinish(struct bio *bp, struct devstat *stat, int error)
4496 {
4497 
4498 	if (error) {
4499 		bp->bio_error = error;
4500 		bp->bio_flags |= BIO_ERROR;
4501 	}
4502 	if (stat != NULL)
4503 		devstat_end_transaction_bio(stat, bp);
4504 	biodone(bp);
4505 }
4506 
4507 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
4508 void
4509 biotrack_buf(struct bio *bp, const char *location)
4510 {
4511 
4512 	buf_track(bp->bio_track_bp, location);
4513 }
4514 #endif
4515 
4516 /*
4517  *	bufwait:
4518  *
4519  *	Wait for buffer I/O completion, returning error status.  The buffer
4520  *	is left locked and B_DONE on return.  B_EINTR is converted into an EINTR
4521  *	error and cleared.
4522  */
4523 int
4524 bufwait(struct buf *bp)
4525 {
4526 	if (bp->b_iocmd == BIO_READ)
4527 		bwait(bp, PRIBIO, "biord");
4528 	else
4529 		bwait(bp, PRIBIO, "biowr");
4530 	if (bp->b_flags & B_EINTR) {
4531 		bp->b_flags &= ~B_EINTR;
4532 		return (EINTR);
4533 	}
4534 	if (bp->b_ioflags & BIO_ERROR) {
4535 		return (bp->b_error ? bp->b_error : EIO);
4536 	} else {
4537 		return (0);
4538 	}
4539 }
4540 
4541 /*
4542  *	bufdone:
4543  *
4544  *	Finish I/O on a buffer, optionally calling a completion function.
4545  *	This is usually called from an interrupt so process blocking is
4546  *	not allowed.
4547  *
4548  *	biodone is also responsible for setting B_CACHE in a B_VMIO bp.
4549  *	In a non-VMIO bp, B_CACHE will be set on the next getblk()
4550  *	assuming B_INVAL is clear.
4551  *
4552  *	For the VMIO case, we set B_CACHE if the op was a read and no
4553  *	read error occurred, or if the op was a write.  B_CACHE is never
4554  *	set if the buffer is invalid or otherwise uncacheable.
4555  *
4556  *	bufdone does not mess with B_INVAL, allowing the I/O routine or the
4557  *	initiator to leave B_INVAL set to brelse the buffer out of existence
4558  *	in the biodone routine.
4559  */
4560 void
4561 bufdone(struct buf *bp)
4562 {
4563 	struct bufobj *dropobj;
4564 	void    (*biodone)(struct buf *);
4565 
4566 	buf_track(bp, __func__);
4567 	CTR3(KTR_BUF, "bufdone(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
4568 	dropobj = NULL;
4569 
4570 	KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
4571 
4572 	runningbufwakeup(bp);
4573 	if (bp->b_iocmd == BIO_WRITE)
4574 		dropobj = bp->b_bufobj;
4575 	/* call optional completion function if requested */
4576 	if (bp->b_iodone != NULL) {
4577 		biodone = bp->b_iodone;
4578 		bp->b_iodone = NULL;
4579 		(*biodone) (bp);
4580 		if (dropobj)
4581 			bufobj_wdrop(dropobj);
4582 		return;
4583 	}
4584 	if (bp->b_flags & B_VMIO) {
4585 		/*
4586 		 * Set B_CACHE if the op was a normal read and no error
4587 		 * occurred.  B_CACHE is set for writes in the b*write()
4588 		 * routines.
4589 		 */
4590 		if (bp->b_iocmd == BIO_READ &&
4591 		    !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
4592 		    !(bp->b_ioflags & BIO_ERROR))
4593 			bp->b_flags |= B_CACHE;
4594 		vfs_vmio_iodone(bp);
4595 	}
4596 	if (!LIST_EMPTY(&bp->b_dep))
4597 		buf_complete(bp);
4598 	if ((bp->b_flags & B_CKHASH) != 0) {
4599 		KASSERT(bp->b_iocmd == BIO_READ,
4600 		    ("bufdone: b_iocmd %d not BIO_READ", bp->b_iocmd));
4601 		KASSERT(buf_mapped(bp), ("bufdone: bp %p not mapped", bp));
4602 		(*bp->b_ckhashcalc)(bp);
4603 	}
4604 	/*
4605 	 * For asynchronous completions, release the buffer now. The brelse
4606 	 * will do a wakeup there if necessary - so no need to do a wakeup
4607 	 * here in the async case. The sync case always needs to do a wakeup.
4608 	 */
4609 	if (bp->b_flags & B_ASYNC) {
4610 		if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) ||
4611 		    (bp->b_ioflags & BIO_ERROR))
4612 			brelse(bp);
4613 		else
4614 			bqrelse(bp);
4615 	} else
4616 		bdone(bp);
4617 	if (dropobj)
4618 		bufobj_wdrop(dropobj);
4619 }
4620 
4621 /*
4622  * This routine is called in lieu of iodone in the case of
4623  * incomplete I/O.  This keeps the busy status for pages
4624  * consistent.
4625  */
4626 void
4627 vfs_unbusy_pages(struct buf *bp)
4628 {
4629 	int i;
4630 	vm_object_t obj;
4631 	vm_page_t m;
4632 
4633 	runningbufwakeup(bp);
4634 	if (!(bp->b_flags & B_VMIO))
4635 		return;
4636 
4637 	obj = bp->b_bufobj->bo_object;
4638 	for (i = 0; i < bp->b_npages; i++) {
4639 		m = bp->b_pages[i];
4640 		if (m == bogus_page) {
4641 			m = vm_page_relookup(obj, OFF_TO_IDX(bp->b_offset) + i);
4642 			if (!m)
4643 				panic("vfs_unbusy_pages: page missing\n");
4644 			bp->b_pages[i] = m;
4645 			if (buf_mapped(bp)) {
4646 				BUF_CHECK_MAPPED(bp);
4647 				pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
4648 				    bp->b_pages, bp->b_npages);
4649 			} else
4650 				BUF_CHECK_UNMAPPED(bp);
4651 		}
4652 		vm_page_sunbusy(m);
4653 	}
4654 	vm_object_pip_wakeupn(obj, bp->b_npages);
4655 }
4656 
4657 /*
4658  * vfs_page_set_valid:
4659  *
4660  *	Set the valid bits in a page based on the supplied offset.   The
4661  *	range is restricted to the buffer's size.
4662  *
4663  *	This routine is typically called after a read completes.
4664  */
4665 static void
4666 vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m)
4667 {
4668 	vm_ooffset_t eoff;
4669 
4670 	/*
4671 	 * Compute the end offset, eoff, such that [off, eoff) does not span a
4672 	 * page boundary and eoff is not greater than the end of the buffer.
4673 	 * The end of the buffer, in this case, is our file EOF, not the
4674 	 * allocation size of the buffer.
4675 	 */
4676 	eoff = (off + PAGE_SIZE) & ~(vm_ooffset_t)PAGE_MASK;
4677 	if (eoff > bp->b_offset + bp->b_bcount)
4678 		eoff = bp->b_offset + bp->b_bcount;
4679 
4680 	/*
4681 	 * Set valid range.  This is typically the entire buffer and thus the
4682 	 * entire page.
4683 	 */
4684 	if (eoff > off)
4685 		vm_page_set_valid_range(m, off & PAGE_MASK, eoff - off);
4686 }
4687 
4688 /*
4689  * vfs_page_set_validclean:
4690  *
4691  *	Set the valid bits and clear the dirty bits in a page based on the
4692  *	supplied offset.   The range is restricted to the buffer's size.
4693  */
4694 static void
4695 vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m)
4696 {
4697 	vm_ooffset_t soff, eoff;
4698 
4699 	/*
4700 	 * Start and end offsets in buffer.  eoff - soff may not cross a
4701 	 * page boundary or cross the end of the buffer.  The end of the
4702 	 * buffer, in this case, is our file EOF, not the allocation size
4703 	 * of the buffer.
4704 	 */
4705 	soff = off;
4706 	eoff = (off + PAGE_SIZE) & ~(off_t)PAGE_MASK;
4707 	if (eoff > bp->b_offset + bp->b_bcount)
4708 		eoff = bp->b_offset + bp->b_bcount;
4709 
4710 	/*
4711 	 * Set valid range.  This is typically the entire buffer and thus the
4712 	 * entire page.
4713 	 */
4714 	if (eoff > soff) {
4715 		vm_page_set_validclean(
4716 		    m,
4717 		   (vm_offset_t) (soff & PAGE_MASK),
4718 		   (vm_offset_t) (eoff - soff)
4719 		);
4720 	}
4721 }
4722 
4723 /*
4724  * Acquire a shared busy on all pages in the buf.
4725  */
4726 void
4727 vfs_busy_pages_acquire(struct buf *bp)
4728 {
4729 	int i;
4730 
4731 	for (i = 0; i < bp->b_npages; i++)
4732 		vm_page_busy_acquire(bp->b_pages[i], VM_ALLOC_SBUSY);
4733 }
4734 
4735 void
4736 vfs_busy_pages_release(struct buf *bp)
4737 {
4738 	int i;
4739 
4740 	for (i = 0; i < bp->b_npages; i++)
4741 		vm_page_sunbusy(bp->b_pages[i]);
4742 }
4743 
4744 /*
4745  * This routine is called before a device strategy routine.
4746  * It is used to tell the VM system that paging I/O is in
4747  * progress, and treat the pages associated with the buffer
4748  * almost as being exclusive busy.  Also the object paging_in_progress
4749  * flag is handled to make sure that the object doesn't become
4750  * inconsistent.
4751  *
4752  * Since I/O has not been initiated yet, certain buffer flags
4753  * such as BIO_ERROR or B_INVAL may be in an inconsistent state
4754  * and should be ignored.
4755  */
4756 void
4757 vfs_busy_pages(struct buf *bp, int clear_modify)
4758 {
4759 	vm_object_t obj;
4760 	vm_ooffset_t foff;
4761 	vm_page_t m;
4762 	int i;
4763 	bool bogus;
4764 
4765 	if (!(bp->b_flags & B_VMIO))
4766 		return;
4767 
4768 	obj = bp->b_bufobj->bo_object;
4769 	foff = bp->b_offset;
4770 	KASSERT(bp->b_offset != NOOFFSET,
4771 	    ("vfs_busy_pages: no buffer offset"));
4772 	if ((bp->b_flags & B_CLUSTER) == 0) {
4773 		vm_object_pip_add(obj, bp->b_npages);
4774 		vfs_busy_pages_acquire(bp);
4775 	}
4776 	if (bp->b_bufsize != 0)
4777 		vfs_setdirty_range(bp);
4778 	bogus = false;
4779 	for (i = 0; i < bp->b_npages; i++) {
4780 		m = bp->b_pages[i];
4781 		vm_page_assert_sbusied(m);
4782 
4783 		/*
4784 		 * When readying a buffer for a read ( i.e
4785 		 * clear_modify == 0 ), it is important to do
4786 		 * bogus_page replacement for valid pages in
4787 		 * partially instantiated buffers.  Partially
4788 		 * instantiated buffers can, in turn, occur when
4789 		 * reconstituting a buffer from its VM backing store
4790 		 * base.  We only have to do this if B_CACHE is
4791 		 * clear ( which causes the I/O to occur in the
4792 		 * first place ).  The replacement prevents the read
4793 		 * I/O from overwriting potentially dirty VM-backed
4794 		 * pages.  XXX bogus page replacement is, uh, bogus.
4795 		 * It may not work properly with small-block devices.
4796 		 * We need to find a better way.
4797 		 */
4798 		if (clear_modify) {
4799 			pmap_remove_write(m);
4800 			vfs_page_set_validclean(bp, foff, m);
4801 		} else if (vm_page_all_valid(m) &&
4802 		    (bp->b_flags & B_CACHE) == 0) {
4803 			bp->b_pages[i] = bogus_page;
4804 			bogus = true;
4805 		}
4806 		foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
4807 	}
4808 	if (bogus && buf_mapped(bp)) {
4809 		BUF_CHECK_MAPPED(bp);
4810 		pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
4811 		    bp->b_pages, bp->b_npages);
4812 	}
4813 }
4814 
4815 /*
4816  *	vfs_bio_set_valid:
4817  *
4818  *	Set the range within the buffer to valid.  The range is
4819  *	relative to the beginning of the buffer, b_offset.  Note that
4820  *	b_offset itself may be offset from the beginning of the first
4821  *	page.
4822  */
4823 void
4824 vfs_bio_set_valid(struct buf *bp, int base, int size)
4825 {
4826 	int i, n;
4827 	vm_page_t m;
4828 
4829 	if (!(bp->b_flags & B_VMIO))
4830 		return;
4831 
4832 	/*
4833 	 * Fixup base to be relative to beginning of first page.
4834 	 * Set initial n to be the maximum number of bytes in the
4835 	 * first page that can be validated.
4836 	 */
4837 	base += (bp->b_offset & PAGE_MASK);
4838 	n = PAGE_SIZE - (base & PAGE_MASK);
4839 
4840 	/*
4841 	 * Busy may not be strictly necessary here because the pages are
4842 	 * unlikely to be fully valid and the vnode lock will synchronize
4843 	 * their access via getpages.  It is grabbed for consistency with
4844 	 * other page validation.
4845 	 */
4846 	vfs_busy_pages_acquire(bp);
4847 	for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4848 		m = bp->b_pages[i];
4849 		if (n > size)
4850 			n = size;
4851 		vm_page_set_valid_range(m, base & PAGE_MASK, n);
4852 		base += n;
4853 		size -= n;
4854 		n = PAGE_SIZE;
4855 	}
4856 	vfs_busy_pages_release(bp);
4857 }
4858 
4859 /*
4860  *	vfs_bio_clrbuf:
4861  *
4862  *	If the specified buffer is a non-VMIO buffer, clear the entire
4863  *	buffer.  If the specified buffer is a VMIO buffer, clear and
4864  *	validate only the previously invalid portions of the buffer.
4865  *	This routine essentially fakes an I/O, so we need to clear
4866  *	BIO_ERROR and B_INVAL.
4867  *
4868  *	Note that while we only theoretically need to clear through b_bcount,
4869  *	we go ahead and clear through b_bufsize.
4870  */
4871 void
4872 vfs_bio_clrbuf(struct buf *bp)
4873 {
4874 	int i, j, sa, ea, slide, zbits;
4875 	vm_page_bits_t mask;
4876 
4877 	if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) {
4878 		clrbuf(bp);
4879 		return;
4880 	}
4881 	bp->b_flags &= ~B_INVAL;
4882 	bp->b_ioflags &= ~BIO_ERROR;
4883 	vfs_busy_pages_acquire(bp);
4884 	sa = bp->b_offset & PAGE_MASK;
4885 	slide = 0;
4886 	for (i = 0; i < bp->b_npages; i++, sa = 0) {
4887 		slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize);
4888 		ea = slide & PAGE_MASK;
4889 		if (ea == 0)
4890 			ea = PAGE_SIZE;
4891 		if (bp->b_pages[i] == bogus_page)
4892 			continue;
4893 		j = sa / DEV_BSIZE;
4894 		zbits = (sizeof(vm_page_bits_t) * NBBY) -
4895 		    (ea - sa) / DEV_BSIZE;
4896 		mask = (VM_PAGE_BITS_ALL >> zbits) << j;
4897 		if ((bp->b_pages[i]->valid & mask) == mask)
4898 			continue;
4899 		if ((bp->b_pages[i]->valid & mask) == 0)
4900 			pmap_zero_page_area(bp->b_pages[i], sa, ea - sa);
4901 		else {
4902 			for (; sa < ea; sa += DEV_BSIZE, j++) {
4903 				if ((bp->b_pages[i]->valid & (1 << j)) == 0) {
4904 					pmap_zero_page_area(bp->b_pages[i],
4905 					    sa, DEV_BSIZE);
4906 				}
4907 			}
4908 		}
4909 		vm_page_set_valid_range(bp->b_pages[i], j * DEV_BSIZE,
4910 		    roundup2(ea - sa, DEV_BSIZE));
4911 	}
4912 	vfs_busy_pages_release(bp);
4913 	bp->b_resid = 0;
4914 }
4915 
4916 void
4917 vfs_bio_bzero_buf(struct buf *bp, int base, int size)
4918 {
4919 	vm_page_t m;
4920 	int i, n;
4921 
4922 	if (buf_mapped(bp)) {
4923 		BUF_CHECK_MAPPED(bp);
4924 		bzero(bp->b_data + base, size);
4925 	} else {
4926 		BUF_CHECK_UNMAPPED(bp);
4927 		n = PAGE_SIZE - (base & PAGE_MASK);
4928 		for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4929 			m = bp->b_pages[i];
4930 			if (n > size)
4931 				n = size;
4932 			pmap_zero_page_area(m, base & PAGE_MASK, n);
4933 			base += n;
4934 			size -= n;
4935 			n = PAGE_SIZE;
4936 		}
4937 	}
4938 }
4939 
4940 /*
4941  * Update buffer flags based on I/O request parameters, optionally releasing the
4942  * buffer.  If it's VMIO or direct I/O, the buffer pages are released to the VM,
4943  * where they may be placed on a page queue (VMIO) or freed immediately (direct
4944  * I/O).  Otherwise the buffer is released to the cache.
4945  */
4946 static void
4947 b_io_dismiss(struct buf *bp, int ioflag, bool release)
4948 {
4949 
4950 	KASSERT((ioflag & IO_NOREUSE) == 0 || (ioflag & IO_VMIO) != 0,
4951 	    ("buf %p non-VMIO noreuse", bp));
4952 
4953 	if ((ioflag & IO_DIRECT) != 0)
4954 		bp->b_flags |= B_DIRECT;
4955 	if ((ioflag & IO_EXT) != 0)
4956 		bp->b_xflags |= BX_ALTDATA;
4957 	if ((ioflag & (IO_VMIO | IO_DIRECT)) != 0 && LIST_EMPTY(&bp->b_dep)) {
4958 		bp->b_flags |= B_RELBUF;
4959 		if ((ioflag & IO_NOREUSE) != 0)
4960 			bp->b_flags |= B_NOREUSE;
4961 		if (release)
4962 			brelse(bp);
4963 	} else if (release)
4964 		bqrelse(bp);
4965 }
4966 
4967 void
4968 vfs_bio_brelse(struct buf *bp, int ioflag)
4969 {
4970 
4971 	b_io_dismiss(bp, ioflag, true);
4972 }
4973 
4974 void
4975 vfs_bio_set_flags(struct buf *bp, int ioflag)
4976 {
4977 
4978 	b_io_dismiss(bp, ioflag, false);
4979 }
4980 
4981 /*
4982  * vm_hold_load_pages and vm_hold_free_pages get pages into
4983  * a buffers address space.  The pages are anonymous and are
4984  * not associated with a file object.
4985  */
4986 static void
4987 vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
4988 {
4989 	vm_offset_t pg;
4990 	vm_page_t p;
4991 	int index;
4992 
4993 	BUF_CHECK_MAPPED(bp);
4994 
4995 	to = round_page(to);
4996 	from = round_page(from);
4997 	index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
4998 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
4999 	KASSERT(to - from <= maxbcachebuf,
5000 	    ("vm_hold_load_pages too large %p %#jx %#jx %u",
5001 	    bp, (uintmax_t)from, (uintmax_t)to, maxbcachebuf));
5002 
5003 	for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
5004 		/*
5005 		 * note: must allocate system pages since blocking here
5006 		 * could interfere with paging I/O, no matter which
5007 		 * process we are.
5008 		 */
5009 		p = vm_page_alloc_noobj(VM_ALLOC_SYSTEM | VM_ALLOC_WIRED |
5010 		    VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT) | VM_ALLOC_WAITOK);
5011 		pmap_qenter(pg, &p, 1);
5012 		bp->b_pages[index] = p;
5013 	}
5014 	bp->b_npages = index;
5015 }
5016 
5017 /* Return pages associated with this buf to the vm system */
5018 static void
5019 vm_hold_free_pages(struct buf *bp, int newbsize)
5020 {
5021 	vm_offset_t from;
5022 	vm_page_t p;
5023 	int index, newnpages;
5024 
5025 	BUF_CHECK_MAPPED(bp);
5026 
5027 	from = round_page((vm_offset_t)bp->b_data + newbsize);
5028 	newnpages = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
5029 	if (bp->b_npages > newnpages)
5030 		pmap_qremove(from, bp->b_npages - newnpages);
5031 	for (index = newnpages; index < bp->b_npages; index++) {
5032 		p = bp->b_pages[index];
5033 		bp->b_pages[index] = NULL;
5034 		vm_page_unwire_noq(p);
5035 		vm_page_free(p);
5036 	}
5037 	bp->b_npages = newnpages;
5038 }
5039 
5040 /*
5041  * Map an IO request into kernel virtual address space.
5042  *
5043  * All requests are (re)mapped into kernel VA space.
5044  * Notice that we use b_bufsize for the size of the buffer
5045  * to be mapped.  b_bcount might be modified by the driver.
5046  *
5047  * Note that even if the caller determines that the address space should
5048  * be valid, a race or a smaller-file mapped into a larger space may
5049  * actually cause vmapbuf() to fail, so all callers of vmapbuf() MUST
5050  * check the return value.
5051  *
5052  * This function only works with pager buffers.
5053  */
5054 int
5055 vmapbuf(struct buf *bp, void *uaddr, size_t len, int mapbuf)
5056 {
5057 	vm_prot_t prot;
5058 	int pidx;
5059 
5060 	MPASS((bp->b_flags & B_MAXPHYS) != 0);
5061 	prot = VM_PROT_READ;
5062 	if (bp->b_iocmd == BIO_READ)
5063 		prot |= VM_PROT_WRITE;	/* Less backwards than it looks */
5064 	pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
5065 	    (vm_offset_t)uaddr, len, prot, bp->b_pages, PBUF_PAGES);
5066 	if (pidx < 0)
5067 		return (-1);
5068 	bp->b_bufsize = len;
5069 	bp->b_npages = pidx;
5070 	bp->b_offset = ((vm_offset_t)uaddr) & PAGE_MASK;
5071 	if (mapbuf || !unmapped_buf_allowed) {
5072 		pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
5073 		bp->b_data = bp->b_kvabase + bp->b_offset;
5074 	} else
5075 		bp->b_data = unmapped_buf;
5076 	return (0);
5077 }
5078 
5079 /*
5080  * Free the io map PTEs associated with this IO operation.
5081  * We also invalidate the TLB entries and restore the original b_addr.
5082  *
5083  * This function only works with pager buffers.
5084  */
5085 void
5086 vunmapbuf(struct buf *bp)
5087 {
5088 	int npages;
5089 
5090 	npages = bp->b_npages;
5091 	if (buf_mapped(bp))
5092 		pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
5093 	vm_page_unhold_pages(bp->b_pages, npages);
5094 
5095 	bp->b_data = unmapped_buf;
5096 }
5097 
5098 void
5099 bdone(struct buf *bp)
5100 {
5101 	struct mtx *mtxp;
5102 
5103 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
5104 	mtx_lock(mtxp);
5105 	bp->b_flags |= B_DONE;
5106 	wakeup(bp);
5107 	mtx_unlock(mtxp);
5108 }
5109 
5110 void
5111 bwait(struct buf *bp, u_char pri, const char *wchan)
5112 {
5113 	struct mtx *mtxp;
5114 
5115 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
5116 	mtx_lock(mtxp);
5117 	while ((bp->b_flags & B_DONE) == 0)
5118 		msleep(bp, mtxp, pri, wchan, 0);
5119 	mtx_unlock(mtxp);
5120 }
5121 
5122 int
5123 bufsync(struct bufobj *bo, int waitfor)
5124 {
5125 
5126 	return (VOP_FSYNC(bo2vnode(bo), waitfor, curthread));
5127 }
5128 
5129 void
5130 bufstrategy(struct bufobj *bo, struct buf *bp)
5131 {
5132 	int i __unused;
5133 	struct vnode *vp;
5134 
5135 	vp = bp->b_vp;
5136 	KASSERT(vp == bo->bo_private, ("Inconsistent vnode bufstrategy"));
5137 	KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
5138 	    ("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp));
5139 	i = VOP_STRATEGY(vp, bp);
5140 	KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp));
5141 }
5142 
5143 /*
5144  * Initialize a struct bufobj before use.  Memory is assumed zero filled.
5145  */
5146 void
5147 bufobj_init(struct bufobj *bo, void *private)
5148 {
5149 	static volatile int bufobj_cleanq;
5150 
5151         bo->bo_domain =
5152             atomic_fetchadd_int(&bufobj_cleanq, 1) % buf_domains;
5153         rw_init(BO_LOCKPTR(bo), "bufobj interlock");
5154         bo->bo_private = private;
5155         TAILQ_INIT(&bo->bo_clean.bv_hd);
5156 	pctrie_init(&bo->bo_clean.bv_root);
5157         TAILQ_INIT(&bo->bo_dirty.bv_hd);
5158 	pctrie_init(&bo->bo_dirty.bv_root);
5159 }
5160 
5161 void
5162 bufobj_wrefl(struct bufobj *bo)
5163 {
5164 
5165 	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
5166 	ASSERT_BO_WLOCKED(bo);
5167 	bo->bo_numoutput++;
5168 }
5169 
5170 void
5171 bufobj_wref(struct bufobj *bo)
5172 {
5173 
5174 	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
5175 	BO_LOCK(bo);
5176 	bo->bo_numoutput++;
5177 	BO_UNLOCK(bo);
5178 }
5179 
5180 void
5181 bufobj_wdrop(struct bufobj *bo)
5182 {
5183 
5184 	KASSERT(bo != NULL, ("NULL bo in bufobj_wdrop"));
5185 	BO_LOCK(bo);
5186 	KASSERT(bo->bo_numoutput > 0, ("bufobj_wdrop non-positive count"));
5187 	if ((--bo->bo_numoutput == 0) && (bo->bo_flag & BO_WWAIT)) {
5188 		bo->bo_flag &= ~BO_WWAIT;
5189 		wakeup(&bo->bo_numoutput);
5190 	}
5191 	BO_UNLOCK(bo);
5192 }
5193 
5194 int
5195 bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
5196 {
5197 	int error;
5198 
5199 	KASSERT(bo != NULL, ("NULL bo in bufobj_wwait"));
5200 	ASSERT_BO_WLOCKED(bo);
5201 	error = 0;
5202 	while (bo->bo_numoutput) {
5203 		bo->bo_flag |= BO_WWAIT;
5204 		error = msleep(&bo->bo_numoutput, BO_LOCKPTR(bo),
5205 		    slpflag | (PRIBIO + 1), "bo_wwait", timeo);
5206 		if (error)
5207 			break;
5208 	}
5209 	return (error);
5210 }
5211 
5212 /*
5213  * Set bio_data or bio_ma for struct bio from the struct buf.
5214  */
5215 void
5216 bdata2bio(struct buf *bp, struct bio *bip)
5217 {
5218 
5219 	if (!buf_mapped(bp)) {
5220 		KASSERT(unmapped_buf_allowed, ("unmapped"));
5221 		bip->bio_ma = bp->b_pages;
5222 		bip->bio_ma_n = bp->b_npages;
5223 		bip->bio_data = unmapped_buf;
5224 		bip->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
5225 		bip->bio_flags |= BIO_UNMAPPED;
5226 		KASSERT(round_page(bip->bio_ma_offset + bip->bio_length) /
5227 		    PAGE_SIZE == bp->b_npages,
5228 		    ("Buffer %p too short: %d %lld %d", bp, bip->bio_ma_offset,
5229 		    (long long)bip->bio_length, bip->bio_ma_n));
5230 	} else {
5231 		bip->bio_data = bp->b_data;
5232 		bip->bio_ma = NULL;
5233 	}
5234 }
5235 
5236 struct memdesc
5237 memdesc_bio(struct bio *bio)
5238 {
5239 	if ((bio->bio_flags & BIO_VLIST) != 0)
5240 		return (memdesc_vlist((struct bus_dma_segment *)bio->bio_data,
5241 		    bio->bio_ma_n));
5242 
5243 	if ((bio->bio_flags & BIO_UNMAPPED) != 0)
5244 		return (memdesc_vmpages(bio->bio_ma, bio->bio_bcount,
5245 		    bio->bio_ma_offset));
5246 
5247 	return (memdesc_vaddr(bio->bio_data, bio->bio_bcount));
5248 }
5249 
5250 static int buf_pager_relbuf;
5251 SYSCTL_INT(_vfs, OID_AUTO, buf_pager_relbuf, CTLFLAG_RWTUN,
5252     &buf_pager_relbuf, 0,
5253     "Make buffer pager release buffers after reading");
5254 
5255 /*
5256  * The buffer pager.  It uses buffer reads to validate pages.
5257  *
5258  * In contrast to the generic local pager from vm/vnode_pager.c, this
5259  * pager correctly and easily handles volumes where the underlying
5260  * device block size is greater than the machine page size.  The
5261  * buffer cache transparently extends the requested page run to be
5262  * aligned at the block boundary, and does the necessary bogus page
5263  * replacements in the addends to avoid obliterating already valid
5264  * pages.
5265  *
5266  * The only non-trivial issue is that the exclusive busy state for
5267  * pages, which is assumed by the vm_pager_getpages() interface, is
5268  * incompatible with the VMIO buffer cache's desire to share-busy the
5269  * pages.  This function performs a trivial downgrade of the pages'
5270  * state before reading buffers, and a less trivial upgrade from the
5271  * shared-busy to excl-busy state after the read.
5272  */
5273 int
5274 vfs_bio_getpages(struct vnode *vp, vm_page_t *ma, int count,
5275     int *rbehind, int *rahead, vbg_get_lblkno_t get_lblkno,
5276     vbg_get_blksize_t get_blksize)
5277 {
5278 	vm_page_t m;
5279 	vm_object_t object;
5280 	struct buf *bp;
5281 	struct mount *mp;
5282 	daddr_t lbn, lbnp;
5283 	vm_ooffset_t la, lb, poff, poffe;
5284 	long bo_bs, bsize;
5285 	int br_flags, error, i, pgsin, pgsin_a, pgsin_b;
5286 	bool redo, lpart;
5287 
5288 	object = vp->v_object;
5289 	mp = vp->v_mount;
5290 	error = 0;
5291 	la = IDX_TO_OFF(ma[count - 1]->pindex);
5292 	if (la >= object->un_pager.vnp.vnp_size)
5293 		return (VM_PAGER_BAD);
5294 
5295 	/*
5296 	 * Change the meaning of la from where the last requested page starts
5297 	 * to where it ends, because that's the end of the requested region
5298 	 * and the start of the potential read-ahead region.
5299 	 */
5300 	la += PAGE_SIZE;
5301 	lpart = la > object->un_pager.vnp.vnp_size;
5302 	error = get_blksize(vp, get_lblkno(vp, IDX_TO_OFF(ma[0]->pindex)),
5303 	    &bo_bs);
5304 	if (error != 0)
5305 		return (VM_PAGER_ERROR);
5306 
5307 	/*
5308 	 * Calculate read-ahead, behind and total pages.
5309 	 */
5310 	pgsin = count;
5311 	lb = IDX_TO_OFF(ma[0]->pindex);
5312 	pgsin_b = OFF_TO_IDX(lb - rounddown2(lb, bo_bs));
5313 	pgsin += pgsin_b;
5314 	if (rbehind != NULL)
5315 		*rbehind = pgsin_b;
5316 	pgsin_a = OFF_TO_IDX(roundup2(la, bo_bs) - la);
5317 	if (la + IDX_TO_OFF(pgsin_a) >= object->un_pager.vnp.vnp_size)
5318 		pgsin_a = OFF_TO_IDX(roundup2(object->un_pager.vnp.vnp_size,
5319 		    PAGE_SIZE) - la);
5320 	pgsin += pgsin_a;
5321 	if (rahead != NULL)
5322 		*rahead = pgsin_a;
5323 	VM_CNT_INC(v_vnodein);
5324 	VM_CNT_ADD(v_vnodepgsin, pgsin);
5325 
5326 	br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS)
5327 	    != 0) ? GB_UNMAPPED : 0;
5328 again:
5329 	for (i = 0; i < count; i++) {
5330 		if (ma[i] != bogus_page)
5331 			vm_page_busy_downgrade(ma[i]);
5332 	}
5333 
5334 	lbnp = -1;
5335 	for (i = 0; i < count; i++) {
5336 		m = ma[i];
5337 		if (m == bogus_page)
5338 			continue;
5339 
5340 		/*
5341 		 * Pages are shared busy and the object lock is not
5342 		 * owned, which together allow for the pages'
5343 		 * invalidation.  The racy test for validity avoids
5344 		 * useless creation of the buffer for the most typical
5345 		 * case when invalidation is not used in redo or for
5346 		 * parallel read.  The shared->excl upgrade loop at
5347 		 * the end of the function catches the race in a
5348 		 * reliable way (protected by the object lock).
5349 		 */
5350 		if (vm_page_all_valid(m))
5351 			continue;
5352 
5353 		poff = IDX_TO_OFF(m->pindex);
5354 		poffe = MIN(poff + PAGE_SIZE, object->un_pager.vnp.vnp_size);
5355 		for (; poff < poffe; poff += bsize) {
5356 			lbn = get_lblkno(vp, poff);
5357 			if (lbn == lbnp)
5358 				goto next_page;
5359 			lbnp = lbn;
5360 
5361 			error = get_blksize(vp, lbn, &bsize);
5362 			if (error == 0)
5363 				error = bread_gb(vp, lbn, bsize,
5364 				    curthread->td_ucred, br_flags, &bp);
5365 			if (error != 0)
5366 				goto end_pages;
5367 			if (bp->b_rcred == curthread->td_ucred) {
5368 				crfree(bp->b_rcred);
5369 				bp->b_rcred = NOCRED;
5370 			}
5371 			if (LIST_EMPTY(&bp->b_dep)) {
5372 				/*
5373 				 * Invalidation clears m->valid, but
5374 				 * may leave B_CACHE flag if the
5375 				 * buffer existed at the invalidation
5376 				 * time.  In this case, recycle the
5377 				 * buffer to do real read on next
5378 				 * bread() after redo.
5379 				 *
5380 				 * Otherwise B_RELBUF is not strictly
5381 				 * necessary, enable to reduce buf
5382 				 * cache pressure.
5383 				 */
5384 				if (buf_pager_relbuf ||
5385 				    !vm_page_all_valid(m))
5386 					bp->b_flags |= B_RELBUF;
5387 
5388 				bp->b_flags &= ~B_NOCACHE;
5389 				brelse(bp);
5390 			} else {
5391 				bqrelse(bp);
5392 			}
5393 		}
5394 		KASSERT(1 /* racy, enable for debugging */ ||
5395 		    vm_page_all_valid(m) || i == count - 1,
5396 		    ("buf %d %p invalid", i, m));
5397 		if (i == count - 1 && lpart) {
5398 			if (!vm_page_none_valid(m) &&
5399 			    !vm_page_all_valid(m))
5400 				vm_page_zero_invalid(m, TRUE);
5401 		}
5402 next_page:;
5403 	}
5404 end_pages:
5405 
5406 	redo = false;
5407 	for (i = 0; i < count; i++) {
5408 		if (ma[i] == bogus_page)
5409 			continue;
5410 		if (vm_page_busy_tryupgrade(ma[i]) == 0) {
5411 			vm_page_sunbusy(ma[i]);
5412 			ma[i] = vm_page_grab_unlocked(object, ma[i]->pindex,
5413 			    VM_ALLOC_NORMAL);
5414 		}
5415 
5416 		/*
5417 		 * Since the pages were only sbusy while neither the
5418 		 * buffer nor the object lock was held by us, or
5419 		 * reallocated while vm_page_grab() slept for busy
5420 		 * relinguish, they could have been invalidated.
5421 		 * Recheck the valid bits and re-read as needed.
5422 		 *
5423 		 * Note that the last page is made fully valid in the
5424 		 * read loop, and partial validity for the page at
5425 		 * index count - 1 could mean that the page was
5426 		 * invalidated or removed, so we must restart for
5427 		 * safety as well.
5428 		 */
5429 		if (!vm_page_all_valid(ma[i]))
5430 			redo = true;
5431 	}
5432 	if (redo && error == 0)
5433 		goto again;
5434 	return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
5435 }
5436 
5437 #include "opt_ddb.h"
5438 #ifdef DDB
5439 #include <ddb/ddb.h>
5440 
5441 /* DDB command to show buffer data */
5442 DB_SHOW_COMMAND(buffer, db_show_buffer)
5443 {
5444 	/* get args */
5445 	struct buf *bp = (struct buf *)addr;
5446 #ifdef FULL_BUF_TRACKING
5447 	uint32_t i, j;
5448 #endif
5449 
5450 	if (!have_addr) {
5451 		db_printf("usage: show buffer <addr>\n");
5452 		return;
5453 	}
5454 
5455 	db_printf("buf at %p\n", bp);
5456 	db_printf("b_flags = 0x%b, b_xflags=0x%b\n",
5457 	    (u_int)bp->b_flags, PRINT_BUF_FLAGS,
5458 	    (u_int)bp->b_xflags, PRINT_BUF_XFLAGS);
5459 	db_printf("b_vflags=0x%b b_ioflags0x%b\n",
5460 	    (u_int)bp->b_vflags, PRINT_BUF_VFLAGS,
5461 	    (u_int)bp->b_ioflags, PRINT_BIO_FLAGS);
5462 	db_printf(
5463 	    "b_error = %d, b_bufsize = %ld, b_bcount = %ld, b_resid = %ld\n"
5464 	    "b_bufobj = (%p), b_data = %p\n, b_blkno = %jd, b_lblkno = %jd, "
5465 	    "b_vp = %p, b_dep = %p\n",
5466 	    bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
5467 	    bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
5468 	    (intmax_t)bp->b_lblkno, bp->b_vp, bp->b_dep.lh_first);
5469 	db_printf("b_kvabase = %p, b_kvasize = %d\n",
5470 	    bp->b_kvabase, bp->b_kvasize);
5471 	if (bp->b_npages) {
5472 		int i;
5473 		db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
5474 		for (i = 0; i < bp->b_npages; i++) {
5475 			vm_page_t m;
5476 			m = bp->b_pages[i];
5477 			if (m != NULL)
5478 				db_printf("(%p, 0x%lx, 0x%lx)", m->object,
5479 				    (u_long)m->pindex,
5480 				    (u_long)VM_PAGE_TO_PHYS(m));
5481 			else
5482 				db_printf("( ??? )");
5483 			if ((i + 1) < bp->b_npages)
5484 				db_printf(",");
5485 		}
5486 		db_printf("\n");
5487 	}
5488 	BUF_LOCKPRINTINFO(bp);
5489 #if defined(FULL_BUF_TRACKING)
5490 	db_printf("b_io_tracking: b_io_tcnt = %u\n", bp->b_io_tcnt);
5491 
5492 	i = bp->b_io_tcnt % BUF_TRACKING_SIZE;
5493 	for (j = 1; j <= BUF_TRACKING_SIZE; j++) {
5494 		if (bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)] == NULL)
5495 			continue;
5496 		db_printf(" %2u: %s\n", j,
5497 		    bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)]);
5498 	}
5499 #elif defined(BUF_TRACKING)
5500 	db_printf("b_io_tracking: %s\n", bp->b_io_tracking);
5501 #endif
5502 	db_printf(" ");
5503 }
5504 
5505 DB_SHOW_COMMAND_FLAGS(bufqueues, bufqueues, DB_CMD_MEMSAFE)
5506 {
5507 	struct bufdomain *bd;
5508 	struct buf *bp;
5509 	long total;
5510 	int i, j, cnt;
5511 
5512 	db_printf("bqempty: %d\n", bqempty.bq_len);
5513 
5514 	for (i = 0; i < buf_domains; i++) {
5515 		bd = &bdomain[i];
5516 		db_printf("Buf domain %d\n", i);
5517 		db_printf("\tfreebufs\t%d\n", bd->bd_freebuffers);
5518 		db_printf("\tlofreebufs\t%d\n", bd->bd_lofreebuffers);
5519 		db_printf("\thifreebufs\t%d\n", bd->bd_hifreebuffers);
5520 		db_printf("\n");
5521 		db_printf("\tbufspace\t%ld\n", bd->bd_bufspace);
5522 		db_printf("\tmaxbufspace\t%ld\n", bd->bd_maxbufspace);
5523 		db_printf("\thibufspace\t%ld\n", bd->bd_hibufspace);
5524 		db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace);
5525 		db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh);
5526 		db_printf("\n");
5527 		db_printf("\tnumdirtybuffers\t%d\n", bd->bd_numdirtybuffers);
5528 		db_printf("\tlodirtybuffers\t%d\n", bd->bd_lodirtybuffers);
5529 		db_printf("\thidirtybuffers\t%d\n", bd->bd_hidirtybuffers);
5530 		db_printf("\tdirtybufthresh\t%d\n", bd->bd_dirtybufthresh);
5531 		db_printf("\n");
5532 		total = 0;
5533 		TAILQ_FOREACH(bp, &bd->bd_cleanq->bq_queue, b_freelist)
5534 			total += bp->b_bufsize;
5535 		db_printf("\tcleanq count\t%d (%ld)\n",
5536 		    bd->bd_cleanq->bq_len, total);
5537 		total = 0;
5538 		TAILQ_FOREACH(bp, &bd->bd_dirtyq.bq_queue, b_freelist)
5539 			total += bp->b_bufsize;
5540 		db_printf("\tdirtyq count\t%d (%ld)\n",
5541 		    bd->bd_dirtyq.bq_len, total);
5542 		db_printf("\twakeup\t\t%d\n", bd->bd_wanted);
5543 		db_printf("\tlim\t\t%d\n", bd->bd_lim);
5544 		db_printf("\tCPU ");
5545 		for (j = 0; j <= mp_maxid; j++)
5546 			db_printf("%d, ", bd->bd_subq[j].bq_len);
5547 		db_printf("\n");
5548 		cnt = 0;
5549 		total = 0;
5550 		for (j = 0; j < nbuf; j++) {
5551 			bp = nbufp(j);
5552 			if (bp->b_domain == i && BUF_ISLOCKED(bp)) {
5553 				cnt++;
5554 				total += bp->b_bufsize;
5555 			}
5556 		}
5557 		db_printf("\tLocked buffers: %d space %ld\n", cnt, total);
5558 		cnt = 0;
5559 		total = 0;
5560 		for (j = 0; j < nbuf; j++) {
5561 			bp = nbufp(j);
5562 			if (bp->b_domain == i) {
5563 				cnt++;
5564 				total += bp->b_bufsize;
5565 			}
5566 		}
5567 		db_printf("\tTotal buffers: %d space %ld\n", cnt, total);
5568 	}
5569 }
5570 
5571 DB_SHOW_COMMAND_FLAGS(lockedbufs, lockedbufs, DB_CMD_MEMSAFE)
5572 {
5573 	struct buf *bp;
5574 	int i;
5575 
5576 	for (i = 0; i < nbuf; i++) {
5577 		bp = nbufp(i);
5578 		if (BUF_ISLOCKED(bp)) {
5579 			db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5580 			db_printf("\n");
5581 			if (db_pager_quit)
5582 				break;
5583 		}
5584 	}
5585 }
5586 
5587 DB_SHOW_COMMAND(vnodebufs, db_show_vnodebufs)
5588 {
5589 	struct vnode *vp;
5590 	struct buf *bp;
5591 
5592 	if (!have_addr) {
5593 		db_printf("usage: show vnodebufs <addr>\n");
5594 		return;
5595 	}
5596 	vp = (struct vnode *)addr;
5597 	db_printf("Clean buffers:\n");
5598 	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_clean.bv_hd, b_bobufs) {
5599 		db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5600 		db_printf("\n");
5601 	}
5602 	db_printf("Dirty buffers:\n");
5603 	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
5604 		db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5605 		db_printf("\n");
5606 	}
5607 }
5608 
5609 DB_COMMAND_FLAGS(countfreebufs, db_coundfreebufs, DB_CMD_MEMSAFE)
5610 {
5611 	struct buf *bp;
5612 	int i, used = 0, nfree = 0;
5613 
5614 	if (have_addr) {
5615 		db_printf("usage: countfreebufs\n");
5616 		return;
5617 	}
5618 
5619 	for (i = 0; i < nbuf; i++) {
5620 		bp = nbufp(i);
5621 		if (bp->b_qindex == QUEUE_EMPTY)
5622 			nfree++;
5623 		else
5624 			used++;
5625 	}
5626 
5627 	db_printf("Counted %d free, %d used (%d tot)\n", nfree, used,
5628 	    nfree + used);
5629 	db_printf("numfreebuffers is %d\n", numfreebuffers);
5630 }
5631 #endif /* DDB */
5632