xref: /freebsd/sys/kern/vfs_bio.c (revision e03a056de0baa06f7b38033e56a33a19526b542c)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
3  *
4  * Copyright (c) 2004 Poul-Henning Kamp
5  * Copyright (c) 1994,1997 John S. Dyson
6  * Copyright (c) 2013 The FreeBSD Foundation
7  * All rights reserved.
8  *
9  * Portions of this software were developed by Konstantin Belousov
10  * under sponsorship from the FreeBSD Foundation.
11  *
12  * Redistribution and use in source and binary forms, with or without
13  * modification, are permitted provided that the following conditions
14  * are met:
15  * 1. Redistributions of source code must retain the above copyright
16  *    notice, this list of conditions and the following disclaimer.
17  * 2. Redistributions in binary form must reproduce the above copyright
18  *    notice, this list of conditions and the following disclaimer in the
19  *    documentation and/or other materials provided with the distribution.
20  *
21  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
22  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
25  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31  * SUCH DAMAGE.
32  */
33 
34 /*
35  * this file contains a new buffer I/O scheme implementing a coherent
36  * VM object and buffer cache scheme.  Pains have been taken to make
37  * sure that the performance degradation associated with schemes such
38  * as this is not realized.
39  *
40  * Author:  John S. Dyson
41  * Significant help during the development and debugging phases
42  * had been provided by David Greenman, also of the FreeBSD core team.
43  *
44  * see man buf(9) for more info.
45  */
46 
47 #include <sys/param.h>
48 #include <sys/systm.h>
49 #include <sys/asan.h>
50 #include <sys/bio.h>
51 #include <sys/bitset.h>
52 #include <sys/boottrace.h>
53 #include <sys/buf.h>
54 #include <sys/conf.h>
55 #include <sys/counter.h>
56 #include <sys/devicestat.h>
57 #include <sys/eventhandler.h>
58 #include <sys/fail.h>
59 #include <sys/ktr.h>
60 #include <sys/limits.h>
61 #include <sys/lock.h>
62 #include <sys/malloc.h>
63 #include <sys/memdesc.h>
64 #include <sys/mount.h>
65 #include <sys/mutex.h>
66 #include <sys/kernel.h>
67 #include <sys/kthread.h>
68 #include <sys/pctrie.h>
69 #include <sys/proc.h>
70 #include <sys/racct.h>
71 #include <sys/refcount.h>
72 #include <sys/resourcevar.h>
73 #include <sys/rwlock.h>
74 #include <sys/sched.h>
75 #include <sys/smp.h>
76 #include <sys/sysctl.h>
77 #include <sys/syscallsubr.h>
78 #include <sys/vmem.h>
79 #include <sys/vmmeter.h>
80 #include <sys/vnode.h>
81 #include <sys/watchdog.h>
82 #include <geom/geom.h>
83 #include <vm/vm.h>
84 #include <vm/vm_param.h>
85 #include <vm/vm_kern.h>
86 #include <vm/vm_object.h>
87 #include <vm/vm_page.h>
88 #include <vm/vm_pageout.h>
89 #include <vm/vm_pager.h>
90 #include <vm/vm_extern.h>
91 #include <vm/vm_map.h>
92 #include <vm/swap_pager.h>
93 
94 static MALLOC_DEFINE(M_BIOBUF, "biobuf", "BIO buffer");
95 
96 struct	bio_ops bioops;		/* I/O operation notification */
97 
98 struct	buf_ops buf_ops_bio = {
99 	.bop_name	=	"buf_ops_bio",
100 	.bop_write	=	bufwrite,
101 	.bop_strategy	=	bufstrategy,
102 	.bop_sync	=	bufsync,
103 	.bop_bdflush	=	bufbdflush,
104 };
105 
106 struct bufqueue {
107 	struct mtx_padalign	bq_lock;
108 	TAILQ_HEAD(, buf)	bq_queue;
109 	uint8_t			bq_index;
110 	uint16_t		bq_subqueue;
111 	int			bq_len;
112 } __aligned(CACHE_LINE_SIZE);
113 
114 #define	BQ_LOCKPTR(bq)		(&(bq)->bq_lock)
115 #define	BQ_LOCK(bq)		mtx_lock(BQ_LOCKPTR((bq)))
116 #define	BQ_UNLOCK(bq)		mtx_unlock(BQ_LOCKPTR((bq)))
117 #define	BQ_ASSERT_LOCKED(bq)	mtx_assert(BQ_LOCKPTR((bq)), MA_OWNED)
118 
119 struct bufdomain {
120 	struct bufqueue	*bd_subq;
121 	struct bufqueue bd_dirtyq;
122 	struct bufqueue	*bd_cleanq;
123 	struct mtx_padalign bd_run_lock;
124 	/* Constants */
125 	long		bd_maxbufspace;
126 	long		bd_hibufspace;
127 	long 		bd_lobufspace;
128 	long 		bd_bufspacethresh;
129 	int		bd_hifreebuffers;
130 	int		bd_lofreebuffers;
131 	int		bd_hidirtybuffers;
132 	int		bd_lodirtybuffers;
133 	int		bd_dirtybufthresh;
134 	int		bd_lim;
135 	/* atomics */
136 	int		bd_wanted;
137 	bool		bd_shutdown;
138 	int __aligned(CACHE_LINE_SIZE)	bd_numdirtybuffers;
139 	int __aligned(CACHE_LINE_SIZE)	bd_running;
140 	long __aligned(CACHE_LINE_SIZE) bd_bufspace;
141 	int __aligned(CACHE_LINE_SIZE)	bd_freebuffers;
142 } __aligned(CACHE_LINE_SIZE);
143 
144 #define	BD_LOCKPTR(bd)		(&(bd)->bd_cleanq->bq_lock)
145 #define	BD_LOCK(bd)		mtx_lock(BD_LOCKPTR((bd)))
146 #define	BD_UNLOCK(bd)		mtx_unlock(BD_LOCKPTR((bd)))
147 #define	BD_ASSERT_LOCKED(bd)	mtx_assert(BD_LOCKPTR((bd)), MA_OWNED)
148 #define	BD_RUN_LOCKPTR(bd)	(&(bd)->bd_run_lock)
149 #define	BD_RUN_LOCK(bd)		mtx_lock(BD_RUN_LOCKPTR((bd)))
150 #define	BD_RUN_UNLOCK(bd)	mtx_unlock(BD_RUN_LOCKPTR((bd)))
151 #define	BD_DOMAIN(bd)		(bd - bdomain)
152 
153 static char *buf;		/* buffer header pool */
154 static struct buf *
nbufp(unsigned i)155 nbufp(unsigned i)
156 {
157 	return ((struct buf *)(buf + (sizeof(struct buf) +
158 	    sizeof(vm_page_t) * atop(maxbcachebuf)) * i));
159 }
160 
161 caddr_t __read_mostly unmapped_buf;
162 #ifdef INVARIANTS
163 caddr_t	poisoned_buf = (void *)-1;
164 #endif
165 
166 /* Used below and for softdep flushing threads in ufs/ffs/ffs_softdep.c */
167 struct proc *bufdaemonproc;
168 
169 static void vm_hold_free_pages(struct buf *bp, int newbsize);
170 static void vm_hold_load_pages(struct buf *bp, vm_offset_t from,
171 		vm_offset_t to);
172 static void vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m);
173 static void vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off,
174 		vm_page_t m);
175 static void vfs_clean_pages_dirty_buf(struct buf *bp);
176 static void vfs_setdirty_range(struct buf *bp);
177 static void vfs_vmio_invalidate(struct buf *bp);
178 static void vfs_vmio_truncate(struct buf *bp, int npages);
179 static void vfs_vmio_extend(struct buf *bp, int npages, int size);
180 static int vfs_bio_clcheck(struct vnode *vp, int size,
181 		daddr_t lblkno, daddr_t blkno);
182 static void breada(struct vnode *, daddr_t *, int *, int, struct ucred *, int,
183 		void (*)(struct buf *));
184 static int buf_flush(struct vnode *vp, struct bufdomain *, int);
185 static int flushbufqueues(struct vnode *, struct bufdomain *, int, int);
186 static void buf_daemon(void);
187 static __inline void bd_wakeup(void);
188 static int sysctl_runningspace(SYSCTL_HANDLER_ARGS);
189 static void bufkva_reclaim(vmem_t *, int);
190 static void bufkva_free(struct buf *);
191 static int buf_import(void *, void **, int, int, int);
192 static void buf_release(void *, void **, int);
193 static void maxbcachebuf_adjust(void);
194 static inline struct bufdomain *bufdomain(struct buf *);
195 static void bq_remove(struct bufqueue *bq, struct buf *bp);
196 static void bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock);
197 static int buf_recycle(struct bufdomain *, bool kva);
198 static void bq_init(struct bufqueue *bq, int qindex, int cpu,
199 	    const char *lockname);
200 static void bd_init(struct bufdomain *bd);
201 static int bd_flushall(struct bufdomain *bd);
202 static int sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS);
203 static int sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS);
204 
205 static int sysctl_bufspace(SYSCTL_HANDLER_ARGS);
206 int vmiodirenable = TRUE;
207 SYSCTL_INT(_vfs, OID_AUTO, vmiodirenable, CTLFLAG_RW, &vmiodirenable, 0,
208     "Use the VM system for directory writes");
209 long runningbufspace;
210 SYSCTL_LONG(_vfs, OID_AUTO, runningbufspace, CTLFLAG_RD, &runningbufspace, 0,
211     "Amount of presently outstanding async buffer io");
212 SYSCTL_PROC(_vfs, OID_AUTO, bufspace, CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RD,
213     NULL, 0, sysctl_bufspace, "L", "Physical memory used for buffers");
214 static counter_u64_t bufkvaspace;
215 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufkvaspace, CTLFLAG_RD, &bufkvaspace,
216     "Kernel virtual memory used for buffers");
217 static long maxbufspace;
218 SYSCTL_PROC(_vfs, OID_AUTO, maxbufspace,
219     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &maxbufspace,
220     __offsetof(struct bufdomain, bd_maxbufspace), sysctl_bufdomain_long, "L",
221     "Maximum allowed value of bufspace (including metadata)");
222 static long bufmallocspace;
223 SYSCTL_LONG(_vfs, OID_AUTO, bufmallocspace, CTLFLAG_RD, &bufmallocspace, 0,
224     "Amount of malloced memory for buffers");
225 static long maxbufmallocspace;
226 SYSCTL_LONG(_vfs, OID_AUTO, maxmallocbufspace, CTLFLAG_RW, &maxbufmallocspace,
227     0, "Maximum amount of malloced memory for buffers");
228 static long lobufspace;
229 SYSCTL_PROC(_vfs, OID_AUTO, lobufspace,
230     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &lobufspace,
231     __offsetof(struct bufdomain, bd_lobufspace), sysctl_bufdomain_long, "L",
232     "Minimum amount of buffers we want to have");
233 long hibufspace;
234 SYSCTL_PROC(_vfs, OID_AUTO, hibufspace,
235     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &hibufspace,
236     __offsetof(struct bufdomain, bd_hibufspace), sysctl_bufdomain_long, "L",
237     "Maximum allowed value of bufspace (excluding metadata)");
238 long bufspacethresh;
239 SYSCTL_PROC(_vfs, OID_AUTO, bufspacethresh,
240     CTLTYPE_LONG|CTLFLAG_MPSAFE|CTLFLAG_RW, &bufspacethresh,
241     __offsetof(struct bufdomain, bd_bufspacethresh), sysctl_bufdomain_long, "L",
242     "Bufspace consumed before waking the daemon to free some");
243 static counter_u64_t buffreekvacnt;
244 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, buffreekvacnt, CTLFLAG_RW, &buffreekvacnt,
245     "Number of times we have freed the KVA space from some buffer");
246 static counter_u64_t bufdefragcnt;
247 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, bufdefragcnt, CTLFLAG_RW, &bufdefragcnt,
248     "Number of times we have had to repeat buffer allocation to defragment");
249 static long lorunningspace;
250 SYSCTL_PROC(_vfs, OID_AUTO, lorunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
251     CTLFLAG_RW, &lorunningspace, 0, sysctl_runningspace, "L",
252     "Minimum preferred space used for in-progress I/O");
253 static long hirunningspace;
254 SYSCTL_PROC(_vfs, OID_AUTO, hirunningspace, CTLTYPE_LONG | CTLFLAG_MPSAFE |
255     CTLFLAG_RW, &hirunningspace, 0, sysctl_runningspace, "L",
256     "Maximum amount of space to use for in-progress I/O");
257 int dirtybufferflushes;
258 SYSCTL_INT(_vfs, OID_AUTO, dirtybufferflushes, CTLFLAG_RW, &dirtybufferflushes,
259     0, "Number of bdwrite to bawrite conversions to limit dirty buffers");
260 int bdwriteskip;
261 SYSCTL_INT(_vfs, OID_AUTO, bdwriteskip, CTLFLAG_RW, &bdwriteskip,
262     0, "Number of buffers supplied to bdwrite with snapshot deadlock risk");
263 int altbufferflushes;
264 SYSCTL_INT(_vfs, OID_AUTO, altbufferflushes, CTLFLAG_RW | CTLFLAG_STATS,
265     &altbufferflushes, 0, "Number of fsync flushes to limit dirty buffers");
266 static int recursiveflushes;
267 SYSCTL_INT(_vfs, OID_AUTO, recursiveflushes, CTLFLAG_RW | CTLFLAG_STATS,
268     &recursiveflushes, 0, "Number of flushes skipped due to being recursive");
269 static int sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS);
270 SYSCTL_PROC(_vfs, OID_AUTO, numdirtybuffers,
271     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RD, NULL, 0, sysctl_numdirtybuffers, "I",
272     "Number of buffers that are dirty (has unwritten changes) at the moment");
273 static int lodirtybuffers;
274 SYSCTL_PROC(_vfs, OID_AUTO, lodirtybuffers,
275     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lodirtybuffers,
276     __offsetof(struct bufdomain, bd_lodirtybuffers), sysctl_bufdomain_int, "I",
277     "How many buffers we want to have free before bufdaemon can sleep");
278 static int hidirtybuffers;
279 SYSCTL_PROC(_vfs, OID_AUTO, hidirtybuffers,
280     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hidirtybuffers,
281     __offsetof(struct bufdomain, bd_hidirtybuffers), sysctl_bufdomain_int, "I",
282     "When the number of dirty buffers is considered severe");
283 int dirtybufthresh;
284 SYSCTL_PROC(_vfs, OID_AUTO, dirtybufthresh,
285     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &dirtybufthresh,
286     __offsetof(struct bufdomain, bd_dirtybufthresh), sysctl_bufdomain_int, "I",
287     "Number of bdwrite to bawrite conversions to clear dirty buffers");
288 static int numfreebuffers;
289 SYSCTL_INT(_vfs, OID_AUTO, numfreebuffers, CTLFLAG_RD, &numfreebuffers, 0,
290     "Number of free buffers");
291 static int lofreebuffers;
292 SYSCTL_PROC(_vfs, OID_AUTO, lofreebuffers,
293     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &lofreebuffers,
294     __offsetof(struct bufdomain, bd_lofreebuffers), sysctl_bufdomain_int, "I",
295    "Target number of free buffers");
296 static int hifreebuffers;
297 SYSCTL_PROC(_vfs, OID_AUTO, hifreebuffers,
298     CTLTYPE_INT|CTLFLAG_MPSAFE|CTLFLAG_RW, &hifreebuffers,
299     __offsetof(struct bufdomain, bd_hifreebuffers), sysctl_bufdomain_int, "I",
300    "Threshold for clean buffer recycling");
301 static counter_u64_t getnewbufcalls;
302 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufcalls, CTLFLAG_RD,
303    &getnewbufcalls, "Number of calls to getnewbuf");
304 static counter_u64_t getnewbufrestarts;
305 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, getnewbufrestarts, CTLFLAG_RD,
306     &getnewbufrestarts,
307     "Number of times getnewbuf has had to restart a buffer acquisition");
308 static counter_u64_t mappingrestarts;
309 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, mappingrestarts, CTLFLAG_RD,
310     &mappingrestarts,
311     "Number of times getblk has had to restart a buffer mapping for "
312     "unmapped buffer");
313 static counter_u64_t numbufallocfails;
314 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, numbufallocfails, CTLFLAG_RW,
315     &numbufallocfails, "Number of times buffer allocations failed");
316 static int flushbufqtarget = 100;
317 SYSCTL_INT(_vfs, OID_AUTO, flushbufqtarget, CTLFLAG_RW, &flushbufqtarget, 0,
318     "Amount of work to do in flushbufqueues when helping bufdaemon");
319 static counter_u64_t notbufdflushes;
320 SYSCTL_COUNTER_U64(_vfs, OID_AUTO, notbufdflushes, CTLFLAG_RD, &notbufdflushes,
321     "Number of dirty buffer flushes done by the bufdaemon helpers");
322 static long barrierwrites;
323 SYSCTL_LONG(_vfs, OID_AUTO, barrierwrites, CTLFLAG_RW | CTLFLAG_STATS,
324     &barrierwrites, 0, "Number of barrier writes");
325 SYSCTL_INT(_vfs, OID_AUTO, unmapped_buf_allowed,
326     CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
327     &unmapped_buf_allowed, 0,
328     "Permit the use of the unmapped i/o");
329 int maxbcachebuf = MAXBCACHEBUF;
330 SYSCTL_INT(_vfs, OID_AUTO, maxbcachebuf, CTLFLAG_RDTUN, &maxbcachebuf, 0,
331     "Maximum size of a buffer cache block");
332 
333 /*
334  * This lock synchronizes access to bd_request.
335  */
336 static struct mtx_padalign __exclusive_cache_line bdlock;
337 
338 /*
339  * This lock protects the runningbufreq and synchronizes runningbufwakeup and
340  * waitrunningbufspace().
341  */
342 static struct mtx_padalign __exclusive_cache_line rbreqlock;
343 
344 /*
345  * Lock that protects bdirtywait.
346  */
347 static struct mtx_padalign __exclusive_cache_line bdirtylock;
348 
349 /*
350  * bufdaemon shutdown request and sleep channel.
351  */
352 static bool bd_shutdown;
353 
354 /*
355  * Wakeup point for bufdaemon, as well as indicator of whether it is already
356  * active.  Set to 1 when the bufdaemon is already "on" the queue, 0 when it
357  * is idling.
358  */
359 static int bd_request;
360 
361 /*
362  * Request for the buf daemon to write more buffers than is indicated by
363  * lodirtybuf.  This may be necessary to push out excess dependencies or
364  * defragment the address space where a simple count of the number of dirty
365  * buffers is insufficient to characterize the demand for flushing them.
366  */
367 static int bd_speedupreq;
368 
369 /*
370  * Synchronization (sleep/wakeup) variable for active buffer space requests.
371  * Set when wait starts, cleared prior to wakeup().
372  * Used in runningbufwakeup() and waitrunningbufspace().
373  */
374 static int runningbufreq;
375 
376 /*
377  * Synchronization for bwillwrite() waiters.
378  */
379 static int bdirtywait;
380 
381 /*
382  * Definitions for the buffer free lists.
383  */
384 #define QUEUE_NONE	0	/* on no queue */
385 #define QUEUE_EMPTY	1	/* empty buffer headers */
386 #define QUEUE_DIRTY	2	/* B_DELWRI buffers */
387 #define QUEUE_CLEAN	3	/* non-B_DELWRI buffers */
388 #define QUEUE_SENTINEL	4	/* not an queue index, but mark for sentinel */
389 
390 /* Maximum number of buffer domains. */
391 #define	BUF_DOMAINS	8
392 
393 struct bufdomainset bdlodirty;		/* Domains > lodirty */
394 struct bufdomainset bdhidirty;		/* Domains > hidirty */
395 
396 /* Configured number of clean queues. */
397 static int __read_mostly buf_domains;
398 
399 BITSET_DEFINE(bufdomainset, BUF_DOMAINS);
400 struct bufdomain __exclusive_cache_line bdomain[BUF_DOMAINS];
401 struct bufqueue __exclusive_cache_line bqempty;
402 
403 /*
404  * per-cpu empty buffer cache.
405  */
406 uma_zone_t buf_zone;
407 
408 static int
sysctl_runningspace(SYSCTL_HANDLER_ARGS)409 sysctl_runningspace(SYSCTL_HANDLER_ARGS)
410 {
411 	long value;
412 	int error;
413 
414 	value = *(long *)arg1;
415 	error = sysctl_handle_long(oidp, &value, 0, req);
416 	if (error != 0 || req->newptr == NULL)
417 		return (error);
418 	mtx_lock(&rbreqlock);
419 	if (arg1 == &hirunningspace) {
420 		if (value < lorunningspace)
421 			error = EINVAL;
422 		else
423 			hirunningspace = value;
424 	} else {
425 		KASSERT(arg1 == &lorunningspace,
426 		    ("%s: unknown arg1", __func__));
427 		if (value > hirunningspace)
428 			error = EINVAL;
429 		else
430 			lorunningspace = value;
431 	}
432 	mtx_unlock(&rbreqlock);
433 	return (error);
434 }
435 
436 static int
sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS)437 sysctl_bufdomain_int(SYSCTL_HANDLER_ARGS)
438 {
439 	int error;
440 	int value;
441 	int i;
442 
443 	value = *(int *)arg1;
444 	error = sysctl_handle_int(oidp, &value, 0, req);
445 	if (error != 0 || req->newptr == NULL)
446 		return (error);
447 	*(int *)arg1 = value;
448 	for (i = 0; i < buf_domains; i++)
449 		*(int *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
450 		    value / buf_domains;
451 
452 	return (error);
453 }
454 
455 static int
sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS)456 sysctl_bufdomain_long(SYSCTL_HANDLER_ARGS)
457 {
458 	long value;
459 	int error;
460 	int i;
461 
462 	value = *(long *)arg1;
463 	error = sysctl_handle_long(oidp, &value, 0, req);
464 	if (error != 0 || req->newptr == NULL)
465 		return (error);
466 	*(long *)arg1 = value;
467 	for (i = 0; i < buf_domains; i++)
468 		*(long *)(uintptr_t)(((uintptr_t)&bdomain[i]) + arg2) =
469 		    value / buf_domains;
470 
471 	return (error);
472 }
473 
474 #if defined(COMPAT_FREEBSD4) || defined(COMPAT_FREEBSD5) || \
475     defined(COMPAT_FREEBSD6) || defined(COMPAT_FREEBSD7)
476 static int
sysctl_bufspace(SYSCTL_HANDLER_ARGS)477 sysctl_bufspace(SYSCTL_HANDLER_ARGS)
478 {
479 	long lvalue;
480 	int ivalue;
481 	int i;
482 
483 	lvalue = 0;
484 	for (i = 0; i < buf_domains; i++)
485 		lvalue += bdomain[i].bd_bufspace;
486 	if (sizeof(int) == sizeof(long) || req->oldlen >= sizeof(long))
487 		return (sysctl_handle_long(oidp, &lvalue, 0, req));
488 	if (lvalue > INT_MAX)
489 		/* On overflow, still write out a long to trigger ENOMEM. */
490 		return (sysctl_handle_long(oidp, &lvalue, 0, req));
491 	ivalue = lvalue;
492 	return (sysctl_handle_int(oidp, &ivalue, 0, req));
493 }
494 #else
495 static int
sysctl_bufspace(SYSCTL_HANDLER_ARGS)496 sysctl_bufspace(SYSCTL_HANDLER_ARGS)
497 {
498 	long lvalue;
499 	int i;
500 
501 	lvalue = 0;
502 	for (i = 0; i < buf_domains; i++)
503 		lvalue += bdomain[i].bd_bufspace;
504 	return (sysctl_handle_long(oidp, &lvalue, 0, req));
505 }
506 #endif
507 
508 static int
sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS)509 sysctl_numdirtybuffers(SYSCTL_HANDLER_ARGS)
510 {
511 	int value;
512 	int i;
513 
514 	value = 0;
515 	for (i = 0; i < buf_domains; i++)
516 		value += bdomain[i].bd_numdirtybuffers;
517 	return (sysctl_handle_int(oidp, &value, 0, req));
518 }
519 
520 /*
521  *	bdirtywakeup:
522  *
523  *	Wakeup any bwillwrite() waiters.
524  */
525 static void
bdirtywakeup(void)526 bdirtywakeup(void)
527 {
528 	mtx_lock(&bdirtylock);
529 	if (bdirtywait) {
530 		bdirtywait = 0;
531 		wakeup(&bdirtywait);
532 	}
533 	mtx_unlock(&bdirtylock);
534 }
535 
536 /*
537  *	bd_clear:
538  *
539  *	Clear a domain from the appropriate bitsets when dirtybuffers
540  *	is decremented.
541  */
542 static void
bd_clear(struct bufdomain * bd)543 bd_clear(struct bufdomain *bd)
544 {
545 
546 	mtx_lock(&bdirtylock);
547 	if (bd->bd_numdirtybuffers <= bd->bd_lodirtybuffers)
548 		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
549 	if (bd->bd_numdirtybuffers <= bd->bd_hidirtybuffers)
550 		BIT_CLR(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
551 	mtx_unlock(&bdirtylock);
552 }
553 
554 /*
555  *	bd_set:
556  *
557  *	Set a domain in the appropriate bitsets when dirtybuffers
558  *	is incremented.
559  */
560 static void
bd_set(struct bufdomain * bd)561 bd_set(struct bufdomain *bd)
562 {
563 
564 	mtx_lock(&bdirtylock);
565 	if (bd->bd_numdirtybuffers > bd->bd_lodirtybuffers)
566 		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdlodirty);
567 	if (bd->bd_numdirtybuffers > bd->bd_hidirtybuffers)
568 		BIT_SET(BUF_DOMAINS, BD_DOMAIN(bd), &bdhidirty);
569 	mtx_unlock(&bdirtylock);
570 }
571 
572 /*
573  *	bdirtysub:
574  *
575  *	Decrement the numdirtybuffers count by one and wakeup any
576  *	threads blocked in bwillwrite().
577  */
578 static void
bdirtysub(struct buf * bp)579 bdirtysub(struct buf *bp)
580 {
581 	struct bufdomain *bd;
582 	int num;
583 
584 	bd = bufdomain(bp);
585 	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, -1);
586 	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
587 		bdirtywakeup();
588 	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
589 		bd_clear(bd);
590 }
591 
592 /*
593  *	bdirtyadd:
594  *
595  *	Increment the numdirtybuffers count by one and wakeup the buf
596  *	daemon if needed.
597  */
598 static void
bdirtyadd(struct buf * bp)599 bdirtyadd(struct buf *bp)
600 {
601 	struct bufdomain *bd;
602 	int num;
603 
604 	/*
605 	 * Only do the wakeup once as we cross the boundary.  The
606 	 * buf daemon will keep running until the condition clears.
607 	 */
608 	bd = bufdomain(bp);
609 	num = atomic_fetchadd_int(&bd->bd_numdirtybuffers, 1);
610 	if (num == (bd->bd_lodirtybuffers + bd->bd_hidirtybuffers) / 2)
611 		bd_wakeup();
612 	if (num == bd->bd_lodirtybuffers || num == bd->bd_hidirtybuffers)
613 		bd_set(bd);
614 }
615 
616 /*
617  *	bufspace_daemon_wakeup:
618  *
619  *	Wakeup the daemons responsible for freeing clean bufs.
620  */
621 static void
bufspace_daemon_wakeup(struct bufdomain * bd)622 bufspace_daemon_wakeup(struct bufdomain *bd)
623 {
624 
625 	/*
626 	 * avoid the lock if the daemon is running.
627 	 */
628 	if (atomic_fetchadd_int(&bd->bd_running, 1) == 0) {
629 		BD_RUN_LOCK(bd);
630 		atomic_store_int(&bd->bd_running, 1);
631 		wakeup(&bd->bd_running);
632 		BD_RUN_UNLOCK(bd);
633 	}
634 }
635 
636 /*
637  *	bufspace_adjust:
638  *
639  *	Adjust the reported bufspace for a KVA managed buffer, possibly
640  * 	waking any waiters.
641  */
642 static void
bufspace_adjust(struct buf * bp,int bufsize)643 bufspace_adjust(struct buf *bp, int bufsize)
644 {
645 	struct bufdomain *bd;
646 	long space;
647 	int diff;
648 
649 	KASSERT((bp->b_flags & B_MALLOC) == 0,
650 	    ("bufspace_adjust: malloc buf %p", bp));
651 	bd = bufdomain(bp);
652 	diff = bufsize - bp->b_bufsize;
653 	if (diff < 0) {
654 		atomic_subtract_long(&bd->bd_bufspace, -diff);
655 	} else if (diff > 0) {
656 		space = atomic_fetchadd_long(&bd->bd_bufspace, diff);
657 		/* Wake up the daemon on the transition. */
658 		if (space < bd->bd_bufspacethresh &&
659 		    space + diff >= bd->bd_bufspacethresh)
660 			bufspace_daemon_wakeup(bd);
661 	}
662 	bp->b_bufsize = bufsize;
663 }
664 
665 /*
666  *	bufspace_reserve:
667  *
668  *	Reserve bufspace before calling allocbuf().  metadata has a
669  *	different space limit than data.
670  */
671 static int
bufspace_reserve(struct bufdomain * bd,int size,bool metadata)672 bufspace_reserve(struct bufdomain *bd, int size, bool metadata)
673 {
674 	long limit, new;
675 	long space;
676 
677 	if (metadata)
678 		limit = bd->bd_maxbufspace;
679 	else
680 		limit = bd->bd_hibufspace;
681 	space = atomic_fetchadd_long(&bd->bd_bufspace, size);
682 	new = space + size;
683 	if (new > limit) {
684 		atomic_subtract_long(&bd->bd_bufspace, size);
685 		return (ENOSPC);
686 	}
687 
688 	/* Wake up the daemon on the transition. */
689 	if (space < bd->bd_bufspacethresh && new >= bd->bd_bufspacethresh)
690 		bufspace_daemon_wakeup(bd);
691 
692 	return (0);
693 }
694 
695 /*
696  *	bufspace_release:
697  *
698  *	Release reserved bufspace after bufspace_adjust() has consumed it.
699  */
700 static void
bufspace_release(struct bufdomain * bd,int size)701 bufspace_release(struct bufdomain *bd, int size)
702 {
703 
704 	atomic_subtract_long(&bd->bd_bufspace, size);
705 }
706 
707 /*
708  *	bufspace_wait:
709  *
710  *	Wait for bufspace, acting as the buf daemon if a locked vnode is
711  *	supplied.  bd_wanted must be set prior to polling for space.  The
712  *	operation must be re-tried on return.
713  */
714 static void
bufspace_wait(struct bufdomain * bd,struct vnode * vp,int gbflags,int slpflag,int slptimeo)715 bufspace_wait(struct bufdomain *bd, struct vnode *vp, int gbflags,
716     int slpflag, int slptimeo)
717 {
718 	struct thread *td;
719 	int error, fl, norunbuf;
720 
721 	if ((gbflags & GB_NOWAIT_BD) != 0)
722 		return;
723 
724 	td = curthread;
725 	BD_LOCK(bd);
726 	while (bd->bd_wanted) {
727 		if (vp != NULL && vp->v_type != VCHR &&
728 		    (td->td_pflags & TDP_BUFNEED) == 0) {
729 			BD_UNLOCK(bd);
730 			/*
731 			 * getblk() is called with a vnode locked, and
732 			 * some majority of the dirty buffers may as
733 			 * well belong to the vnode.  Flushing the
734 			 * buffers there would make a progress that
735 			 * cannot be achieved by the buf_daemon, that
736 			 * cannot lock the vnode.
737 			 */
738 			norunbuf = ~(TDP_BUFNEED | TDP_NORUNNINGBUF) |
739 			    (td->td_pflags & TDP_NORUNNINGBUF);
740 
741 			/*
742 			 * Play bufdaemon.  The getnewbuf() function
743 			 * may be called while the thread owns lock
744 			 * for another dirty buffer for the same
745 			 * vnode, which makes it impossible to use
746 			 * VOP_FSYNC() there, due to the buffer lock
747 			 * recursion.
748 			 */
749 			td->td_pflags |= TDP_BUFNEED | TDP_NORUNNINGBUF;
750 			fl = buf_flush(vp, bd, flushbufqtarget);
751 			td->td_pflags &= norunbuf;
752 			BD_LOCK(bd);
753 			if (fl != 0)
754 				continue;
755 			if (bd->bd_wanted == 0)
756 				break;
757 		}
758 		error = msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
759 		    (PRIBIO + 4) | slpflag, "newbuf", slptimeo);
760 		if (error != 0)
761 			break;
762 	}
763 	BD_UNLOCK(bd);
764 }
765 
766 static void
bufspace_daemon_shutdown(void * arg,int howto __unused)767 bufspace_daemon_shutdown(void *arg, int howto __unused)
768 {
769 	struct bufdomain *bd = arg;
770 	int error;
771 
772 	if (KERNEL_PANICKED())
773 		return;
774 
775 	BD_RUN_LOCK(bd);
776 	bd->bd_shutdown = true;
777 	wakeup(&bd->bd_running);
778 	error = msleep(&bd->bd_shutdown, BD_RUN_LOCKPTR(bd), 0,
779 	    "bufspace_shutdown", 60 * hz);
780 	BD_RUN_UNLOCK(bd);
781 	if (error != 0)
782 		printf("bufspacedaemon wait error: %d\n", error);
783 }
784 
785 /*
786  *	bufspace_daemon:
787  *
788  *	buffer space management daemon.  Tries to maintain some marginal
789  *	amount of free buffer space so that requesting processes neither
790  *	block nor work to reclaim buffers.
791  */
792 static void
bufspace_daemon(void * arg)793 bufspace_daemon(void *arg)
794 {
795 	struct bufdomain *bd = arg;
796 
797 	EVENTHANDLER_REGISTER(shutdown_pre_sync, bufspace_daemon_shutdown, bd,
798 	    SHUTDOWN_PRI_LAST + 100);
799 
800 	BD_RUN_LOCK(bd);
801 	while (!bd->bd_shutdown) {
802 		BD_RUN_UNLOCK(bd);
803 
804 		/*
805 		 * Free buffers from the clean queue until we meet our
806 		 * targets.
807 		 *
808 		 * Theory of operation:  The buffer cache is most efficient
809 		 * when some free buffer headers and space are always
810 		 * available to getnewbuf().  This daemon attempts to prevent
811 		 * the excessive blocking and synchronization associated
812 		 * with shortfall.  It goes through three phases according
813 		 * demand:
814 		 *
815 		 * 1)	The daemon wakes up voluntarily once per-second
816 		 *	during idle periods when the counters are below
817 		 *	the wakeup thresholds (bufspacethresh, lofreebuffers).
818 		 *
819 		 * 2)	The daemon wakes up as we cross the thresholds
820 		 *	ahead of any potential blocking.  This may bounce
821 		 *	slightly according to the rate of consumption and
822 		 *	release.
823 		 *
824 		 * 3)	The daemon and consumers are starved for working
825 		 *	clean buffers.  This is the 'bufspace' sleep below
826 		 *	which will inefficiently trade bufs with bqrelse
827 		 *	until we return to condition 2.
828 		 */
829 		while (bd->bd_bufspace > bd->bd_lobufspace ||
830 		    bd->bd_freebuffers < bd->bd_hifreebuffers) {
831 			if (buf_recycle(bd, false) != 0) {
832 				if (bd_flushall(bd))
833 					continue;
834 				/*
835 				 * Speedup dirty if we've run out of clean
836 				 * buffers.  This is possible in particular
837 				 * because softdep may held many bufs locked
838 				 * pending writes to other bufs which are
839 				 * marked for delayed write, exhausting
840 				 * clean space until they are written.
841 				 */
842 				bd_speedup();
843 				BD_LOCK(bd);
844 				if (bd->bd_wanted) {
845 					msleep(&bd->bd_wanted, BD_LOCKPTR(bd),
846 					    PRIBIO|PDROP, "bufspace", hz/10);
847 				} else
848 					BD_UNLOCK(bd);
849 			}
850 			maybe_yield();
851 		}
852 
853 		/*
854 		 * Re-check our limits and sleep.  bd_running must be
855 		 * cleared prior to checking the limits to avoid missed
856 		 * wakeups.  The waker will adjust one of bufspace or
857 		 * freebuffers prior to checking bd_running.
858 		 */
859 		BD_RUN_LOCK(bd);
860 		if (bd->bd_shutdown)
861 			break;
862 		atomic_store_int(&bd->bd_running, 0);
863 		if (bd->bd_bufspace < bd->bd_bufspacethresh &&
864 		    bd->bd_freebuffers > bd->bd_lofreebuffers) {
865 			msleep(&bd->bd_running, BD_RUN_LOCKPTR(bd),
866 			    PRIBIO, "-", hz);
867 		} else {
868 			/* Avoid spurious wakeups while running. */
869 			atomic_store_int(&bd->bd_running, 1);
870 		}
871 	}
872 	wakeup(&bd->bd_shutdown);
873 	BD_RUN_UNLOCK(bd);
874 	kthread_exit();
875 }
876 
877 /*
878  *	bufmallocadjust:
879  *
880  *	Adjust the reported bufspace for a malloc managed buffer, possibly
881  *	waking any waiters.
882  */
883 static void
bufmallocadjust(struct buf * bp,int bufsize)884 bufmallocadjust(struct buf *bp, int bufsize)
885 {
886 	int diff;
887 
888 	KASSERT((bp->b_flags & B_MALLOC) != 0,
889 	    ("bufmallocadjust: non-malloc buf %p", bp));
890 	diff = bufsize - bp->b_bufsize;
891 	if (diff < 0)
892 		atomic_subtract_long(&bufmallocspace, -diff);
893 	else
894 		atomic_add_long(&bufmallocspace, diff);
895 	bp->b_bufsize = bufsize;
896 }
897 
898 /*
899  *	runningwakeup:
900  *
901  *	Wake up processes that are waiting on asynchronous writes to fall
902  *	below lorunningspace.
903  */
904 static void
runningwakeup(void)905 runningwakeup(void)
906 {
907 
908 	mtx_lock(&rbreqlock);
909 	if (runningbufreq) {
910 		runningbufreq = 0;
911 		wakeup(&runningbufreq);
912 	}
913 	mtx_unlock(&rbreqlock);
914 }
915 
916 /*
917  *	runningbufwakeup:
918  *
919  *	Decrement the outstanding write count according.
920  */
921 void
runningbufwakeup(struct buf * bp)922 runningbufwakeup(struct buf *bp)
923 {
924 	long space, bspace;
925 
926 	bspace = bp->b_runningbufspace;
927 	if (bspace == 0)
928 		return;
929 	space = atomic_fetchadd_long(&runningbufspace, -bspace);
930 	KASSERT(space >= bspace, ("runningbufspace underflow %ld %ld",
931 	    space, bspace));
932 	bp->b_runningbufspace = 0;
933 	/*
934 	 * Only acquire the lock and wakeup on the transition from exceeding
935 	 * the threshold to falling below it.
936 	 */
937 	if (space < lorunningspace)
938 		return;
939 	if (space - bspace > lorunningspace)
940 		return;
941 	runningwakeup();
942 }
943 
944 /*
945  *	waitrunningbufspace()
946  *
947  *	runningbufspace is a measure of the amount of I/O currently
948  *	running.  This routine is used in async-write situations to
949  *	prevent creating huge backups of pending writes to a device.
950  *	Only asynchronous writes are governed by this function.
951  *
952  *	This does NOT turn an async write into a sync write.  It waits
953  *	for earlier writes to complete and generally returns before the
954  *	caller's write has reached the device.
955  */
956 void
waitrunningbufspace(void)957 waitrunningbufspace(void)
958 {
959 
960 	mtx_lock(&rbreqlock);
961 	while (runningbufspace > hirunningspace) {
962 		runningbufreq = 1;
963 		msleep(&runningbufreq, &rbreqlock, PVM, "wdrain", 0);
964 	}
965 	mtx_unlock(&rbreqlock);
966 }
967 
968 /*
969  *	vfs_buf_test_cache:
970  *
971  *	Called when a buffer is extended.  This function clears the B_CACHE
972  *	bit if the newly extended portion of the buffer does not contain
973  *	valid data.
974  */
975 static __inline void
vfs_buf_test_cache(struct buf * bp,vm_ooffset_t foff,vm_offset_t off,vm_offset_t size,vm_page_t m)976 vfs_buf_test_cache(struct buf *bp, vm_ooffset_t foff, vm_offset_t off,
977     vm_offset_t size, vm_page_t m)
978 {
979 
980 	/*
981 	 * This function and its results are protected by higher level
982 	 * synchronization requiring vnode and buf locks to page in and
983 	 * validate pages.
984 	 */
985 	if (bp->b_flags & B_CACHE) {
986 		int base = (foff + off) & PAGE_MASK;
987 		if (vm_page_is_valid(m, base, size) == 0)
988 			bp->b_flags &= ~B_CACHE;
989 	}
990 }
991 
992 /* Wake up the buffer daemon if necessary */
993 static void
bd_wakeup(void)994 bd_wakeup(void)
995 {
996 
997 	mtx_lock(&bdlock);
998 	if (bd_request == 0) {
999 		bd_request = 1;
1000 		wakeup(&bd_request);
1001 	}
1002 	mtx_unlock(&bdlock);
1003 }
1004 
1005 /*
1006  * Adjust the maxbcachbuf tunable.
1007  */
1008 static void
maxbcachebuf_adjust(void)1009 maxbcachebuf_adjust(void)
1010 {
1011 	int i;
1012 
1013 	/*
1014 	 * maxbcachebuf must be a power of 2 >= MAXBSIZE.
1015 	 */
1016 	i = 2;
1017 	while (i * 2 <= maxbcachebuf)
1018 		i *= 2;
1019 	maxbcachebuf = i;
1020 	if (maxbcachebuf < MAXBSIZE)
1021 		maxbcachebuf = MAXBSIZE;
1022 	if (maxbcachebuf > maxphys)
1023 		maxbcachebuf = maxphys;
1024 	if (bootverbose != 0 && maxbcachebuf != MAXBCACHEBUF)
1025 		printf("maxbcachebuf=%d\n", maxbcachebuf);
1026 }
1027 
1028 /*
1029  * bd_speedup - speedup the buffer cache flushing code
1030  */
1031 void
bd_speedup(void)1032 bd_speedup(void)
1033 {
1034 	int needwake;
1035 
1036 	mtx_lock(&bdlock);
1037 	needwake = 0;
1038 	if (bd_speedupreq == 0 || bd_request == 0)
1039 		needwake = 1;
1040 	bd_speedupreq = 1;
1041 	bd_request = 1;
1042 	if (needwake)
1043 		wakeup(&bd_request);
1044 	mtx_unlock(&bdlock);
1045 }
1046 
1047 #ifdef __i386__
1048 #define	TRANSIENT_DENOM	5
1049 #else
1050 #define	TRANSIENT_DENOM 10
1051 #endif
1052 
1053 /*
1054  * Calculating buffer cache scaling values and reserve space for buffer
1055  * headers.  This is called during low level kernel initialization and
1056  * may be called more then once.  We CANNOT write to the memory area
1057  * being reserved at this time.
1058  */
1059 caddr_t
kern_vfs_bio_buffer_alloc(caddr_t v,long physmem_est)1060 kern_vfs_bio_buffer_alloc(caddr_t v, long physmem_est)
1061 {
1062 	int tuned_nbuf;
1063 	long maxbuf, maxbuf_sz, buf_sz,	biotmap_sz;
1064 
1065 	/*
1066 	 * With KASAN or KMSAN enabled, the kernel map is shadowed.  Account for
1067 	 * this when sizing maps based on the amount of physical memory
1068 	 * available.
1069 	 */
1070 #if defined(KASAN)
1071 	physmem_est = (physmem_est * KASAN_SHADOW_SCALE) /
1072 	    (KASAN_SHADOW_SCALE + 1);
1073 #elif defined(KMSAN)
1074 	physmem_est /= 3;
1075 
1076 	/*
1077 	 * KMSAN cannot reliably determine whether buffer data is initialized
1078 	 * unless it is updated through a KVA mapping.
1079 	 */
1080 	unmapped_buf_allowed = 0;
1081 #endif
1082 
1083 	/*
1084 	 * physmem_est is in pages.  Convert it to kilobytes (assumes
1085 	 * PAGE_SIZE is >= 1K)
1086 	 */
1087 	physmem_est = physmem_est * (PAGE_SIZE / 1024);
1088 
1089 	maxbcachebuf_adjust();
1090 	/*
1091 	 * The nominal buffer size (and minimum KVA allocation) is BKVASIZE.
1092 	 * For the first 64MB of ram nominally allocate sufficient buffers to
1093 	 * cover 1/4 of our ram.  Beyond the first 64MB allocate additional
1094 	 * buffers to cover 1/10 of our ram over 64MB.  When auto-sizing
1095 	 * the buffer cache we limit the eventual kva reservation to
1096 	 * maxbcache bytes.
1097 	 *
1098 	 * factor represents the 1/4 x ram conversion.
1099 	 */
1100 	if (nbuf == 0) {
1101 		int factor = 4 * BKVASIZE / 1024;
1102 
1103 		nbuf = 50;
1104 		if (physmem_est > 4096)
1105 			nbuf += min((physmem_est - 4096) / factor,
1106 			    65536 / factor);
1107 		if (physmem_est > 65536)
1108 			nbuf += min((physmem_est - 65536) * 2 / (factor * 5),
1109 			    32 * 1024 * 1024 / (factor * 5));
1110 
1111 		if (maxbcache && nbuf > maxbcache / BKVASIZE)
1112 			nbuf = maxbcache / BKVASIZE;
1113 		tuned_nbuf = 1;
1114 	} else
1115 		tuned_nbuf = 0;
1116 
1117 	/* XXX Avoid unsigned long overflows later on with maxbufspace. */
1118 	maxbuf = (LONG_MAX / 3) / BKVASIZE;
1119 	if (nbuf > maxbuf) {
1120 		if (!tuned_nbuf)
1121 			printf("Warning: nbufs lowered from %d to %ld\n", nbuf,
1122 			    maxbuf);
1123 		nbuf = maxbuf;
1124 	}
1125 
1126 	/*
1127 	 * Ideal allocation size for the transient bio submap is 10%
1128 	 * of the maximal space buffer map.  This roughly corresponds
1129 	 * to the amount of the buffer mapped for typical UFS load.
1130 	 *
1131 	 * Clip the buffer map to reserve space for the transient
1132 	 * BIOs, if its extent is bigger than 90% (80% on i386) of the
1133 	 * maximum buffer map extent on the platform.
1134 	 *
1135 	 * The fall-back to the maxbuf in case of maxbcache unset,
1136 	 * allows to not trim the buffer KVA for the architectures
1137 	 * with ample KVA space.
1138 	 */
1139 	if (bio_transient_maxcnt == 0 && unmapped_buf_allowed) {
1140 		maxbuf_sz = maxbcache != 0 ? maxbcache : maxbuf * BKVASIZE;
1141 		buf_sz = (long)nbuf * BKVASIZE;
1142 		if (buf_sz < maxbuf_sz / TRANSIENT_DENOM *
1143 		    (TRANSIENT_DENOM - 1)) {
1144 			/*
1145 			 * There is more KVA than memory.  Do not
1146 			 * adjust buffer map size, and assign the rest
1147 			 * of maxbuf to transient map.
1148 			 */
1149 			biotmap_sz = maxbuf_sz - buf_sz;
1150 		} else {
1151 			/*
1152 			 * Buffer map spans all KVA we could afford on
1153 			 * this platform.  Give 10% (20% on i386) of
1154 			 * the buffer map to the transient bio map.
1155 			 */
1156 			biotmap_sz = buf_sz / TRANSIENT_DENOM;
1157 			buf_sz -= biotmap_sz;
1158 		}
1159 		if (biotmap_sz / INT_MAX > maxphys)
1160 			bio_transient_maxcnt = INT_MAX;
1161 		else
1162 			bio_transient_maxcnt = biotmap_sz / maxphys;
1163 		/*
1164 		 * Artificially limit to 1024 simultaneous in-flight I/Os
1165 		 * using the transient mapping.
1166 		 */
1167 		if (bio_transient_maxcnt > 1024)
1168 			bio_transient_maxcnt = 1024;
1169 		if (tuned_nbuf)
1170 			nbuf = buf_sz / BKVASIZE;
1171 	}
1172 
1173 	if (nswbuf == 0) {
1174 		/*
1175 		 * Pager buffers are allocated for short periods, so scale the
1176 		 * number of reserved buffers based on the number of CPUs rather
1177 		 * than amount of memory.
1178 		 */
1179 		nswbuf = min(nbuf / 4, 32 * mp_ncpus);
1180 		if (nswbuf < NSWBUF_MIN)
1181 			nswbuf = NSWBUF_MIN;
1182 	}
1183 
1184 	/*
1185 	 * Reserve space for the buffer cache buffers
1186 	 */
1187 	buf = (char *)v;
1188 	v = (caddr_t)buf + (sizeof(struct buf) + sizeof(vm_page_t) *
1189 	    atop(maxbcachebuf)) * nbuf;
1190 
1191 	return (v);
1192 }
1193 
1194 /*
1195  * Single global constant for BUF_WMESG, to avoid getting multiple
1196  * references.
1197  */
1198 static const char buf_wmesg[] = "bufwait";
1199 
1200 /* Initialize the buffer subsystem.  Called before use of any buffers. */
1201 void
bufinit(void)1202 bufinit(void)
1203 {
1204 	struct buf *bp;
1205 	int i;
1206 
1207 	TSENTER();
1208 	KASSERT(maxbcachebuf >= MAXBSIZE,
1209 	    ("maxbcachebuf (%d) must be >= MAXBSIZE (%d)\n", maxbcachebuf,
1210 	    MAXBSIZE));
1211 	bq_init(&bqempty, QUEUE_EMPTY, -1, "bufq empty lock");
1212 	mtx_init(&rbreqlock, "runningbufspace lock", NULL, MTX_DEF);
1213 	mtx_init(&bdlock, "buffer daemon lock", NULL, MTX_DEF);
1214 	mtx_init(&bdirtylock, "dirty buf lock", NULL, MTX_DEF);
1215 
1216 	unmapped_buf = (caddr_t)kva_alloc(maxphys);
1217 #ifdef INVARIANTS
1218 	poisoned_buf = unmapped_buf;
1219 #endif
1220 
1221 	/* finally, initialize each buffer header and stick on empty q */
1222 	for (i = 0; i < nbuf; i++) {
1223 		bp = nbufp(i);
1224 		bzero(bp, sizeof(*bp) + sizeof(vm_page_t) * atop(maxbcachebuf));
1225 		bp->b_flags = B_INVAL;
1226 		bp->b_rcred = NOCRED;
1227 		bp->b_wcred = NOCRED;
1228 		bp->b_qindex = QUEUE_NONE;
1229 		bp->b_domain = -1;
1230 		bp->b_subqueue = mp_maxid + 1;
1231 		bp->b_xflags = 0;
1232 		bp->b_data = bp->b_kvabase = unmapped_buf;
1233 		LIST_INIT(&bp->b_dep);
1234 		BUF_LOCKINIT(bp, buf_wmesg);
1235 		bq_insert(&bqempty, bp, false);
1236 	}
1237 
1238 	/*
1239 	 * maxbufspace is the absolute maximum amount of buffer space we are
1240 	 * allowed to reserve in KVM and in real terms.  The absolute maximum
1241 	 * is nominally used by metadata.  hibufspace is the nominal maximum
1242 	 * used by most other requests.  The differential is required to
1243 	 * ensure that metadata deadlocks don't occur.
1244 	 *
1245 	 * maxbufspace is based on BKVASIZE.  Allocating buffers larger then
1246 	 * this may result in KVM fragmentation which is not handled optimally
1247 	 * by the system. XXX This is less true with vmem.  We could use
1248 	 * PAGE_SIZE.
1249 	 */
1250 	maxbufspace = (long)nbuf * BKVASIZE;
1251 	hibufspace = lmax(3 * maxbufspace / 4, maxbufspace - maxbcachebuf * 10);
1252 	lobufspace = (hibufspace / 20) * 19; /* 95% */
1253 	bufspacethresh = lobufspace + (hibufspace - lobufspace) / 2;
1254 
1255 	/*
1256 	 * Note: The upper limit for hirunningspace was chosen arbitrarily and
1257 	 * may need further tuning. It corresponds to 128 outstanding write IO
1258 	 * requests, which fits with many RAID controllers' tagged queuing
1259 	 * limits.
1260 	 *
1261 	 * The lower 1 MiB limit is the historical upper limit for
1262 	 * hirunningspace.
1263 	 */
1264 	hirunningspace = lmax(lmin(roundup(hibufspace / 64, maxbcachebuf),
1265 	    128 * maxphys), 1024 * 1024);
1266 	lorunningspace = roundup((hirunningspace * 2) / 3, maxbcachebuf);
1267 
1268 	/*
1269 	 * Limit the amount of malloc memory since it is wired permanently into
1270 	 * the kernel space.  Even though this is accounted for in the buffer
1271 	 * allocation, we don't want the malloced region to grow uncontrolled.
1272 	 * The malloc scheme improves memory utilization significantly on
1273 	 * average (small) directories.
1274 	 */
1275 	maxbufmallocspace = hibufspace / 20;
1276 
1277 	/*
1278 	 * Reduce the chance of a deadlock occurring by limiting the number
1279 	 * of delayed-write dirty buffers we allow to stack up.
1280 	 */
1281 	hidirtybuffers = nbuf / 4 + 20;
1282 	dirtybufthresh = hidirtybuffers * 9 / 10;
1283 	/*
1284 	 * To support extreme low-memory systems, make sure hidirtybuffers
1285 	 * cannot eat up all available buffer space.  This occurs when our
1286 	 * minimum cannot be met.  We try to size hidirtybuffers to 3/4 our
1287 	 * buffer space assuming BKVASIZE'd buffers.
1288 	 */
1289 	while ((long)hidirtybuffers * BKVASIZE > 3 * hibufspace / 4) {
1290 		hidirtybuffers >>= 1;
1291 	}
1292 	lodirtybuffers = hidirtybuffers / 2;
1293 
1294 	/*
1295 	 * lofreebuffers should be sufficient to avoid stalling waiting on
1296 	 * buf headers under heavy utilization.  The bufs in per-cpu caches
1297 	 * are counted as free but will be unavailable to threads executing
1298 	 * on other cpus.
1299 	 *
1300 	 * hifreebuffers is the free target for the bufspace daemon.  This
1301 	 * should be set appropriately to limit work per-iteration.
1302 	 */
1303 	lofreebuffers = MIN((nbuf / 25) + (20 * mp_ncpus), 128 * mp_ncpus);
1304 	hifreebuffers = (3 * lofreebuffers) / 2;
1305 	numfreebuffers = nbuf;
1306 
1307 	/* Setup the kva and free list allocators. */
1308 	vmem_set_reclaim(buffer_arena, bufkva_reclaim);
1309 	buf_zone = uma_zcache_create("buf free cache",
1310 	    sizeof(struct buf) + sizeof(vm_page_t) * atop(maxbcachebuf),
1311 	    NULL, NULL, NULL, NULL, buf_import, buf_release, NULL, 0);
1312 
1313 	/*
1314 	 * Size the clean queue according to the amount of buffer space.
1315 	 * One queue per-256mb up to the max.  More queues gives better
1316 	 * concurrency but less accurate LRU.
1317 	 */
1318 	buf_domains = MIN(howmany(maxbufspace, 256*1024*1024), BUF_DOMAINS);
1319 	for (i = 0 ; i < buf_domains; i++) {
1320 		struct bufdomain *bd;
1321 
1322 		bd = &bdomain[i];
1323 		bd_init(bd);
1324 		bd->bd_freebuffers = nbuf / buf_domains;
1325 		bd->bd_hifreebuffers = hifreebuffers / buf_domains;
1326 		bd->bd_lofreebuffers = lofreebuffers / buf_domains;
1327 		bd->bd_bufspace = 0;
1328 		bd->bd_maxbufspace = maxbufspace / buf_domains;
1329 		bd->bd_hibufspace = hibufspace / buf_domains;
1330 		bd->bd_lobufspace = lobufspace / buf_domains;
1331 		bd->bd_bufspacethresh = bufspacethresh / buf_domains;
1332 		bd->bd_numdirtybuffers = 0;
1333 		bd->bd_hidirtybuffers = hidirtybuffers / buf_domains;
1334 		bd->bd_lodirtybuffers = lodirtybuffers / buf_domains;
1335 		bd->bd_dirtybufthresh = dirtybufthresh / buf_domains;
1336 		/* Don't allow more than 2% of bufs in the per-cpu caches. */
1337 		bd->bd_lim = nbuf / buf_domains / 50 / mp_ncpus;
1338 	}
1339 	getnewbufcalls = counter_u64_alloc(M_WAITOK);
1340 	getnewbufrestarts = counter_u64_alloc(M_WAITOK);
1341 	mappingrestarts = counter_u64_alloc(M_WAITOK);
1342 	numbufallocfails = counter_u64_alloc(M_WAITOK);
1343 	notbufdflushes = counter_u64_alloc(M_WAITOK);
1344 	buffreekvacnt = counter_u64_alloc(M_WAITOK);
1345 	bufdefragcnt = counter_u64_alloc(M_WAITOK);
1346 	bufkvaspace = counter_u64_alloc(M_WAITOK);
1347 	TSEXIT();
1348 }
1349 
1350 #ifdef INVARIANTS
1351 static inline void
vfs_buf_check_mapped(struct buf * bp)1352 vfs_buf_check_mapped(struct buf *bp)
1353 {
1354 
1355 	KASSERT(bp->b_kvabase != unmapped_buf,
1356 	    ("mapped buf: b_kvabase was not updated %p", bp));
1357 	KASSERT(bp->b_data != unmapped_buf,
1358 	    ("mapped buf: b_data was not updated %p", bp));
1359 	KASSERT(bp->b_data < unmapped_buf || bp->b_data >= unmapped_buf +
1360 	    maxphys, ("b_data + b_offset unmapped %p", bp));
1361 }
1362 
1363 static inline void
vfs_buf_check_unmapped(struct buf * bp)1364 vfs_buf_check_unmapped(struct buf *bp)
1365 {
1366 
1367 	KASSERT(bp->b_data == unmapped_buf,
1368 	    ("unmapped buf: corrupted b_data %p", bp));
1369 }
1370 
1371 #define	BUF_CHECK_MAPPED(bp) vfs_buf_check_mapped(bp)
1372 #define	BUF_CHECK_UNMAPPED(bp) vfs_buf_check_unmapped(bp)
1373 #else
1374 #define	BUF_CHECK_MAPPED(bp) do {} while (0)
1375 #define	BUF_CHECK_UNMAPPED(bp) do {} while (0)
1376 #endif
1377 
1378 static int
isbufbusy(struct buf * bp)1379 isbufbusy(struct buf *bp)
1380 {
1381 	if (((bp->b_flags & B_INVAL) == 0 && BUF_ISLOCKED(bp)) ||
1382 	    ((bp->b_flags & (B_DELWRI | B_INVAL)) == B_DELWRI))
1383 		return (1);
1384 	return (0);
1385 }
1386 
1387 /*
1388  * Shutdown the system cleanly to prepare for reboot, halt, or power off.
1389  */
1390 void
bufshutdown(int show_busybufs)1391 bufshutdown(int show_busybufs)
1392 {
1393 	static int first_buf_printf = 1;
1394 	struct buf *bp;
1395 	int i, iter, nbusy, pbusy;
1396 #ifndef PREEMPTION
1397 	int subiter;
1398 #endif
1399 
1400 	/*
1401 	 * Sync filesystems for shutdown
1402 	 */
1403 	wdog_kern_pat(WD_LASTVAL);
1404 	kern_sync(curthread);
1405 
1406 	/*
1407 	 * With soft updates, some buffers that are
1408 	 * written will be remarked as dirty until other
1409 	 * buffers are written.
1410 	 */
1411 	for (iter = pbusy = 0; iter < 20; iter++) {
1412 		nbusy = 0;
1413 		for (i = nbuf - 1; i >= 0; i--) {
1414 			bp = nbufp(i);
1415 			if (isbufbusy(bp))
1416 				nbusy++;
1417 		}
1418 		if (nbusy == 0) {
1419 			if (first_buf_printf)
1420 				printf("All buffers synced.");
1421 			break;
1422 		}
1423 		if (first_buf_printf) {
1424 			printf("Syncing disks, buffers remaining... ");
1425 			first_buf_printf = 0;
1426 		}
1427 		printf("%d ", nbusy);
1428 		if (nbusy < pbusy)
1429 			iter = 0;
1430 		pbusy = nbusy;
1431 
1432 		wdog_kern_pat(WD_LASTVAL);
1433 		kern_sync(curthread);
1434 
1435 #ifdef PREEMPTION
1436 		/*
1437 		 * Spin for a while to allow interrupt threads to run.
1438 		 */
1439 		DELAY(50000 * iter);
1440 #else
1441 		/*
1442 		 * Context switch several times to allow interrupt
1443 		 * threads to run.
1444 		 */
1445 		for (subiter = 0; subiter < 50 * iter; subiter++) {
1446 			sched_relinquish(curthread);
1447 			DELAY(1000);
1448 		}
1449 #endif
1450 	}
1451 	printf("\n");
1452 	/*
1453 	 * Count only busy local buffers to prevent forcing
1454 	 * a fsck if we're just a client of a wedged NFS server
1455 	 */
1456 	nbusy = 0;
1457 	for (i = nbuf - 1; i >= 0; i--) {
1458 		bp = nbufp(i);
1459 		if (isbufbusy(bp)) {
1460 #if 0
1461 /* XXX: This is bogus.  We should probably have a BO_REMOTE flag instead */
1462 			if (bp->b_dev == NULL) {
1463 				TAILQ_REMOVE(&mountlist,
1464 				    bp->b_vp->v_mount, mnt_list);
1465 				continue;
1466 			}
1467 #endif
1468 			nbusy++;
1469 			if (show_busybufs > 0) {
1470 				printf(
1471 	    "%d: buf:%p, vnode:%p, flags:%0x, blkno:%jd, lblkno:%jd, buflock:",
1472 				    nbusy, bp, bp->b_vp, bp->b_flags,
1473 				    (intmax_t)bp->b_blkno,
1474 				    (intmax_t)bp->b_lblkno);
1475 				BUF_LOCKPRINTINFO(bp);
1476 				if (show_busybufs > 1)
1477 					vn_printf(bp->b_vp,
1478 					    "vnode content: ");
1479 			}
1480 		}
1481 	}
1482 	if (nbusy) {
1483 		/*
1484 		 * Failed to sync all blocks. Indicate this and don't
1485 		 * unmount filesystems (thus forcing an fsck on reboot).
1486 		 */
1487 		BOOTTRACE("shutdown failed to sync buffers");
1488 		printf("Giving up on %d buffers\n", nbusy);
1489 		DELAY(5000000);	/* 5 seconds */
1490 		swapoff_all();
1491 	} else {
1492 		BOOTTRACE("shutdown sync complete");
1493 		if (!first_buf_printf)
1494 			printf("Final sync complete\n");
1495 
1496 		/*
1497 		 * Unmount filesystems and perform swapoff, to quiesce
1498 		 * the system as much as possible.  In particular, no
1499 		 * I/O should be initiated from top levels since it
1500 		 * might be abruptly terminated by reset, or otherwise
1501 		 * erronously handled because other parts of the
1502 		 * system are disabled.
1503 		 *
1504 		 * Swapoff before unmount, because file-backed swap is
1505 		 * non-operational after unmount of the underlying
1506 		 * filesystem.
1507 		 */
1508 		if (!KERNEL_PANICKED()) {
1509 			swapoff_all();
1510 			vfs_unmountall();
1511 		}
1512 		BOOTTRACE("shutdown unmounted all filesystems");
1513 	}
1514 	DELAY(100000);		/* wait for console output to finish */
1515 }
1516 
1517 static void
bpmap_qenter(struct buf * bp)1518 bpmap_qenter(struct buf *bp)
1519 {
1520 
1521 	BUF_CHECK_MAPPED(bp);
1522 
1523 	/*
1524 	 * bp->b_data is relative to bp->b_offset, but
1525 	 * bp->b_offset may be offset into the first page.
1526 	 */
1527 	bp->b_data = (caddr_t)trunc_page((vm_offset_t)bp->b_data);
1528 	pmap_qenter((vm_offset_t)bp->b_data, bp->b_pages, bp->b_npages);
1529 	bp->b_data = (caddr_t)((vm_offset_t)bp->b_data |
1530 	    (vm_offset_t)(bp->b_offset & PAGE_MASK));
1531 }
1532 
1533 static inline struct bufdomain *
bufdomain(struct buf * bp)1534 bufdomain(struct buf *bp)
1535 {
1536 
1537 	return (&bdomain[bp->b_domain]);
1538 }
1539 
1540 static struct bufqueue *
bufqueue(struct buf * bp)1541 bufqueue(struct buf *bp)
1542 {
1543 
1544 	switch (bp->b_qindex) {
1545 	case QUEUE_NONE:
1546 		/* FALLTHROUGH */
1547 	case QUEUE_SENTINEL:
1548 		return (NULL);
1549 	case QUEUE_EMPTY:
1550 		return (&bqempty);
1551 	case QUEUE_DIRTY:
1552 		return (&bufdomain(bp)->bd_dirtyq);
1553 	case QUEUE_CLEAN:
1554 		return (&bufdomain(bp)->bd_subq[bp->b_subqueue]);
1555 	default:
1556 		break;
1557 	}
1558 	panic("bufqueue(%p): Unhandled type %d\n", bp, bp->b_qindex);
1559 }
1560 
1561 /*
1562  * Return the locked bufqueue that bp is a member of.
1563  */
1564 static struct bufqueue *
bufqueue_acquire(struct buf * bp)1565 bufqueue_acquire(struct buf *bp)
1566 {
1567 	struct bufqueue *bq, *nbq;
1568 
1569 	/*
1570 	 * bp can be pushed from a per-cpu queue to the
1571 	 * cleanq while we're waiting on the lock.  Retry
1572 	 * if the queues don't match.
1573 	 */
1574 	bq = bufqueue(bp);
1575 	BQ_LOCK(bq);
1576 	for (;;) {
1577 		nbq = bufqueue(bp);
1578 		if (bq == nbq)
1579 			break;
1580 		BQ_UNLOCK(bq);
1581 		BQ_LOCK(nbq);
1582 		bq = nbq;
1583 	}
1584 	return (bq);
1585 }
1586 
1587 /*
1588  *	binsfree:
1589  *
1590  *	Insert the buffer into the appropriate free list.  Requires a
1591  *	locked buffer on entry and buffer is unlocked before return.
1592  */
1593 static void
binsfree(struct buf * bp,int qindex)1594 binsfree(struct buf *bp, int qindex)
1595 {
1596 	struct bufdomain *bd;
1597 	struct bufqueue *bq;
1598 
1599 	KASSERT(qindex == QUEUE_CLEAN || qindex == QUEUE_DIRTY,
1600 	    ("binsfree: Invalid qindex %d", qindex));
1601 	BUF_ASSERT_XLOCKED(bp);
1602 
1603 	/*
1604 	 * Handle delayed bremfree() processing.
1605 	 */
1606 	if (bp->b_flags & B_REMFREE) {
1607 		if (bp->b_qindex == qindex) {
1608 			bp->b_flags |= B_REUSE;
1609 			bp->b_flags &= ~B_REMFREE;
1610 			BUF_UNLOCK(bp);
1611 			return;
1612 		}
1613 		bq = bufqueue_acquire(bp);
1614 		bq_remove(bq, bp);
1615 		BQ_UNLOCK(bq);
1616 	}
1617 	bd = bufdomain(bp);
1618 	if (qindex == QUEUE_CLEAN) {
1619 		if (bd->bd_lim != 0)
1620 			bq = &bd->bd_subq[PCPU_GET(cpuid)];
1621 		else
1622 			bq = bd->bd_cleanq;
1623 	} else
1624 		bq = &bd->bd_dirtyq;
1625 	bq_insert(bq, bp, true);
1626 }
1627 
1628 /*
1629  * buf_free:
1630  *
1631  *	Free a buffer to the buf zone once it no longer has valid contents.
1632  */
1633 static void
buf_free(struct buf * bp)1634 buf_free(struct buf *bp)
1635 {
1636 
1637 	if (bp->b_flags & B_REMFREE)
1638 		bremfreef(bp);
1639 	if (bp->b_vflags & BV_BKGRDINPROG)
1640 		panic("losing buffer 1");
1641 	if (bp->b_rcred != NOCRED) {
1642 		crfree(bp->b_rcred);
1643 		bp->b_rcred = NOCRED;
1644 	}
1645 	if (bp->b_wcred != NOCRED) {
1646 		crfree(bp->b_wcred);
1647 		bp->b_wcred = NOCRED;
1648 	}
1649 	if (!LIST_EMPTY(&bp->b_dep))
1650 		buf_deallocate(bp);
1651 	bufkva_free(bp);
1652 	atomic_add_int(&bufdomain(bp)->bd_freebuffers, 1);
1653 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
1654 	BUF_UNLOCK(bp);
1655 	uma_zfree(buf_zone, bp);
1656 }
1657 
1658 /*
1659  * buf_import:
1660  *
1661  *	Import bufs into the uma cache from the buf list.  The system still
1662  *	expects a static array of bufs and much of the synchronization
1663  *	around bufs assumes type stable storage.  As a result, UMA is used
1664  *	only as a per-cpu cache of bufs still maintained on a global list.
1665  */
1666 static int
buf_import(void * arg,void ** store,int cnt,int domain,int flags)1667 buf_import(void *arg, void **store, int cnt, int domain, int flags)
1668 {
1669 	struct buf *bp;
1670 	int i;
1671 
1672 	BQ_LOCK(&bqempty);
1673 	for (i = 0; i < cnt; i++) {
1674 		bp = TAILQ_FIRST(&bqempty.bq_queue);
1675 		if (bp == NULL)
1676 			break;
1677 		bq_remove(&bqempty, bp);
1678 		store[i] = bp;
1679 	}
1680 	BQ_UNLOCK(&bqempty);
1681 
1682 	return (i);
1683 }
1684 
1685 /*
1686  * buf_release:
1687  *
1688  *	Release bufs from the uma cache back to the buffer queues.
1689  */
1690 static void
buf_release(void * arg,void ** store,int cnt)1691 buf_release(void *arg, void **store, int cnt)
1692 {
1693 	struct bufqueue *bq;
1694 	struct buf *bp;
1695         int i;
1696 
1697 	bq = &bqempty;
1698 	BQ_LOCK(bq);
1699         for (i = 0; i < cnt; i++) {
1700 		bp = store[i];
1701 		/* Inline bq_insert() to batch locking. */
1702 		TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
1703 		bp->b_flags &= ~(B_AGE | B_REUSE);
1704 		bq->bq_len++;
1705 		bp->b_qindex = bq->bq_index;
1706 	}
1707 	BQ_UNLOCK(bq);
1708 }
1709 
1710 /*
1711  * buf_alloc:
1712  *
1713  *	Allocate an empty buffer header.
1714  */
1715 static struct buf *
buf_alloc(struct bufdomain * bd)1716 buf_alloc(struct bufdomain *bd)
1717 {
1718 	struct buf *bp;
1719 	int freebufs, error;
1720 
1721 	/*
1722 	 * We can only run out of bufs in the buf zone if the average buf
1723 	 * is less than BKVASIZE.  In this case the actual wait/block will
1724 	 * come from buf_reycle() failing to flush one of these small bufs.
1725 	 */
1726 	bp = NULL;
1727 	freebufs = atomic_fetchadd_int(&bd->bd_freebuffers, -1);
1728 	if (freebufs > 0)
1729 		bp = uma_zalloc(buf_zone, M_NOWAIT);
1730 	if (bp == NULL) {
1731 		atomic_add_int(&bd->bd_freebuffers, 1);
1732 		bufspace_daemon_wakeup(bd);
1733 		counter_u64_add(numbufallocfails, 1);
1734 		return (NULL);
1735 	}
1736 	/*
1737 	 * Wake-up the bufspace daemon on transition below threshold.
1738 	 */
1739 	if (freebufs == bd->bd_lofreebuffers)
1740 		bufspace_daemon_wakeup(bd);
1741 
1742 	error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWITNESS, NULL);
1743 	KASSERT(error == 0, ("%s: BUF_LOCK on free buf %p: %d.", __func__, bp,
1744 	    error));
1745 	(void)error;
1746 
1747 	KASSERT(bp->b_vp == NULL,
1748 	    ("bp: %p still has vnode %p.", bp, bp->b_vp));
1749 	KASSERT((bp->b_flags & (B_DELWRI | B_NOREUSE)) == 0,
1750 	    ("invalid buffer %p flags %#x", bp, bp->b_flags));
1751 	KASSERT((bp->b_xflags & (BX_VNCLEAN|BX_VNDIRTY)) == 0,
1752 	    ("bp: %p still on a buffer list. xflags %X", bp, bp->b_xflags));
1753 	KASSERT(bp->b_npages == 0,
1754 	    ("bp: %p still has %d vm pages\n", bp, bp->b_npages));
1755 	KASSERT(bp->b_kvasize == 0, ("bp: %p still has kva\n", bp));
1756 	KASSERT(bp->b_bufsize == 0, ("bp: %p still has bufspace\n", bp));
1757 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
1758 
1759 	bp->b_domain = BD_DOMAIN(bd);
1760 	bp->b_flags = 0;
1761 	bp->b_ioflags = 0;
1762 	bp->b_xflags = 0;
1763 	bp->b_vflags = 0;
1764 	bp->b_vp = NULL;
1765 	bp->b_blkno = bp->b_lblkno = 0;
1766 	bp->b_offset = NOOFFSET;
1767 	bp->b_iodone = 0;
1768 	bp->b_error = 0;
1769 	bp->b_resid = 0;
1770 	bp->b_bcount = 0;
1771 	bp->b_npages = 0;
1772 	bp->b_dirtyoff = bp->b_dirtyend = 0;
1773 	bp->b_bufobj = NULL;
1774 	bp->b_data = bp->b_kvabase = unmapped_buf;
1775 	bp->b_fsprivate1 = NULL;
1776 	bp->b_fsprivate2 = NULL;
1777 	bp->b_fsprivate3 = NULL;
1778 	LIST_INIT(&bp->b_dep);
1779 
1780 	return (bp);
1781 }
1782 
1783 /*
1784  *	buf_recycle:
1785  *
1786  *	Free a buffer from the given bufqueue.  kva controls whether the
1787  *	freed buf must own some kva resources.  This is used for
1788  *	defragmenting.
1789  */
1790 static int
buf_recycle(struct bufdomain * bd,bool kva)1791 buf_recycle(struct bufdomain *bd, bool kva)
1792 {
1793 	struct bufqueue *bq;
1794 	struct buf *bp, *nbp;
1795 
1796 	if (kva)
1797 		counter_u64_add(bufdefragcnt, 1);
1798 	nbp = NULL;
1799 	bq = bd->bd_cleanq;
1800 	BQ_LOCK(bq);
1801 	KASSERT(BQ_LOCKPTR(bq) == BD_LOCKPTR(bd),
1802 	    ("buf_recycle: Locks don't match"));
1803 	nbp = TAILQ_FIRST(&bq->bq_queue);
1804 
1805 	/*
1806 	 * Run scan, possibly freeing data and/or kva mappings on the fly
1807 	 * depending.
1808 	 */
1809 	while ((bp = nbp) != NULL) {
1810 		/*
1811 		 * Calculate next bp (we can only use it if we do not
1812 		 * release the bqlock).
1813 		 */
1814 		nbp = TAILQ_NEXT(bp, b_freelist);
1815 
1816 		/*
1817 		 * If we are defragging then we need a buffer with
1818 		 * some kva to reclaim.
1819 		 */
1820 		if (kva && bp->b_kvasize == 0)
1821 			continue;
1822 
1823 		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
1824 			continue;
1825 
1826 		/*
1827 		 * Implement a second chance algorithm for frequently
1828 		 * accessed buffers.
1829 		 */
1830 		if ((bp->b_flags & B_REUSE) != 0) {
1831 			TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
1832 			TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
1833 			bp->b_flags &= ~B_REUSE;
1834 			BUF_UNLOCK(bp);
1835 			continue;
1836 		}
1837 
1838 		/*
1839 		 * Skip buffers with background writes in progress.
1840 		 */
1841 		if ((bp->b_vflags & BV_BKGRDINPROG) != 0) {
1842 			BUF_UNLOCK(bp);
1843 			continue;
1844 		}
1845 
1846 		KASSERT(bp->b_qindex == QUEUE_CLEAN,
1847 		    ("buf_recycle: inconsistent queue %d bp %p",
1848 		    bp->b_qindex, bp));
1849 		KASSERT(bp->b_domain == BD_DOMAIN(bd),
1850 		    ("getnewbuf: queue domain %d doesn't match request %d",
1851 		    bp->b_domain, (int)BD_DOMAIN(bd)));
1852 		/*
1853 		 * NOTE:  nbp is now entirely invalid.  We can only restart
1854 		 * the scan from this point on.
1855 		 */
1856 		bq_remove(bq, bp);
1857 		BQ_UNLOCK(bq);
1858 
1859 		/*
1860 		 * Requeue the background write buffer with error and
1861 		 * restart the scan.
1862 		 */
1863 		if ((bp->b_vflags & BV_BKGRDERR) != 0) {
1864 			bqrelse(bp);
1865 			BQ_LOCK(bq);
1866 			nbp = TAILQ_FIRST(&bq->bq_queue);
1867 			continue;
1868 		}
1869 		bp->b_flags |= B_INVAL;
1870 		brelse(bp);
1871 		return (0);
1872 	}
1873 	bd->bd_wanted = 1;
1874 	BQ_UNLOCK(bq);
1875 
1876 	return (ENOBUFS);
1877 }
1878 
1879 /*
1880  *	bremfree:
1881  *
1882  *	Mark the buffer for removal from the appropriate free list.
1883  *
1884  */
1885 void
bremfree(struct buf * bp)1886 bremfree(struct buf *bp)
1887 {
1888 
1889 	CTR3(KTR_BUF, "bremfree(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
1890 	KASSERT((bp->b_flags & B_REMFREE) == 0,
1891 	    ("bremfree: buffer %p already marked for delayed removal.", bp));
1892 	KASSERT(bp->b_qindex != QUEUE_NONE,
1893 	    ("bremfree: buffer %p not on a queue.", bp));
1894 	BUF_ASSERT_XLOCKED(bp);
1895 
1896 	bp->b_flags |= B_REMFREE;
1897 }
1898 
1899 /*
1900  *	bremfreef:
1901  *
1902  *	Force an immediate removal from a free list.  Used only in nfs when
1903  *	it abuses the b_freelist pointer.
1904  */
1905 void
bremfreef(struct buf * bp)1906 bremfreef(struct buf *bp)
1907 {
1908 	struct bufqueue *bq;
1909 
1910 	bq = bufqueue_acquire(bp);
1911 	bq_remove(bq, bp);
1912 	BQ_UNLOCK(bq);
1913 }
1914 
1915 static void
bq_init(struct bufqueue * bq,int qindex,int subqueue,const char * lockname)1916 bq_init(struct bufqueue *bq, int qindex, int subqueue, const char *lockname)
1917 {
1918 
1919 	mtx_init(&bq->bq_lock, lockname, NULL, MTX_DEF);
1920 	TAILQ_INIT(&bq->bq_queue);
1921 	bq->bq_len = 0;
1922 	bq->bq_index = qindex;
1923 	bq->bq_subqueue = subqueue;
1924 }
1925 
1926 static void
bd_init(struct bufdomain * bd)1927 bd_init(struct bufdomain *bd)
1928 {
1929 	int i;
1930 
1931 	/* Per-CPU clean buf queues, plus one global queue. */
1932 	bd->bd_subq = mallocarray(mp_maxid + 2, sizeof(struct bufqueue),
1933 	    M_BIOBUF, M_WAITOK | M_ZERO);
1934 	bd->bd_cleanq = &bd->bd_subq[mp_maxid + 1];
1935 	bq_init(bd->bd_cleanq, QUEUE_CLEAN, mp_maxid + 1, "bufq clean lock");
1936 	bq_init(&bd->bd_dirtyq, QUEUE_DIRTY, -1, "bufq dirty lock");
1937 	for (i = 0; i <= mp_maxid; i++)
1938 		bq_init(&bd->bd_subq[i], QUEUE_CLEAN, i,
1939 		    "bufq clean subqueue lock");
1940 	mtx_init(&bd->bd_run_lock, "bufspace daemon run lock", NULL, MTX_DEF);
1941 }
1942 
1943 /*
1944  *	bq_remove:
1945  *
1946  *	Removes a buffer from the free list, must be called with the
1947  *	correct qlock held.
1948  */
1949 static void
bq_remove(struct bufqueue * bq,struct buf * bp)1950 bq_remove(struct bufqueue *bq, struct buf *bp)
1951 {
1952 
1953 	CTR3(KTR_BUF, "bq_remove(%p) vp %p flags %X",
1954 	    bp, bp->b_vp, bp->b_flags);
1955 	KASSERT(bp->b_qindex != QUEUE_NONE,
1956 	    ("bq_remove: buffer %p not on a queue.", bp));
1957 	KASSERT(bufqueue(bp) == bq,
1958 	    ("bq_remove: Remove buffer %p from wrong queue.", bp));
1959 
1960 	BQ_ASSERT_LOCKED(bq);
1961 	if (bp->b_qindex != QUEUE_EMPTY) {
1962 		BUF_ASSERT_XLOCKED(bp);
1963 	}
1964 	KASSERT(bq->bq_len >= 1,
1965 	    ("queue %d underflow", bp->b_qindex));
1966 	TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
1967 	bq->bq_len--;
1968 	bp->b_qindex = QUEUE_NONE;
1969 	bp->b_flags &= ~(B_REMFREE | B_REUSE);
1970 }
1971 
1972 static void
bd_flush(struct bufdomain * bd,struct bufqueue * bq)1973 bd_flush(struct bufdomain *bd, struct bufqueue *bq)
1974 {
1975 	struct buf *bp;
1976 
1977 	BQ_ASSERT_LOCKED(bq);
1978 	if (bq != bd->bd_cleanq) {
1979 		BD_LOCK(bd);
1980 		while ((bp = TAILQ_FIRST(&bq->bq_queue)) != NULL) {
1981 			TAILQ_REMOVE(&bq->bq_queue, bp, b_freelist);
1982 			TAILQ_INSERT_TAIL(&bd->bd_cleanq->bq_queue, bp,
1983 			    b_freelist);
1984 			bp->b_subqueue = bd->bd_cleanq->bq_subqueue;
1985 		}
1986 		bd->bd_cleanq->bq_len += bq->bq_len;
1987 		bq->bq_len = 0;
1988 	}
1989 	if (bd->bd_wanted) {
1990 		bd->bd_wanted = 0;
1991 		wakeup(&bd->bd_wanted);
1992 	}
1993 	if (bq != bd->bd_cleanq)
1994 		BD_UNLOCK(bd);
1995 }
1996 
1997 static int
bd_flushall(struct bufdomain * bd)1998 bd_flushall(struct bufdomain *bd)
1999 {
2000 	struct bufqueue *bq;
2001 	int flushed;
2002 	int i;
2003 
2004 	if (bd->bd_lim == 0)
2005 		return (0);
2006 	flushed = 0;
2007 	for (i = 0; i <= mp_maxid; i++) {
2008 		bq = &bd->bd_subq[i];
2009 		if (bq->bq_len == 0)
2010 			continue;
2011 		BQ_LOCK(bq);
2012 		bd_flush(bd, bq);
2013 		BQ_UNLOCK(bq);
2014 		flushed++;
2015 	}
2016 
2017 	return (flushed);
2018 }
2019 
2020 static void
bq_insert(struct bufqueue * bq,struct buf * bp,bool unlock)2021 bq_insert(struct bufqueue *bq, struct buf *bp, bool unlock)
2022 {
2023 	struct bufdomain *bd;
2024 
2025 	if (bp->b_qindex != QUEUE_NONE)
2026 		panic("bq_insert: free buffer %p onto another queue?", bp);
2027 
2028 	bd = bufdomain(bp);
2029 	if (bp->b_flags & B_AGE) {
2030 		/* Place this buf directly on the real queue. */
2031 		if (bq->bq_index == QUEUE_CLEAN)
2032 			bq = bd->bd_cleanq;
2033 		BQ_LOCK(bq);
2034 		TAILQ_INSERT_HEAD(&bq->bq_queue, bp, b_freelist);
2035 	} else {
2036 		BQ_LOCK(bq);
2037 		TAILQ_INSERT_TAIL(&bq->bq_queue, bp, b_freelist);
2038 	}
2039 	bp->b_flags &= ~(B_AGE | B_REUSE);
2040 	bq->bq_len++;
2041 	bp->b_qindex = bq->bq_index;
2042 	bp->b_subqueue = bq->bq_subqueue;
2043 
2044 	/*
2045 	 * Unlock before we notify so that we don't wakeup a waiter that
2046 	 * fails a trylock on the buf and sleeps again.
2047 	 */
2048 	if (unlock)
2049 		BUF_UNLOCK(bp);
2050 
2051 	if (bp->b_qindex == QUEUE_CLEAN) {
2052 		/*
2053 		 * Flush the per-cpu queue and notify any waiters.
2054 		 */
2055 		if (bd->bd_wanted || (bq != bd->bd_cleanq &&
2056 		    bq->bq_len >= bd->bd_lim))
2057 			bd_flush(bd, bq);
2058 	}
2059 	BQ_UNLOCK(bq);
2060 }
2061 
2062 /*
2063  *	bufkva_free:
2064  *
2065  *	Free the kva allocation for a buffer.
2066  *
2067  */
2068 static void
bufkva_free(struct buf * bp)2069 bufkva_free(struct buf *bp)
2070 {
2071 
2072 #ifdef INVARIANTS
2073 	if (bp->b_kvasize == 0) {
2074 		KASSERT(bp->b_kvabase == unmapped_buf &&
2075 		    bp->b_data == unmapped_buf,
2076 		    ("Leaked KVA space on %p", bp));
2077 	} else if (buf_mapped(bp))
2078 		BUF_CHECK_MAPPED(bp);
2079 	else
2080 		BUF_CHECK_UNMAPPED(bp);
2081 #endif
2082 	if (bp->b_kvasize == 0)
2083 		return;
2084 
2085 	vmem_free(buffer_arena, (vm_offset_t)bp->b_kvabase, bp->b_kvasize);
2086 	counter_u64_add(bufkvaspace, -bp->b_kvasize);
2087 	counter_u64_add(buffreekvacnt, 1);
2088 	bp->b_data = bp->b_kvabase = unmapped_buf;
2089 	bp->b_kvasize = 0;
2090 }
2091 
2092 /*
2093  *	bufkva_alloc:
2094  *
2095  *	Allocate the buffer KVA and set b_kvasize and b_kvabase.
2096  */
2097 static int
bufkva_alloc(struct buf * bp,int maxsize,int gbflags)2098 bufkva_alloc(struct buf *bp, int maxsize, int gbflags)
2099 {
2100 	vm_offset_t addr;
2101 	int error;
2102 
2103 	KASSERT((gbflags & GB_UNMAPPED) == 0 || (gbflags & GB_KVAALLOC) != 0,
2104 	    ("Invalid gbflags 0x%x in %s", gbflags, __func__));
2105 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
2106 	KASSERT(maxsize <= maxbcachebuf,
2107 	    ("bufkva_alloc kva too large %d %u", maxsize, maxbcachebuf));
2108 
2109 	bufkva_free(bp);
2110 
2111 	addr = 0;
2112 	error = vmem_alloc(buffer_arena, maxsize, M_BESTFIT | M_NOWAIT, &addr);
2113 	if (error != 0) {
2114 		/*
2115 		 * Buffer map is too fragmented.  Request the caller
2116 		 * to defragment the map.
2117 		 */
2118 		return (error);
2119 	}
2120 	bp->b_kvabase = (caddr_t)addr;
2121 	bp->b_kvasize = maxsize;
2122 	counter_u64_add(bufkvaspace, bp->b_kvasize);
2123 	if ((gbflags & GB_UNMAPPED) != 0) {
2124 		bp->b_data = unmapped_buf;
2125 		BUF_CHECK_UNMAPPED(bp);
2126 	} else {
2127 		bp->b_data = bp->b_kvabase;
2128 		BUF_CHECK_MAPPED(bp);
2129 	}
2130 	return (0);
2131 }
2132 
2133 /*
2134  *	bufkva_reclaim:
2135  *
2136  *	Reclaim buffer kva by freeing buffers holding kva.  This is a vmem
2137  *	callback that fires to avoid returning failure.
2138  */
2139 static void
bufkva_reclaim(vmem_t * vmem,int flags)2140 bufkva_reclaim(vmem_t *vmem, int flags)
2141 {
2142 	bool done;
2143 	int q;
2144 	int i;
2145 
2146 	done = false;
2147 	for (i = 0; i < 5; i++) {
2148 		for (q = 0; q < buf_domains; q++)
2149 			if (buf_recycle(&bdomain[q], true) != 0)
2150 				done = true;
2151 		if (done)
2152 			break;
2153 	}
2154 	return;
2155 }
2156 
2157 /*
2158  * Attempt to initiate asynchronous I/O on read-ahead blocks.  We must
2159  * clear BIO_ERROR and B_INVAL prior to initiating I/O . If B_CACHE is set,
2160  * the buffer is valid and we do not have to do anything.
2161  */
2162 static void
breada(struct vnode * vp,daddr_t * rablkno,int * rabsize,int cnt,struct ucred * cred,int flags,void (* ckhashfunc)(struct buf *))2163 breada(struct vnode * vp, daddr_t * rablkno, int * rabsize, int cnt,
2164     struct ucred * cred, int flags, void (*ckhashfunc)(struct buf *))
2165 {
2166 	struct buf *rabp;
2167 	struct thread *td;
2168 	int i;
2169 
2170 	td = curthread;
2171 
2172 	for (i = 0; i < cnt; i++, rablkno++, rabsize++) {
2173 		if (inmem(vp, *rablkno))
2174 			continue;
2175 		rabp = getblk(vp, *rablkno, *rabsize, 0, 0, 0);
2176 		if ((rabp->b_flags & B_CACHE) != 0) {
2177 			brelse(rabp);
2178 			continue;
2179 		}
2180 #ifdef RACCT
2181 		if (racct_enable) {
2182 			PROC_LOCK(curproc);
2183 			racct_add_buf(curproc, rabp, 0);
2184 			PROC_UNLOCK(curproc);
2185 		}
2186 #endif /* RACCT */
2187 		td->td_ru.ru_inblock++;
2188 		rabp->b_flags |= B_ASYNC;
2189 		rabp->b_flags &= ~B_INVAL;
2190 		if ((flags & GB_CKHASH) != 0) {
2191 			rabp->b_flags |= B_CKHASH;
2192 			rabp->b_ckhashcalc = ckhashfunc;
2193 		}
2194 		rabp->b_ioflags &= ~BIO_ERROR;
2195 		rabp->b_iocmd = BIO_READ;
2196 		if (rabp->b_rcred == NOCRED && cred != NOCRED)
2197 			rabp->b_rcred = crhold(cred);
2198 		vfs_busy_pages(rabp, 0);
2199 		BUF_KERNPROC(rabp);
2200 		rabp->b_iooffset = dbtob(rabp->b_blkno);
2201 		bstrategy(rabp);
2202 	}
2203 }
2204 
2205 /*
2206  * Entry point for bread() and breadn() via #defines in sys/buf.h.
2207  *
2208  * Get a buffer with the specified data.  Look in the cache first.  We
2209  * must clear BIO_ERROR and B_INVAL prior to initiating I/O.  If B_CACHE
2210  * is set, the buffer is valid and we do not have to do anything, see
2211  * getblk(). Also starts asynchronous I/O on read-ahead blocks.
2212  *
2213  * Always return a NULL buffer pointer (in bpp) when returning an error.
2214  *
2215  * The blkno parameter is the logical block being requested. Normally
2216  * the mapping of logical block number to disk block address is done
2217  * by calling VOP_BMAP(). However, if the mapping is already known, the
2218  * disk block address can be passed using the dblkno parameter. If the
2219  * disk block address is not known, then the same value should be passed
2220  * for blkno and dblkno.
2221  */
2222 int
breadn_flags(struct vnode * vp,daddr_t blkno,daddr_t dblkno,int size,daddr_t * rablkno,int * rabsize,int cnt,struct ucred * cred,int flags,void (* ckhashfunc)(struct buf *),struct buf ** bpp)2223 breadn_flags(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size,
2224     daddr_t *rablkno, int *rabsize, int cnt, struct ucred *cred, int flags,
2225     void (*ckhashfunc)(struct buf *), struct buf **bpp)
2226 {
2227 	struct buf *bp;
2228 	struct thread *td;
2229 	int error, readwait, rv;
2230 
2231 	CTR3(KTR_BUF, "breadn(%p, %jd, %d)", vp, blkno, size);
2232 	td = curthread;
2233 	/*
2234 	 * Can only return NULL if GB_LOCK_NOWAIT or GB_SPARSE flags
2235 	 * are specified.
2236 	 */
2237 	error = getblkx(vp, blkno, dblkno, size, 0, 0, flags, &bp);
2238 	if (error != 0) {
2239 		*bpp = NULL;
2240 		return (error);
2241 	}
2242 	KASSERT(blkno == bp->b_lblkno,
2243 	    ("getblkx returned buffer for blkno %jd instead of blkno %jd",
2244 	    (intmax_t)bp->b_lblkno, (intmax_t)blkno));
2245 	flags &= ~GB_NOSPARSE;
2246 	*bpp = bp;
2247 
2248 	/*
2249 	 * If not found in cache, do some I/O
2250 	 */
2251 	readwait = 0;
2252 	if ((bp->b_flags & B_CACHE) == 0) {
2253 #ifdef RACCT
2254 		if (racct_enable) {
2255 			PROC_LOCK(td->td_proc);
2256 			racct_add_buf(td->td_proc, bp, 0);
2257 			PROC_UNLOCK(td->td_proc);
2258 		}
2259 #endif /* RACCT */
2260 		td->td_ru.ru_inblock++;
2261 		bp->b_iocmd = BIO_READ;
2262 		bp->b_flags &= ~B_INVAL;
2263 		if ((flags & GB_CKHASH) != 0) {
2264 			bp->b_flags |= B_CKHASH;
2265 			bp->b_ckhashcalc = ckhashfunc;
2266 		}
2267 		if ((flags & GB_CVTENXIO) != 0)
2268 			bp->b_xflags |= BX_CVTENXIO;
2269 		bp->b_ioflags &= ~BIO_ERROR;
2270 		if (bp->b_rcred == NOCRED && cred != NOCRED)
2271 			bp->b_rcred = crhold(cred);
2272 		vfs_busy_pages(bp, 0);
2273 		bp->b_iooffset = dbtob(bp->b_blkno);
2274 		bstrategy(bp);
2275 		++readwait;
2276 	}
2277 
2278 	/*
2279 	 * Attempt to initiate asynchronous I/O on read-ahead blocks.
2280 	 */
2281 	breada(vp, rablkno, rabsize, cnt, cred, flags, ckhashfunc);
2282 
2283 	rv = 0;
2284 	if (readwait) {
2285 		rv = bufwait(bp);
2286 		if (rv != 0) {
2287 			brelse(bp);
2288 			*bpp = NULL;
2289 		}
2290 	}
2291 	return (rv);
2292 }
2293 
2294 /*
2295  * Write, release buffer on completion.  (Done by iodone
2296  * if async).  Do not bother writing anything if the buffer
2297  * is invalid.
2298  *
2299  * Note that we set B_CACHE here, indicating that buffer is
2300  * fully valid and thus cacheable.  This is true even of NFS
2301  * now so we set it generally.  This could be set either here
2302  * or in biodone() since the I/O is synchronous.  We put it
2303  * here.
2304  */
2305 int
bufwrite(struct buf * bp)2306 bufwrite(struct buf *bp)
2307 {
2308 	struct vnode *vp;
2309 	long space;
2310 	int oldflags, retval;
2311 	bool vp_md;
2312 
2313 	CTR3(KTR_BUF, "bufwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2314 	if ((bp->b_bufobj->bo_flag & BO_DEAD) != 0) {
2315 		bp->b_flags |= B_INVAL | B_RELBUF;
2316 		bp->b_flags &= ~B_CACHE;
2317 		brelse(bp);
2318 		return (ENXIO);
2319 	}
2320 	if ((bp->b_flags & B_INVAL) != 0) {
2321 		brelse(bp);
2322 		return (0);
2323 	}
2324 
2325 	if ((bp->b_flags & B_BARRIER) != 0)
2326 		atomic_add_long(&barrierwrites, 1);
2327 
2328 	oldflags = bp->b_flags;
2329 
2330 	KASSERT((bp->b_vflags & BV_BKGRDINPROG) == 0,
2331 	    ("FFS background buffer should not get here %p", bp));
2332 
2333 	vp = bp->b_vp;
2334 	vp_md = vp != NULL && (vp->v_vflag & VV_MD) != 0;
2335 
2336 	/*
2337 	 * Mark the buffer clean.  Increment the bufobj write count
2338 	 * before bundirty() call, to prevent other thread from seeing
2339 	 * empty dirty list and zero counter for writes in progress,
2340 	 * falsely indicating that the bufobj is clean.
2341 	 */
2342 	bufobj_wref(bp->b_bufobj);
2343 	bundirty(bp);
2344 
2345 	bp->b_flags &= ~B_DONE;
2346 	bp->b_ioflags &= ~BIO_ERROR;
2347 	bp->b_flags |= B_CACHE;
2348 	bp->b_iocmd = BIO_WRITE;
2349 
2350 	vfs_busy_pages(bp, 1);
2351 
2352 	/*
2353 	 * Normal bwrites pipeline writes
2354 	 */
2355 	bp->b_runningbufspace = bp->b_bufsize;
2356 	space = atomic_fetchadd_long(&runningbufspace, bp->b_runningbufspace);
2357 
2358 #ifdef RACCT
2359 	if (racct_enable) {
2360 		PROC_LOCK(curproc);
2361 		racct_add_buf(curproc, bp, 1);
2362 		PROC_UNLOCK(curproc);
2363 	}
2364 #endif /* RACCT */
2365 	curthread->td_ru.ru_oublock++;
2366 	if ((oldflags & B_ASYNC) != 0)
2367 		BUF_KERNPROC(bp);
2368 	bp->b_iooffset = dbtob(bp->b_blkno);
2369 	buf_track(bp, __func__);
2370 	bstrategy(bp);
2371 
2372 	if ((oldflags & B_ASYNC) == 0) {
2373 		retval = bufwait(bp);
2374 		brelse(bp);
2375 		return (retval);
2376 	} else if (space > hirunningspace) {
2377 		/*
2378 		 * Don't allow the async write to saturate the I/O
2379 		 * system.  We do not block here if it is the update
2380 		 * or syncer daemon trying to clean up as that can
2381 		 * lead to deadlock.
2382 		 */
2383 		if ((curthread->td_pflags & TDP_NORUNNINGBUF) == 0 && !vp_md)
2384 			waitrunningbufspace();
2385 	}
2386 
2387 	return (0);
2388 }
2389 
2390 void
bufbdflush(struct bufobj * bo,struct buf * bp)2391 bufbdflush(struct bufobj *bo, struct buf *bp)
2392 {
2393 	struct buf *nbp;
2394 	struct bufdomain *bd;
2395 
2396 	bd = &bdomain[bo->bo_domain];
2397 	if (bo->bo_dirty.bv_cnt > bd->bd_dirtybufthresh + 10) {
2398 		(void) VOP_FSYNC(bp->b_vp, MNT_NOWAIT, curthread);
2399 		altbufferflushes++;
2400 	} else if (bo->bo_dirty.bv_cnt > bd->bd_dirtybufthresh) {
2401 		BO_LOCK(bo);
2402 		/*
2403 		 * Try to find a buffer to flush.
2404 		 */
2405 		TAILQ_FOREACH(nbp, &bo->bo_dirty.bv_hd, b_bobufs) {
2406 			if ((nbp->b_vflags & BV_BKGRDINPROG) ||
2407 			    BUF_LOCK(nbp,
2408 				     LK_EXCLUSIVE | LK_NOWAIT, NULL))
2409 				continue;
2410 			if (bp == nbp)
2411 				panic("bdwrite: found ourselves");
2412 			BO_UNLOCK(bo);
2413 			/* Don't countdeps with the bo lock held. */
2414 			if (buf_countdeps(nbp, 0)) {
2415 				BO_LOCK(bo);
2416 				BUF_UNLOCK(nbp);
2417 				continue;
2418 			}
2419 			if (nbp->b_flags & B_CLUSTEROK) {
2420 				vfs_bio_awrite(nbp);
2421 			} else {
2422 				bremfree(nbp);
2423 				bawrite(nbp);
2424 			}
2425 			dirtybufferflushes++;
2426 			break;
2427 		}
2428 		if (nbp == NULL)
2429 			BO_UNLOCK(bo);
2430 	}
2431 }
2432 
2433 /*
2434  * Delayed write. (Buffer is marked dirty).  Do not bother writing
2435  * anything if the buffer is marked invalid.
2436  *
2437  * Note that since the buffer must be completely valid, we can safely
2438  * set B_CACHE.  In fact, we have to set B_CACHE here rather then in
2439  * biodone() in order to prevent getblk from writing the buffer
2440  * out synchronously.
2441  */
2442 void
bdwrite(struct buf * bp)2443 bdwrite(struct buf *bp)
2444 {
2445 	struct thread *td = curthread;
2446 	struct vnode *vp;
2447 	struct bufobj *bo;
2448 
2449 	CTR3(KTR_BUF, "bdwrite(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2450 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2451 	KASSERT((bp->b_flags & B_BARRIER) == 0,
2452 	    ("Barrier request in delayed write %p", bp));
2453 
2454 	if (bp->b_flags & B_INVAL) {
2455 		brelse(bp);
2456 		return;
2457 	}
2458 
2459 	/*
2460 	 * If we have too many dirty buffers, don't create any more.
2461 	 * If we are wildly over our limit, then force a complete
2462 	 * cleanup. Otherwise, just keep the situation from getting
2463 	 * out of control. Note that we have to avoid a recursive
2464 	 * disaster and not try to clean up after our own cleanup!
2465 	 */
2466 	vp = bp->b_vp;
2467 	bo = bp->b_bufobj;
2468 	if ((td->td_pflags & (TDP_COWINPROGRESS|TDP_INBDFLUSH)) == 0) {
2469 		td->td_pflags |= TDP_INBDFLUSH;
2470 		BO_BDFLUSH(bo, bp);
2471 		td->td_pflags &= ~TDP_INBDFLUSH;
2472 	} else
2473 		recursiveflushes++;
2474 
2475 	bdirty(bp);
2476 	/*
2477 	 * Set B_CACHE, indicating that the buffer is fully valid.  This is
2478 	 * true even of NFS now.
2479 	 */
2480 	bp->b_flags |= B_CACHE;
2481 
2482 	/*
2483 	 * This bmap keeps the system from needing to do the bmap later,
2484 	 * perhaps when the system is attempting to do a sync.  Since it
2485 	 * is likely that the indirect block -- or whatever other datastructure
2486 	 * that the filesystem needs is still in memory now, it is a good
2487 	 * thing to do this.  Note also, that if the pageout daemon is
2488 	 * requesting a sync -- there might not be enough memory to do
2489 	 * the bmap then...  So, this is important to do.
2490 	 */
2491 	if (vp->v_type != VCHR && bp->b_lblkno == bp->b_blkno) {
2492 		VOP_BMAP(vp, bp->b_lblkno, NULL, &bp->b_blkno, NULL, NULL);
2493 	}
2494 
2495 	buf_track(bp, __func__);
2496 
2497 	/*
2498 	 * Set the *dirty* buffer range based upon the VM system dirty
2499 	 * pages.
2500 	 *
2501 	 * Mark the buffer pages as clean.  We need to do this here to
2502 	 * satisfy the vnode_pager and the pageout daemon, so that it
2503 	 * thinks that the pages have been "cleaned".  Note that since
2504 	 * the pages are in a delayed write buffer -- the VFS layer
2505 	 * "will" see that the pages get written out on the next sync,
2506 	 * or perhaps the cluster will be completed.
2507 	 */
2508 	vfs_clean_pages_dirty_buf(bp);
2509 	bqrelse(bp);
2510 
2511 	/*
2512 	 * note: we cannot initiate I/O from a bdwrite even if we wanted to,
2513 	 * due to the softdep code.
2514 	 */
2515 }
2516 
2517 /*
2518  *	bdirty:
2519  *
2520  *	Turn buffer into delayed write request.  We must clear BIO_READ and
2521  *	B_RELBUF, and we must set B_DELWRI.  We reassign the buffer to
2522  *	itself to properly update it in the dirty/clean lists.  We mark it
2523  *	B_DONE to ensure that any asynchronization of the buffer properly
2524  *	clears B_DONE ( else a panic will occur later ).
2525  *
2526  *	bdirty() is kinda like bdwrite() - we have to clear B_INVAL which
2527  *	might have been set pre-getblk().  Unlike bwrite/bdwrite, bdirty()
2528  *	should only be called if the buffer is known-good.
2529  *
2530  *	Since the buffer is not on a queue, we do not update the numfreebuffers
2531  *	count.
2532  *
2533  *	The buffer must be on QUEUE_NONE.
2534  */
2535 void
bdirty(struct buf * bp)2536 bdirty(struct buf *bp)
2537 {
2538 
2539 	CTR3(KTR_BUF, "bdirty(%p) vp %p flags %X",
2540 	    bp, bp->b_vp, bp->b_flags);
2541 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2542 	KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
2543 	    ("bdirty: buffer %p still on queue %d", bp, bp->b_qindex));
2544 	bp->b_flags &= ~(B_RELBUF);
2545 	bp->b_iocmd = BIO_WRITE;
2546 
2547 	if ((bp->b_flags & B_DELWRI) == 0) {
2548 		bp->b_flags |= /* XXX B_DONE | */ B_DELWRI;
2549 		reassignbuf(bp);
2550 		bdirtyadd(bp);
2551 	}
2552 }
2553 
2554 /*
2555  *	bundirty:
2556  *
2557  *	Clear B_DELWRI for buffer.
2558  *
2559  *	Since the buffer is not on a queue, we do not update the numfreebuffers
2560  *	count.
2561  *
2562  *	The buffer must be on QUEUE_NONE.
2563  */
2564 
2565 void
bundirty(struct buf * bp)2566 bundirty(struct buf *bp)
2567 {
2568 
2569 	CTR3(KTR_BUF, "bundirty(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2570 	KASSERT(bp->b_bufobj != NULL, ("No b_bufobj %p", bp));
2571 	KASSERT(bp->b_flags & B_REMFREE || bp->b_qindex == QUEUE_NONE,
2572 	    ("bundirty: buffer %p still on queue %d", bp, bp->b_qindex));
2573 
2574 	if (bp->b_flags & B_DELWRI) {
2575 		bp->b_flags &= ~B_DELWRI;
2576 		reassignbuf(bp);
2577 		bdirtysub(bp);
2578 	}
2579 	/*
2580 	 * Since it is now being written, we can clear its deferred write flag.
2581 	 */
2582 	bp->b_flags &= ~B_DEFERRED;
2583 }
2584 
2585 /*
2586  *	bawrite:
2587  *
2588  *	Asynchronous write.  Start output on a buffer, but do not wait for
2589  *	it to complete.  The buffer is released when the output completes.
2590  *
2591  *	bwrite() ( or the VOP routine anyway ) is responsible for handling
2592  *	B_INVAL buffers.  Not us.
2593  */
2594 void
bawrite(struct buf * bp)2595 bawrite(struct buf *bp)
2596 {
2597 
2598 	bp->b_flags |= B_ASYNC;
2599 	(void) bwrite(bp);
2600 }
2601 
2602 /*
2603  *	babarrierwrite:
2604  *
2605  *	Asynchronous barrier write.  Start output on a buffer, but do not
2606  *	wait for it to complete.  Place a write barrier after this write so
2607  *	that this buffer and all buffers written before it are committed to
2608  *	the disk before any buffers written after this write are committed
2609  *	to the disk.  The buffer is released when the output completes.
2610  */
2611 void
babarrierwrite(struct buf * bp)2612 babarrierwrite(struct buf *bp)
2613 {
2614 
2615 	bp->b_flags |= B_ASYNC | B_BARRIER;
2616 	(void) bwrite(bp);
2617 }
2618 
2619 /*
2620  *	bbarrierwrite:
2621  *
2622  *	Synchronous barrier write.  Start output on a buffer and wait for
2623  *	it to complete.  Place a write barrier after this write so that
2624  *	this buffer and all buffers written before it are committed to
2625  *	the disk before any buffers written after this write are committed
2626  *	to the disk.  The buffer is released when the output completes.
2627  */
2628 int
bbarrierwrite(struct buf * bp)2629 bbarrierwrite(struct buf *bp)
2630 {
2631 
2632 	bp->b_flags |= B_BARRIER;
2633 	return (bwrite(bp));
2634 }
2635 
2636 /*
2637  *	bwillwrite:
2638  *
2639  *	Called prior to the locking of any vnodes when we are expecting to
2640  *	write.  We do not want to starve the buffer cache with too many
2641  *	dirty buffers so we block here.  By blocking prior to the locking
2642  *	of any vnodes we attempt to avoid the situation where a locked vnode
2643  *	prevents the various system daemons from flushing related buffers.
2644  */
2645 void
bwillwrite(void)2646 bwillwrite(void)
2647 {
2648 
2649 	if (buf_dirty_count_severe()) {
2650 		mtx_lock(&bdirtylock);
2651 		while (buf_dirty_count_severe()) {
2652 			bdirtywait = 1;
2653 			msleep(&bdirtywait, &bdirtylock, (PRIBIO + 4),
2654 			    "flswai", 0);
2655 		}
2656 		mtx_unlock(&bdirtylock);
2657 	}
2658 }
2659 
2660 /*
2661  * Return true if we have too many dirty buffers.
2662  */
2663 int
buf_dirty_count_severe(void)2664 buf_dirty_count_severe(void)
2665 {
2666 
2667 	return (!BIT_EMPTY(BUF_DOMAINS, &bdhidirty));
2668 }
2669 
2670 /*
2671  *	brelse:
2672  *
2673  *	Release a busy buffer and, if requested, free its resources.  The
2674  *	buffer will be stashed in the appropriate bufqueue[] allowing it
2675  *	to be accessed later as a cache entity or reused for other purposes.
2676  */
2677 void
brelse(struct buf * bp)2678 brelse(struct buf *bp)
2679 {
2680 	struct mount *v_mnt;
2681 	int qindex;
2682 
2683 	/*
2684 	 * Many functions erroneously call brelse with a NULL bp under rare
2685 	 * error conditions. Simply return when called with a NULL bp.
2686 	 */
2687 	if (bp == NULL)
2688 		return;
2689 	CTR3(KTR_BUF, "brelse(%p) vp %p flags %X",
2690 	    bp, bp->b_vp, bp->b_flags);
2691 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
2692 	    ("brelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
2693 	KASSERT((bp->b_flags & B_VMIO) != 0 || (bp->b_flags & B_NOREUSE) == 0,
2694 	    ("brelse: non-VMIO buffer marked NOREUSE"));
2695 
2696 	if (BUF_LOCKRECURSED(bp)) {
2697 		/*
2698 		 * Do not process, in particular, do not handle the
2699 		 * B_INVAL/B_RELBUF and do not release to free list.
2700 		 */
2701 		BUF_UNLOCK(bp);
2702 		return;
2703 	}
2704 
2705 	if (bp->b_flags & B_MANAGED) {
2706 		bqrelse(bp);
2707 		return;
2708 	}
2709 
2710 	if (LIST_EMPTY(&bp->b_dep)) {
2711 		bp->b_flags &= ~B_IOSTARTED;
2712 	} else {
2713 		KASSERT((bp->b_flags & B_IOSTARTED) == 0,
2714 		    ("brelse: SU io not finished bp %p", bp));
2715 	}
2716 
2717 	if ((bp->b_vflags & (BV_BKGRDINPROG | BV_BKGRDERR)) == BV_BKGRDERR) {
2718 		BO_LOCK(bp->b_bufobj);
2719 		bp->b_vflags &= ~BV_BKGRDERR;
2720 		BO_UNLOCK(bp->b_bufobj);
2721 		bdirty(bp);
2722 	}
2723 
2724 	if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
2725 	    (bp->b_flags & B_INVALONERR)) {
2726 		/*
2727 		 * Forced invalidation of dirty buffer contents, to be used
2728 		 * after a failed write in the rare case that the loss of the
2729 		 * contents is acceptable.  The buffer is invalidated and
2730 		 * freed.
2731 		 */
2732 		bp->b_flags |= B_INVAL | B_RELBUF | B_NOCACHE;
2733 		bp->b_flags &= ~(B_ASYNC | B_CACHE);
2734 	}
2735 
2736 	if (bp->b_iocmd == BIO_WRITE && (bp->b_ioflags & BIO_ERROR) &&
2737 	    (bp->b_error != ENXIO || !LIST_EMPTY(&bp->b_dep)) &&
2738 	    !(bp->b_flags & B_INVAL)) {
2739 		/*
2740 		 * Failed write, redirty.  All errors except ENXIO (which
2741 		 * means the device is gone) are treated as being
2742 		 * transient.
2743 		 *
2744 		 * XXX Treating EIO as transient is not correct; the
2745 		 * contract with the local storage device drivers is that
2746 		 * they will only return EIO once the I/O is no longer
2747 		 * retriable.  Network I/O also respects this through the
2748 		 * guarantees of TCP and/or the internal retries of NFS.
2749 		 * ENOMEM might be transient, but we also have no way of
2750 		 * knowing when its ok to retry/reschedule.  In general,
2751 		 * this entire case should be made obsolete through better
2752 		 * error handling/recovery and resource scheduling.
2753 		 *
2754 		 * Do this also for buffers that failed with ENXIO, but have
2755 		 * non-empty dependencies - the soft updates code might need
2756 		 * to access the buffer to untangle them.
2757 		 *
2758 		 * Must clear BIO_ERROR to prevent pages from being scrapped.
2759 		 */
2760 		bp->b_ioflags &= ~BIO_ERROR;
2761 		bdirty(bp);
2762 	} else if ((bp->b_flags & (B_NOCACHE | B_INVAL)) ||
2763 	    (bp->b_ioflags & BIO_ERROR) || (bp->b_bufsize <= 0)) {
2764 		/*
2765 		 * Either a failed read I/O, or we were asked to free or not
2766 		 * cache the buffer, or we failed to write to a device that's
2767 		 * no longer present.
2768 		 */
2769 		bp->b_flags |= B_INVAL;
2770 		if (!LIST_EMPTY(&bp->b_dep))
2771 			buf_deallocate(bp);
2772 		if (bp->b_flags & B_DELWRI)
2773 			bdirtysub(bp);
2774 		bp->b_flags &= ~(B_DELWRI | B_CACHE);
2775 		if ((bp->b_flags & B_VMIO) == 0) {
2776 			allocbuf(bp, 0);
2777 			if (bp->b_vp)
2778 				brelvp(bp);
2779 		}
2780 	}
2781 
2782 	/*
2783 	 * We must clear B_RELBUF if B_DELWRI is set.  If vfs_vmio_truncate()
2784 	 * is called with B_DELWRI set, the underlying pages may wind up
2785 	 * getting freed causing a previous write (bdwrite()) to get 'lost'
2786 	 * because pages associated with a B_DELWRI bp are marked clean.
2787 	 *
2788 	 * We still allow the B_INVAL case to call vfs_vmio_truncate(), even
2789 	 * if B_DELWRI is set.
2790 	 */
2791 	if (bp->b_flags & B_DELWRI)
2792 		bp->b_flags &= ~B_RELBUF;
2793 
2794 	/*
2795 	 * VMIO buffer rundown.  It is not very necessary to keep a VMIO buffer
2796 	 * constituted, not even NFS buffers now.  Two flags effect this.  If
2797 	 * B_INVAL, the struct buf is invalidated but the VM object is kept
2798 	 * around ( i.e. so it is trivial to reconstitute the buffer later ).
2799 	 *
2800 	 * If BIO_ERROR or B_NOCACHE is set, pages in the VM object will be
2801 	 * invalidated.  BIO_ERROR cannot be set for a failed write unless the
2802 	 * buffer is also B_INVAL because it hits the re-dirtying code above.
2803 	 *
2804 	 * Normally we can do this whether a buffer is B_DELWRI or not.  If
2805 	 * the buffer is an NFS buffer, it is tracking piecemeal writes or
2806 	 * the commit state and we cannot afford to lose the buffer. If the
2807 	 * buffer has a background write in progress, we need to keep it
2808 	 * around to prevent it from being reconstituted and starting a second
2809 	 * background write.
2810 	 */
2811 
2812 	v_mnt = bp->b_vp != NULL ? bp->b_vp->v_mount : NULL;
2813 
2814 	if ((bp->b_flags & B_VMIO) && (bp->b_flags & B_NOCACHE ||
2815 	    (bp->b_ioflags & BIO_ERROR && bp->b_iocmd == BIO_READ)) &&
2816 	    (v_mnt == NULL || (v_mnt->mnt_vfc->vfc_flags & VFCF_NETWORK) == 0 ||
2817 	    vn_isdisk(bp->b_vp) || (bp->b_flags & B_DELWRI) == 0)) {
2818 		vfs_vmio_invalidate(bp);
2819 		allocbuf(bp, 0);
2820 	}
2821 
2822 	if ((bp->b_flags & (B_INVAL | B_RELBUF)) != 0 ||
2823 	    (bp->b_flags & (B_DELWRI | B_NOREUSE)) == B_NOREUSE) {
2824 		allocbuf(bp, 0);
2825 		bp->b_flags &= ~B_NOREUSE;
2826 		if (bp->b_vp != NULL)
2827 			brelvp(bp);
2828 	}
2829 
2830 	/*
2831 	 * If the buffer has junk contents signal it and eventually
2832 	 * clean up B_DELWRI and diassociate the vnode so that gbincore()
2833 	 * doesn't find it.
2834 	 */
2835 	if (bp->b_bufsize == 0 || (bp->b_ioflags & BIO_ERROR) != 0 ||
2836 	    (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF)) != 0)
2837 		bp->b_flags |= B_INVAL;
2838 	if (bp->b_flags & B_INVAL) {
2839 		if (bp->b_flags & B_DELWRI)
2840 			bundirty(bp);
2841 		if (bp->b_vp)
2842 			brelvp(bp);
2843 	}
2844 
2845 	buf_track(bp, __func__);
2846 
2847 	/* buffers with no memory */
2848 	if (bp->b_bufsize == 0) {
2849 		buf_free(bp);
2850 		return;
2851 	}
2852 	/* buffers with junk contents */
2853 	if (bp->b_flags & (B_INVAL | B_NOCACHE | B_RELBUF) ||
2854 	    (bp->b_ioflags & BIO_ERROR)) {
2855 		bp->b_xflags &= ~(BX_BKGRDWRITE | BX_ALTDATA);
2856 		if (bp->b_vflags & BV_BKGRDINPROG)
2857 			panic("losing buffer 2");
2858 		qindex = QUEUE_CLEAN;
2859 		bp->b_flags |= B_AGE;
2860 	/* remaining buffers */
2861 	} else if (bp->b_flags & B_DELWRI)
2862 		qindex = QUEUE_DIRTY;
2863 	else
2864 		qindex = QUEUE_CLEAN;
2865 
2866 	if ((bp->b_flags & B_DELWRI) == 0 && (bp->b_xflags & BX_VNDIRTY))
2867 		panic("brelse: not dirty");
2868 
2869 	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_RELBUF | B_DIRECT);
2870 	bp->b_xflags &= ~(BX_CVTENXIO);
2871 	/* binsfree unlocks bp. */
2872 	binsfree(bp, qindex);
2873 }
2874 
2875 /*
2876  * Release a buffer back to the appropriate queue but do not try to free
2877  * it.  The buffer is expected to be used again soon.
2878  *
2879  * bqrelse() is used by bdwrite() to requeue a delayed write, and used by
2880  * biodone() to requeue an async I/O on completion.  It is also used when
2881  * known good buffers need to be requeued but we think we may need the data
2882  * again soon.
2883  *
2884  * XXX we should be able to leave the B_RELBUF hint set on completion.
2885  */
2886 void
bqrelse(struct buf * bp)2887 bqrelse(struct buf *bp)
2888 {
2889 	int qindex;
2890 
2891 	CTR3(KTR_BUF, "bqrelse(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
2892 	KASSERT(!(bp->b_flags & (B_CLUSTER|B_PAGING)),
2893 	    ("bqrelse: inappropriate B_PAGING or B_CLUSTER bp %p", bp));
2894 
2895 	qindex = QUEUE_NONE;
2896 	if (BUF_LOCKRECURSED(bp)) {
2897 		/* do not release to free list */
2898 		BUF_UNLOCK(bp);
2899 		return;
2900 	}
2901 	bp->b_flags &= ~(B_ASYNC | B_NOCACHE | B_AGE | B_RELBUF);
2902 	bp->b_xflags &= ~(BX_CVTENXIO);
2903 
2904 	if (LIST_EMPTY(&bp->b_dep)) {
2905 		bp->b_flags &= ~B_IOSTARTED;
2906 	} else {
2907 		KASSERT((bp->b_flags & B_IOSTARTED) == 0,
2908 		    ("bqrelse: SU io not finished bp %p", bp));
2909 	}
2910 
2911 	if (bp->b_flags & B_MANAGED) {
2912 		if (bp->b_flags & B_REMFREE)
2913 			bremfreef(bp);
2914 		goto out;
2915 	}
2916 
2917 	/* buffers with stale but valid contents */
2918 	if ((bp->b_flags & B_DELWRI) != 0 || (bp->b_vflags & (BV_BKGRDINPROG |
2919 	    BV_BKGRDERR)) == BV_BKGRDERR) {
2920 		BO_LOCK(bp->b_bufobj);
2921 		bp->b_vflags &= ~BV_BKGRDERR;
2922 		BO_UNLOCK(bp->b_bufobj);
2923 		qindex = QUEUE_DIRTY;
2924 	} else {
2925 		if ((bp->b_flags & B_DELWRI) == 0 &&
2926 		    (bp->b_xflags & BX_VNDIRTY))
2927 			panic("bqrelse: not dirty");
2928 		if ((bp->b_flags & B_NOREUSE) != 0) {
2929 			brelse(bp);
2930 			return;
2931 		}
2932 		qindex = QUEUE_CLEAN;
2933 	}
2934 	buf_track(bp, __func__);
2935 	/* binsfree unlocks bp. */
2936 	binsfree(bp, qindex);
2937 	return;
2938 
2939 out:
2940 	buf_track(bp, __func__);
2941 	/* unlock */
2942 	BUF_UNLOCK(bp);
2943 }
2944 
2945 /*
2946  * Complete I/O to a VMIO backed page.  Validate the pages as appropriate,
2947  * restore bogus pages.
2948  */
2949 static void
vfs_vmio_iodone(struct buf * bp)2950 vfs_vmio_iodone(struct buf *bp)
2951 {
2952 	vm_ooffset_t foff;
2953 	vm_page_t m;
2954 	vm_object_t obj;
2955 	struct vnode *vp __unused;
2956 	int i, iosize, resid;
2957 	bool bogus;
2958 
2959 	obj = bp->b_bufobj->bo_object;
2960 	KASSERT(blockcount_read(&obj->paging_in_progress) >= bp->b_npages,
2961 	    ("vfs_vmio_iodone: paging in progress(%d) < b_npages(%d)",
2962 	    blockcount_read(&obj->paging_in_progress), bp->b_npages));
2963 
2964 	vp = bp->b_vp;
2965 	VNPASS(vp->v_holdcnt > 0, vp);
2966 	VNPASS(vp->v_object != NULL, vp);
2967 
2968 	foff = bp->b_offset;
2969 	KASSERT(bp->b_offset != NOOFFSET,
2970 	    ("vfs_vmio_iodone: bp %p has no buffer offset", bp));
2971 
2972 	bogus = false;
2973 	iosize = bp->b_bcount - bp->b_resid;
2974 	for (i = 0; i < bp->b_npages; i++) {
2975 		resid = ((foff + PAGE_SIZE) & ~(off_t)PAGE_MASK) - foff;
2976 		if (resid > iosize)
2977 			resid = iosize;
2978 
2979 		/*
2980 		 * cleanup bogus pages, restoring the originals
2981 		 */
2982 		m = bp->b_pages[i];
2983 		if (m == bogus_page) {
2984 			bogus = true;
2985 			m = vm_page_relookup(obj, OFF_TO_IDX(foff));
2986 			if (m == NULL)
2987 				panic("biodone: page disappeared!");
2988 			bp->b_pages[i] = m;
2989 		} else if ((bp->b_iocmd == BIO_READ) && resid > 0) {
2990 			/*
2991 			 * In the write case, the valid and clean bits are
2992 			 * already changed correctly ( see bdwrite() ), so we
2993 			 * only need to do this here in the read case.
2994 			 */
2995 			KASSERT((m->dirty & vm_page_bits(foff & PAGE_MASK,
2996 			    resid)) == 0, ("vfs_vmio_iodone: page %p "
2997 			    "has unexpected dirty bits", m));
2998 			vfs_page_set_valid(bp, foff, m);
2999 		}
3000 		KASSERT(OFF_TO_IDX(foff) == m->pindex,
3001 		    ("vfs_vmio_iodone: foff(%jd)/pindex(%ju) mismatch",
3002 		    (intmax_t)foff, (uintmax_t)m->pindex));
3003 
3004 		vm_page_sunbusy(m);
3005 		foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
3006 		iosize -= resid;
3007 	}
3008 	vm_object_pip_wakeupn(obj, bp->b_npages);
3009 	if (bogus && buf_mapped(bp)) {
3010 		BUF_CHECK_MAPPED(bp);
3011 		pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
3012 		    bp->b_pages, bp->b_npages);
3013 	}
3014 }
3015 
3016 /*
3017  * Perform page invalidation when a buffer is released.  The fully invalid
3018  * pages will be reclaimed later in vfs_vmio_truncate().
3019  */
3020 static void
vfs_vmio_invalidate(struct buf * bp)3021 vfs_vmio_invalidate(struct buf *bp)
3022 {
3023 	vm_object_t obj;
3024 	vm_page_t m;
3025 	int flags, i, resid, poffset, presid;
3026 
3027 	if (buf_mapped(bp)) {
3028 		BUF_CHECK_MAPPED(bp);
3029 		pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
3030 	} else
3031 		BUF_CHECK_UNMAPPED(bp);
3032 	/*
3033 	 * Get the base offset and length of the buffer.  Note that
3034 	 * in the VMIO case if the buffer block size is not
3035 	 * page-aligned then b_data pointer may not be page-aligned.
3036 	 * But our b_pages[] array *IS* page aligned.
3037 	 *
3038 	 * block sizes less then DEV_BSIZE (usually 512) are not
3039 	 * supported due to the page granularity bits (m->valid,
3040 	 * m->dirty, etc...).
3041 	 *
3042 	 * See man buf(9) for more information
3043 	 */
3044 	flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
3045 	obj = bp->b_bufobj->bo_object;
3046 	resid = bp->b_bufsize;
3047 	poffset = bp->b_offset & PAGE_MASK;
3048 	VM_OBJECT_WLOCK(obj);
3049 	for (i = 0; i < bp->b_npages; i++) {
3050 		m = bp->b_pages[i];
3051 		if (m == bogus_page)
3052 			panic("vfs_vmio_invalidate: Unexpected bogus page.");
3053 		bp->b_pages[i] = NULL;
3054 
3055 		presid = resid > (PAGE_SIZE - poffset) ?
3056 		    (PAGE_SIZE - poffset) : resid;
3057 		KASSERT(presid >= 0, ("brelse: extra page"));
3058 		vm_page_busy_acquire(m, VM_ALLOC_SBUSY);
3059 		if (pmap_page_wired_mappings(m) == 0)
3060 			vm_page_set_invalid(m, poffset, presid);
3061 		vm_page_sunbusy(m);
3062 		vm_page_release_locked(m, flags);
3063 		resid -= presid;
3064 		poffset = 0;
3065 	}
3066 	VM_OBJECT_WUNLOCK(obj);
3067 	bp->b_npages = 0;
3068 }
3069 
3070 /*
3071  * Page-granular truncation of an existing VMIO buffer.
3072  */
3073 static void
vfs_vmio_truncate(struct buf * bp,int desiredpages)3074 vfs_vmio_truncate(struct buf *bp, int desiredpages)
3075 {
3076 	vm_object_t obj;
3077 	vm_page_t m;
3078 	int flags, i;
3079 
3080 	if (bp->b_npages == desiredpages)
3081 		return;
3082 
3083 	if (buf_mapped(bp)) {
3084 		BUF_CHECK_MAPPED(bp);
3085 		pmap_qremove((vm_offset_t)trunc_page((vm_offset_t)bp->b_data) +
3086 		    (desiredpages << PAGE_SHIFT), bp->b_npages - desiredpages);
3087 	} else
3088 		BUF_CHECK_UNMAPPED(bp);
3089 
3090 	/*
3091 	 * The object lock is needed only if we will attempt to free pages.
3092 	 */
3093 	flags = (bp->b_flags & B_NOREUSE) != 0 ? VPR_NOREUSE : 0;
3094 	if ((bp->b_flags & B_DIRECT) != 0) {
3095 		flags |= VPR_TRYFREE;
3096 		obj = bp->b_bufobj->bo_object;
3097 		VM_OBJECT_WLOCK(obj);
3098 	} else {
3099 		obj = NULL;
3100 	}
3101 	for (i = desiredpages; i < bp->b_npages; i++) {
3102 		m = bp->b_pages[i];
3103 		KASSERT(m != bogus_page, ("allocbuf: bogus page found"));
3104 		bp->b_pages[i] = NULL;
3105 		if (obj != NULL)
3106 			vm_page_release_locked(m, flags);
3107 		else
3108 			vm_page_release(m, flags);
3109 	}
3110 	if (obj != NULL)
3111 		VM_OBJECT_WUNLOCK(obj);
3112 	bp->b_npages = desiredpages;
3113 }
3114 
3115 /*
3116  * Byte granular extension of VMIO buffers.
3117  */
3118 static void
vfs_vmio_extend(struct buf * bp,int desiredpages,int size)3119 vfs_vmio_extend(struct buf *bp, int desiredpages, int size)
3120 {
3121 	/*
3122 	 * We are growing the buffer, possibly in a
3123 	 * byte-granular fashion.
3124 	 */
3125 	vm_object_t obj;
3126 	vm_offset_t toff;
3127 	vm_offset_t tinc;
3128 	vm_page_t m;
3129 
3130 	/*
3131 	 * Step 1, bring in the VM pages from the object, allocating
3132 	 * them if necessary.  We must clear B_CACHE if these pages
3133 	 * are not valid for the range covered by the buffer.
3134 	 */
3135 	obj = bp->b_bufobj->bo_object;
3136 	if (bp->b_npages < desiredpages) {
3137 		KASSERT(desiredpages <= atop(maxbcachebuf),
3138 		    ("vfs_vmio_extend past maxbcachebuf %p %d %u",
3139 		    bp, desiredpages, maxbcachebuf));
3140 
3141 		/*
3142 		 * We must allocate system pages since blocking
3143 		 * here could interfere with paging I/O, no
3144 		 * matter which process we are.
3145 		 *
3146 		 * Only exclusive busy can be tested here.
3147 		 * Blocking on shared busy might lead to
3148 		 * deadlocks once allocbuf() is called after
3149 		 * pages are vfs_busy_pages().
3150 		 */
3151 		(void)vm_page_grab_pages_unlocked(obj,
3152 		    OFF_TO_IDX(bp->b_offset) + bp->b_npages,
3153 		    VM_ALLOC_SYSTEM | VM_ALLOC_IGN_SBUSY |
3154 		    VM_ALLOC_NOBUSY | VM_ALLOC_WIRED,
3155 		    &bp->b_pages[bp->b_npages], desiredpages - bp->b_npages);
3156 		bp->b_npages = desiredpages;
3157 	}
3158 
3159 	/*
3160 	 * Step 2.  We've loaded the pages into the buffer,
3161 	 * we have to figure out if we can still have B_CACHE
3162 	 * set.  Note that B_CACHE is set according to the
3163 	 * byte-granular range ( bcount and size ), not the
3164 	 * aligned range ( newbsize ).
3165 	 *
3166 	 * The VM test is against m->valid, which is DEV_BSIZE
3167 	 * aligned.  Needless to say, the validity of the data
3168 	 * needs to also be DEV_BSIZE aligned.  Note that this
3169 	 * fails with NFS if the server or some other client
3170 	 * extends the file's EOF.  If our buffer is resized,
3171 	 * B_CACHE may remain set! XXX
3172 	 */
3173 	toff = bp->b_bcount;
3174 	tinc = PAGE_SIZE - ((bp->b_offset + toff) & PAGE_MASK);
3175 	while ((bp->b_flags & B_CACHE) && toff < size) {
3176 		vm_pindex_t pi;
3177 
3178 		if (tinc > (size - toff))
3179 			tinc = size - toff;
3180 		pi = ((bp->b_offset & PAGE_MASK) + toff) >> PAGE_SHIFT;
3181 		m = bp->b_pages[pi];
3182 		vfs_buf_test_cache(bp, bp->b_offset, toff, tinc, m);
3183 		toff += tinc;
3184 		tinc = PAGE_SIZE;
3185 	}
3186 
3187 	/*
3188 	 * Step 3, fixup the KVA pmap.
3189 	 */
3190 	if (buf_mapped(bp))
3191 		bpmap_qenter(bp);
3192 	else
3193 		BUF_CHECK_UNMAPPED(bp);
3194 }
3195 
3196 /*
3197  * Check to see if a block at a particular lbn is available for a clustered
3198  * write.
3199  */
3200 static int
vfs_bio_clcheck(struct vnode * vp,int size,daddr_t lblkno,daddr_t blkno)3201 vfs_bio_clcheck(struct vnode *vp, int size, daddr_t lblkno, daddr_t blkno)
3202 {
3203 	struct buf *bpa;
3204 	int match;
3205 
3206 	match = 0;
3207 
3208 	/* If the buf isn't in core skip it */
3209 	if ((bpa = gbincore(&vp->v_bufobj, lblkno)) == NULL)
3210 		return (0);
3211 
3212 	/* If the buf is busy we don't want to wait for it */
3213 	if (BUF_LOCK(bpa, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
3214 		return (0);
3215 
3216 	/* Only cluster with valid clusterable delayed write buffers */
3217 	if ((bpa->b_flags & (B_DELWRI | B_CLUSTEROK | B_INVAL)) !=
3218 	    (B_DELWRI | B_CLUSTEROK))
3219 		goto done;
3220 
3221 	if (bpa->b_bufsize != size)
3222 		goto done;
3223 
3224 	/*
3225 	 * Check to see if it is in the expected place on disk and that the
3226 	 * block has been mapped.
3227 	 */
3228 	if ((bpa->b_blkno != bpa->b_lblkno) && (bpa->b_blkno == blkno))
3229 		match = 1;
3230 done:
3231 	BUF_UNLOCK(bpa);
3232 	return (match);
3233 }
3234 
3235 /*
3236  *	vfs_bio_awrite:
3237  *
3238  *	Implement clustered async writes for clearing out B_DELWRI buffers.
3239  *	This is much better then the old way of writing only one buffer at
3240  *	a time.  Note that we may not be presented with the buffers in the
3241  *	correct order, so we search for the cluster in both directions.
3242  */
3243 int
vfs_bio_awrite(struct buf * bp)3244 vfs_bio_awrite(struct buf *bp)
3245 {
3246 	struct bufobj *bo;
3247 	int i;
3248 	int j;
3249 	daddr_t lblkno = bp->b_lblkno;
3250 	struct vnode *vp = bp->b_vp;
3251 	int ncl;
3252 	int nwritten;
3253 	int size;
3254 	int maxcl;
3255 	int gbflags;
3256 
3257 	bo = &vp->v_bufobj;
3258 	gbflags = (bp->b_data == unmapped_buf) ? GB_UNMAPPED : 0;
3259 	/*
3260 	 * right now we support clustered writing only to regular files.  If
3261 	 * we find a clusterable block we could be in the middle of a cluster
3262 	 * rather then at the beginning.
3263 	 */
3264 	if ((vp->v_type == VREG) &&
3265 	    (vp->v_mount != 0) && /* Only on nodes that have the size info */
3266 	    (bp->b_flags & (B_CLUSTEROK | B_INVAL)) == B_CLUSTEROK) {
3267 		size = vp->v_mount->mnt_stat.f_iosize;
3268 		maxcl = maxphys / size;
3269 
3270 		BO_RLOCK(bo);
3271 		for (i = 1; i < maxcl; i++)
3272 			if (vfs_bio_clcheck(vp, size, lblkno + i,
3273 			    bp->b_blkno + ((i * size) >> DEV_BSHIFT)) == 0)
3274 				break;
3275 
3276 		for (j = 1; i + j <= maxcl && j <= lblkno; j++)
3277 			if (vfs_bio_clcheck(vp, size, lblkno - j,
3278 			    bp->b_blkno - ((j * size) >> DEV_BSHIFT)) == 0)
3279 				break;
3280 		BO_RUNLOCK(bo);
3281 		--j;
3282 		ncl = i + j;
3283 		/*
3284 		 * this is a possible cluster write
3285 		 */
3286 		if (ncl != 1) {
3287 			BUF_UNLOCK(bp);
3288 			nwritten = cluster_wbuild(vp, size, lblkno - j, ncl,
3289 			    gbflags);
3290 			return (nwritten);
3291 		}
3292 	}
3293 	bremfree(bp);
3294 	bp->b_flags |= B_ASYNC;
3295 	/*
3296 	 * default (old) behavior, writing out only one block
3297 	 *
3298 	 * XXX returns b_bufsize instead of b_bcount for nwritten?
3299 	 */
3300 	nwritten = bp->b_bufsize;
3301 	(void) bwrite(bp);
3302 
3303 	return (nwritten);
3304 }
3305 
3306 /*
3307  *	getnewbuf_kva:
3308  *
3309  *	Allocate KVA for an empty buf header according to gbflags.
3310  */
3311 static int
getnewbuf_kva(struct buf * bp,int gbflags,int maxsize)3312 getnewbuf_kva(struct buf *bp, int gbflags, int maxsize)
3313 {
3314 
3315 	if ((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_UNMAPPED) {
3316 		/*
3317 		 * In order to keep fragmentation sane we only allocate kva
3318 		 * in BKVASIZE chunks.  XXX with vmem we can do page size.
3319 		 */
3320 		maxsize = (maxsize + BKVAMASK) & ~BKVAMASK;
3321 
3322 		if (maxsize != bp->b_kvasize &&
3323 		    bufkva_alloc(bp, maxsize, gbflags))
3324 			return (ENOSPC);
3325 	}
3326 	return (0);
3327 }
3328 
3329 /*
3330  *	getnewbuf:
3331  *
3332  *	Find and initialize a new buffer header, freeing up existing buffers
3333  *	in the bufqueues as necessary.  The new buffer is returned locked.
3334  *
3335  *	We block if:
3336  *		We have insufficient buffer headers
3337  *		We have insufficient buffer space
3338  *		buffer_arena is too fragmented ( space reservation fails )
3339  *		If we have to flush dirty buffers ( but we try to avoid this )
3340  *
3341  *	The caller is responsible for releasing the reserved bufspace after
3342  *	allocbuf() is called.
3343  */
3344 static struct buf *
getnewbuf(struct vnode * vp,int slpflag,int slptimeo,int maxsize,int gbflags)3345 getnewbuf(struct vnode *vp, int slpflag, int slptimeo, int maxsize, int gbflags)
3346 {
3347 	struct bufdomain *bd;
3348 	struct buf *bp;
3349 	bool metadata, reserved;
3350 
3351 	bp = NULL;
3352 	KASSERT((gbflags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
3353 	    ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
3354 	if (!unmapped_buf_allowed)
3355 		gbflags &= ~(GB_UNMAPPED | GB_KVAALLOC);
3356 
3357 	if (vp == NULL || (vp->v_vflag & (VV_MD | VV_SYSTEM)) != 0 ||
3358 	    vp->v_type == VCHR)
3359 		metadata = true;
3360 	else
3361 		metadata = false;
3362 	if (vp == NULL)
3363 		bd = &bdomain[0];
3364 	else
3365 		bd = &bdomain[vp->v_bufobj.bo_domain];
3366 
3367 	counter_u64_add(getnewbufcalls, 1);
3368 	reserved = false;
3369 	do {
3370 		if (reserved == false &&
3371 		    bufspace_reserve(bd, maxsize, metadata) != 0) {
3372 			counter_u64_add(getnewbufrestarts, 1);
3373 			continue;
3374 		}
3375 		reserved = true;
3376 		if ((bp = buf_alloc(bd)) == NULL) {
3377 			counter_u64_add(getnewbufrestarts, 1);
3378 			continue;
3379 		}
3380 		if (getnewbuf_kva(bp, gbflags, maxsize) == 0)
3381 			return (bp);
3382 		break;
3383 	} while (buf_recycle(bd, false) == 0);
3384 
3385 	if (reserved)
3386 		bufspace_release(bd, maxsize);
3387 	if (bp != NULL) {
3388 		bp->b_flags |= B_INVAL;
3389 		brelse(bp);
3390 	}
3391 	bufspace_wait(bd, vp, gbflags, slpflag, slptimeo);
3392 
3393 	return (NULL);
3394 }
3395 
3396 /*
3397  *	buf_daemon:
3398  *
3399  *	buffer flushing daemon.  Buffers are normally flushed by the
3400  *	update daemon but if it cannot keep up this process starts to
3401  *	take the load in an attempt to prevent getnewbuf() from blocking.
3402  */
3403 static struct kproc_desc buf_kp = {
3404 	"bufdaemon",
3405 	buf_daemon,
3406 	&bufdaemonproc
3407 };
3408 SYSINIT(bufdaemon, SI_SUB_KTHREAD_BUF, SI_ORDER_FIRST, kproc_start, &buf_kp);
3409 
3410 static int
buf_flush(struct vnode * vp,struct bufdomain * bd,int target)3411 buf_flush(struct vnode *vp, struct bufdomain *bd, int target)
3412 {
3413 	int flushed;
3414 
3415 	flushed = flushbufqueues(vp, bd, target, 0);
3416 	if (flushed == 0) {
3417 		/*
3418 		 * Could not find any buffers without rollback
3419 		 * dependencies, so just write the first one
3420 		 * in the hopes of eventually making progress.
3421 		 */
3422 		if (vp != NULL && target > 2)
3423 			target /= 2;
3424 		flushbufqueues(vp, bd, target, 1);
3425 	}
3426 	return (flushed);
3427 }
3428 
3429 static void
buf_daemon_shutdown(void * arg __unused,int howto __unused)3430 buf_daemon_shutdown(void *arg __unused, int howto __unused)
3431 {
3432 	int error;
3433 
3434 	if (KERNEL_PANICKED())
3435 		return;
3436 
3437 	mtx_lock(&bdlock);
3438 	bd_shutdown = true;
3439 	wakeup(&bd_request);
3440 	error = msleep(&bd_shutdown, &bdlock, 0, "buf_daemon_shutdown",
3441 	    60 * hz);
3442 	mtx_unlock(&bdlock);
3443 	if (error != 0)
3444 		printf("bufdaemon wait error: %d\n", error);
3445 }
3446 
3447 static void
buf_daemon(void)3448 buf_daemon(void)
3449 {
3450 	struct bufdomain *bd;
3451 	int speedupreq;
3452 	int lodirty;
3453 	int i;
3454 
3455 	/*
3456 	 * This process needs to be suspended prior to shutdown sync.
3457 	 */
3458 	EVENTHANDLER_REGISTER(shutdown_pre_sync, buf_daemon_shutdown, NULL,
3459 	    SHUTDOWN_PRI_LAST + 100);
3460 
3461 	/*
3462 	 * Start the buf clean daemons as children threads.
3463 	 */
3464 	for (i = 0 ; i < buf_domains; i++) {
3465 		int error;
3466 
3467 		error = kthread_add((void (*)(void *))bufspace_daemon,
3468 		    &bdomain[i], curproc, NULL, 0, 0, "bufspacedaemon-%d", i);
3469 		if (error)
3470 			panic("error %d spawning bufspace daemon", error);
3471 	}
3472 
3473 	/*
3474 	 * This process is allowed to take the buffer cache to the limit
3475 	 */
3476 	curthread->td_pflags |= TDP_NORUNNINGBUF | TDP_BUFNEED;
3477 	mtx_lock(&bdlock);
3478 	while (!bd_shutdown) {
3479 		bd_request = 0;
3480 		mtx_unlock(&bdlock);
3481 
3482 		/*
3483 		 * Save speedupreq for this pass and reset to capture new
3484 		 * requests.
3485 		 */
3486 		speedupreq = bd_speedupreq;
3487 		bd_speedupreq = 0;
3488 
3489 		/*
3490 		 * Flush each domain sequentially according to its level and
3491 		 * the speedup request.
3492 		 */
3493 		for (i = 0; i < buf_domains; i++) {
3494 			bd = &bdomain[i];
3495 			if (speedupreq)
3496 				lodirty = bd->bd_numdirtybuffers / 2;
3497 			else
3498 				lodirty = bd->bd_lodirtybuffers;
3499 			while (bd->bd_numdirtybuffers > lodirty) {
3500 				if (buf_flush(NULL, bd,
3501 				    bd->bd_numdirtybuffers - lodirty) == 0)
3502 					break;
3503 				kern_yield(PRI_USER);
3504 			}
3505 		}
3506 
3507 		/*
3508 		 * Only clear bd_request if we have reached our low water
3509 		 * mark.  The buf_daemon normally waits 1 second and
3510 		 * then incrementally flushes any dirty buffers that have
3511 		 * built up, within reason.
3512 		 *
3513 		 * If we were unable to hit our low water mark and couldn't
3514 		 * find any flushable buffers, we sleep for a short period
3515 		 * to avoid endless loops on unlockable buffers.
3516 		 */
3517 		mtx_lock(&bdlock);
3518 		if (bd_shutdown)
3519 			break;
3520 		if (BIT_EMPTY(BUF_DOMAINS, &bdlodirty)) {
3521 			/*
3522 			 * We reached our low water mark, reset the
3523 			 * request and sleep until we are needed again.
3524 			 * The sleep is just so the suspend code works.
3525 			 */
3526 			bd_request = 0;
3527 			/*
3528 			 * Do an extra wakeup in case dirty threshold
3529 			 * changed via sysctl and the explicit transition
3530 			 * out of shortfall was missed.
3531 			 */
3532 			bdirtywakeup();
3533 			if (runningbufspace <= lorunningspace)
3534 				runningwakeup();
3535 			msleep(&bd_request, &bdlock, PVM, "psleep", hz);
3536 		} else {
3537 			/*
3538 			 * We couldn't find any flushable dirty buffers but
3539 			 * still have too many dirty buffers, we
3540 			 * have to sleep and try again.  (rare)
3541 			 */
3542 			msleep(&bd_request, &bdlock, PVM, "qsleep", hz / 10);
3543 		}
3544 	}
3545 	wakeup(&bd_shutdown);
3546 	mtx_unlock(&bdlock);
3547 	kthread_exit();
3548 }
3549 
3550 /*
3551  *	flushbufqueues:
3552  *
3553  *	Try to flush a buffer in the dirty queue.  We must be careful to
3554  *	free up B_INVAL buffers instead of write them, which NFS is
3555  *	particularly sensitive to.
3556  */
3557 static int flushwithdeps = 0;
3558 SYSCTL_INT(_vfs, OID_AUTO, flushwithdeps, CTLFLAG_RW | CTLFLAG_STATS,
3559     &flushwithdeps, 0,
3560     "Number of buffers flushed with dependencies that require rollbacks");
3561 
3562 static int
flushbufqueues(struct vnode * lvp,struct bufdomain * bd,int target,int flushdeps)3563 flushbufqueues(struct vnode *lvp, struct bufdomain *bd, int target,
3564     int flushdeps)
3565 {
3566 	struct bufqueue *bq;
3567 	struct buf *sentinel;
3568 	struct vnode *vp;
3569 	struct mount *mp;
3570 	struct buf *bp;
3571 	int hasdeps;
3572 	int flushed;
3573 	int error;
3574 	bool unlock;
3575 
3576 	flushed = 0;
3577 	bq = &bd->bd_dirtyq;
3578 	bp = NULL;
3579 	sentinel = malloc(sizeof(struct buf), M_TEMP, M_WAITOK | M_ZERO);
3580 	sentinel->b_qindex = QUEUE_SENTINEL;
3581 	BQ_LOCK(bq);
3582 	TAILQ_INSERT_HEAD(&bq->bq_queue, sentinel, b_freelist);
3583 	BQ_UNLOCK(bq);
3584 	while (flushed != target) {
3585 		maybe_yield();
3586 		BQ_LOCK(bq);
3587 		bp = TAILQ_NEXT(sentinel, b_freelist);
3588 		if (bp != NULL) {
3589 			TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
3590 			TAILQ_INSERT_AFTER(&bq->bq_queue, bp, sentinel,
3591 			    b_freelist);
3592 		} else {
3593 			BQ_UNLOCK(bq);
3594 			break;
3595 		}
3596 		/*
3597 		 * Skip sentinels inserted by other invocations of the
3598 		 * flushbufqueues(), taking care to not reorder them.
3599 		 *
3600 		 * Only flush the buffers that belong to the
3601 		 * vnode locked by the curthread.
3602 		 */
3603 		if (bp->b_qindex == QUEUE_SENTINEL || (lvp != NULL &&
3604 		    bp->b_vp != lvp)) {
3605 			BQ_UNLOCK(bq);
3606 			continue;
3607 		}
3608 		error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL);
3609 		BQ_UNLOCK(bq);
3610 		if (error != 0)
3611 			continue;
3612 
3613 		/*
3614 		 * BKGRDINPROG can only be set with the buf and bufobj
3615 		 * locks both held.  We tolerate a race to clear it here.
3616 		 */
3617 		if ((bp->b_vflags & BV_BKGRDINPROG) != 0 ||
3618 		    (bp->b_flags & B_DELWRI) == 0) {
3619 			BUF_UNLOCK(bp);
3620 			continue;
3621 		}
3622 		if (bp->b_flags & B_INVAL) {
3623 			bremfreef(bp);
3624 			brelse(bp);
3625 			flushed++;
3626 			continue;
3627 		}
3628 
3629 		if (!LIST_EMPTY(&bp->b_dep) && buf_countdeps(bp, 0)) {
3630 			if (flushdeps == 0) {
3631 				BUF_UNLOCK(bp);
3632 				continue;
3633 			}
3634 			hasdeps = 1;
3635 		} else
3636 			hasdeps = 0;
3637 		/*
3638 		 * We must hold the lock on a vnode before writing
3639 		 * one of its buffers. Otherwise we may confuse, or
3640 		 * in the case of a snapshot vnode, deadlock the
3641 		 * system.
3642 		 *
3643 		 * The lock order here is the reverse of the normal
3644 		 * of vnode followed by buf lock.  This is ok because
3645 		 * the NOWAIT will prevent deadlock.
3646 		 */
3647 		vp = bp->b_vp;
3648 		if (vn_start_write(vp, &mp, V_NOWAIT) != 0) {
3649 			BUF_UNLOCK(bp);
3650 			continue;
3651 		}
3652 		if (lvp == NULL) {
3653 			unlock = true;
3654 			error = vn_lock(vp, LK_EXCLUSIVE | LK_NOWAIT);
3655 		} else {
3656 			ASSERT_VOP_LOCKED(vp, "getbuf");
3657 			unlock = false;
3658 			error = VOP_ISLOCKED(vp) == LK_EXCLUSIVE ? 0 :
3659 			    vn_lock(vp, LK_TRYUPGRADE);
3660 		}
3661 		if (error == 0) {
3662 			CTR3(KTR_BUF, "flushbufqueue(%p) vp %p flags %X",
3663 			    bp, bp->b_vp, bp->b_flags);
3664 			if (curproc == bufdaemonproc) {
3665 				vfs_bio_awrite(bp);
3666 			} else {
3667 				bremfree(bp);
3668 				bwrite(bp);
3669 				counter_u64_add(notbufdflushes, 1);
3670 			}
3671 			vn_finished_write(mp);
3672 			if (unlock)
3673 				VOP_UNLOCK(vp);
3674 			flushwithdeps += hasdeps;
3675 			flushed++;
3676 
3677 			/*
3678 			 * Sleeping on runningbufspace while holding
3679 			 * vnode lock leads to deadlock.
3680 			 */
3681 			if (curproc == bufdaemonproc &&
3682 			    runningbufspace > hirunningspace)
3683 				waitrunningbufspace();
3684 			continue;
3685 		}
3686 		vn_finished_write(mp);
3687 		BUF_UNLOCK(bp);
3688 	}
3689 	BQ_LOCK(bq);
3690 	TAILQ_REMOVE(&bq->bq_queue, sentinel, b_freelist);
3691 	BQ_UNLOCK(bq);
3692 	free(sentinel, M_TEMP);
3693 	return (flushed);
3694 }
3695 
3696 /*
3697  * Check to see if a block is currently memory resident.
3698  */
3699 struct buf *
incore(struct bufobj * bo,daddr_t blkno)3700 incore(struct bufobj *bo, daddr_t blkno)
3701 {
3702 	return (gbincore_unlocked(bo, blkno));
3703 }
3704 
3705 /*
3706  * Returns true if no I/O is needed to access the
3707  * associated VM object.  This is like incore except
3708  * it also hunts around in the VM system for the data.
3709  */
3710 bool
inmem(struct vnode * vp,daddr_t blkno)3711 inmem(struct vnode * vp, daddr_t blkno)
3712 {
3713 	vm_object_t obj;
3714 	vm_offset_t toff, tinc, size;
3715 	vm_page_t m, n;
3716 	vm_ooffset_t off;
3717 	int valid;
3718 
3719 	ASSERT_VOP_LOCKED(vp, "inmem");
3720 
3721 	if (incore(&vp->v_bufobj, blkno))
3722 		return (true);
3723 	if (vp->v_mount == NULL)
3724 		return (false);
3725 	obj = vp->v_object;
3726 	if (obj == NULL)
3727 		return (false);
3728 
3729 	size = PAGE_SIZE;
3730 	if (size > vp->v_mount->mnt_stat.f_iosize)
3731 		size = vp->v_mount->mnt_stat.f_iosize;
3732 	off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
3733 
3734 	for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
3735 		m = vm_page_lookup_unlocked(obj, OFF_TO_IDX(off + toff));
3736 recheck:
3737 		if (m == NULL)
3738 			return (false);
3739 
3740 		tinc = size;
3741 		if (tinc > PAGE_SIZE - ((toff + off) & PAGE_MASK))
3742 			tinc = PAGE_SIZE - ((toff + off) & PAGE_MASK);
3743 		/*
3744 		 * Consider page validity only if page mapping didn't change
3745 		 * during the check.
3746 		 */
3747 		valid = vm_page_is_valid(m,
3748 		    (vm_offset_t)((toff + off) & PAGE_MASK), tinc);
3749 		n = vm_page_lookup_unlocked(obj, OFF_TO_IDX(off + toff));
3750 		if (m != n) {
3751 			m = n;
3752 			goto recheck;
3753 		}
3754 		if (!valid)
3755 			return (false);
3756 	}
3757 	return (true);
3758 }
3759 
3760 /*
3761  * Set the dirty range for a buffer based on the status of the dirty
3762  * bits in the pages comprising the buffer.  The range is limited
3763  * to the size of the buffer.
3764  *
3765  * Tell the VM system that the pages associated with this buffer
3766  * are clean.  This is used for delayed writes where the data is
3767  * going to go to disk eventually without additional VM intevention.
3768  *
3769  * Note that while we only really need to clean through to b_bcount, we
3770  * just go ahead and clean through to b_bufsize.
3771  */
3772 static void
vfs_clean_pages_dirty_buf(struct buf * bp)3773 vfs_clean_pages_dirty_buf(struct buf *bp)
3774 {
3775 	vm_ooffset_t foff, noff, eoff;
3776 	vm_page_t m;
3777 	int i;
3778 
3779 	if ((bp->b_flags & B_VMIO) == 0 || bp->b_bufsize == 0)
3780 		return;
3781 
3782 	foff = bp->b_offset;
3783 	KASSERT(bp->b_offset != NOOFFSET,
3784 	    ("vfs_clean_pages_dirty_buf: no buffer offset"));
3785 
3786 	vfs_busy_pages_acquire(bp);
3787 	vfs_setdirty_range(bp);
3788 	for (i = 0; i < bp->b_npages; i++) {
3789 		noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
3790 		eoff = noff;
3791 		if (eoff > bp->b_offset + bp->b_bufsize)
3792 			eoff = bp->b_offset + bp->b_bufsize;
3793 		m = bp->b_pages[i];
3794 		vfs_page_set_validclean(bp, foff, m);
3795 		/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
3796 		foff = noff;
3797 	}
3798 	vfs_busy_pages_release(bp);
3799 }
3800 
3801 static void
vfs_setdirty_range(struct buf * bp)3802 vfs_setdirty_range(struct buf *bp)
3803 {
3804 	vm_offset_t boffset;
3805 	vm_offset_t eoffset;
3806 	int i;
3807 
3808 	/*
3809 	 * test the pages to see if they have been modified directly
3810 	 * by users through the VM system.
3811 	 */
3812 	for (i = 0; i < bp->b_npages; i++)
3813 		vm_page_test_dirty(bp->b_pages[i]);
3814 
3815 	/*
3816 	 * Calculate the encompassing dirty range, boffset and eoffset,
3817 	 * (eoffset - boffset) bytes.
3818 	 */
3819 
3820 	for (i = 0; i < bp->b_npages; i++) {
3821 		if (bp->b_pages[i]->dirty)
3822 			break;
3823 	}
3824 	boffset = (i << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3825 
3826 	for (i = bp->b_npages - 1; i >= 0; --i) {
3827 		if (bp->b_pages[i]->dirty) {
3828 			break;
3829 		}
3830 	}
3831 	eoffset = ((i + 1) << PAGE_SHIFT) - (bp->b_offset & PAGE_MASK);
3832 
3833 	/*
3834 	 * Fit it to the buffer.
3835 	 */
3836 
3837 	if (eoffset > bp->b_bcount)
3838 		eoffset = bp->b_bcount;
3839 
3840 	/*
3841 	 * If we have a good dirty range, merge with the existing
3842 	 * dirty range.
3843 	 */
3844 
3845 	if (boffset < eoffset) {
3846 		if (bp->b_dirtyoff > boffset)
3847 			bp->b_dirtyoff = boffset;
3848 		if (bp->b_dirtyend < eoffset)
3849 			bp->b_dirtyend = eoffset;
3850 	}
3851 }
3852 
3853 /*
3854  * Allocate the KVA mapping for an existing buffer.
3855  * If an unmapped buffer is provided but a mapped buffer is requested, take
3856  * also care to properly setup mappings between pages and KVA.
3857  */
3858 static void
bp_unmapped_get_kva(struct buf * bp,daddr_t blkno,int size,int gbflags)3859 bp_unmapped_get_kva(struct buf *bp, daddr_t blkno, int size, int gbflags)
3860 {
3861 	int bsize, maxsize, need_mapping, need_kva;
3862 	off_t offset;
3863 
3864 	need_mapping = bp->b_data == unmapped_buf &&
3865 	    (gbflags & GB_UNMAPPED) == 0;
3866 	need_kva = bp->b_kvabase == unmapped_buf &&
3867 	    bp->b_data == unmapped_buf &&
3868 	    (gbflags & GB_KVAALLOC) != 0;
3869 	if (!need_mapping && !need_kva)
3870 		return;
3871 
3872 	BUF_CHECK_UNMAPPED(bp);
3873 
3874 	if (need_mapping && bp->b_kvabase != unmapped_buf) {
3875 		/*
3876 		 * Buffer is not mapped, but the KVA was already
3877 		 * reserved at the time of the instantiation.  Use the
3878 		 * allocated space.
3879 		 */
3880 		goto has_addr;
3881 	}
3882 
3883 	/*
3884 	 * Calculate the amount of the address space we would reserve
3885 	 * if the buffer was mapped.
3886 	 */
3887 	bsize = vn_isdisk(bp->b_vp) ? DEV_BSIZE : bp->b_bufobj->bo_bsize;
3888 	KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
3889 	offset = blkno * bsize;
3890 	maxsize = size + (offset & PAGE_MASK);
3891 	maxsize = imax(maxsize, bsize);
3892 
3893 	while (bufkva_alloc(bp, maxsize, gbflags) != 0) {
3894 		if ((gbflags & GB_NOWAIT_BD) != 0) {
3895 			/*
3896 			 * XXXKIB: defragmentation cannot
3897 			 * succeed, not sure what else to do.
3898 			 */
3899 			panic("GB_NOWAIT_BD and GB_UNMAPPED %p", bp);
3900 		}
3901 		counter_u64_add(mappingrestarts, 1);
3902 		bufspace_wait(bufdomain(bp), bp->b_vp, gbflags, 0, 0);
3903 	}
3904 has_addr:
3905 	if (need_mapping) {
3906 		/* b_offset is handled by bpmap_qenter. */
3907 		bp->b_data = bp->b_kvabase;
3908 		BUF_CHECK_MAPPED(bp);
3909 		bpmap_qenter(bp);
3910 	}
3911 }
3912 
3913 struct buf *
getblk(struct vnode * vp,daddr_t blkno,int size,int slpflag,int slptimeo,int flags)3914 getblk(struct vnode *vp, daddr_t blkno, int size, int slpflag, int slptimeo,
3915     int flags)
3916 {
3917 	struct buf *bp;
3918 	int error;
3919 
3920 	error = getblkx(vp, blkno, blkno, size, slpflag, slptimeo, flags, &bp);
3921 	if (error != 0)
3922 		return (NULL);
3923 	return (bp);
3924 }
3925 
3926 /*
3927  *	getblkx:
3928  *
3929  *	Get a block given a specified block and offset into a file/device.
3930  *	The buffers B_DONE bit will be cleared on return, making it almost
3931  * 	ready for an I/O initiation.  B_INVAL may or may not be set on
3932  *	return.  The caller should clear B_INVAL prior to initiating a
3933  *	READ.
3934  *
3935  *	For a non-VMIO buffer, B_CACHE is set to the opposite of B_INVAL for
3936  *	an existing buffer.
3937  *
3938  *	For a VMIO buffer, B_CACHE is modified according to the backing VM.
3939  *	If getblk()ing a previously 0-sized invalid buffer, B_CACHE is set
3940  *	and then cleared based on the backing VM.  If the previous buffer is
3941  *	non-0-sized but invalid, B_CACHE will be cleared.
3942  *
3943  *	If getblk() must create a new buffer, the new buffer is returned with
3944  *	both B_INVAL and B_CACHE clear unless it is a VMIO buffer, in which
3945  *	case it is returned with B_INVAL clear and B_CACHE set based on the
3946  *	backing VM.
3947  *
3948  *	getblk() also forces a bwrite() for any B_DELWRI buffer whose
3949  *	B_CACHE bit is clear.
3950  *
3951  *	What this means, basically, is that the caller should use B_CACHE to
3952  *	determine whether the buffer is fully valid or not and should clear
3953  *	B_INVAL prior to issuing a read.  If the caller intends to validate
3954  *	the buffer by loading its data area with something, the caller needs
3955  *	to clear B_INVAL.  If the caller does this without issuing an I/O,
3956  *	the caller should set B_CACHE ( as an optimization ), else the caller
3957  *	should issue the I/O and biodone() will set B_CACHE if the I/O was
3958  *	a write attempt or if it was a successful read.  If the caller
3959  *	intends to issue a READ, the caller must clear B_INVAL and BIO_ERROR
3960  *	prior to issuing the READ.  biodone() will *not* clear B_INVAL.
3961  *
3962  *	The blkno parameter is the logical block being requested. Normally
3963  *	the mapping of logical block number to disk block address is done
3964  *	by calling VOP_BMAP(). However, if the mapping is already known, the
3965  *	disk block address can be passed using the dblkno parameter. If the
3966  *	disk block address is not known, then the same value should be passed
3967  *	for blkno and dblkno.
3968  */
3969 int
getblkx(struct vnode * vp,daddr_t blkno,daddr_t dblkno,int size,int slpflag,int slptimeo,int flags,struct buf ** bpp)3970 getblkx(struct vnode *vp, daddr_t blkno, daddr_t dblkno, int size, int slpflag,
3971     int slptimeo, int flags, struct buf **bpp)
3972 {
3973 	struct buf *bp;
3974 	struct bufobj *bo;
3975 	daddr_t d_blkno;
3976 	int bsize, error, maxsize, vmio;
3977 	off_t offset;
3978 
3979 	CTR3(KTR_BUF, "getblk(%p, %ld, %d)", vp, (long)blkno, size);
3980 	KASSERT((flags & (GB_UNMAPPED | GB_KVAALLOC)) != GB_KVAALLOC,
3981 	    ("GB_KVAALLOC only makes sense with GB_UNMAPPED"));
3982 	if (vp->v_type != VCHR)
3983 		ASSERT_VOP_LOCKED(vp, "getblk");
3984 	if (size > maxbcachebuf) {
3985 		printf("getblkx: size(%d) > maxbcachebuf(%d)\n", size,
3986 		    maxbcachebuf);
3987 		return (EIO);
3988 	}
3989 	if (!unmapped_buf_allowed)
3990 		flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
3991 
3992 	bo = &vp->v_bufobj;
3993 	d_blkno = dblkno;
3994 
3995 	/* Attempt lockless lookup first. */
3996 	bp = gbincore_unlocked(bo, blkno);
3997 	if (bp == NULL) {
3998 		/*
3999 		 * With GB_NOCREAT we must be sure about not finding the buffer
4000 		 * as it may have been reassigned during unlocked lookup.
4001 		 * If BO_NONSTERILE is still unset, no reassign has occurred.
4002 		 */
4003 		if ((flags & GB_NOCREAT) != 0) {
4004 			/* Ensure bo_flag is loaded after gbincore_unlocked. */
4005 			atomic_thread_fence_acq();
4006 			if ((bo->bo_flag & BO_NONSTERILE) == 0)
4007 				return (EEXIST);
4008 			goto loop;
4009 		}
4010 		goto newbuf_unlocked;
4011 	}
4012 
4013 	error = BUF_TIMELOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL, "getblku", 0,
4014 	    0);
4015 	if (error != 0) {
4016 		KASSERT(error == EBUSY,
4017 		    ("getblk: unexpected error %d from buf try-lock", error));
4018 		/*
4019 		 * We failed a buf try-lock.
4020 		 *
4021 		 * With GB_LOCK_NOWAIT, just return, rather than taking the
4022 		 * bufobj interlock and trying again, since we would probably
4023 		 * fail again anyway.  This is okay even if the buf's identity
4024 		 * changed and we contended on the wrong lock, as changing
4025 		 * identity itself requires the buf lock, and we could have
4026 		 * contended on the right lock.
4027 		 */
4028 		if ((flags & GB_LOCK_NOWAIT) != 0)
4029 			return (error);
4030 		goto loop;
4031 	}
4032 
4033 	/* Verify buf identify has not changed since lookup. */
4034 	if (bp->b_bufobj == bo && bp->b_lblkno == blkno)
4035 		goto foundbuf_fastpath;
4036 
4037 	/* It changed, fallback to locked lookup. */
4038 	BUF_UNLOCK_RAW(bp);
4039 
4040 	/* As above, with GB_LOCK_NOWAIT, just return. */
4041 	if ((flags & GB_LOCK_NOWAIT) != 0)
4042 		return (EBUSY);
4043 
4044 loop:
4045 	BO_RLOCK(bo);
4046 	bp = gbincore(bo, blkno);
4047 	if (bp != NULL) {
4048 		int lockflags;
4049 
4050 		/*
4051 		 * Buffer is in-core.  If the buffer is not busy nor managed,
4052 		 * it must be on a queue.
4053 		 */
4054 		lockflags = LK_EXCLUSIVE | LK_INTERLOCK |
4055 		    ((flags & GB_LOCK_NOWAIT) != 0 ? LK_NOWAIT : LK_SLEEPFAIL);
4056 #ifdef WITNESS
4057 		lockflags |= (flags & GB_NOWITNESS) != 0 ? LK_NOWITNESS : 0;
4058 #endif
4059 
4060 		error = BUF_TIMELOCK(bp, lockflags,
4061 		    BO_LOCKPTR(bo), "getblk", slpflag, slptimeo);
4062 
4063 		/*
4064 		 * If we slept and got the lock we have to restart in case
4065 		 * the buffer changed identities.
4066 		 */
4067 		if (error == ENOLCK)
4068 			goto loop;
4069 		/* We timed out or were interrupted. */
4070 		else if (error != 0)
4071 			return (error);
4072 
4073 foundbuf_fastpath:
4074 		/* If recursed, assume caller knows the rules. */
4075 		if (BUF_LOCKRECURSED(bp))
4076 			goto end;
4077 
4078 		/*
4079 		 * The buffer is locked.  B_CACHE is cleared if the buffer is
4080 		 * invalid.  Otherwise, for a non-VMIO buffer, B_CACHE is set
4081 		 * and for a VMIO buffer B_CACHE is adjusted according to the
4082 		 * backing VM cache.
4083 		 */
4084 		if (bp->b_flags & B_INVAL)
4085 			bp->b_flags &= ~B_CACHE;
4086 		else if ((bp->b_flags & (B_VMIO | B_INVAL)) == 0)
4087 			bp->b_flags |= B_CACHE;
4088 		if (bp->b_flags & B_MANAGED)
4089 			MPASS(bp->b_qindex == QUEUE_NONE);
4090 		else
4091 			bremfree(bp);
4092 
4093 		/*
4094 		 * check for size inconsistencies for non-VMIO case.
4095 		 */
4096 		if (bp->b_bcount != size) {
4097 			if ((bp->b_flags & B_VMIO) == 0 ||
4098 			    (size > bp->b_kvasize)) {
4099 				if (bp->b_flags & B_DELWRI) {
4100 					bp->b_flags |= B_NOCACHE;
4101 					bwrite(bp);
4102 				} else {
4103 					if (LIST_EMPTY(&bp->b_dep)) {
4104 						bp->b_flags |= B_RELBUF;
4105 						brelse(bp);
4106 					} else {
4107 						bp->b_flags |= B_NOCACHE;
4108 						bwrite(bp);
4109 					}
4110 				}
4111 				goto loop;
4112 			}
4113 		}
4114 
4115 		/*
4116 		 * Handle the case of unmapped buffer which should
4117 		 * become mapped, or the buffer for which KVA
4118 		 * reservation is requested.
4119 		 */
4120 		bp_unmapped_get_kva(bp, blkno, size, flags);
4121 
4122 		/*
4123 		 * If the size is inconsistent in the VMIO case, we can resize
4124 		 * the buffer.  This might lead to B_CACHE getting set or
4125 		 * cleared.  If the size has not changed, B_CACHE remains
4126 		 * unchanged from its previous state.
4127 		 */
4128 		allocbuf(bp, size);
4129 
4130 		KASSERT(bp->b_offset != NOOFFSET,
4131 		    ("getblk: no buffer offset"));
4132 
4133 		/*
4134 		 * A buffer with B_DELWRI set and B_CACHE clear must
4135 		 * be committed before we can return the buffer in
4136 		 * order to prevent the caller from issuing a read
4137 		 * ( due to B_CACHE not being set ) and overwriting
4138 		 * it.
4139 		 *
4140 		 * Most callers, including NFS and FFS, need this to
4141 		 * operate properly either because they assume they
4142 		 * can issue a read if B_CACHE is not set, or because
4143 		 * ( for example ) an uncached B_DELWRI might loop due
4144 		 * to softupdates re-dirtying the buffer.  In the latter
4145 		 * case, B_CACHE is set after the first write completes,
4146 		 * preventing further loops.
4147 		 * NOTE!  b*write() sets B_CACHE.  If we cleared B_CACHE
4148 		 * above while extending the buffer, we cannot allow the
4149 		 * buffer to remain with B_CACHE set after the write
4150 		 * completes or it will represent a corrupt state.  To
4151 		 * deal with this we set B_NOCACHE to scrap the buffer
4152 		 * after the write.
4153 		 *
4154 		 * We might be able to do something fancy, like setting
4155 		 * B_CACHE in bwrite() except if B_DELWRI is already set,
4156 		 * so the below call doesn't set B_CACHE, but that gets real
4157 		 * confusing.  This is much easier.
4158 		 */
4159 
4160 		if ((bp->b_flags & (B_CACHE|B_DELWRI)) == B_DELWRI) {
4161 			bp->b_flags |= B_NOCACHE;
4162 			bwrite(bp);
4163 			goto loop;
4164 		}
4165 		bp->b_flags &= ~B_DONE;
4166 	} else {
4167 		/*
4168 		 * Buffer is not in-core, create new buffer.  The buffer
4169 		 * returned by getnewbuf() is locked.  Note that the returned
4170 		 * buffer is also considered valid (not marked B_INVAL).
4171 		 */
4172 		BO_RUNLOCK(bo);
4173 newbuf_unlocked:
4174 		/*
4175 		 * If the user does not want us to create the buffer, bail out
4176 		 * here.
4177 		 */
4178 		if (flags & GB_NOCREAT)
4179 			return (EEXIST);
4180 
4181 		bsize = vn_isdisk(vp) ? DEV_BSIZE : bo->bo_bsize;
4182 		KASSERT(bsize != 0, ("bsize == 0, check bo->bo_bsize"));
4183 		offset = blkno * bsize;
4184 		vmio = vp->v_object != NULL;
4185 		if (vmio) {
4186 			maxsize = size + (offset & PAGE_MASK);
4187 			if (maxsize > maxbcachebuf) {
4188 				printf(
4189 			    "getblkx: maxsize(%d) > maxbcachebuf(%d)\n",
4190 				    maxsize, maxbcachebuf);
4191 				return (EIO);
4192 			}
4193 		} else {
4194 			maxsize = size;
4195 			/* Do not allow non-VMIO notmapped buffers. */
4196 			flags &= ~(GB_UNMAPPED | GB_KVAALLOC);
4197 		}
4198 		maxsize = imax(maxsize, bsize);
4199 		if ((flags & GB_NOSPARSE) != 0 && vmio &&
4200 		    !vn_isdisk(vp)) {
4201 			error = VOP_BMAP(vp, blkno, NULL, &d_blkno, 0, 0);
4202 			KASSERT(error != EOPNOTSUPP,
4203 			    ("GB_NOSPARSE from fs not supporting bmap, vp %p",
4204 			    vp));
4205 			if (error != 0)
4206 				return (error);
4207 			if (d_blkno == -1)
4208 				return (EJUSTRETURN);
4209 		}
4210 
4211 		bp = getnewbuf(vp, slpflag, slptimeo, maxsize, flags);
4212 		if (bp == NULL) {
4213 			if (slpflag || slptimeo)
4214 				return (ETIMEDOUT);
4215 			/*
4216 			 * XXX This is here until the sleep path is diagnosed
4217 			 * enough to work under very low memory conditions.
4218 			 *
4219 			 * There's an issue on low memory, 4BSD+non-preempt
4220 			 * systems (eg MIPS routers with 32MB RAM) where buffer
4221 			 * exhaustion occurs without sleeping for buffer
4222 			 * reclaimation.  This just sticks in a loop and
4223 			 * constantly attempts to allocate a buffer, which
4224 			 * hits exhaustion and tries to wakeup bufdaemon.
4225 			 * This never happens because we never yield.
4226 			 *
4227 			 * The real solution is to identify and fix these cases
4228 			 * so we aren't effectively busy-waiting in a loop
4229 			 * until the reclaimation path has cycles to run.
4230 			 */
4231 			kern_yield(PRI_USER);
4232 			goto loop;
4233 		}
4234 
4235 		/*
4236 		 *
4237 		 * Insert the buffer into the hash, so that it can
4238 		 * be found by incore.
4239 		 *
4240 		 * We don't hold the bufobj interlock while allocating the new
4241 		 * buffer.  Consequently, we can race on buffer creation.  This
4242 		 * can be a problem whether the vnode is locked or not.  If the
4243 		 * buffer is created out from under us, we have to throw away
4244 		 * the one we just created.
4245 		 */
4246 		bp->b_lblkno = blkno;
4247 		bp->b_blkno = d_blkno;
4248 		bp->b_offset = offset;
4249 		error = bgetvp(vp, bp);
4250 		if (error != 0) {
4251 			KASSERT(error == EEXIST,
4252 			    ("getblk: unexpected error %d from bgetvp",
4253 			    error));
4254 			bp->b_flags |= B_INVAL;
4255 			bufspace_release(bufdomain(bp), maxsize);
4256 			brelse(bp);
4257 			goto loop;
4258 		}
4259 
4260 		/*
4261 		 * set B_VMIO bit.  allocbuf() the buffer bigger.  Since the
4262 		 * buffer size starts out as 0, B_CACHE will be set by
4263 		 * allocbuf() for the VMIO case prior to it testing the
4264 		 * backing store for validity.
4265 		 */
4266 
4267 		if (vmio) {
4268 			bp->b_flags |= B_VMIO;
4269 			KASSERT(vp->v_object == bp->b_bufobj->bo_object,
4270 			    ("ARGH! different b_bufobj->bo_object %p %p %p\n",
4271 			    bp, vp->v_object, bp->b_bufobj->bo_object));
4272 		} else {
4273 			bp->b_flags &= ~B_VMIO;
4274 			KASSERT(bp->b_bufobj->bo_object == NULL,
4275 			    ("ARGH! has b_bufobj->bo_object %p %p\n",
4276 			    bp, bp->b_bufobj->bo_object));
4277 			BUF_CHECK_MAPPED(bp);
4278 		}
4279 
4280 		allocbuf(bp, size);
4281 		bufspace_release(bufdomain(bp), maxsize);
4282 		bp->b_flags &= ~B_DONE;
4283 	}
4284 	CTR4(KTR_BUF, "getblk(%p, %ld, %d) = %p", vp, (long)blkno, size, bp);
4285 end:
4286 	buf_track(bp, __func__);
4287 	KASSERT(bp->b_bufobj == bo,
4288 	    ("bp %p wrong b_bufobj %p should be %p", bp, bp->b_bufobj, bo));
4289 	*bpp = bp;
4290 	return (0);
4291 }
4292 
4293 /*
4294  * Get an empty, disassociated buffer of given size.  The buffer is initially
4295  * set to B_INVAL.
4296  */
4297 struct buf *
geteblk(int size,int flags)4298 geteblk(int size, int flags)
4299 {
4300 	struct buf *bp;
4301 	int maxsize;
4302 
4303 	maxsize = (size + BKVAMASK) & ~BKVAMASK;
4304 	while ((bp = getnewbuf(NULL, 0, 0, maxsize, flags)) == NULL) {
4305 		if ((flags & GB_NOWAIT_BD) &&
4306 		    (curthread->td_pflags & TDP_BUFNEED) != 0)
4307 			return (NULL);
4308 	}
4309 	allocbuf(bp, size);
4310 	bufspace_release(bufdomain(bp), maxsize);
4311 	bp->b_flags |= B_INVAL;	/* b_dep cleared by getnewbuf() */
4312 	return (bp);
4313 }
4314 
4315 /*
4316  * Truncate the backing store for a non-vmio buffer.
4317  */
4318 static void
vfs_nonvmio_truncate(struct buf * bp,int newbsize)4319 vfs_nonvmio_truncate(struct buf *bp, int newbsize)
4320 {
4321 
4322 	if (bp->b_flags & B_MALLOC) {
4323 		/*
4324 		 * malloced buffers are not shrunk
4325 		 */
4326 		if (newbsize == 0) {
4327 			bufmallocadjust(bp, 0);
4328 			free(bp->b_data, M_BIOBUF);
4329 			bp->b_data = bp->b_kvabase;
4330 			bp->b_flags &= ~B_MALLOC;
4331 		}
4332 		return;
4333 	}
4334 	vm_hold_free_pages(bp, newbsize);
4335 	bufspace_adjust(bp, newbsize);
4336 }
4337 
4338 /*
4339  * Extend the backing for a non-VMIO buffer.
4340  */
4341 static void
vfs_nonvmio_extend(struct buf * bp,int newbsize)4342 vfs_nonvmio_extend(struct buf *bp, int newbsize)
4343 {
4344 	caddr_t origbuf;
4345 	int origbufsize;
4346 
4347 	/*
4348 	 * We only use malloced memory on the first allocation.
4349 	 * and revert to page-allocated memory when the buffer
4350 	 * grows.
4351 	 *
4352 	 * There is a potential smp race here that could lead
4353 	 * to bufmallocspace slightly passing the max.  It
4354 	 * is probably extremely rare and not worth worrying
4355 	 * over.
4356 	 */
4357 	if (bp->b_bufsize == 0 && newbsize <= PAGE_SIZE/2 &&
4358 	    bufmallocspace < maxbufmallocspace) {
4359 		bp->b_data = malloc(newbsize, M_BIOBUF, M_WAITOK);
4360 		bp->b_flags |= B_MALLOC;
4361 		bufmallocadjust(bp, newbsize);
4362 		return;
4363 	}
4364 
4365 	/*
4366 	 * If the buffer is growing on its other-than-first
4367 	 * allocation then we revert to the page-allocation
4368 	 * scheme.
4369 	 */
4370 	origbuf = NULL;
4371 	origbufsize = 0;
4372 	if (bp->b_flags & B_MALLOC) {
4373 		origbuf = bp->b_data;
4374 		origbufsize = bp->b_bufsize;
4375 		bp->b_data = bp->b_kvabase;
4376 		bufmallocadjust(bp, 0);
4377 		bp->b_flags &= ~B_MALLOC;
4378 		newbsize = round_page(newbsize);
4379 	}
4380 	vm_hold_load_pages(bp, (vm_offset_t) bp->b_data + bp->b_bufsize,
4381 	    (vm_offset_t) bp->b_data + newbsize);
4382 	if (origbuf != NULL) {
4383 		bcopy(origbuf, bp->b_data, origbufsize);
4384 		free(origbuf, M_BIOBUF);
4385 	}
4386 	bufspace_adjust(bp, newbsize);
4387 }
4388 
4389 /*
4390  * This code constitutes the buffer memory from either anonymous system
4391  * memory (in the case of non-VMIO operations) or from an associated
4392  * VM object (in the case of VMIO operations).  This code is able to
4393  * resize a buffer up or down.
4394  *
4395  * Note that this code is tricky, and has many complications to resolve
4396  * deadlock or inconsistent data situations.  Tread lightly!!!
4397  * There are B_CACHE and B_DELWRI interactions that must be dealt with by
4398  * the caller.  Calling this code willy nilly can result in the loss of data.
4399  *
4400  * allocbuf() only adjusts B_CACHE for VMIO buffers.  getblk() deals with
4401  * B_CACHE for the non-VMIO case.
4402  */
4403 int
allocbuf(struct buf * bp,int size)4404 allocbuf(struct buf *bp, int size)
4405 {
4406 	int newbsize;
4407 
4408 	if (bp->b_bcount == size)
4409 		return (1);
4410 
4411 	KASSERT(bp->b_kvasize == 0 || bp->b_kvasize >= size,
4412 	    ("allocbuf: buffer too small %p %#x %#x",
4413 	    bp, bp->b_kvasize, size));
4414 
4415 	newbsize = roundup2(size, DEV_BSIZE);
4416 	if ((bp->b_flags & B_VMIO) == 0) {
4417 		if ((bp->b_flags & B_MALLOC) == 0)
4418 			newbsize = round_page(newbsize);
4419 		/*
4420 		 * Just get anonymous memory from the kernel.  Don't
4421 		 * mess with B_CACHE.
4422 		 */
4423 		if (newbsize < bp->b_bufsize)
4424 			vfs_nonvmio_truncate(bp, newbsize);
4425 		else if (newbsize > bp->b_bufsize)
4426 			vfs_nonvmio_extend(bp, newbsize);
4427 	} else {
4428 		int desiredpages;
4429 
4430 		desiredpages = size == 0 ? 0 :
4431 		    num_pages((bp->b_offset & PAGE_MASK) + newbsize);
4432 
4433 		KASSERT((bp->b_flags & B_MALLOC) == 0,
4434 		    ("allocbuf: VMIO buffer can't be malloced %p", bp));
4435 
4436 		/*
4437 		 * Set B_CACHE initially if buffer is 0 length or will become
4438 		 * 0-length.
4439 		 */
4440 		if (size == 0 || bp->b_bufsize == 0)
4441 			bp->b_flags |= B_CACHE;
4442 
4443 		if (newbsize < bp->b_bufsize)
4444 			vfs_vmio_truncate(bp, desiredpages);
4445 		/* XXX This looks as if it should be newbsize > b_bufsize */
4446 		else if (size > bp->b_bcount)
4447 			vfs_vmio_extend(bp, desiredpages, size);
4448 		bufspace_adjust(bp, newbsize);
4449 	}
4450 	bp->b_bcount = size;		/* requested buffer size. */
4451 	return (1);
4452 }
4453 
4454 extern int inflight_transient_maps;
4455 
4456 static struct bio_queue nondump_bios;
4457 
4458 void
biodone(struct bio * bp)4459 biodone(struct bio *bp)
4460 {
4461 	struct mtx *mtxp;
4462 	void (*done)(struct bio *);
4463 	vm_offset_t start, end;
4464 
4465 	biotrack(bp, __func__);
4466 
4467 	/*
4468 	 * Avoid completing I/O when dumping after a panic since that may
4469 	 * result in a deadlock in the filesystem or pager code.  Note that
4470 	 * this doesn't affect dumps that were started manually since we aim
4471 	 * to keep the system usable after it has been resumed.
4472 	 */
4473 	if (__predict_false(dumping && SCHEDULER_STOPPED())) {
4474 		TAILQ_INSERT_HEAD(&nondump_bios, bp, bio_queue);
4475 		return;
4476 	}
4477 	if ((bp->bio_flags & BIO_TRANSIENT_MAPPING) != 0) {
4478 		bp->bio_flags &= ~BIO_TRANSIENT_MAPPING;
4479 		bp->bio_flags |= BIO_UNMAPPED;
4480 		start = trunc_page((vm_offset_t)bp->bio_data);
4481 		end = round_page((vm_offset_t)bp->bio_data + bp->bio_length);
4482 		bp->bio_data = unmapped_buf;
4483 		pmap_qremove(start, atop(end - start));
4484 		vmem_free(transient_arena, start, end - start);
4485 		atomic_add_int(&inflight_transient_maps, -1);
4486 	}
4487 	done = bp->bio_done;
4488 	/*
4489 	 * The check for done == biodone is to allow biodone to be
4490 	 * used as a bio_done routine.
4491 	 */
4492 	if (done == NULL || done == biodone) {
4493 		mtxp = mtx_pool_find(mtxpool_sleep, bp);
4494 		mtx_lock(mtxp);
4495 		bp->bio_flags |= BIO_DONE;
4496 		wakeup(bp);
4497 		mtx_unlock(mtxp);
4498 	} else
4499 		done(bp);
4500 }
4501 
4502 /*
4503  * Wait for a BIO to finish.
4504  */
4505 int
biowait(struct bio * bp,const char * wmesg)4506 biowait(struct bio *bp, const char *wmesg)
4507 {
4508 	struct mtx *mtxp;
4509 
4510 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
4511 	mtx_lock(mtxp);
4512 	while ((bp->bio_flags & BIO_DONE) == 0)
4513 		msleep(bp, mtxp, PRIBIO, wmesg, 0);
4514 	mtx_unlock(mtxp);
4515 	if (bp->bio_error != 0)
4516 		return (bp->bio_error);
4517 	if (!(bp->bio_flags & BIO_ERROR))
4518 		return (0);
4519 	return (EIO);
4520 }
4521 
4522 void
biofinish(struct bio * bp,struct devstat * stat,int error)4523 biofinish(struct bio *bp, struct devstat *stat, int error)
4524 {
4525 
4526 	if (error) {
4527 		bp->bio_error = error;
4528 		bp->bio_flags |= BIO_ERROR;
4529 	}
4530 	if (stat != NULL)
4531 		devstat_end_transaction_bio(stat, bp);
4532 	biodone(bp);
4533 }
4534 
4535 #if defined(BUF_TRACKING) || defined(FULL_BUF_TRACKING)
4536 void
biotrack_buf(struct bio * bp,const char * location)4537 biotrack_buf(struct bio *bp, const char *location)
4538 {
4539 
4540 	buf_track(bp->bio_track_bp, location);
4541 }
4542 #endif
4543 
4544 /*
4545  *	bufwait:
4546  *
4547  *	Wait for buffer I/O completion, returning error status.  The buffer
4548  *	is left locked and B_DONE on return.  B_EINTR is converted into an EINTR
4549  *	error and cleared.
4550  */
4551 int
bufwait(struct buf * bp)4552 bufwait(struct buf *bp)
4553 {
4554 	if (bp->b_iocmd == BIO_READ)
4555 		bwait(bp, PRIBIO, "biord");
4556 	else
4557 		bwait(bp, PRIBIO, "biowr");
4558 	if (bp->b_flags & B_EINTR) {
4559 		bp->b_flags &= ~B_EINTR;
4560 		return (EINTR);
4561 	}
4562 	if (bp->b_ioflags & BIO_ERROR) {
4563 		return (bp->b_error ? bp->b_error : EIO);
4564 	} else {
4565 		return (0);
4566 	}
4567 }
4568 
4569 /*
4570  *	bufdone:
4571  *
4572  *	Finish I/O on a buffer, optionally calling a completion function.
4573  *	This is usually called from an interrupt so process blocking is
4574  *	not allowed.
4575  *
4576  *	biodone is also responsible for setting B_CACHE in a B_VMIO bp.
4577  *	In a non-VMIO bp, B_CACHE will be set on the next getblk()
4578  *	assuming B_INVAL is clear.
4579  *
4580  *	For the VMIO case, we set B_CACHE if the op was a read and no
4581  *	read error occurred, or if the op was a write.  B_CACHE is never
4582  *	set if the buffer is invalid or otherwise uncacheable.
4583  *
4584  *	bufdone does not mess with B_INVAL, allowing the I/O routine or the
4585  *	initiator to leave B_INVAL set to brelse the buffer out of existence
4586  *	in the biodone routine.
4587  */
4588 void
bufdone(struct buf * bp)4589 bufdone(struct buf *bp)
4590 {
4591 	struct bufobj *dropobj;
4592 	void    (*biodone)(struct buf *);
4593 
4594 	buf_track(bp, __func__);
4595 	CTR3(KTR_BUF, "bufdone(%p) vp %p flags %X", bp, bp->b_vp, bp->b_flags);
4596 	dropobj = NULL;
4597 
4598 	KASSERT(!(bp->b_flags & B_DONE), ("biodone: bp %p already done", bp));
4599 
4600 	runningbufwakeup(bp);
4601 	if (bp->b_iocmd == BIO_WRITE)
4602 		dropobj = bp->b_bufobj;
4603 	/* call optional completion function if requested */
4604 	if (bp->b_iodone != NULL) {
4605 		biodone = bp->b_iodone;
4606 		bp->b_iodone = NULL;
4607 		(*biodone) (bp);
4608 		if (dropobj)
4609 			bufobj_wdrop(dropobj);
4610 		return;
4611 	}
4612 	if (bp->b_flags & B_VMIO) {
4613 		/*
4614 		 * Set B_CACHE if the op was a normal read and no error
4615 		 * occurred.  B_CACHE is set for writes in the b*write()
4616 		 * routines.
4617 		 */
4618 		if (bp->b_iocmd == BIO_READ &&
4619 		    !(bp->b_flags & (B_INVAL|B_NOCACHE)) &&
4620 		    !(bp->b_ioflags & BIO_ERROR))
4621 			bp->b_flags |= B_CACHE;
4622 		vfs_vmio_iodone(bp);
4623 	}
4624 	if (!LIST_EMPTY(&bp->b_dep))
4625 		buf_complete(bp);
4626 	if ((bp->b_flags & B_CKHASH) != 0) {
4627 		KASSERT(bp->b_iocmd == BIO_READ,
4628 		    ("bufdone: b_iocmd %d not BIO_READ", bp->b_iocmd));
4629 		KASSERT(buf_mapped(bp), ("bufdone: bp %p not mapped", bp));
4630 		(*bp->b_ckhashcalc)(bp);
4631 	}
4632 	/*
4633 	 * For asynchronous completions, release the buffer now. The brelse
4634 	 * will do a wakeup there if necessary - so no need to do a wakeup
4635 	 * here in the async case. The sync case always needs to do a wakeup.
4636 	 */
4637 	if (bp->b_flags & B_ASYNC) {
4638 		if ((bp->b_flags & (B_NOCACHE | B_INVAL | B_RELBUF)) ||
4639 		    (bp->b_ioflags & BIO_ERROR))
4640 			brelse(bp);
4641 		else
4642 			bqrelse(bp);
4643 	} else
4644 		bdone(bp);
4645 	if (dropobj)
4646 		bufobj_wdrop(dropobj);
4647 }
4648 
4649 /*
4650  * This routine is called in lieu of iodone in the case of
4651  * incomplete I/O.  This keeps the busy status for pages
4652  * consistent.
4653  */
4654 void
vfs_unbusy_pages(struct buf * bp)4655 vfs_unbusy_pages(struct buf *bp)
4656 {
4657 	int i;
4658 	vm_object_t obj;
4659 	vm_page_t m;
4660 
4661 	runningbufwakeup(bp);
4662 	if (!(bp->b_flags & B_VMIO))
4663 		return;
4664 
4665 	obj = bp->b_bufobj->bo_object;
4666 	for (i = 0; i < bp->b_npages; i++) {
4667 		m = bp->b_pages[i];
4668 		if (m == bogus_page) {
4669 			m = vm_page_relookup(obj, OFF_TO_IDX(bp->b_offset) + i);
4670 			if (!m)
4671 				panic("vfs_unbusy_pages: page missing\n");
4672 			bp->b_pages[i] = m;
4673 			if (buf_mapped(bp)) {
4674 				BUF_CHECK_MAPPED(bp);
4675 				pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
4676 				    bp->b_pages, bp->b_npages);
4677 			} else
4678 				BUF_CHECK_UNMAPPED(bp);
4679 		}
4680 		vm_page_sunbusy(m);
4681 	}
4682 	vm_object_pip_wakeupn(obj, bp->b_npages);
4683 }
4684 
4685 /*
4686  * vfs_page_set_valid:
4687  *
4688  *	Set the valid bits in a page based on the supplied offset.   The
4689  *	range is restricted to the buffer's size.
4690  *
4691  *	This routine is typically called after a read completes.
4692  */
4693 static void
vfs_page_set_valid(struct buf * bp,vm_ooffset_t off,vm_page_t m)4694 vfs_page_set_valid(struct buf *bp, vm_ooffset_t off, vm_page_t m)
4695 {
4696 	vm_ooffset_t eoff;
4697 
4698 	/*
4699 	 * Compute the end offset, eoff, such that [off, eoff) does not span a
4700 	 * page boundary and eoff is not greater than the end of the buffer.
4701 	 * The end of the buffer, in this case, is our file EOF, not the
4702 	 * allocation size of the buffer.
4703 	 */
4704 	eoff = (off + PAGE_SIZE) & ~(vm_ooffset_t)PAGE_MASK;
4705 	if (eoff > bp->b_offset + bp->b_bcount)
4706 		eoff = bp->b_offset + bp->b_bcount;
4707 
4708 	/*
4709 	 * Set valid range.  This is typically the entire buffer and thus the
4710 	 * entire page.
4711 	 */
4712 	if (eoff > off)
4713 		vm_page_set_valid_range(m, off & PAGE_MASK, eoff - off);
4714 }
4715 
4716 /*
4717  * vfs_page_set_validclean:
4718  *
4719  *	Set the valid bits and clear the dirty bits in a page based on the
4720  *	supplied offset.   The range is restricted to the buffer's size.
4721  */
4722 static void
vfs_page_set_validclean(struct buf * bp,vm_ooffset_t off,vm_page_t m)4723 vfs_page_set_validclean(struct buf *bp, vm_ooffset_t off, vm_page_t m)
4724 {
4725 	vm_ooffset_t soff, eoff;
4726 
4727 	/*
4728 	 * Start and end offsets in buffer.  eoff - soff may not cross a
4729 	 * page boundary or cross the end of the buffer.  The end of the
4730 	 * buffer, in this case, is our file EOF, not the allocation size
4731 	 * of the buffer.
4732 	 */
4733 	soff = off;
4734 	eoff = (off + PAGE_SIZE) & ~(off_t)PAGE_MASK;
4735 	if (eoff > bp->b_offset + bp->b_bcount)
4736 		eoff = bp->b_offset + bp->b_bcount;
4737 
4738 	/*
4739 	 * Set valid range.  This is typically the entire buffer and thus the
4740 	 * entire page.
4741 	 */
4742 	if (eoff > soff) {
4743 		vm_page_set_validclean(
4744 		    m,
4745 		   (vm_offset_t) (soff & PAGE_MASK),
4746 		   (vm_offset_t) (eoff - soff)
4747 		);
4748 	}
4749 }
4750 
4751 /*
4752  * Acquire a shared busy on all pages in the buf.
4753  */
4754 void
vfs_busy_pages_acquire(struct buf * bp)4755 vfs_busy_pages_acquire(struct buf *bp)
4756 {
4757 	int i;
4758 
4759 	for (i = 0; i < bp->b_npages; i++)
4760 		vm_page_busy_acquire(bp->b_pages[i], VM_ALLOC_SBUSY);
4761 }
4762 
4763 void
vfs_busy_pages_release(struct buf * bp)4764 vfs_busy_pages_release(struct buf *bp)
4765 {
4766 	int i;
4767 
4768 	for (i = 0; i < bp->b_npages; i++)
4769 		vm_page_sunbusy(bp->b_pages[i]);
4770 }
4771 
4772 /*
4773  * This routine is called before a device strategy routine.
4774  * It is used to tell the VM system that paging I/O is in
4775  * progress, and treat the pages associated with the buffer
4776  * almost as being exclusive busy.  Also the object paging_in_progress
4777  * flag is handled to make sure that the object doesn't become
4778  * inconsistent.
4779  *
4780  * Since I/O has not been initiated yet, certain buffer flags
4781  * such as BIO_ERROR or B_INVAL may be in an inconsistent state
4782  * and should be ignored.
4783  */
4784 void
vfs_busy_pages(struct buf * bp,int clear_modify)4785 vfs_busy_pages(struct buf *bp, int clear_modify)
4786 {
4787 	vm_object_t obj;
4788 	vm_ooffset_t foff;
4789 	vm_page_t m;
4790 	int i;
4791 	bool bogus;
4792 
4793 	if (!(bp->b_flags & B_VMIO))
4794 		return;
4795 
4796 	obj = bp->b_bufobj->bo_object;
4797 	foff = bp->b_offset;
4798 	KASSERT(bp->b_offset != NOOFFSET,
4799 	    ("vfs_busy_pages: no buffer offset"));
4800 	if ((bp->b_flags & B_CLUSTER) == 0) {
4801 		vm_object_pip_add(obj, bp->b_npages);
4802 		vfs_busy_pages_acquire(bp);
4803 	}
4804 	if (bp->b_bufsize != 0)
4805 		vfs_setdirty_range(bp);
4806 	bogus = false;
4807 	for (i = 0; i < bp->b_npages; i++) {
4808 		m = bp->b_pages[i];
4809 		vm_page_assert_sbusied(m);
4810 
4811 		/*
4812 		 * When readying a buffer for a read ( i.e
4813 		 * clear_modify == 0 ), it is important to do
4814 		 * bogus_page replacement for valid pages in
4815 		 * partially instantiated buffers.  Partially
4816 		 * instantiated buffers can, in turn, occur when
4817 		 * reconstituting a buffer from its VM backing store
4818 		 * base.  We only have to do this if B_CACHE is
4819 		 * clear ( which causes the I/O to occur in the
4820 		 * first place ).  The replacement prevents the read
4821 		 * I/O from overwriting potentially dirty VM-backed
4822 		 * pages.  XXX bogus page replacement is, uh, bogus.
4823 		 * It may not work properly with small-block devices.
4824 		 * We need to find a better way.
4825 		 */
4826 		if (clear_modify) {
4827 			pmap_remove_write(m);
4828 			vfs_page_set_validclean(bp, foff, m);
4829 		} else if (vm_page_all_valid(m) &&
4830 		    (bp->b_flags & B_CACHE) == 0) {
4831 			bp->b_pages[i] = bogus_page;
4832 			bogus = true;
4833 		}
4834 		foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
4835 	}
4836 	if (bogus && buf_mapped(bp)) {
4837 		BUF_CHECK_MAPPED(bp);
4838 		pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
4839 		    bp->b_pages, bp->b_npages);
4840 	}
4841 }
4842 
4843 /*
4844  *	vfs_bio_set_valid:
4845  *
4846  *	Set the range within the buffer to valid.  The range is
4847  *	relative to the beginning of the buffer, b_offset.  Note that
4848  *	b_offset itself may be offset from the beginning of the first
4849  *	page.
4850  */
4851 void
vfs_bio_set_valid(struct buf * bp,int base,int size)4852 vfs_bio_set_valid(struct buf *bp, int base, int size)
4853 {
4854 	int i, n;
4855 	vm_page_t m;
4856 
4857 	if (!(bp->b_flags & B_VMIO))
4858 		return;
4859 
4860 	/*
4861 	 * Fixup base to be relative to beginning of first page.
4862 	 * Set initial n to be the maximum number of bytes in the
4863 	 * first page that can be validated.
4864 	 */
4865 	base += (bp->b_offset & PAGE_MASK);
4866 	n = PAGE_SIZE - (base & PAGE_MASK);
4867 
4868 	/*
4869 	 * Busy may not be strictly necessary here because the pages are
4870 	 * unlikely to be fully valid and the vnode lock will synchronize
4871 	 * their access via getpages.  It is grabbed for consistency with
4872 	 * other page validation.
4873 	 */
4874 	vfs_busy_pages_acquire(bp);
4875 	for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4876 		m = bp->b_pages[i];
4877 		if (n > size)
4878 			n = size;
4879 		vm_page_set_valid_range(m, base & PAGE_MASK, n);
4880 		base += n;
4881 		size -= n;
4882 		n = PAGE_SIZE;
4883 	}
4884 	vfs_busy_pages_release(bp);
4885 }
4886 
4887 /*
4888  *	vfs_bio_clrbuf:
4889  *
4890  *	If the specified buffer is a non-VMIO buffer, clear the entire
4891  *	buffer.  If the specified buffer is a VMIO buffer, clear and
4892  *	validate only the previously invalid portions of the buffer.
4893  *	This routine essentially fakes an I/O, so we need to clear
4894  *	BIO_ERROR and B_INVAL.
4895  *
4896  *	Note that while we only theoretically need to clear through b_bcount,
4897  *	we go ahead and clear through b_bufsize.
4898  */
4899 void
vfs_bio_clrbuf(struct buf * bp)4900 vfs_bio_clrbuf(struct buf *bp)
4901 {
4902 	int i, j, sa, ea, slide, zbits;
4903 	vm_page_bits_t mask;
4904 
4905 	if ((bp->b_flags & (B_VMIO | B_MALLOC)) != B_VMIO) {
4906 		clrbuf(bp);
4907 		return;
4908 	}
4909 	bp->b_flags &= ~B_INVAL;
4910 	bp->b_ioflags &= ~BIO_ERROR;
4911 	vfs_busy_pages_acquire(bp);
4912 	sa = bp->b_offset & PAGE_MASK;
4913 	slide = 0;
4914 	for (i = 0; i < bp->b_npages; i++, sa = 0) {
4915 		slide = imin(slide + PAGE_SIZE, bp->b_offset + bp->b_bufsize);
4916 		ea = slide & PAGE_MASK;
4917 		if (ea == 0)
4918 			ea = PAGE_SIZE;
4919 		if (bp->b_pages[i] == bogus_page)
4920 			continue;
4921 		j = sa / DEV_BSIZE;
4922 		zbits = (sizeof(vm_page_bits_t) * NBBY) -
4923 		    (ea - sa) / DEV_BSIZE;
4924 		mask = (VM_PAGE_BITS_ALL >> zbits) << j;
4925 		if ((bp->b_pages[i]->valid & mask) == mask)
4926 			continue;
4927 		if ((bp->b_pages[i]->valid & mask) == 0)
4928 			pmap_zero_page_area(bp->b_pages[i], sa, ea - sa);
4929 		else {
4930 			for (; sa < ea; sa += DEV_BSIZE, j++) {
4931 				if ((bp->b_pages[i]->valid & (1 << j)) == 0) {
4932 					pmap_zero_page_area(bp->b_pages[i],
4933 					    sa, DEV_BSIZE);
4934 				}
4935 			}
4936 		}
4937 		vm_page_set_valid_range(bp->b_pages[i], j * DEV_BSIZE,
4938 		    roundup2(ea - sa, DEV_BSIZE));
4939 	}
4940 	vfs_busy_pages_release(bp);
4941 	bp->b_resid = 0;
4942 }
4943 
4944 void
vfs_bio_bzero_buf(struct buf * bp,int base,int size)4945 vfs_bio_bzero_buf(struct buf *bp, int base, int size)
4946 {
4947 	vm_page_t m;
4948 	int i, n;
4949 
4950 	if (buf_mapped(bp)) {
4951 		BUF_CHECK_MAPPED(bp);
4952 		bzero(bp->b_data + base, size);
4953 	} else {
4954 		BUF_CHECK_UNMAPPED(bp);
4955 		n = PAGE_SIZE - (base & PAGE_MASK);
4956 		for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
4957 			m = bp->b_pages[i];
4958 			if (n > size)
4959 				n = size;
4960 			pmap_zero_page_area(m, base & PAGE_MASK, n);
4961 			base += n;
4962 			size -= n;
4963 			n = PAGE_SIZE;
4964 		}
4965 	}
4966 }
4967 
4968 /*
4969  * Update buffer flags based on I/O request parameters, optionally releasing the
4970  * buffer.  If it's VMIO or direct I/O, the buffer pages are released to the VM,
4971  * where they may be placed on a page queue (VMIO) or freed immediately (direct
4972  * I/O).  Otherwise the buffer is released to the cache.
4973  */
4974 static void
b_io_dismiss(struct buf * bp,int ioflag,bool release)4975 b_io_dismiss(struct buf *bp, int ioflag, bool release)
4976 {
4977 
4978 	KASSERT((ioflag & IO_NOREUSE) == 0 || (ioflag & IO_VMIO) != 0,
4979 	    ("buf %p non-VMIO noreuse", bp));
4980 
4981 	if ((ioflag & IO_DIRECT) != 0)
4982 		bp->b_flags |= B_DIRECT;
4983 	if ((ioflag & IO_EXT) != 0)
4984 		bp->b_xflags |= BX_ALTDATA;
4985 	if ((ioflag & (IO_VMIO | IO_DIRECT)) != 0 && LIST_EMPTY(&bp->b_dep)) {
4986 		bp->b_flags |= B_RELBUF;
4987 		if ((ioflag & IO_NOREUSE) != 0)
4988 			bp->b_flags |= B_NOREUSE;
4989 		if (release)
4990 			brelse(bp);
4991 	} else if (release)
4992 		bqrelse(bp);
4993 }
4994 
4995 void
vfs_bio_brelse(struct buf * bp,int ioflag)4996 vfs_bio_brelse(struct buf *bp, int ioflag)
4997 {
4998 
4999 	b_io_dismiss(bp, ioflag, true);
5000 }
5001 
5002 void
vfs_bio_set_flags(struct buf * bp,int ioflag)5003 vfs_bio_set_flags(struct buf *bp, int ioflag)
5004 {
5005 
5006 	b_io_dismiss(bp, ioflag, false);
5007 }
5008 
5009 /*
5010  * vm_hold_load_pages and vm_hold_free_pages get pages into
5011  * a buffers address space.  The pages are anonymous and are
5012  * not associated with a file object.
5013  */
5014 static void
vm_hold_load_pages(struct buf * bp,vm_offset_t from,vm_offset_t to)5015 vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
5016 {
5017 	vm_offset_t pg;
5018 	vm_page_t p;
5019 	int index;
5020 
5021 	BUF_CHECK_MAPPED(bp);
5022 
5023 	to = round_page(to);
5024 	from = round_page(from);
5025 	index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
5026 	MPASS((bp->b_flags & B_MAXPHYS) == 0);
5027 	KASSERT(to - from <= maxbcachebuf,
5028 	    ("vm_hold_load_pages too large %p %#jx %#jx %u",
5029 	    bp, (uintmax_t)from, (uintmax_t)to, maxbcachebuf));
5030 
5031 	for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
5032 		/*
5033 		 * note: must allocate system pages since blocking here
5034 		 * could interfere with paging I/O, no matter which
5035 		 * process we are.
5036 		 */
5037 		p = vm_page_alloc_noobj(VM_ALLOC_SYSTEM | VM_ALLOC_WIRED |
5038 		    VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT) | VM_ALLOC_WAITOK);
5039 		pmap_qenter(pg, &p, 1);
5040 		bp->b_pages[index] = p;
5041 	}
5042 	bp->b_npages = index;
5043 }
5044 
5045 /* Return pages associated with this buf to the vm system */
5046 static void
vm_hold_free_pages(struct buf * bp,int newbsize)5047 vm_hold_free_pages(struct buf *bp, int newbsize)
5048 {
5049 	vm_offset_t from;
5050 	vm_page_t p;
5051 	int index, newnpages;
5052 
5053 	BUF_CHECK_MAPPED(bp);
5054 
5055 	from = round_page((vm_offset_t)bp->b_data + newbsize);
5056 	newnpages = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
5057 	if (bp->b_npages > newnpages)
5058 		pmap_qremove(from, bp->b_npages - newnpages);
5059 	for (index = newnpages; index < bp->b_npages; index++) {
5060 		p = bp->b_pages[index];
5061 		bp->b_pages[index] = NULL;
5062 		vm_page_unwire_noq(p);
5063 		vm_page_free(p);
5064 	}
5065 	bp->b_npages = newnpages;
5066 }
5067 
5068 /*
5069  * Map an IO request into kernel virtual address space.
5070  *
5071  * All requests are (re)mapped into kernel VA space.
5072  * Notice that we use b_bufsize for the size of the buffer
5073  * to be mapped.  b_bcount might be modified by the driver.
5074  *
5075  * Note that even if the caller determines that the address space should
5076  * be valid, a race or a smaller-file mapped into a larger space may
5077  * actually cause vmapbuf() to fail, so all callers of vmapbuf() MUST
5078  * check the return value.
5079  *
5080  * This function only works with pager buffers.
5081  */
5082 int
vmapbuf(struct buf * bp,void * uaddr,size_t len,int mapbuf)5083 vmapbuf(struct buf *bp, void *uaddr, size_t len, int mapbuf)
5084 {
5085 	vm_prot_t prot;
5086 	int pidx;
5087 
5088 	MPASS((bp->b_flags & B_MAXPHYS) != 0);
5089 	prot = VM_PROT_READ;
5090 	if (bp->b_iocmd == BIO_READ)
5091 		prot |= VM_PROT_WRITE;	/* Less backwards than it looks */
5092 	pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
5093 	    (vm_offset_t)uaddr, len, prot, bp->b_pages, PBUF_PAGES);
5094 	if (pidx < 0)
5095 		return (-1);
5096 	bp->b_bufsize = len;
5097 	bp->b_npages = pidx;
5098 	bp->b_offset = ((vm_offset_t)uaddr) & PAGE_MASK;
5099 	if (mapbuf || !unmapped_buf_allowed) {
5100 		pmap_qenter((vm_offset_t)bp->b_kvabase, bp->b_pages, pidx);
5101 		bp->b_data = bp->b_kvabase + bp->b_offset;
5102 	} else
5103 		bp->b_data = unmapped_buf;
5104 	return (0);
5105 }
5106 
5107 /*
5108  * Free the io map PTEs associated with this IO operation.
5109  * We also invalidate the TLB entries and restore the original b_addr.
5110  *
5111  * This function only works with pager buffers.
5112  */
5113 void
vunmapbuf(struct buf * bp)5114 vunmapbuf(struct buf *bp)
5115 {
5116 	int npages;
5117 
5118 	npages = bp->b_npages;
5119 	if (buf_mapped(bp))
5120 		pmap_qremove(trunc_page((vm_offset_t)bp->b_data), npages);
5121 	vm_page_unhold_pages(bp->b_pages, npages);
5122 
5123 	bp->b_data = unmapped_buf;
5124 }
5125 
5126 void
bdone(struct buf * bp)5127 bdone(struct buf *bp)
5128 {
5129 	struct mtx *mtxp;
5130 
5131 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
5132 	mtx_lock(mtxp);
5133 	bp->b_flags |= B_DONE;
5134 	wakeup(bp);
5135 	mtx_unlock(mtxp);
5136 }
5137 
5138 void
bwait(struct buf * bp,u_char pri,const char * wchan)5139 bwait(struct buf *bp, u_char pri, const char *wchan)
5140 {
5141 	struct mtx *mtxp;
5142 
5143 	mtxp = mtx_pool_find(mtxpool_sleep, bp);
5144 	mtx_lock(mtxp);
5145 	while ((bp->b_flags & B_DONE) == 0)
5146 		msleep(bp, mtxp, pri, wchan, 0);
5147 	mtx_unlock(mtxp);
5148 }
5149 
5150 int
bufsync(struct bufobj * bo,int waitfor)5151 bufsync(struct bufobj *bo, int waitfor)
5152 {
5153 
5154 	return (VOP_FSYNC(bo2vnode(bo), waitfor, curthread));
5155 }
5156 
5157 void
bufstrategy(struct bufobj * bo,struct buf * bp)5158 bufstrategy(struct bufobj *bo, struct buf *bp)
5159 {
5160 	int i __unused;
5161 	struct vnode *vp;
5162 
5163 	vp = bp->b_vp;
5164 	KASSERT(vp == bo->bo_private, ("Inconsistent vnode bufstrategy"));
5165 	KASSERT(vp->v_type != VCHR && vp->v_type != VBLK,
5166 	    ("Wrong vnode in bufstrategy(bp=%p, vp=%p)", bp, vp));
5167 	i = VOP_STRATEGY(vp, bp);
5168 	KASSERT(i == 0, ("VOP_STRATEGY failed bp=%p vp=%p", bp, bp->b_vp));
5169 }
5170 
5171 /*
5172  * Initialize a struct bufobj before use.  Memory is assumed zero filled.
5173  */
5174 void
bufobj_init(struct bufobj * bo,void * private)5175 bufobj_init(struct bufobj *bo, void *private)
5176 {
5177 	static volatile int bufobj_cleanq;
5178 
5179         bo->bo_domain =
5180             atomic_fetchadd_int(&bufobj_cleanq, 1) % buf_domains;
5181         rw_init(BO_LOCKPTR(bo), "bufobj interlock");
5182         bo->bo_private = private;
5183         TAILQ_INIT(&bo->bo_clean.bv_hd);
5184 	pctrie_init(&bo->bo_clean.bv_root);
5185         TAILQ_INIT(&bo->bo_dirty.bv_hd);
5186 	pctrie_init(&bo->bo_dirty.bv_root);
5187 }
5188 
5189 void
bufobj_wrefl(struct bufobj * bo)5190 bufobj_wrefl(struct bufobj *bo)
5191 {
5192 
5193 	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
5194 	ASSERT_BO_WLOCKED(bo);
5195 	bo->bo_numoutput++;
5196 }
5197 
5198 void
bufobj_wref(struct bufobj * bo)5199 bufobj_wref(struct bufobj *bo)
5200 {
5201 
5202 	KASSERT(bo != NULL, ("NULL bo in bufobj_wref"));
5203 	BO_LOCK(bo);
5204 	bo->bo_numoutput++;
5205 	BO_UNLOCK(bo);
5206 }
5207 
5208 void
bufobj_wdrop(struct bufobj * bo)5209 bufobj_wdrop(struct bufobj *bo)
5210 {
5211 
5212 	KASSERT(bo != NULL, ("NULL bo in bufobj_wdrop"));
5213 	BO_LOCK(bo);
5214 	KASSERT(bo->bo_numoutput > 0, ("bufobj_wdrop non-positive count"));
5215 	if ((--bo->bo_numoutput == 0) && (bo->bo_flag & BO_WWAIT)) {
5216 		bo->bo_flag &= ~BO_WWAIT;
5217 		wakeup(&bo->bo_numoutput);
5218 	}
5219 	BO_UNLOCK(bo);
5220 }
5221 
5222 int
bufobj_wwait(struct bufobj * bo,int slpflag,int timeo)5223 bufobj_wwait(struct bufobj *bo, int slpflag, int timeo)
5224 {
5225 	int error;
5226 
5227 	KASSERT(bo != NULL, ("NULL bo in bufobj_wwait"));
5228 	ASSERT_BO_WLOCKED(bo);
5229 	error = 0;
5230 	while (bo->bo_numoutput) {
5231 		bo->bo_flag |= BO_WWAIT;
5232 		error = msleep(&bo->bo_numoutput, BO_LOCKPTR(bo),
5233 		    slpflag | (PRIBIO + 1), "bo_wwait", timeo);
5234 		if (error)
5235 			break;
5236 	}
5237 	return (error);
5238 }
5239 
5240 /*
5241  * Set bio_data or bio_ma for struct bio from the struct buf.
5242  */
5243 void
bdata2bio(struct buf * bp,struct bio * bip)5244 bdata2bio(struct buf *bp, struct bio *bip)
5245 {
5246 
5247 	if (!buf_mapped(bp)) {
5248 		KASSERT(unmapped_buf_allowed, ("unmapped"));
5249 		bip->bio_ma = bp->b_pages;
5250 		bip->bio_ma_n = bp->b_npages;
5251 		bip->bio_data = unmapped_buf;
5252 		bip->bio_ma_offset = (vm_offset_t)bp->b_offset & PAGE_MASK;
5253 		bip->bio_flags |= BIO_UNMAPPED;
5254 		KASSERT(round_page(bip->bio_ma_offset + bip->bio_length) /
5255 		    PAGE_SIZE == bp->b_npages,
5256 		    ("Buffer %p too short: %d %lld %d", bp, bip->bio_ma_offset,
5257 		    (long long)bip->bio_length, bip->bio_ma_n));
5258 	} else {
5259 		bip->bio_data = bp->b_data;
5260 		bip->bio_ma = NULL;
5261 	}
5262 }
5263 
5264 struct memdesc
memdesc_bio(struct bio * bio)5265 memdesc_bio(struct bio *bio)
5266 {
5267 	if ((bio->bio_flags & BIO_VLIST) != 0)
5268 		return (memdesc_vlist((struct bus_dma_segment *)bio->bio_data,
5269 		    bio->bio_ma_n));
5270 
5271 	if ((bio->bio_flags & BIO_UNMAPPED) != 0)
5272 		return (memdesc_vmpages(bio->bio_ma, bio->bio_bcount,
5273 		    bio->bio_ma_offset));
5274 
5275 	return (memdesc_vaddr(bio->bio_data, bio->bio_bcount));
5276 }
5277 
5278 static int buf_pager_relbuf;
5279 SYSCTL_INT(_vfs, OID_AUTO, buf_pager_relbuf, CTLFLAG_RWTUN,
5280     &buf_pager_relbuf, 0,
5281     "Make buffer pager release buffers after reading");
5282 
5283 /*
5284  * The buffer pager.  It uses buffer reads to validate pages.
5285  *
5286  * In contrast to the generic local pager from vm/vnode_pager.c, this
5287  * pager correctly and easily handles volumes where the underlying
5288  * device block size is greater than the machine page size.  The
5289  * buffer cache transparently extends the requested page run to be
5290  * aligned at the block boundary, and does the necessary bogus page
5291  * replacements in the addends to avoid obliterating already valid
5292  * pages.
5293  *
5294  * The only non-trivial issue is that the exclusive busy state for
5295  * pages, which is assumed by the vm_pager_getpages() interface, is
5296  * incompatible with the VMIO buffer cache's desire to share-busy the
5297  * pages.  This function performs a trivial downgrade of the pages'
5298  * state before reading buffers, and a less trivial upgrade from the
5299  * shared-busy to excl-busy state after the read.
5300  */
5301 int
vfs_bio_getpages(struct vnode * vp,vm_page_t * ma,int count,int * rbehind,int * rahead,vbg_get_lblkno_t get_lblkno,vbg_get_blksize_t get_blksize)5302 vfs_bio_getpages(struct vnode *vp, vm_page_t *ma, int count,
5303     int *rbehind, int *rahead, vbg_get_lblkno_t get_lblkno,
5304     vbg_get_blksize_t get_blksize)
5305 {
5306 	vm_page_t m;
5307 	vm_object_t object;
5308 	struct buf *bp;
5309 	struct mount *mp;
5310 	daddr_t lbn, lbnp;
5311 	vm_ooffset_t la, lb, poff, poffe;
5312 	long bo_bs, bsize;
5313 	int br_flags, error, i, pgsin, pgsin_a, pgsin_b;
5314 	bool redo, lpart;
5315 
5316 	object = vp->v_object;
5317 	mp = vp->v_mount;
5318 	error = 0;
5319 	la = IDX_TO_OFF(ma[count - 1]->pindex);
5320 	if (la >= object->un_pager.vnp.vnp_size)
5321 		return (VM_PAGER_BAD);
5322 
5323 	/*
5324 	 * Change the meaning of la from where the last requested page starts
5325 	 * to where it ends, because that's the end of the requested region
5326 	 * and the start of the potential read-ahead region.
5327 	 */
5328 	la += PAGE_SIZE;
5329 	lpart = la > object->un_pager.vnp.vnp_size;
5330 	error = get_blksize(vp, get_lblkno(vp, IDX_TO_OFF(ma[0]->pindex)),
5331 	    &bo_bs);
5332 	if (error != 0)
5333 		return (VM_PAGER_ERROR);
5334 
5335 	/*
5336 	 * Calculate read-ahead, behind and total pages.
5337 	 */
5338 	pgsin = count;
5339 	lb = IDX_TO_OFF(ma[0]->pindex);
5340 	pgsin_b = OFF_TO_IDX(lb - rounddown2(lb, bo_bs));
5341 	pgsin += pgsin_b;
5342 	if (rbehind != NULL)
5343 		*rbehind = pgsin_b;
5344 	pgsin_a = OFF_TO_IDX(roundup2(la, bo_bs) - la);
5345 	if (la + IDX_TO_OFF(pgsin_a) >= object->un_pager.vnp.vnp_size)
5346 		pgsin_a = OFF_TO_IDX(roundup2(object->un_pager.vnp.vnp_size,
5347 		    PAGE_SIZE) - la);
5348 	pgsin += pgsin_a;
5349 	if (rahead != NULL)
5350 		*rahead = pgsin_a;
5351 	VM_CNT_INC(v_vnodein);
5352 	VM_CNT_ADD(v_vnodepgsin, pgsin);
5353 
5354 	br_flags = (mp != NULL && (mp->mnt_kern_flag & MNTK_UNMAPPED_BUFS)
5355 	    != 0) ? GB_UNMAPPED : 0;
5356 again:
5357 	for (i = 0; i < count; i++) {
5358 		if (ma[i] != bogus_page)
5359 			vm_page_busy_downgrade(ma[i]);
5360 	}
5361 
5362 	lbnp = -1;
5363 	for (i = 0; i < count; i++) {
5364 		m = ma[i];
5365 		if (m == bogus_page)
5366 			continue;
5367 
5368 		/*
5369 		 * Pages are shared busy and the object lock is not
5370 		 * owned, which together allow for the pages'
5371 		 * invalidation.  The racy test for validity avoids
5372 		 * useless creation of the buffer for the most typical
5373 		 * case when invalidation is not used in redo or for
5374 		 * parallel read.  The shared->excl upgrade loop at
5375 		 * the end of the function catches the race in a
5376 		 * reliable way (protected by the object lock).
5377 		 */
5378 		if (vm_page_all_valid(m))
5379 			continue;
5380 
5381 		poff = IDX_TO_OFF(m->pindex);
5382 		poffe = MIN(poff + PAGE_SIZE, object->un_pager.vnp.vnp_size);
5383 		for (; poff < poffe; poff += bsize) {
5384 			lbn = get_lblkno(vp, poff);
5385 			if (lbn == lbnp)
5386 				goto next_page;
5387 			lbnp = lbn;
5388 
5389 			error = get_blksize(vp, lbn, &bsize);
5390 			if (error == 0)
5391 				error = bread_gb(vp, lbn, bsize,
5392 				    curthread->td_ucred, br_flags, &bp);
5393 			if (error != 0)
5394 				goto end_pages;
5395 			if (bp->b_rcred == curthread->td_ucred) {
5396 				crfree(bp->b_rcred);
5397 				bp->b_rcred = NOCRED;
5398 			}
5399 			if (LIST_EMPTY(&bp->b_dep)) {
5400 				/*
5401 				 * Invalidation clears m->valid, but
5402 				 * may leave B_CACHE flag if the
5403 				 * buffer existed at the invalidation
5404 				 * time.  In this case, recycle the
5405 				 * buffer to do real read on next
5406 				 * bread() after redo.
5407 				 *
5408 				 * Otherwise B_RELBUF is not strictly
5409 				 * necessary, enable to reduce buf
5410 				 * cache pressure.
5411 				 */
5412 				if (buf_pager_relbuf ||
5413 				    !vm_page_all_valid(m))
5414 					bp->b_flags |= B_RELBUF;
5415 
5416 				bp->b_flags &= ~B_NOCACHE;
5417 				brelse(bp);
5418 			} else {
5419 				bqrelse(bp);
5420 			}
5421 		}
5422 		KASSERT(1 /* racy, enable for debugging */ ||
5423 		    vm_page_all_valid(m) || i == count - 1,
5424 		    ("buf %d %p invalid", i, m));
5425 		if (i == count - 1 && lpart) {
5426 			if (!vm_page_none_valid(m) &&
5427 			    !vm_page_all_valid(m))
5428 				vm_page_zero_invalid(m, TRUE);
5429 		}
5430 next_page:;
5431 	}
5432 end_pages:
5433 
5434 	redo = false;
5435 	for (i = 0; i < count; i++) {
5436 		if (ma[i] == bogus_page)
5437 			continue;
5438 		if (vm_page_busy_tryupgrade(ma[i]) == 0) {
5439 			vm_page_sunbusy(ma[i]);
5440 			ma[i] = vm_page_grab_unlocked(object, ma[i]->pindex,
5441 			    VM_ALLOC_NORMAL);
5442 		}
5443 
5444 		/*
5445 		 * Since the pages were only sbusy while neither the
5446 		 * buffer nor the object lock was held by us, or
5447 		 * reallocated while vm_page_grab() slept for busy
5448 		 * relinguish, they could have been invalidated.
5449 		 * Recheck the valid bits and re-read as needed.
5450 		 *
5451 		 * Note that the last page is made fully valid in the
5452 		 * read loop, and partial validity for the page at
5453 		 * index count - 1 could mean that the page was
5454 		 * invalidated or removed, so we must restart for
5455 		 * safety as well.
5456 		 */
5457 		if (!vm_page_all_valid(ma[i]))
5458 			redo = true;
5459 	}
5460 	if (redo && error == 0)
5461 		goto again;
5462 	return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
5463 }
5464 
5465 #include "opt_ddb.h"
5466 #ifdef DDB
5467 #include <ddb/ddb.h>
5468 
5469 /* DDB command to show buffer data */
DB_SHOW_COMMAND(buffer,db_show_buffer)5470 DB_SHOW_COMMAND(buffer, db_show_buffer)
5471 {
5472 	/* get args */
5473 	struct buf *bp = (struct buf *)addr;
5474 #ifdef FULL_BUF_TRACKING
5475 	uint32_t i, j;
5476 #endif
5477 
5478 	if (!have_addr) {
5479 		db_printf("usage: show buffer <addr>\n");
5480 		return;
5481 	}
5482 
5483 	db_printf("buf at %p\n", bp);
5484 	db_printf("b_flags = 0x%b, b_xflags = 0x%b\n",
5485 	    (u_int)bp->b_flags, PRINT_BUF_FLAGS,
5486 	    (u_int)bp->b_xflags, PRINT_BUF_XFLAGS);
5487 	db_printf("b_vflags = 0x%b, b_ioflags = 0x%b\n",
5488 	    (u_int)bp->b_vflags, PRINT_BUF_VFLAGS,
5489 	    (u_int)bp->b_ioflags, PRINT_BIO_FLAGS);
5490 	db_printf(
5491 	    "b_error = %d, b_bufsize = %ld, b_bcount = %ld, b_resid = %ld\n"
5492 	    "b_bufobj = %p, b_data = %p\n"
5493 	    "b_blkno = %jd, b_lblkno = %jd, b_vp = %p, b_dep = %p\n",
5494 	    bp->b_error, bp->b_bufsize, bp->b_bcount, bp->b_resid,
5495 	    bp->b_bufobj, bp->b_data, (intmax_t)bp->b_blkno,
5496 	    (intmax_t)bp->b_lblkno, bp->b_vp, bp->b_dep.lh_first);
5497 	db_printf("b_kvabase = %p, b_kvasize = %d\n",
5498 	    bp->b_kvabase, bp->b_kvasize);
5499 	if (bp->b_npages) {
5500 		int i;
5501 		db_printf("b_npages = %d, pages(OBJ, IDX, PA): ", bp->b_npages);
5502 		for (i = 0; i < bp->b_npages; i++) {
5503 			vm_page_t m;
5504 			m = bp->b_pages[i];
5505 			if (m != NULL)
5506 				db_printf("(%p, 0x%lx, 0x%lx)", m->object,
5507 				    (u_long)m->pindex,
5508 				    (u_long)VM_PAGE_TO_PHYS(m));
5509 			else
5510 				db_printf("( ??? )");
5511 			if ((i + 1) < bp->b_npages)
5512 				db_printf(",");
5513 		}
5514 		db_printf("\n");
5515 	}
5516 	BUF_LOCKPRINTINFO(bp);
5517 #if defined(FULL_BUF_TRACKING)
5518 	db_printf("b_io_tracking: b_io_tcnt = %u\n", bp->b_io_tcnt);
5519 
5520 	i = bp->b_io_tcnt % BUF_TRACKING_SIZE;
5521 	for (j = 1; j <= BUF_TRACKING_SIZE; j++) {
5522 		if (bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)] == NULL)
5523 			continue;
5524 		db_printf(" %2u: %s\n", j,
5525 		    bp->b_io_tracking[BUF_TRACKING_ENTRY(i - j)]);
5526 	}
5527 #elif defined(BUF_TRACKING)
5528 	db_printf("b_io_tracking: %s\n", bp->b_io_tracking);
5529 #endif
5530 }
5531 
DB_SHOW_COMMAND_FLAGS(bufqueues,bufqueues,DB_CMD_MEMSAFE)5532 DB_SHOW_COMMAND_FLAGS(bufqueues, bufqueues, DB_CMD_MEMSAFE)
5533 {
5534 	struct bufdomain *bd;
5535 	struct buf *bp;
5536 	long total;
5537 	int i, j, cnt;
5538 
5539 	db_printf("bqempty: %d\n", bqempty.bq_len);
5540 
5541 	for (i = 0; i < buf_domains; i++) {
5542 		bd = &bdomain[i];
5543 		db_printf("Buf domain %d\n", i);
5544 		db_printf("\tfreebufs\t%d\n", bd->bd_freebuffers);
5545 		db_printf("\tlofreebufs\t%d\n", bd->bd_lofreebuffers);
5546 		db_printf("\thifreebufs\t%d\n", bd->bd_hifreebuffers);
5547 		db_printf("\n");
5548 		db_printf("\tbufspace\t%ld\n", bd->bd_bufspace);
5549 		db_printf("\tmaxbufspace\t%ld\n", bd->bd_maxbufspace);
5550 		db_printf("\thibufspace\t%ld\n", bd->bd_hibufspace);
5551 		db_printf("\tlobufspace\t%ld\n", bd->bd_lobufspace);
5552 		db_printf("\tbufspacethresh\t%ld\n", bd->bd_bufspacethresh);
5553 		db_printf("\n");
5554 		db_printf("\tnumdirtybuffers\t%d\n", bd->bd_numdirtybuffers);
5555 		db_printf("\tlodirtybuffers\t%d\n", bd->bd_lodirtybuffers);
5556 		db_printf("\thidirtybuffers\t%d\n", bd->bd_hidirtybuffers);
5557 		db_printf("\tdirtybufthresh\t%d\n", bd->bd_dirtybufthresh);
5558 		db_printf("\n");
5559 		total = 0;
5560 		TAILQ_FOREACH(bp, &bd->bd_cleanq->bq_queue, b_freelist)
5561 			total += bp->b_bufsize;
5562 		db_printf("\tcleanq count\t%d (%ld)\n",
5563 		    bd->bd_cleanq->bq_len, total);
5564 		total = 0;
5565 		TAILQ_FOREACH(bp, &bd->bd_dirtyq.bq_queue, b_freelist)
5566 			total += bp->b_bufsize;
5567 		db_printf("\tdirtyq count\t%d (%ld)\n",
5568 		    bd->bd_dirtyq.bq_len, total);
5569 		db_printf("\twakeup\t\t%d\n", bd->bd_wanted);
5570 		db_printf("\tlim\t\t%d\n", bd->bd_lim);
5571 		db_printf("\tCPU ");
5572 		for (j = 0; j <= mp_maxid; j++)
5573 			db_printf("%d, ", bd->bd_subq[j].bq_len);
5574 		db_printf("\n");
5575 		cnt = 0;
5576 		total = 0;
5577 		for (j = 0; j < nbuf; j++) {
5578 			bp = nbufp(j);
5579 			if (bp->b_domain == i && BUF_ISLOCKED(bp)) {
5580 				cnt++;
5581 				total += bp->b_bufsize;
5582 			}
5583 		}
5584 		db_printf("\tLocked buffers: %d space %ld\n", cnt, total);
5585 		cnt = 0;
5586 		total = 0;
5587 		for (j = 0; j < nbuf; j++) {
5588 			bp = nbufp(j);
5589 			if (bp->b_domain == i) {
5590 				cnt++;
5591 				total += bp->b_bufsize;
5592 			}
5593 		}
5594 		db_printf("\tTotal buffers: %d space %ld\n", cnt, total);
5595 	}
5596 }
5597 
DB_SHOW_COMMAND_FLAGS(lockedbufs,lockedbufs,DB_CMD_MEMSAFE)5598 DB_SHOW_COMMAND_FLAGS(lockedbufs, lockedbufs, DB_CMD_MEMSAFE)
5599 {
5600 	struct buf *bp;
5601 	int i;
5602 
5603 	for (i = 0; i < nbuf; i++) {
5604 		bp = nbufp(i);
5605 		if (BUF_ISLOCKED(bp)) {
5606 			db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5607 			db_printf("\n");
5608 			if (db_pager_quit)
5609 				break;
5610 		}
5611 	}
5612 }
5613 
DB_SHOW_COMMAND(vnodebufs,db_show_vnodebufs)5614 DB_SHOW_COMMAND(vnodebufs, db_show_vnodebufs)
5615 {
5616 	struct vnode *vp;
5617 	struct buf *bp;
5618 
5619 	if (!have_addr) {
5620 		db_printf("usage: show vnodebufs <addr>\n");
5621 		return;
5622 	}
5623 	vp = (struct vnode *)addr;
5624 	db_printf("Clean buffers:\n");
5625 	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_clean.bv_hd, b_bobufs) {
5626 		db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5627 		db_printf("\n");
5628 	}
5629 	db_printf("Dirty buffers:\n");
5630 	TAILQ_FOREACH(bp, &vp->v_bufobj.bo_dirty.bv_hd, b_bobufs) {
5631 		db_show_buffer((uintptr_t)bp, 1, 0, NULL);
5632 		db_printf("\n");
5633 	}
5634 }
5635 
DB_COMMAND_FLAGS(countfreebufs,db_coundfreebufs,DB_CMD_MEMSAFE)5636 DB_COMMAND_FLAGS(countfreebufs, db_coundfreebufs, DB_CMD_MEMSAFE)
5637 {
5638 	struct buf *bp;
5639 	int i, used = 0, nfree = 0;
5640 
5641 	if (have_addr) {
5642 		db_printf("usage: countfreebufs\n");
5643 		return;
5644 	}
5645 
5646 	for (i = 0; i < nbuf; i++) {
5647 		bp = nbufp(i);
5648 		if (bp->b_qindex == QUEUE_EMPTY)
5649 			nfree++;
5650 		else
5651 			used++;
5652 	}
5653 
5654 	db_printf("Counted %d free, %d used (%d tot)\n", nfree, used,
5655 	    nfree + used);
5656 	db_printf("numfreebuffers is %d\n", numfreebuffers);
5657 }
5658 #endif /* DDB */
5659