xref: /freebsd/sys/ufs/ffs/ffs_snapshot.c (revision f15e18a642cb3f7ebc747f8e9cdf11274140107d)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright 2000 Marshall Kirk McKusick. All Rights Reserved.
5  *
6  * Further information about snapshots can be obtained from:
7  *
8  *	Marshall Kirk McKusick		http://www.mckusick.com/softdep/
9  *	1614 Oxford Street		mckusick@mckusick.com
10  *	Berkeley, CA 94709-1608		+1-510-843-9542
11  *	USA
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  *
17  * 1. Redistributions of source code must retain the above copyright
18  *    notice, this list of conditions and the following disclaimer.
19  * 2. Redistributions in binary form must reproduce the above copyright
20  *    notice, this list of conditions and the following disclaimer in the
21  *    documentation and/or other materials provided with the distribution.
22  *
23  * THIS SOFTWARE IS PROVIDED BY MARSHALL KIRK MCKUSICK ``AS IS'' AND ANY
24  * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
25  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
26  * DISCLAIMED.  IN NO EVENT SHALL MARSHALL KIRK MCKUSICK BE LIABLE FOR
27  * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
28  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
29  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
30  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
31  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
32  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
33  * SUCH DAMAGE.
34  *
35  *	@(#)ffs_snapshot.c	8.11 (McKusick) 7/23/00
36  */
37 
38 #include <sys/cdefs.h>
39 __FBSDID("$FreeBSD$");
40 
41 #include "opt_quota.h"
42 
43 #include <sys/param.h>
44 #include <sys/kernel.h>
45 #include <sys/systm.h>
46 #include <sys/conf.h>
47 #include <sys/gsb_crc32.h>
48 #include <sys/bio.h>
49 #include <sys/buf.h>
50 #include <sys/fcntl.h>
51 #include <sys/proc.h>
52 #include <sys/namei.h>
53 #include <sys/sched.h>
54 #include <sys/stat.h>
55 #include <sys/malloc.h>
56 #include <sys/mount.h>
57 #include <sys/resource.h>
58 #include <sys/resourcevar.h>
59 #include <sys/rwlock.h>
60 #include <sys/vnode.h>
61 
62 #include <geom/geom.h>
63 
64 #include <ufs/ufs/extattr.h>
65 #include <ufs/ufs/quota.h>
66 #include <ufs/ufs/ufsmount.h>
67 #include <ufs/ufs/inode.h>
68 #include <ufs/ufs/ufs_extern.h>
69 
70 #include <ufs/ffs/fs.h>
71 #include <ufs/ffs/ffs_extern.h>
72 
73 #define KERNCRED thread0.td_ucred
74 
75 #include "opt_ffs.h"
76 
77 #ifdef NO_FFS_SNAPSHOT
78 int
79 ffs_snapshot(mp, snapfile)
80 	struct mount *mp;
81 	char *snapfile;
82 {
83 	return (EINVAL);
84 }
85 
86 int
87 ffs_snapblkfree(fs, devvp, bno, size, inum, vtype, wkhd)
88 	struct fs *fs;
89 	struct vnode *devvp;
90 	ufs2_daddr_t bno;
91 	long size;
92 	ino_t inum;
93 	enum vtype vtype;
94 	struct workhead *wkhd;
95 {
96 	return (EINVAL);
97 }
98 
99 void
100 ffs_snapremove(vp)
101 	struct vnode *vp;
102 {
103 }
104 
105 void
106 ffs_snapshot_mount(mp)
107 	struct mount *mp;
108 {
109 }
110 
111 void
112 ffs_snapshot_unmount(mp)
113 	struct mount *mp;
114 {
115 }
116 
117 void
118 ffs_snapgone(ip)
119 	struct inode *ip;
120 {
121 }
122 
123 int
124 ffs_copyonwrite(devvp, bp)
125 	struct vnode *devvp;
126 	struct buf *bp;
127 {
128 	return (EINVAL);
129 }
130 
131 void
132 ffs_sync_snap(mp, waitfor)
133 	struct mount *mp;
134 	int waitfor;
135 {
136 }
137 
138 #else
139 FEATURE(ffs_snapshot, "FFS snapshot support");
140 
141 LIST_HEAD(, snapdata) snapfree;
142 static struct mtx snapfree_lock;
143 MTX_SYSINIT(ffs_snapfree, &snapfree_lock, "snapdata free list", MTX_DEF);
144 
145 static int cgaccount(int, struct vnode *, struct buf *, int);
146 static int expunge_ufs1(struct vnode *, struct inode *, struct fs *,
147     int (*)(struct vnode *, ufs1_daddr_t *, ufs1_daddr_t *, struct fs *,
148     ufs_lbn_t, int), int, int);
149 static int indiracct_ufs1(struct vnode *, struct vnode *, int,
150     ufs1_daddr_t, ufs_lbn_t, ufs_lbn_t, ufs_lbn_t, ufs_lbn_t, struct fs *,
151     int (*)(struct vnode *, ufs1_daddr_t *, ufs1_daddr_t *, struct fs *,
152     ufs_lbn_t, int), int);
153 static int fullacct_ufs1(struct vnode *, ufs1_daddr_t *, ufs1_daddr_t *,
154     struct fs *, ufs_lbn_t, int);
155 static int snapacct_ufs1(struct vnode *, ufs1_daddr_t *, ufs1_daddr_t *,
156     struct fs *, ufs_lbn_t, int);
157 static int mapacct_ufs1(struct vnode *, ufs1_daddr_t *, ufs1_daddr_t *,
158     struct fs *, ufs_lbn_t, int);
159 static int expunge_ufs2(struct vnode *, struct inode *, struct fs *,
160     int (*)(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *, struct fs *,
161     ufs_lbn_t, int), int, int);
162 static int indiracct_ufs2(struct vnode *, struct vnode *, int,
163     ufs2_daddr_t, ufs_lbn_t, ufs_lbn_t, ufs_lbn_t, ufs_lbn_t, struct fs *,
164     int (*)(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *, struct fs *,
165     ufs_lbn_t, int), int);
166 static int fullacct_ufs2(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
167     struct fs *, ufs_lbn_t, int);
168 static int snapacct_ufs2(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
169     struct fs *, ufs_lbn_t, int);
170 static int mapacct_ufs2(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
171     struct fs *, ufs_lbn_t, int);
172 static int readblock(struct vnode *vp, struct buf *, ufs2_daddr_t);
173 static void try_free_snapdata(struct vnode *devvp);
174 static struct snapdata *ffs_snapdata_acquire(struct vnode *devvp);
175 static int ffs_bp_snapblk(struct vnode *, struct buf *);
176 
177 /*
178  * To ensure the consistency of snapshots across crashes, we must
179  * synchronously write out copied blocks before allowing the
180  * originals to be modified. Because of the rather severe speed
181  * penalty that this imposes, the code normally only ensures
182  * persistence for the filesystem metadata contained within a
183  * snapshot. Setting the following flag allows this crash
184  * persistence to be enabled for file contents.
185  */
186 int dopersistence = 0;
187 
188 #ifdef DIAGNOSTIC
189 #include <sys/sysctl.h>
190 SYSCTL_INT(_debug, OID_AUTO, dopersistence, CTLFLAG_RW, &dopersistence, 0, "");
191 static int snapdebug = 0;
192 SYSCTL_INT(_debug, OID_AUTO, snapdebug, CTLFLAG_RW, &snapdebug, 0, "");
193 int collectsnapstats = 0;
194 SYSCTL_INT(_debug, OID_AUTO, collectsnapstats, CTLFLAG_RW, &collectsnapstats,
195 	0, "");
196 #endif /* DIAGNOSTIC */
197 
198 /*
199  * Create a snapshot file and initialize it for the filesystem.
200  */
201 int
202 ffs_snapshot(mp, snapfile)
203 	struct mount *mp;
204 	char *snapfile;
205 {
206 	ufs2_daddr_t numblks, blkno, *blkp, *snapblklist;
207 	int error, cg, snaploc;
208 	int i, size, len, loc;
209 	ufs2_daddr_t blockno;
210 	uint64_t flag;
211 	char saved_nice = 0;
212 	long redo = 0, snaplistsize = 0;
213 	int32_t *lp;
214 	void *space;
215 	struct fs *copy_fs = NULL, *fs;
216 	struct thread *td = curthread;
217 	struct inode *ip, *xp;
218 	struct buf *bp, *nbp, *ibp;
219 	struct nameidata nd;
220 	struct mount *wrtmp;
221 	struct vattr vat;
222 	struct vnode *vp, *xvp, *mvp, *devvp;
223 	struct uio auio;
224 	struct iovec aiov;
225 	struct snapdata *sn;
226 	struct ufsmount *ump;
227 #ifdef DIAGNOSTIC
228 	struct timespec starttime = {0, 0}, endtime;
229 #endif
230 
231 	ump = VFSTOUFS(mp);
232 	fs = ump->um_fs;
233 	sn = NULL;
234 	/*
235 	 * At the moment, journaled soft updates cannot support
236 	 * taking snapshots.
237 	 */
238 	if (MOUNTEDSUJ(mp)) {
239 		vfs_mount_error(mp, "%s: Snapshots are not yet supported when "
240 		    "running with journaled soft updates", fs->fs_fsmnt);
241 		return (EOPNOTSUPP);
242 	}
243 	MNT_ILOCK(mp);
244 	flag = mp->mnt_flag;
245 	MNT_IUNLOCK(mp);
246 	/*
247 	 * Need to serialize access to snapshot code per filesystem.
248 	 */
249 	/*
250 	 * Assign a snapshot slot in the superblock.
251 	 */
252 	UFS_LOCK(ump);
253 	for (snaploc = 0; snaploc < FSMAXSNAP; snaploc++)
254 		if (fs->fs_snapinum[snaploc] == 0)
255 			break;
256 	UFS_UNLOCK(ump);
257 	if (snaploc == FSMAXSNAP)
258 		return (ENOSPC);
259 	/*
260 	 * Create the snapshot file.
261 	 */
262 restart:
263 	NDINIT(&nd, CREATE, LOCKPARENT | LOCKLEAF | NOCACHE, UIO_SYSSPACE,
264 	    snapfile, td);
265 	if ((error = namei(&nd)) != 0)
266 		return (error);
267 	if (nd.ni_vp != NULL) {
268 		vput(nd.ni_vp);
269 		error = EEXIST;
270 	}
271 	if (nd.ni_dvp->v_mount != mp)
272 		error = EXDEV;
273 	if (error) {
274 		NDFREE(&nd, NDF_ONLY_PNBUF);
275 		if (nd.ni_dvp == nd.ni_vp)
276 			vrele(nd.ni_dvp);
277 		else
278 			vput(nd.ni_dvp);
279 		return (error);
280 	}
281 	VATTR_NULL(&vat);
282 	vat.va_type = VREG;
283 	vat.va_mode = S_IRUSR;
284 	vat.va_vaflags |= VA_EXCLUSIVE;
285 	if (VOP_GETWRITEMOUNT(nd.ni_dvp, &wrtmp))
286 		wrtmp = NULL;
287 	if (wrtmp != mp)
288 		panic("ffs_snapshot: mount mismatch");
289 	vfs_rel(wrtmp);
290 	if (vn_start_write(NULL, &wrtmp, V_NOWAIT) != 0) {
291 		NDFREE(&nd, NDF_ONLY_PNBUF);
292 		vput(nd.ni_dvp);
293 		if ((error = vn_start_write(NULL, &wrtmp,
294 		    V_XSLEEP | PCATCH)) != 0)
295 			return (error);
296 		goto restart;
297 	}
298 	error = VOP_CREATE(nd.ni_dvp, &nd.ni_vp, &nd.ni_cnd, &vat);
299 	if (error) {
300 		VOP_VPUT_PAIR(nd.ni_dvp, NULL, true);
301 		NDFREE(&nd, NDF_ONLY_PNBUF);
302 		vn_finished_write(wrtmp);
303 		if (error == ERELOOKUP)
304 			goto restart;
305 		return (error);
306 	}
307 	vp = nd.ni_vp;
308 	vref(nd.ni_dvp);
309 	VOP_VPUT_PAIR(nd.ni_dvp, &vp, false);
310 	if (VN_IS_DOOMED(vp)) {
311 		error = EBADF;
312 		goto out;
313 	}
314 	vnode_create_vobject(nd.ni_vp, fs->fs_size, td);
315 	vp->v_vflag |= VV_SYSTEM;
316 	ip = VTOI(vp);
317 	devvp = ITODEVVP(ip);
318 	/*
319 	 * Calculate the size of the filesystem then allocate the block
320 	 * immediately following the last block of the filesystem that
321 	 * will contain the snapshot list. This operation allows us to
322 	 * set the size of the snapshot.
323 	 */
324 	numblks = howmany(fs->fs_size, fs->fs_frag);
325 	error = UFS_BALLOC(vp, lblktosize(fs, (off_t)numblks),
326 	    fs->fs_bsize, KERNCRED, BA_CLRBUF, &bp);
327 	if (error)
328 		goto out;
329 	bawrite(bp);
330 	ip->i_size = lblktosize(fs, (off_t)(numblks + 1));
331 	DIP_SET(ip, i_size, ip->i_size);
332 	UFS_INODE_SET_FLAG(ip, IN_SIZEMOD | IN_CHANGE | IN_UPDATE);
333 	/*
334 	 * Preallocate critical data structures so that we can copy
335 	 * them in without further allocation after we suspend all
336 	 * operations on the filesystem. We would like to just release
337 	 * the allocated buffers without writing them since they will
338 	 * be filled in below once we are ready to go, but this upsets
339 	 * the soft update code, so we go ahead and write the new buffers.
340 	 *
341 	 * Allocate all indirect blocks and mark all of them as not
342 	 * needing to be copied.
343 	 */
344 	for (blkno = UFS_NDADDR; blkno < numblks; blkno += NINDIR(fs)) {
345 		error = UFS_BALLOC(vp, lblktosize(fs, (off_t)blkno),
346 		    fs->fs_bsize, td->td_ucred, BA_METAONLY, &ibp);
347 		if (error)
348 			goto out;
349 		bawrite(ibp);
350 	}
351 	/*
352 	 * Allocate copies for the superblock and its summary information.
353 	 */
354 	error = UFS_BALLOC(vp, fs->fs_sblockloc, fs->fs_sbsize, KERNCRED,
355 	    0, &nbp);
356 	if (error)
357 		goto out;
358 	bawrite(nbp);
359 	blkno = fragstoblks(fs, fs->fs_csaddr);
360 	len = howmany(fs->fs_cssize, fs->fs_bsize);
361 	for (loc = 0; loc < len; loc++) {
362 		error = UFS_BALLOC(vp, lblktosize(fs, (off_t)(blkno + loc)),
363 		    fs->fs_bsize, KERNCRED, 0, &nbp);
364 		if (error)
365 			goto out;
366 		bawrite(nbp);
367 	}
368 	/*
369 	 * Allocate all cylinder group blocks.
370 	 */
371 	for (cg = 0; cg < fs->fs_ncg; cg++) {
372 		error = UFS_BALLOC(vp, lfragtosize(fs, cgtod(fs, cg)),
373 		    fs->fs_bsize, KERNCRED, 0, &nbp);
374 		if (error)
375 			goto out;
376 		bawrite(nbp);
377 		if (cg % 10 == 0) {
378 			error = ffs_syncvnode(vp, MNT_WAIT, 0);
379 			/* vp possibly reclaimed if unlocked */
380 			if (error != 0)
381 				goto out;
382 		}
383 	}
384 	/*
385 	 * Copy all the cylinder group maps. Although the
386 	 * filesystem is still active, we hope that only a few
387 	 * cylinder groups will change between now and when we
388 	 * suspend operations. Thus, we will be able to quickly
389 	 * touch up the few cylinder groups that changed during
390 	 * the suspension period.
391 	 */
392 	len = roundup2(howmany(fs->fs_ncg, NBBY), sizeof(int));
393 	space = malloc(len, M_DEVBUF, M_WAITOK | M_ZERO);
394 	UFS_LOCK(ump);
395 	fs->fs_active = space;
396 	UFS_UNLOCK(ump);
397 	for (cg = 0; cg < fs->fs_ncg; cg++) {
398 		error = UFS_BALLOC(vp, lfragtosize(fs, cgtod(fs, cg)),
399 		    fs->fs_bsize, KERNCRED, 0, &nbp);
400 		if (error)
401 			goto out;
402 		error = cgaccount(cg, vp, nbp, 1);
403 		bawrite(nbp);
404 		if (cg % 10 == 0 && error == 0)
405 			error = ffs_syncvnode(vp, MNT_WAIT, 0);
406 		if (error)
407 			goto out;
408 	}
409 	/*
410 	 * Change inode to snapshot type file.
411 	 */
412 	ip->i_flags |= SF_SNAPSHOT;
413 	DIP_SET(ip, i_flags, ip->i_flags);
414 	UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
415 	/*
416 	 * Ensure that the snapshot is completely on disk.
417 	 * Since we have marked it as a snapshot it is safe to
418 	 * unlock it as no process will be allowed to write to it.
419 	 */
420 	if ((error = ffs_syncvnode(vp, MNT_WAIT, 0)) != 0)
421 		goto out;
422 	VOP_UNLOCK(vp);
423 	/*
424 	 * All allocations are done, so we can now snapshot the system.
425 	 *
426 	 * Recind nice scheduling while running with the filesystem suspended.
427 	 */
428 	if (td->td_proc->p_nice > 0) {
429 		struct proc *p;
430 
431 		p = td->td_proc;
432 		PROC_LOCK(p);
433 		saved_nice = p->p_nice;
434 		sched_nice(p, 0);
435 		PROC_UNLOCK(p);
436 	}
437 	/*
438 	 * Suspend operation on filesystem.
439 	 */
440 	for (;;) {
441 		vn_finished_write(wrtmp);
442 		if ((error = vfs_write_suspend(vp->v_mount, 0)) != 0) {
443 			vn_start_write(NULL, &wrtmp, V_WAIT);
444 			vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
445 			goto out;
446 		}
447 		if (mp->mnt_kern_flag & MNTK_SUSPENDED)
448 			break;
449 		vn_start_write(NULL, &wrtmp, V_WAIT);
450 	}
451 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
452 	if (ip->i_effnlink == 0) {
453 		error = ENOENT;		/* Snapshot file unlinked */
454 		goto resumefs;
455 	}
456 #ifdef DIAGNOSTIC
457 	if (collectsnapstats)
458 		nanotime(&starttime);
459 #endif
460 
461 	/*
462 	 * First, copy all the cylinder group maps that have changed.
463 	 */
464 	for (cg = 0; cg < fs->fs_ncg; cg++) {
465 		if ((ACTIVECGNUM(fs, cg) & ACTIVECGOFF(cg)) != 0)
466 			continue;
467 		redo++;
468 		error = UFS_BALLOC(vp, lfragtosize(fs, cgtod(fs, cg)),
469 		    fs->fs_bsize, KERNCRED, 0, &nbp);
470 		if (error)
471 			goto resumefs;
472 		error = cgaccount(cg, vp, nbp, 2);
473 		bawrite(nbp);
474 		if (error)
475 			goto resumefs;
476 	}
477 	/*
478 	 * Grab a copy of the superblock and its summary information.
479 	 * We delay writing it until the suspension is released below.
480 	 */
481 	copy_fs = malloc((u_long)fs->fs_bsize, M_UFSMNT, M_WAITOK);
482 	bcopy(fs, copy_fs, fs->fs_sbsize);
483 	copy_fs->fs_si = malloc(sizeof(struct fs_summary_info), M_UFSMNT,
484 	    M_ZERO | M_WAITOK);
485 	if ((fs->fs_flags & (FS_UNCLEAN | FS_NEEDSFSCK)) == 0)
486 		copy_fs->fs_clean = 1;
487 	size = fs->fs_bsize < SBLOCKSIZE ? fs->fs_bsize : SBLOCKSIZE;
488 	if (fs->fs_sbsize < size)
489 		bzero(&((char *)copy_fs)[fs->fs_sbsize],
490 		    size - fs->fs_sbsize);
491 	size = blkroundup(fs, fs->fs_cssize);
492 	if (fs->fs_contigsumsize > 0)
493 		size += fs->fs_ncg * sizeof(int32_t);
494 	space = malloc((u_long)size, M_UFSMNT, M_WAITOK);
495 	copy_fs->fs_csp = space;
496 	bcopy(fs->fs_csp, copy_fs->fs_csp, fs->fs_cssize);
497 	space = (char *)space + fs->fs_cssize;
498 	loc = howmany(fs->fs_cssize, fs->fs_fsize);
499 	i = fs->fs_frag - loc % fs->fs_frag;
500 	len = (i == fs->fs_frag) ? 0 : i * fs->fs_fsize;
501 	if (len > 0) {
502 		if ((error = bread(devvp, fsbtodb(fs, fs->fs_csaddr + loc),
503 		    len, KERNCRED, &bp)) != 0) {
504 			brelse(bp);
505 			goto resumefs;
506 		}
507 		bcopy(bp->b_data, space, (u_int)len);
508 		space = (char *)space + len;
509 		bp->b_flags |= B_INVAL | B_NOCACHE;
510 		brelse(bp);
511 	}
512 	if (fs->fs_contigsumsize > 0) {
513 		copy_fs->fs_maxcluster = lp = space;
514 		for (i = 0; i < fs->fs_ncg; i++)
515 			*lp++ = fs->fs_contigsumsize;
516 	}
517 	/*
518 	 * We must check for active files that have been unlinked
519 	 * (e.g., with a zero link count). We have to expunge all
520 	 * trace of these files from the snapshot so that they are
521 	 * not reclaimed prematurely by fsck or unnecessarily dumped.
522 	 * We turn off the MNTK_SUSPENDED flag to avoid a panic from
523 	 * spec_strategy about writing on a suspended filesystem.
524 	 * Note that we skip unlinked snapshot files as they will
525 	 * be handled separately below.
526 	 *
527 	 * We also calculate the size needed for the snapshot list.
528 	 * Initial number of entries is composed of:
529 	 * - one for each cylinder group map
530 	 * - one for each block used by superblock summary table
531 	 * - one for each snapshot inode block
532 	 * - one for the superblock
533 	 * - one for the snapshot list
534 	 * The direct block entries in the snapshot are always
535 	 * copied (see reason below). Note that the superblock and
536 	 * the first cylinder group will almost always be allocated
537 	 * in the direct blocks, but we add the slop for them in case
538 	 * they do not end up there. The snapshot list size may get
539 	 * expanded by one because of an update of an inode block for
540 	 * an unlinked but still open file when it is expunged.
541 	 *
542 	 * Because the direct block pointers are always copied, they
543 	 * are not added to the list. Instead ffs_copyonwrite()
544 	 * explicitly checks for them before checking the snapshot list.
545 	 */
546 	snaplistsize = fs->fs_ncg + howmany(fs->fs_cssize, fs->fs_bsize) +
547 	    FSMAXSNAP + /* superblock */ 1 + /* snaplist */ 1;
548 	MNT_ILOCK(mp);
549 	mp->mnt_kern_flag &= ~MNTK_SUSPENDED;
550 	MNT_IUNLOCK(mp);
551 loop:
552 	MNT_VNODE_FOREACH_ALL(xvp, mp, mvp) {
553 		if ((xvp->v_usecount == 0 &&
554 		     (xvp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == 0) ||
555 		    xvp->v_type == VNON ||
556 		    IS_SNAPSHOT(VTOI(xvp))) {
557 			VI_UNLOCK(xvp);
558 			continue;
559 		}
560 		/*
561 		 * We can skip parent directory vnode because it must have
562 		 * this snapshot file in it.
563 		 */
564 		if (xvp == nd.ni_dvp) {
565 			VI_UNLOCK(xvp);
566 			continue;
567 		}
568 		vholdl(xvp);
569 		if (vn_lock(xvp, LK_EXCLUSIVE | LK_INTERLOCK) != 0) {
570 			MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
571 			vdrop(xvp);
572 			goto loop;
573 		}
574 		VI_LOCK(xvp);
575 		if (xvp->v_usecount == 0 &&
576 		    (xvp->v_iflag & (VI_OWEINACT | VI_DOINGINACT)) == 0) {
577 			VI_UNLOCK(xvp);
578 			VOP_UNLOCK(xvp);
579 			vdrop(xvp);
580 			continue;
581 		}
582 		VI_UNLOCK(xvp);
583 #ifdef DIAGNOSTIC
584 		if (snapdebug)
585 			vn_printf(xvp, "ffs_snapshot: busy vnode ");
586 #endif
587 		if (VOP_GETATTR(xvp, &vat, td->td_ucred) == 0 &&
588 		    vat.va_nlink > 0) {
589 			VOP_UNLOCK(xvp);
590 			vdrop(xvp);
591 			continue;
592 		}
593 		xp = VTOI(xvp);
594 		if (ffs_checkfreefile(copy_fs, vp, xp->i_number)) {
595 			VOP_UNLOCK(xvp);
596 			vdrop(xvp);
597 			continue;
598 		}
599 		/*
600 		 * If there is a fragment, clear it here.
601 		 */
602 		blkno = 0;
603 		loc = howmany(xp->i_size, fs->fs_bsize) - 1;
604 		if (loc < UFS_NDADDR) {
605 			len = fragroundup(fs, blkoff(fs, xp->i_size));
606 			if (len != 0 && len < fs->fs_bsize) {
607 				ffs_blkfree(ump, copy_fs, vp,
608 				    DIP(xp, i_db[loc]), len, xp->i_number,
609 				    xvp->v_type, NULL, SINGLETON_KEY);
610 				blkno = DIP(xp, i_db[loc]);
611 				DIP_SET(xp, i_db[loc], 0);
612 			}
613 		}
614 		snaplistsize += 1;
615 		if (I_IS_UFS1(xp))
616 			error = expunge_ufs1(vp, xp, copy_fs, fullacct_ufs1,
617 			    BLK_NOCOPY, 1);
618 		else
619 			error = expunge_ufs2(vp, xp, copy_fs, fullacct_ufs2,
620 			    BLK_NOCOPY, 1);
621 		if (blkno)
622 			DIP_SET(xp, i_db[loc], blkno);
623 		if (!error)
624 			error = ffs_freefile(ump, copy_fs, vp, xp->i_number,
625 			    xp->i_mode, NULL);
626 		VOP_UNLOCK(xvp);
627 		vdrop(xvp);
628 		if (error) {
629 			MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
630 			goto resumefs;
631 		}
632 	}
633 	/*
634 	 * Erase the journal file from the snapshot.
635 	 */
636 	if (fs->fs_flags & FS_SUJ) {
637 		error = softdep_journal_lookup(mp, &xvp);
638 		if (error)
639 			goto resumefs;
640 		xp = VTOI(xvp);
641 		if (I_IS_UFS1(xp))
642 			error = expunge_ufs1(vp, xp, copy_fs, fullacct_ufs1,
643 			    BLK_NOCOPY, 0);
644 		else
645 			error = expunge_ufs2(vp, xp, copy_fs, fullacct_ufs2,
646 			    BLK_NOCOPY, 0);
647 		vput(xvp);
648 	}
649 	/*
650 	 * Acquire a lock on the snapdata structure, creating it if necessary.
651 	 */
652 	sn = ffs_snapdata_acquire(devvp);
653 	/*
654 	 * Change vnode to use shared snapshot lock instead of the original
655 	 * private lock.
656 	 */
657 	vp->v_vnlock = &sn->sn_lock;
658 	lockmgr(&vp->v_lock, LK_RELEASE, NULL);
659 	xp = TAILQ_FIRST(&sn->sn_head);
660 	/*
661 	 * If this is the first snapshot on this filesystem, then we need
662 	 * to allocate the space for the list of preallocated snapshot blocks.
663 	 * This list will be refined below, but this preliminary one will
664 	 * keep us out of deadlock until the full one is ready.
665 	 */
666 	if (xp == NULL) {
667 		snapblklist = malloc(snaplistsize * sizeof(daddr_t),
668 		    M_UFSMNT, M_WAITOK);
669 		blkp = &snapblklist[1];
670 		*blkp++ = lblkno(fs, fs->fs_sblockloc);
671 		blkno = fragstoblks(fs, fs->fs_csaddr);
672 		for (cg = 0; cg < fs->fs_ncg; cg++) {
673 			if (fragstoblks(fs, cgtod(fs, cg) > blkno))
674 				break;
675 			*blkp++ = fragstoblks(fs, cgtod(fs, cg));
676 		}
677 		len = howmany(fs->fs_cssize, fs->fs_bsize);
678 		for (loc = 0; loc < len; loc++)
679 			*blkp++ = blkno + loc;
680 		for (; cg < fs->fs_ncg; cg++)
681 			*blkp++ = fragstoblks(fs, cgtod(fs, cg));
682 		snapblklist[0] = blkp - snapblklist;
683 		VI_LOCK(devvp);
684 		if (sn->sn_blklist != NULL)
685 			panic("ffs_snapshot: non-empty list");
686 		sn->sn_blklist = snapblklist;
687 		sn->sn_listsize = blkp - snapblklist;
688 		VI_UNLOCK(devvp);
689 	}
690 	/*
691 	 * Preallocate all the direct blocks in the snapshot inode so
692 	 * that we never have to write the inode itself to commit an
693 	 * update to the contents of the snapshot. Note that once
694 	 * created, the size of the snapshot will never change, so
695 	 * there will never be a need to write the inode except to
696 	 * update the non-integrity-critical time fields and
697 	 * allocated-block count.
698 	 */
699 	for (blockno = 0; blockno < UFS_NDADDR; blockno++) {
700 		if (DIP(ip, i_db[blockno]) != 0)
701 			continue;
702 		error = UFS_BALLOC(vp, lblktosize(fs, blockno),
703 		    fs->fs_bsize, KERNCRED, BA_CLRBUF, &bp);
704 		if (error)
705 			goto resumefs;
706 		error = readblock(vp, bp, blockno);
707 		bawrite(bp);
708 		if (error != 0)
709 			goto resumefs;
710 	}
711 	/*
712 	 * Record snapshot inode. Since this is the newest snapshot,
713 	 * it must be placed at the end of the list.
714 	 */
715 	VI_LOCK(devvp);
716 	fs->fs_snapinum[snaploc] = ip->i_number;
717 	if (ip->i_nextsnap.tqe_prev != 0)
718 		panic("ffs_snapshot: %ju already on list",
719 		    (uintmax_t)ip->i_number);
720 	TAILQ_INSERT_TAIL(&sn->sn_head, ip, i_nextsnap);
721 	devvp->v_vflag |= VV_COPYONWRITE;
722 	VI_UNLOCK(devvp);
723 resumefs:
724 	ASSERT_VOP_LOCKED(vp, "ffs_snapshot vp");
725 	if (error != 0 && copy_fs != NULL) {
726 		free(copy_fs->fs_csp, M_UFSMNT);
727 		free(copy_fs->fs_si, M_UFSMNT);
728 		free(copy_fs, M_UFSMNT);
729 		copy_fs = NULL;
730 	}
731 	KASSERT(error != 0 || (sn != NULL && copy_fs != NULL),
732 		("missing snapshot setup parameters"));
733 	/*
734 	 * Resume operation on filesystem.
735 	 */
736 	vfs_write_resume(vp->v_mount, VR_START_WRITE | VR_NO_SUSPCLR);
737 #ifdef DIAGNOSTIC
738 	if (collectsnapstats && starttime.tv_sec > 0) {
739 		nanotime(&endtime);
740 		timespecsub(&endtime, &starttime, &endtime);
741 		printf("%s: suspended %ld.%03ld sec, redo %ld of %d\n",
742 		    vp->v_mount->mnt_stat.f_mntonname, (long)endtime.tv_sec,
743 		    endtime.tv_nsec / 1000000, redo, fs->fs_ncg);
744 	}
745 #endif
746 	if (copy_fs == NULL)
747 		goto out;
748 	/*
749 	 * Copy allocation information from all the snapshots in
750 	 * this snapshot and then expunge them from its view.
751 	 */
752 	TAILQ_FOREACH(xp, &sn->sn_head, i_nextsnap) {
753 		if (xp == ip)
754 			break;
755 		if (I_IS_UFS1(xp))
756 			error = expunge_ufs1(vp, xp, fs, snapacct_ufs1,
757 			    BLK_SNAP, 0);
758 		else
759 			error = expunge_ufs2(vp, xp, fs, snapacct_ufs2,
760 			    BLK_SNAP, 0);
761 		if (error == 0 && xp->i_effnlink == 0) {
762 			error = ffs_freefile(ump,
763 					     copy_fs,
764 					     vp,
765 					     xp->i_number,
766 					     xp->i_mode, NULL);
767 		}
768 		if (error) {
769 			fs->fs_snapinum[snaploc] = 0;
770 			goto done;
771 		}
772 	}
773 	/*
774 	 * Allocate space for the full list of preallocated snapshot blocks.
775 	 */
776 	snapblklist = malloc(snaplistsize * sizeof(daddr_t),
777 	    M_UFSMNT, M_WAITOK);
778 	ip->i_snapblklist = &snapblklist[1];
779 	/*
780 	 * Expunge the blocks used by the snapshots from the set of
781 	 * blocks marked as used in the snapshot bitmaps. Also, collect
782 	 * the list of allocated blocks in i_snapblklist.
783 	 */
784 	if (I_IS_UFS1(ip))
785 		error = expunge_ufs1(vp, ip, copy_fs, mapacct_ufs1,
786 		    BLK_SNAP, 0);
787 	else
788 		error = expunge_ufs2(vp, ip, copy_fs, mapacct_ufs2,
789 		    BLK_SNAP, 0);
790 	if (error) {
791 		fs->fs_snapinum[snaploc] = 0;
792 		free(snapblklist, M_UFSMNT);
793 		goto done;
794 	}
795 	if (snaplistsize < ip->i_snapblklist - snapblklist)
796 		panic("ffs_snapshot: list too small");
797 	snaplistsize = ip->i_snapblklist - snapblklist;
798 	snapblklist[0] = snaplistsize;
799 	ip->i_snapblklist = 0;
800 	/*
801 	 * Write out the list of allocated blocks to the end of the snapshot.
802 	 */
803 	auio.uio_iov = &aiov;
804 	auio.uio_iovcnt = 1;
805 	aiov.iov_base = (void *)snapblklist;
806 	aiov.iov_len = snaplistsize * sizeof(daddr_t);
807 	auio.uio_resid = aiov.iov_len;
808 	auio.uio_offset = lblktosize(fs, (off_t)numblks);
809 	auio.uio_segflg = UIO_SYSSPACE;
810 	auio.uio_rw = UIO_WRITE;
811 	auio.uio_td = td;
812 	if ((error = VOP_WRITE(vp, &auio, IO_UNIT, td->td_ucred)) != 0) {
813 		fs->fs_snapinum[snaploc] = 0;
814 		free(snapblklist, M_UFSMNT);
815 		goto done;
816 	}
817 	/*
818 	 * Write the superblock and its summary information
819 	 * to the snapshot.
820 	 */
821 	blkno = fragstoblks(fs, fs->fs_csaddr);
822 	len = howmany(fs->fs_cssize, fs->fs_bsize);
823 	space = copy_fs->fs_csp;
824 	for (loc = 0; loc < len; loc++) {
825 		error = bread(vp, blkno + loc, fs->fs_bsize, KERNCRED, &nbp);
826 		if (error) {
827 			fs->fs_snapinum[snaploc] = 0;
828 			free(snapblklist, M_UFSMNT);
829 			goto done;
830 		}
831 		bcopy(space, nbp->b_data, fs->fs_bsize);
832 		space = (char *)space + fs->fs_bsize;
833 		bawrite(nbp);
834 	}
835 	error = bread(vp, lblkno(fs, fs->fs_sblockloc), fs->fs_bsize,
836 	    KERNCRED, &nbp);
837 	if (error) {
838 		brelse(nbp);
839 	} else {
840 		loc = blkoff(fs, fs->fs_sblockloc);
841 		copy_fs->fs_fmod = 0;
842 		copy_fs->fs_ckhash = ffs_calc_sbhash(copy_fs);
843 		bcopy((char *)copy_fs, &nbp->b_data[loc], (u_int)fs->fs_sbsize);
844 		bawrite(nbp);
845 	}
846 	/*
847 	 * As this is the newest list, it is the most inclusive, so
848 	 * should replace the previous list.
849 	 */
850 	VI_LOCK(devvp);
851 	space = sn->sn_blklist;
852 	sn->sn_blklist = snapblklist;
853 	sn->sn_listsize = snaplistsize;
854 	VI_UNLOCK(devvp);
855 	if (space != NULL)
856 		free(space, M_UFSMNT);
857 done:
858 	free(copy_fs->fs_csp, M_UFSMNT);
859 	free(copy_fs->fs_si, M_UFSMNT);
860 	free(copy_fs, M_UFSMNT);
861 	copy_fs = NULL;
862 out:
863 	NDFREE(&nd, NDF_ONLY_PNBUF);
864 	if (saved_nice > 0) {
865 		struct proc *p;
866 
867 		p = td->td_proc;
868 		PROC_LOCK(p);
869 		sched_nice(td->td_proc, saved_nice);
870 		PROC_UNLOCK(td->td_proc);
871 	}
872 	UFS_LOCK(ump);
873 	if (fs->fs_active != 0) {
874 		free(fs->fs_active, M_DEVBUF);
875 		fs->fs_active = 0;
876 	}
877 	UFS_UNLOCK(ump);
878 	MNT_ILOCK(mp);
879 	mp->mnt_flag = (mp->mnt_flag & MNT_QUOTA) | (flag & ~MNT_QUOTA);
880 	MNT_IUNLOCK(mp);
881 	if (error)
882 		(void) ffs_truncate(vp, (off_t)0, 0, NOCRED);
883 	(void) ffs_syncvnode(vp, MNT_WAIT, 0);
884 	if (error)
885 		vput(vp);
886 	else
887 		VOP_UNLOCK(vp);
888 	vrele(nd.ni_dvp);
889 	vn_finished_write(wrtmp);
890 	process_deferred_inactive(mp);
891 	return (error);
892 }
893 
894 /*
895  * Copy a cylinder group map. All the unallocated blocks are marked
896  * BLK_NOCOPY so that the snapshot knows that it need not copy them
897  * if they are later written. If passno is one, then this is a first
898  * pass, so only setting needs to be done. If passno is 2, then this
899  * is a revision to a previous pass which must be undone as the
900  * replacement pass is done.
901  */
902 static int
903 cgaccount(cg, vp, nbp, passno)
904 	int cg;
905 	struct vnode *vp;
906 	struct buf *nbp;
907 	int passno;
908 {
909 	struct buf *bp, *ibp;
910 	struct inode *ip;
911 	struct cg *cgp;
912 	struct fs *fs;
913 	ufs2_daddr_t base, numblks;
914 	int error, len, loc, indiroff;
915 
916 	ip = VTOI(vp);
917 	fs = ITOFS(ip);
918 	if ((error = ffs_getcg(fs, ITODEVVP(ip), cg, 0, &bp, &cgp)) != 0)
919 		return (error);
920 	UFS_LOCK(ITOUMP(ip));
921 	ACTIVESET(fs, cg);
922 	/*
923 	 * Recomputation of summary information might not have been performed
924 	 * at mount time.  Sync up summary information for current cylinder
925 	 * group while data is in memory to ensure that result of background
926 	 * fsck is slightly more consistent.
927 	 */
928 	fs->fs_cs(fs, cg) = cgp->cg_cs;
929 	UFS_UNLOCK(ITOUMP(ip));
930 	bcopy(bp->b_data, nbp->b_data, fs->fs_cgsize);
931 	if (fs->fs_cgsize < fs->fs_bsize)
932 		bzero(&nbp->b_data[fs->fs_cgsize],
933 		    fs->fs_bsize - fs->fs_cgsize);
934 	cgp = (struct cg *)nbp->b_data;
935 	bqrelse(bp);
936 	if (passno == 2)
937 		nbp->b_flags |= B_VALIDSUSPWRT;
938 	numblks = howmany(fs->fs_size, fs->fs_frag);
939 	len = howmany(fs->fs_fpg, fs->fs_frag);
940 	base = cgbase(fs, cg) / fs->fs_frag;
941 	if (base + len >= numblks)
942 		len = numblks - base - 1;
943 	loc = 0;
944 	if (base < UFS_NDADDR) {
945 		for ( ; loc < UFS_NDADDR; loc++) {
946 			if (ffs_isblock(fs, cg_blksfree(cgp), loc))
947 				DIP_SET(ip, i_db[loc], BLK_NOCOPY);
948 			else if (passno == 2 && DIP(ip, i_db[loc])== BLK_NOCOPY)
949 				DIP_SET(ip, i_db[loc], 0);
950 			else if (passno == 1 && DIP(ip, i_db[loc])== BLK_NOCOPY)
951 				panic("ffs_snapshot: lost direct block");
952 		}
953 	}
954 	error = UFS_BALLOC(vp, lblktosize(fs, (off_t)(base + loc)),
955 	    fs->fs_bsize, KERNCRED, BA_METAONLY, &ibp);
956 	if (error) {
957 		goto out;
958 	}
959 	indiroff = (base + loc - UFS_NDADDR) % NINDIR(fs);
960 	for ( ; loc < len; loc++, indiroff++) {
961 		if (indiroff >= NINDIR(fs)) {
962 			if (passno == 2)
963 				ibp->b_flags |= B_VALIDSUSPWRT;
964 			bawrite(ibp);
965 			error = UFS_BALLOC(vp,
966 			    lblktosize(fs, (off_t)(base + loc)),
967 			    fs->fs_bsize, KERNCRED, BA_METAONLY, &ibp);
968 			if (error) {
969 				goto out;
970 			}
971 			indiroff = 0;
972 		}
973 		if (I_IS_UFS1(ip)) {
974 			if (ffs_isblock(fs, cg_blksfree(cgp), loc))
975 				((ufs1_daddr_t *)(ibp->b_data))[indiroff] =
976 				    BLK_NOCOPY;
977 			else if (passno == 2 && ((ufs1_daddr_t *)(ibp->b_data))
978 			    [indiroff] == BLK_NOCOPY)
979 				((ufs1_daddr_t *)(ibp->b_data))[indiroff] = 0;
980 			else if (passno == 1 && ((ufs1_daddr_t *)(ibp->b_data))
981 			    [indiroff] == BLK_NOCOPY)
982 				panic("ffs_snapshot: lost indirect block");
983 			continue;
984 		}
985 		if (ffs_isblock(fs, cg_blksfree(cgp), loc))
986 			((ufs2_daddr_t *)(ibp->b_data))[indiroff] = BLK_NOCOPY;
987 		else if (passno == 2 &&
988 		    ((ufs2_daddr_t *)(ibp->b_data)) [indiroff] == BLK_NOCOPY)
989 			((ufs2_daddr_t *)(ibp->b_data))[indiroff] = 0;
990 		else if (passno == 1 &&
991 		    ((ufs2_daddr_t *)(ibp->b_data)) [indiroff] == BLK_NOCOPY)
992 			panic("ffs_snapshot: lost indirect block");
993 	}
994 	if (passno == 2)
995 		ibp->b_flags |= B_VALIDSUSPWRT;
996 	bdwrite(ibp);
997 out:
998 	/*
999 	 * We have to calculate the crc32c here rather than just setting the
1000 	 * BX_CYLGRP b_xflags because the allocation of the block for the
1001 	 * the cylinder group map will always be a full size block (fs_bsize)
1002 	 * even though the cylinder group may be smaller (fs_cgsize). The
1003 	 * crc32c must be computed only over fs_cgsize whereas the BX_CYLGRP
1004 	 * flag causes it to be computed over the size of the buffer.
1005 	 */
1006 	if ((fs->fs_metackhash & CK_CYLGRP) != 0) {
1007 		((struct cg *)nbp->b_data)->cg_ckhash = 0;
1008 		((struct cg *)nbp->b_data)->cg_ckhash =
1009 		    calculate_crc32c(~0L, nbp->b_data, fs->fs_cgsize);
1010 	}
1011 	return (error);
1012 }
1013 
1014 /*
1015  * Before expunging a snapshot inode, note all the
1016  * blocks that it claims with BLK_SNAP so that fsck will
1017  * be able to account for those blocks properly and so
1018  * that this snapshot knows that it need not copy them
1019  * if the other snapshot holding them is freed. This code
1020  * is reproduced once each for UFS1 and UFS2.
1021  */
1022 static int
1023 expunge_ufs1(snapvp, cancelip, fs, acctfunc, expungetype, clearmode)
1024 	struct vnode *snapvp;
1025 	struct inode *cancelip;
1026 	struct fs *fs;
1027 	int (*acctfunc)(struct vnode *, ufs1_daddr_t *, ufs1_daddr_t *,
1028 	    struct fs *, ufs_lbn_t, int);
1029 	int expungetype;
1030 	int clearmode;
1031 {
1032 	int i, error, indiroff;
1033 	ufs_lbn_t lbn, rlbn;
1034 	ufs2_daddr_t len, blkno, numblks, blksperindir;
1035 	struct ufs1_dinode *dip;
1036 	struct thread *td = curthread;
1037 	struct buf *bp;
1038 
1039 	/*
1040 	 * Prepare to expunge the inode. If its inode block has not
1041 	 * yet been copied, then allocate and fill the copy.
1042 	 */
1043 	lbn = fragstoblks(fs, ino_to_fsba(fs, cancelip->i_number));
1044 	blkno = 0;
1045 	if (lbn < UFS_NDADDR) {
1046 		blkno = VTOI(snapvp)->i_din1->di_db[lbn];
1047 	} else {
1048 		if (DOINGSOFTDEP(snapvp))
1049 			softdep_prealloc(snapvp, MNT_WAIT);
1050 		td->td_pflags |= TDP_COWINPROGRESS;
1051 		error = ffs_balloc_ufs1(snapvp, lblktosize(fs, (off_t)lbn),
1052 		   fs->fs_bsize, KERNCRED, BA_METAONLY, &bp);
1053 		td->td_pflags &= ~TDP_COWINPROGRESS;
1054 		if (error)
1055 			return (error);
1056 		indiroff = (lbn - UFS_NDADDR) % NINDIR(fs);
1057 		blkno = ((ufs1_daddr_t *)(bp->b_data))[indiroff];
1058 		bqrelse(bp);
1059 	}
1060 	if (blkno != 0) {
1061 		if ((error = bread(snapvp, lbn, fs->fs_bsize, KERNCRED, &bp)))
1062 			return (error);
1063 	} else {
1064 		error = ffs_balloc_ufs1(snapvp, lblktosize(fs, (off_t)lbn),
1065 		    fs->fs_bsize, KERNCRED, 0, &bp);
1066 		if (error)
1067 			return (error);
1068 		if ((error = readblock(snapvp, bp, lbn)) != 0)
1069 			return (error);
1070 	}
1071 	/*
1072 	 * Set a snapshot inode to be a zero length file, regular files
1073 	 * or unlinked snapshots to be completely unallocated.
1074 	 */
1075 	dip = (struct ufs1_dinode *)bp->b_data +
1076 	    ino_to_fsbo(fs, cancelip->i_number);
1077 	if (clearmode || cancelip->i_effnlink == 0)
1078 		dip->di_mode = 0;
1079 	dip->di_size = 0;
1080 	dip->di_blocks = 0;
1081 	dip->di_flags &= ~SF_SNAPSHOT;
1082 	bzero(&dip->di_db[0], (UFS_NDADDR + UFS_NIADDR) * sizeof(ufs1_daddr_t));
1083 	bdwrite(bp);
1084 	/*
1085 	 * Now go through and expunge all the blocks in the file
1086 	 * using the function requested.
1087 	 */
1088 	numblks = howmany(cancelip->i_size, fs->fs_bsize);
1089 	if ((error = (*acctfunc)(snapvp, &cancelip->i_din1->di_db[0],
1090 	    &cancelip->i_din1->di_db[UFS_NDADDR], fs, 0, expungetype)))
1091 		return (error);
1092 	if ((error = (*acctfunc)(snapvp, &cancelip->i_din1->di_ib[0],
1093 	    &cancelip->i_din1->di_ib[UFS_NIADDR], fs, -1, expungetype)))
1094 		return (error);
1095 	blksperindir = 1;
1096 	lbn = -UFS_NDADDR;
1097 	len = numblks - UFS_NDADDR;
1098 	rlbn = UFS_NDADDR;
1099 	for (i = 0; len > 0 && i < UFS_NIADDR; i++) {
1100 		error = indiracct_ufs1(snapvp, ITOV(cancelip), i,
1101 		    cancelip->i_din1->di_ib[i], lbn, rlbn, len,
1102 		    blksperindir, fs, acctfunc, expungetype);
1103 		if (error)
1104 			return (error);
1105 		blksperindir *= NINDIR(fs);
1106 		lbn -= blksperindir + 1;
1107 		len -= blksperindir;
1108 		rlbn += blksperindir;
1109 	}
1110 	return (0);
1111 }
1112 
1113 /*
1114  * Descend an indirect block chain for vnode cancelvp accounting for all
1115  * its indirect blocks in snapvp.
1116  */
1117 static int
1118 indiracct_ufs1(snapvp, cancelvp, level, blkno, lbn, rlbn, remblks,
1119 	    blksperindir, fs, acctfunc, expungetype)
1120 	struct vnode *snapvp;
1121 	struct vnode *cancelvp;
1122 	int level;
1123 	ufs1_daddr_t blkno;
1124 	ufs_lbn_t lbn;
1125 	ufs_lbn_t rlbn;
1126 	ufs_lbn_t remblks;
1127 	ufs_lbn_t blksperindir;
1128 	struct fs *fs;
1129 	int (*acctfunc)(struct vnode *, ufs1_daddr_t *, ufs1_daddr_t *,
1130 	    struct fs *, ufs_lbn_t, int);
1131 	int expungetype;
1132 {
1133 	int error, num, i;
1134 	ufs_lbn_t subblksperindir;
1135 	struct indir indirs[UFS_NIADDR + 2];
1136 	ufs1_daddr_t last, *bap;
1137 	struct buf *bp;
1138 
1139 	if (blkno == 0) {
1140 		if (expungetype == BLK_NOCOPY)
1141 			return (0);
1142 		panic("indiracct_ufs1: missing indir");
1143 	}
1144 	if ((error = ufs_getlbns(cancelvp, rlbn, indirs, &num)) != 0)
1145 		return (error);
1146 	if (lbn != indirs[num - 1 - level].in_lbn || num < 2)
1147 		panic("indiracct_ufs1: botched params");
1148 	/*
1149 	 * We have to expand bread here since it will deadlock looking
1150 	 * up the block number for any blocks that are not in the cache.
1151 	 */
1152 	bp = getblk(cancelvp, lbn, fs->fs_bsize, 0, 0, 0);
1153 	bp->b_blkno = fsbtodb(fs, blkno);
1154 	if ((bp->b_flags & (B_DONE | B_DELWRI)) == 0 &&
1155 	    (error = readblock(cancelvp, bp, fragstoblks(fs, blkno)))) {
1156 		brelse(bp);
1157 		return (error);
1158 	}
1159 	/*
1160 	 * Account for the block pointers in this indirect block.
1161 	 */
1162 	last = howmany(remblks, blksperindir);
1163 	if (last > NINDIR(fs))
1164 		last = NINDIR(fs);
1165 	bap = malloc(fs->fs_bsize, M_DEVBUF, M_WAITOK);
1166 	bcopy(bp->b_data, (caddr_t)bap, fs->fs_bsize);
1167 	bqrelse(bp);
1168 	error = (*acctfunc)(snapvp, &bap[0], &bap[last], fs,
1169 	    level == 0 ? rlbn : -1, expungetype);
1170 	if (error || level == 0)
1171 		goto out;
1172 	/*
1173 	 * Account for the block pointers in each of the indirect blocks
1174 	 * in the levels below us.
1175 	 */
1176 	subblksperindir = blksperindir / NINDIR(fs);
1177 	for (lbn++, level--, i = 0; i < last; i++) {
1178 		error = indiracct_ufs1(snapvp, cancelvp, level, bap[i], lbn,
1179 		    rlbn, remblks, subblksperindir, fs, acctfunc, expungetype);
1180 		if (error)
1181 			goto out;
1182 		rlbn += blksperindir;
1183 		lbn -= blksperindir;
1184 		remblks -= blksperindir;
1185 	}
1186 out:
1187 	free(bap, M_DEVBUF);
1188 	return (error);
1189 }
1190 
1191 /*
1192  * Do both snap accounting and map accounting.
1193  */
1194 static int
1195 fullacct_ufs1(vp, oldblkp, lastblkp, fs, lblkno, exptype)
1196 	struct vnode *vp;
1197 	ufs1_daddr_t *oldblkp, *lastblkp;
1198 	struct fs *fs;
1199 	ufs_lbn_t lblkno;
1200 	int exptype;	/* BLK_SNAP or BLK_NOCOPY */
1201 {
1202 	int error;
1203 
1204 	if ((error = snapacct_ufs1(vp, oldblkp, lastblkp, fs, lblkno, exptype)))
1205 		return (error);
1206 	return (mapacct_ufs1(vp, oldblkp, lastblkp, fs, lblkno, exptype));
1207 }
1208 
1209 /*
1210  * Identify a set of blocks allocated in a snapshot inode.
1211  */
1212 static int
1213 snapacct_ufs1(vp, oldblkp, lastblkp, fs, lblkno, expungetype)
1214 	struct vnode *vp;
1215 	ufs1_daddr_t *oldblkp, *lastblkp;
1216 	struct fs *fs;
1217 	ufs_lbn_t lblkno;
1218 	int expungetype;	/* BLK_SNAP or BLK_NOCOPY */
1219 {
1220 	struct inode *ip = VTOI(vp);
1221 	ufs1_daddr_t blkno, *blkp;
1222 	ufs_lbn_t lbn;
1223 	struct buf *ibp;
1224 	int error;
1225 
1226 	for ( ; oldblkp < lastblkp; oldblkp++) {
1227 		blkno = *oldblkp;
1228 		if (blkno == 0 || blkno == BLK_NOCOPY || blkno == BLK_SNAP)
1229 			continue;
1230 		lbn = fragstoblks(fs, blkno);
1231 		if (lbn < UFS_NDADDR) {
1232 			blkp = &ip->i_din1->di_db[lbn];
1233 			UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
1234 		} else {
1235 			error = ffs_balloc_ufs1(vp, lblktosize(fs, (off_t)lbn),
1236 			    fs->fs_bsize, KERNCRED, BA_METAONLY, &ibp);
1237 			if (error)
1238 				return (error);
1239 			blkp = &((ufs1_daddr_t *)(ibp->b_data))
1240 			    [(lbn - UFS_NDADDR) % NINDIR(fs)];
1241 		}
1242 		/*
1243 		 * If we are expunging a snapshot vnode and we
1244 		 * find a block marked BLK_NOCOPY, then it is
1245 		 * one that has been allocated to this snapshot after
1246 		 * we took our current snapshot and can be ignored.
1247 		 */
1248 		if (expungetype == BLK_SNAP && *blkp == BLK_NOCOPY) {
1249 			if (lbn >= UFS_NDADDR)
1250 				brelse(ibp);
1251 		} else {
1252 			if (*blkp != 0)
1253 				panic("snapacct_ufs1: bad block");
1254 			*blkp = expungetype;
1255 			if (lbn >= UFS_NDADDR)
1256 				bdwrite(ibp);
1257 		}
1258 	}
1259 	return (0);
1260 }
1261 
1262 /*
1263  * Account for a set of blocks allocated in a snapshot inode.
1264  */
1265 static int
1266 mapacct_ufs1(vp, oldblkp, lastblkp, fs, lblkno, expungetype)
1267 	struct vnode *vp;
1268 	ufs1_daddr_t *oldblkp, *lastblkp;
1269 	struct fs *fs;
1270 	ufs_lbn_t lblkno;
1271 	int expungetype;
1272 {
1273 	ufs1_daddr_t blkno;
1274 	struct inode *ip;
1275 	ino_t inum;
1276 	int acctit;
1277 
1278 	ip = VTOI(vp);
1279 	inum = ip->i_number;
1280 	if (lblkno == -1)
1281 		acctit = 0;
1282 	else
1283 		acctit = 1;
1284 	for ( ; oldblkp < lastblkp; oldblkp++, lblkno++) {
1285 		blkno = *oldblkp;
1286 		if (blkno == 0 || blkno == BLK_NOCOPY)
1287 			continue;
1288 		if (acctit && expungetype == BLK_SNAP && blkno != BLK_SNAP)
1289 			*ip->i_snapblklist++ = lblkno;
1290 		if (blkno == BLK_SNAP)
1291 			blkno = blkstofrags(fs, lblkno);
1292 		ffs_blkfree(ITOUMP(ip), fs, vp, blkno, fs->fs_bsize, inum,
1293 		    vp->v_type, NULL, SINGLETON_KEY);
1294 	}
1295 	return (0);
1296 }
1297 
1298 /*
1299  * Before expunging a snapshot inode, note all the
1300  * blocks that it claims with BLK_SNAP so that fsck will
1301  * be able to account for those blocks properly and so
1302  * that this snapshot knows that it need not copy them
1303  * if the other snapshot holding them is freed. This code
1304  * is reproduced once each for UFS1 and UFS2.
1305  */
1306 static int
1307 expunge_ufs2(snapvp, cancelip, fs, acctfunc, expungetype, clearmode)
1308 	struct vnode *snapvp;
1309 	struct inode *cancelip;
1310 	struct fs *fs;
1311 	int (*acctfunc)(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
1312 	    struct fs *, ufs_lbn_t, int);
1313 	int expungetype;
1314 	int clearmode;
1315 {
1316 	int i, error, indiroff;
1317 	ufs_lbn_t lbn, rlbn;
1318 	ufs2_daddr_t len, blkno, numblks, blksperindir;
1319 	struct ufs2_dinode *dip;
1320 	struct thread *td = curthread;
1321 	struct buf *bp;
1322 
1323 	/*
1324 	 * Prepare to expunge the inode. If its inode block has not
1325 	 * yet been copied, then allocate and fill the copy.
1326 	 */
1327 	lbn = fragstoblks(fs, ino_to_fsba(fs, cancelip->i_number));
1328 	blkno = 0;
1329 	if (lbn < UFS_NDADDR) {
1330 		blkno = VTOI(snapvp)->i_din2->di_db[lbn];
1331 	} else {
1332 		if (DOINGSOFTDEP(snapvp))
1333 			softdep_prealloc(snapvp, MNT_WAIT);
1334 		td->td_pflags |= TDP_COWINPROGRESS;
1335 		error = ffs_balloc_ufs2(snapvp, lblktosize(fs, (off_t)lbn),
1336 		   fs->fs_bsize, KERNCRED, BA_METAONLY, &bp);
1337 		td->td_pflags &= ~TDP_COWINPROGRESS;
1338 		if (error)
1339 			return (error);
1340 		indiroff = (lbn - UFS_NDADDR) % NINDIR(fs);
1341 		blkno = ((ufs2_daddr_t *)(bp->b_data))[indiroff];
1342 		bqrelse(bp);
1343 	}
1344 	if (blkno != 0) {
1345 		if ((error = bread(snapvp, lbn, fs->fs_bsize, KERNCRED, &bp)))
1346 			return (error);
1347 	} else {
1348 		error = ffs_balloc_ufs2(snapvp, lblktosize(fs, (off_t)lbn),
1349 		    fs->fs_bsize, KERNCRED, 0, &bp);
1350 		if (error)
1351 			return (error);
1352 		if ((error = readblock(snapvp, bp, lbn)) != 0)
1353 			return (error);
1354 	}
1355 	/*
1356 	 * Set a snapshot inode to be a zero length file, regular files
1357 	 * to be completely unallocated.
1358 	 */
1359 	dip = (struct ufs2_dinode *)bp->b_data +
1360 	    ino_to_fsbo(fs, cancelip->i_number);
1361 	dip->di_size = 0;
1362 	dip->di_blocks = 0;
1363 	dip->di_flags &= ~SF_SNAPSHOT;
1364 	bzero(&dip->di_db[0], (UFS_NDADDR + UFS_NIADDR) * sizeof(ufs2_daddr_t));
1365 	if (clearmode || cancelip->i_effnlink == 0)
1366 		dip->di_mode = 0;
1367 	else
1368 		ffs_update_dinode_ckhash(fs, dip);
1369 	bdwrite(bp);
1370 	/*
1371 	 * Now go through and expunge all the blocks in the file
1372 	 * using the function requested.
1373 	 */
1374 	numblks = howmany(cancelip->i_size, fs->fs_bsize);
1375 	if ((error = (*acctfunc)(snapvp, &cancelip->i_din2->di_db[0],
1376 	    &cancelip->i_din2->di_db[UFS_NDADDR], fs, 0, expungetype)))
1377 		return (error);
1378 	if ((error = (*acctfunc)(snapvp, &cancelip->i_din2->di_ib[0],
1379 	    &cancelip->i_din2->di_ib[UFS_NIADDR], fs, -1, expungetype)))
1380 		return (error);
1381 	blksperindir = 1;
1382 	lbn = -UFS_NDADDR;
1383 	len = numblks - UFS_NDADDR;
1384 	rlbn = UFS_NDADDR;
1385 	for (i = 0; len > 0 && i < UFS_NIADDR; i++) {
1386 		error = indiracct_ufs2(snapvp, ITOV(cancelip), i,
1387 		    cancelip->i_din2->di_ib[i], lbn, rlbn, len,
1388 		    blksperindir, fs, acctfunc, expungetype);
1389 		if (error)
1390 			return (error);
1391 		blksperindir *= NINDIR(fs);
1392 		lbn -= blksperindir + 1;
1393 		len -= blksperindir;
1394 		rlbn += blksperindir;
1395 	}
1396 	return (0);
1397 }
1398 
1399 /*
1400  * Descend an indirect block chain for vnode cancelvp accounting for all
1401  * its indirect blocks in snapvp.
1402  */
1403 static int
1404 indiracct_ufs2(snapvp, cancelvp, level, blkno, lbn, rlbn, remblks,
1405 	    blksperindir, fs, acctfunc, expungetype)
1406 	struct vnode *snapvp;
1407 	struct vnode *cancelvp;
1408 	int level;
1409 	ufs2_daddr_t blkno;
1410 	ufs_lbn_t lbn;
1411 	ufs_lbn_t rlbn;
1412 	ufs_lbn_t remblks;
1413 	ufs_lbn_t blksperindir;
1414 	struct fs *fs;
1415 	int (*acctfunc)(struct vnode *, ufs2_daddr_t *, ufs2_daddr_t *,
1416 	    struct fs *, ufs_lbn_t, int);
1417 	int expungetype;
1418 {
1419 	int error, num, i;
1420 	ufs_lbn_t subblksperindir;
1421 	struct indir indirs[UFS_NIADDR + 2];
1422 	ufs2_daddr_t last, *bap;
1423 	struct buf *bp;
1424 
1425 	if (blkno == 0) {
1426 		if (expungetype == BLK_NOCOPY)
1427 			return (0);
1428 		panic("indiracct_ufs2: missing indir");
1429 	}
1430 	if ((error = ufs_getlbns(cancelvp, rlbn, indirs, &num)) != 0)
1431 		return (error);
1432 	if (lbn != indirs[num - 1 - level].in_lbn || num < 2)
1433 		panic("indiracct_ufs2: botched params");
1434 	/*
1435 	 * We have to expand bread here since it will deadlock looking
1436 	 * up the block number for any blocks that are not in the cache.
1437 	 */
1438 	bp = getblk(cancelvp, lbn, fs->fs_bsize, 0, 0, 0);
1439 	bp->b_blkno = fsbtodb(fs, blkno);
1440 	if ((bp->b_flags & B_CACHE) == 0 &&
1441 	    (error = readblock(cancelvp, bp, fragstoblks(fs, blkno)))) {
1442 		brelse(bp);
1443 		return (error);
1444 	}
1445 	/*
1446 	 * Account for the block pointers in this indirect block.
1447 	 */
1448 	last = howmany(remblks, blksperindir);
1449 	if (last > NINDIR(fs))
1450 		last = NINDIR(fs);
1451 	bap = malloc(fs->fs_bsize, M_DEVBUF, M_WAITOK);
1452 	bcopy(bp->b_data, (caddr_t)bap, fs->fs_bsize);
1453 	bqrelse(bp);
1454 	error = (*acctfunc)(snapvp, &bap[0], &bap[last], fs,
1455 	    level == 0 ? rlbn : -1, expungetype);
1456 	if (error || level == 0)
1457 		goto out;
1458 	/*
1459 	 * Account for the block pointers in each of the indirect blocks
1460 	 * in the levels below us.
1461 	 */
1462 	subblksperindir = blksperindir / NINDIR(fs);
1463 	for (lbn++, level--, i = 0; i < last; i++) {
1464 		error = indiracct_ufs2(snapvp, cancelvp, level, bap[i], lbn,
1465 		    rlbn, remblks, subblksperindir, fs, acctfunc, expungetype);
1466 		if (error)
1467 			goto out;
1468 		rlbn += blksperindir;
1469 		lbn -= blksperindir;
1470 		remblks -= blksperindir;
1471 	}
1472 out:
1473 	free(bap, M_DEVBUF);
1474 	return (error);
1475 }
1476 
1477 /*
1478  * Do both snap accounting and map accounting.
1479  */
1480 static int
1481 fullacct_ufs2(vp, oldblkp, lastblkp, fs, lblkno, exptype)
1482 	struct vnode *vp;
1483 	ufs2_daddr_t *oldblkp, *lastblkp;
1484 	struct fs *fs;
1485 	ufs_lbn_t lblkno;
1486 	int exptype;	/* BLK_SNAP or BLK_NOCOPY */
1487 {
1488 	int error;
1489 
1490 	if ((error = snapacct_ufs2(vp, oldblkp, lastblkp, fs, lblkno, exptype)))
1491 		return (error);
1492 	return (mapacct_ufs2(vp, oldblkp, lastblkp, fs, lblkno, exptype));
1493 }
1494 
1495 /*
1496  * Identify a set of blocks allocated in a snapshot inode.
1497  */
1498 static int
1499 snapacct_ufs2(vp, oldblkp, lastblkp, fs, lblkno, expungetype)
1500 	struct vnode *vp;
1501 	ufs2_daddr_t *oldblkp, *lastblkp;
1502 	struct fs *fs;
1503 	ufs_lbn_t lblkno;
1504 	int expungetype;	/* BLK_SNAP or BLK_NOCOPY */
1505 {
1506 	struct inode *ip = VTOI(vp);
1507 	ufs2_daddr_t blkno, *blkp;
1508 	ufs_lbn_t lbn;
1509 	struct buf *ibp;
1510 	int error;
1511 
1512 	for ( ; oldblkp < lastblkp; oldblkp++) {
1513 		blkno = *oldblkp;
1514 		if (blkno == 0 || blkno == BLK_NOCOPY || blkno == BLK_SNAP)
1515 			continue;
1516 		lbn = fragstoblks(fs, blkno);
1517 		if (lbn < UFS_NDADDR) {
1518 			blkp = &ip->i_din2->di_db[lbn];
1519 			UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
1520 		} else {
1521 			error = ffs_balloc_ufs2(vp, lblktosize(fs, (off_t)lbn),
1522 			    fs->fs_bsize, KERNCRED, BA_METAONLY, &ibp);
1523 			if (error)
1524 				return (error);
1525 			blkp = &((ufs2_daddr_t *)(ibp->b_data))
1526 			    [(lbn - UFS_NDADDR) % NINDIR(fs)];
1527 		}
1528 		/*
1529 		 * If we are expunging a snapshot vnode and we
1530 		 * find a block marked BLK_NOCOPY, then it is
1531 		 * one that has been allocated to this snapshot after
1532 		 * we took our current snapshot and can be ignored.
1533 		 */
1534 		if (expungetype == BLK_SNAP && *blkp == BLK_NOCOPY) {
1535 			if (lbn >= UFS_NDADDR)
1536 				brelse(ibp);
1537 		} else {
1538 			if (*blkp != 0)
1539 				panic("snapacct_ufs2: bad block");
1540 			*blkp = expungetype;
1541 			if (lbn >= UFS_NDADDR)
1542 				bdwrite(ibp);
1543 		}
1544 	}
1545 	return (0);
1546 }
1547 
1548 /*
1549  * Account for a set of blocks allocated in a snapshot inode.
1550  */
1551 static int
1552 mapacct_ufs2(vp, oldblkp, lastblkp, fs, lblkno, expungetype)
1553 	struct vnode *vp;
1554 	ufs2_daddr_t *oldblkp, *lastblkp;
1555 	struct fs *fs;
1556 	ufs_lbn_t lblkno;
1557 	int expungetype;
1558 {
1559 	ufs2_daddr_t blkno;
1560 	struct inode *ip;
1561 	ino_t inum;
1562 	int acctit;
1563 
1564 	ip = VTOI(vp);
1565 	inum = ip->i_number;
1566 	if (lblkno == -1)
1567 		acctit = 0;
1568 	else
1569 		acctit = 1;
1570 	for ( ; oldblkp < lastblkp; oldblkp++, lblkno++) {
1571 		blkno = *oldblkp;
1572 		if (blkno == 0 || blkno == BLK_NOCOPY)
1573 			continue;
1574 		if (acctit && expungetype == BLK_SNAP && blkno != BLK_SNAP &&
1575 		    lblkno >= UFS_NDADDR)
1576 			*ip->i_snapblklist++ = lblkno;
1577 		if (blkno == BLK_SNAP)
1578 			blkno = blkstofrags(fs, lblkno);
1579 		ffs_blkfree(ITOUMP(ip), fs, vp, blkno, fs->fs_bsize, inum,
1580 		    vp->v_type, NULL, SINGLETON_KEY);
1581 	}
1582 	return (0);
1583 }
1584 
1585 /*
1586  * Decrement extra reference on snapshot when last name is removed.
1587  * It will not be freed until the last open reference goes away.
1588  */
1589 void
1590 ffs_snapgone(ip)
1591 	struct inode *ip;
1592 {
1593 	struct inode *xp;
1594 	struct fs *fs;
1595 	int snaploc;
1596 	struct snapdata *sn;
1597 	struct ufsmount *ump;
1598 
1599 	/*
1600 	 * Find snapshot in incore list.
1601 	 */
1602 	xp = NULL;
1603 	sn = ITODEVVP(ip)->v_rdev->si_snapdata;
1604 	if (sn != NULL)
1605 		TAILQ_FOREACH(xp, &sn->sn_head, i_nextsnap)
1606 			if (xp == ip)
1607 				break;
1608 	if (xp != NULL)
1609 		vrele(ITOV(ip));
1610 #ifdef DIAGNOSTIC
1611 	else if (snapdebug)
1612 		printf("ffs_snapgone: lost snapshot vnode %ju\n",
1613 		    (uintmax_t)ip->i_number);
1614 #endif
1615 	/*
1616 	 * Delete snapshot inode from superblock. Keep list dense.
1617 	 */
1618 	ump = ITOUMP(ip);
1619 	fs = ump->um_fs;
1620 	UFS_LOCK(ump);
1621 	for (snaploc = 0; snaploc < FSMAXSNAP; snaploc++)
1622 		if (fs->fs_snapinum[snaploc] == ip->i_number)
1623 			break;
1624 	if (snaploc < FSMAXSNAP) {
1625 		for (snaploc++; snaploc < FSMAXSNAP; snaploc++) {
1626 			if (fs->fs_snapinum[snaploc] == 0)
1627 				break;
1628 			fs->fs_snapinum[snaploc - 1] = fs->fs_snapinum[snaploc];
1629 		}
1630 		fs->fs_snapinum[snaploc - 1] = 0;
1631 	}
1632 	UFS_UNLOCK(ump);
1633 }
1634 
1635 /*
1636  * Prepare a snapshot file for being removed.
1637  */
1638 void
1639 ffs_snapremove(vp)
1640 	struct vnode *vp;
1641 {
1642 	struct inode *ip;
1643 	struct vnode *devvp;
1644 	struct buf *ibp;
1645 	struct fs *fs;
1646 	ufs2_daddr_t numblks, blkno, dblk;
1647 	int error, i, last, loc;
1648 	struct snapdata *sn;
1649 
1650 	ip = VTOI(vp);
1651 	fs = ITOFS(ip);
1652 	devvp = ITODEVVP(ip);
1653 	/*
1654 	 * If active, delete from incore list (this snapshot may
1655 	 * already have been in the process of being deleted, so
1656 	 * would not have been active).
1657 	 *
1658 	 * Clear copy-on-write flag if last snapshot.
1659 	 */
1660 	VI_LOCK(devvp);
1661 	if (ip->i_nextsnap.tqe_prev != 0) {
1662 		sn = devvp->v_rdev->si_snapdata;
1663 		TAILQ_REMOVE(&sn->sn_head, ip, i_nextsnap);
1664 		ip->i_nextsnap.tqe_prev = 0;
1665 		VI_UNLOCK(devvp);
1666 		lockmgr(&vp->v_lock, LK_EXCLUSIVE, NULL);
1667 		for (i = 0; i < sn->sn_lock.lk_recurse; i++)
1668 			lockmgr(&vp->v_lock, LK_EXCLUSIVE, NULL);
1669 		KASSERT(vp->v_vnlock == &sn->sn_lock,
1670 			("ffs_snapremove: lost lock mutation"));
1671 		vp->v_vnlock = &vp->v_lock;
1672 		VI_LOCK(devvp);
1673 		while (sn->sn_lock.lk_recurse > 0)
1674 			lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
1675 		lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
1676 		try_free_snapdata(devvp);
1677 	} else
1678 		VI_UNLOCK(devvp);
1679 	/*
1680 	 * Clear all BLK_NOCOPY fields. Pass any block claims to other
1681 	 * snapshots that want them (see ffs_snapblkfree below).
1682 	 */
1683 	for (blkno = 1; blkno < UFS_NDADDR; blkno++) {
1684 		dblk = DIP(ip, i_db[blkno]);
1685 		if (dblk == 0)
1686 			continue;
1687 		if (dblk == BLK_NOCOPY || dblk == BLK_SNAP)
1688 			DIP_SET(ip, i_db[blkno], 0);
1689 		else if ((dblk == blkstofrags(fs, blkno) &&
1690 		     ffs_snapblkfree(fs, ITODEVVP(ip), dblk, fs->fs_bsize,
1691 		     ip->i_number, vp->v_type, NULL))) {
1692 			DIP_SET(ip, i_blocks, DIP(ip, i_blocks) -
1693 			    btodb(fs->fs_bsize));
1694 			DIP_SET(ip, i_db[blkno], 0);
1695 		}
1696 	}
1697 	numblks = howmany(ip->i_size, fs->fs_bsize);
1698 	for (blkno = UFS_NDADDR; blkno < numblks; blkno += NINDIR(fs)) {
1699 		error = UFS_BALLOC(vp, lblktosize(fs, (off_t)blkno),
1700 		    fs->fs_bsize, KERNCRED, BA_METAONLY, &ibp);
1701 		if (error)
1702 			continue;
1703 		if (fs->fs_size - blkno > NINDIR(fs))
1704 			last = NINDIR(fs);
1705 		else
1706 			last = fs->fs_size - blkno;
1707 		for (loc = 0; loc < last; loc++) {
1708 			if (I_IS_UFS1(ip)) {
1709 				dblk = ((ufs1_daddr_t *)(ibp->b_data))[loc];
1710 				if (dblk == 0)
1711 					continue;
1712 				if (dblk == BLK_NOCOPY || dblk == BLK_SNAP)
1713 					((ufs1_daddr_t *)(ibp->b_data))[loc]= 0;
1714 				else if ((dblk == blkstofrags(fs, blkno) &&
1715 				     ffs_snapblkfree(fs, ITODEVVP(ip), dblk,
1716 				     fs->fs_bsize, ip->i_number, vp->v_type,
1717 				     NULL))) {
1718 					ip->i_din1->di_blocks -=
1719 					    btodb(fs->fs_bsize);
1720 					((ufs1_daddr_t *)(ibp->b_data))[loc]= 0;
1721 				}
1722 				continue;
1723 			}
1724 			dblk = ((ufs2_daddr_t *)(ibp->b_data))[loc];
1725 			if (dblk == 0)
1726 				continue;
1727 			if (dblk == BLK_NOCOPY || dblk == BLK_SNAP)
1728 				((ufs2_daddr_t *)(ibp->b_data))[loc] = 0;
1729 			else if ((dblk == blkstofrags(fs, blkno) &&
1730 			     ffs_snapblkfree(fs, ITODEVVP(ip), dblk,
1731 			     fs->fs_bsize, ip->i_number, vp->v_type, NULL))) {
1732 				ip->i_din2->di_blocks -= btodb(fs->fs_bsize);
1733 				((ufs2_daddr_t *)(ibp->b_data))[loc] = 0;
1734 			}
1735 		}
1736 		bawrite(ibp);
1737 	}
1738 	/*
1739 	 * Clear snapshot flag and drop reference.
1740 	 */
1741 	ip->i_flags &= ~SF_SNAPSHOT;
1742 	DIP_SET(ip, i_flags, ip->i_flags);
1743 	UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
1744 	/*
1745 	 * The dirtied indirects must be written out before
1746 	 * softdep_setup_freeblocks() is called.  Otherwise indir_trunc()
1747 	 * may find indirect pointers using the magic BLK_* values.
1748 	 */
1749 	if (DOINGSOFTDEP(vp))
1750 		ffs_syncvnode(vp, MNT_WAIT, 0);
1751 #ifdef QUOTA
1752 	/*
1753 	 * Reenable disk quotas for ex-snapshot file.
1754 	 */
1755 	if (!getinoquota(ip))
1756 		(void) chkdq(ip, DIP(ip, i_blocks), KERNCRED, FORCE);
1757 #endif
1758 }
1759 
1760 /*
1761  * Notification that a block is being freed. Return zero if the free
1762  * should be allowed to proceed. Return non-zero if the snapshot file
1763  * wants to claim the block. The block will be claimed if it is an
1764  * uncopied part of one of the snapshots. It will be freed if it is
1765  * either a BLK_NOCOPY or has already been copied in all of the snapshots.
1766  * If a fragment is being freed, then all snapshots that care about
1767  * it must make a copy since a snapshot file can only claim full sized
1768  * blocks. Note that if more than one snapshot file maps the block,
1769  * we can pick one at random to claim it. Since none of the snapshots
1770  * can change, we are assurred that they will all see the same unmodified
1771  * image. When deleting a snapshot file (see ffs_snapremove above), we
1772  * must push any of these claimed blocks to one of the other snapshots
1773  * that maps it. These claimed blocks are easily identified as they will
1774  * have a block number equal to their logical block number within the
1775  * snapshot. A copied block can never have this property because they
1776  * must always have been allocated from a BLK_NOCOPY location.
1777  */
1778 int
1779 ffs_snapblkfree(fs, devvp, bno, size, inum, vtype, wkhd)
1780 	struct fs *fs;
1781 	struct vnode *devvp;
1782 	ufs2_daddr_t bno;
1783 	long size;
1784 	ino_t inum;
1785 	enum vtype vtype;
1786 	struct workhead *wkhd;
1787 {
1788 	struct buf *ibp, *cbp, *savedcbp = NULL;
1789 	struct thread *td = curthread;
1790 	struct inode *ip;
1791 	struct vnode *vp = NULL;
1792 	ufs_lbn_t lbn;
1793 	ufs2_daddr_t blkno;
1794 	int indiroff = 0, error = 0, claimedblk = 0;
1795 	struct snapdata *sn;
1796 
1797 	lbn = fragstoblks(fs, bno);
1798 retry:
1799 	VI_LOCK(devvp);
1800 	sn = devvp->v_rdev->si_snapdata;
1801 	if (sn == NULL) {
1802 		VI_UNLOCK(devvp);
1803 		return (0);
1804 	}
1805 	if (lockmgr(&sn->sn_lock, LK_INTERLOCK | LK_EXCLUSIVE | LK_SLEEPFAIL,
1806 	    VI_MTX(devvp)) != 0)
1807 		goto retry;
1808 	TAILQ_FOREACH(ip, &sn->sn_head, i_nextsnap) {
1809 		vp = ITOV(ip);
1810 		if (DOINGSOFTDEP(vp))
1811 			softdep_prealloc(vp, MNT_WAIT);
1812 		/*
1813 		 * Lookup block being written.
1814 		 */
1815 		if (lbn < UFS_NDADDR) {
1816 			blkno = DIP(ip, i_db[lbn]);
1817 		} else {
1818 			td->td_pflags |= TDP_COWINPROGRESS;
1819 			error = UFS_BALLOC(vp, lblktosize(fs, (off_t)lbn),
1820 			    fs->fs_bsize, KERNCRED, BA_METAONLY, &ibp);
1821 			td->td_pflags &= ~TDP_COWINPROGRESS;
1822 			if (error)
1823 				break;
1824 			indiroff = (lbn - UFS_NDADDR) % NINDIR(fs);
1825 			if (I_IS_UFS1(ip))
1826 				blkno=((ufs1_daddr_t *)(ibp->b_data))[indiroff];
1827 			else
1828 				blkno=((ufs2_daddr_t *)(ibp->b_data))[indiroff];
1829 		}
1830 		/*
1831 		 * Check to see if block needs to be copied.
1832 		 */
1833 		if (blkno == 0) {
1834 			/*
1835 			 * A block that we map is being freed. If it has not
1836 			 * been claimed yet, we will claim or copy it (below).
1837 			 */
1838 			claimedblk = 1;
1839 		} else if (blkno == BLK_SNAP) {
1840 			/*
1841 			 * No previous snapshot claimed the block,
1842 			 * so it will be freed and become a BLK_NOCOPY
1843 			 * (don't care) for us.
1844 			 */
1845 			if (claimedblk)
1846 				panic("snapblkfree: inconsistent block type");
1847 			if (lbn < UFS_NDADDR) {
1848 				DIP_SET(ip, i_db[lbn], BLK_NOCOPY);
1849 				UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
1850 			} else if (I_IS_UFS1(ip)) {
1851 				((ufs1_daddr_t *)(ibp->b_data))[indiroff] =
1852 				    BLK_NOCOPY;
1853 				bdwrite(ibp);
1854 			} else {
1855 				((ufs2_daddr_t *)(ibp->b_data))[indiroff] =
1856 				    BLK_NOCOPY;
1857 				bdwrite(ibp);
1858 			}
1859 			continue;
1860 		} else /* BLK_NOCOPY or default */ {
1861 			/*
1862 			 * If the snapshot has already copied the block
1863 			 * (default), or does not care about the block,
1864 			 * it is not needed.
1865 			 */
1866 			if (lbn >= UFS_NDADDR)
1867 				bqrelse(ibp);
1868 			continue;
1869 		}
1870 		/*
1871 		 * If this is a full size block, we will just grab it
1872 		 * and assign it to the snapshot inode. Otherwise we
1873 		 * will proceed to copy it. See explanation for this
1874 		 * routine as to why only a single snapshot needs to
1875 		 * claim this block.
1876 		 */
1877 		if (size == fs->fs_bsize) {
1878 #ifdef DIAGNOSTIC
1879 			if (snapdebug)
1880 				printf("%s %ju lbn %jd from inum %ju\n",
1881 				    "Grabonremove: snapino",
1882 				    (uintmax_t)ip->i_number,
1883 				    (intmax_t)lbn, (uintmax_t)inum);
1884 #endif
1885 			/*
1886 			 * If journaling is tracking this write we must add
1887 			 * the work to the inode or indirect being written.
1888 			 */
1889 			if (wkhd != NULL) {
1890 				if (lbn < UFS_NDADDR)
1891 					softdep_inode_append(ip,
1892 					    curthread->td_ucred, wkhd);
1893 				else
1894 					softdep_buf_append(ibp, wkhd);
1895 			}
1896 			if (lbn < UFS_NDADDR) {
1897 				DIP_SET(ip, i_db[lbn], bno);
1898 			} else if (I_IS_UFS1(ip)) {
1899 				((ufs1_daddr_t *)(ibp->b_data))[indiroff] = bno;
1900 				bdwrite(ibp);
1901 			} else {
1902 				((ufs2_daddr_t *)(ibp->b_data))[indiroff] = bno;
1903 				bdwrite(ibp);
1904 			}
1905 			DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + btodb(size));
1906 			UFS_INODE_SET_FLAG(ip, IN_CHANGE | IN_UPDATE);
1907 			lockmgr(vp->v_vnlock, LK_RELEASE, NULL);
1908 			return (1);
1909 		}
1910 		if (lbn >= UFS_NDADDR)
1911 			bqrelse(ibp);
1912 		/*
1913 		 * Allocate the block into which to do the copy. Note that this
1914 		 * allocation will never require any additional allocations for
1915 		 * the snapshot inode.
1916 		 */
1917 		td->td_pflags |= TDP_COWINPROGRESS;
1918 		error = UFS_BALLOC(vp, lblktosize(fs, (off_t)lbn),
1919 		    fs->fs_bsize, KERNCRED, 0, &cbp);
1920 		td->td_pflags &= ~TDP_COWINPROGRESS;
1921 		if (error)
1922 			break;
1923 #ifdef DIAGNOSTIC
1924 		if (snapdebug)
1925 			printf("%s%ju lbn %jd %s %ju size %ld to blkno %jd\n",
1926 			    "Copyonremove: snapino ", (uintmax_t)ip->i_number,
1927 			    (intmax_t)lbn, "for inum", (uintmax_t)inum, size,
1928 			    (intmax_t)cbp->b_blkno);
1929 #endif
1930 		/*
1931 		 * If we have already read the old block contents, then
1932 		 * simply copy them to the new block. Note that we need
1933 		 * to synchronously write snapshots that have not been
1934 		 * unlinked, and hence will be visible after a crash,
1935 		 * to ensure their integrity. At a minimum we ensure the
1936 		 * integrity of the filesystem metadata, but use the
1937 		 * dopersistence sysctl-setable flag to decide on the
1938 		 * persistence needed for file content data.
1939 		 */
1940 		if (savedcbp != NULL) {
1941 			bcopy(savedcbp->b_data, cbp->b_data, fs->fs_bsize);
1942 			bawrite(cbp);
1943 			if ((vtype == VDIR || dopersistence) &&
1944 			    ip->i_effnlink > 0)
1945 				(void) ffs_syncvnode(vp, MNT_WAIT, NO_INO_UPDT);
1946 			continue;
1947 		}
1948 		/*
1949 		 * Otherwise, read the old block contents into the buffer.
1950 		 */
1951 		if ((error = readblock(vp, cbp, lbn)) != 0) {
1952 			bzero(cbp->b_data, fs->fs_bsize);
1953 			bawrite(cbp);
1954 			if ((vtype == VDIR || dopersistence) &&
1955 			    ip->i_effnlink > 0)
1956 				(void) ffs_syncvnode(vp, MNT_WAIT, NO_INO_UPDT);
1957 			break;
1958 		}
1959 		savedcbp = cbp;
1960 	}
1961 	/*
1962 	 * Note that we need to synchronously write snapshots that
1963 	 * have not been unlinked, and hence will be visible after
1964 	 * a crash, to ensure their integrity. At a minimum we
1965 	 * ensure the integrity of the filesystem metadata, but
1966 	 * use the dopersistence sysctl-setable flag to decide on
1967 	 * the persistence needed for file content data.
1968 	 */
1969 	if (savedcbp) {
1970 		vp = savedcbp->b_vp;
1971 		bawrite(savedcbp);
1972 		if ((vtype == VDIR || dopersistence) &&
1973 		    VTOI(vp)->i_effnlink > 0)
1974 			(void) ffs_syncvnode(vp, MNT_WAIT, NO_INO_UPDT);
1975 	}
1976 	/*
1977 	 * If we have been unable to allocate a block in which to do
1978 	 * the copy, then return non-zero so that the fragment will
1979 	 * not be freed. Although space will be lost, the snapshot
1980 	 * will stay consistent.
1981 	 */
1982 	if (error != 0 && wkhd != NULL)
1983 		softdep_freework(wkhd);
1984 	lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
1985 	return (error);
1986 }
1987 
1988 /*
1989  * Associate snapshot files when mounting.
1990  */
1991 void
1992 ffs_snapshot_mount(mp)
1993 	struct mount *mp;
1994 {
1995 	struct ufsmount *ump = VFSTOUFS(mp);
1996 	struct vnode *devvp = ump->um_devvp;
1997 	struct fs *fs = ump->um_fs;
1998 	struct thread *td = curthread;
1999 	struct snapdata *sn;
2000 	struct vnode *vp;
2001 	struct vnode *lastvp;
2002 	struct inode *ip;
2003 	struct uio auio;
2004 	struct iovec aiov;
2005 	void *snapblklist;
2006 	char *reason;
2007 	daddr_t snaplistsize;
2008 	int error, snaploc, loc;
2009 
2010 	/*
2011 	 * XXX The following needs to be set before ffs_truncate or
2012 	 * VOP_READ can be called.
2013 	 */
2014 	mp->mnt_stat.f_iosize = fs->fs_bsize;
2015 	/*
2016 	 * Process each snapshot listed in the superblock.
2017 	 */
2018 	vp = NULL;
2019 	lastvp = NULL;
2020 	sn = NULL;
2021 	for (snaploc = 0; snaploc < FSMAXSNAP; snaploc++) {
2022 		if (fs->fs_snapinum[snaploc] == 0)
2023 			break;
2024 		if ((error = ffs_vget(mp, fs->fs_snapinum[snaploc],
2025 		    LK_EXCLUSIVE, &vp)) != 0){
2026 			printf("ffs_snapshot_mount: vget failed %d\n", error);
2027 			continue;
2028 		}
2029 		ip = VTOI(vp);
2030 		if (vp->v_type != VREG) {
2031 			reason = "non-file snapshot";
2032 		} else if (!IS_SNAPSHOT(ip)) {
2033 			reason = "non-snapshot";
2034 		} else if (ip->i_size ==
2035 		    lblktosize(fs, howmany(fs->fs_size, fs->fs_frag))) {
2036 			reason = "old format snapshot";
2037 			(void)ffs_truncate(vp, (off_t)0, 0, NOCRED);
2038 			(void)ffs_syncvnode(vp, MNT_WAIT, 0);
2039 		} else {
2040 			reason = NULL;
2041 		}
2042 		if (reason != NULL) {
2043 			printf("ffs_snapshot_mount: %s inode %d\n",
2044 			    reason, fs->fs_snapinum[snaploc]);
2045 			vput(vp);
2046 			vp = NULL;
2047 			for (loc = snaploc + 1; loc < FSMAXSNAP; loc++) {
2048 				if (fs->fs_snapinum[loc] == 0)
2049 					break;
2050 				fs->fs_snapinum[loc - 1] = fs->fs_snapinum[loc];
2051 			}
2052 			fs->fs_snapinum[loc - 1] = 0;
2053 			snaploc--;
2054 			continue;
2055 		}
2056 		/*
2057 		 * Acquire a lock on the snapdata structure, creating it if
2058 		 * necessary.
2059 		 */
2060 		sn = ffs_snapdata_acquire(devvp);
2061 		/*
2062 		 * Change vnode to use shared snapshot lock instead of the
2063 		 * original private lock.
2064 		 */
2065 		vp->v_vnlock = &sn->sn_lock;
2066 		lockmgr(&vp->v_lock, LK_RELEASE, NULL);
2067 		/*
2068 		 * Link it onto the active snapshot list.
2069 		 */
2070 		VI_LOCK(devvp);
2071 		if (ip->i_nextsnap.tqe_prev != 0)
2072 			panic("ffs_snapshot_mount: %ju already on list",
2073 			    (uintmax_t)ip->i_number);
2074 		else
2075 			TAILQ_INSERT_TAIL(&sn->sn_head, ip, i_nextsnap);
2076 		vp->v_vflag |= VV_SYSTEM;
2077 		VI_UNLOCK(devvp);
2078 		VOP_UNLOCK(vp);
2079 		lastvp = vp;
2080 	}
2081 	vp = lastvp;
2082 	/*
2083 	 * No usable snapshots found.
2084 	 */
2085 	if (sn == NULL || vp == NULL)
2086 		return;
2087 	/*
2088 	 * Allocate the space for the block hints list. We always want to
2089 	 * use the list from the newest snapshot.
2090 	 */
2091 	auio.uio_iov = &aiov;
2092 	auio.uio_iovcnt = 1;
2093 	aiov.iov_base = (void *)&snaplistsize;
2094 	aiov.iov_len = sizeof(snaplistsize);
2095 	auio.uio_resid = aiov.iov_len;
2096 	auio.uio_offset =
2097 	    lblktosize(fs, howmany(fs->fs_size, fs->fs_frag));
2098 	auio.uio_segflg = UIO_SYSSPACE;
2099 	auio.uio_rw = UIO_READ;
2100 	auio.uio_td = td;
2101 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
2102 	if ((error = VOP_READ(vp, &auio, IO_UNIT, td->td_ucred)) != 0) {
2103 		printf("ffs_snapshot_mount: read_1 failed %d\n", error);
2104 		VOP_UNLOCK(vp);
2105 		return;
2106 	}
2107 	snapblklist = malloc(snaplistsize * sizeof(daddr_t),
2108 	    M_UFSMNT, M_WAITOK);
2109 	auio.uio_iovcnt = 1;
2110 	aiov.iov_base = snapblklist;
2111 	aiov.iov_len = snaplistsize * sizeof (daddr_t);
2112 	auio.uio_resid = aiov.iov_len;
2113 	auio.uio_offset -= sizeof(snaplistsize);
2114 	if ((error = VOP_READ(vp, &auio, IO_UNIT, td->td_ucred)) != 0) {
2115 		printf("ffs_snapshot_mount: read_2 failed %d\n", error);
2116 		VOP_UNLOCK(vp);
2117 		free(snapblklist, M_UFSMNT);
2118 		return;
2119 	}
2120 	VOP_UNLOCK(vp);
2121 	VI_LOCK(devvp);
2122 	ASSERT_VOP_LOCKED(devvp, "ffs_snapshot_mount");
2123 	sn->sn_listsize = snaplistsize;
2124 	sn->sn_blklist = (daddr_t *)snapblklist;
2125 	devvp->v_vflag |= VV_COPYONWRITE;
2126 	VI_UNLOCK(devvp);
2127 }
2128 
2129 /*
2130  * Disassociate snapshot files when unmounting.
2131  */
2132 void
2133 ffs_snapshot_unmount(mp)
2134 	struct mount *mp;
2135 {
2136 	struct vnode *devvp = VFSTOUFS(mp)->um_devvp;
2137 	struct snapdata *sn;
2138 	struct inode *xp;
2139 	struct vnode *vp;
2140 
2141 	VI_LOCK(devvp);
2142 	sn = devvp->v_rdev->si_snapdata;
2143 	while (sn != NULL && (xp = TAILQ_FIRST(&sn->sn_head)) != NULL) {
2144 		vp = ITOV(xp);
2145 		TAILQ_REMOVE(&sn->sn_head, xp, i_nextsnap);
2146 		xp->i_nextsnap.tqe_prev = 0;
2147 		lockmgr(&sn->sn_lock, LK_INTERLOCK | LK_EXCLUSIVE,
2148 		    VI_MTX(devvp));
2149 		/*
2150 		 * Avoid LOR with above snapshot lock. The LK_NOWAIT should
2151 		 * never fail as the lock is currently unused. Rather than
2152 		 * panic, we recover by doing the blocking lock.
2153 		 */
2154 		if (lockmgr(&vp->v_lock, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) {
2155 			printf("ffs_snapshot_unmount: Unexpected LK_NOWAIT "
2156 			    "failure\n");
2157 			lockmgr(&vp->v_lock, LK_EXCLUSIVE, NULL);
2158 		}
2159 		KASSERT(vp->v_vnlock == &sn->sn_lock,
2160 		("ffs_snapshot_unmount: lost lock mutation"));
2161 		vp->v_vnlock = &vp->v_lock;
2162 		lockmgr(&vp->v_lock, LK_RELEASE, NULL);
2163 		lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
2164 		if (xp->i_effnlink > 0)
2165 			vrele(vp);
2166 		VI_LOCK(devvp);
2167 		sn = devvp->v_rdev->si_snapdata;
2168 	}
2169 	try_free_snapdata(devvp);
2170 	ASSERT_VOP_LOCKED(devvp, "ffs_snapshot_unmount");
2171 }
2172 
2173 /*
2174  * Check the buffer block to be belong to device buffer that shall be
2175  * locked after snaplk. devvp shall be locked on entry, and will be
2176  * leaved locked upon exit.
2177  */
2178 static int
2179 ffs_bp_snapblk(devvp, bp)
2180 	struct vnode *devvp;
2181 	struct buf *bp;
2182 {
2183 	struct snapdata *sn;
2184 	struct fs *fs;
2185 	ufs2_daddr_t lbn, *snapblklist;
2186 	int lower, upper, mid;
2187 
2188 	ASSERT_VI_LOCKED(devvp, "ffs_bp_snapblk");
2189 	KASSERT(devvp->v_type == VCHR, ("Not a device %p", devvp));
2190 	sn = devvp->v_rdev->si_snapdata;
2191 	if (sn == NULL || TAILQ_FIRST(&sn->sn_head) == NULL)
2192 		return (0);
2193 	fs = ITOFS(TAILQ_FIRST(&sn->sn_head));
2194 	lbn = fragstoblks(fs, dbtofsb(fs, bp->b_blkno));
2195 	snapblklist = sn->sn_blklist;
2196 	upper = sn->sn_listsize - 1;
2197 	lower = 1;
2198 	while (lower <= upper) {
2199 		mid = (lower + upper) / 2;
2200 		if (snapblklist[mid] == lbn)
2201 			break;
2202 		if (snapblklist[mid] < lbn)
2203 			lower = mid + 1;
2204 		else
2205 			upper = mid - 1;
2206 	}
2207 	if (lower <= upper)
2208 		return (1);
2209 	return (0);
2210 }
2211 
2212 void
2213 ffs_bdflush(bo, bp)
2214 	struct bufobj *bo;
2215 	struct buf *bp;
2216 {
2217 	struct thread *td;
2218 	struct vnode *vp, *devvp;
2219 	struct buf *nbp;
2220 	int bp_bdskip;
2221 
2222 	if (bo->bo_dirty.bv_cnt <= dirtybufthresh)
2223 		return;
2224 
2225 	td = curthread;
2226 	vp = bp->b_vp;
2227 	devvp = bo2vnode(bo);
2228 	KASSERT(vp == devvp, ("devvp != vp %p %p", bo, bp));
2229 
2230 	VI_LOCK(devvp);
2231 	bp_bdskip = ffs_bp_snapblk(devvp, bp);
2232 	if (bp_bdskip)
2233 		bdwriteskip++;
2234 	VI_UNLOCK(devvp);
2235 	if (bo->bo_dirty.bv_cnt > dirtybufthresh + 10 && !bp_bdskip) {
2236 		(void) VOP_FSYNC(vp, MNT_NOWAIT, td);
2237 		altbufferflushes++;
2238 	} else {
2239 		BO_LOCK(bo);
2240 		/*
2241 		 * Try to find a buffer to flush.
2242 		 */
2243 		TAILQ_FOREACH(nbp, &bo->bo_dirty.bv_hd, b_bobufs) {
2244 			if ((nbp->b_vflags & BV_BKGRDINPROG) ||
2245 			    BUF_LOCK(nbp,
2246 				     LK_EXCLUSIVE | LK_NOWAIT, NULL))
2247 				continue;
2248 			if (bp == nbp)
2249 				panic("bdwrite: found ourselves");
2250 			BO_UNLOCK(bo);
2251 			/*
2252 			 * Don't countdeps with the bo lock
2253 			 * held.
2254 			 */
2255 			if (buf_countdeps(nbp, 0)) {
2256 				BO_LOCK(bo);
2257 				BUF_UNLOCK(nbp);
2258 				continue;
2259 			}
2260 			if (bp_bdskip) {
2261 				VI_LOCK(devvp);
2262 				if (!ffs_bp_snapblk(vp, nbp)) {
2263 					VI_UNLOCK(devvp);
2264 					BO_LOCK(bo);
2265 					BUF_UNLOCK(nbp);
2266 					continue;
2267 				}
2268 				VI_UNLOCK(devvp);
2269 			}
2270 			if (nbp->b_flags & B_CLUSTEROK) {
2271 				vfs_bio_awrite(nbp);
2272 			} else {
2273 				bremfree(nbp);
2274 				bawrite(nbp);
2275 			}
2276 			dirtybufferflushes++;
2277 			break;
2278 		}
2279 		if (nbp == NULL)
2280 			BO_UNLOCK(bo);
2281 	}
2282 }
2283 
2284 /*
2285  * Check for need to copy block that is about to be written,
2286  * copying the block if necessary.
2287  */
2288 int
2289 ffs_copyonwrite(devvp, bp)
2290 	struct vnode *devvp;
2291 	struct buf *bp;
2292 {
2293 	struct snapdata *sn;
2294 	struct buf *ibp, *cbp, *savedcbp = NULL;
2295 	struct thread *td = curthread;
2296 	struct fs *fs;
2297 	struct inode *ip;
2298 	struct vnode *vp = NULL;
2299 	ufs2_daddr_t lbn, blkno, *snapblklist;
2300 	int lower, upper, mid, indiroff, error = 0;
2301 	int launched_async_io, prev_norunningbuf;
2302 	long saved_runningbufspace;
2303 
2304 	if (devvp != bp->b_vp && IS_SNAPSHOT(VTOI(bp->b_vp)))
2305 		return (0);		/* Update on a snapshot file */
2306 	if (td->td_pflags & TDP_COWINPROGRESS)
2307 		panic("ffs_copyonwrite: recursive call");
2308 	/*
2309 	 * First check to see if it is in the preallocated list.
2310 	 * By doing this check we avoid several potential deadlocks.
2311 	 */
2312 	VI_LOCK(devvp);
2313 	sn = devvp->v_rdev->si_snapdata;
2314 	if (sn == NULL ||
2315 	    TAILQ_EMPTY(&sn->sn_head)) {
2316 		VI_UNLOCK(devvp);
2317 		return (0);		/* No snapshot */
2318 	}
2319 	ip = TAILQ_FIRST(&sn->sn_head);
2320 	fs = ITOFS(ip);
2321 	lbn = fragstoblks(fs, dbtofsb(fs, bp->b_blkno));
2322 	if (lbn < UFS_NDADDR) {
2323 		VI_UNLOCK(devvp);
2324 		return (0);		/* Direct blocks are always copied */
2325 	}
2326 	snapblklist = sn->sn_blklist;
2327 	upper = sn->sn_listsize - 1;
2328 	lower = 1;
2329 	while (lower <= upper) {
2330 		mid = (lower + upper) / 2;
2331 		if (snapblklist[mid] == lbn)
2332 			break;
2333 		if (snapblklist[mid] < lbn)
2334 			lower = mid + 1;
2335 		else
2336 			upper = mid - 1;
2337 	}
2338 	if (lower <= upper) {
2339 		VI_UNLOCK(devvp);
2340 		return (0);
2341 	}
2342 	launched_async_io = 0;
2343 	prev_norunningbuf = td->td_pflags & TDP_NORUNNINGBUF;
2344 	/*
2345 	 * Since I/O on bp isn't yet in progress and it may be blocked
2346 	 * for a long time waiting on snaplk, back it out of
2347 	 * runningbufspace, possibly waking other threads waiting for space.
2348 	 */
2349 	saved_runningbufspace = bp->b_runningbufspace;
2350 	if (saved_runningbufspace != 0)
2351 		runningbufwakeup(bp);
2352 	/*
2353 	 * Not in the precomputed list, so check the snapshots.
2354 	 */
2355 	while (lockmgr(&sn->sn_lock, LK_INTERLOCK | LK_EXCLUSIVE | LK_SLEEPFAIL,
2356 	    VI_MTX(devvp)) != 0) {
2357 		VI_LOCK(devvp);
2358 		sn = devvp->v_rdev->si_snapdata;
2359 		if (sn == NULL ||
2360 		    TAILQ_EMPTY(&sn->sn_head)) {
2361 			VI_UNLOCK(devvp);
2362 			if (saved_runningbufspace != 0) {
2363 				bp->b_runningbufspace = saved_runningbufspace;
2364 				atomic_add_long(&runningbufspace,
2365 					       bp->b_runningbufspace);
2366 			}
2367 			return (0);		/* Snapshot gone */
2368 		}
2369 	}
2370 	TAILQ_FOREACH(ip, &sn->sn_head, i_nextsnap) {
2371 		vp = ITOV(ip);
2372 		if (DOINGSOFTDEP(vp))
2373 			softdep_prealloc(vp, MNT_WAIT);
2374 		/*
2375 		 * We ensure that everything of our own that needs to be
2376 		 * copied will be done at the time that ffs_snapshot is
2377 		 * called. Thus we can skip the check here which can
2378 		 * deadlock in doing the lookup in UFS_BALLOC.
2379 		 */
2380 		if (bp->b_vp == vp)
2381 			continue;
2382 		/*
2383 		 * Check to see if block needs to be copied. We do not have
2384 		 * to hold the snapshot lock while doing this lookup as it
2385 		 * will never require any additional allocations for the
2386 		 * snapshot inode.
2387 		 */
2388 		if (lbn < UFS_NDADDR) {
2389 			blkno = DIP(ip, i_db[lbn]);
2390 		} else {
2391 			td->td_pflags |= TDP_COWINPROGRESS | TDP_NORUNNINGBUF;
2392 			error = UFS_BALLOC(vp, lblktosize(fs, (off_t)lbn),
2393 			   fs->fs_bsize, KERNCRED, BA_METAONLY, &ibp);
2394 			td->td_pflags &= ~TDP_COWINPROGRESS;
2395 			if (error)
2396 				break;
2397 			indiroff = (lbn - UFS_NDADDR) % NINDIR(fs);
2398 			if (I_IS_UFS1(ip))
2399 				blkno=((ufs1_daddr_t *)(ibp->b_data))[indiroff];
2400 			else
2401 				blkno=((ufs2_daddr_t *)(ibp->b_data))[indiroff];
2402 			bqrelse(ibp);
2403 		}
2404 #ifdef INVARIANTS
2405 		if (blkno == BLK_SNAP && bp->b_lblkno >= 0)
2406 			panic("ffs_copyonwrite: bad copy block");
2407 #endif
2408 		if (blkno != 0)
2409 			continue;
2410 		/*
2411 		 * Allocate the block into which to do the copy. Since
2412 		 * multiple processes may all try to copy the same block,
2413 		 * we have to recheck our need to do a copy if we sleep
2414 		 * waiting for the lock.
2415 		 *
2416 		 * Because all snapshots on a filesystem share a single
2417 		 * lock, we ensure that we will never be in competition
2418 		 * with another process to allocate a block.
2419 		 */
2420 		td->td_pflags |= TDP_COWINPROGRESS | TDP_NORUNNINGBUF;
2421 		error = UFS_BALLOC(vp, lblktosize(fs, (off_t)lbn),
2422 		    fs->fs_bsize, KERNCRED, 0, &cbp);
2423 		td->td_pflags &= ~TDP_COWINPROGRESS;
2424 		if (error)
2425 			break;
2426 #ifdef DIAGNOSTIC
2427 		if (snapdebug) {
2428 			printf("Copyonwrite: snapino %ju lbn %jd for ",
2429 			    (uintmax_t)ip->i_number, (intmax_t)lbn);
2430 			if (bp->b_vp == devvp)
2431 				printf("fs metadata");
2432 			else
2433 				printf("inum %ju",
2434 				    (uintmax_t)VTOI(bp->b_vp)->i_number);
2435 			printf(" lblkno %jd to blkno %jd\n",
2436 			    (intmax_t)bp->b_lblkno, (intmax_t)cbp->b_blkno);
2437 		}
2438 #endif
2439 		/*
2440 		 * If we have already read the old block contents, then
2441 		 * simply copy them to the new block. Note that we need
2442 		 * to synchronously write snapshots that have not been
2443 		 * unlinked, and hence will be visible after a crash,
2444 		 * to ensure their integrity. At a minimum we ensure the
2445 		 * integrity of the filesystem metadata, but use the
2446 		 * dopersistence sysctl-setable flag to decide on the
2447 		 * persistence needed for file content data.
2448 		 */
2449 		if (savedcbp != NULL) {
2450 			bcopy(savedcbp->b_data, cbp->b_data, fs->fs_bsize);
2451 			bawrite(cbp);
2452 			if ((devvp == bp->b_vp || bp->b_vp->v_type == VDIR ||
2453 			    dopersistence) && ip->i_effnlink > 0)
2454 				(void) ffs_syncvnode(vp, MNT_WAIT, NO_INO_UPDT);
2455 			else
2456 				launched_async_io = 1;
2457 			continue;
2458 		}
2459 		/*
2460 		 * Otherwise, read the old block contents into the buffer.
2461 		 */
2462 		if ((error = readblock(vp, cbp, lbn)) != 0) {
2463 			bzero(cbp->b_data, fs->fs_bsize);
2464 			bawrite(cbp);
2465 			if ((devvp == bp->b_vp || bp->b_vp->v_type == VDIR ||
2466 			    dopersistence) && ip->i_effnlink > 0)
2467 				(void) ffs_syncvnode(vp, MNT_WAIT, NO_INO_UPDT);
2468 			else
2469 				launched_async_io = 1;
2470 			break;
2471 		}
2472 		savedcbp = cbp;
2473 	}
2474 	/*
2475 	 * Note that we need to synchronously write snapshots that
2476 	 * have not been unlinked, and hence will be visible after
2477 	 * a crash, to ensure their integrity. At a minimum we
2478 	 * ensure the integrity of the filesystem metadata, but
2479 	 * use the dopersistence sysctl-setable flag to decide on
2480 	 * the persistence needed for file content data.
2481 	 */
2482 	if (savedcbp) {
2483 		vp = savedcbp->b_vp;
2484 		bawrite(savedcbp);
2485 		if ((devvp == bp->b_vp || bp->b_vp->v_type == VDIR ||
2486 		    dopersistence) && VTOI(vp)->i_effnlink > 0)
2487 			(void) ffs_syncvnode(vp, MNT_WAIT, NO_INO_UPDT);
2488 		else
2489 			launched_async_io = 1;
2490 	}
2491 	lockmgr(vp->v_vnlock, LK_RELEASE, NULL);
2492 	td->td_pflags = (td->td_pflags & ~TDP_NORUNNINGBUF) |
2493 		prev_norunningbuf;
2494 	if (launched_async_io && (td->td_pflags & TDP_NORUNNINGBUF) == 0)
2495 		waitrunningbufspace();
2496 	/*
2497 	 * I/O on bp will now be started, so count it in runningbufspace.
2498 	 */
2499 	if (saved_runningbufspace != 0) {
2500 		bp->b_runningbufspace = saved_runningbufspace;
2501 		atomic_add_long(&runningbufspace, bp->b_runningbufspace);
2502 	}
2503 	return (error);
2504 }
2505 
2506 /*
2507  * sync snapshots to force freework records waiting on snapshots to claim
2508  * blocks to free.
2509  */
2510 void
2511 ffs_sync_snap(mp, waitfor)
2512 	struct mount *mp;
2513 	int waitfor;
2514 {
2515 	struct snapdata *sn;
2516 	struct vnode *devvp;
2517 	struct vnode *vp;
2518 	struct inode *ip;
2519 
2520 	devvp = VFSTOUFS(mp)->um_devvp;
2521 	if ((devvp->v_vflag & VV_COPYONWRITE) == 0)
2522 		return;
2523 	for (;;) {
2524 		VI_LOCK(devvp);
2525 		sn = devvp->v_rdev->si_snapdata;
2526 		if (sn == NULL) {
2527 			VI_UNLOCK(devvp);
2528 			return;
2529 		}
2530 		if (lockmgr(&sn->sn_lock,
2531 		    LK_INTERLOCK | LK_EXCLUSIVE | LK_SLEEPFAIL,
2532 		    VI_MTX(devvp)) == 0)
2533 			break;
2534 	}
2535 	TAILQ_FOREACH(ip, &sn->sn_head, i_nextsnap) {
2536 		vp = ITOV(ip);
2537 		ffs_syncvnode(vp, waitfor, NO_INO_UPDT);
2538 	}
2539 	lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
2540 }
2541 
2542 /*
2543  * Read the specified block into the given buffer.
2544  * Much of this boiler-plate comes from bwrite().
2545  */
2546 static int
2547 readblock(vp, bp, lbn)
2548 	struct vnode *vp;
2549 	struct buf *bp;
2550 	ufs2_daddr_t lbn;
2551 {
2552 	struct inode *ip;
2553 	struct bio *bip;
2554 	struct fs *fs;
2555 
2556 	ip = VTOI(vp);
2557 	fs = ITOFS(ip);
2558 
2559 	bip = g_alloc_bio();
2560 	bip->bio_cmd = BIO_READ;
2561 	bip->bio_offset = dbtob(fsbtodb(fs, blkstofrags(fs, lbn)));
2562 	bip->bio_data = bp->b_data;
2563 	bip->bio_length = bp->b_bcount;
2564 	bip->bio_done = NULL;
2565 
2566 	g_io_request(bip, ITODEVVP(ip)->v_bufobj.bo_private);
2567 	bp->b_error = biowait(bip, "snaprdb");
2568 	g_destroy_bio(bip);
2569 	return (bp->b_error);
2570 }
2571 
2572 #endif
2573 
2574 /*
2575  * Process file deletes that were deferred by ufs_inactive() due to
2576  * the file system being suspended. Transfer IN_LAZYACCESS into
2577  * IN_MODIFIED for vnodes that were accessed during suspension.
2578  */
2579 void
2580 process_deferred_inactive(struct mount *mp)
2581 {
2582 	struct vnode *vp, *mvp;
2583 	struct inode *ip;
2584 	int error;
2585 
2586 	(void) vn_start_secondary_write(NULL, &mp, V_WAIT);
2587  loop:
2588 	MNT_VNODE_FOREACH_ALL(vp, mp, mvp) {
2589 		/*
2590 		 * IN_LAZYACCESS is checked here without holding any
2591 		 * vnode lock, but this flag is set only while holding
2592 		 * vnode interlock.
2593 		 */
2594 		if (vp->v_type == VNON ||
2595 		    ((VTOI(vp)->i_flag & IN_LAZYACCESS) == 0 &&
2596 		    ((vp->v_iflag & VI_OWEINACT) == 0 || vp->v_usecount > 0))) {
2597 			VI_UNLOCK(vp);
2598 			continue;
2599 		}
2600 		vholdl(vp);
2601 retry_vnode:
2602 		error = vn_lock(vp, LK_EXCLUSIVE | LK_INTERLOCK);
2603 		if (error != 0) {
2604 			vdrop(vp);
2605 			if (error == ENOENT)
2606 				continue;	/* vnode recycled */
2607 			MNT_VNODE_FOREACH_ALL_ABORT(mp, mvp);
2608 			goto loop;
2609 		}
2610 		ip = VTOI(vp);
2611 		if ((ip->i_flag & IN_LAZYACCESS) != 0) {
2612 			ip->i_flag &= ~IN_LAZYACCESS;
2613 			UFS_INODE_SET_FLAG(ip, IN_MODIFIED);
2614 		}
2615 		VI_LOCK(vp);
2616 		error = vinactive(vp);
2617 		if (error == ERELOOKUP && vp->v_usecount == 0) {
2618 			VI_UNLOCK(vp);
2619 			VOP_UNLOCK(vp);
2620 			goto retry_vnode;
2621 		}
2622 		VI_UNLOCK(vp);
2623 		VOP_UNLOCK(vp);
2624 		vdrop(vp);
2625 	}
2626 	vn_finished_secondary_write(mp);
2627 }
2628 
2629 #ifndef NO_FFS_SNAPSHOT
2630 
2631 static struct snapdata *
2632 ffs_snapdata_alloc(void)
2633 {
2634 	struct snapdata *sn;
2635 
2636 	/*
2637 	 * Fetch a snapdata from the free list if there is one available.
2638 	 */
2639 	mtx_lock(&snapfree_lock);
2640 	sn = LIST_FIRST(&snapfree);
2641 	if (sn != NULL)
2642 		LIST_REMOVE(sn, sn_link);
2643 	mtx_unlock(&snapfree_lock);
2644 	if (sn != NULL)
2645 		return (sn);
2646 	/*
2647  	 * If there were no free snapdatas allocate one.
2648 	 */
2649 	sn = malloc(sizeof *sn, M_UFSMNT, M_WAITOK | M_ZERO);
2650 	TAILQ_INIT(&sn->sn_head);
2651 	lockinit(&sn->sn_lock, PVFS, "snaplk", VLKTIMEOUT,
2652 	    LK_CANRECURSE | LK_NOSHARE);
2653 	return (sn);
2654 }
2655 
2656 /*
2657  * The snapdata is never freed because we can not be certain that
2658  * there are no threads sleeping on the snap lock.  Persisting
2659  * them permanently avoids costly synchronization in ffs_lock().
2660  */
2661 static void
2662 ffs_snapdata_free(struct snapdata *sn)
2663 {
2664 	mtx_lock(&snapfree_lock);
2665 	LIST_INSERT_HEAD(&snapfree, sn, sn_link);
2666 	mtx_unlock(&snapfree_lock);
2667 }
2668 
2669 /* Try to free snapdata associated with devvp */
2670 static void
2671 try_free_snapdata(struct vnode *devvp)
2672 {
2673 	struct snapdata *sn;
2674 	ufs2_daddr_t *snapblklist;
2675 
2676 	ASSERT_VI_LOCKED(devvp, "try_free_snapdata");
2677 	sn = devvp->v_rdev->si_snapdata;
2678 
2679 	if (sn == NULL || TAILQ_FIRST(&sn->sn_head) != NULL ||
2680 	    (devvp->v_vflag & VV_COPYONWRITE) == 0) {
2681 		VI_UNLOCK(devvp);
2682 		return;
2683 	}
2684 
2685 	devvp->v_rdev->si_snapdata = NULL;
2686 	devvp->v_vflag &= ~VV_COPYONWRITE;
2687 	lockmgr(&sn->sn_lock, LK_DRAIN|LK_INTERLOCK, VI_MTX(devvp));
2688 	snapblklist = sn->sn_blklist;
2689 	sn->sn_blklist = NULL;
2690 	sn->sn_listsize = 0;
2691 	lockmgr(&sn->sn_lock, LK_RELEASE, NULL);
2692 	if (snapblklist != NULL)
2693 		free(snapblklist, M_UFSMNT);
2694 	ffs_snapdata_free(sn);
2695 }
2696 
2697 static struct snapdata *
2698 ffs_snapdata_acquire(struct vnode *devvp)
2699 {
2700 	struct snapdata *nsn, *sn;
2701 	int error;
2702 
2703 	/*
2704 	 * Allocate a free snapdata.  This is done before acquiring the
2705 	 * devvp lock to avoid allocation while the devvp interlock is
2706 	 * held.
2707 	 */
2708 	nsn = ffs_snapdata_alloc();
2709 
2710 	for (;;) {
2711 		VI_LOCK(devvp);
2712 		sn = devvp->v_rdev->si_snapdata;
2713 		if (sn == NULL) {
2714 			/*
2715 			 * This is the first snapshot on this
2716 			 * filesystem and we use our pre-allocated
2717 			 * snapdata.  Publish sn with the sn_lock
2718 			 * owned by us, to avoid the race.
2719 			 */
2720 			error = lockmgr(&nsn->sn_lock, LK_EXCLUSIVE |
2721 			    LK_NOWAIT, NULL);
2722 			if (error != 0)
2723 				panic("leaked sn, lockmgr error %d", error);
2724 			sn = devvp->v_rdev->si_snapdata = nsn;
2725 			VI_UNLOCK(devvp);
2726 			nsn = NULL;
2727 			break;
2728 		}
2729 
2730 		/*
2731 		 * There is a snapshots which already exists on this
2732 		 * filesystem, grab a reference to the common lock.
2733 		 */
2734 		error = lockmgr(&sn->sn_lock, LK_INTERLOCK |
2735 		    LK_EXCLUSIVE | LK_SLEEPFAIL, VI_MTX(devvp));
2736 		if (error == 0)
2737 			break;
2738 	}
2739 
2740 	/*
2741 	 * Free any unused snapdata.
2742 	 */
2743 	if (nsn != NULL)
2744 		ffs_snapdata_free(nsn);
2745 
2746 	return (sn);
2747 }
2748 
2749 #endif
2750