1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 1998, 2000 Marshall Kirk McKusick. 5 * Copyright 2009, 2010 Jeffrey W. Roberson <jeff@FreeBSD.org> 6 * All rights reserved. 7 * 8 * The soft updates code is derived from the appendix of a University 9 * of Michigan technical report (Gregory R. Ganger and Yale N. Patt, 10 * "Soft Updates: A Solution to the Metadata Update Problem in File 11 * Systems", CSE-TR-254-95, August 1995). 12 * 13 * Further information about soft updates can be obtained from: 14 * 15 * Marshall Kirk McKusick http://www.mckusick.com/softdep/ 16 * 1614 Oxford Street mckusick@mckusick.com 17 * Berkeley, CA 94709-1608 +1-510-843-9542 18 * USA 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 24 * 1. Redistributions of source code must retain the above copyright 25 * notice, this list of conditions and the following disclaimer. 26 * 2. Redistributions in binary form must reproduce the above copyright 27 * notice, this list of conditions and the following disclaimer in the 28 * documentation and/or other materials provided with the distribution. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 31 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 32 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 33 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, 34 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 35 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 36 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 37 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 38 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 39 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 40 * 41 * from: @(#)ffs_softdep.c 9.59 (McKusick) 6/21/00 42 */ 43 44 #include <sys/cdefs.h> 45 __FBSDID("$FreeBSD$"); 46 47 #include "opt_ffs.h" 48 #include "opt_quota.h" 49 #include "opt_ddb.h" 50 51 #include <sys/param.h> 52 #include <sys/kernel.h> 53 #include <sys/systm.h> 54 #include <sys/bio.h> 55 #include <sys/buf.h> 56 #include <sys/kdb.h> 57 #include <sys/kthread.h> 58 #include <sys/ktr.h> 59 #include <sys/limits.h> 60 #include <sys/lock.h> 61 #include <sys/malloc.h> 62 #include <sys/mount.h> 63 #include <sys/mutex.h> 64 #include <sys/namei.h> 65 #include <sys/priv.h> 66 #include <sys/proc.h> 67 #include <sys/racct.h> 68 #include <sys/rwlock.h> 69 #include <sys/stat.h> 70 #include <sys/sysctl.h> 71 #include <sys/syslog.h> 72 #include <sys/vnode.h> 73 #include <sys/conf.h> 74 75 #include <ufs/ufs/dir.h> 76 #include <ufs/ufs/extattr.h> 77 #include <ufs/ufs/quota.h> 78 #include <ufs/ufs/inode.h> 79 #include <ufs/ufs/ufsmount.h> 80 #include <ufs/ffs/fs.h> 81 #include <ufs/ffs/softdep.h> 82 #include <ufs/ffs/ffs_extern.h> 83 #include <ufs/ufs/ufs_extern.h> 84 85 #include <vm/vm.h> 86 #include <vm/vm_extern.h> 87 #include <vm/vm_object.h> 88 89 #include <geom/geom.h> 90 91 #include <ddb/ddb.h> 92 93 #define KTR_SUJ 0 /* Define to KTR_SPARE. */ 94 95 #ifndef SOFTUPDATES 96 97 int 98 softdep_flushfiles(oldmnt, flags, td) 99 struct mount *oldmnt; 100 int flags; 101 struct thread *td; 102 { 103 104 panic("softdep_flushfiles called"); 105 } 106 107 int 108 softdep_mount(devvp, mp, fs, cred) 109 struct vnode *devvp; 110 struct mount *mp; 111 struct fs *fs; 112 struct ucred *cred; 113 { 114 115 return (0); 116 } 117 118 void 119 softdep_initialize() 120 { 121 122 return; 123 } 124 125 void 126 softdep_uninitialize() 127 { 128 129 return; 130 } 131 132 void 133 softdep_unmount(mp) 134 struct mount *mp; 135 { 136 137 panic("softdep_unmount called"); 138 } 139 140 void 141 softdep_setup_sbupdate(ump, fs, bp) 142 struct ufsmount *ump; 143 struct fs *fs; 144 struct buf *bp; 145 { 146 147 panic("softdep_setup_sbupdate called"); 148 } 149 150 void 151 softdep_setup_inomapdep(bp, ip, newinum, mode) 152 struct buf *bp; 153 struct inode *ip; 154 ino_t newinum; 155 int mode; 156 { 157 158 panic("softdep_setup_inomapdep called"); 159 } 160 161 void 162 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 163 struct buf *bp; 164 struct mount *mp; 165 ufs2_daddr_t newblkno; 166 int frags; 167 int oldfrags; 168 { 169 170 panic("softdep_setup_blkmapdep called"); 171 } 172 173 void 174 softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 175 struct inode *ip; 176 ufs_lbn_t lbn; 177 ufs2_daddr_t newblkno; 178 ufs2_daddr_t oldblkno; 179 long newsize; 180 long oldsize; 181 struct buf *bp; 182 { 183 184 panic("softdep_setup_allocdirect called"); 185 } 186 187 void 188 softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 189 struct inode *ip; 190 ufs_lbn_t lbn; 191 ufs2_daddr_t newblkno; 192 ufs2_daddr_t oldblkno; 193 long newsize; 194 long oldsize; 195 struct buf *bp; 196 { 197 198 panic("softdep_setup_allocext called"); 199 } 200 201 void 202 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 203 struct inode *ip; 204 ufs_lbn_t lbn; 205 struct buf *bp; 206 int ptrno; 207 ufs2_daddr_t newblkno; 208 ufs2_daddr_t oldblkno; 209 struct buf *nbp; 210 { 211 212 panic("softdep_setup_allocindir_page called"); 213 } 214 215 void 216 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 217 struct buf *nbp; 218 struct inode *ip; 219 struct buf *bp; 220 int ptrno; 221 ufs2_daddr_t newblkno; 222 { 223 224 panic("softdep_setup_allocindir_meta called"); 225 } 226 227 void 228 softdep_journal_freeblocks(ip, cred, length, flags) 229 struct inode *ip; 230 struct ucred *cred; 231 off_t length; 232 int flags; 233 { 234 235 panic("softdep_journal_freeblocks called"); 236 } 237 238 void 239 softdep_journal_fsync(ip) 240 struct inode *ip; 241 { 242 243 panic("softdep_journal_fsync called"); 244 } 245 246 void 247 softdep_setup_freeblocks(ip, length, flags) 248 struct inode *ip; 249 off_t length; 250 int flags; 251 { 252 253 panic("softdep_setup_freeblocks called"); 254 } 255 256 void 257 softdep_freefile(pvp, ino, mode) 258 struct vnode *pvp; 259 ino_t ino; 260 int mode; 261 { 262 263 panic("softdep_freefile called"); 264 } 265 266 int 267 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 268 struct buf *bp; 269 struct inode *dp; 270 off_t diroffset; 271 ino_t newinum; 272 struct buf *newdirbp; 273 int isnewblk; 274 { 275 276 panic("softdep_setup_directory_add called"); 277 } 278 279 void 280 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 281 struct buf *bp; 282 struct inode *dp; 283 caddr_t base; 284 caddr_t oldloc; 285 caddr_t newloc; 286 int entrysize; 287 { 288 289 panic("softdep_change_directoryentry_offset called"); 290 } 291 292 void 293 softdep_setup_remove(bp, dp, ip, isrmdir) 294 struct buf *bp; 295 struct inode *dp; 296 struct inode *ip; 297 int isrmdir; 298 { 299 300 panic("softdep_setup_remove called"); 301 } 302 303 void 304 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 305 struct buf *bp; 306 struct inode *dp; 307 struct inode *ip; 308 ino_t newinum; 309 int isrmdir; 310 { 311 312 panic("softdep_setup_directory_change called"); 313 } 314 315 void 316 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 317 struct mount *mp; 318 struct buf *bp; 319 ufs2_daddr_t blkno; 320 int frags; 321 struct workhead *wkhd; 322 { 323 324 panic("%s called", __FUNCTION__); 325 } 326 327 void 328 softdep_setup_inofree(mp, bp, ino, wkhd) 329 struct mount *mp; 330 struct buf *bp; 331 ino_t ino; 332 struct workhead *wkhd; 333 { 334 335 panic("%s called", __FUNCTION__); 336 } 337 338 void 339 softdep_setup_unlink(dp, ip) 340 struct inode *dp; 341 struct inode *ip; 342 { 343 344 panic("%s called", __FUNCTION__); 345 } 346 347 void 348 softdep_setup_link(dp, ip) 349 struct inode *dp; 350 struct inode *ip; 351 { 352 353 panic("%s called", __FUNCTION__); 354 } 355 356 void 357 softdep_revert_link(dp, ip) 358 struct inode *dp; 359 struct inode *ip; 360 { 361 362 panic("%s called", __FUNCTION__); 363 } 364 365 void 366 softdep_setup_rmdir(dp, ip) 367 struct inode *dp; 368 struct inode *ip; 369 { 370 371 panic("%s called", __FUNCTION__); 372 } 373 374 void 375 softdep_revert_rmdir(dp, ip) 376 struct inode *dp; 377 struct inode *ip; 378 { 379 380 panic("%s called", __FUNCTION__); 381 } 382 383 void 384 softdep_setup_create(dp, ip) 385 struct inode *dp; 386 struct inode *ip; 387 { 388 389 panic("%s called", __FUNCTION__); 390 } 391 392 void 393 softdep_revert_create(dp, ip) 394 struct inode *dp; 395 struct inode *ip; 396 { 397 398 panic("%s called", __FUNCTION__); 399 } 400 401 void 402 softdep_setup_mkdir(dp, ip) 403 struct inode *dp; 404 struct inode *ip; 405 { 406 407 panic("%s called", __FUNCTION__); 408 } 409 410 void 411 softdep_revert_mkdir(dp, ip) 412 struct inode *dp; 413 struct inode *ip; 414 { 415 416 panic("%s called", __FUNCTION__); 417 } 418 419 void 420 softdep_setup_dotdot_link(dp, ip) 421 struct inode *dp; 422 struct inode *ip; 423 { 424 425 panic("%s called", __FUNCTION__); 426 } 427 428 int 429 softdep_prealloc(vp, waitok) 430 struct vnode *vp; 431 int waitok; 432 { 433 434 panic("%s called", __FUNCTION__); 435 } 436 437 int 438 softdep_journal_lookup(mp, vpp) 439 struct mount *mp; 440 struct vnode **vpp; 441 { 442 443 return (ENOENT); 444 } 445 446 void 447 softdep_change_linkcnt(ip) 448 struct inode *ip; 449 { 450 451 panic("softdep_change_linkcnt called"); 452 } 453 454 void 455 softdep_load_inodeblock(ip) 456 struct inode *ip; 457 { 458 459 panic("softdep_load_inodeblock called"); 460 } 461 462 void 463 softdep_update_inodeblock(ip, bp, waitfor) 464 struct inode *ip; 465 struct buf *bp; 466 int waitfor; 467 { 468 469 panic("softdep_update_inodeblock called"); 470 } 471 472 int 473 softdep_fsync(vp) 474 struct vnode *vp; /* the "in_core" copy of the inode */ 475 { 476 477 return (0); 478 } 479 480 void 481 softdep_fsync_mountdev(vp) 482 struct vnode *vp; 483 { 484 485 return; 486 } 487 488 int 489 softdep_flushworklist(oldmnt, countp, td) 490 struct mount *oldmnt; 491 int *countp; 492 struct thread *td; 493 { 494 495 *countp = 0; 496 return (0); 497 } 498 499 int 500 softdep_sync_metadata(struct vnode *vp) 501 { 502 503 panic("softdep_sync_metadata called"); 504 } 505 506 int 507 softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor) 508 { 509 510 panic("softdep_sync_buf called"); 511 } 512 513 int 514 softdep_slowdown(vp) 515 struct vnode *vp; 516 { 517 518 panic("softdep_slowdown called"); 519 } 520 521 int 522 softdep_request_cleanup(fs, vp, cred, resource) 523 struct fs *fs; 524 struct vnode *vp; 525 struct ucred *cred; 526 int resource; 527 { 528 529 return (0); 530 } 531 532 int 533 softdep_check_suspend(struct mount *mp, 534 struct vnode *devvp, 535 int softdep_depcnt, 536 int softdep_accdepcnt, 537 int secondary_writes, 538 int secondary_accwrites) 539 { 540 struct bufobj *bo; 541 int error; 542 543 (void) softdep_depcnt, 544 (void) softdep_accdepcnt; 545 546 bo = &devvp->v_bufobj; 547 ASSERT_BO_WLOCKED(bo); 548 549 MNT_ILOCK(mp); 550 while (mp->mnt_secondary_writes != 0) { 551 BO_UNLOCK(bo); 552 msleep(&mp->mnt_secondary_writes, MNT_MTX(mp), 553 (PUSER - 1) | PDROP, "secwr", 0); 554 BO_LOCK(bo); 555 MNT_ILOCK(mp); 556 } 557 558 /* 559 * Reasons for needing more work before suspend: 560 * - Dirty buffers on devvp. 561 * - Secondary writes occurred after start of vnode sync loop 562 */ 563 error = 0; 564 if (bo->bo_numoutput > 0 || 565 bo->bo_dirty.bv_cnt > 0 || 566 secondary_writes != 0 || 567 mp->mnt_secondary_writes != 0 || 568 secondary_accwrites != mp->mnt_secondary_accwrites) 569 error = EAGAIN; 570 BO_UNLOCK(bo); 571 return (error); 572 } 573 574 void 575 softdep_get_depcounts(struct mount *mp, 576 int *softdepactivep, 577 int *softdepactiveaccp) 578 { 579 (void) mp; 580 *softdepactivep = 0; 581 *softdepactiveaccp = 0; 582 } 583 584 void 585 softdep_buf_append(bp, wkhd) 586 struct buf *bp; 587 struct workhead *wkhd; 588 { 589 590 panic("softdep_buf_appendwork called"); 591 } 592 593 void 594 softdep_inode_append(ip, cred, wkhd) 595 struct inode *ip; 596 struct ucred *cred; 597 struct workhead *wkhd; 598 { 599 600 panic("softdep_inode_appendwork called"); 601 } 602 603 void 604 softdep_freework(wkhd) 605 struct workhead *wkhd; 606 { 607 608 panic("softdep_freework called"); 609 } 610 611 #else 612 613 FEATURE(softupdates, "FFS soft-updates support"); 614 615 static SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW, 0, 616 "soft updates stats"); 617 static SYSCTL_NODE(_debug_softdep, OID_AUTO, total, CTLFLAG_RW, 0, 618 "total dependencies allocated"); 619 static SYSCTL_NODE(_debug_softdep, OID_AUTO, highuse, CTLFLAG_RW, 0, 620 "high use dependencies allocated"); 621 static SYSCTL_NODE(_debug_softdep, OID_AUTO, current, CTLFLAG_RW, 0, 622 "current dependencies allocated"); 623 static SYSCTL_NODE(_debug_softdep, OID_AUTO, write, CTLFLAG_RW, 0, 624 "current dependencies written"); 625 626 unsigned long dep_current[D_LAST + 1]; 627 unsigned long dep_highuse[D_LAST + 1]; 628 unsigned long dep_total[D_LAST + 1]; 629 unsigned long dep_write[D_LAST + 1]; 630 631 #define SOFTDEP_TYPE(type, str, long) \ 632 static MALLOC_DEFINE(M_ ## type, #str, long); \ 633 SYSCTL_ULONG(_debug_softdep_total, OID_AUTO, str, CTLFLAG_RD, \ 634 &dep_total[D_ ## type], 0, ""); \ 635 SYSCTL_ULONG(_debug_softdep_current, OID_AUTO, str, CTLFLAG_RD, \ 636 &dep_current[D_ ## type], 0, ""); \ 637 SYSCTL_ULONG(_debug_softdep_highuse, OID_AUTO, str, CTLFLAG_RD, \ 638 &dep_highuse[D_ ## type], 0, ""); \ 639 SYSCTL_ULONG(_debug_softdep_write, OID_AUTO, str, CTLFLAG_RD, \ 640 &dep_write[D_ ## type], 0, ""); 641 642 SOFTDEP_TYPE(PAGEDEP, pagedep, "File page dependencies"); 643 SOFTDEP_TYPE(INODEDEP, inodedep, "Inode dependencies"); 644 SOFTDEP_TYPE(BMSAFEMAP, bmsafemap, 645 "Block or frag allocated from cyl group map"); 646 SOFTDEP_TYPE(NEWBLK, newblk, "New block or frag allocation dependency"); 647 SOFTDEP_TYPE(ALLOCDIRECT, allocdirect, "Block or frag dependency for an inode"); 648 SOFTDEP_TYPE(INDIRDEP, indirdep, "Indirect block dependencies"); 649 SOFTDEP_TYPE(ALLOCINDIR, allocindir, "Block dependency for an indirect block"); 650 SOFTDEP_TYPE(FREEFRAG, freefrag, "Previously used frag for an inode"); 651 SOFTDEP_TYPE(FREEBLKS, freeblks, "Blocks freed from an inode"); 652 SOFTDEP_TYPE(FREEFILE, freefile, "Inode deallocated"); 653 SOFTDEP_TYPE(DIRADD, diradd, "New directory entry"); 654 SOFTDEP_TYPE(MKDIR, mkdir, "New directory"); 655 SOFTDEP_TYPE(DIRREM, dirrem, "Directory entry deleted"); 656 SOFTDEP_TYPE(NEWDIRBLK, newdirblk, "Unclaimed new directory block"); 657 SOFTDEP_TYPE(FREEWORK, freework, "free an inode block"); 658 SOFTDEP_TYPE(FREEDEP, freedep, "track a block free"); 659 SOFTDEP_TYPE(JADDREF, jaddref, "Journal inode ref add"); 660 SOFTDEP_TYPE(JREMREF, jremref, "Journal inode ref remove"); 661 SOFTDEP_TYPE(JMVREF, jmvref, "Journal inode ref move"); 662 SOFTDEP_TYPE(JNEWBLK, jnewblk, "Journal new block"); 663 SOFTDEP_TYPE(JFREEBLK, jfreeblk, "Journal free block"); 664 SOFTDEP_TYPE(JFREEFRAG, jfreefrag, "Journal free frag"); 665 SOFTDEP_TYPE(JSEG, jseg, "Journal segment"); 666 SOFTDEP_TYPE(JSEGDEP, jsegdep, "Journal segment complete"); 667 SOFTDEP_TYPE(SBDEP, sbdep, "Superblock write dependency"); 668 SOFTDEP_TYPE(JTRUNC, jtrunc, "Journal inode truncation"); 669 SOFTDEP_TYPE(JFSYNC, jfsync, "Journal fsync complete"); 670 671 static MALLOC_DEFINE(M_SENTINEL, "sentinel", "Worklist sentinel"); 672 673 static MALLOC_DEFINE(M_SAVEDINO, "savedino", "Saved inodes"); 674 static MALLOC_DEFINE(M_JBLOCKS, "jblocks", "Journal block locations"); 675 static MALLOC_DEFINE(M_MOUNTDATA, "softdep", "Softdep per-mount data"); 676 677 #define M_SOFTDEP_FLAGS (M_WAITOK) 678 679 /* 680 * translate from workitem type to memory type 681 * MUST match the defines above, such that memtype[D_XXX] == M_XXX 682 */ 683 static struct malloc_type *memtype[] = { 684 NULL, 685 M_PAGEDEP, 686 M_INODEDEP, 687 M_BMSAFEMAP, 688 M_NEWBLK, 689 M_ALLOCDIRECT, 690 M_INDIRDEP, 691 M_ALLOCINDIR, 692 M_FREEFRAG, 693 M_FREEBLKS, 694 M_FREEFILE, 695 M_DIRADD, 696 M_MKDIR, 697 M_DIRREM, 698 M_NEWDIRBLK, 699 M_FREEWORK, 700 M_FREEDEP, 701 M_JADDREF, 702 M_JREMREF, 703 M_JMVREF, 704 M_JNEWBLK, 705 M_JFREEBLK, 706 M_JFREEFRAG, 707 M_JSEG, 708 M_JSEGDEP, 709 M_SBDEP, 710 M_JTRUNC, 711 M_JFSYNC, 712 M_SENTINEL 713 }; 714 715 #define DtoM(type) (memtype[type]) 716 717 /* 718 * Names of malloc types. 719 */ 720 #define TYPENAME(type) \ 721 ((unsigned)(type) <= D_LAST && (unsigned)(type) >= D_FIRST ? \ 722 memtype[type]->ks_shortdesc : "???") 723 /* 724 * End system adaptation definitions. 725 */ 726 727 #define DOTDOT_OFFSET offsetof(struct dirtemplate, dotdot_ino) 728 #define DOT_OFFSET offsetof(struct dirtemplate, dot_ino) 729 730 /* 731 * Internal function prototypes. 732 */ 733 static void check_clear_deps(struct mount *); 734 static void softdep_error(char *, int); 735 static int softdep_process_worklist(struct mount *, int); 736 static int softdep_waitidle(struct mount *, int); 737 static void drain_output(struct vnode *); 738 static struct buf *getdirtybuf(struct buf *, struct rwlock *, int); 739 static int check_inodedep_free(struct inodedep *); 740 static void clear_remove(struct mount *); 741 static void clear_inodedeps(struct mount *); 742 static void unlinked_inodedep(struct mount *, struct inodedep *); 743 static void clear_unlinked_inodedep(struct inodedep *); 744 static struct inodedep *first_unlinked_inodedep(struct ufsmount *); 745 static int flush_pagedep_deps(struct vnode *, struct mount *, 746 struct diraddhd *); 747 static int free_pagedep(struct pagedep *); 748 static int flush_newblk_dep(struct vnode *, struct mount *, ufs_lbn_t); 749 static int flush_inodedep_deps(struct vnode *, struct mount *, ino_t); 750 static int flush_deplist(struct allocdirectlst *, int, int *); 751 static int sync_cgs(struct mount *, int); 752 static int handle_written_filepage(struct pagedep *, struct buf *, int); 753 static int handle_written_sbdep(struct sbdep *, struct buf *); 754 static void initiate_write_sbdep(struct sbdep *); 755 static void diradd_inode_written(struct diradd *, struct inodedep *); 756 static int handle_written_indirdep(struct indirdep *, struct buf *, 757 struct buf**, int); 758 static int handle_written_inodeblock(struct inodedep *, struct buf *, int); 759 static int jnewblk_rollforward(struct jnewblk *, struct fs *, struct cg *, 760 uint8_t *); 761 static int handle_written_bmsafemap(struct bmsafemap *, struct buf *, int); 762 static void handle_written_jaddref(struct jaddref *); 763 static void handle_written_jremref(struct jremref *); 764 static void handle_written_jseg(struct jseg *, struct buf *); 765 static void handle_written_jnewblk(struct jnewblk *); 766 static void handle_written_jblkdep(struct jblkdep *); 767 static void handle_written_jfreefrag(struct jfreefrag *); 768 static void complete_jseg(struct jseg *); 769 static void complete_jsegs(struct jseg *); 770 static void jseg_write(struct ufsmount *ump, struct jseg *, uint8_t *); 771 static void jaddref_write(struct jaddref *, struct jseg *, uint8_t *); 772 static void jremref_write(struct jremref *, struct jseg *, uint8_t *); 773 static void jmvref_write(struct jmvref *, struct jseg *, uint8_t *); 774 static void jtrunc_write(struct jtrunc *, struct jseg *, uint8_t *); 775 static void jfsync_write(struct jfsync *, struct jseg *, uint8_t *data); 776 static void jnewblk_write(struct jnewblk *, struct jseg *, uint8_t *); 777 static void jfreeblk_write(struct jfreeblk *, struct jseg *, uint8_t *); 778 static void jfreefrag_write(struct jfreefrag *, struct jseg *, uint8_t *); 779 static inline void inoref_write(struct inoref *, struct jseg *, 780 struct jrefrec *); 781 static void handle_allocdirect_partdone(struct allocdirect *, 782 struct workhead *); 783 static struct jnewblk *cancel_newblk(struct newblk *, struct worklist *, 784 struct workhead *); 785 static void indirdep_complete(struct indirdep *); 786 static int indirblk_lookup(struct mount *, ufs2_daddr_t); 787 static void indirblk_insert(struct freework *); 788 static void indirblk_remove(struct freework *); 789 static void handle_allocindir_partdone(struct allocindir *); 790 static void initiate_write_filepage(struct pagedep *, struct buf *); 791 static void initiate_write_indirdep(struct indirdep*, struct buf *); 792 static void handle_written_mkdir(struct mkdir *, int); 793 static int jnewblk_rollback(struct jnewblk *, struct fs *, struct cg *, 794 uint8_t *); 795 static void initiate_write_bmsafemap(struct bmsafemap *, struct buf *); 796 static void initiate_write_inodeblock_ufs1(struct inodedep *, struct buf *); 797 static void initiate_write_inodeblock_ufs2(struct inodedep *, struct buf *); 798 static void handle_workitem_freefile(struct freefile *); 799 static int handle_workitem_remove(struct dirrem *, int); 800 static struct dirrem *newdirrem(struct buf *, struct inode *, 801 struct inode *, int, struct dirrem **); 802 static struct indirdep *indirdep_lookup(struct mount *, struct inode *, 803 struct buf *); 804 static void cancel_indirdep(struct indirdep *, struct buf *, 805 struct freeblks *); 806 static void free_indirdep(struct indirdep *); 807 static void free_diradd(struct diradd *, struct workhead *); 808 static void merge_diradd(struct inodedep *, struct diradd *); 809 static void complete_diradd(struct diradd *); 810 static struct diradd *diradd_lookup(struct pagedep *, int); 811 static struct jremref *cancel_diradd_dotdot(struct inode *, struct dirrem *, 812 struct jremref *); 813 static struct jremref *cancel_mkdir_dotdot(struct inode *, struct dirrem *, 814 struct jremref *); 815 static void cancel_diradd(struct diradd *, struct dirrem *, struct jremref *, 816 struct jremref *, struct jremref *); 817 static void dirrem_journal(struct dirrem *, struct jremref *, struct jremref *, 818 struct jremref *); 819 static void cancel_allocindir(struct allocindir *, struct buf *bp, 820 struct freeblks *, int); 821 static int setup_trunc_indir(struct freeblks *, struct inode *, 822 ufs_lbn_t, ufs_lbn_t, ufs2_daddr_t); 823 static void complete_trunc_indir(struct freework *); 824 static void trunc_indirdep(struct indirdep *, struct freeblks *, struct buf *, 825 int); 826 static void complete_mkdir(struct mkdir *); 827 static void free_newdirblk(struct newdirblk *); 828 static void free_jremref(struct jremref *); 829 static void free_jaddref(struct jaddref *); 830 static void free_jsegdep(struct jsegdep *); 831 static void free_jsegs(struct jblocks *); 832 static void rele_jseg(struct jseg *); 833 static void free_jseg(struct jseg *, struct jblocks *); 834 static void free_jnewblk(struct jnewblk *); 835 static void free_jblkdep(struct jblkdep *); 836 static void free_jfreefrag(struct jfreefrag *); 837 static void free_freedep(struct freedep *); 838 static void journal_jremref(struct dirrem *, struct jremref *, 839 struct inodedep *); 840 static void cancel_jnewblk(struct jnewblk *, struct workhead *); 841 static int cancel_jaddref(struct jaddref *, struct inodedep *, 842 struct workhead *); 843 static void cancel_jfreefrag(struct jfreefrag *); 844 static inline void setup_freedirect(struct freeblks *, struct inode *, 845 int, int); 846 static inline void setup_freeext(struct freeblks *, struct inode *, int, int); 847 static inline void setup_freeindir(struct freeblks *, struct inode *, int, 848 ufs_lbn_t, int); 849 static inline struct freeblks *newfreeblks(struct mount *, struct inode *); 850 static void freeblks_free(struct ufsmount *, struct freeblks *, int); 851 static void indir_trunc(struct freework *, ufs2_daddr_t, ufs_lbn_t); 852 static ufs2_daddr_t blkcount(struct fs *, ufs2_daddr_t, off_t); 853 static int trunc_check_buf(struct buf *, int *, ufs_lbn_t, int, int); 854 static void trunc_dependencies(struct inode *, struct freeblks *, ufs_lbn_t, 855 int, int); 856 static void trunc_pages(struct inode *, off_t, ufs2_daddr_t, int); 857 static int cancel_pagedep(struct pagedep *, struct freeblks *, int); 858 static int deallocate_dependencies(struct buf *, struct freeblks *, int); 859 static void newblk_freefrag(struct newblk*); 860 static void free_newblk(struct newblk *); 861 static void cancel_allocdirect(struct allocdirectlst *, 862 struct allocdirect *, struct freeblks *); 863 static int check_inode_unwritten(struct inodedep *); 864 static int free_inodedep(struct inodedep *); 865 static void freework_freeblock(struct freework *, u_long); 866 static void freework_enqueue(struct freework *); 867 static int handle_workitem_freeblocks(struct freeblks *, int); 868 static int handle_complete_freeblocks(struct freeblks *, int); 869 static void handle_workitem_indirblk(struct freework *); 870 static void handle_written_freework(struct freework *); 871 static void merge_inode_lists(struct allocdirectlst *,struct allocdirectlst *); 872 static struct worklist *jnewblk_merge(struct worklist *, struct worklist *, 873 struct workhead *); 874 static struct freefrag *setup_allocindir_phase2(struct buf *, struct inode *, 875 struct inodedep *, struct allocindir *, ufs_lbn_t); 876 static struct allocindir *newallocindir(struct inode *, int, ufs2_daddr_t, 877 ufs2_daddr_t, ufs_lbn_t); 878 static void handle_workitem_freefrag(struct freefrag *); 879 static struct freefrag *newfreefrag(struct inode *, ufs2_daddr_t, long, 880 ufs_lbn_t, u_long); 881 static void allocdirect_merge(struct allocdirectlst *, 882 struct allocdirect *, struct allocdirect *); 883 static struct freefrag *allocindir_merge(struct allocindir *, 884 struct allocindir *); 885 static int bmsafemap_find(struct bmsafemap_hashhead *, int, 886 struct bmsafemap **); 887 static struct bmsafemap *bmsafemap_lookup(struct mount *, struct buf *, 888 int cg, struct bmsafemap *); 889 static int newblk_find(struct newblk_hashhead *, ufs2_daddr_t, int, 890 struct newblk **); 891 static int newblk_lookup(struct mount *, ufs2_daddr_t, int, struct newblk **); 892 static int inodedep_find(struct inodedep_hashhead *, ino_t, 893 struct inodedep **); 894 static int inodedep_lookup(struct mount *, ino_t, int, struct inodedep **); 895 static int pagedep_lookup(struct mount *, struct buf *bp, ino_t, ufs_lbn_t, 896 int, struct pagedep **); 897 static int pagedep_find(struct pagedep_hashhead *, ino_t, ufs_lbn_t, 898 struct pagedep **); 899 static void pause_timer(void *); 900 static int request_cleanup(struct mount *, int); 901 static int softdep_request_cleanup_flush(struct mount *, struct ufsmount *); 902 static void schedule_cleanup(struct mount *); 903 static void softdep_ast_cleanup_proc(struct thread *); 904 static struct ufsmount *softdep_bp_to_mp(struct buf *bp); 905 static int process_worklist_item(struct mount *, int, int); 906 static void process_removes(struct vnode *); 907 static void process_truncates(struct vnode *); 908 static void jwork_move(struct workhead *, struct workhead *); 909 static void jwork_insert(struct workhead *, struct jsegdep *); 910 static void add_to_worklist(struct worklist *, int); 911 static void wake_worklist(struct worklist *); 912 static void wait_worklist(struct worklist *, char *); 913 static void remove_from_worklist(struct worklist *); 914 static void softdep_flush(void *); 915 static void softdep_flushjournal(struct mount *); 916 static int softdep_speedup(struct ufsmount *); 917 static void worklist_speedup(struct mount *); 918 static int journal_mount(struct mount *, struct fs *, struct ucred *); 919 static void journal_unmount(struct ufsmount *); 920 static int journal_space(struct ufsmount *, int); 921 static void journal_suspend(struct ufsmount *); 922 static int journal_unsuspend(struct ufsmount *ump); 923 static void softdep_prelink(struct vnode *, struct vnode *); 924 static void add_to_journal(struct worklist *); 925 static void remove_from_journal(struct worklist *); 926 static bool softdep_excess_items(struct ufsmount *, int); 927 static void softdep_process_journal(struct mount *, struct worklist *, int); 928 static struct jremref *newjremref(struct dirrem *, struct inode *, 929 struct inode *ip, off_t, nlink_t); 930 static struct jaddref *newjaddref(struct inode *, ino_t, off_t, int16_t, 931 uint16_t); 932 static inline void newinoref(struct inoref *, ino_t, ino_t, off_t, nlink_t, 933 uint16_t); 934 static inline struct jsegdep *inoref_jseg(struct inoref *); 935 static struct jmvref *newjmvref(struct inode *, ino_t, off_t, off_t); 936 static struct jfreeblk *newjfreeblk(struct freeblks *, ufs_lbn_t, 937 ufs2_daddr_t, int); 938 static void adjust_newfreework(struct freeblks *, int); 939 static struct jtrunc *newjtrunc(struct freeblks *, off_t, int); 940 static void move_newblock_dep(struct jaddref *, struct inodedep *); 941 static void cancel_jfreeblk(struct freeblks *, ufs2_daddr_t); 942 static struct jfreefrag *newjfreefrag(struct freefrag *, struct inode *, 943 ufs2_daddr_t, long, ufs_lbn_t); 944 static struct freework *newfreework(struct ufsmount *, struct freeblks *, 945 struct freework *, ufs_lbn_t, ufs2_daddr_t, int, int, int); 946 static int jwait(struct worklist *, int); 947 static struct inodedep *inodedep_lookup_ip(struct inode *); 948 static int bmsafemap_backgroundwrite(struct bmsafemap *, struct buf *); 949 static struct freefile *handle_bufwait(struct inodedep *, struct workhead *); 950 static void handle_jwork(struct workhead *); 951 static struct mkdir *setup_newdir(struct diradd *, ino_t, ino_t, struct buf *, 952 struct mkdir **); 953 static struct jblocks *jblocks_create(void); 954 static ufs2_daddr_t jblocks_alloc(struct jblocks *, int, int *); 955 static void jblocks_free(struct jblocks *, struct mount *, int); 956 static void jblocks_destroy(struct jblocks *); 957 static void jblocks_add(struct jblocks *, ufs2_daddr_t, int); 958 959 /* 960 * Exported softdep operations. 961 */ 962 static void softdep_disk_io_initiation(struct buf *); 963 static void softdep_disk_write_complete(struct buf *); 964 static void softdep_deallocate_dependencies(struct buf *); 965 static int softdep_count_dependencies(struct buf *bp, int); 966 967 /* 968 * Global lock over all of soft updates. 969 */ 970 static struct mtx lk; 971 MTX_SYSINIT(softdep_lock, &lk, "global softdep", MTX_DEF); 972 973 #define ACQUIRE_GBLLOCK(lk) mtx_lock(lk) 974 #define FREE_GBLLOCK(lk) mtx_unlock(lk) 975 #define GBLLOCK_OWNED(lk) mtx_assert((lk), MA_OWNED) 976 977 /* 978 * Per-filesystem soft-updates locking. 979 */ 980 #define LOCK_PTR(ump) (&(ump)->um_softdep->sd_fslock) 981 #define TRY_ACQUIRE_LOCK(ump) rw_try_wlock(&(ump)->um_softdep->sd_fslock) 982 #define ACQUIRE_LOCK(ump) rw_wlock(&(ump)->um_softdep->sd_fslock) 983 #define FREE_LOCK(ump) rw_wunlock(&(ump)->um_softdep->sd_fslock) 984 #define LOCK_OWNED(ump) rw_assert(&(ump)->um_softdep->sd_fslock, \ 985 RA_WLOCKED) 986 987 #define BUF_AREC(bp) lockallowrecurse(&(bp)->b_lock) 988 #define BUF_NOREC(bp) lockdisablerecurse(&(bp)->b_lock) 989 990 /* 991 * Worklist queue management. 992 * These routines require that the lock be held. 993 */ 994 #ifndef /* NOT */ INVARIANTS 995 #define WORKLIST_INSERT(head, item) do { \ 996 (item)->wk_state |= ONWORKLIST; \ 997 LIST_INSERT_HEAD(head, item, wk_list); \ 998 } while (0) 999 #define WORKLIST_REMOVE(item) do { \ 1000 (item)->wk_state &= ~ONWORKLIST; \ 1001 LIST_REMOVE(item, wk_list); \ 1002 } while (0) 1003 #define WORKLIST_INSERT_UNLOCKED WORKLIST_INSERT 1004 #define WORKLIST_REMOVE_UNLOCKED WORKLIST_REMOVE 1005 1006 #else /* INVARIANTS */ 1007 static void worklist_insert(struct workhead *, struct worklist *, int, 1008 const char *, int); 1009 static void worklist_remove(struct worklist *, int, const char *, int); 1010 1011 #define WORKLIST_INSERT(head, item) \ 1012 worklist_insert(head, item, 1, __func__, __LINE__) 1013 #define WORKLIST_INSERT_UNLOCKED(head, item)\ 1014 worklist_insert(head, item, 0, __func__, __LINE__) 1015 #define WORKLIST_REMOVE(item)\ 1016 worklist_remove(item, 1, __func__, __LINE__) 1017 #define WORKLIST_REMOVE_UNLOCKED(item)\ 1018 worklist_remove(item, 0, __func__, __LINE__) 1019 1020 static void 1021 worklist_insert(head, item, locked, func, line) 1022 struct workhead *head; 1023 struct worklist *item; 1024 int locked; 1025 const char *func; 1026 int line; 1027 { 1028 1029 if (locked) 1030 LOCK_OWNED(VFSTOUFS(item->wk_mp)); 1031 if (item->wk_state & ONWORKLIST) 1032 panic("worklist_insert: %p %s(0x%X) already on list, " 1033 "added in function %s at line %d", 1034 item, TYPENAME(item->wk_type), item->wk_state, 1035 item->wk_func, item->wk_line); 1036 item->wk_state |= ONWORKLIST; 1037 item->wk_func = func; 1038 item->wk_line = line; 1039 LIST_INSERT_HEAD(head, item, wk_list); 1040 } 1041 1042 static void 1043 worklist_remove(item, locked, func, line) 1044 struct worklist *item; 1045 int locked; 1046 const char *func; 1047 int line; 1048 { 1049 1050 if (locked) 1051 LOCK_OWNED(VFSTOUFS(item->wk_mp)); 1052 if ((item->wk_state & ONWORKLIST) == 0) 1053 panic("worklist_remove: %p %s(0x%X) not on list, " 1054 "removed in function %s at line %d", 1055 item, TYPENAME(item->wk_type), item->wk_state, 1056 item->wk_func, item->wk_line); 1057 item->wk_state &= ~ONWORKLIST; 1058 item->wk_func = func; 1059 item->wk_line = line; 1060 LIST_REMOVE(item, wk_list); 1061 } 1062 #endif /* INVARIANTS */ 1063 1064 /* 1065 * Merge two jsegdeps keeping only the oldest one as newer references 1066 * can't be discarded until after older references. 1067 */ 1068 static inline struct jsegdep * 1069 jsegdep_merge(struct jsegdep *one, struct jsegdep *two) 1070 { 1071 struct jsegdep *swp; 1072 1073 if (two == NULL) 1074 return (one); 1075 1076 if (one->jd_seg->js_seq > two->jd_seg->js_seq) { 1077 swp = one; 1078 one = two; 1079 two = swp; 1080 } 1081 WORKLIST_REMOVE(&two->jd_list); 1082 free_jsegdep(two); 1083 1084 return (one); 1085 } 1086 1087 /* 1088 * If two freedeps are compatible free one to reduce list size. 1089 */ 1090 static inline struct freedep * 1091 freedep_merge(struct freedep *one, struct freedep *two) 1092 { 1093 if (two == NULL) 1094 return (one); 1095 1096 if (one->fd_freework == two->fd_freework) { 1097 WORKLIST_REMOVE(&two->fd_list); 1098 free_freedep(two); 1099 } 1100 return (one); 1101 } 1102 1103 /* 1104 * Move journal work from one list to another. Duplicate freedeps and 1105 * jsegdeps are coalesced to keep the lists as small as possible. 1106 */ 1107 static void 1108 jwork_move(dst, src) 1109 struct workhead *dst; 1110 struct workhead *src; 1111 { 1112 struct freedep *freedep; 1113 struct jsegdep *jsegdep; 1114 struct worklist *wkn; 1115 struct worklist *wk; 1116 1117 KASSERT(dst != src, 1118 ("jwork_move: dst == src")); 1119 freedep = NULL; 1120 jsegdep = NULL; 1121 LIST_FOREACH_SAFE(wk, dst, wk_list, wkn) { 1122 if (wk->wk_type == D_JSEGDEP) 1123 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1124 else if (wk->wk_type == D_FREEDEP) 1125 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1126 } 1127 1128 while ((wk = LIST_FIRST(src)) != NULL) { 1129 WORKLIST_REMOVE(wk); 1130 WORKLIST_INSERT(dst, wk); 1131 if (wk->wk_type == D_JSEGDEP) { 1132 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1133 continue; 1134 } 1135 if (wk->wk_type == D_FREEDEP) 1136 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1137 } 1138 } 1139 1140 static void 1141 jwork_insert(dst, jsegdep) 1142 struct workhead *dst; 1143 struct jsegdep *jsegdep; 1144 { 1145 struct jsegdep *jsegdepn; 1146 struct worklist *wk; 1147 1148 LIST_FOREACH(wk, dst, wk_list) 1149 if (wk->wk_type == D_JSEGDEP) 1150 break; 1151 if (wk == NULL) { 1152 WORKLIST_INSERT(dst, &jsegdep->jd_list); 1153 return; 1154 } 1155 jsegdepn = WK_JSEGDEP(wk); 1156 if (jsegdep->jd_seg->js_seq < jsegdepn->jd_seg->js_seq) { 1157 WORKLIST_REMOVE(wk); 1158 free_jsegdep(jsegdepn); 1159 WORKLIST_INSERT(dst, &jsegdep->jd_list); 1160 } else 1161 free_jsegdep(jsegdep); 1162 } 1163 1164 /* 1165 * Routines for tracking and managing workitems. 1166 */ 1167 static void workitem_free(struct worklist *, int); 1168 static void workitem_alloc(struct worklist *, int, struct mount *); 1169 static void workitem_reassign(struct worklist *, int); 1170 1171 #define WORKITEM_FREE(item, type) \ 1172 workitem_free((struct worklist *)(item), (type)) 1173 #define WORKITEM_REASSIGN(item, type) \ 1174 workitem_reassign((struct worklist *)(item), (type)) 1175 1176 static void 1177 workitem_free(item, type) 1178 struct worklist *item; 1179 int type; 1180 { 1181 struct ufsmount *ump; 1182 1183 #ifdef INVARIANTS 1184 if (item->wk_state & ONWORKLIST) 1185 panic("workitem_free: %s(0x%X) still on list, " 1186 "added in function %s at line %d", 1187 TYPENAME(item->wk_type), item->wk_state, 1188 item->wk_func, item->wk_line); 1189 if (item->wk_type != type && type != D_NEWBLK) 1190 panic("workitem_free: type mismatch %s != %s", 1191 TYPENAME(item->wk_type), TYPENAME(type)); 1192 #endif 1193 if (item->wk_state & IOWAITING) 1194 wakeup(item); 1195 ump = VFSTOUFS(item->wk_mp); 1196 LOCK_OWNED(ump); 1197 KASSERT(ump->softdep_deps > 0, 1198 ("workitem_free: %s: softdep_deps going negative", 1199 ump->um_fs->fs_fsmnt)); 1200 if (--ump->softdep_deps == 0 && ump->softdep_req) 1201 wakeup(&ump->softdep_deps); 1202 KASSERT(dep_current[item->wk_type] > 0, 1203 ("workitem_free: %s: dep_current[%s] going negative", 1204 ump->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1205 KASSERT(ump->softdep_curdeps[item->wk_type] > 0, 1206 ("workitem_free: %s: softdep_curdeps[%s] going negative", 1207 ump->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1208 atomic_subtract_long(&dep_current[item->wk_type], 1); 1209 ump->softdep_curdeps[item->wk_type] -= 1; 1210 free(item, DtoM(type)); 1211 } 1212 1213 static void 1214 workitem_alloc(item, type, mp) 1215 struct worklist *item; 1216 int type; 1217 struct mount *mp; 1218 { 1219 struct ufsmount *ump; 1220 1221 item->wk_type = type; 1222 item->wk_mp = mp; 1223 item->wk_state = 0; 1224 1225 ump = VFSTOUFS(mp); 1226 ACQUIRE_GBLLOCK(&lk); 1227 dep_current[type]++; 1228 if (dep_current[type] > dep_highuse[type]) 1229 dep_highuse[type] = dep_current[type]; 1230 dep_total[type]++; 1231 FREE_GBLLOCK(&lk); 1232 ACQUIRE_LOCK(ump); 1233 ump->softdep_curdeps[type] += 1; 1234 ump->softdep_deps++; 1235 ump->softdep_accdeps++; 1236 FREE_LOCK(ump); 1237 } 1238 1239 static void 1240 workitem_reassign(item, newtype) 1241 struct worklist *item; 1242 int newtype; 1243 { 1244 struct ufsmount *ump; 1245 1246 ump = VFSTOUFS(item->wk_mp); 1247 LOCK_OWNED(ump); 1248 KASSERT(ump->softdep_curdeps[item->wk_type] > 0, 1249 ("workitem_reassign: %s: softdep_curdeps[%s] going negative", 1250 VFSTOUFS(item->wk_mp)->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1251 ump->softdep_curdeps[item->wk_type] -= 1; 1252 ump->softdep_curdeps[newtype] += 1; 1253 KASSERT(dep_current[item->wk_type] > 0, 1254 ("workitem_reassign: %s: dep_current[%s] going negative", 1255 VFSTOUFS(item->wk_mp)->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1256 ACQUIRE_GBLLOCK(&lk); 1257 dep_current[newtype]++; 1258 dep_current[item->wk_type]--; 1259 if (dep_current[newtype] > dep_highuse[newtype]) 1260 dep_highuse[newtype] = dep_current[newtype]; 1261 dep_total[newtype]++; 1262 FREE_GBLLOCK(&lk); 1263 item->wk_type = newtype; 1264 } 1265 1266 /* 1267 * Workitem queue management 1268 */ 1269 static int max_softdeps; /* maximum number of structs before slowdown */ 1270 static int tickdelay = 2; /* number of ticks to pause during slowdown */ 1271 static int proc_waiting; /* tracks whether we have a timeout posted */ 1272 static int *stat_countp; /* statistic to count in proc_waiting timeout */ 1273 static struct callout softdep_callout; 1274 static int req_clear_inodedeps; /* syncer process flush some inodedeps */ 1275 static int req_clear_remove; /* syncer process flush some freeblks */ 1276 static int softdep_flushcache = 0; /* Should we do BIO_FLUSH? */ 1277 1278 /* 1279 * runtime statistics 1280 */ 1281 static int stat_flush_threads; /* number of softdep flushing threads */ 1282 static int stat_worklist_push; /* number of worklist cleanups */ 1283 static int stat_blk_limit_push; /* number of times block limit neared */ 1284 static int stat_ino_limit_push; /* number of times inode limit neared */ 1285 static int stat_blk_limit_hit; /* number of times block slowdown imposed */ 1286 static int stat_ino_limit_hit; /* number of times inode slowdown imposed */ 1287 static int stat_sync_limit_hit; /* number of synchronous slowdowns imposed */ 1288 static int stat_indir_blk_ptrs; /* bufs redirtied as indir ptrs not written */ 1289 static int stat_inode_bitmap; /* bufs redirtied as inode bitmap not written */ 1290 static int stat_direct_blk_ptrs;/* bufs redirtied as direct ptrs not written */ 1291 static int stat_dir_entry; /* bufs redirtied as dir entry cannot write */ 1292 static int stat_jaddref; /* bufs redirtied as ino bitmap can not write */ 1293 static int stat_jnewblk; /* bufs redirtied as blk bitmap can not write */ 1294 static int stat_journal_min; /* Times hit journal min threshold */ 1295 static int stat_journal_low; /* Times hit journal low threshold */ 1296 static int stat_journal_wait; /* Times blocked in jwait(). */ 1297 static int stat_jwait_filepage; /* Times blocked in jwait() for filepage. */ 1298 static int stat_jwait_freeblks; /* Times blocked in jwait() for freeblks. */ 1299 static int stat_jwait_inode; /* Times blocked in jwait() for inodes. */ 1300 static int stat_jwait_newblk; /* Times blocked in jwait() for newblks. */ 1301 static int stat_cleanup_high_delay; /* Maximum cleanup delay (in ticks) */ 1302 static int stat_cleanup_blkrequests; /* Number of block cleanup requests */ 1303 static int stat_cleanup_inorequests; /* Number of inode cleanup requests */ 1304 static int stat_cleanup_retries; /* Number of cleanups that needed to flush */ 1305 static int stat_cleanup_failures; /* Number of cleanup requests that failed */ 1306 static int stat_emptyjblocks; /* Number of potentially empty journal blocks */ 1307 1308 SYSCTL_INT(_debug_softdep, OID_AUTO, max_softdeps, CTLFLAG_RW, 1309 &max_softdeps, 0, ""); 1310 SYSCTL_INT(_debug_softdep, OID_AUTO, tickdelay, CTLFLAG_RW, 1311 &tickdelay, 0, ""); 1312 SYSCTL_INT(_debug_softdep, OID_AUTO, flush_threads, CTLFLAG_RD, 1313 &stat_flush_threads, 0, ""); 1314 SYSCTL_INT(_debug_softdep, OID_AUTO, worklist_push, 1315 CTLFLAG_RW | CTLFLAG_STATS, &stat_worklist_push, 0,""); 1316 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_push, 1317 CTLFLAG_RW | CTLFLAG_STATS, &stat_blk_limit_push, 0,""); 1318 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_push, 1319 CTLFLAG_RW | CTLFLAG_STATS, &stat_ino_limit_push, 0,""); 1320 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_hit, 1321 CTLFLAG_RW | CTLFLAG_STATS, &stat_blk_limit_hit, 0, ""); 1322 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_hit, 1323 CTLFLAG_RW | CTLFLAG_STATS, &stat_ino_limit_hit, 0, ""); 1324 SYSCTL_INT(_debug_softdep, OID_AUTO, sync_limit_hit, 1325 CTLFLAG_RW | CTLFLAG_STATS, &stat_sync_limit_hit, 0, ""); 1326 SYSCTL_INT(_debug_softdep, OID_AUTO, indir_blk_ptrs, 1327 CTLFLAG_RW | CTLFLAG_STATS, &stat_indir_blk_ptrs, 0, ""); 1328 SYSCTL_INT(_debug_softdep, OID_AUTO, inode_bitmap, 1329 CTLFLAG_RW | CTLFLAG_STATS, &stat_inode_bitmap, 0, ""); 1330 SYSCTL_INT(_debug_softdep, OID_AUTO, direct_blk_ptrs, 1331 CTLFLAG_RW | CTLFLAG_STATS, &stat_direct_blk_ptrs, 0, ""); 1332 SYSCTL_INT(_debug_softdep, OID_AUTO, dir_entry, 1333 CTLFLAG_RW | CTLFLAG_STATS, &stat_dir_entry, 0, ""); 1334 SYSCTL_INT(_debug_softdep, OID_AUTO, jaddref_rollback, 1335 CTLFLAG_RW | CTLFLAG_STATS, &stat_jaddref, 0, ""); 1336 SYSCTL_INT(_debug_softdep, OID_AUTO, jnewblk_rollback, 1337 CTLFLAG_RW | CTLFLAG_STATS, &stat_jnewblk, 0, ""); 1338 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_low, 1339 CTLFLAG_RW | CTLFLAG_STATS, &stat_journal_low, 0, ""); 1340 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_min, 1341 CTLFLAG_RW | CTLFLAG_STATS, &stat_journal_min, 0, ""); 1342 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_wait, 1343 CTLFLAG_RW | CTLFLAG_STATS, &stat_journal_wait, 0, ""); 1344 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_filepage, 1345 CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_filepage, 0, ""); 1346 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_freeblks, 1347 CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_freeblks, 0, ""); 1348 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_inode, 1349 CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_inode, 0, ""); 1350 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_newblk, 1351 CTLFLAG_RW | CTLFLAG_STATS, &stat_jwait_newblk, 0, ""); 1352 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_blkrequests, 1353 CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_blkrequests, 0, ""); 1354 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_inorequests, 1355 CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_inorequests, 0, ""); 1356 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_high_delay, 1357 CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_high_delay, 0, ""); 1358 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_retries, 1359 CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_retries, 0, ""); 1360 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_failures, 1361 CTLFLAG_RW | CTLFLAG_STATS, &stat_cleanup_failures, 0, ""); 1362 1363 SYSCTL_INT(_debug_softdep, OID_AUTO, flushcache, CTLFLAG_RW, 1364 &softdep_flushcache, 0, ""); 1365 SYSCTL_INT(_debug_softdep, OID_AUTO, emptyjblocks, CTLFLAG_RD, 1366 &stat_emptyjblocks, 0, ""); 1367 1368 SYSCTL_DECL(_vfs_ffs); 1369 1370 /* Whether to recompute the summary at mount time */ 1371 static int compute_summary_at_mount = 0; 1372 SYSCTL_INT(_vfs_ffs, OID_AUTO, compute_summary_at_mount, CTLFLAG_RW, 1373 &compute_summary_at_mount, 0, "Recompute summary at mount"); 1374 static int print_threads = 0; 1375 SYSCTL_INT(_debug_softdep, OID_AUTO, print_threads, CTLFLAG_RW, 1376 &print_threads, 0, "Notify flusher thread start/stop"); 1377 1378 /* List of all filesystems mounted with soft updates */ 1379 static TAILQ_HEAD(, mount_softdeps) softdepmounts; 1380 1381 /* 1382 * This function cleans the worklist for a filesystem. 1383 * Each filesystem running with soft dependencies gets its own 1384 * thread to run in this function. The thread is started up in 1385 * softdep_mount and shutdown in softdep_unmount. They show up 1386 * as part of the kernel "bufdaemon" process whose process 1387 * entry is available in bufdaemonproc. 1388 */ 1389 static int searchfailed; 1390 extern struct proc *bufdaemonproc; 1391 static void 1392 softdep_flush(addr) 1393 void *addr; 1394 { 1395 struct mount *mp; 1396 struct thread *td; 1397 struct ufsmount *ump; 1398 1399 td = curthread; 1400 td->td_pflags |= TDP_NORUNNINGBUF; 1401 mp = (struct mount *)addr; 1402 ump = VFSTOUFS(mp); 1403 atomic_add_int(&stat_flush_threads, 1); 1404 ACQUIRE_LOCK(ump); 1405 ump->softdep_flags &= ~FLUSH_STARTING; 1406 wakeup(&ump->softdep_flushtd); 1407 FREE_LOCK(ump); 1408 if (print_threads) { 1409 if (stat_flush_threads == 1) 1410 printf("Running %s at pid %d\n", bufdaemonproc->p_comm, 1411 bufdaemonproc->p_pid); 1412 printf("Start thread %s\n", td->td_name); 1413 } 1414 for (;;) { 1415 while (softdep_process_worklist(mp, 0) > 0 || 1416 (MOUNTEDSUJ(mp) && 1417 VFSTOUFS(mp)->softdep_jblocks->jb_suspended)) 1418 kthread_suspend_check(); 1419 ACQUIRE_LOCK(ump); 1420 if ((ump->softdep_flags & (FLUSH_CLEANUP | FLUSH_EXIT)) == 0) 1421 msleep(&ump->softdep_flushtd, LOCK_PTR(ump), PVM, 1422 "sdflush", hz / 2); 1423 ump->softdep_flags &= ~FLUSH_CLEANUP; 1424 /* 1425 * Check to see if we are done and need to exit. 1426 */ 1427 if ((ump->softdep_flags & FLUSH_EXIT) == 0) { 1428 FREE_LOCK(ump); 1429 continue; 1430 } 1431 ump->softdep_flags &= ~FLUSH_EXIT; 1432 FREE_LOCK(ump); 1433 wakeup(&ump->softdep_flags); 1434 if (print_threads) 1435 printf("Stop thread %s: searchfailed %d, did cleanups %d\n", td->td_name, searchfailed, ump->um_softdep->sd_cleanups); 1436 atomic_subtract_int(&stat_flush_threads, 1); 1437 kthread_exit(); 1438 panic("kthread_exit failed\n"); 1439 } 1440 } 1441 1442 static void 1443 worklist_speedup(mp) 1444 struct mount *mp; 1445 { 1446 struct ufsmount *ump; 1447 1448 ump = VFSTOUFS(mp); 1449 LOCK_OWNED(ump); 1450 if ((ump->softdep_flags & (FLUSH_CLEANUP | FLUSH_EXIT)) == 0) 1451 ump->softdep_flags |= FLUSH_CLEANUP; 1452 wakeup(&ump->softdep_flushtd); 1453 } 1454 1455 static int 1456 softdep_speedup(ump) 1457 struct ufsmount *ump; 1458 { 1459 struct ufsmount *altump; 1460 struct mount_softdeps *sdp; 1461 1462 LOCK_OWNED(ump); 1463 worklist_speedup(ump->um_mountp); 1464 bd_speedup(); 1465 /* 1466 * If we have global shortages, then we need other 1467 * filesystems to help with the cleanup. Here we wakeup a 1468 * flusher thread for a filesystem that is over its fair 1469 * share of resources. 1470 */ 1471 if (req_clear_inodedeps || req_clear_remove) { 1472 ACQUIRE_GBLLOCK(&lk); 1473 TAILQ_FOREACH(sdp, &softdepmounts, sd_next) { 1474 if ((altump = sdp->sd_ump) == ump) 1475 continue; 1476 if (((req_clear_inodedeps && 1477 altump->softdep_curdeps[D_INODEDEP] > 1478 max_softdeps / stat_flush_threads) || 1479 (req_clear_remove && 1480 altump->softdep_curdeps[D_DIRREM] > 1481 (max_softdeps / 2) / stat_flush_threads)) && 1482 TRY_ACQUIRE_LOCK(altump)) 1483 break; 1484 } 1485 if (sdp == NULL) { 1486 searchfailed++; 1487 FREE_GBLLOCK(&lk); 1488 } else { 1489 /* 1490 * Move to the end of the list so we pick a 1491 * different one on out next try. 1492 */ 1493 TAILQ_REMOVE(&softdepmounts, sdp, sd_next); 1494 TAILQ_INSERT_TAIL(&softdepmounts, sdp, sd_next); 1495 FREE_GBLLOCK(&lk); 1496 if ((altump->softdep_flags & 1497 (FLUSH_CLEANUP | FLUSH_EXIT)) == 0) 1498 altump->softdep_flags |= FLUSH_CLEANUP; 1499 altump->um_softdep->sd_cleanups++; 1500 wakeup(&altump->softdep_flushtd); 1501 FREE_LOCK(altump); 1502 } 1503 } 1504 return (speedup_syncer()); 1505 } 1506 1507 /* 1508 * Add an item to the end of the work queue. 1509 * This routine requires that the lock be held. 1510 * This is the only routine that adds items to the list. 1511 * The following routine is the only one that removes items 1512 * and does so in order from first to last. 1513 */ 1514 1515 #define WK_HEAD 0x0001 /* Add to HEAD. */ 1516 #define WK_NODELAY 0x0002 /* Process immediately. */ 1517 1518 static void 1519 add_to_worklist(wk, flags) 1520 struct worklist *wk; 1521 int flags; 1522 { 1523 struct ufsmount *ump; 1524 1525 ump = VFSTOUFS(wk->wk_mp); 1526 LOCK_OWNED(ump); 1527 if (wk->wk_state & ONWORKLIST) 1528 panic("add_to_worklist: %s(0x%X) already on list", 1529 TYPENAME(wk->wk_type), wk->wk_state); 1530 wk->wk_state |= ONWORKLIST; 1531 if (ump->softdep_on_worklist == 0) { 1532 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list); 1533 ump->softdep_worklist_tail = wk; 1534 } else if (flags & WK_HEAD) { 1535 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list); 1536 } else { 1537 LIST_INSERT_AFTER(ump->softdep_worklist_tail, wk, wk_list); 1538 ump->softdep_worklist_tail = wk; 1539 } 1540 ump->softdep_on_worklist += 1; 1541 if (flags & WK_NODELAY) 1542 worklist_speedup(wk->wk_mp); 1543 } 1544 1545 /* 1546 * Remove the item to be processed. If we are removing the last 1547 * item on the list, we need to recalculate the tail pointer. 1548 */ 1549 static void 1550 remove_from_worklist(wk) 1551 struct worklist *wk; 1552 { 1553 struct ufsmount *ump; 1554 1555 ump = VFSTOUFS(wk->wk_mp); 1556 if (ump->softdep_worklist_tail == wk) 1557 ump->softdep_worklist_tail = 1558 (struct worklist *)wk->wk_list.le_prev; 1559 WORKLIST_REMOVE(wk); 1560 ump->softdep_on_worklist -= 1; 1561 } 1562 1563 static void 1564 wake_worklist(wk) 1565 struct worklist *wk; 1566 { 1567 if (wk->wk_state & IOWAITING) { 1568 wk->wk_state &= ~IOWAITING; 1569 wakeup(wk); 1570 } 1571 } 1572 1573 static void 1574 wait_worklist(wk, wmesg) 1575 struct worklist *wk; 1576 char *wmesg; 1577 { 1578 struct ufsmount *ump; 1579 1580 ump = VFSTOUFS(wk->wk_mp); 1581 wk->wk_state |= IOWAITING; 1582 msleep(wk, LOCK_PTR(ump), PVM, wmesg, 0); 1583 } 1584 1585 /* 1586 * Process that runs once per second to handle items in the background queue. 1587 * 1588 * Note that we ensure that everything is done in the order in which they 1589 * appear in the queue. The code below depends on this property to ensure 1590 * that blocks of a file are freed before the inode itself is freed. This 1591 * ordering ensures that no new <vfsid, inum, lbn> triples will be generated 1592 * until all the old ones have been purged from the dependency lists. 1593 */ 1594 static int 1595 softdep_process_worklist(mp, full) 1596 struct mount *mp; 1597 int full; 1598 { 1599 int cnt, matchcnt; 1600 struct ufsmount *ump; 1601 long starttime; 1602 1603 KASSERT(mp != NULL, ("softdep_process_worklist: NULL mp")); 1604 if (MOUNTEDSOFTDEP(mp) == 0) 1605 return (0); 1606 matchcnt = 0; 1607 ump = VFSTOUFS(mp); 1608 ACQUIRE_LOCK(ump); 1609 starttime = time_second; 1610 softdep_process_journal(mp, NULL, full ? MNT_WAIT : 0); 1611 check_clear_deps(mp); 1612 while (ump->softdep_on_worklist > 0) { 1613 if ((cnt = process_worklist_item(mp, 10, LK_NOWAIT)) == 0) 1614 break; 1615 else 1616 matchcnt += cnt; 1617 check_clear_deps(mp); 1618 /* 1619 * We do not generally want to stop for buffer space, but if 1620 * we are really being a buffer hog, we will stop and wait. 1621 */ 1622 if (should_yield()) { 1623 FREE_LOCK(ump); 1624 kern_yield(PRI_USER); 1625 bwillwrite(); 1626 ACQUIRE_LOCK(ump); 1627 } 1628 /* 1629 * Never allow processing to run for more than one 1630 * second. This gives the syncer thread the opportunity 1631 * to pause if appropriate. 1632 */ 1633 if (!full && starttime != time_second) 1634 break; 1635 } 1636 if (full == 0) 1637 journal_unsuspend(ump); 1638 FREE_LOCK(ump); 1639 return (matchcnt); 1640 } 1641 1642 /* 1643 * Process all removes associated with a vnode if we are running out of 1644 * journal space. Any other process which attempts to flush these will 1645 * be unable as we have the vnodes locked. 1646 */ 1647 static void 1648 process_removes(vp) 1649 struct vnode *vp; 1650 { 1651 struct inodedep *inodedep; 1652 struct dirrem *dirrem; 1653 struct ufsmount *ump; 1654 struct mount *mp; 1655 ino_t inum; 1656 1657 mp = vp->v_mount; 1658 ump = VFSTOUFS(mp); 1659 LOCK_OWNED(ump); 1660 inum = VTOI(vp)->i_number; 1661 for (;;) { 1662 top: 1663 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0) 1664 return; 1665 LIST_FOREACH(dirrem, &inodedep->id_dirremhd, dm_inonext) { 1666 /* 1667 * If another thread is trying to lock this vnode 1668 * it will fail but we must wait for it to do so 1669 * before we can proceed. 1670 */ 1671 if (dirrem->dm_state & INPROGRESS) { 1672 wait_worklist(&dirrem->dm_list, "pwrwait"); 1673 goto top; 1674 } 1675 if ((dirrem->dm_state & (COMPLETE | ONWORKLIST)) == 1676 (COMPLETE | ONWORKLIST)) 1677 break; 1678 } 1679 if (dirrem == NULL) 1680 return; 1681 remove_from_worklist(&dirrem->dm_list); 1682 FREE_LOCK(ump); 1683 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1684 panic("process_removes: suspended filesystem"); 1685 handle_workitem_remove(dirrem, 0); 1686 vn_finished_secondary_write(mp); 1687 ACQUIRE_LOCK(ump); 1688 } 1689 } 1690 1691 /* 1692 * Process all truncations associated with a vnode if we are running out 1693 * of journal space. This is called when the vnode lock is already held 1694 * and no other process can clear the truncation. This function returns 1695 * a value greater than zero if it did any work. 1696 */ 1697 static void 1698 process_truncates(vp) 1699 struct vnode *vp; 1700 { 1701 struct inodedep *inodedep; 1702 struct freeblks *freeblks; 1703 struct ufsmount *ump; 1704 struct mount *mp; 1705 ino_t inum; 1706 int cgwait; 1707 1708 mp = vp->v_mount; 1709 ump = VFSTOUFS(mp); 1710 LOCK_OWNED(ump); 1711 inum = VTOI(vp)->i_number; 1712 for (;;) { 1713 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0) 1714 return; 1715 cgwait = 0; 1716 TAILQ_FOREACH(freeblks, &inodedep->id_freeblklst, fb_next) { 1717 /* Journal entries not yet written. */ 1718 if (!LIST_EMPTY(&freeblks->fb_jblkdephd)) { 1719 jwait(&LIST_FIRST( 1720 &freeblks->fb_jblkdephd)->jb_list, 1721 MNT_WAIT); 1722 break; 1723 } 1724 /* Another thread is executing this item. */ 1725 if (freeblks->fb_state & INPROGRESS) { 1726 wait_worklist(&freeblks->fb_list, "ptrwait"); 1727 break; 1728 } 1729 /* Freeblks is waiting on a inode write. */ 1730 if ((freeblks->fb_state & COMPLETE) == 0) { 1731 FREE_LOCK(ump); 1732 ffs_update(vp, 1); 1733 ACQUIRE_LOCK(ump); 1734 break; 1735 } 1736 if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST)) == 1737 (ALLCOMPLETE | ONWORKLIST)) { 1738 remove_from_worklist(&freeblks->fb_list); 1739 freeblks->fb_state |= INPROGRESS; 1740 FREE_LOCK(ump); 1741 if (vn_start_secondary_write(NULL, &mp, 1742 V_NOWAIT)) 1743 panic("process_truncates: " 1744 "suspended filesystem"); 1745 handle_workitem_freeblocks(freeblks, 0); 1746 vn_finished_secondary_write(mp); 1747 ACQUIRE_LOCK(ump); 1748 break; 1749 } 1750 if (freeblks->fb_cgwait) 1751 cgwait++; 1752 } 1753 if (cgwait) { 1754 FREE_LOCK(ump); 1755 sync_cgs(mp, MNT_WAIT); 1756 ffs_sync_snap(mp, MNT_WAIT); 1757 ACQUIRE_LOCK(ump); 1758 continue; 1759 } 1760 if (freeblks == NULL) 1761 break; 1762 } 1763 return; 1764 } 1765 1766 /* 1767 * Process one item on the worklist. 1768 */ 1769 static int 1770 process_worklist_item(mp, target, flags) 1771 struct mount *mp; 1772 int target; 1773 int flags; 1774 { 1775 struct worklist sentinel; 1776 struct worklist *wk; 1777 struct ufsmount *ump; 1778 int matchcnt; 1779 int error; 1780 1781 KASSERT(mp != NULL, ("process_worklist_item: NULL mp")); 1782 /* 1783 * If we are being called because of a process doing a 1784 * copy-on-write, then it is not safe to write as we may 1785 * recurse into the copy-on-write routine. 1786 */ 1787 if (curthread->td_pflags & TDP_COWINPROGRESS) 1788 return (-1); 1789 PHOLD(curproc); /* Don't let the stack go away. */ 1790 ump = VFSTOUFS(mp); 1791 LOCK_OWNED(ump); 1792 matchcnt = 0; 1793 sentinel.wk_mp = NULL; 1794 sentinel.wk_type = D_SENTINEL; 1795 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, &sentinel, wk_list); 1796 for (wk = LIST_NEXT(&sentinel, wk_list); wk != NULL; 1797 wk = LIST_NEXT(&sentinel, wk_list)) { 1798 if (wk->wk_type == D_SENTINEL) { 1799 LIST_REMOVE(&sentinel, wk_list); 1800 LIST_INSERT_AFTER(wk, &sentinel, wk_list); 1801 continue; 1802 } 1803 if (wk->wk_state & INPROGRESS) 1804 panic("process_worklist_item: %p already in progress.", 1805 wk); 1806 wk->wk_state |= INPROGRESS; 1807 remove_from_worklist(wk); 1808 FREE_LOCK(ump); 1809 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1810 panic("process_worklist_item: suspended filesystem"); 1811 switch (wk->wk_type) { 1812 case D_DIRREM: 1813 /* removal of a directory entry */ 1814 error = handle_workitem_remove(WK_DIRREM(wk), flags); 1815 break; 1816 1817 case D_FREEBLKS: 1818 /* releasing blocks and/or fragments from a file */ 1819 error = handle_workitem_freeblocks(WK_FREEBLKS(wk), 1820 flags); 1821 break; 1822 1823 case D_FREEFRAG: 1824 /* releasing a fragment when replaced as a file grows */ 1825 handle_workitem_freefrag(WK_FREEFRAG(wk)); 1826 error = 0; 1827 break; 1828 1829 case D_FREEFILE: 1830 /* releasing an inode when its link count drops to 0 */ 1831 handle_workitem_freefile(WK_FREEFILE(wk)); 1832 error = 0; 1833 break; 1834 1835 default: 1836 panic("%s_process_worklist: Unknown type %s", 1837 "softdep", TYPENAME(wk->wk_type)); 1838 /* NOTREACHED */ 1839 } 1840 vn_finished_secondary_write(mp); 1841 ACQUIRE_LOCK(ump); 1842 if (error == 0) { 1843 if (++matchcnt == target) 1844 break; 1845 continue; 1846 } 1847 /* 1848 * We have to retry the worklist item later. Wake up any 1849 * waiters who may be able to complete it immediately and 1850 * add the item back to the head so we don't try to execute 1851 * it again. 1852 */ 1853 wk->wk_state &= ~INPROGRESS; 1854 wake_worklist(wk); 1855 add_to_worklist(wk, WK_HEAD); 1856 } 1857 /* Sentinal could've become the tail from remove_from_worklist. */ 1858 if (ump->softdep_worklist_tail == &sentinel) 1859 ump->softdep_worklist_tail = 1860 (struct worklist *)sentinel.wk_list.le_prev; 1861 LIST_REMOVE(&sentinel, wk_list); 1862 PRELE(curproc); 1863 return (matchcnt); 1864 } 1865 1866 /* 1867 * Move dependencies from one buffer to another. 1868 */ 1869 int 1870 softdep_move_dependencies(oldbp, newbp) 1871 struct buf *oldbp; 1872 struct buf *newbp; 1873 { 1874 struct worklist *wk, *wktail; 1875 struct ufsmount *ump; 1876 int dirty; 1877 1878 if ((wk = LIST_FIRST(&oldbp->b_dep)) == NULL) 1879 return (0); 1880 KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0, 1881 ("softdep_move_dependencies called on non-softdep filesystem")); 1882 dirty = 0; 1883 wktail = NULL; 1884 ump = VFSTOUFS(wk->wk_mp); 1885 ACQUIRE_LOCK(ump); 1886 while ((wk = LIST_FIRST(&oldbp->b_dep)) != NULL) { 1887 LIST_REMOVE(wk, wk_list); 1888 if (wk->wk_type == D_BMSAFEMAP && 1889 bmsafemap_backgroundwrite(WK_BMSAFEMAP(wk), newbp)) 1890 dirty = 1; 1891 if (wktail == NULL) 1892 LIST_INSERT_HEAD(&newbp->b_dep, wk, wk_list); 1893 else 1894 LIST_INSERT_AFTER(wktail, wk, wk_list); 1895 wktail = wk; 1896 } 1897 FREE_LOCK(ump); 1898 1899 return (dirty); 1900 } 1901 1902 /* 1903 * Purge the work list of all items associated with a particular mount point. 1904 */ 1905 int 1906 softdep_flushworklist(oldmnt, countp, td) 1907 struct mount *oldmnt; 1908 int *countp; 1909 struct thread *td; 1910 { 1911 struct vnode *devvp; 1912 struct ufsmount *ump; 1913 int count, error; 1914 1915 /* 1916 * Alternately flush the block device associated with the mount 1917 * point and process any dependencies that the flushing 1918 * creates. We continue until no more worklist dependencies 1919 * are found. 1920 */ 1921 *countp = 0; 1922 error = 0; 1923 ump = VFSTOUFS(oldmnt); 1924 devvp = ump->um_devvp; 1925 while ((count = softdep_process_worklist(oldmnt, 1)) > 0) { 1926 *countp += count; 1927 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY); 1928 error = VOP_FSYNC(devvp, MNT_WAIT, td); 1929 VOP_UNLOCK(devvp, 0); 1930 if (error != 0) 1931 break; 1932 } 1933 return (error); 1934 } 1935 1936 #define SU_WAITIDLE_RETRIES 20 1937 static int 1938 softdep_waitidle(struct mount *mp, int flags __unused) 1939 { 1940 struct ufsmount *ump; 1941 struct vnode *devvp; 1942 struct thread *td; 1943 int error, i; 1944 1945 ump = VFSTOUFS(mp); 1946 devvp = ump->um_devvp; 1947 td = curthread; 1948 error = 0; 1949 ACQUIRE_LOCK(ump); 1950 for (i = 0; i < SU_WAITIDLE_RETRIES && ump->softdep_deps != 0; i++) { 1951 ump->softdep_req = 1; 1952 KASSERT((flags & FORCECLOSE) == 0 || 1953 ump->softdep_on_worklist == 0, 1954 ("softdep_waitidle: work added after flush")); 1955 msleep(&ump->softdep_deps, LOCK_PTR(ump), PVM | PDROP, 1956 "softdeps", 10 * hz); 1957 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY); 1958 error = VOP_FSYNC(devvp, MNT_WAIT, td); 1959 VOP_UNLOCK(devvp, 0); 1960 ACQUIRE_LOCK(ump); 1961 if (error != 0) 1962 break; 1963 } 1964 ump->softdep_req = 0; 1965 if (i == SU_WAITIDLE_RETRIES && error == 0 && ump->softdep_deps != 0) { 1966 error = EBUSY; 1967 printf("softdep_waitidle: Failed to flush worklist for %p\n", 1968 mp); 1969 } 1970 FREE_LOCK(ump); 1971 return (error); 1972 } 1973 1974 /* 1975 * Flush all vnodes and worklist items associated with a specified mount point. 1976 */ 1977 int 1978 softdep_flushfiles(oldmnt, flags, td) 1979 struct mount *oldmnt; 1980 int flags; 1981 struct thread *td; 1982 { 1983 #ifdef QUOTA 1984 struct ufsmount *ump; 1985 int i; 1986 #endif 1987 int error, early, depcount, loopcnt, retry_flush_count, retry; 1988 int morework; 1989 1990 KASSERT(MOUNTEDSOFTDEP(oldmnt) != 0, 1991 ("softdep_flushfiles called on non-softdep filesystem")); 1992 loopcnt = 10; 1993 retry_flush_count = 3; 1994 retry_flush: 1995 error = 0; 1996 1997 /* 1998 * Alternately flush the vnodes associated with the mount 1999 * point and process any dependencies that the flushing 2000 * creates. In theory, this loop can happen at most twice, 2001 * but we give it a few extra just to be sure. 2002 */ 2003 for (; loopcnt > 0; loopcnt--) { 2004 /* 2005 * Do another flush in case any vnodes were brought in 2006 * as part of the cleanup operations. 2007 */ 2008 early = retry_flush_count == 1 || (oldmnt->mnt_kern_flag & 2009 MNTK_UNMOUNT) == 0 ? 0 : EARLYFLUSH; 2010 if ((error = ffs_flushfiles(oldmnt, flags | early, td)) != 0) 2011 break; 2012 if ((error = softdep_flushworklist(oldmnt, &depcount, td)) != 0 || 2013 depcount == 0) 2014 break; 2015 } 2016 /* 2017 * If we are unmounting then it is an error to fail. If we 2018 * are simply trying to downgrade to read-only, then filesystem 2019 * activity can keep us busy forever, so we just fail with EBUSY. 2020 */ 2021 if (loopcnt == 0) { 2022 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) 2023 panic("softdep_flushfiles: looping"); 2024 error = EBUSY; 2025 } 2026 if (!error) 2027 error = softdep_waitidle(oldmnt, flags); 2028 if (!error) { 2029 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) { 2030 retry = 0; 2031 MNT_ILOCK(oldmnt); 2032 morework = oldmnt->mnt_nvnodelistsize > 0; 2033 #ifdef QUOTA 2034 ump = VFSTOUFS(oldmnt); 2035 UFS_LOCK(ump); 2036 for (i = 0; i < MAXQUOTAS; i++) { 2037 if (ump->um_quotas[i] != NULLVP) 2038 morework = 1; 2039 } 2040 UFS_UNLOCK(ump); 2041 #endif 2042 if (morework) { 2043 if (--retry_flush_count > 0) { 2044 retry = 1; 2045 loopcnt = 3; 2046 } else 2047 error = EBUSY; 2048 } 2049 MNT_IUNLOCK(oldmnt); 2050 if (retry) 2051 goto retry_flush; 2052 } 2053 } 2054 return (error); 2055 } 2056 2057 /* 2058 * Structure hashing. 2059 * 2060 * There are four types of structures that can be looked up: 2061 * 1) pagedep structures identified by mount point, inode number, 2062 * and logical block. 2063 * 2) inodedep structures identified by mount point and inode number. 2064 * 3) newblk structures identified by mount point and 2065 * physical block number. 2066 * 4) bmsafemap structures identified by mount point and 2067 * cylinder group number. 2068 * 2069 * The "pagedep" and "inodedep" dependency structures are hashed 2070 * separately from the file blocks and inodes to which they correspond. 2071 * This separation helps when the in-memory copy of an inode or 2072 * file block must be replaced. It also obviates the need to access 2073 * an inode or file page when simply updating (or de-allocating) 2074 * dependency structures. Lookup of newblk structures is needed to 2075 * find newly allocated blocks when trying to associate them with 2076 * their allocdirect or allocindir structure. 2077 * 2078 * The lookup routines optionally create and hash a new instance when 2079 * an existing entry is not found. The bmsafemap lookup routine always 2080 * allocates a new structure if an existing one is not found. 2081 */ 2082 #define DEPALLOC 0x0001 /* allocate structure if lookup fails */ 2083 2084 /* 2085 * Structures and routines associated with pagedep caching. 2086 */ 2087 #define PAGEDEP_HASH(ump, inum, lbn) \ 2088 (&(ump)->pagedep_hashtbl[((inum) + (lbn)) & (ump)->pagedep_hash_size]) 2089 2090 static int 2091 pagedep_find(pagedephd, ino, lbn, pagedeppp) 2092 struct pagedep_hashhead *pagedephd; 2093 ino_t ino; 2094 ufs_lbn_t lbn; 2095 struct pagedep **pagedeppp; 2096 { 2097 struct pagedep *pagedep; 2098 2099 LIST_FOREACH(pagedep, pagedephd, pd_hash) { 2100 if (ino == pagedep->pd_ino && lbn == pagedep->pd_lbn) { 2101 *pagedeppp = pagedep; 2102 return (1); 2103 } 2104 } 2105 *pagedeppp = NULL; 2106 return (0); 2107 } 2108 /* 2109 * Look up a pagedep. Return 1 if found, 0 otherwise. 2110 * If not found, allocate if DEPALLOC flag is passed. 2111 * Found or allocated entry is returned in pagedeppp. 2112 */ 2113 static int 2114 pagedep_lookup(mp, bp, ino, lbn, flags, pagedeppp) 2115 struct mount *mp; 2116 struct buf *bp; 2117 ino_t ino; 2118 ufs_lbn_t lbn; 2119 int flags; 2120 struct pagedep **pagedeppp; 2121 { 2122 struct pagedep *pagedep; 2123 struct pagedep_hashhead *pagedephd; 2124 struct worklist *wk; 2125 struct ufsmount *ump; 2126 int ret; 2127 int i; 2128 2129 ump = VFSTOUFS(mp); 2130 LOCK_OWNED(ump); 2131 if (bp) { 2132 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 2133 if (wk->wk_type == D_PAGEDEP) { 2134 *pagedeppp = WK_PAGEDEP(wk); 2135 return (1); 2136 } 2137 } 2138 } 2139 pagedephd = PAGEDEP_HASH(ump, ino, lbn); 2140 ret = pagedep_find(pagedephd, ino, lbn, pagedeppp); 2141 if (ret) { 2142 if (((*pagedeppp)->pd_state & ONWORKLIST) == 0 && bp) 2143 WORKLIST_INSERT(&bp->b_dep, &(*pagedeppp)->pd_list); 2144 return (1); 2145 } 2146 if ((flags & DEPALLOC) == 0) 2147 return (0); 2148 FREE_LOCK(ump); 2149 pagedep = malloc(sizeof(struct pagedep), 2150 M_PAGEDEP, M_SOFTDEP_FLAGS|M_ZERO); 2151 workitem_alloc(&pagedep->pd_list, D_PAGEDEP, mp); 2152 ACQUIRE_LOCK(ump); 2153 ret = pagedep_find(pagedephd, ino, lbn, pagedeppp); 2154 if (*pagedeppp) { 2155 /* 2156 * This should never happen since we only create pagedeps 2157 * with the vnode lock held. Could be an assert. 2158 */ 2159 WORKITEM_FREE(pagedep, D_PAGEDEP); 2160 return (ret); 2161 } 2162 pagedep->pd_ino = ino; 2163 pagedep->pd_lbn = lbn; 2164 LIST_INIT(&pagedep->pd_dirremhd); 2165 LIST_INIT(&pagedep->pd_pendinghd); 2166 for (i = 0; i < DAHASHSZ; i++) 2167 LIST_INIT(&pagedep->pd_diraddhd[i]); 2168 LIST_INSERT_HEAD(pagedephd, pagedep, pd_hash); 2169 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 2170 *pagedeppp = pagedep; 2171 return (0); 2172 } 2173 2174 /* 2175 * Structures and routines associated with inodedep caching. 2176 */ 2177 #define INODEDEP_HASH(ump, inum) \ 2178 (&(ump)->inodedep_hashtbl[(inum) & (ump)->inodedep_hash_size]) 2179 2180 static int 2181 inodedep_find(inodedephd, inum, inodedeppp) 2182 struct inodedep_hashhead *inodedephd; 2183 ino_t inum; 2184 struct inodedep **inodedeppp; 2185 { 2186 struct inodedep *inodedep; 2187 2188 LIST_FOREACH(inodedep, inodedephd, id_hash) 2189 if (inum == inodedep->id_ino) 2190 break; 2191 if (inodedep) { 2192 *inodedeppp = inodedep; 2193 return (1); 2194 } 2195 *inodedeppp = NULL; 2196 2197 return (0); 2198 } 2199 /* 2200 * Look up an inodedep. Return 1 if found, 0 if not found. 2201 * If not found, allocate if DEPALLOC flag is passed. 2202 * Found or allocated entry is returned in inodedeppp. 2203 */ 2204 static int 2205 inodedep_lookup(mp, inum, flags, inodedeppp) 2206 struct mount *mp; 2207 ino_t inum; 2208 int flags; 2209 struct inodedep **inodedeppp; 2210 { 2211 struct inodedep *inodedep; 2212 struct inodedep_hashhead *inodedephd; 2213 struct ufsmount *ump; 2214 struct fs *fs; 2215 2216 ump = VFSTOUFS(mp); 2217 LOCK_OWNED(ump); 2218 fs = ump->um_fs; 2219 inodedephd = INODEDEP_HASH(ump, inum); 2220 2221 if (inodedep_find(inodedephd, inum, inodedeppp)) 2222 return (1); 2223 if ((flags & DEPALLOC) == 0) 2224 return (0); 2225 /* 2226 * If the system is over its limit and our filesystem is 2227 * responsible for more than our share of that usage and 2228 * we are not in a rush, request some inodedep cleanup. 2229 */ 2230 if (softdep_excess_items(ump, D_INODEDEP)) 2231 schedule_cleanup(mp); 2232 else 2233 FREE_LOCK(ump); 2234 inodedep = malloc(sizeof(struct inodedep), 2235 M_INODEDEP, M_SOFTDEP_FLAGS); 2236 workitem_alloc(&inodedep->id_list, D_INODEDEP, mp); 2237 ACQUIRE_LOCK(ump); 2238 if (inodedep_find(inodedephd, inum, inodedeppp)) { 2239 WORKITEM_FREE(inodedep, D_INODEDEP); 2240 return (1); 2241 } 2242 inodedep->id_fs = fs; 2243 inodedep->id_ino = inum; 2244 inodedep->id_state = ALLCOMPLETE; 2245 inodedep->id_nlinkdelta = 0; 2246 inodedep->id_savedino1 = NULL; 2247 inodedep->id_savedsize = -1; 2248 inodedep->id_savedextsize = -1; 2249 inodedep->id_savednlink = -1; 2250 inodedep->id_bmsafemap = NULL; 2251 inodedep->id_mkdiradd = NULL; 2252 LIST_INIT(&inodedep->id_dirremhd); 2253 LIST_INIT(&inodedep->id_pendinghd); 2254 LIST_INIT(&inodedep->id_inowait); 2255 LIST_INIT(&inodedep->id_bufwait); 2256 TAILQ_INIT(&inodedep->id_inoreflst); 2257 TAILQ_INIT(&inodedep->id_inoupdt); 2258 TAILQ_INIT(&inodedep->id_newinoupdt); 2259 TAILQ_INIT(&inodedep->id_extupdt); 2260 TAILQ_INIT(&inodedep->id_newextupdt); 2261 TAILQ_INIT(&inodedep->id_freeblklst); 2262 LIST_INSERT_HEAD(inodedephd, inodedep, id_hash); 2263 *inodedeppp = inodedep; 2264 return (0); 2265 } 2266 2267 /* 2268 * Structures and routines associated with newblk caching. 2269 */ 2270 #define NEWBLK_HASH(ump, inum) \ 2271 (&(ump)->newblk_hashtbl[(inum) & (ump)->newblk_hash_size]) 2272 2273 static int 2274 newblk_find(newblkhd, newblkno, flags, newblkpp) 2275 struct newblk_hashhead *newblkhd; 2276 ufs2_daddr_t newblkno; 2277 int flags; 2278 struct newblk **newblkpp; 2279 { 2280 struct newblk *newblk; 2281 2282 LIST_FOREACH(newblk, newblkhd, nb_hash) { 2283 if (newblkno != newblk->nb_newblkno) 2284 continue; 2285 /* 2286 * If we're creating a new dependency don't match those that 2287 * have already been converted to allocdirects. This is for 2288 * a frag extend. 2289 */ 2290 if ((flags & DEPALLOC) && newblk->nb_list.wk_type != D_NEWBLK) 2291 continue; 2292 break; 2293 } 2294 if (newblk) { 2295 *newblkpp = newblk; 2296 return (1); 2297 } 2298 *newblkpp = NULL; 2299 return (0); 2300 } 2301 2302 /* 2303 * Look up a newblk. Return 1 if found, 0 if not found. 2304 * If not found, allocate if DEPALLOC flag is passed. 2305 * Found or allocated entry is returned in newblkpp. 2306 */ 2307 static int 2308 newblk_lookup(mp, newblkno, flags, newblkpp) 2309 struct mount *mp; 2310 ufs2_daddr_t newblkno; 2311 int flags; 2312 struct newblk **newblkpp; 2313 { 2314 struct newblk *newblk; 2315 struct newblk_hashhead *newblkhd; 2316 struct ufsmount *ump; 2317 2318 ump = VFSTOUFS(mp); 2319 LOCK_OWNED(ump); 2320 newblkhd = NEWBLK_HASH(ump, newblkno); 2321 if (newblk_find(newblkhd, newblkno, flags, newblkpp)) 2322 return (1); 2323 if ((flags & DEPALLOC) == 0) 2324 return (0); 2325 if (softdep_excess_items(ump, D_NEWBLK) || 2326 softdep_excess_items(ump, D_ALLOCDIRECT) || 2327 softdep_excess_items(ump, D_ALLOCINDIR)) 2328 schedule_cleanup(mp); 2329 else 2330 FREE_LOCK(ump); 2331 newblk = malloc(sizeof(union allblk), M_NEWBLK, 2332 M_SOFTDEP_FLAGS | M_ZERO); 2333 workitem_alloc(&newblk->nb_list, D_NEWBLK, mp); 2334 ACQUIRE_LOCK(ump); 2335 if (newblk_find(newblkhd, newblkno, flags, newblkpp)) { 2336 WORKITEM_FREE(newblk, D_NEWBLK); 2337 return (1); 2338 } 2339 newblk->nb_freefrag = NULL; 2340 LIST_INIT(&newblk->nb_indirdeps); 2341 LIST_INIT(&newblk->nb_newdirblk); 2342 LIST_INIT(&newblk->nb_jwork); 2343 newblk->nb_state = ATTACHED; 2344 newblk->nb_newblkno = newblkno; 2345 LIST_INSERT_HEAD(newblkhd, newblk, nb_hash); 2346 *newblkpp = newblk; 2347 return (0); 2348 } 2349 2350 /* 2351 * Structures and routines associated with freed indirect block caching. 2352 */ 2353 #define INDIR_HASH(ump, blkno) \ 2354 (&(ump)->indir_hashtbl[(blkno) & (ump)->indir_hash_size]) 2355 2356 /* 2357 * Lookup an indirect block in the indir hash table. The freework is 2358 * removed and potentially freed. The caller must do a blocking journal 2359 * write before writing to the blkno. 2360 */ 2361 static int 2362 indirblk_lookup(mp, blkno) 2363 struct mount *mp; 2364 ufs2_daddr_t blkno; 2365 { 2366 struct freework *freework; 2367 struct indir_hashhead *wkhd; 2368 struct ufsmount *ump; 2369 2370 ump = VFSTOUFS(mp); 2371 wkhd = INDIR_HASH(ump, blkno); 2372 TAILQ_FOREACH(freework, wkhd, fw_next) { 2373 if (freework->fw_blkno != blkno) 2374 continue; 2375 indirblk_remove(freework); 2376 return (1); 2377 } 2378 return (0); 2379 } 2380 2381 /* 2382 * Insert an indirect block represented by freework into the indirblk 2383 * hash table so that it may prevent the block from being re-used prior 2384 * to the journal being written. 2385 */ 2386 static void 2387 indirblk_insert(freework) 2388 struct freework *freework; 2389 { 2390 struct jblocks *jblocks; 2391 struct jseg *jseg; 2392 struct ufsmount *ump; 2393 2394 ump = VFSTOUFS(freework->fw_list.wk_mp); 2395 jblocks = ump->softdep_jblocks; 2396 jseg = TAILQ_LAST(&jblocks->jb_segs, jseglst); 2397 if (jseg == NULL) 2398 return; 2399 2400 LIST_INSERT_HEAD(&jseg->js_indirs, freework, fw_segs); 2401 TAILQ_INSERT_HEAD(INDIR_HASH(ump, freework->fw_blkno), freework, 2402 fw_next); 2403 freework->fw_state &= ~DEPCOMPLETE; 2404 } 2405 2406 static void 2407 indirblk_remove(freework) 2408 struct freework *freework; 2409 { 2410 struct ufsmount *ump; 2411 2412 ump = VFSTOUFS(freework->fw_list.wk_mp); 2413 LIST_REMOVE(freework, fw_segs); 2414 TAILQ_REMOVE(INDIR_HASH(ump, freework->fw_blkno), freework, fw_next); 2415 freework->fw_state |= DEPCOMPLETE; 2416 if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE) 2417 WORKITEM_FREE(freework, D_FREEWORK); 2418 } 2419 2420 /* 2421 * Executed during filesystem system initialization before 2422 * mounting any filesystems. 2423 */ 2424 void 2425 softdep_initialize() 2426 { 2427 2428 TAILQ_INIT(&softdepmounts); 2429 #ifdef __LP64__ 2430 max_softdeps = desiredvnodes * 4; 2431 #else 2432 max_softdeps = desiredvnodes * 2; 2433 #endif 2434 2435 /* initialise bioops hack */ 2436 bioops.io_start = softdep_disk_io_initiation; 2437 bioops.io_complete = softdep_disk_write_complete; 2438 bioops.io_deallocate = softdep_deallocate_dependencies; 2439 bioops.io_countdeps = softdep_count_dependencies; 2440 softdep_ast_cleanup = softdep_ast_cleanup_proc; 2441 2442 /* Initialize the callout with an mtx. */ 2443 callout_init_mtx(&softdep_callout, &lk, 0); 2444 } 2445 2446 /* 2447 * Executed after all filesystems have been unmounted during 2448 * filesystem module unload. 2449 */ 2450 void 2451 softdep_uninitialize() 2452 { 2453 2454 /* clear bioops hack */ 2455 bioops.io_start = NULL; 2456 bioops.io_complete = NULL; 2457 bioops.io_deallocate = NULL; 2458 bioops.io_countdeps = NULL; 2459 softdep_ast_cleanup = NULL; 2460 2461 callout_drain(&softdep_callout); 2462 } 2463 2464 /* 2465 * Called at mount time to notify the dependency code that a 2466 * filesystem wishes to use it. 2467 */ 2468 int 2469 softdep_mount(devvp, mp, fs, cred) 2470 struct vnode *devvp; 2471 struct mount *mp; 2472 struct fs *fs; 2473 struct ucred *cred; 2474 { 2475 struct csum_total cstotal; 2476 struct mount_softdeps *sdp; 2477 struct ufsmount *ump; 2478 struct cg *cgp; 2479 struct buf *bp; 2480 u_int cyl, i; 2481 int error; 2482 2483 sdp = malloc(sizeof(struct mount_softdeps), M_MOUNTDATA, 2484 M_WAITOK | M_ZERO); 2485 MNT_ILOCK(mp); 2486 mp->mnt_flag = (mp->mnt_flag & ~MNT_ASYNC) | MNT_SOFTDEP; 2487 if ((mp->mnt_kern_flag & MNTK_SOFTDEP) == 0) { 2488 mp->mnt_kern_flag = (mp->mnt_kern_flag & ~MNTK_ASYNC) | 2489 MNTK_SOFTDEP | MNTK_NOASYNC; 2490 } 2491 ump = VFSTOUFS(mp); 2492 ump->um_softdep = sdp; 2493 MNT_IUNLOCK(mp); 2494 rw_init(LOCK_PTR(ump), "per-fs softdep"); 2495 sdp->sd_ump = ump; 2496 LIST_INIT(&ump->softdep_workitem_pending); 2497 LIST_INIT(&ump->softdep_journal_pending); 2498 TAILQ_INIT(&ump->softdep_unlinked); 2499 LIST_INIT(&ump->softdep_dirtycg); 2500 ump->softdep_worklist_tail = NULL; 2501 ump->softdep_on_worklist = 0; 2502 ump->softdep_deps = 0; 2503 LIST_INIT(&ump->softdep_mkdirlisthd); 2504 ump->pagedep_hashtbl = hashinit(desiredvnodes / 5, M_PAGEDEP, 2505 &ump->pagedep_hash_size); 2506 ump->pagedep_nextclean = 0; 2507 ump->inodedep_hashtbl = hashinit(desiredvnodes, M_INODEDEP, 2508 &ump->inodedep_hash_size); 2509 ump->inodedep_nextclean = 0; 2510 ump->newblk_hashtbl = hashinit(max_softdeps / 2, M_NEWBLK, 2511 &ump->newblk_hash_size); 2512 ump->bmsafemap_hashtbl = hashinit(1024, M_BMSAFEMAP, 2513 &ump->bmsafemap_hash_size); 2514 i = 1 << (ffs(desiredvnodes / 10) - 1); 2515 ump->indir_hashtbl = malloc(i * sizeof(struct indir_hashhead), 2516 M_FREEWORK, M_WAITOK); 2517 ump->indir_hash_size = i - 1; 2518 for (i = 0; i <= ump->indir_hash_size; i++) 2519 TAILQ_INIT(&ump->indir_hashtbl[i]); 2520 ACQUIRE_GBLLOCK(&lk); 2521 TAILQ_INSERT_TAIL(&softdepmounts, sdp, sd_next); 2522 FREE_GBLLOCK(&lk); 2523 if ((fs->fs_flags & FS_SUJ) && 2524 (error = journal_mount(mp, fs, cred)) != 0) { 2525 printf("Failed to start journal: %d\n", error); 2526 softdep_unmount(mp); 2527 return (error); 2528 } 2529 /* 2530 * Start our flushing thread in the bufdaemon process. 2531 */ 2532 ACQUIRE_LOCK(ump); 2533 ump->softdep_flags |= FLUSH_STARTING; 2534 FREE_LOCK(ump); 2535 kproc_kthread_add(&softdep_flush, mp, &bufdaemonproc, 2536 &ump->softdep_flushtd, 0, 0, "softdepflush", "%s worker", 2537 mp->mnt_stat.f_mntonname); 2538 ACQUIRE_LOCK(ump); 2539 while ((ump->softdep_flags & FLUSH_STARTING) != 0) { 2540 msleep(&ump->softdep_flushtd, LOCK_PTR(ump), PVM, "sdstart", 2541 hz / 2); 2542 } 2543 FREE_LOCK(ump); 2544 /* 2545 * When doing soft updates, the counters in the 2546 * superblock may have gotten out of sync. Recomputation 2547 * can take a long time and can be deferred for background 2548 * fsck. However, the old behavior of scanning the cylinder 2549 * groups and recalculating them at mount time is available 2550 * by setting vfs.ffs.compute_summary_at_mount to one. 2551 */ 2552 if (compute_summary_at_mount == 0 || fs->fs_clean != 0) 2553 return (0); 2554 bzero(&cstotal, sizeof cstotal); 2555 for (cyl = 0; cyl < fs->fs_ncg; cyl++) { 2556 if ((error = bread(devvp, fsbtodb(fs, cgtod(fs, cyl)), 2557 fs->fs_cgsize, cred, &bp)) != 0) { 2558 brelse(bp); 2559 softdep_unmount(mp); 2560 return (error); 2561 } 2562 cgp = (struct cg *)bp->b_data; 2563 cstotal.cs_nffree += cgp->cg_cs.cs_nffree; 2564 cstotal.cs_nbfree += cgp->cg_cs.cs_nbfree; 2565 cstotal.cs_nifree += cgp->cg_cs.cs_nifree; 2566 cstotal.cs_ndir += cgp->cg_cs.cs_ndir; 2567 fs->fs_cs(fs, cyl) = cgp->cg_cs; 2568 brelse(bp); 2569 } 2570 #ifdef INVARIANTS 2571 if (bcmp(&cstotal, &fs->fs_cstotal, sizeof cstotal)) 2572 printf("%s: superblock summary recomputed\n", fs->fs_fsmnt); 2573 #endif 2574 bcopy(&cstotal, &fs->fs_cstotal, sizeof cstotal); 2575 return (0); 2576 } 2577 2578 void 2579 softdep_unmount(mp) 2580 struct mount *mp; 2581 { 2582 struct ufsmount *ump; 2583 #ifdef INVARIANTS 2584 int i; 2585 #endif 2586 2587 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 2588 ("softdep_unmount called on non-softdep filesystem")); 2589 ump = VFSTOUFS(mp); 2590 MNT_ILOCK(mp); 2591 mp->mnt_flag &= ~MNT_SOFTDEP; 2592 if (MOUNTEDSUJ(mp) == 0) { 2593 MNT_IUNLOCK(mp); 2594 } else { 2595 mp->mnt_flag &= ~MNT_SUJ; 2596 MNT_IUNLOCK(mp); 2597 journal_unmount(ump); 2598 } 2599 /* 2600 * Shut down our flushing thread. Check for NULL is if 2601 * softdep_mount errors out before the thread has been created. 2602 */ 2603 if (ump->softdep_flushtd != NULL) { 2604 ACQUIRE_LOCK(ump); 2605 ump->softdep_flags |= FLUSH_EXIT; 2606 wakeup(&ump->softdep_flushtd); 2607 msleep(&ump->softdep_flags, LOCK_PTR(ump), PVM | PDROP, 2608 "sdwait", 0); 2609 KASSERT((ump->softdep_flags & FLUSH_EXIT) == 0, 2610 ("Thread shutdown failed")); 2611 } 2612 /* 2613 * Free up our resources. 2614 */ 2615 ACQUIRE_GBLLOCK(&lk); 2616 TAILQ_REMOVE(&softdepmounts, ump->um_softdep, sd_next); 2617 FREE_GBLLOCK(&lk); 2618 rw_destroy(LOCK_PTR(ump)); 2619 hashdestroy(ump->pagedep_hashtbl, M_PAGEDEP, ump->pagedep_hash_size); 2620 hashdestroy(ump->inodedep_hashtbl, M_INODEDEP, ump->inodedep_hash_size); 2621 hashdestroy(ump->newblk_hashtbl, M_NEWBLK, ump->newblk_hash_size); 2622 hashdestroy(ump->bmsafemap_hashtbl, M_BMSAFEMAP, 2623 ump->bmsafemap_hash_size); 2624 free(ump->indir_hashtbl, M_FREEWORK); 2625 #ifdef INVARIANTS 2626 for (i = 0; i <= D_LAST; i++) 2627 KASSERT(ump->softdep_curdeps[i] == 0, 2628 ("Unmount %s: Dep type %s != 0 (%ld)", ump->um_fs->fs_fsmnt, 2629 TYPENAME(i), ump->softdep_curdeps[i])); 2630 #endif 2631 free(ump->um_softdep, M_MOUNTDATA); 2632 } 2633 2634 static struct jblocks * 2635 jblocks_create(void) 2636 { 2637 struct jblocks *jblocks; 2638 2639 jblocks = malloc(sizeof(*jblocks), M_JBLOCKS, M_WAITOK | M_ZERO); 2640 TAILQ_INIT(&jblocks->jb_segs); 2641 jblocks->jb_avail = 10; 2642 jblocks->jb_extent = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2643 M_JBLOCKS, M_WAITOK | M_ZERO); 2644 2645 return (jblocks); 2646 } 2647 2648 static ufs2_daddr_t 2649 jblocks_alloc(jblocks, bytes, actual) 2650 struct jblocks *jblocks; 2651 int bytes; 2652 int *actual; 2653 { 2654 ufs2_daddr_t daddr; 2655 struct jextent *jext; 2656 int freecnt; 2657 int blocks; 2658 2659 blocks = bytes / DEV_BSIZE; 2660 jext = &jblocks->jb_extent[jblocks->jb_head]; 2661 freecnt = jext->je_blocks - jblocks->jb_off; 2662 if (freecnt == 0) { 2663 jblocks->jb_off = 0; 2664 if (++jblocks->jb_head > jblocks->jb_used) 2665 jblocks->jb_head = 0; 2666 jext = &jblocks->jb_extent[jblocks->jb_head]; 2667 freecnt = jext->je_blocks; 2668 } 2669 if (freecnt > blocks) 2670 freecnt = blocks; 2671 *actual = freecnt * DEV_BSIZE; 2672 daddr = jext->je_daddr + jblocks->jb_off; 2673 jblocks->jb_off += freecnt; 2674 jblocks->jb_free -= freecnt; 2675 2676 return (daddr); 2677 } 2678 2679 static void 2680 jblocks_free(jblocks, mp, bytes) 2681 struct jblocks *jblocks; 2682 struct mount *mp; 2683 int bytes; 2684 { 2685 2686 LOCK_OWNED(VFSTOUFS(mp)); 2687 jblocks->jb_free += bytes / DEV_BSIZE; 2688 if (jblocks->jb_suspended) 2689 worklist_speedup(mp); 2690 wakeup(jblocks); 2691 } 2692 2693 static void 2694 jblocks_destroy(jblocks) 2695 struct jblocks *jblocks; 2696 { 2697 2698 if (jblocks->jb_extent) 2699 free(jblocks->jb_extent, M_JBLOCKS); 2700 free(jblocks, M_JBLOCKS); 2701 } 2702 2703 static void 2704 jblocks_add(jblocks, daddr, blocks) 2705 struct jblocks *jblocks; 2706 ufs2_daddr_t daddr; 2707 int blocks; 2708 { 2709 struct jextent *jext; 2710 2711 jblocks->jb_blocks += blocks; 2712 jblocks->jb_free += blocks; 2713 jext = &jblocks->jb_extent[jblocks->jb_used]; 2714 /* Adding the first block. */ 2715 if (jext->je_daddr == 0) { 2716 jext->je_daddr = daddr; 2717 jext->je_blocks = blocks; 2718 return; 2719 } 2720 /* Extending the last extent. */ 2721 if (jext->je_daddr + jext->je_blocks == daddr) { 2722 jext->je_blocks += blocks; 2723 return; 2724 } 2725 /* Adding a new extent. */ 2726 if (++jblocks->jb_used == jblocks->jb_avail) { 2727 jblocks->jb_avail *= 2; 2728 jext = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2729 M_JBLOCKS, M_WAITOK | M_ZERO); 2730 memcpy(jext, jblocks->jb_extent, 2731 sizeof(struct jextent) * jblocks->jb_used); 2732 free(jblocks->jb_extent, M_JBLOCKS); 2733 jblocks->jb_extent = jext; 2734 } 2735 jext = &jblocks->jb_extent[jblocks->jb_used]; 2736 jext->je_daddr = daddr; 2737 jext->je_blocks = blocks; 2738 return; 2739 } 2740 2741 int 2742 softdep_journal_lookup(mp, vpp) 2743 struct mount *mp; 2744 struct vnode **vpp; 2745 { 2746 struct componentname cnp; 2747 struct vnode *dvp; 2748 ino_t sujournal; 2749 int error; 2750 2751 error = VFS_VGET(mp, UFS_ROOTINO, LK_EXCLUSIVE, &dvp); 2752 if (error) 2753 return (error); 2754 bzero(&cnp, sizeof(cnp)); 2755 cnp.cn_nameiop = LOOKUP; 2756 cnp.cn_flags = ISLASTCN; 2757 cnp.cn_thread = curthread; 2758 cnp.cn_cred = curthread->td_ucred; 2759 cnp.cn_pnbuf = SUJ_FILE; 2760 cnp.cn_nameptr = SUJ_FILE; 2761 cnp.cn_namelen = strlen(SUJ_FILE); 2762 error = ufs_lookup_ino(dvp, NULL, &cnp, &sujournal); 2763 vput(dvp); 2764 if (error != 0) 2765 return (error); 2766 error = VFS_VGET(mp, sujournal, LK_EXCLUSIVE, vpp); 2767 return (error); 2768 } 2769 2770 /* 2771 * Open and verify the journal file. 2772 */ 2773 static int 2774 journal_mount(mp, fs, cred) 2775 struct mount *mp; 2776 struct fs *fs; 2777 struct ucred *cred; 2778 { 2779 struct jblocks *jblocks; 2780 struct ufsmount *ump; 2781 struct vnode *vp; 2782 struct inode *ip; 2783 ufs2_daddr_t blkno; 2784 int bcount; 2785 int error; 2786 int i; 2787 2788 ump = VFSTOUFS(mp); 2789 ump->softdep_journal_tail = NULL; 2790 ump->softdep_on_journal = 0; 2791 ump->softdep_accdeps = 0; 2792 ump->softdep_req = 0; 2793 ump->softdep_jblocks = NULL; 2794 error = softdep_journal_lookup(mp, &vp); 2795 if (error != 0) { 2796 printf("Failed to find journal. Use tunefs to create one\n"); 2797 return (error); 2798 } 2799 ip = VTOI(vp); 2800 if (ip->i_size < SUJ_MIN) { 2801 error = ENOSPC; 2802 goto out; 2803 } 2804 bcount = lblkno(fs, ip->i_size); /* Only use whole blocks. */ 2805 jblocks = jblocks_create(); 2806 for (i = 0; i < bcount; i++) { 2807 error = ufs_bmaparray(vp, i, &blkno, NULL, NULL, NULL); 2808 if (error) 2809 break; 2810 jblocks_add(jblocks, blkno, fsbtodb(fs, fs->fs_frag)); 2811 } 2812 if (error) { 2813 jblocks_destroy(jblocks); 2814 goto out; 2815 } 2816 jblocks->jb_low = jblocks->jb_free / 3; /* Reserve 33%. */ 2817 jblocks->jb_min = jblocks->jb_free / 10; /* Suspend at 10%. */ 2818 ump->softdep_jblocks = jblocks; 2819 out: 2820 if (error == 0) { 2821 MNT_ILOCK(mp); 2822 mp->mnt_flag |= MNT_SUJ; 2823 mp->mnt_flag &= ~MNT_SOFTDEP; 2824 MNT_IUNLOCK(mp); 2825 /* 2826 * Only validate the journal contents if the 2827 * filesystem is clean, otherwise we write the logs 2828 * but they'll never be used. If the filesystem was 2829 * still dirty when we mounted it the journal is 2830 * invalid and a new journal can only be valid if it 2831 * starts from a clean mount. 2832 */ 2833 if (fs->fs_clean) { 2834 DIP_SET(ip, i_modrev, fs->fs_mtime); 2835 ip->i_flags |= IN_MODIFIED; 2836 ffs_update(vp, 1); 2837 } 2838 } 2839 vput(vp); 2840 return (error); 2841 } 2842 2843 static void 2844 journal_unmount(ump) 2845 struct ufsmount *ump; 2846 { 2847 2848 if (ump->softdep_jblocks) 2849 jblocks_destroy(ump->softdep_jblocks); 2850 ump->softdep_jblocks = NULL; 2851 } 2852 2853 /* 2854 * Called when a journal record is ready to be written. Space is allocated 2855 * and the journal entry is created when the journal is flushed to stable 2856 * store. 2857 */ 2858 static void 2859 add_to_journal(wk) 2860 struct worklist *wk; 2861 { 2862 struct ufsmount *ump; 2863 2864 ump = VFSTOUFS(wk->wk_mp); 2865 LOCK_OWNED(ump); 2866 if (wk->wk_state & ONWORKLIST) 2867 panic("add_to_journal: %s(0x%X) already on list", 2868 TYPENAME(wk->wk_type), wk->wk_state); 2869 wk->wk_state |= ONWORKLIST | DEPCOMPLETE; 2870 if (LIST_EMPTY(&ump->softdep_journal_pending)) { 2871 ump->softdep_jblocks->jb_age = ticks; 2872 LIST_INSERT_HEAD(&ump->softdep_journal_pending, wk, wk_list); 2873 } else 2874 LIST_INSERT_AFTER(ump->softdep_journal_tail, wk, wk_list); 2875 ump->softdep_journal_tail = wk; 2876 ump->softdep_on_journal += 1; 2877 } 2878 2879 /* 2880 * Remove an arbitrary item for the journal worklist maintain the tail 2881 * pointer. This happens when a new operation obviates the need to 2882 * journal an old operation. 2883 */ 2884 static void 2885 remove_from_journal(wk) 2886 struct worklist *wk; 2887 { 2888 struct ufsmount *ump; 2889 2890 ump = VFSTOUFS(wk->wk_mp); 2891 LOCK_OWNED(ump); 2892 #ifdef INVARIANTS 2893 { 2894 struct worklist *wkn; 2895 2896 LIST_FOREACH(wkn, &ump->softdep_journal_pending, wk_list) 2897 if (wkn == wk) 2898 break; 2899 if (wkn == NULL) 2900 panic("remove_from_journal: %p is not in journal", wk); 2901 } 2902 #endif 2903 /* 2904 * We emulate a TAILQ to save space in most structures which do not 2905 * require TAILQ semantics. Here we must update the tail position 2906 * when removing the tail which is not the final entry. This works 2907 * only if the worklist linkage are at the beginning of the structure. 2908 */ 2909 if (ump->softdep_journal_tail == wk) 2910 ump->softdep_journal_tail = 2911 (struct worklist *)wk->wk_list.le_prev; 2912 WORKLIST_REMOVE(wk); 2913 ump->softdep_on_journal -= 1; 2914 } 2915 2916 /* 2917 * Check for journal space as well as dependency limits so the prelink 2918 * code can throttle both journaled and non-journaled filesystems. 2919 * Threshold is 0 for low and 1 for min. 2920 */ 2921 static int 2922 journal_space(ump, thresh) 2923 struct ufsmount *ump; 2924 int thresh; 2925 { 2926 struct jblocks *jblocks; 2927 int limit, avail; 2928 2929 jblocks = ump->softdep_jblocks; 2930 if (jblocks == NULL) 2931 return (1); 2932 /* 2933 * We use a tighter restriction here to prevent request_cleanup() 2934 * running in threads from running into locks we currently hold. 2935 * We have to be over the limit and our filesystem has to be 2936 * responsible for more than our share of that usage. 2937 */ 2938 limit = (max_softdeps / 10) * 9; 2939 if (dep_current[D_INODEDEP] > limit && 2940 ump->softdep_curdeps[D_INODEDEP] > limit / stat_flush_threads) 2941 return (0); 2942 if (thresh) 2943 thresh = jblocks->jb_min; 2944 else 2945 thresh = jblocks->jb_low; 2946 avail = (ump->softdep_on_journal * JREC_SIZE) / DEV_BSIZE; 2947 avail = jblocks->jb_free - avail; 2948 2949 return (avail > thresh); 2950 } 2951 2952 static void 2953 journal_suspend(ump) 2954 struct ufsmount *ump; 2955 { 2956 struct jblocks *jblocks; 2957 struct mount *mp; 2958 bool set; 2959 2960 mp = UFSTOVFS(ump); 2961 if ((mp->mnt_kern_flag & MNTK_SUSPEND) != 0) 2962 return; 2963 2964 jblocks = ump->softdep_jblocks; 2965 vfs_op_enter(mp); 2966 set = false; 2967 MNT_ILOCK(mp); 2968 if ((mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 2969 stat_journal_min++; 2970 mp->mnt_kern_flag |= MNTK_SUSPEND; 2971 mp->mnt_susp_owner = ump->softdep_flushtd; 2972 set = true; 2973 } 2974 jblocks->jb_suspended = 1; 2975 MNT_IUNLOCK(mp); 2976 if (!set) 2977 vfs_op_exit(mp); 2978 } 2979 2980 static int 2981 journal_unsuspend(struct ufsmount *ump) 2982 { 2983 struct jblocks *jblocks; 2984 struct mount *mp; 2985 2986 mp = UFSTOVFS(ump); 2987 jblocks = ump->softdep_jblocks; 2988 2989 if (jblocks != NULL && jblocks->jb_suspended && 2990 journal_space(ump, jblocks->jb_min)) { 2991 jblocks->jb_suspended = 0; 2992 FREE_LOCK(ump); 2993 mp->mnt_susp_owner = curthread; 2994 vfs_write_resume(mp, 0); 2995 ACQUIRE_LOCK(ump); 2996 return (1); 2997 } 2998 return (0); 2999 } 3000 3001 /* 3002 * Called before any allocation function to be certain that there is 3003 * sufficient space in the journal prior to creating any new records. 3004 * Since in the case of block allocation we may have multiple locked 3005 * buffers at the time of the actual allocation we can not block 3006 * when the journal records are created. Doing so would create a deadlock 3007 * if any of these buffers needed to be flushed to reclaim space. Instead 3008 * we require a sufficiently large amount of available space such that 3009 * each thread in the system could have passed this allocation check and 3010 * still have sufficient free space. With 20% of a minimum journal size 3011 * of 1MB we have 6553 records available. 3012 */ 3013 int 3014 softdep_prealloc(vp, waitok) 3015 struct vnode *vp; 3016 int waitok; 3017 { 3018 struct ufsmount *ump; 3019 3020 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 3021 ("softdep_prealloc called on non-softdep filesystem")); 3022 /* 3023 * Nothing to do if we are not running journaled soft updates. 3024 * If we currently hold the snapshot lock, we must avoid 3025 * handling other resources that could cause deadlock. Do not 3026 * touch quotas vnode since it is typically recursed with 3027 * other vnode locks held. 3028 */ 3029 if (DOINGSUJ(vp) == 0 || IS_SNAPSHOT(VTOI(vp)) || 3030 (vp->v_vflag & VV_SYSTEM) != 0) 3031 return (0); 3032 ump = VFSTOUFS(vp->v_mount); 3033 ACQUIRE_LOCK(ump); 3034 if (journal_space(ump, 0)) { 3035 FREE_LOCK(ump); 3036 return (0); 3037 } 3038 stat_journal_low++; 3039 FREE_LOCK(ump); 3040 if (waitok == MNT_NOWAIT) 3041 return (ENOSPC); 3042 /* 3043 * Attempt to sync this vnode once to flush any journal 3044 * work attached to it. 3045 */ 3046 if ((curthread->td_pflags & TDP_COWINPROGRESS) == 0) 3047 ffs_syncvnode(vp, waitok, 0); 3048 ACQUIRE_LOCK(ump); 3049 process_removes(vp); 3050 process_truncates(vp); 3051 if (journal_space(ump, 0) == 0) { 3052 softdep_speedup(ump); 3053 if (journal_space(ump, 1) == 0) 3054 journal_suspend(ump); 3055 } 3056 FREE_LOCK(ump); 3057 3058 return (0); 3059 } 3060 3061 /* 3062 * Before adjusting a link count on a vnode verify that we have sufficient 3063 * journal space. If not, process operations that depend on the currently 3064 * locked pair of vnodes to try to flush space as the syncer, buf daemon, 3065 * and softdep flush threads can not acquire these locks to reclaim space. 3066 */ 3067 static void 3068 softdep_prelink(dvp, vp) 3069 struct vnode *dvp; 3070 struct vnode *vp; 3071 { 3072 struct ufsmount *ump; 3073 3074 ump = VFSTOUFS(dvp->v_mount); 3075 LOCK_OWNED(ump); 3076 /* 3077 * Nothing to do if we have sufficient journal space. 3078 * If we currently hold the snapshot lock, we must avoid 3079 * handling other resources that could cause deadlock. 3080 */ 3081 if (journal_space(ump, 0) || (vp && IS_SNAPSHOT(VTOI(vp)))) 3082 return; 3083 stat_journal_low++; 3084 FREE_LOCK(ump); 3085 if (vp) 3086 ffs_syncvnode(vp, MNT_NOWAIT, 0); 3087 ffs_syncvnode(dvp, MNT_WAIT, 0); 3088 ACQUIRE_LOCK(ump); 3089 /* Process vp before dvp as it may create .. removes. */ 3090 if (vp) { 3091 process_removes(vp); 3092 process_truncates(vp); 3093 } 3094 process_removes(dvp); 3095 process_truncates(dvp); 3096 softdep_speedup(ump); 3097 process_worklist_item(UFSTOVFS(ump), 2, LK_NOWAIT); 3098 if (journal_space(ump, 0) == 0) { 3099 softdep_speedup(ump); 3100 if (journal_space(ump, 1) == 0) 3101 journal_suspend(ump); 3102 } 3103 } 3104 3105 static void 3106 jseg_write(ump, jseg, data) 3107 struct ufsmount *ump; 3108 struct jseg *jseg; 3109 uint8_t *data; 3110 { 3111 struct jsegrec *rec; 3112 3113 rec = (struct jsegrec *)data; 3114 rec->jsr_seq = jseg->js_seq; 3115 rec->jsr_oldest = jseg->js_oldseq; 3116 rec->jsr_cnt = jseg->js_cnt; 3117 rec->jsr_blocks = jseg->js_size / ump->um_devvp->v_bufobj.bo_bsize; 3118 rec->jsr_crc = 0; 3119 rec->jsr_time = ump->um_fs->fs_mtime; 3120 } 3121 3122 static inline void 3123 inoref_write(inoref, jseg, rec) 3124 struct inoref *inoref; 3125 struct jseg *jseg; 3126 struct jrefrec *rec; 3127 { 3128 3129 inoref->if_jsegdep->jd_seg = jseg; 3130 rec->jr_ino = inoref->if_ino; 3131 rec->jr_parent = inoref->if_parent; 3132 rec->jr_nlink = inoref->if_nlink; 3133 rec->jr_mode = inoref->if_mode; 3134 rec->jr_diroff = inoref->if_diroff; 3135 } 3136 3137 static void 3138 jaddref_write(jaddref, jseg, data) 3139 struct jaddref *jaddref; 3140 struct jseg *jseg; 3141 uint8_t *data; 3142 { 3143 struct jrefrec *rec; 3144 3145 rec = (struct jrefrec *)data; 3146 rec->jr_op = JOP_ADDREF; 3147 inoref_write(&jaddref->ja_ref, jseg, rec); 3148 } 3149 3150 static void 3151 jremref_write(jremref, jseg, data) 3152 struct jremref *jremref; 3153 struct jseg *jseg; 3154 uint8_t *data; 3155 { 3156 struct jrefrec *rec; 3157 3158 rec = (struct jrefrec *)data; 3159 rec->jr_op = JOP_REMREF; 3160 inoref_write(&jremref->jr_ref, jseg, rec); 3161 } 3162 3163 static void 3164 jmvref_write(jmvref, jseg, data) 3165 struct jmvref *jmvref; 3166 struct jseg *jseg; 3167 uint8_t *data; 3168 { 3169 struct jmvrec *rec; 3170 3171 rec = (struct jmvrec *)data; 3172 rec->jm_op = JOP_MVREF; 3173 rec->jm_ino = jmvref->jm_ino; 3174 rec->jm_parent = jmvref->jm_parent; 3175 rec->jm_oldoff = jmvref->jm_oldoff; 3176 rec->jm_newoff = jmvref->jm_newoff; 3177 } 3178 3179 static void 3180 jnewblk_write(jnewblk, jseg, data) 3181 struct jnewblk *jnewblk; 3182 struct jseg *jseg; 3183 uint8_t *data; 3184 { 3185 struct jblkrec *rec; 3186 3187 jnewblk->jn_jsegdep->jd_seg = jseg; 3188 rec = (struct jblkrec *)data; 3189 rec->jb_op = JOP_NEWBLK; 3190 rec->jb_ino = jnewblk->jn_ino; 3191 rec->jb_blkno = jnewblk->jn_blkno; 3192 rec->jb_lbn = jnewblk->jn_lbn; 3193 rec->jb_frags = jnewblk->jn_frags; 3194 rec->jb_oldfrags = jnewblk->jn_oldfrags; 3195 } 3196 3197 static void 3198 jfreeblk_write(jfreeblk, jseg, data) 3199 struct jfreeblk *jfreeblk; 3200 struct jseg *jseg; 3201 uint8_t *data; 3202 { 3203 struct jblkrec *rec; 3204 3205 jfreeblk->jf_dep.jb_jsegdep->jd_seg = jseg; 3206 rec = (struct jblkrec *)data; 3207 rec->jb_op = JOP_FREEBLK; 3208 rec->jb_ino = jfreeblk->jf_ino; 3209 rec->jb_blkno = jfreeblk->jf_blkno; 3210 rec->jb_lbn = jfreeblk->jf_lbn; 3211 rec->jb_frags = jfreeblk->jf_frags; 3212 rec->jb_oldfrags = 0; 3213 } 3214 3215 static void 3216 jfreefrag_write(jfreefrag, jseg, data) 3217 struct jfreefrag *jfreefrag; 3218 struct jseg *jseg; 3219 uint8_t *data; 3220 { 3221 struct jblkrec *rec; 3222 3223 jfreefrag->fr_jsegdep->jd_seg = jseg; 3224 rec = (struct jblkrec *)data; 3225 rec->jb_op = JOP_FREEBLK; 3226 rec->jb_ino = jfreefrag->fr_ino; 3227 rec->jb_blkno = jfreefrag->fr_blkno; 3228 rec->jb_lbn = jfreefrag->fr_lbn; 3229 rec->jb_frags = jfreefrag->fr_frags; 3230 rec->jb_oldfrags = 0; 3231 } 3232 3233 static void 3234 jtrunc_write(jtrunc, jseg, data) 3235 struct jtrunc *jtrunc; 3236 struct jseg *jseg; 3237 uint8_t *data; 3238 { 3239 struct jtrncrec *rec; 3240 3241 jtrunc->jt_dep.jb_jsegdep->jd_seg = jseg; 3242 rec = (struct jtrncrec *)data; 3243 rec->jt_op = JOP_TRUNC; 3244 rec->jt_ino = jtrunc->jt_ino; 3245 rec->jt_size = jtrunc->jt_size; 3246 rec->jt_extsize = jtrunc->jt_extsize; 3247 } 3248 3249 static void 3250 jfsync_write(jfsync, jseg, data) 3251 struct jfsync *jfsync; 3252 struct jseg *jseg; 3253 uint8_t *data; 3254 { 3255 struct jtrncrec *rec; 3256 3257 rec = (struct jtrncrec *)data; 3258 rec->jt_op = JOP_SYNC; 3259 rec->jt_ino = jfsync->jfs_ino; 3260 rec->jt_size = jfsync->jfs_size; 3261 rec->jt_extsize = jfsync->jfs_extsize; 3262 } 3263 3264 static void 3265 softdep_flushjournal(mp) 3266 struct mount *mp; 3267 { 3268 struct jblocks *jblocks; 3269 struct ufsmount *ump; 3270 3271 if (MOUNTEDSUJ(mp) == 0) 3272 return; 3273 ump = VFSTOUFS(mp); 3274 jblocks = ump->softdep_jblocks; 3275 ACQUIRE_LOCK(ump); 3276 while (ump->softdep_on_journal) { 3277 jblocks->jb_needseg = 1; 3278 softdep_process_journal(mp, NULL, MNT_WAIT); 3279 } 3280 FREE_LOCK(ump); 3281 } 3282 3283 static void softdep_synchronize_completed(struct bio *); 3284 static void softdep_synchronize(struct bio *, struct ufsmount *, void *); 3285 3286 static void 3287 softdep_synchronize_completed(bp) 3288 struct bio *bp; 3289 { 3290 struct jseg *oldest; 3291 struct jseg *jseg; 3292 struct ufsmount *ump; 3293 3294 /* 3295 * caller1 marks the last segment written before we issued the 3296 * synchronize cache. 3297 */ 3298 jseg = bp->bio_caller1; 3299 if (jseg == NULL) { 3300 g_destroy_bio(bp); 3301 return; 3302 } 3303 ump = VFSTOUFS(jseg->js_list.wk_mp); 3304 ACQUIRE_LOCK(ump); 3305 oldest = NULL; 3306 /* 3307 * Mark all the journal entries waiting on the synchronize cache 3308 * as completed so they may continue on. 3309 */ 3310 while (jseg != NULL && (jseg->js_state & COMPLETE) == 0) { 3311 jseg->js_state |= COMPLETE; 3312 oldest = jseg; 3313 jseg = TAILQ_PREV(jseg, jseglst, js_next); 3314 } 3315 /* 3316 * Restart deferred journal entry processing from the oldest 3317 * completed jseg. 3318 */ 3319 if (oldest) 3320 complete_jsegs(oldest); 3321 3322 FREE_LOCK(ump); 3323 g_destroy_bio(bp); 3324 } 3325 3326 /* 3327 * Send BIO_FLUSH/SYNCHRONIZE CACHE to the device to enforce write ordering 3328 * barriers. The journal must be written prior to any blocks that depend 3329 * on it and the journal can not be released until the blocks have be 3330 * written. This code handles both barriers simultaneously. 3331 */ 3332 static void 3333 softdep_synchronize(bp, ump, caller1) 3334 struct bio *bp; 3335 struct ufsmount *ump; 3336 void *caller1; 3337 { 3338 3339 bp->bio_cmd = BIO_FLUSH; 3340 bp->bio_flags |= BIO_ORDERED; 3341 bp->bio_data = NULL; 3342 bp->bio_offset = ump->um_cp->provider->mediasize; 3343 bp->bio_length = 0; 3344 bp->bio_done = softdep_synchronize_completed; 3345 bp->bio_caller1 = caller1; 3346 g_io_request(bp, 3347 (struct g_consumer *)ump->um_devvp->v_bufobj.bo_private); 3348 } 3349 3350 /* 3351 * Flush some journal records to disk. 3352 */ 3353 static void 3354 softdep_process_journal(mp, needwk, flags) 3355 struct mount *mp; 3356 struct worklist *needwk; 3357 int flags; 3358 { 3359 struct jblocks *jblocks; 3360 struct ufsmount *ump; 3361 struct worklist *wk; 3362 struct jseg *jseg; 3363 struct buf *bp; 3364 struct bio *bio; 3365 uint8_t *data; 3366 struct fs *fs; 3367 int shouldflush; 3368 int segwritten; 3369 int jrecmin; /* Minimum records per block. */ 3370 int jrecmax; /* Maximum records per block. */ 3371 int size; 3372 int cnt; 3373 int off; 3374 int devbsize; 3375 3376 if (MOUNTEDSUJ(mp) == 0) 3377 return; 3378 shouldflush = softdep_flushcache; 3379 bio = NULL; 3380 jseg = NULL; 3381 ump = VFSTOUFS(mp); 3382 LOCK_OWNED(ump); 3383 fs = ump->um_fs; 3384 jblocks = ump->softdep_jblocks; 3385 devbsize = ump->um_devvp->v_bufobj.bo_bsize; 3386 /* 3387 * We write anywhere between a disk block and fs block. The upper 3388 * bound is picked to prevent buffer cache fragmentation and limit 3389 * processing time per I/O. 3390 */ 3391 jrecmin = (devbsize / JREC_SIZE) - 1; /* -1 for seg header */ 3392 jrecmax = (fs->fs_bsize / devbsize) * jrecmin; 3393 segwritten = 0; 3394 for (;;) { 3395 cnt = ump->softdep_on_journal; 3396 /* 3397 * Criteria for writing a segment: 3398 * 1) We have a full block. 3399 * 2) We're called from jwait() and haven't found the 3400 * journal item yet. 3401 * 3) Always write if needseg is set. 3402 * 4) If we are called from process_worklist and have 3403 * not yet written anything we write a partial block 3404 * to enforce a 1 second maximum latency on journal 3405 * entries. 3406 */ 3407 if (cnt < (jrecmax - 1) && needwk == NULL && 3408 jblocks->jb_needseg == 0 && (segwritten || cnt == 0)) 3409 break; 3410 cnt++; 3411 /* 3412 * Verify some free journal space. softdep_prealloc() should 3413 * guarantee that we don't run out so this is indicative of 3414 * a problem with the flow control. Try to recover 3415 * gracefully in any event. 3416 */ 3417 while (jblocks->jb_free == 0) { 3418 if (flags != MNT_WAIT) 3419 break; 3420 printf("softdep: Out of journal space!\n"); 3421 softdep_speedup(ump); 3422 msleep(jblocks, LOCK_PTR(ump), PRIBIO, "jblocks", hz); 3423 } 3424 FREE_LOCK(ump); 3425 jseg = malloc(sizeof(*jseg), M_JSEG, M_SOFTDEP_FLAGS); 3426 workitem_alloc(&jseg->js_list, D_JSEG, mp); 3427 LIST_INIT(&jseg->js_entries); 3428 LIST_INIT(&jseg->js_indirs); 3429 jseg->js_state = ATTACHED; 3430 if (shouldflush == 0) 3431 jseg->js_state |= COMPLETE; 3432 else if (bio == NULL) 3433 bio = g_alloc_bio(); 3434 jseg->js_jblocks = jblocks; 3435 bp = geteblk(fs->fs_bsize, 0); 3436 ACQUIRE_LOCK(ump); 3437 /* 3438 * If there was a race while we were allocating the block 3439 * and jseg the entry we care about was likely written. 3440 * We bail out in both the WAIT and NOWAIT case and assume 3441 * the caller will loop if the entry it cares about is 3442 * not written. 3443 */ 3444 cnt = ump->softdep_on_journal; 3445 if (cnt + jblocks->jb_needseg == 0 || jblocks->jb_free == 0) { 3446 bp->b_flags |= B_INVAL | B_NOCACHE; 3447 WORKITEM_FREE(jseg, D_JSEG); 3448 FREE_LOCK(ump); 3449 brelse(bp); 3450 ACQUIRE_LOCK(ump); 3451 break; 3452 } 3453 /* 3454 * Calculate the disk block size required for the available 3455 * records rounded to the min size. 3456 */ 3457 if (cnt == 0) 3458 size = devbsize; 3459 else if (cnt < jrecmax) 3460 size = howmany(cnt, jrecmin) * devbsize; 3461 else 3462 size = fs->fs_bsize; 3463 /* 3464 * Allocate a disk block for this journal data and account 3465 * for truncation of the requested size if enough contiguous 3466 * space was not available. 3467 */ 3468 bp->b_blkno = jblocks_alloc(jblocks, size, &size); 3469 bp->b_lblkno = bp->b_blkno; 3470 bp->b_offset = bp->b_blkno * DEV_BSIZE; 3471 bp->b_bcount = size; 3472 bp->b_flags &= ~B_INVAL; 3473 bp->b_flags |= B_VALIDSUSPWRT | B_NOCOPY; 3474 /* 3475 * Initialize our jseg with cnt records. Assign the next 3476 * sequence number to it and link it in-order. 3477 */ 3478 cnt = MIN(cnt, (size / devbsize) * jrecmin); 3479 jseg->js_buf = bp; 3480 jseg->js_cnt = cnt; 3481 jseg->js_refs = cnt + 1; /* Self ref. */ 3482 jseg->js_size = size; 3483 jseg->js_seq = jblocks->jb_nextseq++; 3484 if (jblocks->jb_oldestseg == NULL) 3485 jblocks->jb_oldestseg = jseg; 3486 jseg->js_oldseq = jblocks->jb_oldestseg->js_seq; 3487 TAILQ_INSERT_TAIL(&jblocks->jb_segs, jseg, js_next); 3488 if (jblocks->jb_writeseg == NULL) 3489 jblocks->jb_writeseg = jseg; 3490 /* 3491 * Start filling in records from the pending list. 3492 */ 3493 data = bp->b_data; 3494 off = 0; 3495 3496 /* 3497 * Always put a header on the first block. 3498 * XXX As with below, there might not be a chance to get 3499 * into the loop. Ensure that something valid is written. 3500 */ 3501 jseg_write(ump, jseg, data); 3502 off += JREC_SIZE; 3503 data = bp->b_data + off; 3504 3505 /* 3506 * XXX Something is wrong here. There's no work to do, 3507 * but we need to perform and I/O and allow it to complete 3508 * anyways. 3509 */ 3510 if (LIST_EMPTY(&ump->softdep_journal_pending)) 3511 stat_emptyjblocks++; 3512 3513 while ((wk = LIST_FIRST(&ump->softdep_journal_pending)) 3514 != NULL) { 3515 if (cnt == 0) 3516 break; 3517 /* Place a segment header on every device block. */ 3518 if ((off % devbsize) == 0) { 3519 jseg_write(ump, jseg, data); 3520 off += JREC_SIZE; 3521 data = bp->b_data + off; 3522 } 3523 if (wk == needwk) 3524 needwk = NULL; 3525 remove_from_journal(wk); 3526 wk->wk_state |= INPROGRESS; 3527 WORKLIST_INSERT(&jseg->js_entries, wk); 3528 switch (wk->wk_type) { 3529 case D_JADDREF: 3530 jaddref_write(WK_JADDREF(wk), jseg, data); 3531 break; 3532 case D_JREMREF: 3533 jremref_write(WK_JREMREF(wk), jseg, data); 3534 break; 3535 case D_JMVREF: 3536 jmvref_write(WK_JMVREF(wk), jseg, data); 3537 break; 3538 case D_JNEWBLK: 3539 jnewblk_write(WK_JNEWBLK(wk), jseg, data); 3540 break; 3541 case D_JFREEBLK: 3542 jfreeblk_write(WK_JFREEBLK(wk), jseg, data); 3543 break; 3544 case D_JFREEFRAG: 3545 jfreefrag_write(WK_JFREEFRAG(wk), jseg, data); 3546 break; 3547 case D_JTRUNC: 3548 jtrunc_write(WK_JTRUNC(wk), jseg, data); 3549 break; 3550 case D_JFSYNC: 3551 jfsync_write(WK_JFSYNC(wk), jseg, data); 3552 break; 3553 default: 3554 panic("process_journal: Unknown type %s", 3555 TYPENAME(wk->wk_type)); 3556 /* NOTREACHED */ 3557 } 3558 off += JREC_SIZE; 3559 data = bp->b_data + off; 3560 cnt--; 3561 } 3562 3563 /* Clear any remaining space so we don't leak kernel data */ 3564 if (size > off) 3565 bzero(data, size - off); 3566 3567 /* 3568 * Write this one buffer and continue. 3569 */ 3570 segwritten = 1; 3571 jblocks->jb_needseg = 0; 3572 WORKLIST_INSERT(&bp->b_dep, &jseg->js_list); 3573 FREE_LOCK(ump); 3574 pbgetvp(ump->um_devvp, bp); 3575 /* 3576 * We only do the blocking wait once we find the journal 3577 * entry we're looking for. 3578 */ 3579 if (needwk == NULL && flags == MNT_WAIT) 3580 bwrite(bp); 3581 else 3582 bawrite(bp); 3583 ACQUIRE_LOCK(ump); 3584 } 3585 /* 3586 * If we wrote a segment issue a synchronize cache so the journal 3587 * is reflected on disk before the data is written. Since reclaiming 3588 * journal space also requires writing a journal record this 3589 * process also enforces a barrier before reclamation. 3590 */ 3591 if (segwritten && shouldflush) { 3592 softdep_synchronize(bio, ump, 3593 TAILQ_LAST(&jblocks->jb_segs, jseglst)); 3594 } else if (bio) 3595 g_destroy_bio(bio); 3596 /* 3597 * If we've suspended the filesystem because we ran out of journal 3598 * space either try to sync it here to make some progress or 3599 * unsuspend it if we already have. 3600 */ 3601 if (flags == 0 && jblocks->jb_suspended) { 3602 if (journal_unsuspend(ump)) 3603 return; 3604 FREE_LOCK(ump); 3605 VFS_SYNC(mp, MNT_NOWAIT); 3606 ffs_sbupdate(ump, MNT_WAIT, 0); 3607 ACQUIRE_LOCK(ump); 3608 } 3609 } 3610 3611 /* 3612 * Complete a jseg, allowing all dependencies awaiting journal writes 3613 * to proceed. Each journal dependency also attaches a jsegdep to dependent 3614 * structures so that the journal segment can be freed to reclaim space. 3615 */ 3616 static void 3617 complete_jseg(jseg) 3618 struct jseg *jseg; 3619 { 3620 struct worklist *wk; 3621 struct jmvref *jmvref; 3622 #ifdef INVARIANTS 3623 int i = 0; 3624 #endif 3625 3626 while ((wk = LIST_FIRST(&jseg->js_entries)) != NULL) { 3627 WORKLIST_REMOVE(wk); 3628 wk->wk_state &= ~INPROGRESS; 3629 wk->wk_state |= COMPLETE; 3630 KASSERT(i++ < jseg->js_cnt, 3631 ("handle_written_jseg: overflow %d >= %d", 3632 i - 1, jseg->js_cnt)); 3633 switch (wk->wk_type) { 3634 case D_JADDREF: 3635 handle_written_jaddref(WK_JADDREF(wk)); 3636 break; 3637 case D_JREMREF: 3638 handle_written_jremref(WK_JREMREF(wk)); 3639 break; 3640 case D_JMVREF: 3641 rele_jseg(jseg); /* No jsegdep. */ 3642 jmvref = WK_JMVREF(wk); 3643 LIST_REMOVE(jmvref, jm_deps); 3644 if ((jmvref->jm_pagedep->pd_state & ONWORKLIST) == 0) 3645 free_pagedep(jmvref->jm_pagedep); 3646 WORKITEM_FREE(jmvref, D_JMVREF); 3647 break; 3648 case D_JNEWBLK: 3649 handle_written_jnewblk(WK_JNEWBLK(wk)); 3650 break; 3651 case D_JFREEBLK: 3652 handle_written_jblkdep(&WK_JFREEBLK(wk)->jf_dep); 3653 break; 3654 case D_JTRUNC: 3655 handle_written_jblkdep(&WK_JTRUNC(wk)->jt_dep); 3656 break; 3657 case D_JFSYNC: 3658 rele_jseg(jseg); /* No jsegdep. */ 3659 WORKITEM_FREE(wk, D_JFSYNC); 3660 break; 3661 case D_JFREEFRAG: 3662 handle_written_jfreefrag(WK_JFREEFRAG(wk)); 3663 break; 3664 default: 3665 panic("handle_written_jseg: Unknown type %s", 3666 TYPENAME(wk->wk_type)); 3667 /* NOTREACHED */ 3668 } 3669 } 3670 /* Release the self reference so the structure may be freed. */ 3671 rele_jseg(jseg); 3672 } 3673 3674 /* 3675 * Determine which jsegs are ready for completion processing. Waits for 3676 * synchronize cache to complete as well as forcing in-order completion 3677 * of journal entries. 3678 */ 3679 static void 3680 complete_jsegs(jseg) 3681 struct jseg *jseg; 3682 { 3683 struct jblocks *jblocks; 3684 struct jseg *jsegn; 3685 3686 jblocks = jseg->js_jblocks; 3687 /* 3688 * Don't allow out of order completions. If this isn't the first 3689 * block wait for it to write before we're done. 3690 */ 3691 if (jseg != jblocks->jb_writeseg) 3692 return; 3693 /* Iterate through available jsegs processing their entries. */ 3694 while (jseg && (jseg->js_state & ALLCOMPLETE) == ALLCOMPLETE) { 3695 jblocks->jb_oldestwrseq = jseg->js_oldseq; 3696 jsegn = TAILQ_NEXT(jseg, js_next); 3697 complete_jseg(jseg); 3698 jseg = jsegn; 3699 } 3700 jblocks->jb_writeseg = jseg; 3701 /* 3702 * Attempt to free jsegs now that oldestwrseq may have advanced. 3703 */ 3704 free_jsegs(jblocks); 3705 } 3706 3707 /* 3708 * Mark a jseg as DEPCOMPLETE and throw away the buffer. Attempt to handle 3709 * the final completions. 3710 */ 3711 static void 3712 handle_written_jseg(jseg, bp) 3713 struct jseg *jseg; 3714 struct buf *bp; 3715 { 3716 3717 if (jseg->js_refs == 0) 3718 panic("handle_written_jseg: No self-reference on %p", jseg); 3719 jseg->js_state |= DEPCOMPLETE; 3720 /* 3721 * We'll never need this buffer again, set flags so it will be 3722 * discarded. 3723 */ 3724 bp->b_flags |= B_INVAL | B_NOCACHE; 3725 pbrelvp(bp); 3726 complete_jsegs(jseg); 3727 } 3728 3729 static inline struct jsegdep * 3730 inoref_jseg(inoref) 3731 struct inoref *inoref; 3732 { 3733 struct jsegdep *jsegdep; 3734 3735 jsegdep = inoref->if_jsegdep; 3736 inoref->if_jsegdep = NULL; 3737 3738 return (jsegdep); 3739 } 3740 3741 /* 3742 * Called once a jremref has made it to stable store. The jremref is marked 3743 * complete and we attempt to free it. Any pagedeps writes sleeping waiting 3744 * for the jremref to complete will be awoken by free_jremref. 3745 */ 3746 static void 3747 handle_written_jremref(jremref) 3748 struct jremref *jremref; 3749 { 3750 struct inodedep *inodedep; 3751 struct jsegdep *jsegdep; 3752 struct dirrem *dirrem; 3753 3754 /* Grab the jsegdep. */ 3755 jsegdep = inoref_jseg(&jremref->jr_ref); 3756 /* 3757 * Remove us from the inoref list. 3758 */ 3759 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 3760 0, &inodedep) == 0) 3761 panic("handle_written_jremref: Lost inodedep"); 3762 TAILQ_REMOVE(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 3763 /* 3764 * Complete the dirrem. 3765 */ 3766 dirrem = jremref->jr_dirrem; 3767 jremref->jr_dirrem = NULL; 3768 LIST_REMOVE(jremref, jr_deps); 3769 jsegdep->jd_state |= jremref->jr_state & MKDIR_PARENT; 3770 jwork_insert(&dirrem->dm_jwork, jsegdep); 3771 if (LIST_EMPTY(&dirrem->dm_jremrefhd) && 3772 (dirrem->dm_state & COMPLETE) != 0) 3773 add_to_worklist(&dirrem->dm_list, 0); 3774 free_jremref(jremref); 3775 } 3776 3777 /* 3778 * Called once a jaddref has made it to stable store. The dependency is 3779 * marked complete and any dependent structures are added to the inode 3780 * bufwait list to be completed as soon as it is written. If a bitmap write 3781 * depends on this entry we move the inode into the inodedephd of the 3782 * bmsafemap dependency and attempt to remove the jaddref from the bmsafemap. 3783 */ 3784 static void 3785 handle_written_jaddref(jaddref) 3786 struct jaddref *jaddref; 3787 { 3788 struct jsegdep *jsegdep; 3789 struct inodedep *inodedep; 3790 struct diradd *diradd; 3791 struct mkdir *mkdir; 3792 3793 /* Grab the jsegdep. */ 3794 jsegdep = inoref_jseg(&jaddref->ja_ref); 3795 mkdir = NULL; 3796 diradd = NULL; 3797 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 3798 0, &inodedep) == 0) 3799 panic("handle_written_jaddref: Lost inodedep."); 3800 if (jaddref->ja_diradd == NULL) 3801 panic("handle_written_jaddref: No dependency"); 3802 if (jaddref->ja_diradd->da_list.wk_type == D_DIRADD) { 3803 diradd = jaddref->ja_diradd; 3804 WORKLIST_INSERT(&inodedep->id_bufwait, &diradd->da_list); 3805 } else if (jaddref->ja_state & MKDIR_PARENT) { 3806 mkdir = jaddref->ja_mkdir; 3807 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir->md_list); 3808 } else if (jaddref->ja_state & MKDIR_BODY) 3809 mkdir = jaddref->ja_mkdir; 3810 else 3811 panic("handle_written_jaddref: Unknown dependency %p", 3812 jaddref->ja_diradd); 3813 jaddref->ja_diradd = NULL; /* also clears ja_mkdir */ 3814 /* 3815 * Remove us from the inode list. 3816 */ 3817 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, if_deps); 3818 /* 3819 * The mkdir may be waiting on the jaddref to clear before freeing. 3820 */ 3821 if (mkdir) { 3822 KASSERT(mkdir->md_list.wk_type == D_MKDIR, 3823 ("handle_written_jaddref: Incorrect type for mkdir %s", 3824 TYPENAME(mkdir->md_list.wk_type))); 3825 mkdir->md_jaddref = NULL; 3826 diradd = mkdir->md_diradd; 3827 mkdir->md_state |= DEPCOMPLETE; 3828 complete_mkdir(mkdir); 3829 } 3830 jwork_insert(&diradd->da_jwork, jsegdep); 3831 if (jaddref->ja_state & NEWBLOCK) { 3832 inodedep->id_state |= ONDEPLIST; 3833 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_inodedephd, 3834 inodedep, id_deps); 3835 } 3836 free_jaddref(jaddref); 3837 } 3838 3839 /* 3840 * Called once a jnewblk journal is written. The allocdirect or allocindir 3841 * is placed in the bmsafemap to await notification of a written bitmap. If 3842 * the operation was canceled we add the segdep to the appropriate 3843 * dependency to free the journal space once the canceling operation 3844 * completes. 3845 */ 3846 static void 3847 handle_written_jnewblk(jnewblk) 3848 struct jnewblk *jnewblk; 3849 { 3850 struct bmsafemap *bmsafemap; 3851 struct freefrag *freefrag; 3852 struct freework *freework; 3853 struct jsegdep *jsegdep; 3854 struct newblk *newblk; 3855 3856 /* Grab the jsegdep. */ 3857 jsegdep = jnewblk->jn_jsegdep; 3858 jnewblk->jn_jsegdep = NULL; 3859 if (jnewblk->jn_dep == NULL) 3860 panic("handle_written_jnewblk: No dependency for the segdep."); 3861 switch (jnewblk->jn_dep->wk_type) { 3862 case D_NEWBLK: 3863 case D_ALLOCDIRECT: 3864 case D_ALLOCINDIR: 3865 /* 3866 * Add the written block to the bmsafemap so it can 3867 * be notified when the bitmap is on disk. 3868 */ 3869 newblk = WK_NEWBLK(jnewblk->jn_dep); 3870 newblk->nb_jnewblk = NULL; 3871 if ((newblk->nb_state & GOINGAWAY) == 0) { 3872 bmsafemap = newblk->nb_bmsafemap; 3873 newblk->nb_state |= ONDEPLIST; 3874 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, 3875 nb_deps); 3876 } 3877 jwork_insert(&newblk->nb_jwork, jsegdep); 3878 break; 3879 case D_FREEFRAG: 3880 /* 3881 * A newblock being removed by a freefrag when replaced by 3882 * frag extension. 3883 */ 3884 freefrag = WK_FREEFRAG(jnewblk->jn_dep); 3885 freefrag->ff_jdep = NULL; 3886 jwork_insert(&freefrag->ff_jwork, jsegdep); 3887 break; 3888 case D_FREEWORK: 3889 /* 3890 * A direct block was removed by truncate. 3891 */ 3892 freework = WK_FREEWORK(jnewblk->jn_dep); 3893 freework->fw_jnewblk = NULL; 3894 jwork_insert(&freework->fw_freeblks->fb_jwork, jsegdep); 3895 break; 3896 default: 3897 panic("handle_written_jnewblk: Unknown type %d.", 3898 jnewblk->jn_dep->wk_type); 3899 } 3900 jnewblk->jn_dep = NULL; 3901 free_jnewblk(jnewblk); 3902 } 3903 3904 /* 3905 * Cancel a jfreefrag that won't be needed, probably due to colliding with 3906 * an in-flight allocation that has not yet been committed. Divorce us 3907 * from the freefrag and mark it DEPCOMPLETE so that it may be added 3908 * to the worklist. 3909 */ 3910 static void 3911 cancel_jfreefrag(jfreefrag) 3912 struct jfreefrag *jfreefrag; 3913 { 3914 struct freefrag *freefrag; 3915 3916 if (jfreefrag->fr_jsegdep) { 3917 free_jsegdep(jfreefrag->fr_jsegdep); 3918 jfreefrag->fr_jsegdep = NULL; 3919 } 3920 freefrag = jfreefrag->fr_freefrag; 3921 jfreefrag->fr_freefrag = NULL; 3922 free_jfreefrag(jfreefrag); 3923 freefrag->ff_state |= DEPCOMPLETE; 3924 CTR1(KTR_SUJ, "cancel_jfreefrag: blkno %jd", freefrag->ff_blkno); 3925 } 3926 3927 /* 3928 * Free a jfreefrag when the parent freefrag is rendered obsolete. 3929 */ 3930 static void 3931 free_jfreefrag(jfreefrag) 3932 struct jfreefrag *jfreefrag; 3933 { 3934 3935 if (jfreefrag->fr_state & INPROGRESS) 3936 WORKLIST_REMOVE(&jfreefrag->fr_list); 3937 else if (jfreefrag->fr_state & ONWORKLIST) 3938 remove_from_journal(&jfreefrag->fr_list); 3939 if (jfreefrag->fr_freefrag != NULL) 3940 panic("free_jfreefrag: Still attached to a freefrag."); 3941 WORKITEM_FREE(jfreefrag, D_JFREEFRAG); 3942 } 3943 3944 /* 3945 * Called when the journal write for a jfreefrag completes. The parent 3946 * freefrag is added to the worklist if this completes its dependencies. 3947 */ 3948 static void 3949 handle_written_jfreefrag(jfreefrag) 3950 struct jfreefrag *jfreefrag; 3951 { 3952 struct jsegdep *jsegdep; 3953 struct freefrag *freefrag; 3954 3955 /* Grab the jsegdep. */ 3956 jsegdep = jfreefrag->fr_jsegdep; 3957 jfreefrag->fr_jsegdep = NULL; 3958 freefrag = jfreefrag->fr_freefrag; 3959 if (freefrag == NULL) 3960 panic("handle_written_jfreefrag: No freefrag."); 3961 freefrag->ff_state |= DEPCOMPLETE; 3962 freefrag->ff_jdep = NULL; 3963 jwork_insert(&freefrag->ff_jwork, jsegdep); 3964 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 3965 add_to_worklist(&freefrag->ff_list, 0); 3966 jfreefrag->fr_freefrag = NULL; 3967 free_jfreefrag(jfreefrag); 3968 } 3969 3970 /* 3971 * Called when the journal write for a jfreeblk completes. The jfreeblk 3972 * is removed from the freeblks list of pending journal writes and the 3973 * jsegdep is moved to the freeblks jwork to be completed when all blocks 3974 * have been reclaimed. 3975 */ 3976 static void 3977 handle_written_jblkdep(jblkdep) 3978 struct jblkdep *jblkdep; 3979 { 3980 struct freeblks *freeblks; 3981 struct jsegdep *jsegdep; 3982 3983 /* Grab the jsegdep. */ 3984 jsegdep = jblkdep->jb_jsegdep; 3985 jblkdep->jb_jsegdep = NULL; 3986 freeblks = jblkdep->jb_freeblks; 3987 LIST_REMOVE(jblkdep, jb_deps); 3988 jwork_insert(&freeblks->fb_jwork, jsegdep); 3989 /* 3990 * If the freeblks is all journaled, we can add it to the worklist. 3991 */ 3992 if (LIST_EMPTY(&freeblks->fb_jblkdephd) && 3993 (freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) 3994 add_to_worklist(&freeblks->fb_list, WK_NODELAY); 3995 3996 free_jblkdep(jblkdep); 3997 } 3998 3999 static struct jsegdep * 4000 newjsegdep(struct worklist *wk) 4001 { 4002 struct jsegdep *jsegdep; 4003 4004 jsegdep = malloc(sizeof(*jsegdep), M_JSEGDEP, M_SOFTDEP_FLAGS); 4005 workitem_alloc(&jsegdep->jd_list, D_JSEGDEP, wk->wk_mp); 4006 jsegdep->jd_seg = NULL; 4007 4008 return (jsegdep); 4009 } 4010 4011 static struct jmvref * 4012 newjmvref(dp, ino, oldoff, newoff) 4013 struct inode *dp; 4014 ino_t ino; 4015 off_t oldoff; 4016 off_t newoff; 4017 { 4018 struct jmvref *jmvref; 4019 4020 jmvref = malloc(sizeof(*jmvref), M_JMVREF, M_SOFTDEP_FLAGS); 4021 workitem_alloc(&jmvref->jm_list, D_JMVREF, ITOVFS(dp)); 4022 jmvref->jm_list.wk_state = ATTACHED | DEPCOMPLETE; 4023 jmvref->jm_parent = dp->i_number; 4024 jmvref->jm_ino = ino; 4025 jmvref->jm_oldoff = oldoff; 4026 jmvref->jm_newoff = newoff; 4027 4028 return (jmvref); 4029 } 4030 4031 /* 4032 * Allocate a new jremref that tracks the removal of ip from dp with the 4033 * directory entry offset of diroff. Mark the entry as ATTACHED and 4034 * DEPCOMPLETE as we have all the information required for the journal write 4035 * and the directory has already been removed from the buffer. The caller 4036 * is responsible for linking the jremref into the pagedep and adding it 4037 * to the journal to write. The MKDIR_PARENT flag is set if we're doing 4038 * a DOTDOT addition so handle_workitem_remove() can properly assign 4039 * the jsegdep when we're done. 4040 */ 4041 static struct jremref * 4042 newjremref(struct dirrem *dirrem, struct inode *dp, struct inode *ip, 4043 off_t diroff, nlink_t nlink) 4044 { 4045 struct jremref *jremref; 4046 4047 jremref = malloc(sizeof(*jremref), M_JREMREF, M_SOFTDEP_FLAGS); 4048 workitem_alloc(&jremref->jr_list, D_JREMREF, ITOVFS(dp)); 4049 jremref->jr_state = ATTACHED; 4050 newinoref(&jremref->jr_ref, ip->i_number, dp->i_number, diroff, 4051 nlink, ip->i_mode); 4052 jremref->jr_dirrem = dirrem; 4053 4054 return (jremref); 4055 } 4056 4057 static inline void 4058 newinoref(struct inoref *inoref, ino_t ino, ino_t parent, off_t diroff, 4059 nlink_t nlink, uint16_t mode) 4060 { 4061 4062 inoref->if_jsegdep = newjsegdep(&inoref->if_list); 4063 inoref->if_diroff = diroff; 4064 inoref->if_ino = ino; 4065 inoref->if_parent = parent; 4066 inoref->if_nlink = nlink; 4067 inoref->if_mode = mode; 4068 } 4069 4070 /* 4071 * Allocate a new jaddref to track the addition of ino to dp at diroff. The 4072 * directory offset may not be known until later. The caller is responsible 4073 * adding the entry to the journal when this information is available. nlink 4074 * should be the link count prior to the addition and mode is only required 4075 * to have the correct FMT. 4076 */ 4077 static struct jaddref * 4078 newjaddref(struct inode *dp, ino_t ino, off_t diroff, int16_t nlink, 4079 uint16_t mode) 4080 { 4081 struct jaddref *jaddref; 4082 4083 jaddref = malloc(sizeof(*jaddref), M_JADDREF, M_SOFTDEP_FLAGS); 4084 workitem_alloc(&jaddref->ja_list, D_JADDREF, ITOVFS(dp)); 4085 jaddref->ja_state = ATTACHED; 4086 jaddref->ja_mkdir = NULL; 4087 newinoref(&jaddref->ja_ref, ino, dp->i_number, diroff, nlink, mode); 4088 4089 return (jaddref); 4090 } 4091 4092 /* 4093 * Create a new free dependency for a freework. The caller is responsible 4094 * for adjusting the reference count when it has the lock held. The freedep 4095 * will track an outstanding bitmap write that will ultimately clear the 4096 * freework to continue. 4097 */ 4098 static struct freedep * 4099 newfreedep(struct freework *freework) 4100 { 4101 struct freedep *freedep; 4102 4103 freedep = malloc(sizeof(*freedep), M_FREEDEP, M_SOFTDEP_FLAGS); 4104 workitem_alloc(&freedep->fd_list, D_FREEDEP, freework->fw_list.wk_mp); 4105 freedep->fd_freework = freework; 4106 4107 return (freedep); 4108 } 4109 4110 /* 4111 * Free a freedep structure once the buffer it is linked to is written. If 4112 * this is the last reference to the freework schedule it for completion. 4113 */ 4114 static void 4115 free_freedep(freedep) 4116 struct freedep *freedep; 4117 { 4118 struct freework *freework; 4119 4120 freework = freedep->fd_freework; 4121 freework->fw_freeblks->fb_cgwait--; 4122 if (--freework->fw_ref == 0) 4123 freework_enqueue(freework); 4124 WORKITEM_FREE(freedep, D_FREEDEP); 4125 } 4126 4127 /* 4128 * Allocate a new freework structure that may be a level in an indirect 4129 * when parent is not NULL or a top level block when it is. The top level 4130 * freework structures are allocated without the per-filesystem lock held 4131 * and before the freeblks is visible outside of softdep_setup_freeblocks(). 4132 */ 4133 static struct freework * 4134 newfreework(ump, freeblks, parent, lbn, nb, frags, off, journal) 4135 struct ufsmount *ump; 4136 struct freeblks *freeblks; 4137 struct freework *parent; 4138 ufs_lbn_t lbn; 4139 ufs2_daddr_t nb; 4140 int frags; 4141 int off; 4142 int journal; 4143 { 4144 struct freework *freework; 4145 4146 freework = malloc(sizeof(*freework), M_FREEWORK, M_SOFTDEP_FLAGS); 4147 workitem_alloc(&freework->fw_list, D_FREEWORK, freeblks->fb_list.wk_mp); 4148 freework->fw_state = ATTACHED; 4149 freework->fw_jnewblk = NULL; 4150 freework->fw_freeblks = freeblks; 4151 freework->fw_parent = parent; 4152 freework->fw_lbn = lbn; 4153 freework->fw_blkno = nb; 4154 freework->fw_frags = frags; 4155 freework->fw_indir = NULL; 4156 freework->fw_ref = (MOUNTEDSUJ(UFSTOVFS(ump)) == 0 || 4157 lbn >= -UFS_NXADDR) ? 0 : NINDIR(ump->um_fs) + 1; 4158 freework->fw_start = freework->fw_off = off; 4159 if (journal) 4160 newjfreeblk(freeblks, lbn, nb, frags); 4161 if (parent == NULL) { 4162 ACQUIRE_LOCK(ump); 4163 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list); 4164 freeblks->fb_ref++; 4165 FREE_LOCK(ump); 4166 } 4167 4168 return (freework); 4169 } 4170 4171 /* 4172 * Eliminate a jfreeblk for a block that does not need journaling. 4173 */ 4174 static void 4175 cancel_jfreeblk(freeblks, blkno) 4176 struct freeblks *freeblks; 4177 ufs2_daddr_t blkno; 4178 { 4179 struct jfreeblk *jfreeblk; 4180 struct jblkdep *jblkdep; 4181 4182 LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps) { 4183 if (jblkdep->jb_list.wk_type != D_JFREEBLK) 4184 continue; 4185 jfreeblk = WK_JFREEBLK(&jblkdep->jb_list); 4186 if (jfreeblk->jf_blkno == blkno) 4187 break; 4188 } 4189 if (jblkdep == NULL) 4190 return; 4191 CTR1(KTR_SUJ, "cancel_jfreeblk: blkno %jd", blkno); 4192 free_jsegdep(jblkdep->jb_jsegdep); 4193 LIST_REMOVE(jblkdep, jb_deps); 4194 WORKITEM_FREE(jfreeblk, D_JFREEBLK); 4195 } 4196 4197 /* 4198 * Allocate a new jfreeblk to journal top level block pointer when truncating 4199 * a file. The caller must add this to the worklist when the per-filesystem 4200 * lock is held. 4201 */ 4202 static struct jfreeblk * 4203 newjfreeblk(freeblks, lbn, blkno, frags) 4204 struct freeblks *freeblks; 4205 ufs_lbn_t lbn; 4206 ufs2_daddr_t blkno; 4207 int frags; 4208 { 4209 struct jfreeblk *jfreeblk; 4210 4211 jfreeblk = malloc(sizeof(*jfreeblk), M_JFREEBLK, M_SOFTDEP_FLAGS); 4212 workitem_alloc(&jfreeblk->jf_dep.jb_list, D_JFREEBLK, 4213 freeblks->fb_list.wk_mp); 4214 jfreeblk->jf_dep.jb_jsegdep = newjsegdep(&jfreeblk->jf_dep.jb_list); 4215 jfreeblk->jf_dep.jb_freeblks = freeblks; 4216 jfreeblk->jf_ino = freeblks->fb_inum; 4217 jfreeblk->jf_lbn = lbn; 4218 jfreeblk->jf_blkno = blkno; 4219 jfreeblk->jf_frags = frags; 4220 LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jfreeblk->jf_dep, jb_deps); 4221 4222 return (jfreeblk); 4223 } 4224 4225 /* 4226 * The journal is only prepared to handle full-size block numbers, so we 4227 * have to adjust the record to reflect the change to a full-size block. 4228 * For example, suppose we have a block made up of fragments 8-15 and 4229 * want to free its last two fragments. We are given a request that says: 4230 * FREEBLK ino=5, blkno=14, lbn=0, frags=2, oldfrags=0 4231 * where frags are the number of fragments to free and oldfrags are the 4232 * number of fragments to keep. To block align it, we have to change it to 4233 * have a valid full-size blkno, so it becomes: 4234 * FREEBLK ino=5, blkno=8, lbn=0, frags=2, oldfrags=6 4235 */ 4236 static void 4237 adjust_newfreework(freeblks, frag_offset) 4238 struct freeblks *freeblks; 4239 int frag_offset; 4240 { 4241 struct jfreeblk *jfreeblk; 4242 4243 KASSERT((LIST_FIRST(&freeblks->fb_jblkdephd) != NULL && 4244 LIST_FIRST(&freeblks->fb_jblkdephd)->jb_list.wk_type == D_JFREEBLK), 4245 ("adjust_newfreework: Missing freeblks dependency")); 4246 4247 jfreeblk = WK_JFREEBLK(LIST_FIRST(&freeblks->fb_jblkdephd)); 4248 jfreeblk->jf_blkno -= frag_offset; 4249 jfreeblk->jf_frags += frag_offset; 4250 } 4251 4252 /* 4253 * Allocate a new jtrunc to track a partial truncation. 4254 */ 4255 static struct jtrunc * 4256 newjtrunc(freeblks, size, extsize) 4257 struct freeblks *freeblks; 4258 off_t size; 4259 int extsize; 4260 { 4261 struct jtrunc *jtrunc; 4262 4263 jtrunc = malloc(sizeof(*jtrunc), M_JTRUNC, M_SOFTDEP_FLAGS); 4264 workitem_alloc(&jtrunc->jt_dep.jb_list, D_JTRUNC, 4265 freeblks->fb_list.wk_mp); 4266 jtrunc->jt_dep.jb_jsegdep = newjsegdep(&jtrunc->jt_dep.jb_list); 4267 jtrunc->jt_dep.jb_freeblks = freeblks; 4268 jtrunc->jt_ino = freeblks->fb_inum; 4269 jtrunc->jt_size = size; 4270 jtrunc->jt_extsize = extsize; 4271 LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jtrunc->jt_dep, jb_deps); 4272 4273 return (jtrunc); 4274 } 4275 4276 /* 4277 * If we're canceling a new bitmap we have to search for another ref 4278 * to move into the bmsafemap dep. This might be better expressed 4279 * with another structure. 4280 */ 4281 static void 4282 move_newblock_dep(jaddref, inodedep) 4283 struct jaddref *jaddref; 4284 struct inodedep *inodedep; 4285 { 4286 struct inoref *inoref; 4287 struct jaddref *jaddrefn; 4288 4289 jaddrefn = NULL; 4290 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 4291 inoref = TAILQ_NEXT(inoref, if_deps)) { 4292 if ((jaddref->ja_state & NEWBLOCK) && 4293 inoref->if_list.wk_type == D_JADDREF) { 4294 jaddrefn = (struct jaddref *)inoref; 4295 break; 4296 } 4297 } 4298 if (jaddrefn == NULL) 4299 return; 4300 jaddrefn->ja_state &= ~(ATTACHED | UNDONE); 4301 jaddrefn->ja_state |= jaddref->ja_state & 4302 (ATTACHED | UNDONE | NEWBLOCK); 4303 jaddref->ja_state &= ~(ATTACHED | UNDONE | NEWBLOCK); 4304 jaddref->ja_state |= ATTACHED; 4305 LIST_REMOVE(jaddref, ja_bmdeps); 4306 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_jaddrefhd, jaddrefn, 4307 ja_bmdeps); 4308 } 4309 4310 /* 4311 * Cancel a jaddref either before it has been written or while it is being 4312 * written. This happens when a link is removed before the add reaches 4313 * the disk. The jaddref dependency is kept linked into the bmsafemap 4314 * and inode to prevent the link count or bitmap from reaching the disk 4315 * until handle_workitem_remove() re-adjusts the counts and bitmaps as 4316 * required. 4317 * 4318 * Returns 1 if the canceled addref requires journaling of the remove and 4319 * 0 otherwise. 4320 */ 4321 static int 4322 cancel_jaddref(jaddref, inodedep, wkhd) 4323 struct jaddref *jaddref; 4324 struct inodedep *inodedep; 4325 struct workhead *wkhd; 4326 { 4327 struct inoref *inoref; 4328 struct jsegdep *jsegdep; 4329 int needsj; 4330 4331 KASSERT((jaddref->ja_state & COMPLETE) == 0, 4332 ("cancel_jaddref: Canceling complete jaddref")); 4333 if (jaddref->ja_state & (INPROGRESS | COMPLETE)) 4334 needsj = 1; 4335 else 4336 needsj = 0; 4337 if (inodedep == NULL) 4338 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 4339 0, &inodedep) == 0) 4340 panic("cancel_jaddref: Lost inodedep"); 4341 /* 4342 * We must adjust the nlink of any reference operation that follows 4343 * us so that it is consistent with the in-memory reference. This 4344 * ensures that inode nlink rollbacks always have the correct link. 4345 */ 4346 if (needsj == 0) { 4347 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 4348 inoref = TAILQ_NEXT(inoref, if_deps)) { 4349 if (inoref->if_state & GOINGAWAY) 4350 break; 4351 inoref->if_nlink--; 4352 } 4353 } 4354 jsegdep = inoref_jseg(&jaddref->ja_ref); 4355 if (jaddref->ja_state & NEWBLOCK) 4356 move_newblock_dep(jaddref, inodedep); 4357 wake_worklist(&jaddref->ja_list); 4358 jaddref->ja_mkdir = NULL; 4359 if (jaddref->ja_state & INPROGRESS) { 4360 jaddref->ja_state &= ~INPROGRESS; 4361 WORKLIST_REMOVE(&jaddref->ja_list); 4362 jwork_insert(wkhd, jsegdep); 4363 } else { 4364 free_jsegdep(jsegdep); 4365 if (jaddref->ja_state & DEPCOMPLETE) 4366 remove_from_journal(&jaddref->ja_list); 4367 } 4368 jaddref->ja_state |= (GOINGAWAY | DEPCOMPLETE); 4369 /* 4370 * Leave NEWBLOCK jaddrefs on the inodedep so handle_workitem_remove 4371 * can arrange for them to be freed with the bitmap. Otherwise we 4372 * no longer need this addref attached to the inoreflst and it 4373 * will incorrectly adjust nlink if we leave it. 4374 */ 4375 if ((jaddref->ja_state & NEWBLOCK) == 0) { 4376 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 4377 if_deps); 4378 jaddref->ja_state |= COMPLETE; 4379 free_jaddref(jaddref); 4380 return (needsj); 4381 } 4382 /* 4383 * Leave the head of the list for jsegdeps for fast merging. 4384 */ 4385 if (LIST_FIRST(wkhd) != NULL) { 4386 jaddref->ja_state |= ONWORKLIST; 4387 LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jaddref->ja_list, wk_list); 4388 } else 4389 WORKLIST_INSERT(wkhd, &jaddref->ja_list); 4390 4391 return (needsj); 4392 } 4393 4394 /* 4395 * Attempt to free a jaddref structure when some work completes. This 4396 * should only succeed once the entry is written and all dependencies have 4397 * been notified. 4398 */ 4399 static void 4400 free_jaddref(jaddref) 4401 struct jaddref *jaddref; 4402 { 4403 4404 if ((jaddref->ja_state & ALLCOMPLETE) != ALLCOMPLETE) 4405 return; 4406 if (jaddref->ja_ref.if_jsegdep) 4407 panic("free_jaddref: segdep attached to jaddref %p(0x%X)\n", 4408 jaddref, jaddref->ja_state); 4409 if (jaddref->ja_state & NEWBLOCK) 4410 LIST_REMOVE(jaddref, ja_bmdeps); 4411 if (jaddref->ja_state & (INPROGRESS | ONWORKLIST)) 4412 panic("free_jaddref: Bad state %p(0x%X)", 4413 jaddref, jaddref->ja_state); 4414 if (jaddref->ja_mkdir != NULL) 4415 panic("free_jaddref: Work pending, 0x%X\n", jaddref->ja_state); 4416 WORKITEM_FREE(jaddref, D_JADDREF); 4417 } 4418 4419 /* 4420 * Free a jremref structure once it has been written or discarded. 4421 */ 4422 static void 4423 free_jremref(jremref) 4424 struct jremref *jremref; 4425 { 4426 4427 if (jremref->jr_ref.if_jsegdep) 4428 free_jsegdep(jremref->jr_ref.if_jsegdep); 4429 if (jremref->jr_state & INPROGRESS) 4430 panic("free_jremref: IO still pending"); 4431 WORKITEM_FREE(jremref, D_JREMREF); 4432 } 4433 4434 /* 4435 * Free a jnewblk structure. 4436 */ 4437 static void 4438 free_jnewblk(jnewblk) 4439 struct jnewblk *jnewblk; 4440 { 4441 4442 if ((jnewblk->jn_state & ALLCOMPLETE) != ALLCOMPLETE) 4443 return; 4444 LIST_REMOVE(jnewblk, jn_deps); 4445 if (jnewblk->jn_dep != NULL) 4446 panic("free_jnewblk: Dependency still attached."); 4447 WORKITEM_FREE(jnewblk, D_JNEWBLK); 4448 } 4449 4450 /* 4451 * Cancel a jnewblk which has been been made redundant by frag extension. 4452 */ 4453 static void 4454 cancel_jnewblk(jnewblk, wkhd) 4455 struct jnewblk *jnewblk; 4456 struct workhead *wkhd; 4457 { 4458 struct jsegdep *jsegdep; 4459 4460 CTR1(KTR_SUJ, "cancel_jnewblk: blkno %jd", jnewblk->jn_blkno); 4461 jsegdep = jnewblk->jn_jsegdep; 4462 if (jnewblk->jn_jsegdep == NULL || jnewblk->jn_dep == NULL) 4463 panic("cancel_jnewblk: Invalid state"); 4464 jnewblk->jn_jsegdep = NULL; 4465 jnewblk->jn_dep = NULL; 4466 jnewblk->jn_state |= GOINGAWAY; 4467 if (jnewblk->jn_state & INPROGRESS) { 4468 jnewblk->jn_state &= ~INPROGRESS; 4469 WORKLIST_REMOVE(&jnewblk->jn_list); 4470 jwork_insert(wkhd, jsegdep); 4471 } else { 4472 free_jsegdep(jsegdep); 4473 remove_from_journal(&jnewblk->jn_list); 4474 } 4475 wake_worklist(&jnewblk->jn_list); 4476 WORKLIST_INSERT(wkhd, &jnewblk->jn_list); 4477 } 4478 4479 static void 4480 free_jblkdep(jblkdep) 4481 struct jblkdep *jblkdep; 4482 { 4483 4484 if (jblkdep->jb_list.wk_type == D_JFREEBLK) 4485 WORKITEM_FREE(jblkdep, D_JFREEBLK); 4486 else if (jblkdep->jb_list.wk_type == D_JTRUNC) 4487 WORKITEM_FREE(jblkdep, D_JTRUNC); 4488 else 4489 panic("free_jblkdep: Unexpected type %s", 4490 TYPENAME(jblkdep->jb_list.wk_type)); 4491 } 4492 4493 /* 4494 * Free a single jseg once it is no longer referenced in memory or on 4495 * disk. Reclaim journal blocks and dependencies waiting for the segment 4496 * to disappear. 4497 */ 4498 static void 4499 free_jseg(jseg, jblocks) 4500 struct jseg *jseg; 4501 struct jblocks *jblocks; 4502 { 4503 struct freework *freework; 4504 4505 /* 4506 * Free freework structures that were lingering to indicate freed 4507 * indirect blocks that forced journal write ordering on reallocate. 4508 */ 4509 while ((freework = LIST_FIRST(&jseg->js_indirs)) != NULL) 4510 indirblk_remove(freework); 4511 if (jblocks->jb_oldestseg == jseg) 4512 jblocks->jb_oldestseg = TAILQ_NEXT(jseg, js_next); 4513 TAILQ_REMOVE(&jblocks->jb_segs, jseg, js_next); 4514 jblocks_free(jblocks, jseg->js_list.wk_mp, jseg->js_size); 4515 KASSERT(LIST_EMPTY(&jseg->js_entries), 4516 ("free_jseg: Freed jseg has valid entries.")); 4517 WORKITEM_FREE(jseg, D_JSEG); 4518 } 4519 4520 /* 4521 * Free all jsegs that meet the criteria for being reclaimed and update 4522 * oldestseg. 4523 */ 4524 static void 4525 free_jsegs(jblocks) 4526 struct jblocks *jblocks; 4527 { 4528 struct jseg *jseg; 4529 4530 /* 4531 * Free only those jsegs which have none allocated before them to 4532 * preserve the journal space ordering. 4533 */ 4534 while ((jseg = TAILQ_FIRST(&jblocks->jb_segs)) != NULL) { 4535 /* 4536 * Only reclaim space when nothing depends on this journal 4537 * set and another set has written that it is no longer 4538 * valid. 4539 */ 4540 if (jseg->js_refs != 0) { 4541 jblocks->jb_oldestseg = jseg; 4542 return; 4543 } 4544 if ((jseg->js_state & ALLCOMPLETE) != ALLCOMPLETE) 4545 break; 4546 if (jseg->js_seq > jblocks->jb_oldestwrseq) 4547 break; 4548 /* 4549 * We can free jsegs that didn't write entries when 4550 * oldestwrseq == js_seq. 4551 */ 4552 if (jseg->js_seq == jblocks->jb_oldestwrseq && 4553 jseg->js_cnt != 0) 4554 break; 4555 free_jseg(jseg, jblocks); 4556 } 4557 /* 4558 * If we exited the loop above we still must discover the 4559 * oldest valid segment. 4560 */ 4561 if (jseg) 4562 for (jseg = jblocks->jb_oldestseg; jseg != NULL; 4563 jseg = TAILQ_NEXT(jseg, js_next)) 4564 if (jseg->js_refs != 0) 4565 break; 4566 jblocks->jb_oldestseg = jseg; 4567 /* 4568 * The journal has no valid records but some jsegs may still be 4569 * waiting on oldestwrseq to advance. We force a small record 4570 * out to permit these lingering records to be reclaimed. 4571 */ 4572 if (jblocks->jb_oldestseg == NULL && !TAILQ_EMPTY(&jblocks->jb_segs)) 4573 jblocks->jb_needseg = 1; 4574 } 4575 4576 /* 4577 * Release one reference to a jseg and free it if the count reaches 0. This 4578 * should eventually reclaim journal space as well. 4579 */ 4580 static void 4581 rele_jseg(jseg) 4582 struct jseg *jseg; 4583 { 4584 4585 KASSERT(jseg->js_refs > 0, 4586 ("free_jseg: Invalid refcnt %d", jseg->js_refs)); 4587 if (--jseg->js_refs != 0) 4588 return; 4589 free_jsegs(jseg->js_jblocks); 4590 } 4591 4592 /* 4593 * Release a jsegdep and decrement the jseg count. 4594 */ 4595 static void 4596 free_jsegdep(jsegdep) 4597 struct jsegdep *jsegdep; 4598 { 4599 4600 if (jsegdep->jd_seg) 4601 rele_jseg(jsegdep->jd_seg); 4602 WORKITEM_FREE(jsegdep, D_JSEGDEP); 4603 } 4604 4605 /* 4606 * Wait for a journal item to make it to disk. Initiate journal processing 4607 * if required. 4608 */ 4609 static int 4610 jwait(wk, waitfor) 4611 struct worklist *wk; 4612 int waitfor; 4613 { 4614 4615 LOCK_OWNED(VFSTOUFS(wk->wk_mp)); 4616 /* 4617 * Blocking journal waits cause slow synchronous behavior. Record 4618 * stats on the frequency of these blocking operations. 4619 */ 4620 if (waitfor == MNT_WAIT) { 4621 stat_journal_wait++; 4622 switch (wk->wk_type) { 4623 case D_JREMREF: 4624 case D_JMVREF: 4625 stat_jwait_filepage++; 4626 break; 4627 case D_JTRUNC: 4628 case D_JFREEBLK: 4629 stat_jwait_freeblks++; 4630 break; 4631 case D_JNEWBLK: 4632 stat_jwait_newblk++; 4633 break; 4634 case D_JADDREF: 4635 stat_jwait_inode++; 4636 break; 4637 default: 4638 break; 4639 } 4640 } 4641 /* 4642 * If IO has not started we process the journal. We can't mark the 4643 * worklist item as IOWAITING because we drop the lock while 4644 * processing the journal and the worklist entry may be freed after 4645 * this point. The caller may call back in and re-issue the request. 4646 */ 4647 if ((wk->wk_state & INPROGRESS) == 0) { 4648 softdep_process_journal(wk->wk_mp, wk, waitfor); 4649 if (waitfor != MNT_WAIT) 4650 return (EBUSY); 4651 return (0); 4652 } 4653 if (waitfor != MNT_WAIT) 4654 return (EBUSY); 4655 wait_worklist(wk, "jwait"); 4656 return (0); 4657 } 4658 4659 /* 4660 * Lookup an inodedep based on an inode pointer and set the nlinkdelta as 4661 * appropriate. This is a convenience function to reduce duplicate code 4662 * for the setup and revert functions below. 4663 */ 4664 static struct inodedep * 4665 inodedep_lookup_ip(ip) 4666 struct inode *ip; 4667 { 4668 struct inodedep *inodedep; 4669 4670 KASSERT(ip->i_nlink >= ip->i_effnlink, 4671 ("inodedep_lookup_ip: bad delta")); 4672 (void) inodedep_lookup(ITOVFS(ip), ip->i_number, DEPALLOC, 4673 &inodedep); 4674 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 4675 KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked")); 4676 4677 return (inodedep); 4678 } 4679 4680 /* 4681 * Called prior to creating a new inode and linking it to a directory. The 4682 * jaddref structure must already be allocated by softdep_setup_inomapdep 4683 * and it is discovered here so we can initialize the mode and update 4684 * nlinkdelta. 4685 */ 4686 void 4687 softdep_setup_create(dp, ip) 4688 struct inode *dp; 4689 struct inode *ip; 4690 { 4691 struct inodedep *inodedep; 4692 struct jaddref *jaddref; 4693 struct vnode *dvp; 4694 4695 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4696 ("softdep_setup_create called on non-softdep filesystem")); 4697 KASSERT(ip->i_nlink == 1, 4698 ("softdep_setup_create: Invalid link count.")); 4699 dvp = ITOV(dp); 4700 ACQUIRE_LOCK(ITOUMP(dp)); 4701 inodedep = inodedep_lookup_ip(ip); 4702 if (DOINGSUJ(dvp)) { 4703 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4704 inoreflst); 4705 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 4706 ("softdep_setup_create: No addref structure present.")); 4707 } 4708 softdep_prelink(dvp, NULL); 4709 FREE_LOCK(ITOUMP(dp)); 4710 } 4711 4712 /* 4713 * Create a jaddref structure to track the addition of a DOTDOT link when 4714 * we are reparenting an inode as part of a rename. This jaddref will be 4715 * found by softdep_setup_directory_change. Adjusts nlinkdelta for 4716 * non-journaling softdep. 4717 */ 4718 void 4719 softdep_setup_dotdot_link(dp, ip) 4720 struct inode *dp; 4721 struct inode *ip; 4722 { 4723 struct inodedep *inodedep; 4724 struct jaddref *jaddref; 4725 struct vnode *dvp; 4726 4727 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4728 ("softdep_setup_dotdot_link called on non-softdep filesystem")); 4729 dvp = ITOV(dp); 4730 jaddref = NULL; 4731 /* 4732 * We don't set MKDIR_PARENT as this is not tied to a mkdir and 4733 * is used as a normal link would be. 4734 */ 4735 if (DOINGSUJ(dvp)) 4736 jaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 4737 dp->i_effnlink - 1, dp->i_mode); 4738 ACQUIRE_LOCK(ITOUMP(dp)); 4739 inodedep = inodedep_lookup_ip(dp); 4740 if (jaddref) 4741 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 4742 if_deps); 4743 softdep_prelink(dvp, ITOV(ip)); 4744 FREE_LOCK(ITOUMP(dp)); 4745 } 4746 4747 /* 4748 * Create a jaddref structure to track a new link to an inode. The directory 4749 * offset is not known until softdep_setup_directory_add or 4750 * softdep_setup_directory_change. Adjusts nlinkdelta for non-journaling 4751 * softdep. 4752 */ 4753 void 4754 softdep_setup_link(dp, ip) 4755 struct inode *dp; 4756 struct inode *ip; 4757 { 4758 struct inodedep *inodedep; 4759 struct jaddref *jaddref; 4760 struct vnode *dvp; 4761 4762 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4763 ("softdep_setup_link called on non-softdep filesystem")); 4764 dvp = ITOV(dp); 4765 jaddref = NULL; 4766 if (DOINGSUJ(dvp)) 4767 jaddref = newjaddref(dp, ip->i_number, 0, ip->i_effnlink - 1, 4768 ip->i_mode); 4769 ACQUIRE_LOCK(ITOUMP(dp)); 4770 inodedep = inodedep_lookup_ip(ip); 4771 if (jaddref) 4772 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 4773 if_deps); 4774 softdep_prelink(dvp, ITOV(ip)); 4775 FREE_LOCK(ITOUMP(dp)); 4776 } 4777 4778 /* 4779 * Called to create the jaddref structures to track . and .. references as 4780 * well as lookup and further initialize the incomplete jaddref created 4781 * by softdep_setup_inomapdep when the inode was allocated. Adjusts 4782 * nlinkdelta for non-journaling softdep. 4783 */ 4784 void 4785 softdep_setup_mkdir(dp, ip) 4786 struct inode *dp; 4787 struct inode *ip; 4788 { 4789 struct inodedep *inodedep; 4790 struct jaddref *dotdotaddref; 4791 struct jaddref *dotaddref; 4792 struct jaddref *jaddref; 4793 struct vnode *dvp; 4794 4795 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4796 ("softdep_setup_mkdir called on non-softdep filesystem")); 4797 dvp = ITOV(dp); 4798 dotaddref = dotdotaddref = NULL; 4799 if (DOINGSUJ(dvp)) { 4800 dotaddref = newjaddref(ip, ip->i_number, DOT_OFFSET, 1, 4801 ip->i_mode); 4802 dotaddref->ja_state |= MKDIR_BODY; 4803 dotdotaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 4804 dp->i_effnlink - 1, dp->i_mode); 4805 dotdotaddref->ja_state |= MKDIR_PARENT; 4806 } 4807 ACQUIRE_LOCK(ITOUMP(dp)); 4808 inodedep = inodedep_lookup_ip(ip); 4809 if (DOINGSUJ(dvp)) { 4810 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4811 inoreflst); 4812 KASSERT(jaddref != NULL, 4813 ("softdep_setup_mkdir: No addref structure present.")); 4814 KASSERT(jaddref->ja_parent == dp->i_number, 4815 ("softdep_setup_mkdir: bad parent %ju", 4816 (uintmax_t)jaddref->ja_parent)); 4817 TAILQ_INSERT_BEFORE(&jaddref->ja_ref, &dotaddref->ja_ref, 4818 if_deps); 4819 } 4820 inodedep = inodedep_lookup_ip(dp); 4821 if (DOINGSUJ(dvp)) 4822 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, 4823 &dotdotaddref->ja_ref, if_deps); 4824 softdep_prelink(ITOV(dp), NULL); 4825 FREE_LOCK(ITOUMP(dp)); 4826 } 4827 4828 /* 4829 * Called to track nlinkdelta of the inode and parent directories prior to 4830 * unlinking a directory. 4831 */ 4832 void 4833 softdep_setup_rmdir(dp, ip) 4834 struct inode *dp; 4835 struct inode *ip; 4836 { 4837 struct vnode *dvp; 4838 4839 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4840 ("softdep_setup_rmdir called on non-softdep filesystem")); 4841 dvp = ITOV(dp); 4842 ACQUIRE_LOCK(ITOUMP(dp)); 4843 (void) inodedep_lookup_ip(ip); 4844 (void) inodedep_lookup_ip(dp); 4845 softdep_prelink(dvp, ITOV(ip)); 4846 FREE_LOCK(ITOUMP(dp)); 4847 } 4848 4849 /* 4850 * Called to track nlinkdelta of the inode and parent directories prior to 4851 * unlink. 4852 */ 4853 void 4854 softdep_setup_unlink(dp, ip) 4855 struct inode *dp; 4856 struct inode *ip; 4857 { 4858 struct vnode *dvp; 4859 4860 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4861 ("softdep_setup_unlink called on non-softdep filesystem")); 4862 dvp = ITOV(dp); 4863 ACQUIRE_LOCK(ITOUMP(dp)); 4864 (void) inodedep_lookup_ip(ip); 4865 (void) inodedep_lookup_ip(dp); 4866 softdep_prelink(dvp, ITOV(ip)); 4867 FREE_LOCK(ITOUMP(dp)); 4868 } 4869 4870 /* 4871 * Called to release the journal structures created by a failed non-directory 4872 * creation. Adjusts nlinkdelta for non-journaling softdep. 4873 */ 4874 void 4875 softdep_revert_create(dp, ip) 4876 struct inode *dp; 4877 struct inode *ip; 4878 { 4879 struct inodedep *inodedep; 4880 struct jaddref *jaddref; 4881 struct vnode *dvp; 4882 4883 KASSERT(MOUNTEDSOFTDEP(ITOVFS((dp))) != 0, 4884 ("softdep_revert_create called on non-softdep filesystem")); 4885 dvp = ITOV(dp); 4886 ACQUIRE_LOCK(ITOUMP(dp)); 4887 inodedep = inodedep_lookup_ip(ip); 4888 if (DOINGSUJ(dvp)) { 4889 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4890 inoreflst); 4891 KASSERT(jaddref->ja_parent == dp->i_number, 4892 ("softdep_revert_create: addref parent mismatch")); 4893 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4894 } 4895 FREE_LOCK(ITOUMP(dp)); 4896 } 4897 4898 /* 4899 * Called to release the journal structures created by a failed link 4900 * addition. Adjusts nlinkdelta for non-journaling softdep. 4901 */ 4902 void 4903 softdep_revert_link(dp, ip) 4904 struct inode *dp; 4905 struct inode *ip; 4906 { 4907 struct inodedep *inodedep; 4908 struct jaddref *jaddref; 4909 struct vnode *dvp; 4910 4911 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4912 ("softdep_revert_link called on non-softdep filesystem")); 4913 dvp = ITOV(dp); 4914 ACQUIRE_LOCK(ITOUMP(dp)); 4915 inodedep = inodedep_lookup_ip(ip); 4916 if (DOINGSUJ(dvp)) { 4917 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4918 inoreflst); 4919 KASSERT(jaddref->ja_parent == dp->i_number, 4920 ("softdep_revert_link: addref parent mismatch")); 4921 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4922 } 4923 FREE_LOCK(ITOUMP(dp)); 4924 } 4925 4926 /* 4927 * Called to release the journal structures created by a failed mkdir 4928 * attempt. Adjusts nlinkdelta for non-journaling softdep. 4929 */ 4930 void 4931 softdep_revert_mkdir(dp, ip) 4932 struct inode *dp; 4933 struct inode *ip; 4934 { 4935 struct inodedep *inodedep; 4936 struct jaddref *jaddref; 4937 struct jaddref *dotaddref; 4938 struct vnode *dvp; 4939 4940 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4941 ("softdep_revert_mkdir called on non-softdep filesystem")); 4942 dvp = ITOV(dp); 4943 4944 ACQUIRE_LOCK(ITOUMP(dp)); 4945 inodedep = inodedep_lookup_ip(dp); 4946 if (DOINGSUJ(dvp)) { 4947 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4948 inoreflst); 4949 KASSERT(jaddref->ja_parent == ip->i_number, 4950 ("softdep_revert_mkdir: dotdot addref parent mismatch")); 4951 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4952 } 4953 inodedep = inodedep_lookup_ip(ip); 4954 if (DOINGSUJ(dvp)) { 4955 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4956 inoreflst); 4957 KASSERT(jaddref->ja_parent == dp->i_number, 4958 ("softdep_revert_mkdir: addref parent mismatch")); 4959 dotaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref, 4960 inoreflst, if_deps); 4961 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4962 KASSERT(dotaddref->ja_parent == ip->i_number, 4963 ("softdep_revert_mkdir: dot addref parent mismatch")); 4964 cancel_jaddref(dotaddref, inodedep, &inodedep->id_inowait); 4965 } 4966 FREE_LOCK(ITOUMP(dp)); 4967 } 4968 4969 /* 4970 * Called to correct nlinkdelta after a failed rmdir. 4971 */ 4972 void 4973 softdep_revert_rmdir(dp, ip) 4974 struct inode *dp; 4975 struct inode *ip; 4976 { 4977 4978 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4979 ("softdep_revert_rmdir called on non-softdep filesystem")); 4980 ACQUIRE_LOCK(ITOUMP(dp)); 4981 (void) inodedep_lookup_ip(ip); 4982 (void) inodedep_lookup_ip(dp); 4983 FREE_LOCK(ITOUMP(dp)); 4984 } 4985 4986 /* 4987 * Protecting the freemaps (or bitmaps). 4988 * 4989 * To eliminate the need to execute fsck before mounting a filesystem 4990 * after a power failure, one must (conservatively) guarantee that the 4991 * on-disk copy of the bitmaps never indicate that a live inode or block is 4992 * free. So, when a block or inode is allocated, the bitmap should be 4993 * updated (on disk) before any new pointers. When a block or inode is 4994 * freed, the bitmap should not be updated until all pointers have been 4995 * reset. The latter dependency is handled by the delayed de-allocation 4996 * approach described below for block and inode de-allocation. The former 4997 * dependency is handled by calling the following procedure when a block or 4998 * inode is allocated. When an inode is allocated an "inodedep" is created 4999 * with its DEPCOMPLETE flag cleared until its bitmap is written to disk. 5000 * Each "inodedep" is also inserted into the hash indexing structure so 5001 * that any additional link additions can be made dependent on the inode 5002 * allocation. 5003 * 5004 * The ufs filesystem maintains a number of free block counts (e.g., per 5005 * cylinder group, per cylinder and per <cylinder, rotational position> pair) 5006 * in addition to the bitmaps. These counts are used to improve efficiency 5007 * during allocation and therefore must be consistent with the bitmaps. 5008 * There is no convenient way to guarantee post-crash consistency of these 5009 * counts with simple update ordering, for two main reasons: (1) The counts 5010 * and bitmaps for a single cylinder group block are not in the same disk 5011 * sector. If a disk write is interrupted (e.g., by power failure), one may 5012 * be written and the other not. (2) Some of the counts are located in the 5013 * superblock rather than the cylinder group block. So, we focus our soft 5014 * updates implementation on protecting the bitmaps. When mounting a 5015 * filesystem, we recompute the auxiliary counts from the bitmaps. 5016 */ 5017 5018 /* 5019 * Called just after updating the cylinder group block to allocate an inode. 5020 */ 5021 void 5022 softdep_setup_inomapdep(bp, ip, newinum, mode) 5023 struct buf *bp; /* buffer for cylgroup block with inode map */ 5024 struct inode *ip; /* inode related to allocation */ 5025 ino_t newinum; /* new inode number being allocated */ 5026 int mode; 5027 { 5028 struct inodedep *inodedep; 5029 struct bmsafemap *bmsafemap; 5030 struct jaddref *jaddref; 5031 struct mount *mp; 5032 struct fs *fs; 5033 5034 mp = ITOVFS(ip); 5035 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5036 ("softdep_setup_inomapdep called on non-softdep filesystem")); 5037 fs = VFSTOUFS(mp)->um_fs; 5038 jaddref = NULL; 5039 5040 /* 5041 * Allocate the journal reference add structure so that the bitmap 5042 * can be dependent on it. 5043 */ 5044 if (MOUNTEDSUJ(mp)) { 5045 jaddref = newjaddref(ip, newinum, 0, 0, mode); 5046 jaddref->ja_state |= NEWBLOCK; 5047 } 5048 5049 /* 5050 * Create a dependency for the newly allocated inode. 5051 * Panic if it already exists as something is seriously wrong. 5052 * Otherwise add it to the dependency list for the buffer holding 5053 * the cylinder group map from which it was allocated. 5054 * 5055 * We have to preallocate a bmsafemap entry in case it is needed 5056 * in bmsafemap_lookup since once we allocate the inodedep, we 5057 * have to finish initializing it before we can FREE_LOCK(). 5058 * By preallocating, we avoid FREE_LOCK() while doing a malloc 5059 * in bmsafemap_lookup. We cannot call bmsafemap_lookup before 5060 * creating the inodedep as it can be freed during the time 5061 * that we FREE_LOCK() while allocating the inodedep. We must 5062 * call workitem_alloc() before entering the locked section as 5063 * it also acquires the lock and we must avoid trying doing so 5064 * recursively. 5065 */ 5066 bmsafemap = malloc(sizeof(struct bmsafemap), 5067 M_BMSAFEMAP, M_SOFTDEP_FLAGS); 5068 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); 5069 ACQUIRE_LOCK(ITOUMP(ip)); 5070 if ((inodedep_lookup(mp, newinum, DEPALLOC, &inodedep))) 5071 panic("softdep_setup_inomapdep: dependency %p for new" 5072 "inode already exists", inodedep); 5073 bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum), bmsafemap); 5074 if (jaddref) { 5075 LIST_INSERT_HEAD(&bmsafemap->sm_jaddrefhd, jaddref, ja_bmdeps); 5076 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 5077 if_deps); 5078 } else { 5079 inodedep->id_state |= ONDEPLIST; 5080 LIST_INSERT_HEAD(&bmsafemap->sm_inodedephd, inodedep, id_deps); 5081 } 5082 inodedep->id_bmsafemap = bmsafemap; 5083 inodedep->id_state &= ~DEPCOMPLETE; 5084 FREE_LOCK(ITOUMP(ip)); 5085 } 5086 5087 /* 5088 * Called just after updating the cylinder group block to 5089 * allocate block or fragment. 5090 */ 5091 void 5092 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 5093 struct buf *bp; /* buffer for cylgroup block with block map */ 5094 struct mount *mp; /* filesystem doing allocation */ 5095 ufs2_daddr_t newblkno; /* number of newly allocated block */ 5096 int frags; /* Number of fragments. */ 5097 int oldfrags; /* Previous number of fragments for extend. */ 5098 { 5099 struct newblk *newblk; 5100 struct bmsafemap *bmsafemap; 5101 struct jnewblk *jnewblk; 5102 struct ufsmount *ump; 5103 struct fs *fs; 5104 5105 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5106 ("softdep_setup_blkmapdep called on non-softdep filesystem")); 5107 ump = VFSTOUFS(mp); 5108 fs = ump->um_fs; 5109 jnewblk = NULL; 5110 /* 5111 * Create a dependency for the newly allocated block. 5112 * Add it to the dependency list for the buffer holding 5113 * the cylinder group map from which it was allocated. 5114 */ 5115 if (MOUNTEDSUJ(mp)) { 5116 jnewblk = malloc(sizeof(*jnewblk), M_JNEWBLK, M_SOFTDEP_FLAGS); 5117 workitem_alloc(&jnewblk->jn_list, D_JNEWBLK, mp); 5118 jnewblk->jn_jsegdep = newjsegdep(&jnewblk->jn_list); 5119 jnewblk->jn_state = ATTACHED; 5120 jnewblk->jn_blkno = newblkno; 5121 jnewblk->jn_frags = frags; 5122 jnewblk->jn_oldfrags = oldfrags; 5123 #ifdef INVARIANTS 5124 { 5125 struct cg *cgp; 5126 uint8_t *blksfree; 5127 long bno; 5128 int i; 5129 5130 cgp = (struct cg *)bp->b_data; 5131 blksfree = cg_blksfree(cgp); 5132 bno = dtogd(fs, jnewblk->jn_blkno); 5133 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 5134 i++) { 5135 if (isset(blksfree, bno + i)) 5136 panic("softdep_setup_blkmapdep: " 5137 "free fragment %d from %d-%d " 5138 "state 0x%X dep %p", i, 5139 jnewblk->jn_oldfrags, 5140 jnewblk->jn_frags, 5141 jnewblk->jn_state, 5142 jnewblk->jn_dep); 5143 } 5144 } 5145 #endif 5146 } 5147 5148 CTR3(KTR_SUJ, 5149 "softdep_setup_blkmapdep: blkno %jd frags %d oldfrags %d", 5150 newblkno, frags, oldfrags); 5151 ACQUIRE_LOCK(ump); 5152 if (newblk_lookup(mp, newblkno, DEPALLOC, &newblk) != 0) 5153 panic("softdep_setup_blkmapdep: found block"); 5154 newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(mp, bp, 5155 dtog(fs, newblkno), NULL); 5156 if (jnewblk) { 5157 jnewblk->jn_dep = (struct worklist *)newblk; 5158 LIST_INSERT_HEAD(&bmsafemap->sm_jnewblkhd, jnewblk, jn_deps); 5159 } else { 5160 newblk->nb_state |= ONDEPLIST; 5161 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps); 5162 } 5163 newblk->nb_bmsafemap = bmsafemap; 5164 newblk->nb_jnewblk = jnewblk; 5165 FREE_LOCK(ump); 5166 } 5167 5168 #define BMSAFEMAP_HASH(ump, cg) \ 5169 (&(ump)->bmsafemap_hashtbl[(cg) & (ump)->bmsafemap_hash_size]) 5170 5171 static int 5172 bmsafemap_find(bmsafemaphd, cg, bmsafemapp) 5173 struct bmsafemap_hashhead *bmsafemaphd; 5174 int cg; 5175 struct bmsafemap **bmsafemapp; 5176 { 5177 struct bmsafemap *bmsafemap; 5178 5179 LIST_FOREACH(bmsafemap, bmsafemaphd, sm_hash) 5180 if (bmsafemap->sm_cg == cg) 5181 break; 5182 if (bmsafemap) { 5183 *bmsafemapp = bmsafemap; 5184 return (1); 5185 } 5186 *bmsafemapp = NULL; 5187 5188 return (0); 5189 } 5190 5191 /* 5192 * Find the bmsafemap associated with a cylinder group buffer. 5193 * If none exists, create one. The buffer must be locked when 5194 * this routine is called and this routine must be called with 5195 * the softdep lock held. To avoid giving up the lock while 5196 * allocating a new bmsafemap, a preallocated bmsafemap may be 5197 * provided. If it is provided but not needed, it is freed. 5198 */ 5199 static struct bmsafemap * 5200 bmsafemap_lookup(mp, bp, cg, newbmsafemap) 5201 struct mount *mp; 5202 struct buf *bp; 5203 int cg; 5204 struct bmsafemap *newbmsafemap; 5205 { 5206 struct bmsafemap_hashhead *bmsafemaphd; 5207 struct bmsafemap *bmsafemap, *collision; 5208 struct worklist *wk; 5209 struct ufsmount *ump; 5210 5211 ump = VFSTOUFS(mp); 5212 LOCK_OWNED(ump); 5213 KASSERT(bp != NULL, ("bmsafemap_lookup: missing buffer")); 5214 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 5215 if (wk->wk_type == D_BMSAFEMAP) { 5216 if (newbmsafemap) 5217 WORKITEM_FREE(newbmsafemap, D_BMSAFEMAP); 5218 return (WK_BMSAFEMAP(wk)); 5219 } 5220 } 5221 bmsafemaphd = BMSAFEMAP_HASH(ump, cg); 5222 if (bmsafemap_find(bmsafemaphd, cg, &bmsafemap) == 1) { 5223 if (newbmsafemap) 5224 WORKITEM_FREE(newbmsafemap, D_BMSAFEMAP); 5225 return (bmsafemap); 5226 } 5227 if (newbmsafemap) { 5228 bmsafemap = newbmsafemap; 5229 } else { 5230 FREE_LOCK(ump); 5231 bmsafemap = malloc(sizeof(struct bmsafemap), 5232 M_BMSAFEMAP, M_SOFTDEP_FLAGS); 5233 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); 5234 ACQUIRE_LOCK(ump); 5235 } 5236 bmsafemap->sm_buf = bp; 5237 LIST_INIT(&bmsafemap->sm_inodedephd); 5238 LIST_INIT(&bmsafemap->sm_inodedepwr); 5239 LIST_INIT(&bmsafemap->sm_newblkhd); 5240 LIST_INIT(&bmsafemap->sm_newblkwr); 5241 LIST_INIT(&bmsafemap->sm_jaddrefhd); 5242 LIST_INIT(&bmsafemap->sm_jnewblkhd); 5243 LIST_INIT(&bmsafemap->sm_freehd); 5244 LIST_INIT(&bmsafemap->sm_freewr); 5245 if (bmsafemap_find(bmsafemaphd, cg, &collision) == 1) { 5246 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 5247 return (collision); 5248 } 5249 bmsafemap->sm_cg = cg; 5250 LIST_INSERT_HEAD(bmsafemaphd, bmsafemap, sm_hash); 5251 LIST_INSERT_HEAD(&ump->softdep_dirtycg, bmsafemap, sm_next); 5252 WORKLIST_INSERT(&bp->b_dep, &bmsafemap->sm_list); 5253 return (bmsafemap); 5254 } 5255 5256 /* 5257 * Direct block allocation dependencies. 5258 * 5259 * When a new block is allocated, the corresponding disk locations must be 5260 * initialized (with zeros or new data) before the on-disk inode points to 5261 * them. Also, the freemap from which the block was allocated must be 5262 * updated (on disk) before the inode's pointer. These two dependencies are 5263 * independent of each other and are needed for all file blocks and indirect 5264 * blocks that are pointed to directly by the inode. Just before the 5265 * "in-core" version of the inode is updated with a newly allocated block 5266 * number, a procedure (below) is called to setup allocation dependency 5267 * structures. These structures are removed when the corresponding 5268 * dependencies are satisfied or when the block allocation becomes obsolete 5269 * (i.e., the file is deleted, the block is de-allocated, or the block is a 5270 * fragment that gets upgraded). All of these cases are handled in 5271 * procedures described later. 5272 * 5273 * When a file extension causes a fragment to be upgraded, either to a larger 5274 * fragment or to a full block, the on-disk location may change (if the 5275 * previous fragment could not simply be extended). In this case, the old 5276 * fragment must be de-allocated, but not until after the inode's pointer has 5277 * been updated. In most cases, this is handled by later procedures, which 5278 * will construct a "freefrag" structure to be added to the workitem queue 5279 * when the inode update is complete (or obsolete). The main exception to 5280 * this is when an allocation occurs while a pending allocation dependency 5281 * (for the same block pointer) remains. This case is handled in the main 5282 * allocation dependency setup procedure by immediately freeing the 5283 * unreferenced fragments. 5284 */ 5285 void 5286 softdep_setup_allocdirect(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 5287 struct inode *ip; /* inode to which block is being added */ 5288 ufs_lbn_t off; /* block pointer within inode */ 5289 ufs2_daddr_t newblkno; /* disk block number being added */ 5290 ufs2_daddr_t oldblkno; /* previous block number, 0 unless frag */ 5291 long newsize; /* size of new block */ 5292 long oldsize; /* size of new block */ 5293 struct buf *bp; /* bp for allocated block */ 5294 { 5295 struct allocdirect *adp, *oldadp; 5296 struct allocdirectlst *adphead; 5297 struct freefrag *freefrag; 5298 struct inodedep *inodedep; 5299 struct pagedep *pagedep; 5300 struct jnewblk *jnewblk; 5301 struct newblk *newblk; 5302 struct mount *mp; 5303 ufs_lbn_t lbn; 5304 5305 lbn = bp->b_lblkno; 5306 mp = ITOVFS(ip); 5307 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5308 ("softdep_setup_allocdirect called on non-softdep filesystem")); 5309 if (oldblkno && oldblkno != newblkno) 5310 /* 5311 * The usual case is that a smaller fragment that 5312 * was just allocated has been replaced with a bigger 5313 * fragment or a full-size block. If it is marked as 5314 * B_DELWRI, the current contents have not been written 5315 * to disk. It is possible that the block was written 5316 * earlier, but very uncommon. If the block has never 5317 * been written, there is no need to send a BIO_DELETE 5318 * for it when it is freed. The gain from avoiding the 5319 * TRIMs for the common case of unwritten blocks far 5320 * exceeds the cost of the write amplification for the 5321 * uncommon case of failing to send a TRIM for a block 5322 * that had been written. 5323 */ 5324 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn, 5325 (bp->b_flags & B_DELWRI) != 0 ? NOTRIM_KEY : SINGLETON_KEY); 5326 else 5327 freefrag = NULL; 5328 5329 CTR6(KTR_SUJ, 5330 "softdep_setup_allocdirect: ino %d blkno %jd oldblkno %jd " 5331 "off %jd newsize %ld oldsize %d", 5332 ip->i_number, newblkno, oldblkno, off, newsize, oldsize); 5333 ACQUIRE_LOCK(ITOUMP(ip)); 5334 if (off >= UFS_NDADDR) { 5335 if (lbn > 0) 5336 panic("softdep_setup_allocdirect: bad lbn %jd, off %jd", 5337 lbn, off); 5338 /* allocating an indirect block */ 5339 if (oldblkno != 0) 5340 panic("softdep_setup_allocdirect: non-zero indir"); 5341 } else { 5342 if (off != lbn) 5343 panic("softdep_setup_allocdirect: lbn %jd != off %jd", 5344 lbn, off); 5345 /* 5346 * Allocating a direct block. 5347 * 5348 * If we are allocating a directory block, then we must 5349 * allocate an associated pagedep to track additions and 5350 * deletions. 5351 */ 5352 if ((ip->i_mode & IFMT) == IFDIR) 5353 pagedep_lookup(mp, bp, ip->i_number, off, DEPALLOC, 5354 &pagedep); 5355 } 5356 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 5357 panic("softdep_setup_allocdirect: lost block"); 5358 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 5359 ("softdep_setup_allocdirect: newblk already initialized")); 5360 /* 5361 * Convert the newblk to an allocdirect. 5362 */ 5363 WORKITEM_REASSIGN(newblk, D_ALLOCDIRECT); 5364 adp = (struct allocdirect *)newblk; 5365 newblk->nb_freefrag = freefrag; 5366 adp->ad_offset = off; 5367 adp->ad_oldblkno = oldblkno; 5368 adp->ad_newsize = newsize; 5369 adp->ad_oldsize = oldsize; 5370 5371 /* 5372 * Finish initializing the journal. 5373 */ 5374 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 5375 jnewblk->jn_ino = ip->i_number; 5376 jnewblk->jn_lbn = lbn; 5377 add_to_journal(&jnewblk->jn_list); 5378 } 5379 if (freefrag && freefrag->ff_jdep != NULL && 5380 freefrag->ff_jdep->wk_type == D_JFREEFRAG) 5381 add_to_journal(freefrag->ff_jdep); 5382 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5383 adp->ad_inodedep = inodedep; 5384 5385 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 5386 /* 5387 * The list of allocdirects must be kept in sorted and ascending 5388 * order so that the rollback routines can quickly determine the 5389 * first uncommitted block (the size of the file stored on disk 5390 * ends at the end of the lowest committed fragment, or if there 5391 * are no fragments, at the end of the highest committed block). 5392 * Since files generally grow, the typical case is that the new 5393 * block is to be added at the end of the list. We speed this 5394 * special case by checking against the last allocdirect in the 5395 * list before laboriously traversing the list looking for the 5396 * insertion point. 5397 */ 5398 adphead = &inodedep->id_newinoupdt; 5399 oldadp = TAILQ_LAST(adphead, allocdirectlst); 5400 if (oldadp == NULL || oldadp->ad_offset <= off) { 5401 /* insert at end of list */ 5402 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 5403 if (oldadp != NULL && oldadp->ad_offset == off) 5404 allocdirect_merge(adphead, adp, oldadp); 5405 FREE_LOCK(ITOUMP(ip)); 5406 return; 5407 } 5408 TAILQ_FOREACH(oldadp, adphead, ad_next) { 5409 if (oldadp->ad_offset >= off) 5410 break; 5411 } 5412 if (oldadp == NULL) 5413 panic("softdep_setup_allocdirect: lost entry"); 5414 /* insert in middle of list */ 5415 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 5416 if (oldadp->ad_offset == off) 5417 allocdirect_merge(adphead, adp, oldadp); 5418 5419 FREE_LOCK(ITOUMP(ip)); 5420 } 5421 5422 /* 5423 * Merge a newer and older journal record to be stored either in a 5424 * newblock or freefrag. This handles aggregating journal records for 5425 * fragment allocation into a second record as well as replacing a 5426 * journal free with an aborted journal allocation. A segment for the 5427 * oldest record will be placed on wkhd if it has been written. If not 5428 * the segment for the newer record will suffice. 5429 */ 5430 static struct worklist * 5431 jnewblk_merge(new, old, wkhd) 5432 struct worklist *new; 5433 struct worklist *old; 5434 struct workhead *wkhd; 5435 { 5436 struct jnewblk *njnewblk; 5437 struct jnewblk *jnewblk; 5438 5439 /* Handle NULLs to simplify callers. */ 5440 if (new == NULL) 5441 return (old); 5442 if (old == NULL) 5443 return (new); 5444 /* Replace a jfreefrag with a jnewblk. */ 5445 if (new->wk_type == D_JFREEFRAG) { 5446 if (WK_JNEWBLK(old)->jn_blkno != WK_JFREEFRAG(new)->fr_blkno) 5447 panic("jnewblk_merge: blkno mismatch: %p, %p", 5448 old, new); 5449 cancel_jfreefrag(WK_JFREEFRAG(new)); 5450 return (old); 5451 } 5452 if (old->wk_type != D_JNEWBLK || new->wk_type != D_JNEWBLK) 5453 panic("jnewblk_merge: Bad type: old %d new %d\n", 5454 old->wk_type, new->wk_type); 5455 /* 5456 * Handle merging of two jnewblk records that describe 5457 * different sets of fragments in the same block. 5458 */ 5459 jnewblk = WK_JNEWBLK(old); 5460 njnewblk = WK_JNEWBLK(new); 5461 if (jnewblk->jn_blkno != njnewblk->jn_blkno) 5462 panic("jnewblk_merge: Merging disparate blocks."); 5463 /* 5464 * The record may be rolled back in the cg. 5465 */ 5466 if (jnewblk->jn_state & UNDONE) { 5467 jnewblk->jn_state &= ~UNDONE; 5468 njnewblk->jn_state |= UNDONE; 5469 njnewblk->jn_state &= ~ATTACHED; 5470 } 5471 /* 5472 * We modify the newer addref and free the older so that if neither 5473 * has been written the most up-to-date copy will be on disk. If 5474 * both have been written but rolled back we only temporarily need 5475 * one of them to fix the bits when the cg write completes. 5476 */ 5477 jnewblk->jn_state |= ATTACHED | COMPLETE; 5478 njnewblk->jn_oldfrags = jnewblk->jn_oldfrags; 5479 cancel_jnewblk(jnewblk, wkhd); 5480 WORKLIST_REMOVE(&jnewblk->jn_list); 5481 free_jnewblk(jnewblk); 5482 return (new); 5483 } 5484 5485 /* 5486 * Replace an old allocdirect dependency with a newer one. 5487 */ 5488 static void 5489 allocdirect_merge(adphead, newadp, oldadp) 5490 struct allocdirectlst *adphead; /* head of list holding allocdirects */ 5491 struct allocdirect *newadp; /* allocdirect being added */ 5492 struct allocdirect *oldadp; /* existing allocdirect being checked */ 5493 { 5494 struct worklist *wk; 5495 struct freefrag *freefrag; 5496 5497 freefrag = NULL; 5498 LOCK_OWNED(VFSTOUFS(newadp->ad_list.wk_mp)); 5499 if (newadp->ad_oldblkno != oldadp->ad_newblkno || 5500 newadp->ad_oldsize != oldadp->ad_newsize || 5501 newadp->ad_offset >= UFS_NDADDR) 5502 panic("%s %jd != new %jd || old size %ld != new %ld", 5503 "allocdirect_merge: old blkno", 5504 (intmax_t)newadp->ad_oldblkno, 5505 (intmax_t)oldadp->ad_newblkno, 5506 newadp->ad_oldsize, oldadp->ad_newsize); 5507 newadp->ad_oldblkno = oldadp->ad_oldblkno; 5508 newadp->ad_oldsize = oldadp->ad_oldsize; 5509 /* 5510 * If the old dependency had a fragment to free or had never 5511 * previously had a block allocated, then the new dependency 5512 * can immediately post its freefrag and adopt the old freefrag. 5513 * This action is done by swapping the freefrag dependencies. 5514 * The new dependency gains the old one's freefrag, and the 5515 * old one gets the new one and then immediately puts it on 5516 * the worklist when it is freed by free_newblk. It is 5517 * not possible to do this swap when the old dependency had a 5518 * non-zero size but no previous fragment to free. This condition 5519 * arises when the new block is an extension of the old block. 5520 * Here, the first part of the fragment allocated to the new 5521 * dependency is part of the block currently claimed on disk by 5522 * the old dependency, so cannot legitimately be freed until the 5523 * conditions for the new dependency are fulfilled. 5524 */ 5525 freefrag = newadp->ad_freefrag; 5526 if (oldadp->ad_freefrag != NULL || oldadp->ad_oldblkno == 0) { 5527 newadp->ad_freefrag = oldadp->ad_freefrag; 5528 oldadp->ad_freefrag = freefrag; 5529 } 5530 /* 5531 * If we are tracking a new directory-block allocation, 5532 * move it from the old allocdirect to the new allocdirect. 5533 */ 5534 if ((wk = LIST_FIRST(&oldadp->ad_newdirblk)) != NULL) { 5535 WORKLIST_REMOVE(wk); 5536 if (!LIST_EMPTY(&oldadp->ad_newdirblk)) 5537 panic("allocdirect_merge: extra newdirblk"); 5538 WORKLIST_INSERT(&newadp->ad_newdirblk, wk); 5539 } 5540 TAILQ_REMOVE(adphead, oldadp, ad_next); 5541 /* 5542 * We need to move any journal dependencies over to the freefrag 5543 * that releases this block if it exists. Otherwise we are 5544 * extending an existing block and we'll wait until that is 5545 * complete to release the journal space and extend the 5546 * new journal to cover this old space as well. 5547 */ 5548 if (freefrag == NULL) { 5549 if (oldadp->ad_newblkno != newadp->ad_newblkno) 5550 panic("allocdirect_merge: %jd != %jd", 5551 oldadp->ad_newblkno, newadp->ad_newblkno); 5552 newadp->ad_block.nb_jnewblk = (struct jnewblk *) 5553 jnewblk_merge(&newadp->ad_block.nb_jnewblk->jn_list, 5554 &oldadp->ad_block.nb_jnewblk->jn_list, 5555 &newadp->ad_block.nb_jwork); 5556 oldadp->ad_block.nb_jnewblk = NULL; 5557 cancel_newblk(&oldadp->ad_block, NULL, 5558 &newadp->ad_block.nb_jwork); 5559 } else { 5560 wk = (struct worklist *) cancel_newblk(&oldadp->ad_block, 5561 &freefrag->ff_list, &freefrag->ff_jwork); 5562 freefrag->ff_jdep = jnewblk_merge(freefrag->ff_jdep, wk, 5563 &freefrag->ff_jwork); 5564 } 5565 free_newblk(&oldadp->ad_block); 5566 } 5567 5568 /* 5569 * Allocate a jfreefrag structure to journal a single block free. 5570 */ 5571 static struct jfreefrag * 5572 newjfreefrag(freefrag, ip, blkno, size, lbn) 5573 struct freefrag *freefrag; 5574 struct inode *ip; 5575 ufs2_daddr_t blkno; 5576 long size; 5577 ufs_lbn_t lbn; 5578 { 5579 struct jfreefrag *jfreefrag; 5580 struct fs *fs; 5581 5582 fs = ITOFS(ip); 5583 jfreefrag = malloc(sizeof(struct jfreefrag), M_JFREEFRAG, 5584 M_SOFTDEP_FLAGS); 5585 workitem_alloc(&jfreefrag->fr_list, D_JFREEFRAG, ITOVFS(ip)); 5586 jfreefrag->fr_jsegdep = newjsegdep(&jfreefrag->fr_list); 5587 jfreefrag->fr_state = ATTACHED | DEPCOMPLETE; 5588 jfreefrag->fr_ino = ip->i_number; 5589 jfreefrag->fr_lbn = lbn; 5590 jfreefrag->fr_blkno = blkno; 5591 jfreefrag->fr_frags = numfrags(fs, size); 5592 jfreefrag->fr_freefrag = freefrag; 5593 5594 return (jfreefrag); 5595 } 5596 5597 /* 5598 * Allocate a new freefrag structure. 5599 */ 5600 static struct freefrag * 5601 newfreefrag(ip, blkno, size, lbn, key) 5602 struct inode *ip; 5603 ufs2_daddr_t blkno; 5604 long size; 5605 ufs_lbn_t lbn; 5606 u_long key; 5607 { 5608 struct freefrag *freefrag; 5609 struct ufsmount *ump; 5610 struct fs *fs; 5611 5612 CTR4(KTR_SUJ, "newfreefrag: ino %d blkno %jd size %ld lbn %jd", 5613 ip->i_number, blkno, size, lbn); 5614 ump = ITOUMP(ip); 5615 fs = ump->um_fs; 5616 if (fragnum(fs, blkno) + numfrags(fs, size) > fs->fs_frag) 5617 panic("newfreefrag: frag size"); 5618 freefrag = malloc(sizeof(struct freefrag), 5619 M_FREEFRAG, M_SOFTDEP_FLAGS); 5620 workitem_alloc(&freefrag->ff_list, D_FREEFRAG, UFSTOVFS(ump)); 5621 freefrag->ff_state = ATTACHED; 5622 LIST_INIT(&freefrag->ff_jwork); 5623 freefrag->ff_inum = ip->i_number; 5624 freefrag->ff_vtype = ITOV(ip)->v_type; 5625 freefrag->ff_blkno = blkno; 5626 freefrag->ff_fragsize = size; 5627 freefrag->ff_key = key; 5628 5629 if (MOUNTEDSUJ(UFSTOVFS(ump))) { 5630 freefrag->ff_jdep = (struct worklist *) 5631 newjfreefrag(freefrag, ip, blkno, size, lbn); 5632 } else { 5633 freefrag->ff_state |= DEPCOMPLETE; 5634 freefrag->ff_jdep = NULL; 5635 } 5636 5637 return (freefrag); 5638 } 5639 5640 /* 5641 * This workitem de-allocates fragments that were replaced during 5642 * file block allocation. 5643 */ 5644 static void 5645 handle_workitem_freefrag(freefrag) 5646 struct freefrag *freefrag; 5647 { 5648 struct ufsmount *ump = VFSTOUFS(freefrag->ff_list.wk_mp); 5649 struct workhead wkhd; 5650 5651 CTR3(KTR_SUJ, 5652 "handle_workitem_freefrag: ino %d blkno %jd size %ld", 5653 freefrag->ff_inum, freefrag->ff_blkno, freefrag->ff_fragsize); 5654 /* 5655 * It would be illegal to add new completion items to the 5656 * freefrag after it was schedule to be done so it must be 5657 * safe to modify the list head here. 5658 */ 5659 LIST_INIT(&wkhd); 5660 ACQUIRE_LOCK(ump); 5661 LIST_SWAP(&freefrag->ff_jwork, &wkhd, worklist, wk_list); 5662 /* 5663 * If the journal has not been written we must cancel it here. 5664 */ 5665 if (freefrag->ff_jdep) { 5666 if (freefrag->ff_jdep->wk_type != D_JNEWBLK) 5667 panic("handle_workitem_freefrag: Unexpected type %d\n", 5668 freefrag->ff_jdep->wk_type); 5669 cancel_jnewblk(WK_JNEWBLK(freefrag->ff_jdep), &wkhd); 5670 } 5671 FREE_LOCK(ump); 5672 ffs_blkfree(ump, ump->um_fs, ump->um_devvp, freefrag->ff_blkno, 5673 freefrag->ff_fragsize, freefrag->ff_inum, freefrag->ff_vtype, 5674 &wkhd, freefrag->ff_key); 5675 ACQUIRE_LOCK(ump); 5676 WORKITEM_FREE(freefrag, D_FREEFRAG); 5677 FREE_LOCK(ump); 5678 } 5679 5680 /* 5681 * Set up a dependency structure for an external attributes data block. 5682 * This routine follows much of the structure of softdep_setup_allocdirect. 5683 * See the description of softdep_setup_allocdirect above for details. 5684 */ 5685 void 5686 softdep_setup_allocext(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 5687 struct inode *ip; 5688 ufs_lbn_t off; 5689 ufs2_daddr_t newblkno; 5690 ufs2_daddr_t oldblkno; 5691 long newsize; 5692 long oldsize; 5693 struct buf *bp; 5694 { 5695 struct allocdirect *adp, *oldadp; 5696 struct allocdirectlst *adphead; 5697 struct freefrag *freefrag; 5698 struct inodedep *inodedep; 5699 struct jnewblk *jnewblk; 5700 struct newblk *newblk; 5701 struct mount *mp; 5702 struct ufsmount *ump; 5703 ufs_lbn_t lbn; 5704 5705 mp = ITOVFS(ip); 5706 ump = VFSTOUFS(mp); 5707 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5708 ("softdep_setup_allocext called on non-softdep filesystem")); 5709 KASSERT(off < UFS_NXADDR, 5710 ("softdep_setup_allocext: lbn %lld > UFS_NXADDR", (long long)off)); 5711 5712 lbn = bp->b_lblkno; 5713 if (oldblkno && oldblkno != newblkno) 5714 /* 5715 * The usual case is that a smaller fragment that 5716 * was just allocated has been replaced with a bigger 5717 * fragment or a full-size block. If it is marked as 5718 * B_DELWRI, the current contents have not been written 5719 * to disk. It is possible that the block was written 5720 * earlier, but very uncommon. If the block has never 5721 * been written, there is no need to send a BIO_DELETE 5722 * for it when it is freed. The gain from avoiding the 5723 * TRIMs for the common case of unwritten blocks far 5724 * exceeds the cost of the write amplification for the 5725 * uncommon case of failing to send a TRIM for a block 5726 * that had been written. 5727 */ 5728 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn, 5729 (bp->b_flags & B_DELWRI) != 0 ? NOTRIM_KEY : SINGLETON_KEY); 5730 else 5731 freefrag = NULL; 5732 5733 ACQUIRE_LOCK(ump); 5734 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 5735 panic("softdep_setup_allocext: lost block"); 5736 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 5737 ("softdep_setup_allocext: newblk already initialized")); 5738 /* 5739 * Convert the newblk to an allocdirect. 5740 */ 5741 WORKITEM_REASSIGN(newblk, D_ALLOCDIRECT); 5742 adp = (struct allocdirect *)newblk; 5743 newblk->nb_freefrag = freefrag; 5744 adp->ad_offset = off; 5745 adp->ad_oldblkno = oldblkno; 5746 adp->ad_newsize = newsize; 5747 adp->ad_oldsize = oldsize; 5748 adp->ad_state |= EXTDATA; 5749 5750 /* 5751 * Finish initializing the journal. 5752 */ 5753 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 5754 jnewblk->jn_ino = ip->i_number; 5755 jnewblk->jn_lbn = lbn; 5756 add_to_journal(&jnewblk->jn_list); 5757 } 5758 if (freefrag && freefrag->ff_jdep != NULL && 5759 freefrag->ff_jdep->wk_type == D_JFREEFRAG) 5760 add_to_journal(freefrag->ff_jdep); 5761 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5762 adp->ad_inodedep = inodedep; 5763 5764 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 5765 /* 5766 * The list of allocdirects must be kept in sorted and ascending 5767 * order so that the rollback routines can quickly determine the 5768 * first uncommitted block (the size of the file stored on disk 5769 * ends at the end of the lowest committed fragment, or if there 5770 * are no fragments, at the end of the highest committed block). 5771 * Since files generally grow, the typical case is that the new 5772 * block is to be added at the end of the list. We speed this 5773 * special case by checking against the last allocdirect in the 5774 * list before laboriously traversing the list looking for the 5775 * insertion point. 5776 */ 5777 adphead = &inodedep->id_newextupdt; 5778 oldadp = TAILQ_LAST(adphead, allocdirectlst); 5779 if (oldadp == NULL || oldadp->ad_offset <= off) { 5780 /* insert at end of list */ 5781 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 5782 if (oldadp != NULL && oldadp->ad_offset == off) 5783 allocdirect_merge(adphead, adp, oldadp); 5784 FREE_LOCK(ump); 5785 return; 5786 } 5787 TAILQ_FOREACH(oldadp, adphead, ad_next) { 5788 if (oldadp->ad_offset >= off) 5789 break; 5790 } 5791 if (oldadp == NULL) 5792 panic("softdep_setup_allocext: lost entry"); 5793 /* insert in middle of list */ 5794 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 5795 if (oldadp->ad_offset == off) 5796 allocdirect_merge(adphead, adp, oldadp); 5797 FREE_LOCK(ump); 5798 } 5799 5800 /* 5801 * Indirect block allocation dependencies. 5802 * 5803 * The same dependencies that exist for a direct block also exist when 5804 * a new block is allocated and pointed to by an entry in a block of 5805 * indirect pointers. The undo/redo states described above are also 5806 * used here. Because an indirect block contains many pointers that 5807 * may have dependencies, a second copy of the entire in-memory indirect 5808 * block is kept. The buffer cache copy is always completely up-to-date. 5809 * The second copy, which is used only as a source for disk writes, 5810 * contains only the safe pointers (i.e., those that have no remaining 5811 * update dependencies). The second copy is freed when all pointers 5812 * are safe. The cache is not allowed to replace indirect blocks with 5813 * pending update dependencies. If a buffer containing an indirect 5814 * block with dependencies is written, these routines will mark it 5815 * dirty again. It can only be successfully written once all the 5816 * dependencies are removed. The ffs_fsync routine in conjunction with 5817 * softdep_sync_metadata work together to get all the dependencies 5818 * removed so that a file can be successfully written to disk. Three 5819 * procedures are used when setting up indirect block pointer 5820 * dependencies. The division is necessary because of the organization 5821 * of the "balloc" routine and because of the distinction between file 5822 * pages and file metadata blocks. 5823 */ 5824 5825 /* 5826 * Allocate a new allocindir structure. 5827 */ 5828 static struct allocindir * 5829 newallocindir(ip, ptrno, newblkno, oldblkno, lbn) 5830 struct inode *ip; /* inode for file being extended */ 5831 int ptrno; /* offset of pointer in indirect block */ 5832 ufs2_daddr_t newblkno; /* disk block number being added */ 5833 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 5834 ufs_lbn_t lbn; 5835 { 5836 struct newblk *newblk; 5837 struct allocindir *aip; 5838 struct freefrag *freefrag; 5839 struct jnewblk *jnewblk; 5840 5841 if (oldblkno) 5842 freefrag = newfreefrag(ip, oldblkno, ITOFS(ip)->fs_bsize, lbn, 5843 SINGLETON_KEY); 5844 else 5845 freefrag = NULL; 5846 ACQUIRE_LOCK(ITOUMP(ip)); 5847 if (newblk_lookup(ITOVFS(ip), newblkno, 0, &newblk) == 0) 5848 panic("new_allocindir: lost block"); 5849 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 5850 ("newallocindir: newblk already initialized")); 5851 WORKITEM_REASSIGN(newblk, D_ALLOCINDIR); 5852 newblk->nb_freefrag = freefrag; 5853 aip = (struct allocindir *)newblk; 5854 aip->ai_offset = ptrno; 5855 aip->ai_oldblkno = oldblkno; 5856 aip->ai_lbn = lbn; 5857 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 5858 jnewblk->jn_ino = ip->i_number; 5859 jnewblk->jn_lbn = lbn; 5860 add_to_journal(&jnewblk->jn_list); 5861 } 5862 if (freefrag && freefrag->ff_jdep != NULL && 5863 freefrag->ff_jdep->wk_type == D_JFREEFRAG) 5864 add_to_journal(freefrag->ff_jdep); 5865 return (aip); 5866 } 5867 5868 /* 5869 * Called just before setting an indirect block pointer 5870 * to a newly allocated file page. 5871 */ 5872 void 5873 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 5874 struct inode *ip; /* inode for file being extended */ 5875 ufs_lbn_t lbn; /* allocated block number within file */ 5876 struct buf *bp; /* buffer with indirect blk referencing page */ 5877 int ptrno; /* offset of pointer in indirect block */ 5878 ufs2_daddr_t newblkno; /* disk block number being added */ 5879 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 5880 struct buf *nbp; /* buffer holding allocated page */ 5881 { 5882 struct inodedep *inodedep; 5883 struct freefrag *freefrag; 5884 struct allocindir *aip; 5885 struct pagedep *pagedep; 5886 struct mount *mp; 5887 struct ufsmount *ump; 5888 5889 mp = ITOVFS(ip); 5890 ump = VFSTOUFS(mp); 5891 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5892 ("softdep_setup_allocindir_page called on non-softdep filesystem")); 5893 KASSERT(lbn == nbp->b_lblkno, 5894 ("softdep_setup_allocindir_page: lbn %jd != lblkno %jd", 5895 lbn, bp->b_lblkno)); 5896 CTR4(KTR_SUJ, 5897 "softdep_setup_allocindir_page: ino %d blkno %jd oldblkno %jd " 5898 "lbn %jd", ip->i_number, newblkno, oldblkno, lbn); 5899 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_page"); 5900 aip = newallocindir(ip, ptrno, newblkno, oldblkno, lbn); 5901 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5902 /* 5903 * If we are allocating a directory page, then we must 5904 * allocate an associated pagedep to track additions and 5905 * deletions. 5906 */ 5907 if ((ip->i_mode & IFMT) == IFDIR) 5908 pagedep_lookup(mp, nbp, ip->i_number, lbn, DEPALLOC, &pagedep); 5909 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 5910 freefrag = setup_allocindir_phase2(bp, ip, inodedep, aip, lbn); 5911 FREE_LOCK(ump); 5912 if (freefrag) 5913 handle_workitem_freefrag(freefrag); 5914 } 5915 5916 /* 5917 * Called just before setting an indirect block pointer to a 5918 * newly allocated indirect block. 5919 */ 5920 void 5921 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 5922 struct buf *nbp; /* newly allocated indirect block */ 5923 struct inode *ip; /* inode for file being extended */ 5924 struct buf *bp; /* indirect block referencing allocated block */ 5925 int ptrno; /* offset of pointer in indirect block */ 5926 ufs2_daddr_t newblkno; /* disk block number being added */ 5927 { 5928 struct inodedep *inodedep; 5929 struct allocindir *aip; 5930 struct ufsmount *ump; 5931 ufs_lbn_t lbn; 5932 5933 ump = ITOUMP(ip); 5934 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 5935 ("softdep_setup_allocindir_meta called on non-softdep filesystem")); 5936 CTR3(KTR_SUJ, 5937 "softdep_setup_allocindir_meta: ino %d blkno %jd ptrno %d", 5938 ip->i_number, newblkno, ptrno); 5939 lbn = nbp->b_lblkno; 5940 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_meta"); 5941 aip = newallocindir(ip, ptrno, newblkno, 0, lbn); 5942 inodedep_lookup(UFSTOVFS(ump), ip->i_number, DEPALLOC, &inodedep); 5943 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 5944 if (setup_allocindir_phase2(bp, ip, inodedep, aip, lbn)) 5945 panic("softdep_setup_allocindir_meta: Block already existed"); 5946 FREE_LOCK(ump); 5947 } 5948 5949 static void 5950 indirdep_complete(indirdep) 5951 struct indirdep *indirdep; 5952 { 5953 struct allocindir *aip; 5954 5955 LIST_REMOVE(indirdep, ir_next); 5956 indirdep->ir_state |= DEPCOMPLETE; 5957 5958 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) { 5959 LIST_REMOVE(aip, ai_next); 5960 free_newblk(&aip->ai_block); 5961 } 5962 /* 5963 * If this indirdep is not attached to a buf it was simply waiting 5964 * on completion to clear completehd. free_indirdep() asserts 5965 * that nothing is dangling. 5966 */ 5967 if ((indirdep->ir_state & ONWORKLIST) == 0) 5968 free_indirdep(indirdep); 5969 } 5970 5971 static struct indirdep * 5972 indirdep_lookup(mp, ip, bp) 5973 struct mount *mp; 5974 struct inode *ip; 5975 struct buf *bp; 5976 { 5977 struct indirdep *indirdep, *newindirdep; 5978 struct newblk *newblk; 5979 struct ufsmount *ump; 5980 struct worklist *wk; 5981 struct fs *fs; 5982 ufs2_daddr_t blkno; 5983 5984 ump = VFSTOUFS(mp); 5985 LOCK_OWNED(ump); 5986 indirdep = NULL; 5987 newindirdep = NULL; 5988 fs = ump->um_fs; 5989 for (;;) { 5990 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 5991 if (wk->wk_type != D_INDIRDEP) 5992 continue; 5993 indirdep = WK_INDIRDEP(wk); 5994 break; 5995 } 5996 /* Found on the buffer worklist, no new structure to free. */ 5997 if (indirdep != NULL && newindirdep == NULL) 5998 return (indirdep); 5999 if (indirdep != NULL && newindirdep != NULL) 6000 panic("indirdep_lookup: simultaneous create"); 6001 /* None found on the buffer and a new structure is ready. */ 6002 if (indirdep == NULL && newindirdep != NULL) 6003 break; 6004 /* None found and no new structure available. */ 6005 FREE_LOCK(ump); 6006 newindirdep = malloc(sizeof(struct indirdep), 6007 M_INDIRDEP, M_SOFTDEP_FLAGS); 6008 workitem_alloc(&newindirdep->ir_list, D_INDIRDEP, mp); 6009 newindirdep->ir_state = ATTACHED; 6010 if (I_IS_UFS1(ip)) 6011 newindirdep->ir_state |= UFS1FMT; 6012 TAILQ_INIT(&newindirdep->ir_trunc); 6013 newindirdep->ir_saveddata = NULL; 6014 LIST_INIT(&newindirdep->ir_deplisthd); 6015 LIST_INIT(&newindirdep->ir_donehd); 6016 LIST_INIT(&newindirdep->ir_writehd); 6017 LIST_INIT(&newindirdep->ir_completehd); 6018 if (bp->b_blkno == bp->b_lblkno) { 6019 ufs_bmaparray(bp->b_vp, bp->b_lblkno, &blkno, bp, 6020 NULL, NULL); 6021 bp->b_blkno = blkno; 6022 } 6023 newindirdep->ir_freeblks = NULL; 6024 newindirdep->ir_savebp = 6025 getblk(ump->um_devvp, bp->b_blkno, bp->b_bcount, 0, 0, 0); 6026 newindirdep->ir_bp = bp; 6027 BUF_KERNPROC(newindirdep->ir_savebp); 6028 bcopy(bp->b_data, newindirdep->ir_savebp->b_data, bp->b_bcount); 6029 ACQUIRE_LOCK(ump); 6030 } 6031 indirdep = newindirdep; 6032 WORKLIST_INSERT(&bp->b_dep, &indirdep->ir_list); 6033 /* 6034 * If the block is not yet allocated we don't set DEPCOMPLETE so 6035 * that we don't free dependencies until the pointers are valid. 6036 * This could search b_dep for D_ALLOCDIRECT/D_ALLOCINDIR rather 6037 * than using the hash. 6038 */ 6039 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk)) 6040 LIST_INSERT_HEAD(&newblk->nb_indirdeps, indirdep, ir_next); 6041 else 6042 indirdep->ir_state |= DEPCOMPLETE; 6043 return (indirdep); 6044 } 6045 6046 /* 6047 * Called to finish the allocation of the "aip" allocated 6048 * by one of the two routines above. 6049 */ 6050 static struct freefrag * 6051 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn) 6052 struct buf *bp; /* in-memory copy of the indirect block */ 6053 struct inode *ip; /* inode for file being extended */ 6054 struct inodedep *inodedep; /* Inodedep for ip */ 6055 struct allocindir *aip; /* allocindir allocated by the above routines */ 6056 ufs_lbn_t lbn; /* Logical block number for this block. */ 6057 { 6058 struct fs *fs; 6059 struct indirdep *indirdep; 6060 struct allocindir *oldaip; 6061 struct freefrag *freefrag; 6062 struct mount *mp; 6063 struct ufsmount *ump; 6064 6065 mp = ITOVFS(ip); 6066 ump = VFSTOUFS(mp); 6067 LOCK_OWNED(ump); 6068 fs = ump->um_fs; 6069 if (bp->b_lblkno >= 0) 6070 panic("setup_allocindir_phase2: not indir blk"); 6071 KASSERT(aip->ai_offset >= 0 && aip->ai_offset < NINDIR(fs), 6072 ("setup_allocindir_phase2: Bad offset %d", aip->ai_offset)); 6073 indirdep = indirdep_lookup(mp, ip, bp); 6074 KASSERT(indirdep->ir_savebp != NULL, 6075 ("setup_allocindir_phase2 NULL ir_savebp")); 6076 aip->ai_indirdep = indirdep; 6077 /* 6078 * Check for an unwritten dependency for this indirect offset. If 6079 * there is, merge the old dependency into the new one. This happens 6080 * as a result of reallocblk only. 6081 */ 6082 freefrag = NULL; 6083 if (aip->ai_oldblkno != 0) { 6084 LIST_FOREACH(oldaip, &indirdep->ir_deplisthd, ai_next) { 6085 if (oldaip->ai_offset == aip->ai_offset) { 6086 freefrag = allocindir_merge(aip, oldaip); 6087 goto done; 6088 } 6089 } 6090 LIST_FOREACH(oldaip, &indirdep->ir_donehd, ai_next) { 6091 if (oldaip->ai_offset == aip->ai_offset) { 6092 freefrag = allocindir_merge(aip, oldaip); 6093 goto done; 6094 } 6095 } 6096 } 6097 done: 6098 LIST_INSERT_HEAD(&indirdep->ir_deplisthd, aip, ai_next); 6099 return (freefrag); 6100 } 6101 6102 /* 6103 * Merge two allocindirs which refer to the same block. Move newblock 6104 * dependencies and setup the freefrags appropriately. 6105 */ 6106 static struct freefrag * 6107 allocindir_merge(aip, oldaip) 6108 struct allocindir *aip; 6109 struct allocindir *oldaip; 6110 { 6111 struct freefrag *freefrag; 6112 struct worklist *wk; 6113 6114 if (oldaip->ai_newblkno != aip->ai_oldblkno) 6115 panic("allocindir_merge: blkno"); 6116 aip->ai_oldblkno = oldaip->ai_oldblkno; 6117 freefrag = aip->ai_freefrag; 6118 aip->ai_freefrag = oldaip->ai_freefrag; 6119 oldaip->ai_freefrag = NULL; 6120 KASSERT(freefrag != NULL, ("setup_allocindir_phase2: No freefrag")); 6121 /* 6122 * If we are tracking a new directory-block allocation, 6123 * move it from the old allocindir to the new allocindir. 6124 */ 6125 if ((wk = LIST_FIRST(&oldaip->ai_newdirblk)) != NULL) { 6126 WORKLIST_REMOVE(wk); 6127 if (!LIST_EMPTY(&oldaip->ai_newdirblk)) 6128 panic("allocindir_merge: extra newdirblk"); 6129 WORKLIST_INSERT(&aip->ai_newdirblk, wk); 6130 } 6131 /* 6132 * We can skip journaling for this freefrag and just complete 6133 * any pending journal work for the allocindir that is being 6134 * removed after the freefrag completes. 6135 */ 6136 if (freefrag->ff_jdep) 6137 cancel_jfreefrag(WK_JFREEFRAG(freefrag->ff_jdep)); 6138 LIST_REMOVE(oldaip, ai_next); 6139 freefrag->ff_jdep = (struct worklist *)cancel_newblk(&oldaip->ai_block, 6140 &freefrag->ff_list, &freefrag->ff_jwork); 6141 free_newblk(&oldaip->ai_block); 6142 6143 return (freefrag); 6144 } 6145 6146 static inline void 6147 setup_freedirect(freeblks, ip, i, needj) 6148 struct freeblks *freeblks; 6149 struct inode *ip; 6150 int i; 6151 int needj; 6152 { 6153 struct ufsmount *ump; 6154 ufs2_daddr_t blkno; 6155 int frags; 6156 6157 blkno = DIP(ip, i_db[i]); 6158 if (blkno == 0) 6159 return; 6160 DIP_SET(ip, i_db[i], 0); 6161 ump = ITOUMP(ip); 6162 frags = sblksize(ump->um_fs, ip->i_size, i); 6163 frags = numfrags(ump->um_fs, frags); 6164 newfreework(ump, freeblks, NULL, i, blkno, frags, 0, needj); 6165 } 6166 6167 static inline void 6168 setup_freeext(freeblks, ip, i, needj) 6169 struct freeblks *freeblks; 6170 struct inode *ip; 6171 int i; 6172 int needj; 6173 { 6174 struct ufsmount *ump; 6175 ufs2_daddr_t blkno; 6176 int frags; 6177 6178 blkno = ip->i_din2->di_extb[i]; 6179 if (blkno == 0) 6180 return; 6181 ip->i_din2->di_extb[i] = 0; 6182 ump = ITOUMP(ip); 6183 frags = sblksize(ump->um_fs, ip->i_din2->di_extsize, i); 6184 frags = numfrags(ump->um_fs, frags); 6185 newfreework(ump, freeblks, NULL, -1 - i, blkno, frags, 0, needj); 6186 } 6187 6188 static inline void 6189 setup_freeindir(freeblks, ip, i, lbn, needj) 6190 struct freeblks *freeblks; 6191 struct inode *ip; 6192 int i; 6193 ufs_lbn_t lbn; 6194 int needj; 6195 { 6196 struct ufsmount *ump; 6197 ufs2_daddr_t blkno; 6198 6199 blkno = DIP(ip, i_ib[i]); 6200 if (blkno == 0) 6201 return; 6202 DIP_SET(ip, i_ib[i], 0); 6203 ump = ITOUMP(ip); 6204 newfreework(ump, freeblks, NULL, lbn, blkno, ump->um_fs->fs_frag, 6205 0, needj); 6206 } 6207 6208 static inline struct freeblks * 6209 newfreeblks(mp, ip) 6210 struct mount *mp; 6211 struct inode *ip; 6212 { 6213 struct freeblks *freeblks; 6214 6215 freeblks = malloc(sizeof(struct freeblks), 6216 M_FREEBLKS, M_SOFTDEP_FLAGS|M_ZERO); 6217 workitem_alloc(&freeblks->fb_list, D_FREEBLKS, mp); 6218 LIST_INIT(&freeblks->fb_jblkdephd); 6219 LIST_INIT(&freeblks->fb_jwork); 6220 freeblks->fb_ref = 0; 6221 freeblks->fb_cgwait = 0; 6222 freeblks->fb_state = ATTACHED; 6223 freeblks->fb_uid = ip->i_uid; 6224 freeblks->fb_inum = ip->i_number; 6225 freeblks->fb_vtype = ITOV(ip)->v_type; 6226 freeblks->fb_modrev = DIP(ip, i_modrev); 6227 freeblks->fb_devvp = ITODEVVP(ip); 6228 freeblks->fb_chkcnt = 0; 6229 freeblks->fb_len = 0; 6230 6231 return (freeblks); 6232 } 6233 6234 static void 6235 trunc_indirdep(indirdep, freeblks, bp, off) 6236 struct indirdep *indirdep; 6237 struct freeblks *freeblks; 6238 struct buf *bp; 6239 int off; 6240 { 6241 struct allocindir *aip, *aipn; 6242 6243 /* 6244 * The first set of allocindirs won't be in savedbp. 6245 */ 6246 LIST_FOREACH_SAFE(aip, &indirdep->ir_deplisthd, ai_next, aipn) 6247 if (aip->ai_offset > off) 6248 cancel_allocindir(aip, bp, freeblks, 1); 6249 LIST_FOREACH_SAFE(aip, &indirdep->ir_donehd, ai_next, aipn) 6250 if (aip->ai_offset > off) 6251 cancel_allocindir(aip, bp, freeblks, 1); 6252 /* 6253 * These will exist in savedbp. 6254 */ 6255 LIST_FOREACH_SAFE(aip, &indirdep->ir_writehd, ai_next, aipn) 6256 if (aip->ai_offset > off) 6257 cancel_allocindir(aip, NULL, freeblks, 0); 6258 LIST_FOREACH_SAFE(aip, &indirdep->ir_completehd, ai_next, aipn) 6259 if (aip->ai_offset > off) 6260 cancel_allocindir(aip, NULL, freeblks, 0); 6261 } 6262 6263 /* 6264 * Follow the chain of indirects down to lastlbn creating a freework 6265 * structure for each. This will be used to start indir_trunc() at 6266 * the right offset and create the journal records for the parrtial 6267 * truncation. A second step will handle the truncated dependencies. 6268 */ 6269 static int 6270 setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno) 6271 struct freeblks *freeblks; 6272 struct inode *ip; 6273 ufs_lbn_t lbn; 6274 ufs_lbn_t lastlbn; 6275 ufs2_daddr_t blkno; 6276 { 6277 struct indirdep *indirdep; 6278 struct indirdep *indirn; 6279 struct freework *freework; 6280 struct newblk *newblk; 6281 struct mount *mp; 6282 struct ufsmount *ump; 6283 struct buf *bp; 6284 uint8_t *start; 6285 uint8_t *end; 6286 ufs_lbn_t lbnadd; 6287 int level; 6288 int error; 6289 int off; 6290 6291 6292 freework = NULL; 6293 if (blkno == 0) 6294 return (0); 6295 mp = freeblks->fb_list.wk_mp; 6296 ump = VFSTOUFS(mp); 6297 /* 6298 * Here, calls to VOP_BMAP() will fail. However, we already have 6299 * the on-disk address, so we just pass it to bread() instead of 6300 * having bread() attempt to calculate it using VOP_BMAP(). 6301 */ 6302 error = breadn_flags(ITOV(ip), lbn, blkptrtodb(ump, blkno), 6303 (int)mp->mnt_stat.f_iosize, NULL, NULL, 0, NOCRED, 0, NULL, &bp); 6304 if (error) 6305 return (error); 6306 level = lbn_level(lbn); 6307 lbnadd = lbn_offset(ump->um_fs, level); 6308 /* 6309 * Compute the offset of the last block we want to keep. Store 6310 * in the freework the first block we want to completely free. 6311 */ 6312 off = (lastlbn - -(lbn + level)) / lbnadd; 6313 if (off + 1 == NINDIR(ump->um_fs)) 6314 goto nowork; 6315 freework = newfreework(ump, freeblks, NULL, lbn, blkno, 0, off + 1, 0); 6316 /* 6317 * Link the freework into the indirdep. This will prevent any new 6318 * allocations from proceeding until we are finished with the 6319 * truncate and the block is written. 6320 */ 6321 ACQUIRE_LOCK(ump); 6322 indirdep = indirdep_lookup(mp, ip, bp); 6323 if (indirdep->ir_freeblks) 6324 panic("setup_trunc_indir: indirdep already truncated."); 6325 TAILQ_INSERT_TAIL(&indirdep->ir_trunc, freework, fw_next); 6326 freework->fw_indir = indirdep; 6327 /* 6328 * Cancel any allocindirs that will not make it to disk. 6329 * We have to do this for all copies of the indirdep that 6330 * live on this newblk. 6331 */ 6332 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 6333 if (newblk_lookup(mp, dbtofsb(ump->um_fs, bp->b_blkno), 0, 6334 &newblk) == 0) 6335 panic("setup_trunc_indir: lost block"); 6336 LIST_FOREACH(indirn, &newblk->nb_indirdeps, ir_next) 6337 trunc_indirdep(indirn, freeblks, bp, off); 6338 } else 6339 trunc_indirdep(indirdep, freeblks, bp, off); 6340 FREE_LOCK(ump); 6341 /* 6342 * Creation is protected by the buf lock. The saveddata is only 6343 * needed if a full truncation follows a partial truncation but it 6344 * is difficult to allocate in that case so we fetch it anyway. 6345 */ 6346 if (indirdep->ir_saveddata == NULL) 6347 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP, 6348 M_SOFTDEP_FLAGS); 6349 nowork: 6350 /* Fetch the blkno of the child and the zero start offset. */ 6351 if (I_IS_UFS1(ip)) { 6352 blkno = ((ufs1_daddr_t *)bp->b_data)[off]; 6353 start = (uint8_t *)&((ufs1_daddr_t *)bp->b_data)[off+1]; 6354 } else { 6355 blkno = ((ufs2_daddr_t *)bp->b_data)[off]; 6356 start = (uint8_t *)&((ufs2_daddr_t *)bp->b_data)[off+1]; 6357 } 6358 if (freework) { 6359 /* Zero the truncated pointers. */ 6360 end = bp->b_data + bp->b_bcount; 6361 bzero(start, end - start); 6362 bdwrite(bp); 6363 } else 6364 bqrelse(bp); 6365 if (level == 0) 6366 return (0); 6367 lbn++; /* adjust level */ 6368 lbn -= (off * lbnadd); 6369 return setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno); 6370 } 6371 6372 /* 6373 * Complete the partial truncation of an indirect block setup by 6374 * setup_trunc_indir(). This zeros the truncated pointers in the saved 6375 * copy and writes them to disk before the freeblks is allowed to complete. 6376 */ 6377 static void 6378 complete_trunc_indir(freework) 6379 struct freework *freework; 6380 { 6381 struct freework *fwn; 6382 struct indirdep *indirdep; 6383 struct ufsmount *ump; 6384 struct buf *bp; 6385 uintptr_t start; 6386 int count; 6387 6388 ump = VFSTOUFS(freework->fw_list.wk_mp); 6389 LOCK_OWNED(ump); 6390 indirdep = freework->fw_indir; 6391 for (;;) { 6392 bp = indirdep->ir_bp; 6393 /* See if the block was discarded. */ 6394 if (bp == NULL) 6395 break; 6396 /* Inline part of getdirtybuf(). We dont want bremfree. */ 6397 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) == 0) 6398 break; 6399 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 6400 LOCK_PTR(ump)) == 0) 6401 BUF_UNLOCK(bp); 6402 ACQUIRE_LOCK(ump); 6403 } 6404 freework->fw_state |= DEPCOMPLETE; 6405 TAILQ_REMOVE(&indirdep->ir_trunc, freework, fw_next); 6406 /* 6407 * Zero the pointers in the saved copy. 6408 */ 6409 if (indirdep->ir_state & UFS1FMT) 6410 start = sizeof(ufs1_daddr_t); 6411 else 6412 start = sizeof(ufs2_daddr_t); 6413 start *= freework->fw_start; 6414 count = indirdep->ir_savebp->b_bcount - start; 6415 start += (uintptr_t)indirdep->ir_savebp->b_data; 6416 bzero((char *)start, count); 6417 /* 6418 * We need to start the next truncation in the list if it has not 6419 * been started yet. 6420 */ 6421 fwn = TAILQ_FIRST(&indirdep->ir_trunc); 6422 if (fwn != NULL) { 6423 if (fwn->fw_freeblks == indirdep->ir_freeblks) 6424 TAILQ_REMOVE(&indirdep->ir_trunc, fwn, fw_next); 6425 if ((fwn->fw_state & ONWORKLIST) == 0) 6426 freework_enqueue(fwn); 6427 } 6428 /* 6429 * If bp is NULL the block was fully truncated, restore 6430 * the saved block list otherwise free it if it is no 6431 * longer needed. 6432 */ 6433 if (TAILQ_EMPTY(&indirdep->ir_trunc)) { 6434 if (bp == NULL) 6435 bcopy(indirdep->ir_saveddata, 6436 indirdep->ir_savebp->b_data, 6437 indirdep->ir_savebp->b_bcount); 6438 free(indirdep->ir_saveddata, M_INDIRDEP); 6439 indirdep->ir_saveddata = NULL; 6440 } 6441 /* 6442 * When bp is NULL there is a full truncation pending. We 6443 * must wait for this full truncation to be journaled before 6444 * we can release this freework because the disk pointers will 6445 * never be written as zero. 6446 */ 6447 if (bp == NULL) { 6448 if (LIST_EMPTY(&indirdep->ir_freeblks->fb_jblkdephd)) 6449 handle_written_freework(freework); 6450 else 6451 WORKLIST_INSERT(&indirdep->ir_freeblks->fb_freeworkhd, 6452 &freework->fw_list); 6453 } else { 6454 /* Complete when the real copy is written. */ 6455 WORKLIST_INSERT(&bp->b_dep, &freework->fw_list); 6456 BUF_UNLOCK(bp); 6457 } 6458 } 6459 6460 /* 6461 * Calculate the number of blocks we are going to release where datablocks 6462 * is the current total and length is the new file size. 6463 */ 6464 static ufs2_daddr_t 6465 blkcount(fs, datablocks, length) 6466 struct fs *fs; 6467 ufs2_daddr_t datablocks; 6468 off_t length; 6469 { 6470 off_t totblks, numblks; 6471 6472 totblks = 0; 6473 numblks = howmany(length, fs->fs_bsize); 6474 if (numblks <= UFS_NDADDR) { 6475 totblks = howmany(length, fs->fs_fsize); 6476 goto out; 6477 } 6478 totblks = blkstofrags(fs, numblks); 6479 numblks -= UFS_NDADDR; 6480 /* 6481 * Count all single, then double, then triple indirects required. 6482 * Subtracting one indirects worth of blocks for each pass 6483 * acknowledges one of each pointed to by the inode. 6484 */ 6485 for (;;) { 6486 totblks += blkstofrags(fs, howmany(numblks, NINDIR(fs))); 6487 numblks -= NINDIR(fs); 6488 if (numblks <= 0) 6489 break; 6490 numblks = howmany(numblks, NINDIR(fs)); 6491 } 6492 out: 6493 totblks = fsbtodb(fs, totblks); 6494 /* 6495 * Handle sparse files. We can't reclaim more blocks than the inode 6496 * references. We will correct it later in handle_complete_freeblks() 6497 * when we know the real count. 6498 */ 6499 if (totblks > datablocks) 6500 return (0); 6501 return (datablocks - totblks); 6502 } 6503 6504 /* 6505 * Handle freeblocks for journaled softupdate filesystems. 6506 * 6507 * Contrary to normal softupdates, we must preserve the block pointers in 6508 * indirects until their subordinates are free. This is to avoid journaling 6509 * every block that is freed which may consume more space than the journal 6510 * itself. The recovery program will see the free block journals at the 6511 * base of the truncated area and traverse them to reclaim space. The 6512 * pointers in the inode may be cleared immediately after the journal 6513 * records are written because each direct and indirect pointer in the 6514 * inode is recorded in a journal. This permits full truncation to proceed 6515 * asynchronously. The write order is journal -> inode -> cgs -> indirects. 6516 * 6517 * The algorithm is as follows: 6518 * 1) Traverse the in-memory state and create journal entries to release 6519 * the relevant blocks and full indirect trees. 6520 * 2) Traverse the indirect block chain adding partial truncation freework 6521 * records to indirects in the path to lastlbn. The freework will 6522 * prevent new allocation dependencies from being satisfied in this 6523 * indirect until the truncation completes. 6524 * 3) Read and lock the inode block, performing an update with the new size 6525 * and pointers. This prevents truncated data from becoming valid on 6526 * disk through step 4. 6527 * 4) Reap unsatisfied dependencies that are beyond the truncated area, 6528 * eliminate journal work for those records that do not require it. 6529 * 5) Schedule the journal records to be written followed by the inode block. 6530 * 6) Allocate any necessary frags for the end of file. 6531 * 7) Zero any partially truncated blocks. 6532 * 6533 * From this truncation proceeds asynchronously using the freework and 6534 * indir_trunc machinery. The file will not be extended again into a 6535 * partially truncated indirect block until all work is completed but 6536 * the normal dependency mechanism ensures that it is rolled back/forward 6537 * as appropriate. Further truncation may occur without delay and is 6538 * serialized in indir_trunc(). 6539 */ 6540 void 6541 softdep_journal_freeblocks(ip, cred, length, flags) 6542 struct inode *ip; /* The inode whose length is to be reduced */ 6543 struct ucred *cred; 6544 off_t length; /* The new length for the file */ 6545 int flags; /* IO_EXT and/or IO_NORMAL */ 6546 { 6547 struct freeblks *freeblks, *fbn; 6548 struct worklist *wk, *wkn; 6549 struct inodedep *inodedep; 6550 struct jblkdep *jblkdep; 6551 struct allocdirect *adp, *adpn; 6552 struct ufsmount *ump; 6553 struct fs *fs; 6554 struct buf *bp; 6555 struct vnode *vp; 6556 struct mount *mp; 6557 ufs2_daddr_t extblocks, datablocks; 6558 ufs_lbn_t tmpval, lbn, lastlbn; 6559 int frags, lastoff, iboff, allocblock, needj, error, i; 6560 6561 ump = ITOUMP(ip); 6562 mp = UFSTOVFS(ump); 6563 fs = ump->um_fs; 6564 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 6565 ("softdep_journal_freeblocks called on non-softdep filesystem")); 6566 vp = ITOV(ip); 6567 needj = 1; 6568 iboff = -1; 6569 allocblock = 0; 6570 extblocks = 0; 6571 datablocks = 0; 6572 frags = 0; 6573 freeblks = newfreeblks(mp, ip); 6574 ACQUIRE_LOCK(ump); 6575 /* 6576 * If we're truncating a removed file that will never be written 6577 * we don't need to journal the block frees. The canceled journals 6578 * for the allocations will suffice. 6579 */ 6580 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6581 if ((inodedep->id_state & (UNLINKED | DEPCOMPLETE)) == UNLINKED && 6582 length == 0) 6583 needj = 0; 6584 CTR3(KTR_SUJ, "softdep_journal_freeblks: ip %d length %ld needj %d", 6585 ip->i_number, length, needj); 6586 FREE_LOCK(ump); 6587 /* 6588 * Calculate the lbn that we are truncating to. This results in -1 6589 * if we're truncating the 0 bytes. So it is the last lbn we want 6590 * to keep, not the first lbn we want to truncate. 6591 */ 6592 lastlbn = lblkno(fs, length + fs->fs_bsize - 1) - 1; 6593 lastoff = blkoff(fs, length); 6594 /* 6595 * Compute frags we are keeping in lastlbn. 0 means all. 6596 */ 6597 if (lastlbn >= 0 && lastlbn < UFS_NDADDR) { 6598 frags = fragroundup(fs, lastoff); 6599 /* adp offset of last valid allocdirect. */ 6600 iboff = lastlbn; 6601 } else if (lastlbn > 0) 6602 iboff = UFS_NDADDR; 6603 if (fs->fs_magic == FS_UFS2_MAGIC) 6604 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize)); 6605 /* 6606 * Handle normal data blocks and indirects. This section saves 6607 * values used after the inode update to complete frag and indirect 6608 * truncation. 6609 */ 6610 if ((flags & IO_NORMAL) != 0) { 6611 /* 6612 * Handle truncation of whole direct and indirect blocks. 6613 */ 6614 for (i = iboff + 1; i < UFS_NDADDR; i++) 6615 setup_freedirect(freeblks, ip, i, needj); 6616 for (i = 0, tmpval = NINDIR(fs), lbn = UFS_NDADDR; 6617 i < UFS_NIADDR; 6618 i++, lbn += tmpval, tmpval *= NINDIR(fs)) { 6619 /* Release a whole indirect tree. */ 6620 if (lbn > lastlbn) { 6621 setup_freeindir(freeblks, ip, i, -lbn -i, 6622 needj); 6623 continue; 6624 } 6625 iboff = i + UFS_NDADDR; 6626 /* 6627 * Traverse partially truncated indirect tree. 6628 */ 6629 if (lbn <= lastlbn && lbn + tmpval - 1 > lastlbn) 6630 setup_trunc_indir(freeblks, ip, -lbn - i, 6631 lastlbn, DIP(ip, i_ib[i])); 6632 } 6633 /* 6634 * Handle partial truncation to a frag boundary. 6635 */ 6636 if (frags) { 6637 ufs2_daddr_t blkno; 6638 long oldfrags; 6639 6640 oldfrags = blksize(fs, ip, lastlbn); 6641 blkno = DIP(ip, i_db[lastlbn]); 6642 if (blkno && oldfrags != frags) { 6643 oldfrags -= frags; 6644 oldfrags = numfrags(fs, oldfrags); 6645 blkno += numfrags(fs, frags); 6646 newfreework(ump, freeblks, NULL, lastlbn, 6647 blkno, oldfrags, 0, needj); 6648 if (needj) 6649 adjust_newfreework(freeblks, 6650 numfrags(fs, frags)); 6651 } else if (blkno == 0) 6652 allocblock = 1; 6653 } 6654 /* 6655 * Add a journal record for partial truncate if we are 6656 * handling indirect blocks. Non-indirects need no extra 6657 * journaling. 6658 */ 6659 if (length != 0 && lastlbn >= UFS_NDADDR) { 6660 ip->i_flag |= IN_TRUNCATED; 6661 newjtrunc(freeblks, length, 0); 6662 } 6663 ip->i_size = length; 6664 DIP_SET(ip, i_size, ip->i_size); 6665 datablocks = DIP(ip, i_blocks) - extblocks; 6666 if (length != 0) 6667 datablocks = blkcount(fs, datablocks, length); 6668 freeblks->fb_len = length; 6669 } 6670 if ((flags & IO_EXT) != 0) { 6671 for (i = 0; i < UFS_NXADDR; i++) 6672 setup_freeext(freeblks, ip, i, needj); 6673 ip->i_din2->di_extsize = 0; 6674 datablocks += extblocks; 6675 } 6676 #ifdef QUOTA 6677 /* Reference the quotas in case the block count is wrong in the end. */ 6678 quotaref(vp, freeblks->fb_quota); 6679 (void) chkdq(ip, -datablocks, NOCRED, FORCE); 6680 #endif 6681 freeblks->fb_chkcnt = -datablocks; 6682 UFS_LOCK(ump); 6683 fs->fs_pendingblocks += datablocks; 6684 UFS_UNLOCK(ump); 6685 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks); 6686 /* 6687 * Handle truncation of incomplete alloc direct dependencies. We 6688 * hold the inode block locked to prevent incomplete dependencies 6689 * from reaching the disk while we are eliminating those that 6690 * have been truncated. This is a partially inlined ffs_update(). 6691 */ 6692 ufs_itimes(vp); 6693 ip->i_flag &= ~(IN_LAZYACCESS | IN_LAZYMOD | IN_MODIFIED); 6694 error = bread(ump->um_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 6695 (int)fs->fs_bsize, cred, &bp); 6696 if (error) { 6697 softdep_error("softdep_journal_freeblocks", error); 6698 return; 6699 } 6700 if (bp->b_bufsize == fs->fs_bsize) 6701 bp->b_flags |= B_CLUSTEROK; 6702 softdep_update_inodeblock(ip, bp, 0); 6703 if (ump->um_fstype == UFS1) { 6704 *((struct ufs1_dinode *)bp->b_data + 6705 ino_to_fsbo(fs, ip->i_number)) = *ip->i_din1; 6706 } else { 6707 ffs_update_dinode_ckhash(fs, ip->i_din2); 6708 *((struct ufs2_dinode *)bp->b_data + 6709 ino_to_fsbo(fs, ip->i_number)) = *ip->i_din2; 6710 } 6711 ACQUIRE_LOCK(ump); 6712 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6713 if ((inodedep->id_state & IOSTARTED) != 0) 6714 panic("softdep_setup_freeblocks: inode busy"); 6715 /* 6716 * Add the freeblks structure to the list of operations that 6717 * must await the zero'ed inode being written to disk. If we 6718 * still have a bitmap dependency (needj), then the inode 6719 * has never been written to disk, so we can process the 6720 * freeblks below once we have deleted the dependencies. 6721 */ 6722 if (needj) 6723 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list); 6724 else 6725 freeblks->fb_state |= COMPLETE; 6726 if ((flags & IO_NORMAL) != 0) { 6727 TAILQ_FOREACH_SAFE(adp, &inodedep->id_inoupdt, ad_next, adpn) { 6728 if (adp->ad_offset > iboff) 6729 cancel_allocdirect(&inodedep->id_inoupdt, adp, 6730 freeblks); 6731 /* 6732 * Truncate the allocdirect. We could eliminate 6733 * or modify journal records as well. 6734 */ 6735 else if (adp->ad_offset == iboff && frags) 6736 adp->ad_newsize = frags; 6737 } 6738 } 6739 if ((flags & IO_EXT) != 0) 6740 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 6741 cancel_allocdirect(&inodedep->id_extupdt, adp, 6742 freeblks); 6743 /* 6744 * Scan the bufwait list for newblock dependencies that will never 6745 * make it to disk. 6746 */ 6747 LIST_FOREACH_SAFE(wk, &inodedep->id_bufwait, wk_list, wkn) { 6748 if (wk->wk_type != D_ALLOCDIRECT) 6749 continue; 6750 adp = WK_ALLOCDIRECT(wk); 6751 if (((flags & IO_NORMAL) != 0 && (adp->ad_offset > iboff)) || 6752 ((flags & IO_EXT) != 0 && (adp->ad_state & EXTDATA))) { 6753 cancel_jfreeblk(freeblks, adp->ad_newblkno); 6754 cancel_newblk(WK_NEWBLK(wk), NULL, &freeblks->fb_jwork); 6755 WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk); 6756 } 6757 } 6758 /* 6759 * Add journal work. 6760 */ 6761 LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps) 6762 add_to_journal(&jblkdep->jb_list); 6763 FREE_LOCK(ump); 6764 bdwrite(bp); 6765 /* 6766 * Truncate dependency structures beyond length. 6767 */ 6768 trunc_dependencies(ip, freeblks, lastlbn, frags, flags); 6769 /* 6770 * This is only set when we need to allocate a fragment because 6771 * none existed at the end of a frag-sized file. It handles only 6772 * allocating a new, zero filled block. 6773 */ 6774 if (allocblock) { 6775 ip->i_size = length - lastoff; 6776 DIP_SET(ip, i_size, ip->i_size); 6777 error = UFS_BALLOC(vp, length - 1, 1, cred, BA_CLRBUF, &bp); 6778 if (error != 0) { 6779 softdep_error("softdep_journal_freeblks", error); 6780 return; 6781 } 6782 ip->i_size = length; 6783 DIP_SET(ip, i_size, length); 6784 ip->i_flag |= IN_CHANGE | IN_UPDATE; 6785 allocbuf(bp, frags); 6786 ffs_update(vp, 0); 6787 bawrite(bp); 6788 } else if (lastoff != 0 && vp->v_type != VDIR) { 6789 int size; 6790 6791 /* 6792 * Zero the end of a truncated frag or block. 6793 */ 6794 size = sblksize(fs, length, lastlbn); 6795 error = bread(vp, lastlbn, size, cred, &bp); 6796 if (error) { 6797 softdep_error("softdep_journal_freeblks", error); 6798 return; 6799 } 6800 bzero((char *)bp->b_data + lastoff, size - lastoff); 6801 bawrite(bp); 6802 6803 } 6804 ACQUIRE_LOCK(ump); 6805 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6806 TAILQ_INSERT_TAIL(&inodedep->id_freeblklst, freeblks, fb_next); 6807 freeblks->fb_state |= DEPCOMPLETE | ONDEPLIST; 6808 /* 6809 * We zero earlier truncations so they don't erroneously 6810 * update i_blocks. 6811 */ 6812 if (freeblks->fb_len == 0 && (flags & IO_NORMAL) != 0) 6813 TAILQ_FOREACH(fbn, &inodedep->id_freeblklst, fb_next) 6814 fbn->fb_len = 0; 6815 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE && 6816 LIST_EMPTY(&freeblks->fb_jblkdephd)) 6817 freeblks->fb_state |= INPROGRESS; 6818 else 6819 freeblks = NULL; 6820 FREE_LOCK(ump); 6821 if (freeblks) 6822 handle_workitem_freeblocks(freeblks, 0); 6823 trunc_pages(ip, length, extblocks, flags); 6824 6825 } 6826 6827 /* 6828 * Flush a JOP_SYNC to the journal. 6829 */ 6830 void 6831 softdep_journal_fsync(ip) 6832 struct inode *ip; 6833 { 6834 struct jfsync *jfsync; 6835 struct ufsmount *ump; 6836 6837 ump = ITOUMP(ip); 6838 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 6839 ("softdep_journal_fsync called on non-softdep filesystem")); 6840 if ((ip->i_flag & IN_TRUNCATED) == 0) 6841 return; 6842 ip->i_flag &= ~IN_TRUNCATED; 6843 jfsync = malloc(sizeof(*jfsync), M_JFSYNC, M_SOFTDEP_FLAGS | M_ZERO); 6844 workitem_alloc(&jfsync->jfs_list, D_JFSYNC, UFSTOVFS(ump)); 6845 jfsync->jfs_size = ip->i_size; 6846 jfsync->jfs_ino = ip->i_number; 6847 ACQUIRE_LOCK(ump); 6848 add_to_journal(&jfsync->jfs_list); 6849 jwait(&jfsync->jfs_list, MNT_WAIT); 6850 FREE_LOCK(ump); 6851 } 6852 6853 /* 6854 * Block de-allocation dependencies. 6855 * 6856 * When blocks are de-allocated, the on-disk pointers must be nullified before 6857 * the blocks are made available for use by other files. (The true 6858 * requirement is that old pointers must be nullified before new on-disk 6859 * pointers are set. We chose this slightly more stringent requirement to 6860 * reduce complexity.) Our implementation handles this dependency by updating 6861 * the inode (or indirect block) appropriately but delaying the actual block 6862 * de-allocation (i.e., freemap and free space count manipulation) until 6863 * after the updated versions reach stable storage. After the disk is 6864 * updated, the blocks can be safely de-allocated whenever it is convenient. 6865 * This implementation handles only the common case of reducing a file's 6866 * length to zero. Other cases are handled by the conventional synchronous 6867 * write approach. 6868 * 6869 * The ffs implementation with which we worked double-checks 6870 * the state of the block pointers and file size as it reduces 6871 * a file's length. Some of this code is replicated here in our 6872 * soft updates implementation. The freeblks->fb_chkcnt field is 6873 * used to transfer a part of this information to the procedure 6874 * that eventually de-allocates the blocks. 6875 * 6876 * This routine should be called from the routine that shortens 6877 * a file's length, before the inode's size or block pointers 6878 * are modified. It will save the block pointer information for 6879 * later release and zero the inode so that the calling routine 6880 * can release it. 6881 */ 6882 void 6883 softdep_setup_freeblocks(ip, length, flags) 6884 struct inode *ip; /* The inode whose length is to be reduced */ 6885 off_t length; /* The new length for the file */ 6886 int flags; /* IO_EXT and/or IO_NORMAL */ 6887 { 6888 struct ufs1_dinode *dp1; 6889 struct ufs2_dinode *dp2; 6890 struct freeblks *freeblks; 6891 struct inodedep *inodedep; 6892 struct allocdirect *adp; 6893 struct ufsmount *ump; 6894 struct buf *bp; 6895 struct fs *fs; 6896 ufs2_daddr_t extblocks, datablocks; 6897 struct mount *mp; 6898 int i, delay, error; 6899 ufs_lbn_t tmpval; 6900 ufs_lbn_t lbn; 6901 6902 ump = ITOUMP(ip); 6903 mp = UFSTOVFS(ump); 6904 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 6905 ("softdep_setup_freeblocks called on non-softdep filesystem")); 6906 CTR2(KTR_SUJ, "softdep_setup_freeblks: ip %d length %ld", 6907 ip->i_number, length); 6908 KASSERT(length == 0, ("softdep_setup_freeblocks: non-zero length")); 6909 fs = ump->um_fs; 6910 if ((error = bread(ump->um_devvp, 6911 fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 6912 (int)fs->fs_bsize, NOCRED, &bp)) != 0) { 6913 brelse(bp); 6914 softdep_error("softdep_setup_freeblocks", error); 6915 return; 6916 } 6917 freeblks = newfreeblks(mp, ip); 6918 extblocks = 0; 6919 datablocks = 0; 6920 if (fs->fs_magic == FS_UFS2_MAGIC) 6921 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize)); 6922 if ((flags & IO_NORMAL) != 0) { 6923 for (i = 0; i < UFS_NDADDR; i++) 6924 setup_freedirect(freeblks, ip, i, 0); 6925 for (i = 0, tmpval = NINDIR(fs), lbn = UFS_NDADDR; 6926 i < UFS_NIADDR; 6927 i++, lbn += tmpval, tmpval *= NINDIR(fs)) 6928 setup_freeindir(freeblks, ip, i, -lbn -i, 0); 6929 ip->i_size = 0; 6930 DIP_SET(ip, i_size, 0); 6931 datablocks = DIP(ip, i_blocks) - extblocks; 6932 } 6933 if ((flags & IO_EXT) != 0) { 6934 for (i = 0; i < UFS_NXADDR; i++) 6935 setup_freeext(freeblks, ip, i, 0); 6936 ip->i_din2->di_extsize = 0; 6937 datablocks += extblocks; 6938 } 6939 #ifdef QUOTA 6940 /* Reference the quotas in case the block count is wrong in the end. */ 6941 quotaref(ITOV(ip), freeblks->fb_quota); 6942 (void) chkdq(ip, -datablocks, NOCRED, FORCE); 6943 #endif 6944 freeblks->fb_chkcnt = -datablocks; 6945 UFS_LOCK(ump); 6946 fs->fs_pendingblocks += datablocks; 6947 UFS_UNLOCK(ump); 6948 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks); 6949 /* 6950 * Push the zero'ed inode to its disk buffer so that we are free 6951 * to delete its dependencies below. Once the dependencies are gone 6952 * the buffer can be safely released. 6953 */ 6954 if (ump->um_fstype == UFS1) { 6955 dp1 = ((struct ufs1_dinode *)bp->b_data + 6956 ino_to_fsbo(fs, ip->i_number)); 6957 ip->i_din1->di_freelink = dp1->di_freelink; 6958 *dp1 = *ip->i_din1; 6959 } else { 6960 dp2 = ((struct ufs2_dinode *)bp->b_data + 6961 ino_to_fsbo(fs, ip->i_number)); 6962 ip->i_din2->di_freelink = dp2->di_freelink; 6963 ffs_update_dinode_ckhash(fs, ip->i_din2); 6964 *dp2 = *ip->i_din2; 6965 } 6966 /* 6967 * Find and eliminate any inode dependencies. 6968 */ 6969 ACQUIRE_LOCK(ump); 6970 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6971 if ((inodedep->id_state & IOSTARTED) != 0) 6972 panic("softdep_setup_freeblocks: inode busy"); 6973 /* 6974 * Add the freeblks structure to the list of operations that 6975 * must await the zero'ed inode being written to disk. If we 6976 * still have a bitmap dependency (delay == 0), then the inode 6977 * has never been written to disk, so we can process the 6978 * freeblks below once we have deleted the dependencies. 6979 */ 6980 delay = (inodedep->id_state & DEPCOMPLETE); 6981 if (delay) 6982 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list); 6983 else 6984 freeblks->fb_state |= COMPLETE; 6985 /* 6986 * Because the file length has been truncated to zero, any 6987 * pending block allocation dependency structures associated 6988 * with this inode are obsolete and can simply be de-allocated. 6989 * We must first merge the two dependency lists to get rid of 6990 * any duplicate freefrag structures, then purge the merged list. 6991 * If we still have a bitmap dependency, then the inode has never 6992 * been written to disk, so we can free any fragments without delay. 6993 */ 6994 if (flags & IO_NORMAL) { 6995 merge_inode_lists(&inodedep->id_newinoupdt, 6996 &inodedep->id_inoupdt); 6997 while ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL) 6998 cancel_allocdirect(&inodedep->id_inoupdt, adp, 6999 freeblks); 7000 } 7001 if (flags & IO_EXT) { 7002 merge_inode_lists(&inodedep->id_newextupdt, 7003 &inodedep->id_extupdt); 7004 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 7005 cancel_allocdirect(&inodedep->id_extupdt, adp, 7006 freeblks); 7007 } 7008 FREE_LOCK(ump); 7009 bdwrite(bp); 7010 trunc_dependencies(ip, freeblks, -1, 0, flags); 7011 ACQUIRE_LOCK(ump); 7012 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 7013 (void) free_inodedep(inodedep); 7014 freeblks->fb_state |= DEPCOMPLETE; 7015 /* 7016 * If the inode with zeroed block pointers is now on disk 7017 * we can start freeing blocks. 7018 */ 7019 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) 7020 freeblks->fb_state |= INPROGRESS; 7021 else 7022 freeblks = NULL; 7023 FREE_LOCK(ump); 7024 if (freeblks) 7025 handle_workitem_freeblocks(freeblks, 0); 7026 trunc_pages(ip, length, extblocks, flags); 7027 } 7028 7029 /* 7030 * Eliminate pages from the page cache that back parts of this inode and 7031 * adjust the vnode pager's idea of our size. This prevents stale data 7032 * from hanging around in the page cache. 7033 */ 7034 static void 7035 trunc_pages(ip, length, extblocks, flags) 7036 struct inode *ip; 7037 off_t length; 7038 ufs2_daddr_t extblocks; 7039 int flags; 7040 { 7041 struct vnode *vp; 7042 struct fs *fs; 7043 ufs_lbn_t lbn; 7044 off_t end, extend; 7045 7046 vp = ITOV(ip); 7047 fs = ITOFS(ip); 7048 extend = OFF_TO_IDX(lblktosize(fs, -extblocks)); 7049 if ((flags & IO_EXT) != 0) 7050 vn_pages_remove(vp, extend, 0); 7051 if ((flags & IO_NORMAL) == 0) 7052 return; 7053 BO_LOCK(&vp->v_bufobj); 7054 drain_output(vp); 7055 BO_UNLOCK(&vp->v_bufobj); 7056 /* 7057 * The vnode pager eliminates file pages we eliminate indirects 7058 * below. 7059 */ 7060 vnode_pager_setsize(vp, length); 7061 /* 7062 * Calculate the end based on the last indirect we want to keep. If 7063 * the block extends into indirects we can just use the negative of 7064 * its lbn. Doubles and triples exist at lower numbers so we must 7065 * be careful not to remove those, if they exist. double and triple 7066 * indirect lbns do not overlap with others so it is not important 7067 * to verify how many levels are required. 7068 */ 7069 lbn = lblkno(fs, length); 7070 if (lbn >= UFS_NDADDR) { 7071 /* Calculate the virtual lbn of the triple indirect. */ 7072 lbn = -lbn - (UFS_NIADDR - 1); 7073 end = OFF_TO_IDX(lblktosize(fs, lbn)); 7074 } else 7075 end = extend; 7076 vn_pages_remove(vp, OFF_TO_IDX(OFF_MAX), end); 7077 } 7078 7079 /* 7080 * See if the buf bp is in the range eliminated by truncation. 7081 */ 7082 static int 7083 trunc_check_buf(bp, blkoffp, lastlbn, lastoff, flags) 7084 struct buf *bp; 7085 int *blkoffp; 7086 ufs_lbn_t lastlbn; 7087 int lastoff; 7088 int flags; 7089 { 7090 ufs_lbn_t lbn; 7091 7092 *blkoffp = 0; 7093 /* Only match ext/normal blocks as appropriate. */ 7094 if (((flags & IO_EXT) == 0 && (bp->b_xflags & BX_ALTDATA)) || 7095 ((flags & IO_NORMAL) == 0 && (bp->b_xflags & BX_ALTDATA) == 0)) 7096 return (0); 7097 /* ALTDATA is always a full truncation. */ 7098 if ((bp->b_xflags & BX_ALTDATA) != 0) 7099 return (1); 7100 /* -1 is full truncation. */ 7101 if (lastlbn == -1) 7102 return (1); 7103 /* 7104 * If this is a partial truncate we only want those 7105 * blocks and indirect blocks that cover the range 7106 * we're after. 7107 */ 7108 lbn = bp->b_lblkno; 7109 if (lbn < 0) 7110 lbn = -(lbn + lbn_level(lbn)); 7111 if (lbn < lastlbn) 7112 return (0); 7113 /* Here we only truncate lblkno if it's partial. */ 7114 if (lbn == lastlbn) { 7115 if (lastoff == 0) 7116 return (0); 7117 *blkoffp = lastoff; 7118 } 7119 return (1); 7120 } 7121 7122 /* 7123 * Eliminate any dependencies that exist in memory beyond lblkno:off 7124 */ 7125 static void 7126 trunc_dependencies(ip, freeblks, lastlbn, lastoff, flags) 7127 struct inode *ip; 7128 struct freeblks *freeblks; 7129 ufs_lbn_t lastlbn; 7130 int lastoff; 7131 int flags; 7132 { 7133 struct bufobj *bo; 7134 struct vnode *vp; 7135 struct buf *bp; 7136 int blkoff; 7137 7138 /* 7139 * We must wait for any I/O in progress to finish so that 7140 * all potential buffers on the dirty list will be visible. 7141 * Once they are all there, walk the list and get rid of 7142 * any dependencies. 7143 */ 7144 vp = ITOV(ip); 7145 bo = &vp->v_bufobj; 7146 BO_LOCK(bo); 7147 drain_output(vp); 7148 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) 7149 bp->b_vflags &= ~BV_SCANNED; 7150 restart: 7151 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) { 7152 if (bp->b_vflags & BV_SCANNED) 7153 continue; 7154 if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) { 7155 bp->b_vflags |= BV_SCANNED; 7156 continue; 7157 } 7158 KASSERT(bp->b_bufobj == bo, ("Wrong object in buffer")); 7159 if ((bp = getdirtybuf(bp, BO_LOCKPTR(bo), MNT_WAIT)) == NULL) 7160 goto restart; 7161 BO_UNLOCK(bo); 7162 if (deallocate_dependencies(bp, freeblks, blkoff)) 7163 bqrelse(bp); 7164 else 7165 brelse(bp); 7166 BO_LOCK(bo); 7167 goto restart; 7168 } 7169 /* 7170 * Now do the work of vtruncbuf while also matching indirect blocks. 7171 */ 7172 TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs) 7173 bp->b_vflags &= ~BV_SCANNED; 7174 cleanrestart: 7175 TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs) { 7176 if (bp->b_vflags & BV_SCANNED) 7177 continue; 7178 if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) { 7179 bp->b_vflags |= BV_SCANNED; 7180 continue; 7181 } 7182 if (BUF_LOCK(bp, 7183 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 7184 BO_LOCKPTR(bo)) == ENOLCK) { 7185 BO_LOCK(bo); 7186 goto cleanrestart; 7187 } 7188 bp->b_vflags |= BV_SCANNED; 7189 bremfree(bp); 7190 if (blkoff != 0) { 7191 allocbuf(bp, blkoff); 7192 bqrelse(bp); 7193 } else { 7194 bp->b_flags |= B_INVAL | B_NOCACHE | B_RELBUF; 7195 brelse(bp); 7196 } 7197 BO_LOCK(bo); 7198 goto cleanrestart; 7199 } 7200 drain_output(vp); 7201 BO_UNLOCK(bo); 7202 } 7203 7204 static int 7205 cancel_pagedep(pagedep, freeblks, blkoff) 7206 struct pagedep *pagedep; 7207 struct freeblks *freeblks; 7208 int blkoff; 7209 { 7210 struct jremref *jremref; 7211 struct jmvref *jmvref; 7212 struct dirrem *dirrem, *tmp; 7213 int i; 7214 7215 /* 7216 * Copy any directory remove dependencies to the list 7217 * to be processed after the freeblks proceeds. If 7218 * directory entry never made it to disk they 7219 * can be dumped directly onto the work list. 7220 */ 7221 LIST_FOREACH_SAFE(dirrem, &pagedep->pd_dirremhd, dm_next, tmp) { 7222 /* Skip this directory removal if it is intended to remain. */ 7223 if (dirrem->dm_offset < blkoff) 7224 continue; 7225 /* 7226 * If there are any dirrems we wait for the journal write 7227 * to complete and then restart the buf scan as the lock 7228 * has been dropped. 7229 */ 7230 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) { 7231 jwait(&jremref->jr_list, MNT_WAIT); 7232 return (ERESTART); 7233 } 7234 LIST_REMOVE(dirrem, dm_next); 7235 dirrem->dm_dirinum = pagedep->pd_ino; 7236 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &dirrem->dm_list); 7237 } 7238 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) { 7239 jwait(&jmvref->jm_list, MNT_WAIT); 7240 return (ERESTART); 7241 } 7242 /* 7243 * When we're partially truncating a pagedep we just want to flush 7244 * journal entries and return. There can not be any adds in the 7245 * truncated portion of the directory and newblk must remain if 7246 * part of the block remains. 7247 */ 7248 if (blkoff != 0) { 7249 struct diradd *dap; 7250 7251 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist) 7252 if (dap->da_offset > blkoff) 7253 panic("cancel_pagedep: diradd %p off %d > %d", 7254 dap, dap->da_offset, blkoff); 7255 for (i = 0; i < DAHASHSZ; i++) 7256 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) 7257 if (dap->da_offset > blkoff) 7258 panic("cancel_pagedep: diradd %p off %d > %d", 7259 dap, dap->da_offset, blkoff); 7260 return (0); 7261 } 7262 /* 7263 * There should be no directory add dependencies present 7264 * as the directory could not be truncated until all 7265 * children were removed. 7266 */ 7267 KASSERT(LIST_FIRST(&pagedep->pd_pendinghd) == NULL, 7268 ("deallocate_dependencies: pendinghd != NULL")); 7269 for (i = 0; i < DAHASHSZ; i++) 7270 KASSERT(LIST_FIRST(&pagedep->pd_diraddhd[i]) == NULL, 7271 ("deallocate_dependencies: diraddhd != NULL")); 7272 if ((pagedep->pd_state & NEWBLOCK) != 0) 7273 free_newdirblk(pagedep->pd_newdirblk); 7274 if (free_pagedep(pagedep) == 0) 7275 panic("Failed to free pagedep %p", pagedep); 7276 return (0); 7277 } 7278 7279 /* 7280 * Reclaim any dependency structures from a buffer that is about to 7281 * be reallocated to a new vnode. The buffer must be locked, thus, 7282 * no I/O completion operations can occur while we are manipulating 7283 * its associated dependencies. The mutex is held so that other I/O's 7284 * associated with related dependencies do not occur. 7285 */ 7286 static int 7287 deallocate_dependencies(bp, freeblks, off) 7288 struct buf *bp; 7289 struct freeblks *freeblks; 7290 int off; 7291 { 7292 struct indirdep *indirdep; 7293 struct pagedep *pagedep; 7294 struct worklist *wk, *wkn; 7295 struct ufsmount *ump; 7296 7297 ump = softdep_bp_to_mp(bp); 7298 if (ump == NULL) 7299 goto done; 7300 ACQUIRE_LOCK(ump); 7301 LIST_FOREACH_SAFE(wk, &bp->b_dep, wk_list, wkn) { 7302 switch (wk->wk_type) { 7303 case D_INDIRDEP: 7304 indirdep = WK_INDIRDEP(wk); 7305 if (bp->b_lblkno >= 0 || 7306 bp->b_blkno != indirdep->ir_savebp->b_lblkno) 7307 panic("deallocate_dependencies: not indir"); 7308 cancel_indirdep(indirdep, bp, freeblks); 7309 continue; 7310 7311 case D_PAGEDEP: 7312 pagedep = WK_PAGEDEP(wk); 7313 if (cancel_pagedep(pagedep, freeblks, off)) { 7314 FREE_LOCK(ump); 7315 return (ERESTART); 7316 } 7317 continue; 7318 7319 case D_ALLOCINDIR: 7320 /* 7321 * Simply remove the allocindir, we'll find it via 7322 * the indirdep where we can clear pointers if 7323 * needed. 7324 */ 7325 WORKLIST_REMOVE(wk); 7326 continue; 7327 7328 case D_FREEWORK: 7329 /* 7330 * A truncation is waiting for the zero'd pointers 7331 * to be written. It can be freed when the freeblks 7332 * is journaled. 7333 */ 7334 WORKLIST_REMOVE(wk); 7335 wk->wk_state |= ONDEPLIST; 7336 WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk); 7337 break; 7338 7339 case D_ALLOCDIRECT: 7340 if (off != 0) 7341 continue; 7342 /* FALLTHROUGH */ 7343 default: 7344 panic("deallocate_dependencies: Unexpected type %s", 7345 TYPENAME(wk->wk_type)); 7346 /* NOTREACHED */ 7347 } 7348 } 7349 FREE_LOCK(ump); 7350 done: 7351 /* 7352 * Don't throw away this buf, we were partially truncating and 7353 * some deps may always remain. 7354 */ 7355 if (off) { 7356 allocbuf(bp, off); 7357 bp->b_vflags |= BV_SCANNED; 7358 return (EBUSY); 7359 } 7360 bp->b_flags |= B_INVAL | B_NOCACHE; 7361 7362 return (0); 7363 } 7364 7365 /* 7366 * An allocdirect is being canceled due to a truncate. We must make sure 7367 * the journal entry is released in concert with the blkfree that releases 7368 * the storage. Completed journal entries must not be released until the 7369 * space is no longer pointed to by the inode or in the bitmap. 7370 */ 7371 static void 7372 cancel_allocdirect(adphead, adp, freeblks) 7373 struct allocdirectlst *adphead; 7374 struct allocdirect *adp; 7375 struct freeblks *freeblks; 7376 { 7377 struct freework *freework; 7378 struct newblk *newblk; 7379 struct worklist *wk; 7380 7381 TAILQ_REMOVE(adphead, adp, ad_next); 7382 newblk = (struct newblk *)adp; 7383 freework = NULL; 7384 /* 7385 * Find the correct freework structure. 7386 */ 7387 LIST_FOREACH(wk, &freeblks->fb_freeworkhd, wk_list) { 7388 if (wk->wk_type != D_FREEWORK) 7389 continue; 7390 freework = WK_FREEWORK(wk); 7391 if (freework->fw_blkno == newblk->nb_newblkno) 7392 break; 7393 } 7394 if (freework == NULL) 7395 panic("cancel_allocdirect: Freework not found"); 7396 /* 7397 * If a newblk exists at all we still have the journal entry that 7398 * initiated the allocation so we do not need to journal the free. 7399 */ 7400 cancel_jfreeblk(freeblks, freework->fw_blkno); 7401 /* 7402 * If the journal hasn't been written the jnewblk must be passed 7403 * to the call to ffs_blkfree that reclaims the space. We accomplish 7404 * this by linking the journal dependency into the freework to be 7405 * freed when freework_freeblock() is called. If the journal has 7406 * been written we can simply reclaim the journal space when the 7407 * freeblks work is complete. 7408 */ 7409 freework->fw_jnewblk = cancel_newblk(newblk, &freework->fw_list, 7410 &freeblks->fb_jwork); 7411 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list); 7412 } 7413 7414 7415 /* 7416 * Cancel a new block allocation. May be an indirect or direct block. We 7417 * remove it from various lists and return any journal record that needs to 7418 * be resolved by the caller. 7419 * 7420 * A special consideration is made for indirects which were never pointed 7421 * at on disk and will never be found once this block is released. 7422 */ 7423 static struct jnewblk * 7424 cancel_newblk(newblk, wk, wkhd) 7425 struct newblk *newblk; 7426 struct worklist *wk; 7427 struct workhead *wkhd; 7428 { 7429 struct jnewblk *jnewblk; 7430 7431 CTR1(KTR_SUJ, "cancel_newblk: blkno %jd", newblk->nb_newblkno); 7432 7433 newblk->nb_state |= GOINGAWAY; 7434 /* 7435 * Previously we traversed the completedhd on each indirdep 7436 * attached to this newblk to cancel them and gather journal 7437 * work. Since we need only the oldest journal segment and 7438 * the lowest point on the tree will always have the oldest 7439 * journal segment we are free to release the segments 7440 * of any subordinates and may leave the indirdep list to 7441 * indirdep_complete() when this newblk is freed. 7442 */ 7443 if (newblk->nb_state & ONDEPLIST) { 7444 newblk->nb_state &= ~ONDEPLIST; 7445 LIST_REMOVE(newblk, nb_deps); 7446 } 7447 if (newblk->nb_state & ONWORKLIST) 7448 WORKLIST_REMOVE(&newblk->nb_list); 7449 /* 7450 * If the journal entry hasn't been written we save a pointer to 7451 * the dependency that frees it until it is written or the 7452 * superseding operation completes. 7453 */ 7454 jnewblk = newblk->nb_jnewblk; 7455 if (jnewblk != NULL && wk != NULL) { 7456 newblk->nb_jnewblk = NULL; 7457 jnewblk->jn_dep = wk; 7458 } 7459 if (!LIST_EMPTY(&newblk->nb_jwork)) 7460 jwork_move(wkhd, &newblk->nb_jwork); 7461 /* 7462 * When truncating we must free the newdirblk early to remove 7463 * the pagedep from the hash before returning. 7464 */ 7465 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL) 7466 free_newdirblk(WK_NEWDIRBLK(wk)); 7467 if (!LIST_EMPTY(&newblk->nb_newdirblk)) 7468 panic("cancel_newblk: extra newdirblk"); 7469 7470 return (jnewblk); 7471 } 7472 7473 /* 7474 * Schedule the freefrag associated with a newblk to be released once 7475 * the pointers are written and the previous block is no longer needed. 7476 */ 7477 static void 7478 newblk_freefrag(newblk) 7479 struct newblk *newblk; 7480 { 7481 struct freefrag *freefrag; 7482 7483 if (newblk->nb_freefrag == NULL) 7484 return; 7485 freefrag = newblk->nb_freefrag; 7486 newblk->nb_freefrag = NULL; 7487 freefrag->ff_state |= COMPLETE; 7488 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 7489 add_to_worklist(&freefrag->ff_list, 0); 7490 } 7491 7492 /* 7493 * Free a newblk. Generate a new freefrag work request if appropriate. 7494 * This must be called after the inode pointer and any direct block pointers 7495 * are valid or fully removed via truncate or frag extension. 7496 */ 7497 static void 7498 free_newblk(newblk) 7499 struct newblk *newblk; 7500 { 7501 struct indirdep *indirdep; 7502 struct worklist *wk; 7503 7504 KASSERT(newblk->nb_jnewblk == NULL, 7505 ("free_newblk: jnewblk %p still attached", newblk->nb_jnewblk)); 7506 KASSERT(newblk->nb_list.wk_type != D_NEWBLK, 7507 ("free_newblk: unclaimed newblk")); 7508 LOCK_OWNED(VFSTOUFS(newblk->nb_list.wk_mp)); 7509 newblk_freefrag(newblk); 7510 if (newblk->nb_state & ONDEPLIST) 7511 LIST_REMOVE(newblk, nb_deps); 7512 if (newblk->nb_state & ONWORKLIST) 7513 WORKLIST_REMOVE(&newblk->nb_list); 7514 LIST_REMOVE(newblk, nb_hash); 7515 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL) 7516 free_newdirblk(WK_NEWDIRBLK(wk)); 7517 if (!LIST_EMPTY(&newblk->nb_newdirblk)) 7518 panic("free_newblk: extra newdirblk"); 7519 while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL) 7520 indirdep_complete(indirdep); 7521 handle_jwork(&newblk->nb_jwork); 7522 WORKITEM_FREE(newblk, D_NEWBLK); 7523 } 7524 7525 /* 7526 * Free a newdirblk. Clear the NEWBLOCK flag on its associated pagedep. 7527 */ 7528 static void 7529 free_newdirblk(newdirblk) 7530 struct newdirblk *newdirblk; 7531 { 7532 struct pagedep *pagedep; 7533 struct diradd *dap; 7534 struct worklist *wk; 7535 7536 LOCK_OWNED(VFSTOUFS(newdirblk->db_list.wk_mp)); 7537 WORKLIST_REMOVE(&newdirblk->db_list); 7538 /* 7539 * If the pagedep is still linked onto the directory buffer 7540 * dependency chain, then some of the entries on the 7541 * pd_pendinghd list may not be committed to disk yet. In 7542 * this case, we will simply clear the NEWBLOCK flag and 7543 * let the pd_pendinghd list be processed when the pagedep 7544 * is next written. If the pagedep is no longer on the buffer 7545 * dependency chain, then all the entries on the pd_pending 7546 * list are committed to disk and we can free them here. 7547 */ 7548 pagedep = newdirblk->db_pagedep; 7549 pagedep->pd_state &= ~NEWBLOCK; 7550 if ((pagedep->pd_state & ONWORKLIST) == 0) { 7551 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 7552 free_diradd(dap, NULL); 7553 /* 7554 * If no dependencies remain, the pagedep will be freed. 7555 */ 7556 free_pagedep(pagedep); 7557 } 7558 /* Should only ever be one item in the list. */ 7559 while ((wk = LIST_FIRST(&newdirblk->db_mkdir)) != NULL) { 7560 WORKLIST_REMOVE(wk); 7561 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 7562 } 7563 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 7564 } 7565 7566 /* 7567 * Prepare an inode to be freed. The actual free operation is not 7568 * done until the zero'ed inode has been written to disk. 7569 */ 7570 void 7571 softdep_freefile(pvp, ino, mode) 7572 struct vnode *pvp; 7573 ino_t ino; 7574 int mode; 7575 { 7576 struct inode *ip = VTOI(pvp); 7577 struct inodedep *inodedep; 7578 struct freefile *freefile; 7579 struct freeblks *freeblks; 7580 struct ufsmount *ump; 7581 7582 ump = ITOUMP(ip); 7583 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 7584 ("softdep_freefile called on non-softdep filesystem")); 7585 /* 7586 * This sets up the inode de-allocation dependency. 7587 */ 7588 freefile = malloc(sizeof(struct freefile), 7589 M_FREEFILE, M_SOFTDEP_FLAGS); 7590 workitem_alloc(&freefile->fx_list, D_FREEFILE, pvp->v_mount); 7591 freefile->fx_mode = mode; 7592 freefile->fx_oldinum = ino; 7593 freefile->fx_devvp = ump->um_devvp; 7594 LIST_INIT(&freefile->fx_jwork); 7595 UFS_LOCK(ump); 7596 ump->um_fs->fs_pendinginodes += 1; 7597 UFS_UNLOCK(ump); 7598 7599 /* 7600 * If the inodedep does not exist, then the zero'ed inode has 7601 * been written to disk. If the allocated inode has never been 7602 * written to disk, then the on-disk inode is zero'ed. In either 7603 * case we can free the file immediately. If the journal was 7604 * canceled before being written the inode will never make it to 7605 * disk and we must send the canceled journal entrys to 7606 * ffs_freefile() to be cleared in conjunction with the bitmap. 7607 * Any blocks waiting on the inode to write can be safely freed 7608 * here as it will never been written. 7609 */ 7610 ACQUIRE_LOCK(ump); 7611 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 7612 if (inodedep) { 7613 /* 7614 * Clear out freeblks that no longer need to reference 7615 * this inode. 7616 */ 7617 while ((freeblks = 7618 TAILQ_FIRST(&inodedep->id_freeblklst)) != NULL) { 7619 TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks, 7620 fb_next); 7621 freeblks->fb_state &= ~ONDEPLIST; 7622 } 7623 /* 7624 * Remove this inode from the unlinked list. 7625 */ 7626 if (inodedep->id_state & UNLINKED) { 7627 /* 7628 * Save the journal work to be freed with the bitmap 7629 * before we clear UNLINKED. Otherwise it can be lost 7630 * if the inode block is written. 7631 */ 7632 handle_bufwait(inodedep, &freefile->fx_jwork); 7633 clear_unlinked_inodedep(inodedep); 7634 /* 7635 * Re-acquire inodedep as we've dropped the 7636 * per-filesystem lock in clear_unlinked_inodedep(). 7637 */ 7638 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 7639 } 7640 } 7641 if (inodedep == NULL || check_inode_unwritten(inodedep)) { 7642 FREE_LOCK(ump); 7643 handle_workitem_freefile(freefile); 7644 return; 7645 } 7646 if ((inodedep->id_state & DEPCOMPLETE) == 0) 7647 inodedep->id_state |= GOINGAWAY; 7648 WORKLIST_INSERT(&inodedep->id_inowait, &freefile->fx_list); 7649 FREE_LOCK(ump); 7650 if (ip->i_number == ino) 7651 ip->i_flag |= IN_MODIFIED; 7652 } 7653 7654 /* 7655 * Check to see if an inode has never been written to disk. If 7656 * so free the inodedep and return success, otherwise return failure. 7657 * 7658 * If we still have a bitmap dependency, then the inode has never 7659 * been written to disk. Drop the dependency as it is no longer 7660 * necessary since the inode is being deallocated. We set the 7661 * ALLCOMPLETE flags since the bitmap now properly shows that the 7662 * inode is not allocated. Even if the inode is actively being 7663 * written, it has been rolled back to its zero'ed state, so we 7664 * are ensured that a zero inode is what is on the disk. For short 7665 * lived files, this change will usually result in removing all the 7666 * dependencies from the inode so that it can be freed immediately. 7667 */ 7668 static int 7669 check_inode_unwritten(inodedep) 7670 struct inodedep *inodedep; 7671 { 7672 7673 LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp)); 7674 7675 if ((inodedep->id_state & (DEPCOMPLETE | UNLINKED)) != 0 || 7676 !LIST_EMPTY(&inodedep->id_dirremhd) || 7677 !LIST_EMPTY(&inodedep->id_pendinghd) || 7678 !LIST_EMPTY(&inodedep->id_bufwait) || 7679 !LIST_EMPTY(&inodedep->id_inowait) || 7680 !TAILQ_EMPTY(&inodedep->id_inoreflst) || 7681 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 7682 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 7683 !TAILQ_EMPTY(&inodedep->id_extupdt) || 7684 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 7685 !TAILQ_EMPTY(&inodedep->id_freeblklst) || 7686 inodedep->id_mkdiradd != NULL || 7687 inodedep->id_nlinkdelta != 0) 7688 return (0); 7689 /* 7690 * Another process might be in initiate_write_inodeblock_ufs[12] 7691 * trying to allocate memory without holding "Softdep Lock". 7692 */ 7693 if ((inodedep->id_state & IOSTARTED) != 0 && 7694 inodedep->id_savedino1 == NULL) 7695 return (0); 7696 7697 if (inodedep->id_state & ONDEPLIST) 7698 LIST_REMOVE(inodedep, id_deps); 7699 inodedep->id_state &= ~ONDEPLIST; 7700 inodedep->id_state |= ALLCOMPLETE; 7701 inodedep->id_bmsafemap = NULL; 7702 if (inodedep->id_state & ONWORKLIST) 7703 WORKLIST_REMOVE(&inodedep->id_list); 7704 if (inodedep->id_savedino1 != NULL) { 7705 free(inodedep->id_savedino1, M_SAVEDINO); 7706 inodedep->id_savedino1 = NULL; 7707 } 7708 if (free_inodedep(inodedep) == 0) 7709 panic("check_inode_unwritten: busy inode"); 7710 return (1); 7711 } 7712 7713 static int 7714 check_inodedep_free(inodedep) 7715 struct inodedep *inodedep; 7716 { 7717 7718 LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp)); 7719 if ((inodedep->id_state & ALLCOMPLETE) != ALLCOMPLETE || 7720 !LIST_EMPTY(&inodedep->id_dirremhd) || 7721 !LIST_EMPTY(&inodedep->id_pendinghd) || 7722 !LIST_EMPTY(&inodedep->id_bufwait) || 7723 !LIST_EMPTY(&inodedep->id_inowait) || 7724 !TAILQ_EMPTY(&inodedep->id_inoreflst) || 7725 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 7726 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 7727 !TAILQ_EMPTY(&inodedep->id_extupdt) || 7728 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 7729 !TAILQ_EMPTY(&inodedep->id_freeblklst) || 7730 inodedep->id_mkdiradd != NULL || 7731 inodedep->id_nlinkdelta != 0 || 7732 inodedep->id_savedino1 != NULL) 7733 return (0); 7734 return (1); 7735 } 7736 7737 /* 7738 * Try to free an inodedep structure. Return 1 if it could be freed. 7739 */ 7740 static int 7741 free_inodedep(inodedep) 7742 struct inodedep *inodedep; 7743 { 7744 7745 LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp)); 7746 if ((inodedep->id_state & (ONWORKLIST | UNLINKED)) != 0 || 7747 !check_inodedep_free(inodedep)) 7748 return (0); 7749 if (inodedep->id_state & ONDEPLIST) 7750 LIST_REMOVE(inodedep, id_deps); 7751 LIST_REMOVE(inodedep, id_hash); 7752 WORKITEM_FREE(inodedep, D_INODEDEP); 7753 return (1); 7754 } 7755 7756 /* 7757 * Free the block referenced by a freework structure. The parent freeblks 7758 * structure is released and completed when the final cg bitmap reaches 7759 * the disk. This routine may be freeing a jnewblk which never made it to 7760 * disk in which case we do not have to wait as the operation is undone 7761 * in memory immediately. 7762 */ 7763 static void 7764 freework_freeblock(freework, key) 7765 struct freework *freework; 7766 u_long key; 7767 { 7768 struct freeblks *freeblks; 7769 struct jnewblk *jnewblk; 7770 struct ufsmount *ump; 7771 struct workhead wkhd; 7772 struct fs *fs; 7773 int bsize; 7774 int needj; 7775 7776 ump = VFSTOUFS(freework->fw_list.wk_mp); 7777 LOCK_OWNED(ump); 7778 /* 7779 * Handle partial truncate separately. 7780 */ 7781 if (freework->fw_indir) { 7782 complete_trunc_indir(freework); 7783 return; 7784 } 7785 freeblks = freework->fw_freeblks; 7786 fs = ump->um_fs; 7787 needj = MOUNTEDSUJ(freeblks->fb_list.wk_mp) != 0; 7788 bsize = lfragtosize(fs, freework->fw_frags); 7789 LIST_INIT(&wkhd); 7790 /* 7791 * DEPCOMPLETE is cleared in indirblk_insert() if the block lives 7792 * on the indirblk hashtable and prevents premature freeing. 7793 */ 7794 freework->fw_state |= DEPCOMPLETE; 7795 /* 7796 * SUJ needs to wait for the segment referencing freed indirect 7797 * blocks to expire so that we know the checker will not confuse 7798 * a re-allocated indirect block with its old contents. 7799 */ 7800 if (needj && freework->fw_lbn <= -UFS_NDADDR) 7801 indirblk_insert(freework); 7802 /* 7803 * If we are canceling an existing jnewblk pass it to the free 7804 * routine, otherwise pass the freeblk which will ultimately 7805 * release the freeblks. If we're not journaling, we can just 7806 * free the freeblks immediately. 7807 */ 7808 jnewblk = freework->fw_jnewblk; 7809 if (jnewblk != NULL) { 7810 cancel_jnewblk(jnewblk, &wkhd); 7811 needj = 0; 7812 } else if (needj) { 7813 freework->fw_state |= DELAYEDFREE; 7814 freeblks->fb_cgwait++; 7815 WORKLIST_INSERT(&wkhd, &freework->fw_list); 7816 } 7817 FREE_LOCK(ump); 7818 freeblks_free(ump, freeblks, btodb(bsize)); 7819 CTR4(KTR_SUJ, 7820 "freework_freeblock: ino %jd blkno %jd lbn %jd size %d", 7821 freeblks->fb_inum, freework->fw_blkno, freework->fw_lbn, bsize); 7822 ffs_blkfree(ump, fs, freeblks->fb_devvp, freework->fw_blkno, bsize, 7823 freeblks->fb_inum, freeblks->fb_vtype, &wkhd, key); 7824 ACQUIRE_LOCK(ump); 7825 /* 7826 * The jnewblk will be discarded and the bits in the map never 7827 * made it to disk. We can immediately free the freeblk. 7828 */ 7829 if (needj == 0) 7830 handle_written_freework(freework); 7831 } 7832 7833 /* 7834 * We enqueue freework items that need processing back on the freeblks and 7835 * add the freeblks to the worklist. This makes it easier to find all work 7836 * required to flush a truncation in process_truncates(). 7837 */ 7838 static void 7839 freework_enqueue(freework) 7840 struct freework *freework; 7841 { 7842 struct freeblks *freeblks; 7843 7844 freeblks = freework->fw_freeblks; 7845 if ((freework->fw_state & INPROGRESS) == 0) 7846 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list); 7847 if ((freeblks->fb_state & 7848 (ONWORKLIST | INPROGRESS | ALLCOMPLETE)) == ALLCOMPLETE && 7849 LIST_EMPTY(&freeblks->fb_jblkdephd)) 7850 add_to_worklist(&freeblks->fb_list, WK_NODELAY); 7851 } 7852 7853 /* 7854 * Start, continue, or finish the process of freeing an indirect block tree. 7855 * The free operation may be paused at any point with fw_off containing the 7856 * offset to restart from. This enables us to implement some flow control 7857 * for large truncates which may fan out and generate a huge number of 7858 * dependencies. 7859 */ 7860 static void 7861 handle_workitem_indirblk(freework) 7862 struct freework *freework; 7863 { 7864 struct freeblks *freeblks; 7865 struct ufsmount *ump; 7866 struct fs *fs; 7867 7868 freeblks = freework->fw_freeblks; 7869 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 7870 fs = ump->um_fs; 7871 if (freework->fw_state & DEPCOMPLETE) { 7872 handle_written_freework(freework); 7873 return; 7874 } 7875 if (freework->fw_off == NINDIR(fs)) { 7876 freework_freeblock(freework, SINGLETON_KEY); 7877 return; 7878 } 7879 freework->fw_state |= INPROGRESS; 7880 FREE_LOCK(ump); 7881 indir_trunc(freework, fsbtodb(fs, freework->fw_blkno), 7882 freework->fw_lbn); 7883 ACQUIRE_LOCK(ump); 7884 } 7885 7886 /* 7887 * Called when a freework structure attached to a cg buf is written. The 7888 * ref on either the parent or the freeblks structure is released and 7889 * the freeblks is added back to the worklist if there is more work to do. 7890 */ 7891 static void 7892 handle_written_freework(freework) 7893 struct freework *freework; 7894 { 7895 struct freeblks *freeblks; 7896 struct freework *parent; 7897 7898 freeblks = freework->fw_freeblks; 7899 parent = freework->fw_parent; 7900 if (freework->fw_state & DELAYEDFREE) 7901 freeblks->fb_cgwait--; 7902 freework->fw_state |= COMPLETE; 7903 if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE) 7904 WORKITEM_FREE(freework, D_FREEWORK); 7905 if (parent) { 7906 if (--parent->fw_ref == 0) 7907 freework_enqueue(parent); 7908 return; 7909 } 7910 if (--freeblks->fb_ref != 0) 7911 return; 7912 if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST | INPROGRESS)) == 7913 ALLCOMPLETE && LIST_EMPTY(&freeblks->fb_jblkdephd)) 7914 add_to_worklist(&freeblks->fb_list, WK_NODELAY); 7915 } 7916 7917 /* 7918 * This workitem routine performs the block de-allocation. 7919 * The workitem is added to the pending list after the updated 7920 * inode block has been written to disk. As mentioned above, 7921 * checks regarding the number of blocks de-allocated (compared 7922 * to the number of blocks allocated for the file) are also 7923 * performed in this function. 7924 */ 7925 static int 7926 handle_workitem_freeblocks(freeblks, flags) 7927 struct freeblks *freeblks; 7928 int flags; 7929 { 7930 struct freework *freework; 7931 struct newblk *newblk; 7932 struct allocindir *aip; 7933 struct ufsmount *ump; 7934 struct worklist *wk; 7935 u_long key; 7936 7937 KASSERT(LIST_EMPTY(&freeblks->fb_jblkdephd), 7938 ("handle_workitem_freeblocks: Journal entries not written.")); 7939 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 7940 key = ffs_blkrelease_start(ump, freeblks->fb_devvp, freeblks->fb_inum); 7941 ACQUIRE_LOCK(ump); 7942 while ((wk = LIST_FIRST(&freeblks->fb_freeworkhd)) != NULL) { 7943 WORKLIST_REMOVE(wk); 7944 switch (wk->wk_type) { 7945 case D_DIRREM: 7946 wk->wk_state |= COMPLETE; 7947 add_to_worklist(wk, 0); 7948 continue; 7949 7950 case D_ALLOCDIRECT: 7951 free_newblk(WK_NEWBLK(wk)); 7952 continue; 7953 7954 case D_ALLOCINDIR: 7955 aip = WK_ALLOCINDIR(wk); 7956 freework = NULL; 7957 if (aip->ai_state & DELAYEDFREE) { 7958 FREE_LOCK(ump); 7959 freework = newfreework(ump, freeblks, NULL, 7960 aip->ai_lbn, aip->ai_newblkno, 7961 ump->um_fs->fs_frag, 0, 0); 7962 ACQUIRE_LOCK(ump); 7963 } 7964 newblk = WK_NEWBLK(wk); 7965 if (newblk->nb_jnewblk) { 7966 freework->fw_jnewblk = newblk->nb_jnewblk; 7967 newblk->nb_jnewblk->jn_dep = &freework->fw_list; 7968 newblk->nb_jnewblk = NULL; 7969 } 7970 free_newblk(newblk); 7971 continue; 7972 7973 case D_FREEWORK: 7974 freework = WK_FREEWORK(wk); 7975 if (freework->fw_lbn <= -UFS_NDADDR) 7976 handle_workitem_indirblk(freework); 7977 else 7978 freework_freeblock(freework, key); 7979 continue; 7980 default: 7981 panic("handle_workitem_freeblocks: Unknown type %s", 7982 TYPENAME(wk->wk_type)); 7983 } 7984 } 7985 if (freeblks->fb_ref != 0) { 7986 freeblks->fb_state &= ~INPROGRESS; 7987 wake_worklist(&freeblks->fb_list); 7988 freeblks = NULL; 7989 } 7990 FREE_LOCK(ump); 7991 ffs_blkrelease_finish(ump, key); 7992 if (freeblks) 7993 return handle_complete_freeblocks(freeblks, flags); 7994 return (0); 7995 } 7996 7997 /* 7998 * Handle completion of block free via truncate. This allows fs_pending 7999 * to track the actual free block count more closely than if we only updated 8000 * it at the end. We must be careful to handle cases where the block count 8001 * on free was incorrect. 8002 */ 8003 static void 8004 freeblks_free(ump, freeblks, blocks) 8005 struct ufsmount *ump; 8006 struct freeblks *freeblks; 8007 int blocks; 8008 { 8009 struct fs *fs; 8010 ufs2_daddr_t remain; 8011 8012 UFS_LOCK(ump); 8013 remain = -freeblks->fb_chkcnt; 8014 freeblks->fb_chkcnt += blocks; 8015 if (remain > 0) { 8016 if (remain < blocks) 8017 blocks = remain; 8018 fs = ump->um_fs; 8019 fs->fs_pendingblocks -= blocks; 8020 } 8021 UFS_UNLOCK(ump); 8022 } 8023 8024 /* 8025 * Once all of the freework workitems are complete we can retire the 8026 * freeblocks dependency and any journal work awaiting completion. This 8027 * can not be called until all other dependencies are stable on disk. 8028 */ 8029 static int 8030 handle_complete_freeblocks(freeblks, flags) 8031 struct freeblks *freeblks; 8032 int flags; 8033 { 8034 struct inodedep *inodedep; 8035 struct inode *ip; 8036 struct vnode *vp; 8037 struct fs *fs; 8038 struct ufsmount *ump; 8039 ufs2_daddr_t spare; 8040 8041 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 8042 fs = ump->um_fs; 8043 flags = LK_EXCLUSIVE | flags; 8044 spare = freeblks->fb_chkcnt; 8045 8046 /* 8047 * If we did not release the expected number of blocks we may have 8048 * to adjust the inode block count here. Only do so if it wasn't 8049 * a truncation to zero and the modrev still matches. 8050 */ 8051 if (spare && freeblks->fb_len != 0) { 8052 if (ffs_vgetf(freeblks->fb_list.wk_mp, freeblks->fb_inum, 8053 flags, &vp, FFSV_FORCEINSMQ) != 0) 8054 return (EBUSY); 8055 ip = VTOI(vp); 8056 if (DIP(ip, i_modrev) == freeblks->fb_modrev) { 8057 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - spare); 8058 ip->i_flag |= IN_CHANGE; 8059 /* 8060 * We must wait so this happens before the 8061 * journal is reclaimed. 8062 */ 8063 ffs_update(vp, 1); 8064 } 8065 vput(vp); 8066 } 8067 if (spare < 0) { 8068 UFS_LOCK(ump); 8069 fs->fs_pendingblocks += spare; 8070 UFS_UNLOCK(ump); 8071 } 8072 #ifdef QUOTA 8073 /* Handle spare. */ 8074 if (spare) 8075 quotaadj(freeblks->fb_quota, ump, -spare); 8076 quotarele(freeblks->fb_quota); 8077 #endif 8078 ACQUIRE_LOCK(ump); 8079 if (freeblks->fb_state & ONDEPLIST) { 8080 inodedep_lookup(freeblks->fb_list.wk_mp, freeblks->fb_inum, 8081 0, &inodedep); 8082 TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks, fb_next); 8083 freeblks->fb_state &= ~ONDEPLIST; 8084 if (TAILQ_EMPTY(&inodedep->id_freeblklst)) 8085 free_inodedep(inodedep); 8086 } 8087 /* 8088 * All of the freeblock deps must be complete prior to this call 8089 * so it's now safe to complete earlier outstanding journal entries. 8090 */ 8091 handle_jwork(&freeblks->fb_jwork); 8092 WORKITEM_FREE(freeblks, D_FREEBLKS); 8093 FREE_LOCK(ump); 8094 return (0); 8095 } 8096 8097 /* 8098 * Release blocks associated with the freeblks and stored in the indirect 8099 * block dbn. If level is greater than SINGLE, the block is an indirect block 8100 * and recursive calls to indirtrunc must be used to cleanse other indirect 8101 * blocks. 8102 * 8103 * This handles partial and complete truncation of blocks. Partial is noted 8104 * with goingaway == 0. In this case the freework is completed after the 8105 * zero'd indirects are written to disk. For full truncation the freework 8106 * is completed after the block is freed. 8107 */ 8108 static void 8109 indir_trunc(freework, dbn, lbn) 8110 struct freework *freework; 8111 ufs2_daddr_t dbn; 8112 ufs_lbn_t lbn; 8113 { 8114 struct freework *nfreework; 8115 struct workhead wkhd; 8116 struct freeblks *freeblks; 8117 struct buf *bp; 8118 struct fs *fs; 8119 struct indirdep *indirdep; 8120 struct mount *mp; 8121 struct ufsmount *ump; 8122 ufs1_daddr_t *bap1; 8123 ufs2_daddr_t nb, nnb, *bap2; 8124 ufs_lbn_t lbnadd, nlbn; 8125 u_long key; 8126 int nblocks, ufs1fmt, freedblocks; 8127 int goingaway, freedeps, needj, level, cnt, i; 8128 8129 freeblks = freework->fw_freeblks; 8130 mp = freeblks->fb_list.wk_mp; 8131 ump = VFSTOUFS(mp); 8132 fs = ump->um_fs; 8133 /* 8134 * Get buffer of block pointers to be freed. There are three cases: 8135 * 8136 * 1) Partial truncate caches the indirdep pointer in the freework 8137 * which provides us a back copy to the save bp which holds the 8138 * pointers we want to clear. When this completes the zero 8139 * pointers are written to the real copy. 8140 * 2) The indirect is being completely truncated, cancel_indirdep() 8141 * eliminated the real copy and placed the indirdep on the saved 8142 * copy. The indirdep and buf are discarded when this completes. 8143 * 3) The indirect was not in memory, we read a copy off of the disk 8144 * using the devvp and drop and invalidate the buffer when we're 8145 * done. 8146 */ 8147 goingaway = 1; 8148 indirdep = NULL; 8149 if (freework->fw_indir != NULL) { 8150 goingaway = 0; 8151 indirdep = freework->fw_indir; 8152 bp = indirdep->ir_savebp; 8153 if (bp == NULL || bp->b_blkno != dbn) 8154 panic("indir_trunc: Bad saved buf %p blkno %jd", 8155 bp, (intmax_t)dbn); 8156 } else if ((bp = incore(&freeblks->fb_devvp->v_bufobj, dbn)) != NULL) { 8157 /* 8158 * The lock prevents the buf dep list from changing and 8159 * indirects on devvp should only ever have one dependency. 8160 */ 8161 indirdep = WK_INDIRDEP(LIST_FIRST(&bp->b_dep)); 8162 if (indirdep == NULL || (indirdep->ir_state & GOINGAWAY) == 0) 8163 panic("indir_trunc: Bad indirdep %p from buf %p", 8164 indirdep, bp); 8165 } else if (bread(freeblks->fb_devvp, dbn, (int)fs->fs_bsize, 8166 NOCRED, &bp) != 0) { 8167 brelse(bp); 8168 return; 8169 } 8170 ACQUIRE_LOCK(ump); 8171 /* Protects against a race with complete_trunc_indir(). */ 8172 freework->fw_state &= ~INPROGRESS; 8173 /* 8174 * If we have an indirdep we need to enforce the truncation order 8175 * and discard it when it is complete. 8176 */ 8177 if (indirdep) { 8178 if (freework != TAILQ_FIRST(&indirdep->ir_trunc) && 8179 !TAILQ_EMPTY(&indirdep->ir_trunc)) { 8180 /* 8181 * Add the complete truncate to the list on the 8182 * indirdep to enforce in-order processing. 8183 */ 8184 if (freework->fw_indir == NULL) 8185 TAILQ_INSERT_TAIL(&indirdep->ir_trunc, 8186 freework, fw_next); 8187 FREE_LOCK(ump); 8188 return; 8189 } 8190 /* 8191 * If we're goingaway, free the indirdep. Otherwise it will 8192 * linger until the write completes. 8193 */ 8194 if (goingaway) 8195 free_indirdep(indirdep); 8196 } 8197 FREE_LOCK(ump); 8198 /* Initialize pointers depending on block size. */ 8199 if (ump->um_fstype == UFS1) { 8200 bap1 = (ufs1_daddr_t *)bp->b_data; 8201 nb = bap1[freework->fw_off]; 8202 ufs1fmt = 1; 8203 bap2 = NULL; 8204 } else { 8205 bap2 = (ufs2_daddr_t *)bp->b_data; 8206 nb = bap2[freework->fw_off]; 8207 ufs1fmt = 0; 8208 bap1 = NULL; 8209 } 8210 level = lbn_level(lbn); 8211 needj = MOUNTEDSUJ(UFSTOVFS(ump)) != 0; 8212 lbnadd = lbn_offset(fs, level); 8213 nblocks = btodb(fs->fs_bsize); 8214 nfreework = freework; 8215 freedeps = 0; 8216 cnt = 0; 8217 /* 8218 * Reclaim blocks. Traverses into nested indirect levels and 8219 * arranges for the current level to be freed when subordinates 8220 * are free when journaling. 8221 */ 8222 key = ffs_blkrelease_start(ump, freeblks->fb_devvp, freeblks->fb_inum); 8223 for (i = freework->fw_off; i < NINDIR(fs); i++, nb = nnb) { 8224 if (UFS_CHECK_BLKNO(mp, freeblks->fb_inum, nb, 8225 fs->fs_bsize) != 0) 8226 nb = 0; 8227 if (i != NINDIR(fs) - 1) { 8228 if (ufs1fmt) 8229 nnb = bap1[i+1]; 8230 else 8231 nnb = bap2[i+1]; 8232 } else 8233 nnb = 0; 8234 if (nb == 0) 8235 continue; 8236 cnt++; 8237 if (level != 0) { 8238 nlbn = (lbn + 1) - (i * lbnadd); 8239 if (needj != 0) { 8240 nfreework = newfreework(ump, freeblks, freework, 8241 nlbn, nb, fs->fs_frag, 0, 0); 8242 freedeps++; 8243 } 8244 indir_trunc(nfreework, fsbtodb(fs, nb), nlbn); 8245 } else { 8246 struct freedep *freedep; 8247 8248 /* 8249 * Attempt to aggregate freedep dependencies for 8250 * all blocks being released to the same CG. 8251 */ 8252 LIST_INIT(&wkhd); 8253 if (needj != 0 && 8254 (nnb == 0 || (dtog(fs, nb) != dtog(fs, nnb)))) { 8255 freedep = newfreedep(freework); 8256 WORKLIST_INSERT_UNLOCKED(&wkhd, 8257 &freedep->fd_list); 8258 freedeps++; 8259 } 8260 CTR3(KTR_SUJ, 8261 "indir_trunc: ino %jd blkno %jd size %d", 8262 freeblks->fb_inum, nb, fs->fs_bsize); 8263 ffs_blkfree(ump, fs, freeblks->fb_devvp, nb, 8264 fs->fs_bsize, freeblks->fb_inum, 8265 freeblks->fb_vtype, &wkhd, key); 8266 } 8267 } 8268 ffs_blkrelease_finish(ump, key); 8269 if (goingaway) { 8270 bp->b_flags |= B_INVAL | B_NOCACHE; 8271 brelse(bp); 8272 } 8273 freedblocks = 0; 8274 if (level == 0) 8275 freedblocks = (nblocks * cnt); 8276 if (needj == 0) 8277 freedblocks += nblocks; 8278 freeblks_free(ump, freeblks, freedblocks); 8279 /* 8280 * If we are journaling set up the ref counts and offset so this 8281 * indirect can be completed when its children are free. 8282 */ 8283 if (needj) { 8284 ACQUIRE_LOCK(ump); 8285 freework->fw_off = i; 8286 freework->fw_ref += freedeps; 8287 freework->fw_ref -= NINDIR(fs) + 1; 8288 if (level == 0) 8289 freeblks->fb_cgwait += freedeps; 8290 if (freework->fw_ref == 0) 8291 freework_freeblock(freework, SINGLETON_KEY); 8292 FREE_LOCK(ump); 8293 return; 8294 } 8295 /* 8296 * If we're not journaling we can free the indirect now. 8297 */ 8298 dbn = dbtofsb(fs, dbn); 8299 CTR3(KTR_SUJ, 8300 "indir_trunc 2: ino %jd blkno %jd size %d", 8301 freeblks->fb_inum, dbn, fs->fs_bsize); 8302 ffs_blkfree(ump, fs, freeblks->fb_devvp, dbn, fs->fs_bsize, 8303 freeblks->fb_inum, freeblks->fb_vtype, NULL, SINGLETON_KEY); 8304 /* Non SUJ softdep does single-threaded truncations. */ 8305 if (freework->fw_blkno == dbn) { 8306 freework->fw_state |= ALLCOMPLETE; 8307 ACQUIRE_LOCK(ump); 8308 handle_written_freework(freework); 8309 FREE_LOCK(ump); 8310 } 8311 return; 8312 } 8313 8314 /* 8315 * Cancel an allocindir when it is removed via truncation. When bp is not 8316 * NULL the indirect never appeared on disk and is scheduled to be freed 8317 * independently of the indir so we can more easily track journal work. 8318 */ 8319 static void 8320 cancel_allocindir(aip, bp, freeblks, trunc) 8321 struct allocindir *aip; 8322 struct buf *bp; 8323 struct freeblks *freeblks; 8324 int trunc; 8325 { 8326 struct indirdep *indirdep; 8327 struct freefrag *freefrag; 8328 struct newblk *newblk; 8329 8330 newblk = (struct newblk *)aip; 8331 LIST_REMOVE(aip, ai_next); 8332 /* 8333 * We must eliminate the pointer in bp if it must be freed on its 8334 * own due to partial truncate or pending journal work. 8335 */ 8336 if (bp && (trunc || newblk->nb_jnewblk)) { 8337 /* 8338 * Clear the pointer and mark the aip to be freed 8339 * directly if it never existed on disk. 8340 */ 8341 aip->ai_state |= DELAYEDFREE; 8342 indirdep = aip->ai_indirdep; 8343 if (indirdep->ir_state & UFS1FMT) 8344 ((ufs1_daddr_t *)bp->b_data)[aip->ai_offset] = 0; 8345 else 8346 ((ufs2_daddr_t *)bp->b_data)[aip->ai_offset] = 0; 8347 } 8348 /* 8349 * When truncating the previous pointer will be freed via 8350 * savedbp. Eliminate the freefrag which would dup free. 8351 */ 8352 if (trunc && (freefrag = newblk->nb_freefrag) != NULL) { 8353 newblk->nb_freefrag = NULL; 8354 if (freefrag->ff_jdep) 8355 cancel_jfreefrag( 8356 WK_JFREEFRAG(freefrag->ff_jdep)); 8357 jwork_move(&freeblks->fb_jwork, &freefrag->ff_jwork); 8358 WORKITEM_FREE(freefrag, D_FREEFRAG); 8359 } 8360 /* 8361 * If the journal hasn't been written the jnewblk must be passed 8362 * to the call to ffs_blkfree that reclaims the space. We accomplish 8363 * this by leaving the journal dependency on the newblk to be freed 8364 * when a freework is created in handle_workitem_freeblocks(). 8365 */ 8366 cancel_newblk(newblk, NULL, &freeblks->fb_jwork); 8367 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list); 8368 } 8369 8370 /* 8371 * Create the mkdir dependencies for . and .. in a new directory. Link them 8372 * in to a newdirblk so any subsequent additions are tracked properly. The 8373 * caller is responsible for adding the mkdir1 dependency to the journal 8374 * and updating id_mkdiradd. This function returns with the per-filesystem 8375 * lock held. 8376 */ 8377 static struct mkdir * 8378 setup_newdir(dap, newinum, dinum, newdirbp, mkdirp) 8379 struct diradd *dap; 8380 ino_t newinum; 8381 ino_t dinum; 8382 struct buf *newdirbp; 8383 struct mkdir **mkdirp; 8384 { 8385 struct newblk *newblk; 8386 struct pagedep *pagedep; 8387 struct inodedep *inodedep; 8388 struct newdirblk *newdirblk; 8389 struct mkdir *mkdir1, *mkdir2; 8390 struct worklist *wk; 8391 struct jaddref *jaddref; 8392 struct ufsmount *ump; 8393 struct mount *mp; 8394 8395 mp = dap->da_list.wk_mp; 8396 ump = VFSTOUFS(mp); 8397 newdirblk = malloc(sizeof(struct newdirblk), M_NEWDIRBLK, 8398 M_SOFTDEP_FLAGS); 8399 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 8400 LIST_INIT(&newdirblk->db_mkdir); 8401 mkdir1 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 8402 workitem_alloc(&mkdir1->md_list, D_MKDIR, mp); 8403 mkdir1->md_state = ATTACHED | MKDIR_BODY; 8404 mkdir1->md_diradd = dap; 8405 mkdir1->md_jaddref = NULL; 8406 mkdir2 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 8407 workitem_alloc(&mkdir2->md_list, D_MKDIR, mp); 8408 mkdir2->md_state = ATTACHED | MKDIR_PARENT; 8409 mkdir2->md_diradd = dap; 8410 mkdir2->md_jaddref = NULL; 8411 if (MOUNTEDSUJ(mp) == 0) { 8412 mkdir1->md_state |= DEPCOMPLETE; 8413 mkdir2->md_state |= DEPCOMPLETE; 8414 } 8415 /* 8416 * Dependency on "." and ".." being written to disk. 8417 */ 8418 mkdir1->md_buf = newdirbp; 8419 ACQUIRE_LOCK(VFSTOUFS(mp)); 8420 LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir1, md_mkdirs); 8421 /* 8422 * We must link the pagedep, allocdirect, and newdirblk for 8423 * the initial file page so the pointer to the new directory 8424 * is not written until the directory contents are live and 8425 * any subsequent additions are not marked live until the 8426 * block is reachable via the inode. 8427 */ 8428 if (pagedep_lookup(mp, newdirbp, newinum, 0, 0, &pagedep) == 0) 8429 panic("setup_newdir: lost pagedep"); 8430 LIST_FOREACH(wk, &newdirbp->b_dep, wk_list) 8431 if (wk->wk_type == D_ALLOCDIRECT) 8432 break; 8433 if (wk == NULL) 8434 panic("setup_newdir: lost allocdirect"); 8435 if (pagedep->pd_state & NEWBLOCK) 8436 panic("setup_newdir: NEWBLOCK already set"); 8437 newblk = WK_NEWBLK(wk); 8438 pagedep->pd_state |= NEWBLOCK; 8439 pagedep->pd_newdirblk = newdirblk; 8440 newdirblk->db_pagedep = pagedep; 8441 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 8442 WORKLIST_INSERT(&newdirblk->db_mkdir, &mkdir1->md_list); 8443 /* 8444 * Look up the inodedep for the parent directory so that we 8445 * can link mkdir2 into the pending dotdot jaddref or 8446 * the inode write if there is none. If the inode is 8447 * ALLCOMPLETE and no jaddref is present all dependencies have 8448 * been satisfied and mkdir2 can be freed. 8449 */ 8450 inodedep_lookup(mp, dinum, 0, &inodedep); 8451 if (MOUNTEDSUJ(mp)) { 8452 if (inodedep == NULL) 8453 panic("setup_newdir: Lost parent."); 8454 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 8455 inoreflst); 8456 KASSERT(jaddref != NULL && jaddref->ja_parent == newinum && 8457 (jaddref->ja_state & MKDIR_PARENT), 8458 ("setup_newdir: bad dotdot jaddref %p", jaddref)); 8459 LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir2, md_mkdirs); 8460 mkdir2->md_jaddref = jaddref; 8461 jaddref->ja_mkdir = mkdir2; 8462 } else if (inodedep == NULL || 8463 (inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 8464 dap->da_state &= ~MKDIR_PARENT; 8465 WORKITEM_FREE(mkdir2, D_MKDIR); 8466 mkdir2 = NULL; 8467 } else { 8468 LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir2, md_mkdirs); 8469 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir2->md_list); 8470 } 8471 *mkdirp = mkdir2; 8472 8473 return (mkdir1); 8474 } 8475 8476 /* 8477 * Directory entry addition dependencies. 8478 * 8479 * When adding a new directory entry, the inode (with its incremented link 8480 * count) must be written to disk before the directory entry's pointer to it. 8481 * Also, if the inode is newly allocated, the corresponding freemap must be 8482 * updated (on disk) before the directory entry's pointer. These requirements 8483 * are met via undo/redo on the directory entry's pointer, which consists 8484 * simply of the inode number. 8485 * 8486 * As directory entries are added and deleted, the free space within a 8487 * directory block can become fragmented. The ufs filesystem will compact 8488 * a fragmented directory block to make space for a new entry. When this 8489 * occurs, the offsets of previously added entries change. Any "diradd" 8490 * dependency structures corresponding to these entries must be updated with 8491 * the new offsets. 8492 */ 8493 8494 /* 8495 * This routine is called after the in-memory inode's link 8496 * count has been incremented, but before the directory entry's 8497 * pointer to the inode has been set. 8498 */ 8499 int 8500 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 8501 struct buf *bp; /* buffer containing directory block */ 8502 struct inode *dp; /* inode for directory */ 8503 off_t diroffset; /* offset of new entry in directory */ 8504 ino_t newinum; /* inode referenced by new directory entry */ 8505 struct buf *newdirbp; /* non-NULL => contents of new mkdir */ 8506 int isnewblk; /* entry is in a newly allocated block */ 8507 { 8508 int offset; /* offset of new entry within directory block */ 8509 ufs_lbn_t lbn; /* block in directory containing new entry */ 8510 struct fs *fs; 8511 struct diradd *dap; 8512 struct newblk *newblk; 8513 struct pagedep *pagedep; 8514 struct inodedep *inodedep; 8515 struct newdirblk *newdirblk; 8516 struct mkdir *mkdir1, *mkdir2; 8517 struct jaddref *jaddref; 8518 struct ufsmount *ump; 8519 struct mount *mp; 8520 int isindir; 8521 8522 mp = ITOVFS(dp); 8523 ump = VFSTOUFS(mp); 8524 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 8525 ("softdep_setup_directory_add called on non-softdep filesystem")); 8526 /* 8527 * Whiteouts have no dependencies. 8528 */ 8529 if (newinum == UFS_WINO) { 8530 if (newdirbp != NULL) 8531 bdwrite(newdirbp); 8532 return (0); 8533 } 8534 jaddref = NULL; 8535 mkdir1 = mkdir2 = NULL; 8536 fs = ump->um_fs; 8537 lbn = lblkno(fs, diroffset); 8538 offset = blkoff(fs, diroffset); 8539 dap = malloc(sizeof(struct diradd), M_DIRADD, 8540 M_SOFTDEP_FLAGS|M_ZERO); 8541 workitem_alloc(&dap->da_list, D_DIRADD, mp); 8542 dap->da_offset = offset; 8543 dap->da_newinum = newinum; 8544 dap->da_state = ATTACHED; 8545 LIST_INIT(&dap->da_jwork); 8546 isindir = bp->b_lblkno >= UFS_NDADDR; 8547 newdirblk = NULL; 8548 if (isnewblk && 8549 (isindir ? blkoff(fs, diroffset) : fragoff(fs, diroffset)) == 0) { 8550 newdirblk = malloc(sizeof(struct newdirblk), 8551 M_NEWDIRBLK, M_SOFTDEP_FLAGS); 8552 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 8553 LIST_INIT(&newdirblk->db_mkdir); 8554 } 8555 /* 8556 * If we're creating a new directory setup the dependencies and set 8557 * the dap state to wait for them. Otherwise it's COMPLETE and 8558 * we can move on. 8559 */ 8560 if (newdirbp == NULL) { 8561 dap->da_state |= DEPCOMPLETE; 8562 ACQUIRE_LOCK(ump); 8563 } else { 8564 dap->da_state |= MKDIR_BODY | MKDIR_PARENT; 8565 mkdir1 = setup_newdir(dap, newinum, dp->i_number, newdirbp, 8566 &mkdir2); 8567 } 8568 /* 8569 * Link into parent directory pagedep to await its being written. 8570 */ 8571 pagedep_lookup(mp, bp, dp->i_number, lbn, DEPALLOC, &pagedep); 8572 #ifdef INVARIANTS 8573 if (diradd_lookup(pagedep, offset) != NULL) 8574 panic("softdep_setup_directory_add: %p already at off %d\n", 8575 diradd_lookup(pagedep, offset), offset); 8576 #endif 8577 dap->da_pagedep = pagedep; 8578 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], dap, 8579 da_pdlist); 8580 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 8581 /* 8582 * If we're journaling, link the diradd into the jaddref so it 8583 * may be completed after the journal entry is written. Otherwise, 8584 * link the diradd into its inodedep. If the inode is not yet 8585 * written place it on the bufwait list, otherwise do the post-inode 8586 * write processing to put it on the id_pendinghd list. 8587 */ 8588 if (MOUNTEDSUJ(mp)) { 8589 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 8590 inoreflst); 8591 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 8592 ("softdep_setup_directory_add: bad jaddref %p", jaddref)); 8593 jaddref->ja_diroff = diroffset; 8594 jaddref->ja_diradd = dap; 8595 add_to_journal(&jaddref->ja_list); 8596 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) 8597 diradd_inode_written(dap, inodedep); 8598 else 8599 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 8600 /* 8601 * Add the journal entries for . and .. links now that the primary 8602 * link is written. 8603 */ 8604 if (mkdir1 != NULL && MOUNTEDSUJ(mp)) { 8605 jaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref, 8606 inoreflst, if_deps); 8607 KASSERT(jaddref != NULL && 8608 jaddref->ja_ino == jaddref->ja_parent && 8609 (jaddref->ja_state & MKDIR_BODY), 8610 ("softdep_setup_directory_add: bad dot jaddref %p", 8611 jaddref)); 8612 mkdir1->md_jaddref = jaddref; 8613 jaddref->ja_mkdir = mkdir1; 8614 /* 8615 * It is important that the dotdot journal entry 8616 * is added prior to the dot entry since dot writes 8617 * both the dot and dotdot links. These both must 8618 * be added after the primary link for the journal 8619 * to remain consistent. 8620 */ 8621 add_to_journal(&mkdir2->md_jaddref->ja_list); 8622 add_to_journal(&jaddref->ja_list); 8623 } 8624 /* 8625 * If we are adding a new directory remember this diradd so that if 8626 * we rename it we can keep the dot and dotdot dependencies. If 8627 * we are adding a new name for an inode that has a mkdiradd we 8628 * must be in rename and we have to move the dot and dotdot 8629 * dependencies to this new name. The old name is being orphaned 8630 * soon. 8631 */ 8632 if (mkdir1 != NULL) { 8633 if (inodedep->id_mkdiradd != NULL) 8634 panic("softdep_setup_directory_add: Existing mkdir"); 8635 inodedep->id_mkdiradd = dap; 8636 } else if (inodedep->id_mkdiradd) 8637 merge_diradd(inodedep, dap); 8638 if (newdirblk != NULL) { 8639 /* 8640 * There is nothing to do if we are already tracking 8641 * this block. 8642 */ 8643 if ((pagedep->pd_state & NEWBLOCK) != 0) { 8644 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 8645 FREE_LOCK(ump); 8646 return (0); 8647 } 8648 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk) 8649 == 0) 8650 panic("softdep_setup_directory_add: lost entry"); 8651 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 8652 pagedep->pd_state |= NEWBLOCK; 8653 pagedep->pd_newdirblk = newdirblk; 8654 newdirblk->db_pagedep = pagedep; 8655 FREE_LOCK(ump); 8656 /* 8657 * If we extended into an indirect signal direnter to sync. 8658 */ 8659 if (isindir) 8660 return (1); 8661 return (0); 8662 } 8663 FREE_LOCK(ump); 8664 return (0); 8665 } 8666 8667 /* 8668 * This procedure is called to change the offset of a directory 8669 * entry when compacting a directory block which must be owned 8670 * exclusively by the caller. Note that the actual entry movement 8671 * must be done in this procedure to ensure that no I/O completions 8672 * occur while the move is in progress. 8673 */ 8674 void 8675 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 8676 struct buf *bp; /* Buffer holding directory block. */ 8677 struct inode *dp; /* inode for directory */ 8678 caddr_t base; /* address of dp->i_offset */ 8679 caddr_t oldloc; /* address of old directory location */ 8680 caddr_t newloc; /* address of new directory location */ 8681 int entrysize; /* size of directory entry */ 8682 { 8683 int offset, oldoffset, newoffset; 8684 struct pagedep *pagedep; 8685 struct jmvref *jmvref; 8686 struct diradd *dap; 8687 struct direct *de; 8688 struct mount *mp; 8689 struct ufsmount *ump; 8690 ufs_lbn_t lbn; 8691 int flags; 8692 8693 mp = ITOVFS(dp); 8694 ump = VFSTOUFS(mp); 8695 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 8696 ("softdep_change_directoryentry_offset called on " 8697 "non-softdep filesystem")); 8698 de = (struct direct *)oldloc; 8699 jmvref = NULL; 8700 flags = 0; 8701 /* 8702 * Moves are always journaled as it would be too complex to 8703 * determine if any affected adds or removes are present in the 8704 * journal. 8705 */ 8706 if (MOUNTEDSUJ(mp)) { 8707 flags = DEPALLOC; 8708 jmvref = newjmvref(dp, de->d_ino, 8709 dp->i_offset + (oldloc - base), 8710 dp->i_offset + (newloc - base)); 8711 } 8712 lbn = lblkno(ump->um_fs, dp->i_offset); 8713 offset = blkoff(ump->um_fs, dp->i_offset); 8714 oldoffset = offset + (oldloc - base); 8715 newoffset = offset + (newloc - base); 8716 ACQUIRE_LOCK(ump); 8717 if (pagedep_lookup(mp, bp, dp->i_number, lbn, flags, &pagedep) == 0) 8718 goto done; 8719 dap = diradd_lookup(pagedep, oldoffset); 8720 if (dap) { 8721 dap->da_offset = newoffset; 8722 newoffset = DIRADDHASH(newoffset); 8723 oldoffset = DIRADDHASH(oldoffset); 8724 if ((dap->da_state & ALLCOMPLETE) != ALLCOMPLETE && 8725 newoffset != oldoffset) { 8726 LIST_REMOVE(dap, da_pdlist); 8727 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[newoffset], 8728 dap, da_pdlist); 8729 } 8730 } 8731 done: 8732 if (jmvref) { 8733 jmvref->jm_pagedep = pagedep; 8734 LIST_INSERT_HEAD(&pagedep->pd_jmvrefhd, jmvref, jm_deps); 8735 add_to_journal(&jmvref->jm_list); 8736 } 8737 bcopy(oldloc, newloc, entrysize); 8738 FREE_LOCK(ump); 8739 } 8740 8741 /* 8742 * Move the mkdir dependencies and journal work from one diradd to another 8743 * when renaming a directory. The new name must depend on the mkdir deps 8744 * completing as the old name did. Directories can only have one valid link 8745 * at a time so one must be canonical. 8746 */ 8747 static void 8748 merge_diradd(inodedep, newdap) 8749 struct inodedep *inodedep; 8750 struct diradd *newdap; 8751 { 8752 struct diradd *olddap; 8753 struct mkdir *mkdir, *nextmd; 8754 struct ufsmount *ump; 8755 short state; 8756 8757 olddap = inodedep->id_mkdiradd; 8758 inodedep->id_mkdiradd = newdap; 8759 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 8760 newdap->da_state &= ~DEPCOMPLETE; 8761 ump = VFSTOUFS(inodedep->id_list.wk_mp); 8762 for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir; 8763 mkdir = nextmd) { 8764 nextmd = LIST_NEXT(mkdir, md_mkdirs); 8765 if (mkdir->md_diradd != olddap) 8766 continue; 8767 mkdir->md_diradd = newdap; 8768 state = mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY); 8769 newdap->da_state |= state; 8770 olddap->da_state &= ~state; 8771 if ((olddap->da_state & 8772 (MKDIR_PARENT | MKDIR_BODY)) == 0) 8773 break; 8774 } 8775 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 8776 panic("merge_diradd: unfound ref"); 8777 } 8778 /* 8779 * Any mkdir related journal items are not safe to be freed until 8780 * the new name is stable. 8781 */ 8782 jwork_move(&newdap->da_jwork, &olddap->da_jwork); 8783 olddap->da_state |= DEPCOMPLETE; 8784 complete_diradd(olddap); 8785 } 8786 8787 /* 8788 * Move the diradd to the pending list when all diradd dependencies are 8789 * complete. 8790 */ 8791 static void 8792 complete_diradd(dap) 8793 struct diradd *dap; 8794 { 8795 struct pagedep *pagedep; 8796 8797 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 8798 if (dap->da_state & DIRCHG) 8799 pagedep = dap->da_previous->dm_pagedep; 8800 else 8801 pagedep = dap->da_pagedep; 8802 LIST_REMOVE(dap, da_pdlist); 8803 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 8804 } 8805 } 8806 8807 /* 8808 * Cancel a diradd when a dirrem overlaps with it. We must cancel the journal 8809 * add entries and conditonally journal the remove. 8810 */ 8811 static void 8812 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref) 8813 struct diradd *dap; 8814 struct dirrem *dirrem; 8815 struct jremref *jremref; 8816 struct jremref *dotremref; 8817 struct jremref *dotdotremref; 8818 { 8819 struct inodedep *inodedep; 8820 struct jaddref *jaddref; 8821 struct inoref *inoref; 8822 struct ufsmount *ump; 8823 struct mkdir *mkdir; 8824 8825 /* 8826 * If no remove references were allocated we're on a non-journaled 8827 * filesystem and can skip the cancel step. 8828 */ 8829 if (jremref == NULL) { 8830 free_diradd(dap, NULL); 8831 return; 8832 } 8833 /* 8834 * Cancel the primary name an free it if it does not require 8835 * journaling. 8836 */ 8837 if (inodedep_lookup(dap->da_list.wk_mp, dap->da_newinum, 8838 0, &inodedep) != 0) { 8839 /* Abort the addref that reference this diradd. */ 8840 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 8841 if (inoref->if_list.wk_type != D_JADDREF) 8842 continue; 8843 jaddref = (struct jaddref *)inoref; 8844 if (jaddref->ja_diradd != dap) 8845 continue; 8846 if (cancel_jaddref(jaddref, inodedep, 8847 &dirrem->dm_jwork) == 0) { 8848 free_jremref(jremref); 8849 jremref = NULL; 8850 } 8851 break; 8852 } 8853 } 8854 /* 8855 * Cancel subordinate names and free them if they do not require 8856 * journaling. 8857 */ 8858 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 8859 ump = VFSTOUFS(dap->da_list.wk_mp); 8860 LIST_FOREACH(mkdir, &ump->softdep_mkdirlisthd, md_mkdirs) { 8861 if (mkdir->md_diradd != dap) 8862 continue; 8863 if ((jaddref = mkdir->md_jaddref) == NULL) 8864 continue; 8865 mkdir->md_jaddref = NULL; 8866 if (mkdir->md_state & MKDIR_PARENT) { 8867 if (cancel_jaddref(jaddref, NULL, 8868 &dirrem->dm_jwork) == 0) { 8869 free_jremref(dotdotremref); 8870 dotdotremref = NULL; 8871 } 8872 } else { 8873 if (cancel_jaddref(jaddref, inodedep, 8874 &dirrem->dm_jwork) == 0) { 8875 free_jremref(dotremref); 8876 dotremref = NULL; 8877 } 8878 } 8879 } 8880 } 8881 8882 if (jremref) 8883 journal_jremref(dirrem, jremref, inodedep); 8884 if (dotremref) 8885 journal_jremref(dirrem, dotremref, inodedep); 8886 if (dotdotremref) 8887 journal_jremref(dirrem, dotdotremref, NULL); 8888 jwork_move(&dirrem->dm_jwork, &dap->da_jwork); 8889 free_diradd(dap, &dirrem->dm_jwork); 8890 } 8891 8892 /* 8893 * Free a diradd dependency structure. 8894 */ 8895 static void 8896 free_diradd(dap, wkhd) 8897 struct diradd *dap; 8898 struct workhead *wkhd; 8899 { 8900 struct dirrem *dirrem; 8901 struct pagedep *pagedep; 8902 struct inodedep *inodedep; 8903 struct mkdir *mkdir, *nextmd; 8904 struct ufsmount *ump; 8905 8906 ump = VFSTOUFS(dap->da_list.wk_mp); 8907 LOCK_OWNED(ump); 8908 LIST_REMOVE(dap, da_pdlist); 8909 if (dap->da_state & ONWORKLIST) 8910 WORKLIST_REMOVE(&dap->da_list); 8911 if ((dap->da_state & DIRCHG) == 0) { 8912 pagedep = dap->da_pagedep; 8913 } else { 8914 dirrem = dap->da_previous; 8915 pagedep = dirrem->dm_pagedep; 8916 dirrem->dm_dirinum = pagedep->pd_ino; 8917 dirrem->dm_state |= COMPLETE; 8918 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 8919 add_to_worklist(&dirrem->dm_list, 0); 8920 } 8921 if (inodedep_lookup(pagedep->pd_list.wk_mp, dap->da_newinum, 8922 0, &inodedep) != 0) 8923 if (inodedep->id_mkdiradd == dap) 8924 inodedep->id_mkdiradd = NULL; 8925 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 8926 for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir; 8927 mkdir = nextmd) { 8928 nextmd = LIST_NEXT(mkdir, md_mkdirs); 8929 if (mkdir->md_diradd != dap) 8930 continue; 8931 dap->da_state &= 8932 ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 8933 LIST_REMOVE(mkdir, md_mkdirs); 8934 if (mkdir->md_state & ONWORKLIST) 8935 WORKLIST_REMOVE(&mkdir->md_list); 8936 if (mkdir->md_jaddref != NULL) 8937 panic("free_diradd: Unexpected jaddref"); 8938 WORKITEM_FREE(mkdir, D_MKDIR); 8939 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) 8940 break; 8941 } 8942 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 8943 panic("free_diradd: unfound ref"); 8944 } 8945 if (inodedep) 8946 free_inodedep(inodedep); 8947 /* 8948 * Free any journal segments waiting for the directory write. 8949 */ 8950 handle_jwork(&dap->da_jwork); 8951 WORKITEM_FREE(dap, D_DIRADD); 8952 } 8953 8954 /* 8955 * Directory entry removal dependencies. 8956 * 8957 * When removing a directory entry, the entry's inode pointer must be 8958 * zero'ed on disk before the corresponding inode's link count is decremented 8959 * (possibly freeing the inode for re-use). This dependency is handled by 8960 * updating the directory entry but delaying the inode count reduction until 8961 * after the directory block has been written to disk. After this point, the 8962 * inode count can be decremented whenever it is convenient. 8963 */ 8964 8965 /* 8966 * This routine should be called immediately after removing 8967 * a directory entry. The inode's link count should not be 8968 * decremented by the calling procedure -- the soft updates 8969 * code will do this task when it is safe. 8970 */ 8971 void 8972 softdep_setup_remove(bp, dp, ip, isrmdir) 8973 struct buf *bp; /* buffer containing directory block */ 8974 struct inode *dp; /* inode for the directory being modified */ 8975 struct inode *ip; /* inode for directory entry being removed */ 8976 int isrmdir; /* indicates if doing RMDIR */ 8977 { 8978 struct dirrem *dirrem, *prevdirrem; 8979 struct inodedep *inodedep; 8980 struct ufsmount *ump; 8981 int direct; 8982 8983 ump = ITOUMP(ip); 8984 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 8985 ("softdep_setup_remove called on non-softdep filesystem")); 8986 /* 8987 * Allocate a new dirrem if appropriate and ACQUIRE_LOCK. We want 8988 * newdirrem() to setup the full directory remove which requires 8989 * isrmdir > 1. 8990 */ 8991 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem); 8992 /* 8993 * Add the dirrem to the inodedep's pending remove list for quick 8994 * discovery later. 8995 */ 8996 if (inodedep_lookup(UFSTOVFS(ump), ip->i_number, 0, &inodedep) == 0) 8997 panic("softdep_setup_remove: Lost inodedep."); 8998 KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked")); 8999 dirrem->dm_state |= ONDEPLIST; 9000 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 9001 9002 /* 9003 * If the COMPLETE flag is clear, then there were no active 9004 * entries and we want to roll back to a zeroed entry until 9005 * the new inode is committed to disk. If the COMPLETE flag is 9006 * set then we have deleted an entry that never made it to 9007 * disk. If the entry we deleted resulted from a name change, 9008 * then the old name still resides on disk. We cannot delete 9009 * its inode (returned to us in prevdirrem) until the zeroed 9010 * directory entry gets to disk. The new inode has never been 9011 * referenced on the disk, so can be deleted immediately. 9012 */ 9013 if ((dirrem->dm_state & COMPLETE) == 0) { 9014 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, dirrem, 9015 dm_next); 9016 FREE_LOCK(ump); 9017 } else { 9018 if (prevdirrem != NULL) 9019 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, 9020 prevdirrem, dm_next); 9021 dirrem->dm_dirinum = dirrem->dm_pagedep->pd_ino; 9022 direct = LIST_EMPTY(&dirrem->dm_jremrefhd); 9023 FREE_LOCK(ump); 9024 if (direct) 9025 handle_workitem_remove(dirrem, 0); 9026 } 9027 } 9028 9029 /* 9030 * Check for an entry matching 'offset' on both the pd_dirraddhd list and the 9031 * pd_pendinghd list of a pagedep. 9032 */ 9033 static struct diradd * 9034 diradd_lookup(pagedep, offset) 9035 struct pagedep *pagedep; 9036 int offset; 9037 { 9038 struct diradd *dap; 9039 9040 LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(offset)], da_pdlist) 9041 if (dap->da_offset == offset) 9042 return (dap); 9043 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist) 9044 if (dap->da_offset == offset) 9045 return (dap); 9046 return (NULL); 9047 } 9048 9049 /* 9050 * Search for a .. diradd dependency in a directory that is being removed. 9051 * If the directory was renamed to a new parent we have a diradd rather 9052 * than a mkdir for the .. entry. We need to cancel it now before 9053 * it is found in truncate(). 9054 */ 9055 static struct jremref * 9056 cancel_diradd_dotdot(ip, dirrem, jremref) 9057 struct inode *ip; 9058 struct dirrem *dirrem; 9059 struct jremref *jremref; 9060 { 9061 struct pagedep *pagedep; 9062 struct diradd *dap; 9063 struct worklist *wk; 9064 9065 if (pagedep_lookup(ITOVFS(ip), NULL, ip->i_number, 0, 0, &pagedep) == 0) 9066 return (jremref); 9067 dap = diradd_lookup(pagedep, DOTDOT_OFFSET); 9068 if (dap == NULL) 9069 return (jremref); 9070 cancel_diradd(dap, dirrem, jremref, NULL, NULL); 9071 /* 9072 * Mark any journal work as belonging to the parent so it is freed 9073 * with the .. reference. 9074 */ 9075 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 9076 wk->wk_state |= MKDIR_PARENT; 9077 return (NULL); 9078 } 9079 9080 /* 9081 * Cancel the MKDIR_PARENT mkdir component of a diradd when we're going to 9082 * replace it with a dirrem/diradd pair as a result of re-parenting a 9083 * directory. This ensures that we don't simultaneously have a mkdir and 9084 * a diradd for the same .. entry. 9085 */ 9086 static struct jremref * 9087 cancel_mkdir_dotdot(ip, dirrem, jremref) 9088 struct inode *ip; 9089 struct dirrem *dirrem; 9090 struct jremref *jremref; 9091 { 9092 struct inodedep *inodedep; 9093 struct jaddref *jaddref; 9094 struct ufsmount *ump; 9095 struct mkdir *mkdir; 9096 struct diradd *dap; 9097 struct mount *mp; 9098 9099 mp = ITOVFS(ip); 9100 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 9101 return (jremref); 9102 dap = inodedep->id_mkdiradd; 9103 if (dap == NULL || (dap->da_state & MKDIR_PARENT) == 0) 9104 return (jremref); 9105 ump = VFSTOUFS(inodedep->id_list.wk_mp); 9106 for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir; 9107 mkdir = LIST_NEXT(mkdir, md_mkdirs)) 9108 if (mkdir->md_diradd == dap && mkdir->md_state & MKDIR_PARENT) 9109 break; 9110 if (mkdir == NULL) 9111 panic("cancel_mkdir_dotdot: Unable to find mkdir\n"); 9112 if ((jaddref = mkdir->md_jaddref) != NULL) { 9113 mkdir->md_jaddref = NULL; 9114 jaddref->ja_state &= ~MKDIR_PARENT; 9115 if (inodedep_lookup(mp, jaddref->ja_ino, 0, &inodedep) == 0) 9116 panic("cancel_mkdir_dotdot: Lost parent inodedep"); 9117 if (cancel_jaddref(jaddref, inodedep, &dirrem->dm_jwork)) { 9118 journal_jremref(dirrem, jremref, inodedep); 9119 jremref = NULL; 9120 } 9121 } 9122 if (mkdir->md_state & ONWORKLIST) 9123 WORKLIST_REMOVE(&mkdir->md_list); 9124 mkdir->md_state |= ALLCOMPLETE; 9125 complete_mkdir(mkdir); 9126 return (jremref); 9127 } 9128 9129 static void 9130 journal_jremref(dirrem, jremref, inodedep) 9131 struct dirrem *dirrem; 9132 struct jremref *jremref; 9133 struct inodedep *inodedep; 9134 { 9135 9136 if (inodedep == NULL) 9137 if (inodedep_lookup(jremref->jr_list.wk_mp, 9138 jremref->jr_ref.if_ino, 0, &inodedep) == 0) 9139 panic("journal_jremref: Lost inodedep"); 9140 LIST_INSERT_HEAD(&dirrem->dm_jremrefhd, jremref, jr_deps); 9141 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 9142 add_to_journal(&jremref->jr_list); 9143 } 9144 9145 static void 9146 dirrem_journal(dirrem, jremref, dotremref, dotdotremref) 9147 struct dirrem *dirrem; 9148 struct jremref *jremref; 9149 struct jremref *dotremref; 9150 struct jremref *dotdotremref; 9151 { 9152 struct inodedep *inodedep; 9153 9154 9155 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 0, 9156 &inodedep) == 0) 9157 panic("dirrem_journal: Lost inodedep"); 9158 journal_jremref(dirrem, jremref, inodedep); 9159 if (dotremref) 9160 journal_jremref(dirrem, dotremref, inodedep); 9161 if (dotdotremref) 9162 journal_jremref(dirrem, dotdotremref, NULL); 9163 } 9164 9165 /* 9166 * Allocate a new dirrem if appropriate and return it along with 9167 * its associated pagedep. Called without a lock, returns with lock. 9168 */ 9169 static struct dirrem * 9170 newdirrem(bp, dp, ip, isrmdir, prevdirremp) 9171 struct buf *bp; /* buffer containing directory block */ 9172 struct inode *dp; /* inode for the directory being modified */ 9173 struct inode *ip; /* inode for directory entry being removed */ 9174 int isrmdir; /* indicates if doing RMDIR */ 9175 struct dirrem **prevdirremp; /* previously referenced inode, if any */ 9176 { 9177 int offset; 9178 ufs_lbn_t lbn; 9179 struct diradd *dap; 9180 struct dirrem *dirrem; 9181 struct pagedep *pagedep; 9182 struct jremref *jremref; 9183 struct jremref *dotremref; 9184 struct jremref *dotdotremref; 9185 struct vnode *dvp; 9186 struct ufsmount *ump; 9187 9188 /* 9189 * Whiteouts have no deletion dependencies. 9190 */ 9191 if (ip == NULL) 9192 panic("newdirrem: whiteout"); 9193 dvp = ITOV(dp); 9194 ump = ITOUMP(dp); 9195 9196 /* 9197 * If the system is over its limit and our filesystem is 9198 * responsible for more than our share of that usage and 9199 * we are not a snapshot, request some inodedep cleanup. 9200 * Limiting the number of dirrem structures will also limit 9201 * the number of freefile and freeblks structures. 9202 */ 9203 ACQUIRE_LOCK(ump); 9204 if (!IS_SNAPSHOT(ip) && softdep_excess_items(ump, D_DIRREM)) 9205 schedule_cleanup(UFSTOVFS(ump)); 9206 else 9207 FREE_LOCK(ump); 9208 dirrem = malloc(sizeof(struct dirrem), M_DIRREM, M_SOFTDEP_FLAGS | 9209 M_ZERO); 9210 workitem_alloc(&dirrem->dm_list, D_DIRREM, dvp->v_mount); 9211 LIST_INIT(&dirrem->dm_jremrefhd); 9212 LIST_INIT(&dirrem->dm_jwork); 9213 dirrem->dm_state = isrmdir ? RMDIR : 0; 9214 dirrem->dm_oldinum = ip->i_number; 9215 *prevdirremp = NULL; 9216 /* 9217 * Allocate remove reference structures to track journal write 9218 * dependencies. We will always have one for the link and 9219 * when doing directories we will always have one more for dot. 9220 * When renaming a directory we skip the dotdot link change so 9221 * this is not needed. 9222 */ 9223 jremref = dotremref = dotdotremref = NULL; 9224 if (DOINGSUJ(dvp)) { 9225 if (isrmdir) { 9226 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 9227 ip->i_effnlink + 2); 9228 dotremref = newjremref(dirrem, ip, ip, DOT_OFFSET, 9229 ip->i_effnlink + 1); 9230 dotdotremref = newjremref(dirrem, ip, dp, DOTDOT_OFFSET, 9231 dp->i_effnlink + 1); 9232 dotdotremref->jr_state |= MKDIR_PARENT; 9233 } else 9234 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 9235 ip->i_effnlink + 1); 9236 } 9237 ACQUIRE_LOCK(ump); 9238 lbn = lblkno(ump->um_fs, dp->i_offset); 9239 offset = blkoff(ump->um_fs, dp->i_offset); 9240 pagedep_lookup(UFSTOVFS(ump), bp, dp->i_number, lbn, DEPALLOC, 9241 &pagedep); 9242 dirrem->dm_pagedep = pagedep; 9243 dirrem->dm_offset = offset; 9244 /* 9245 * If we're renaming a .. link to a new directory, cancel any 9246 * existing MKDIR_PARENT mkdir. If it has already been canceled 9247 * the jremref is preserved for any potential diradd in this 9248 * location. This can not coincide with a rmdir. 9249 */ 9250 if (dp->i_offset == DOTDOT_OFFSET) { 9251 if (isrmdir) 9252 panic("newdirrem: .. directory change during remove?"); 9253 jremref = cancel_mkdir_dotdot(dp, dirrem, jremref); 9254 } 9255 /* 9256 * If we're removing a directory search for the .. dependency now and 9257 * cancel it. Any pending journal work will be added to the dirrem 9258 * to be completed when the workitem remove completes. 9259 */ 9260 if (isrmdir) 9261 dotdotremref = cancel_diradd_dotdot(ip, dirrem, dotdotremref); 9262 /* 9263 * Check for a diradd dependency for the same directory entry. 9264 * If present, then both dependencies become obsolete and can 9265 * be de-allocated. 9266 */ 9267 dap = diradd_lookup(pagedep, offset); 9268 if (dap == NULL) { 9269 /* 9270 * Link the jremref structures into the dirrem so they are 9271 * written prior to the pagedep. 9272 */ 9273 if (jremref) 9274 dirrem_journal(dirrem, jremref, dotremref, 9275 dotdotremref); 9276 return (dirrem); 9277 } 9278 /* 9279 * Must be ATTACHED at this point. 9280 */ 9281 if ((dap->da_state & ATTACHED) == 0) 9282 panic("newdirrem: not ATTACHED"); 9283 if (dap->da_newinum != ip->i_number) 9284 panic("newdirrem: inum %ju should be %ju", 9285 (uintmax_t)ip->i_number, (uintmax_t)dap->da_newinum); 9286 /* 9287 * If we are deleting a changed name that never made it to disk, 9288 * then return the dirrem describing the previous inode (which 9289 * represents the inode currently referenced from this entry on disk). 9290 */ 9291 if ((dap->da_state & DIRCHG) != 0) { 9292 *prevdirremp = dap->da_previous; 9293 dap->da_state &= ~DIRCHG; 9294 dap->da_pagedep = pagedep; 9295 } 9296 /* 9297 * We are deleting an entry that never made it to disk. 9298 * Mark it COMPLETE so we can delete its inode immediately. 9299 */ 9300 dirrem->dm_state |= COMPLETE; 9301 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref); 9302 #ifdef INVARIANTS 9303 if (isrmdir == 0) { 9304 struct worklist *wk; 9305 9306 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 9307 if (wk->wk_state & (MKDIR_BODY | MKDIR_PARENT)) 9308 panic("bad wk %p (0x%X)\n", wk, wk->wk_state); 9309 } 9310 #endif 9311 9312 return (dirrem); 9313 } 9314 9315 /* 9316 * Directory entry change dependencies. 9317 * 9318 * Changing an existing directory entry requires that an add operation 9319 * be completed first followed by a deletion. The semantics for the addition 9320 * are identical to the description of adding a new entry above except 9321 * that the rollback is to the old inode number rather than zero. Once 9322 * the addition dependency is completed, the removal is done as described 9323 * in the removal routine above. 9324 */ 9325 9326 /* 9327 * This routine should be called immediately after changing 9328 * a directory entry. The inode's link count should not be 9329 * decremented by the calling procedure -- the soft updates 9330 * code will perform this task when it is safe. 9331 */ 9332 void 9333 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 9334 struct buf *bp; /* buffer containing directory block */ 9335 struct inode *dp; /* inode for the directory being modified */ 9336 struct inode *ip; /* inode for directory entry being removed */ 9337 ino_t newinum; /* new inode number for changed entry */ 9338 int isrmdir; /* indicates if doing RMDIR */ 9339 { 9340 int offset; 9341 struct diradd *dap = NULL; 9342 struct dirrem *dirrem, *prevdirrem; 9343 struct pagedep *pagedep; 9344 struct inodedep *inodedep; 9345 struct jaddref *jaddref; 9346 struct mount *mp; 9347 struct ufsmount *ump; 9348 9349 mp = ITOVFS(dp); 9350 ump = VFSTOUFS(mp); 9351 offset = blkoff(ump->um_fs, dp->i_offset); 9352 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 9353 ("softdep_setup_directory_change called on non-softdep filesystem")); 9354 9355 /* 9356 * Whiteouts do not need diradd dependencies. 9357 */ 9358 if (newinum != UFS_WINO) { 9359 dap = malloc(sizeof(struct diradd), 9360 M_DIRADD, M_SOFTDEP_FLAGS|M_ZERO); 9361 workitem_alloc(&dap->da_list, D_DIRADD, mp); 9362 dap->da_state = DIRCHG | ATTACHED | DEPCOMPLETE; 9363 dap->da_offset = offset; 9364 dap->da_newinum = newinum; 9365 LIST_INIT(&dap->da_jwork); 9366 } 9367 9368 /* 9369 * Allocate a new dirrem and ACQUIRE_LOCK. 9370 */ 9371 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem); 9372 pagedep = dirrem->dm_pagedep; 9373 /* 9374 * The possible values for isrmdir: 9375 * 0 - non-directory file rename 9376 * 1 - directory rename within same directory 9377 * inum - directory rename to new directory of given inode number 9378 * When renaming to a new directory, we are both deleting and 9379 * creating a new directory entry, so the link count on the new 9380 * directory should not change. Thus we do not need the followup 9381 * dirrem which is usually done in handle_workitem_remove. We set 9382 * the DIRCHG flag to tell handle_workitem_remove to skip the 9383 * followup dirrem. 9384 */ 9385 if (isrmdir > 1) 9386 dirrem->dm_state |= DIRCHG; 9387 9388 /* 9389 * Whiteouts have no additional dependencies, 9390 * so just put the dirrem on the correct list. 9391 */ 9392 if (newinum == UFS_WINO) { 9393 if ((dirrem->dm_state & COMPLETE) == 0) { 9394 LIST_INSERT_HEAD(&pagedep->pd_dirremhd, dirrem, 9395 dm_next); 9396 } else { 9397 dirrem->dm_dirinum = pagedep->pd_ino; 9398 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 9399 add_to_worklist(&dirrem->dm_list, 0); 9400 } 9401 FREE_LOCK(ump); 9402 return; 9403 } 9404 /* 9405 * Add the dirrem to the inodedep's pending remove list for quick 9406 * discovery later. A valid nlinkdelta ensures that this lookup 9407 * will not fail. 9408 */ 9409 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 9410 panic("softdep_setup_directory_change: Lost inodedep."); 9411 dirrem->dm_state |= ONDEPLIST; 9412 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 9413 9414 /* 9415 * If the COMPLETE flag is clear, then there were no active 9416 * entries and we want to roll back to the previous inode until 9417 * the new inode is committed to disk. If the COMPLETE flag is 9418 * set, then we have deleted an entry that never made it to disk. 9419 * If the entry we deleted resulted from a name change, then the old 9420 * inode reference still resides on disk. Any rollback that we do 9421 * needs to be to that old inode (returned to us in prevdirrem). If 9422 * the entry we deleted resulted from a create, then there is 9423 * no entry on the disk, so we want to roll back to zero rather 9424 * than the uncommitted inode. In either of the COMPLETE cases we 9425 * want to immediately free the unwritten and unreferenced inode. 9426 */ 9427 if ((dirrem->dm_state & COMPLETE) == 0) { 9428 dap->da_previous = dirrem; 9429 } else { 9430 if (prevdirrem != NULL) { 9431 dap->da_previous = prevdirrem; 9432 } else { 9433 dap->da_state &= ~DIRCHG; 9434 dap->da_pagedep = pagedep; 9435 } 9436 dirrem->dm_dirinum = pagedep->pd_ino; 9437 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 9438 add_to_worklist(&dirrem->dm_list, 0); 9439 } 9440 /* 9441 * Lookup the jaddref for this journal entry. We must finish 9442 * initializing it and make the diradd write dependent on it. 9443 * If we're not journaling, put it on the id_bufwait list if the 9444 * inode is not yet written. If it is written, do the post-inode 9445 * write processing to put it on the id_pendinghd list. 9446 */ 9447 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 9448 if (MOUNTEDSUJ(mp)) { 9449 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 9450 inoreflst); 9451 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 9452 ("softdep_setup_directory_change: bad jaddref %p", 9453 jaddref)); 9454 jaddref->ja_diroff = dp->i_offset; 9455 jaddref->ja_diradd = dap; 9456 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 9457 dap, da_pdlist); 9458 add_to_journal(&jaddref->ja_list); 9459 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 9460 dap->da_state |= COMPLETE; 9461 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 9462 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 9463 } else { 9464 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 9465 dap, da_pdlist); 9466 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 9467 } 9468 /* 9469 * If we're making a new name for a directory that has not been 9470 * committed when need to move the dot and dotdot references to 9471 * this new name. 9472 */ 9473 if (inodedep->id_mkdiradd && dp->i_offset != DOTDOT_OFFSET) 9474 merge_diradd(inodedep, dap); 9475 FREE_LOCK(ump); 9476 } 9477 9478 /* 9479 * Called whenever the link count on an inode is changed. 9480 * It creates an inode dependency so that the new reference(s) 9481 * to the inode cannot be committed to disk until the updated 9482 * inode has been written. 9483 */ 9484 void 9485 softdep_change_linkcnt(ip) 9486 struct inode *ip; /* the inode with the increased link count */ 9487 { 9488 struct inodedep *inodedep; 9489 struct ufsmount *ump; 9490 9491 ump = ITOUMP(ip); 9492 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 9493 ("softdep_change_linkcnt called on non-softdep filesystem")); 9494 ACQUIRE_LOCK(ump); 9495 inodedep_lookup(UFSTOVFS(ump), ip->i_number, DEPALLOC, &inodedep); 9496 if (ip->i_nlink < ip->i_effnlink) 9497 panic("softdep_change_linkcnt: bad delta"); 9498 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 9499 FREE_LOCK(ump); 9500 } 9501 9502 /* 9503 * Attach a sbdep dependency to the superblock buf so that we can keep 9504 * track of the head of the linked list of referenced but unlinked inodes. 9505 */ 9506 void 9507 softdep_setup_sbupdate(ump, fs, bp) 9508 struct ufsmount *ump; 9509 struct fs *fs; 9510 struct buf *bp; 9511 { 9512 struct sbdep *sbdep; 9513 struct worklist *wk; 9514 9515 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 9516 ("softdep_setup_sbupdate called on non-softdep filesystem")); 9517 LIST_FOREACH(wk, &bp->b_dep, wk_list) 9518 if (wk->wk_type == D_SBDEP) 9519 break; 9520 if (wk != NULL) 9521 return; 9522 sbdep = malloc(sizeof(struct sbdep), M_SBDEP, M_SOFTDEP_FLAGS); 9523 workitem_alloc(&sbdep->sb_list, D_SBDEP, UFSTOVFS(ump)); 9524 sbdep->sb_fs = fs; 9525 sbdep->sb_ump = ump; 9526 ACQUIRE_LOCK(ump); 9527 WORKLIST_INSERT(&bp->b_dep, &sbdep->sb_list); 9528 FREE_LOCK(ump); 9529 } 9530 9531 /* 9532 * Return the first unlinked inodedep which is ready to be the head of the 9533 * list. The inodedep and all those after it must have valid next pointers. 9534 */ 9535 static struct inodedep * 9536 first_unlinked_inodedep(ump) 9537 struct ufsmount *ump; 9538 { 9539 struct inodedep *inodedep; 9540 struct inodedep *idp; 9541 9542 LOCK_OWNED(ump); 9543 for (inodedep = TAILQ_LAST(&ump->softdep_unlinked, inodedeplst); 9544 inodedep; inodedep = idp) { 9545 if ((inodedep->id_state & UNLINKNEXT) == 0) 9546 return (NULL); 9547 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 9548 if (idp == NULL || (idp->id_state & UNLINKNEXT) == 0) 9549 break; 9550 if ((inodedep->id_state & UNLINKPREV) == 0) 9551 break; 9552 } 9553 return (inodedep); 9554 } 9555 9556 /* 9557 * Set the sujfree unlinked head pointer prior to writing a superblock. 9558 */ 9559 static void 9560 initiate_write_sbdep(sbdep) 9561 struct sbdep *sbdep; 9562 { 9563 struct inodedep *inodedep; 9564 struct fs *bpfs; 9565 struct fs *fs; 9566 9567 bpfs = sbdep->sb_fs; 9568 fs = sbdep->sb_ump->um_fs; 9569 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 9570 if (inodedep) { 9571 fs->fs_sujfree = inodedep->id_ino; 9572 inodedep->id_state |= UNLINKPREV; 9573 } else 9574 fs->fs_sujfree = 0; 9575 bpfs->fs_sujfree = fs->fs_sujfree; 9576 /* 9577 * Because we have made changes to the superblock, we need to 9578 * recompute its check-hash. 9579 */ 9580 bpfs->fs_ckhash = ffs_calc_sbhash(bpfs); 9581 } 9582 9583 /* 9584 * After a superblock is written determine whether it must be written again 9585 * due to a changing unlinked list head. 9586 */ 9587 static int 9588 handle_written_sbdep(sbdep, bp) 9589 struct sbdep *sbdep; 9590 struct buf *bp; 9591 { 9592 struct inodedep *inodedep; 9593 struct fs *fs; 9594 9595 LOCK_OWNED(sbdep->sb_ump); 9596 fs = sbdep->sb_fs; 9597 /* 9598 * If the superblock doesn't match the in-memory list start over. 9599 */ 9600 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 9601 if ((inodedep && fs->fs_sujfree != inodedep->id_ino) || 9602 (inodedep == NULL && fs->fs_sujfree != 0)) { 9603 bdirty(bp); 9604 return (1); 9605 } 9606 WORKITEM_FREE(sbdep, D_SBDEP); 9607 if (fs->fs_sujfree == 0) 9608 return (0); 9609 /* 9610 * Now that we have a record of this inode in stable store allow it 9611 * to be written to free up pending work. Inodes may see a lot of 9612 * write activity after they are unlinked which we must not hold up. 9613 */ 9614 for (; inodedep != NULL; inodedep = TAILQ_NEXT(inodedep, id_unlinked)) { 9615 if ((inodedep->id_state & UNLINKLINKS) != UNLINKLINKS) 9616 panic("handle_written_sbdep: Bad inodedep %p (0x%X)", 9617 inodedep, inodedep->id_state); 9618 if (inodedep->id_state & UNLINKONLIST) 9619 break; 9620 inodedep->id_state |= DEPCOMPLETE | UNLINKONLIST; 9621 } 9622 9623 return (0); 9624 } 9625 9626 /* 9627 * Mark an inodedep as unlinked and insert it into the in-memory unlinked list. 9628 */ 9629 static void 9630 unlinked_inodedep(mp, inodedep) 9631 struct mount *mp; 9632 struct inodedep *inodedep; 9633 { 9634 struct ufsmount *ump; 9635 9636 ump = VFSTOUFS(mp); 9637 LOCK_OWNED(ump); 9638 if (MOUNTEDSUJ(mp) == 0) 9639 return; 9640 ump->um_fs->fs_fmod = 1; 9641 if (inodedep->id_state & UNLINKED) 9642 panic("unlinked_inodedep: %p already unlinked\n", inodedep); 9643 inodedep->id_state |= UNLINKED; 9644 TAILQ_INSERT_HEAD(&ump->softdep_unlinked, inodedep, id_unlinked); 9645 } 9646 9647 /* 9648 * Remove an inodedep from the unlinked inodedep list. This may require 9649 * disk writes if the inode has made it that far. 9650 */ 9651 static void 9652 clear_unlinked_inodedep(inodedep) 9653 struct inodedep *inodedep; 9654 { 9655 struct ufs2_dinode *dip; 9656 struct ufsmount *ump; 9657 struct inodedep *idp; 9658 struct inodedep *idn; 9659 struct fs *fs, *bpfs; 9660 struct buf *bp; 9661 ino_t ino; 9662 ino_t nino; 9663 ino_t pino; 9664 int error; 9665 9666 ump = VFSTOUFS(inodedep->id_list.wk_mp); 9667 fs = ump->um_fs; 9668 ino = inodedep->id_ino; 9669 error = 0; 9670 for (;;) { 9671 LOCK_OWNED(ump); 9672 KASSERT((inodedep->id_state & UNLINKED) != 0, 9673 ("clear_unlinked_inodedep: inodedep %p not unlinked", 9674 inodedep)); 9675 /* 9676 * If nothing has yet been written simply remove us from 9677 * the in memory list and return. This is the most common 9678 * case where handle_workitem_remove() loses the final 9679 * reference. 9680 */ 9681 if ((inodedep->id_state & UNLINKLINKS) == 0) 9682 break; 9683 /* 9684 * If we have a NEXT pointer and no PREV pointer we can simply 9685 * clear NEXT's PREV and remove ourselves from the list. Be 9686 * careful not to clear PREV if the superblock points at 9687 * next as well. 9688 */ 9689 idn = TAILQ_NEXT(inodedep, id_unlinked); 9690 if ((inodedep->id_state & UNLINKLINKS) == UNLINKNEXT) { 9691 if (idn && fs->fs_sujfree != idn->id_ino) 9692 idn->id_state &= ~UNLINKPREV; 9693 break; 9694 } 9695 /* 9696 * Here we have an inodedep which is actually linked into 9697 * the list. We must remove it by forcing a write to the 9698 * link before us, whether it be the superblock or an inode. 9699 * Unfortunately the list may change while we're waiting 9700 * on the buf lock for either resource so we must loop until 9701 * we lock the right one. If both the superblock and an 9702 * inode point to this inode we must clear the inode first 9703 * followed by the superblock. 9704 */ 9705 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 9706 pino = 0; 9707 if (idp && (idp->id_state & UNLINKNEXT)) 9708 pino = idp->id_ino; 9709 FREE_LOCK(ump); 9710 if (pino == 0) { 9711 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 9712 (int)fs->fs_sbsize, 0, 0, 0); 9713 } else { 9714 error = bread(ump->um_devvp, 9715 fsbtodb(fs, ino_to_fsba(fs, pino)), 9716 (int)fs->fs_bsize, NOCRED, &bp); 9717 if (error) 9718 brelse(bp); 9719 } 9720 ACQUIRE_LOCK(ump); 9721 if (error) 9722 break; 9723 /* If the list has changed restart the loop. */ 9724 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 9725 nino = 0; 9726 if (idp && (idp->id_state & UNLINKNEXT)) 9727 nino = idp->id_ino; 9728 if (nino != pino || 9729 (inodedep->id_state & UNLINKPREV) != UNLINKPREV) { 9730 FREE_LOCK(ump); 9731 brelse(bp); 9732 ACQUIRE_LOCK(ump); 9733 continue; 9734 } 9735 nino = 0; 9736 idn = TAILQ_NEXT(inodedep, id_unlinked); 9737 if (idn) 9738 nino = idn->id_ino; 9739 /* 9740 * Remove us from the in memory list. After this we cannot 9741 * access the inodedep. 9742 */ 9743 KASSERT((inodedep->id_state & UNLINKED) != 0, 9744 ("clear_unlinked_inodedep: inodedep %p not unlinked", 9745 inodedep)); 9746 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST); 9747 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 9748 FREE_LOCK(ump); 9749 /* 9750 * The predecessor's next pointer is manually updated here 9751 * so that the NEXT flag is never cleared for an element 9752 * that is in the list. 9753 */ 9754 if (pino == 0) { 9755 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 9756 bpfs = (struct fs *)bp->b_data; 9757 ffs_oldfscompat_write(bpfs, ump); 9758 softdep_setup_sbupdate(ump, bpfs, bp); 9759 /* 9760 * Because we may have made changes to the superblock, 9761 * we need to recompute its check-hash. 9762 */ 9763 bpfs->fs_ckhash = ffs_calc_sbhash(bpfs); 9764 } else if (fs->fs_magic == FS_UFS1_MAGIC) { 9765 ((struct ufs1_dinode *)bp->b_data + 9766 ino_to_fsbo(fs, pino))->di_freelink = nino; 9767 } else { 9768 dip = (struct ufs2_dinode *)bp->b_data + 9769 ino_to_fsbo(fs, pino); 9770 dip->di_freelink = nino; 9771 ffs_update_dinode_ckhash(fs, dip); 9772 } 9773 /* 9774 * If the bwrite fails we have no recourse to recover. The 9775 * filesystem is corrupted already. 9776 */ 9777 bwrite(bp); 9778 ACQUIRE_LOCK(ump); 9779 /* 9780 * If the superblock pointer still needs to be cleared force 9781 * a write here. 9782 */ 9783 if (fs->fs_sujfree == ino) { 9784 FREE_LOCK(ump); 9785 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 9786 (int)fs->fs_sbsize, 0, 0, 0); 9787 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 9788 bpfs = (struct fs *)bp->b_data; 9789 ffs_oldfscompat_write(bpfs, ump); 9790 softdep_setup_sbupdate(ump, bpfs, bp); 9791 /* 9792 * Because we may have made changes to the superblock, 9793 * we need to recompute its check-hash. 9794 */ 9795 bpfs->fs_ckhash = ffs_calc_sbhash(bpfs); 9796 bwrite(bp); 9797 ACQUIRE_LOCK(ump); 9798 } 9799 9800 if (fs->fs_sujfree != ino) 9801 return; 9802 panic("clear_unlinked_inodedep: Failed to clear free head"); 9803 } 9804 if (inodedep->id_ino == fs->fs_sujfree) 9805 panic("clear_unlinked_inodedep: Freeing head of free list"); 9806 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST); 9807 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 9808 return; 9809 } 9810 9811 /* 9812 * This workitem decrements the inode's link count. 9813 * If the link count reaches zero, the file is removed. 9814 */ 9815 static int 9816 handle_workitem_remove(dirrem, flags) 9817 struct dirrem *dirrem; 9818 int flags; 9819 { 9820 struct inodedep *inodedep; 9821 struct workhead dotdotwk; 9822 struct worklist *wk; 9823 struct ufsmount *ump; 9824 struct mount *mp; 9825 struct vnode *vp; 9826 struct inode *ip; 9827 ino_t oldinum; 9828 9829 if (dirrem->dm_state & ONWORKLIST) 9830 panic("handle_workitem_remove: dirrem %p still on worklist", 9831 dirrem); 9832 oldinum = dirrem->dm_oldinum; 9833 mp = dirrem->dm_list.wk_mp; 9834 ump = VFSTOUFS(mp); 9835 flags |= LK_EXCLUSIVE; 9836 if (ffs_vgetf(mp, oldinum, flags, &vp, FFSV_FORCEINSMQ) != 0) 9837 return (EBUSY); 9838 ip = VTOI(vp); 9839 ACQUIRE_LOCK(ump); 9840 if ((inodedep_lookup(mp, oldinum, 0, &inodedep)) == 0) 9841 panic("handle_workitem_remove: lost inodedep"); 9842 if (dirrem->dm_state & ONDEPLIST) 9843 LIST_REMOVE(dirrem, dm_inonext); 9844 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 9845 ("handle_workitem_remove: Journal entries not written.")); 9846 9847 /* 9848 * Move all dependencies waiting on the remove to complete 9849 * from the dirrem to the inode inowait list to be completed 9850 * after the inode has been updated and written to disk. Any 9851 * marked MKDIR_PARENT are saved to be completed when the .. ref 9852 * is removed. 9853 */ 9854 LIST_INIT(&dotdotwk); 9855 while ((wk = LIST_FIRST(&dirrem->dm_jwork)) != NULL) { 9856 WORKLIST_REMOVE(wk); 9857 if (wk->wk_state & MKDIR_PARENT) { 9858 wk->wk_state &= ~MKDIR_PARENT; 9859 WORKLIST_INSERT(&dotdotwk, wk); 9860 continue; 9861 } 9862 WORKLIST_INSERT(&inodedep->id_inowait, wk); 9863 } 9864 LIST_SWAP(&dirrem->dm_jwork, &dotdotwk, worklist, wk_list); 9865 /* 9866 * Normal file deletion. 9867 */ 9868 if ((dirrem->dm_state & RMDIR) == 0) { 9869 ip->i_nlink--; 9870 KASSERT(ip->i_nlink >= 0, ("handle_workitem_remove: file ino " 9871 "%ju negative i_nlink %d", (intmax_t)ip->i_number, 9872 ip->i_nlink)); 9873 DIP_SET(ip, i_nlink, ip->i_nlink); 9874 ip->i_flag |= IN_CHANGE; 9875 if (ip->i_nlink < ip->i_effnlink) 9876 panic("handle_workitem_remove: bad file delta"); 9877 if (ip->i_nlink == 0) 9878 unlinked_inodedep(mp, inodedep); 9879 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 9880 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 9881 ("handle_workitem_remove: worklist not empty. %s", 9882 TYPENAME(LIST_FIRST(&dirrem->dm_jwork)->wk_type))); 9883 WORKITEM_FREE(dirrem, D_DIRREM); 9884 FREE_LOCK(ump); 9885 goto out; 9886 } 9887 /* 9888 * Directory deletion. Decrement reference count for both the 9889 * just deleted parent directory entry and the reference for ".". 9890 * Arrange to have the reference count on the parent decremented 9891 * to account for the loss of "..". 9892 */ 9893 ip->i_nlink -= 2; 9894 KASSERT(ip->i_nlink >= 0, ("handle_workitem_remove: directory ino " 9895 "%ju negative i_nlink %d", (intmax_t)ip->i_number, ip->i_nlink)); 9896 DIP_SET(ip, i_nlink, ip->i_nlink); 9897 ip->i_flag |= IN_CHANGE; 9898 if (ip->i_nlink < ip->i_effnlink) 9899 panic("handle_workitem_remove: bad dir delta"); 9900 if (ip->i_nlink == 0) 9901 unlinked_inodedep(mp, inodedep); 9902 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 9903 /* 9904 * Rename a directory to a new parent. Since, we are both deleting 9905 * and creating a new directory entry, the link count on the new 9906 * directory should not change. Thus we skip the followup dirrem. 9907 */ 9908 if (dirrem->dm_state & DIRCHG) { 9909 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 9910 ("handle_workitem_remove: DIRCHG and worklist not empty.")); 9911 WORKITEM_FREE(dirrem, D_DIRREM); 9912 FREE_LOCK(ump); 9913 goto out; 9914 } 9915 dirrem->dm_state = ONDEPLIST; 9916 dirrem->dm_oldinum = dirrem->dm_dirinum; 9917 /* 9918 * Place the dirrem on the parent's diremhd list. 9919 */ 9920 if (inodedep_lookup(mp, dirrem->dm_oldinum, 0, &inodedep) == 0) 9921 panic("handle_workitem_remove: lost dir inodedep"); 9922 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 9923 /* 9924 * If the allocated inode has never been written to disk, then 9925 * the on-disk inode is zero'ed and we can remove the file 9926 * immediately. When journaling if the inode has been marked 9927 * unlinked and not DEPCOMPLETE we know it can never be written. 9928 */ 9929 inodedep_lookup(mp, oldinum, 0, &inodedep); 9930 if (inodedep == NULL || 9931 (inodedep->id_state & (DEPCOMPLETE | UNLINKED)) == UNLINKED || 9932 check_inode_unwritten(inodedep)) { 9933 FREE_LOCK(ump); 9934 vput(vp); 9935 return handle_workitem_remove(dirrem, flags); 9936 } 9937 WORKLIST_INSERT(&inodedep->id_inowait, &dirrem->dm_list); 9938 FREE_LOCK(ump); 9939 ip->i_flag |= IN_CHANGE; 9940 out: 9941 ffs_update(vp, 0); 9942 vput(vp); 9943 return (0); 9944 } 9945 9946 /* 9947 * Inode de-allocation dependencies. 9948 * 9949 * When an inode's link count is reduced to zero, it can be de-allocated. We 9950 * found it convenient to postpone de-allocation until after the inode is 9951 * written to disk with its new link count (zero). At this point, all of the 9952 * on-disk inode's block pointers are nullified and, with careful dependency 9953 * list ordering, all dependencies related to the inode will be satisfied and 9954 * the corresponding dependency structures de-allocated. So, if/when the 9955 * inode is reused, there will be no mixing of old dependencies with new 9956 * ones. This artificial dependency is set up by the block de-allocation 9957 * procedure above (softdep_setup_freeblocks) and completed by the 9958 * following procedure. 9959 */ 9960 static void 9961 handle_workitem_freefile(freefile) 9962 struct freefile *freefile; 9963 { 9964 struct workhead wkhd; 9965 struct fs *fs; 9966 struct ufsmount *ump; 9967 int error; 9968 #ifdef INVARIANTS 9969 struct inodedep *idp; 9970 #endif 9971 9972 ump = VFSTOUFS(freefile->fx_list.wk_mp); 9973 fs = ump->um_fs; 9974 #ifdef INVARIANTS 9975 ACQUIRE_LOCK(ump); 9976 error = inodedep_lookup(UFSTOVFS(ump), freefile->fx_oldinum, 0, &idp); 9977 FREE_LOCK(ump); 9978 if (error) 9979 panic("handle_workitem_freefile: inodedep %p survived", idp); 9980 #endif 9981 UFS_LOCK(ump); 9982 fs->fs_pendinginodes -= 1; 9983 UFS_UNLOCK(ump); 9984 LIST_INIT(&wkhd); 9985 LIST_SWAP(&freefile->fx_jwork, &wkhd, worklist, wk_list); 9986 if ((error = ffs_freefile(ump, fs, freefile->fx_devvp, 9987 freefile->fx_oldinum, freefile->fx_mode, &wkhd)) != 0) 9988 softdep_error("handle_workitem_freefile", error); 9989 ACQUIRE_LOCK(ump); 9990 WORKITEM_FREE(freefile, D_FREEFILE); 9991 FREE_LOCK(ump); 9992 } 9993 9994 9995 /* 9996 * Helper function which unlinks marker element from work list and returns 9997 * the next element on the list. 9998 */ 9999 static __inline struct worklist * 10000 markernext(struct worklist *marker) 10001 { 10002 struct worklist *next; 10003 10004 next = LIST_NEXT(marker, wk_list); 10005 LIST_REMOVE(marker, wk_list); 10006 return next; 10007 } 10008 10009 /* 10010 * Disk writes. 10011 * 10012 * The dependency structures constructed above are most actively used when file 10013 * system blocks are written to disk. No constraints are placed on when a 10014 * block can be written, but unsatisfied update dependencies are made safe by 10015 * modifying (or replacing) the source memory for the duration of the disk 10016 * write. When the disk write completes, the memory block is again brought 10017 * up-to-date. 10018 * 10019 * In-core inode structure reclamation. 10020 * 10021 * Because there are a finite number of "in-core" inode structures, they are 10022 * reused regularly. By transferring all inode-related dependencies to the 10023 * in-memory inode block and indexing them separately (via "inodedep"s), we 10024 * can allow "in-core" inode structures to be reused at any time and avoid 10025 * any increase in contention. 10026 * 10027 * Called just before entering the device driver to initiate a new disk I/O. 10028 * The buffer must be locked, thus, no I/O completion operations can occur 10029 * while we are manipulating its associated dependencies. 10030 */ 10031 static void 10032 softdep_disk_io_initiation(bp) 10033 struct buf *bp; /* structure describing disk write to occur */ 10034 { 10035 struct worklist *wk; 10036 struct worklist marker; 10037 struct inodedep *inodedep; 10038 struct freeblks *freeblks; 10039 struct jblkdep *jblkdep; 10040 struct newblk *newblk; 10041 struct ufsmount *ump; 10042 10043 /* 10044 * We only care about write operations. There should never 10045 * be dependencies for reads. 10046 */ 10047 if (bp->b_iocmd != BIO_WRITE) 10048 panic("softdep_disk_io_initiation: not write"); 10049 10050 if (bp->b_vflags & BV_BKGRDINPROG) 10051 panic("softdep_disk_io_initiation: Writing buffer with " 10052 "background write in progress: %p", bp); 10053 10054 ump = softdep_bp_to_mp(bp); 10055 if (ump == NULL) 10056 return; 10057 10058 marker.wk_type = D_LAST + 1; /* Not a normal workitem */ 10059 PHOLD(curproc); /* Don't swap out kernel stack */ 10060 ACQUIRE_LOCK(ump); 10061 /* 10062 * Do any necessary pre-I/O processing. 10063 */ 10064 for (wk = LIST_FIRST(&bp->b_dep); wk != NULL; 10065 wk = markernext(&marker)) { 10066 LIST_INSERT_AFTER(wk, &marker, wk_list); 10067 switch (wk->wk_type) { 10068 10069 case D_PAGEDEP: 10070 initiate_write_filepage(WK_PAGEDEP(wk), bp); 10071 continue; 10072 10073 case D_INODEDEP: 10074 inodedep = WK_INODEDEP(wk); 10075 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) 10076 initiate_write_inodeblock_ufs1(inodedep, bp); 10077 else 10078 initiate_write_inodeblock_ufs2(inodedep, bp); 10079 continue; 10080 10081 case D_INDIRDEP: 10082 initiate_write_indirdep(WK_INDIRDEP(wk), bp); 10083 continue; 10084 10085 case D_BMSAFEMAP: 10086 initiate_write_bmsafemap(WK_BMSAFEMAP(wk), bp); 10087 continue; 10088 10089 case D_JSEG: 10090 WK_JSEG(wk)->js_buf = NULL; 10091 continue; 10092 10093 case D_FREEBLKS: 10094 freeblks = WK_FREEBLKS(wk); 10095 jblkdep = LIST_FIRST(&freeblks->fb_jblkdephd); 10096 /* 10097 * We have to wait for the freeblks to be journaled 10098 * before we can write an inodeblock with updated 10099 * pointers. Be careful to arrange the marker so 10100 * we revisit the freeblks if it's not removed by 10101 * the first jwait(). 10102 */ 10103 if (jblkdep != NULL) { 10104 LIST_REMOVE(&marker, wk_list); 10105 LIST_INSERT_BEFORE(wk, &marker, wk_list); 10106 jwait(&jblkdep->jb_list, MNT_WAIT); 10107 } 10108 continue; 10109 case D_ALLOCDIRECT: 10110 case D_ALLOCINDIR: 10111 /* 10112 * We have to wait for the jnewblk to be journaled 10113 * before we can write to a block if the contents 10114 * may be confused with an earlier file's indirect 10115 * at recovery time. Handle the marker as described 10116 * above. 10117 */ 10118 newblk = WK_NEWBLK(wk); 10119 if (newblk->nb_jnewblk != NULL && 10120 indirblk_lookup(newblk->nb_list.wk_mp, 10121 newblk->nb_newblkno)) { 10122 LIST_REMOVE(&marker, wk_list); 10123 LIST_INSERT_BEFORE(wk, &marker, wk_list); 10124 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT); 10125 } 10126 continue; 10127 10128 case D_SBDEP: 10129 initiate_write_sbdep(WK_SBDEP(wk)); 10130 continue; 10131 10132 case D_MKDIR: 10133 case D_FREEWORK: 10134 case D_FREEDEP: 10135 case D_JSEGDEP: 10136 continue; 10137 10138 default: 10139 panic("handle_disk_io_initiation: Unexpected type %s", 10140 TYPENAME(wk->wk_type)); 10141 /* NOTREACHED */ 10142 } 10143 } 10144 FREE_LOCK(ump); 10145 PRELE(curproc); /* Allow swapout of kernel stack */ 10146 } 10147 10148 /* 10149 * Called from within the procedure above to deal with unsatisfied 10150 * allocation dependencies in a directory. The buffer must be locked, 10151 * thus, no I/O completion operations can occur while we are 10152 * manipulating its associated dependencies. 10153 */ 10154 static void 10155 initiate_write_filepage(pagedep, bp) 10156 struct pagedep *pagedep; 10157 struct buf *bp; 10158 { 10159 struct jremref *jremref; 10160 struct jmvref *jmvref; 10161 struct dirrem *dirrem; 10162 struct diradd *dap; 10163 struct direct *ep; 10164 int i; 10165 10166 if (pagedep->pd_state & IOSTARTED) { 10167 /* 10168 * This can only happen if there is a driver that does not 10169 * understand chaining. Here biodone will reissue the call 10170 * to strategy for the incomplete buffers. 10171 */ 10172 printf("initiate_write_filepage: already started\n"); 10173 return; 10174 } 10175 pagedep->pd_state |= IOSTARTED; 10176 /* 10177 * Wait for all journal remove dependencies to hit the disk. 10178 * We can not allow any potentially conflicting directory adds 10179 * to be visible before removes and rollback is too difficult. 10180 * The per-filesystem lock may be dropped and re-acquired, however 10181 * we hold the buf locked so the dependency can not go away. 10182 */ 10183 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) 10184 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) 10185 jwait(&jremref->jr_list, MNT_WAIT); 10186 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) 10187 jwait(&jmvref->jm_list, MNT_WAIT); 10188 for (i = 0; i < DAHASHSZ; i++) { 10189 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 10190 ep = (struct direct *) 10191 ((char *)bp->b_data + dap->da_offset); 10192 if (ep->d_ino != dap->da_newinum) 10193 panic("%s: dir inum %ju != new %ju", 10194 "initiate_write_filepage", 10195 (uintmax_t)ep->d_ino, 10196 (uintmax_t)dap->da_newinum); 10197 if (dap->da_state & DIRCHG) 10198 ep->d_ino = dap->da_previous->dm_oldinum; 10199 else 10200 ep->d_ino = 0; 10201 dap->da_state &= ~ATTACHED; 10202 dap->da_state |= UNDONE; 10203 } 10204 } 10205 } 10206 10207 /* 10208 * Version of initiate_write_inodeblock that handles UFS1 dinodes. 10209 * Note that any bug fixes made to this routine must be done in the 10210 * version found below. 10211 * 10212 * Called from within the procedure above to deal with unsatisfied 10213 * allocation dependencies in an inodeblock. The buffer must be 10214 * locked, thus, no I/O completion operations can occur while we 10215 * are manipulating its associated dependencies. 10216 */ 10217 static void 10218 initiate_write_inodeblock_ufs1(inodedep, bp) 10219 struct inodedep *inodedep; 10220 struct buf *bp; /* The inode block */ 10221 { 10222 struct allocdirect *adp, *lastadp; 10223 struct ufs1_dinode *dp; 10224 struct ufs1_dinode *sip; 10225 struct inoref *inoref; 10226 struct ufsmount *ump; 10227 struct fs *fs; 10228 ufs_lbn_t i; 10229 #ifdef INVARIANTS 10230 ufs_lbn_t prevlbn = 0; 10231 #endif 10232 int deplist; 10233 10234 if (inodedep->id_state & IOSTARTED) 10235 panic("initiate_write_inodeblock_ufs1: already started"); 10236 inodedep->id_state |= IOSTARTED; 10237 fs = inodedep->id_fs; 10238 ump = VFSTOUFS(inodedep->id_list.wk_mp); 10239 LOCK_OWNED(ump); 10240 dp = (struct ufs1_dinode *)bp->b_data + 10241 ino_to_fsbo(fs, inodedep->id_ino); 10242 10243 /* 10244 * If we're on the unlinked list but have not yet written our 10245 * next pointer initialize it here. 10246 */ 10247 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 10248 struct inodedep *inon; 10249 10250 inon = TAILQ_NEXT(inodedep, id_unlinked); 10251 dp->di_freelink = inon ? inon->id_ino : 0; 10252 } 10253 /* 10254 * If the bitmap is not yet written, then the allocated 10255 * inode cannot be written to disk. 10256 */ 10257 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 10258 if (inodedep->id_savedino1 != NULL) 10259 panic("initiate_write_inodeblock_ufs1: I/O underway"); 10260 FREE_LOCK(ump); 10261 sip = malloc(sizeof(struct ufs1_dinode), 10262 M_SAVEDINO, M_SOFTDEP_FLAGS); 10263 ACQUIRE_LOCK(ump); 10264 inodedep->id_savedino1 = sip; 10265 *inodedep->id_savedino1 = *dp; 10266 bzero((caddr_t)dp, sizeof(struct ufs1_dinode)); 10267 dp->di_gen = inodedep->id_savedino1->di_gen; 10268 dp->di_freelink = inodedep->id_savedino1->di_freelink; 10269 return; 10270 } 10271 /* 10272 * If no dependencies, then there is nothing to roll back. 10273 */ 10274 inodedep->id_savedsize = dp->di_size; 10275 inodedep->id_savedextsize = 0; 10276 inodedep->id_savednlink = dp->di_nlink; 10277 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 10278 TAILQ_EMPTY(&inodedep->id_inoreflst)) 10279 return; 10280 /* 10281 * Revert the link count to that of the first unwritten journal entry. 10282 */ 10283 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 10284 if (inoref) 10285 dp->di_nlink = inoref->if_nlink; 10286 /* 10287 * Set the dependencies to busy. 10288 */ 10289 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10290 adp = TAILQ_NEXT(adp, ad_next)) { 10291 #ifdef INVARIANTS 10292 if (deplist != 0 && prevlbn >= adp->ad_offset) 10293 panic("softdep_write_inodeblock: lbn order"); 10294 prevlbn = adp->ad_offset; 10295 if (adp->ad_offset < UFS_NDADDR && 10296 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 10297 panic("initiate_write_inodeblock_ufs1: " 10298 "direct pointer #%jd mismatch %d != %jd", 10299 (intmax_t)adp->ad_offset, 10300 dp->di_db[adp->ad_offset], 10301 (intmax_t)adp->ad_newblkno); 10302 if (adp->ad_offset >= UFS_NDADDR && 10303 dp->di_ib[adp->ad_offset - UFS_NDADDR] != adp->ad_newblkno) 10304 panic("initiate_write_inodeblock_ufs1: " 10305 "indirect pointer #%jd mismatch %d != %jd", 10306 (intmax_t)adp->ad_offset - UFS_NDADDR, 10307 dp->di_ib[adp->ad_offset - UFS_NDADDR], 10308 (intmax_t)adp->ad_newblkno); 10309 deplist |= 1 << adp->ad_offset; 10310 if ((adp->ad_state & ATTACHED) == 0) 10311 panic("initiate_write_inodeblock_ufs1: " 10312 "Unknown state 0x%x", adp->ad_state); 10313 #endif /* INVARIANTS */ 10314 adp->ad_state &= ~ATTACHED; 10315 adp->ad_state |= UNDONE; 10316 } 10317 /* 10318 * The on-disk inode cannot claim to be any larger than the last 10319 * fragment that has been written. Otherwise, the on-disk inode 10320 * might have fragments that were not the last block in the file 10321 * which would corrupt the filesystem. 10322 */ 10323 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10324 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 10325 if (adp->ad_offset >= UFS_NDADDR) 10326 break; 10327 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 10328 /* keep going until hitting a rollback to a frag */ 10329 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 10330 continue; 10331 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 10332 for (i = adp->ad_offset + 1; i < UFS_NDADDR; i++) { 10333 #ifdef INVARIANTS 10334 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 10335 panic("initiate_write_inodeblock_ufs1: " 10336 "lost dep1"); 10337 #endif /* INVARIANTS */ 10338 dp->di_db[i] = 0; 10339 } 10340 for (i = 0; i < UFS_NIADDR; i++) { 10341 #ifdef INVARIANTS 10342 if (dp->di_ib[i] != 0 && 10343 (deplist & ((1 << UFS_NDADDR) << i)) == 0) 10344 panic("initiate_write_inodeblock_ufs1: " 10345 "lost dep2"); 10346 #endif /* INVARIANTS */ 10347 dp->di_ib[i] = 0; 10348 } 10349 return; 10350 } 10351 /* 10352 * If we have zero'ed out the last allocated block of the file, 10353 * roll back the size to the last currently allocated block. 10354 * We know that this last allocated block is a full-sized as 10355 * we already checked for fragments in the loop above. 10356 */ 10357 if (lastadp != NULL && 10358 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 10359 for (i = lastadp->ad_offset; i >= 0; i--) 10360 if (dp->di_db[i] != 0) 10361 break; 10362 dp->di_size = (i + 1) * fs->fs_bsize; 10363 } 10364 /* 10365 * The only dependencies are for indirect blocks. 10366 * 10367 * The file size for indirect block additions is not guaranteed. 10368 * Such a guarantee would be non-trivial to achieve. The conventional 10369 * synchronous write implementation also does not make this guarantee. 10370 * Fsck should catch and fix discrepancies. Arguably, the file size 10371 * can be over-estimated without destroying integrity when the file 10372 * moves into the indirect blocks (i.e., is large). If we want to 10373 * postpone fsck, we are stuck with this argument. 10374 */ 10375 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 10376 dp->di_ib[adp->ad_offset - UFS_NDADDR] = 0; 10377 } 10378 10379 /* 10380 * Version of initiate_write_inodeblock that handles UFS2 dinodes. 10381 * Note that any bug fixes made to this routine must be done in the 10382 * version found above. 10383 * 10384 * Called from within the procedure above to deal with unsatisfied 10385 * allocation dependencies in an inodeblock. The buffer must be 10386 * locked, thus, no I/O completion operations can occur while we 10387 * are manipulating its associated dependencies. 10388 */ 10389 static void 10390 initiate_write_inodeblock_ufs2(inodedep, bp) 10391 struct inodedep *inodedep; 10392 struct buf *bp; /* The inode block */ 10393 { 10394 struct allocdirect *adp, *lastadp; 10395 struct ufs2_dinode *dp; 10396 struct ufs2_dinode *sip; 10397 struct inoref *inoref; 10398 struct ufsmount *ump; 10399 struct fs *fs; 10400 ufs_lbn_t i; 10401 #ifdef INVARIANTS 10402 ufs_lbn_t prevlbn = 0; 10403 #endif 10404 int deplist; 10405 10406 if (inodedep->id_state & IOSTARTED) 10407 panic("initiate_write_inodeblock_ufs2: already started"); 10408 inodedep->id_state |= IOSTARTED; 10409 fs = inodedep->id_fs; 10410 ump = VFSTOUFS(inodedep->id_list.wk_mp); 10411 LOCK_OWNED(ump); 10412 dp = (struct ufs2_dinode *)bp->b_data + 10413 ino_to_fsbo(fs, inodedep->id_ino); 10414 10415 /* 10416 * If we're on the unlinked list but have not yet written our 10417 * next pointer initialize it here. 10418 */ 10419 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 10420 struct inodedep *inon; 10421 10422 inon = TAILQ_NEXT(inodedep, id_unlinked); 10423 dp->di_freelink = inon ? inon->id_ino : 0; 10424 ffs_update_dinode_ckhash(fs, dp); 10425 } 10426 /* 10427 * If the bitmap is not yet written, then the allocated 10428 * inode cannot be written to disk. 10429 */ 10430 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 10431 if (inodedep->id_savedino2 != NULL) 10432 panic("initiate_write_inodeblock_ufs2: I/O underway"); 10433 FREE_LOCK(ump); 10434 sip = malloc(sizeof(struct ufs2_dinode), 10435 M_SAVEDINO, M_SOFTDEP_FLAGS); 10436 ACQUIRE_LOCK(ump); 10437 inodedep->id_savedino2 = sip; 10438 *inodedep->id_savedino2 = *dp; 10439 bzero((caddr_t)dp, sizeof(struct ufs2_dinode)); 10440 dp->di_gen = inodedep->id_savedino2->di_gen; 10441 dp->di_freelink = inodedep->id_savedino2->di_freelink; 10442 return; 10443 } 10444 /* 10445 * If no dependencies, then there is nothing to roll back. 10446 */ 10447 inodedep->id_savedsize = dp->di_size; 10448 inodedep->id_savedextsize = dp->di_extsize; 10449 inodedep->id_savednlink = dp->di_nlink; 10450 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 10451 TAILQ_EMPTY(&inodedep->id_extupdt) && 10452 TAILQ_EMPTY(&inodedep->id_inoreflst)) 10453 return; 10454 /* 10455 * Revert the link count to that of the first unwritten journal entry. 10456 */ 10457 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 10458 if (inoref) 10459 dp->di_nlink = inoref->if_nlink; 10460 10461 /* 10462 * Set the ext data dependencies to busy. 10463 */ 10464 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 10465 adp = TAILQ_NEXT(adp, ad_next)) { 10466 #ifdef INVARIANTS 10467 if (deplist != 0 && prevlbn >= adp->ad_offset) 10468 panic("initiate_write_inodeblock_ufs2: lbn order"); 10469 prevlbn = adp->ad_offset; 10470 if (dp->di_extb[adp->ad_offset] != adp->ad_newblkno) 10471 panic("initiate_write_inodeblock_ufs2: " 10472 "ext pointer #%jd mismatch %jd != %jd", 10473 (intmax_t)adp->ad_offset, 10474 (intmax_t)dp->di_extb[adp->ad_offset], 10475 (intmax_t)adp->ad_newblkno); 10476 deplist |= 1 << adp->ad_offset; 10477 if ((adp->ad_state & ATTACHED) == 0) 10478 panic("initiate_write_inodeblock_ufs2: Unknown " 10479 "state 0x%x", adp->ad_state); 10480 #endif /* INVARIANTS */ 10481 adp->ad_state &= ~ATTACHED; 10482 adp->ad_state |= UNDONE; 10483 } 10484 /* 10485 * The on-disk inode cannot claim to be any larger than the last 10486 * fragment that has been written. Otherwise, the on-disk inode 10487 * might have fragments that were not the last block in the ext 10488 * data which would corrupt the filesystem. 10489 */ 10490 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 10491 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 10492 dp->di_extb[adp->ad_offset] = adp->ad_oldblkno; 10493 /* keep going until hitting a rollback to a frag */ 10494 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 10495 continue; 10496 dp->di_extsize = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 10497 for (i = adp->ad_offset + 1; i < UFS_NXADDR; i++) { 10498 #ifdef INVARIANTS 10499 if (dp->di_extb[i] != 0 && (deplist & (1 << i)) == 0) 10500 panic("initiate_write_inodeblock_ufs2: " 10501 "lost dep1"); 10502 #endif /* INVARIANTS */ 10503 dp->di_extb[i] = 0; 10504 } 10505 lastadp = NULL; 10506 break; 10507 } 10508 /* 10509 * If we have zero'ed out the last allocated block of the ext 10510 * data, roll back the size to the last currently allocated block. 10511 * We know that this last allocated block is a full-sized as 10512 * we already checked for fragments in the loop above. 10513 */ 10514 if (lastadp != NULL && 10515 dp->di_extsize <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 10516 for (i = lastadp->ad_offset; i >= 0; i--) 10517 if (dp->di_extb[i] != 0) 10518 break; 10519 dp->di_extsize = (i + 1) * fs->fs_bsize; 10520 } 10521 /* 10522 * Set the file data dependencies to busy. 10523 */ 10524 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10525 adp = TAILQ_NEXT(adp, ad_next)) { 10526 #ifdef INVARIANTS 10527 if (deplist != 0 && prevlbn >= adp->ad_offset) 10528 panic("softdep_write_inodeblock: lbn order"); 10529 if ((adp->ad_state & ATTACHED) == 0) 10530 panic("inodedep %p and adp %p not attached", inodedep, adp); 10531 prevlbn = adp->ad_offset; 10532 if (adp->ad_offset < UFS_NDADDR && 10533 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 10534 panic("initiate_write_inodeblock_ufs2: " 10535 "direct pointer #%jd mismatch %jd != %jd", 10536 (intmax_t)adp->ad_offset, 10537 (intmax_t)dp->di_db[adp->ad_offset], 10538 (intmax_t)adp->ad_newblkno); 10539 if (adp->ad_offset >= UFS_NDADDR && 10540 dp->di_ib[adp->ad_offset - UFS_NDADDR] != adp->ad_newblkno) 10541 panic("initiate_write_inodeblock_ufs2: " 10542 "indirect pointer #%jd mismatch %jd != %jd", 10543 (intmax_t)adp->ad_offset - UFS_NDADDR, 10544 (intmax_t)dp->di_ib[adp->ad_offset - UFS_NDADDR], 10545 (intmax_t)adp->ad_newblkno); 10546 deplist |= 1 << adp->ad_offset; 10547 if ((adp->ad_state & ATTACHED) == 0) 10548 panic("initiate_write_inodeblock_ufs2: Unknown " 10549 "state 0x%x", adp->ad_state); 10550 #endif /* INVARIANTS */ 10551 adp->ad_state &= ~ATTACHED; 10552 adp->ad_state |= UNDONE; 10553 } 10554 /* 10555 * The on-disk inode cannot claim to be any larger than the last 10556 * fragment that has been written. Otherwise, the on-disk inode 10557 * might have fragments that were not the last block in the file 10558 * which would corrupt the filesystem. 10559 */ 10560 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10561 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 10562 if (adp->ad_offset >= UFS_NDADDR) 10563 break; 10564 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 10565 /* keep going until hitting a rollback to a frag */ 10566 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 10567 continue; 10568 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 10569 for (i = adp->ad_offset + 1; i < UFS_NDADDR; i++) { 10570 #ifdef INVARIANTS 10571 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 10572 panic("initiate_write_inodeblock_ufs2: " 10573 "lost dep2"); 10574 #endif /* INVARIANTS */ 10575 dp->di_db[i] = 0; 10576 } 10577 for (i = 0; i < UFS_NIADDR; i++) { 10578 #ifdef INVARIANTS 10579 if (dp->di_ib[i] != 0 && 10580 (deplist & ((1 << UFS_NDADDR) << i)) == 0) 10581 panic("initiate_write_inodeblock_ufs2: " 10582 "lost dep3"); 10583 #endif /* INVARIANTS */ 10584 dp->di_ib[i] = 0; 10585 } 10586 ffs_update_dinode_ckhash(fs, dp); 10587 return; 10588 } 10589 /* 10590 * If we have zero'ed out the last allocated block of the file, 10591 * roll back the size to the last currently allocated block. 10592 * We know that this last allocated block is a full-sized as 10593 * we already checked for fragments in the loop above. 10594 */ 10595 if (lastadp != NULL && 10596 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 10597 for (i = lastadp->ad_offset; i >= 0; i--) 10598 if (dp->di_db[i] != 0) 10599 break; 10600 dp->di_size = (i + 1) * fs->fs_bsize; 10601 } 10602 /* 10603 * The only dependencies are for indirect blocks. 10604 * 10605 * The file size for indirect block additions is not guaranteed. 10606 * Such a guarantee would be non-trivial to achieve. The conventional 10607 * synchronous write implementation also does not make this guarantee. 10608 * Fsck should catch and fix discrepancies. Arguably, the file size 10609 * can be over-estimated without destroying integrity when the file 10610 * moves into the indirect blocks (i.e., is large). If we want to 10611 * postpone fsck, we are stuck with this argument. 10612 */ 10613 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 10614 dp->di_ib[adp->ad_offset - UFS_NDADDR] = 0; 10615 ffs_update_dinode_ckhash(fs, dp); 10616 } 10617 10618 /* 10619 * Cancel an indirdep as a result of truncation. Release all of the 10620 * children allocindirs and place their journal work on the appropriate 10621 * list. 10622 */ 10623 static void 10624 cancel_indirdep(indirdep, bp, freeblks) 10625 struct indirdep *indirdep; 10626 struct buf *bp; 10627 struct freeblks *freeblks; 10628 { 10629 struct allocindir *aip; 10630 10631 /* 10632 * None of the indirect pointers will ever be visible, 10633 * so they can simply be tossed. GOINGAWAY ensures 10634 * that allocated pointers will be saved in the buffer 10635 * cache until they are freed. Note that they will 10636 * only be able to be found by their physical address 10637 * since the inode mapping the logical address will 10638 * be gone. The save buffer used for the safe copy 10639 * was allocated in setup_allocindir_phase2 using 10640 * the physical address so it could be used for this 10641 * purpose. Hence we swap the safe copy with the real 10642 * copy, allowing the safe copy to be freed and holding 10643 * on to the real copy for later use in indir_trunc. 10644 */ 10645 if (indirdep->ir_state & GOINGAWAY) 10646 panic("cancel_indirdep: already gone"); 10647 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 10648 indirdep->ir_state |= DEPCOMPLETE; 10649 LIST_REMOVE(indirdep, ir_next); 10650 } 10651 indirdep->ir_state |= GOINGAWAY; 10652 /* 10653 * Pass in bp for blocks still have journal writes 10654 * pending so we can cancel them on their own. 10655 */ 10656 while ((aip = LIST_FIRST(&indirdep->ir_deplisthd)) != NULL) 10657 cancel_allocindir(aip, bp, freeblks, 0); 10658 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != NULL) 10659 cancel_allocindir(aip, NULL, freeblks, 0); 10660 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != NULL) 10661 cancel_allocindir(aip, NULL, freeblks, 0); 10662 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) 10663 cancel_allocindir(aip, NULL, freeblks, 0); 10664 /* 10665 * If there are pending partial truncations we need to keep the 10666 * old block copy around until they complete. This is because 10667 * the current b_data is not a perfect superset of the available 10668 * blocks. 10669 */ 10670 if (TAILQ_EMPTY(&indirdep->ir_trunc)) 10671 bcopy(bp->b_data, indirdep->ir_savebp->b_data, bp->b_bcount); 10672 else 10673 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount); 10674 WORKLIST_REMOVE(&indirdep->ir_list); 10675 WORKLIST_INSERT(&indirdep->ir_savebp->b_dep, &indirdep->ir_list); 10676 indirdep->ir_bp = NULL; 10677 indirdep->ir_freeblks = freeblks; 10678 } 10679 10680 /* 10681 * Free an indirdep once it no longer has new pointers to track. 10682 */ 10683 static void 10684 free_indirdep(indirdep) 10685 struct indirdep *indirdep; 10686 { 10687 10688 KASSERT(TAILQ_EMPTY(&indirdep->ir_trunc), 10689 ("free_indirdep: Indir trunc list not empty.")); 10690 KASSERT(LIST_EMPTY(&indirdep->ir_completehd), 10691 ("free_indirdep: Complete head not empty.")); 10692 KASSERT(LIST_EMPTY(&indirdep->ir_writehd), 10693 ("free_indirdep: write head not empty.")); 10694 KASSERT(LIST_EMPTY(&indirdep->ir_donehd), 10695 ("free_indirdep: done head not empty.")); 10696 KASSERT(LIST_EMPTY(&indirdep->ir_deplisthd), 10697 ("free_indirdep: deplist head not empty.")); 10698 KASSERT((indirdep->ir_state & DEPCOMPLETE), 10699 ("free_indirdep: %p still on newblk list.", indirdep)); 10700 KASSERT(indirdep->ir_saveddata == NULL, 10701 ("free_indirdep: %p still has saved data.", indirdep)); 10702 if (indirdep->ir_state & ONWORKLIST) 10703 WORKLIST_REMOVE(&indirdep->ir_list); 10704 WORKITEM_FREE(indirdep, D_INDIRDEP); 10705 } 10706 10707 /* 10708 * Called before a write to an indirdep. This routine is responsible for 10709 * rolling back pointers to a safe state which includes only those 10710 * allocindirs which have been completed. 10711 */ 10712 static void 10713 initiate_write_indirdep(indirdep, bp) 10714 struct indirdep *indirdep; 10715 struct buf *bp; 10716 { 10717 struct ufsmount *ump; 10718 10719 indirdep->ir_state |= IOSTARTED; 10720 if (indirdep->ir_state & GOINGAWAY) 10721 panic("disk_io_initiation: indirdep gone"); 10722 /* 10723 * If there are no remaining dependencies, this will be writing 10724 * the real pointers. 10725 */ 10726 if (LIST_EMPTY(&indirdep->ir_deplisthd) && 10727 TAILQ_EMPTY(&indirdep->ir_trunc)) 10728 return; 10729 /* 10730 * Replace up-to-date version with safe version. 10731 */ 10732 if (indirdep->ir_saveddata == NULL) { 10733 ump = VFSTOUFS(indirdep->ir_list.wk_mp); 10734 LOCK_OWNED(ump); 10735 FREE_LOCK(ump); 10736 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP, 10737 M_SOFTDEP_FLAGS); 10738 ACQUIRE_LOCK(ump); 10739 } 10740 indirdep->ir_state &= ~ATTACHED; 10741 indirdep->ir_state |= UNDONE; 10742 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount); 10743 bcopy(indirdep->ir_savebp->b_data, bp->b_data, 10744 bp->b_bcount); 10745 } 10746 10747 /* 10748 * Called when an inode has been cleared in a cg bitmap. This finally 10749 * eliminates any canceled jaddrefs 10750 */ 10751 void 10752 softdep_setup_inofree(mp, bp, ino, wkhd) 10753 struct mount *mp; 10754 struct buf *bp; 10755 ino_t ino; 10756 struct workhead *wkhd; 10757 { 10758 struct worklist *wk, *wkn; 10759 struct inodedep *inodedep; 10760 struct ufsmount *ump; 10761 uint8_t *inosused; 10762 struct cg *cgp; 10763 struct fs *fs; 10764 10765 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 10766 ("softdep_setup_inofree called on non-softdep filesystem")); 10767 ump = VFSTOUFS(mp); 10768 ACQUIRE_LOCK(ump); 10769 fs = ump->um_fs; 10770 cgp = (struct cg *)bp->b_data; 10771 inosused = cg_inosused(cgp); 10772 if (isset(inosused, ino % fs->fs_ipg)) 10773 panic("softdep_setup_inofree: inode %ju not freed.", 10774 (uintmax_t)ino); 10775 if (inodedep_lookup(mp, ino, 0, &inodedep)) 10776 panic("softdep_setup_inofree: ino %ju has existing inodedep %p", 10777 (uintmax_t)ino, inodedep); 10778 if (wkhd) { 10779 LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) { 10780 if (wk->wk_type != D_JADDREF) 10781 continue; 10782 WORKLIST_REMOVE(wk); 10783 /* 10784 * We can free immediately even if the jaddref 10785 * isn't attached in a background write as now 10786 * the bitmaps are reconciled. 10787 */ 10788 wk->wk_state |= COMPLETE | ATTACHED; 10789 free_jaddref(WK_JADDREF(wk)); 10790 } 10791 jwork_move(&bp->b_dep, wkhd); 10792 } 10793 FREE_LOCK(ump); 10794 } 10795 10796 /* 10797 * Called via ffs_blkfree() after a set of frags has been cleared from a cg 10798 * map. Any dependencies waiting for the write to clear are added to the 10799 * buf's list and any jnewblks that are being canceled are discarded 10800 * immediately. 10801 */ 10802 void 10803 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 10804 struct mount *mp; 10805 struct buf *bp; 10806 ufs2_daddr_t blkno; 10807 int frags; 10808 struct workhead *wkhd; 10809 { 10810 struct bmsafemap *bmsafemap; 10811 struct jnewblk *jnewblk; 10812 struct ufsmount *ump; 10813 struct worklist *wk; 10814 struct fs *fs; 10815 #ifdef INVARIANTS 10816 uint8_t *blksfree; 10817 struct cg *cgp; 10818 ufs2_daddr_t jstart; 10819 ufs2_daddr_t jend; 10820 ufs2_daddr_t end; 10821 long bno; 10822 int i; 10823 #endif 10824 10825 CTR3(KTR_SUJ, 10826 "softdep_setup_blkfree: blkno %jd frags %d wk head %p", 10827 blkno, frags, wkhd); 10828 10829 ump = VFSTOUFS(mp); 10830 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 10831 ("softdep_setup_blkfree called on non-softdep filesystem")); 10832 ACQUIRE_LOCK(ump); 10833 /* Lookup the bmsafemap so we track when it is dirty. */ 10834 fs = ump->um_fs; 10835 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL); 10836 /* 10837 * Detach any jnewblks which have been canceled. They must linger 10838 * until the bitmap is cleared again by ffs_blkfree() to prevent 10839 * an unjournaled allocation from hitting the disk. 10840 */ 10841 if (wkhd) { 10842 while ((wk = LIST_FIRST(wkhd)) != NULL) { 10843 CTR2(KTR_SUJ, 10844 "softdep_setup_blkfree: blkno %jd wk type %d", 10845 blkno, wk->wk_type); 10846 WORKLIST_REMOVE(wk); 10847 if (wk->wk_type != D_JNEWBLK) { 10848 WORKLIST_INSERT(&bmsafemap->sm_freehd, wk); 10849 continue; 10850 } 10851 jnewblk = WK_JNEWBLK(wk); 10852 KASSERT(jnewblk->jn_state & GOINGAWAY, 10853 ("softdep_setup_blkfree: jnewblk not canceled.")); 10854 #ifdef INVARIANTS 10855 /* 10856 * Assert that this block is free in the bitmap 10857 * before we discard the jnewblk. 10858 */ 10859 cgp = (struct cg *)bp->b_data; 10860 blksfree = cg_blksfree(cgp); 10861 bno = dtogd(fs, jnewblk->jn_blkno); 10862 for (i = jnewblk->jn_oldfrags; 10863 i < jnewblk->jn_frags; i++) { 10864 if (isset(blksfree, bno + i)) 10865 continue; 10866 panic("softdep_setup_blkfree: not free"); 10867 } 10868 #endif 10869 /* 10870 * Even if it's not attached we can free immediately 10871 * as the new bitmap is correct. 10872 */ 10873 wk->wk_state |= COMPLETE | ATTACHED; 10874 free_jnewblk(jnewblk); 10875 } 10876 } 10877 10878 #ifdef INVARIANTS 10879 /* 10880 * Assert that we are not freeing a block which has an outstanding 10881 * allocation dependency. 10882 */ 10883 fs = VFSTOUFS(mp)->um_fs; 10884 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL); 10885 end = blkno + frags; 10886 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 10887 /* 10888 * Don't match against blocks that will be freed when the 10889 * background write is done. 10890 */ 10891 if ((jnewblk->jn_state & (ATTACHED | COMPLETE | DEPCOMPLETE)) == 10892 (COMPLETE | DEPCOMPLETE)) 10893 continue; 10894 jstart = jnewblk->jn_blkno + jnewblk->jn_oldfrags; 10895 jend = jnewblk->jn_blkno + jnewblk->jn_frags; 10896 if ((blkno >= jstart && blkno < jend) || 10897 (end > jstart && end <= jend)) { 10898 printf("state 0x%X %jd - %d %d dep %p\n", 10899 jnewblk->jn_state, jnewblk->jn_blkno, 10900 jnewblk->jn_oldfrags, jnewblk->jn_frags, 10901 jnewblk->jn_dep); 10902 panic("softdep_setup_blkfree: " 10903 "%jd-%jd(%d) overlaps with %jd-%jd", 10904 blkno, end, frags, jstart, jend); 10905 } 10906 } 10907 #endif 10908 FREE_LOCK(ump); 10909 } 10910 10911 /* 10912 * Revert a block allocation when the journal record that describes it 10913 * is not yet written. 10914 */ 10915 static int 10916 jnewblk_rollback(jnewblk, fs, cgp, blksfree) 10917 struct jnewblk *jnewblk; 10918 struct fs *fs; 10919 struct cg *cgp; 10920 uint8_t *blksfree; 10921 { 10922 ufs1_daddr_t fragno; 10923 long cgbno, bbase; 10924 int frags, blk; 10925 int i; 10926 10927 frags = 0; 10928 cgbno = dtogd(fs, jnewblk->jn_blkno); 10929 /* 10930 * We have to test which frags need to be rolled back. We may 10931 * be operating on a stale copy when doing background writes. 10932 */ 10933 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++) 10934 if (isclr(blksfree, cgbno + i)) 10935 frags++; 10936 if (frags == 0) 10937 return (0); 10938 /* 10939 * This is mostly ffs_blkfree() sans some validation and 10940 * superblock updates. 10941 */ 10942 if (frags == fs->fs_frag) { 10943 fragno = fragstoblks(fs, cgbno); 10944 ffs_setblock(fs, blksfree, fragno); 10945 ffs_clusteracct(fs, cgp, fragno, 1); 10946 cgp->cg_cs.cs_nbfree++; 10947 } else { 10948 cgbno += jnewblk->jn_oldfrags; 10949 bbase = cgbno - fragnum(fs, cgbno); 10950 /* Decrement the old frags. */ 10951 blk = blkmap(fs, blksfree, bbase); 10952 ffs_fragacct(fs, blk, cgp->cg_frsum, -1); 10953 /* Deallocate the fragment */ 10954 for (i = 0; i < frags; i++) 10955 setbit(blksfree, cgbno + i); 10956 cgp->cg_cs.cs_nffree += frags; 10957 /* Add back in counts associated with the new frags */ 10958 blk = blkmap(fs, blksfree, bbase); 10959 ffs_fragacct(fs, blk, cgp->cg_frsum, 1); 10960 /* If a complete block has been reassembled, account for it. */ 10961 fragno = fragstoblks(fs, bbase); 10962 if (ffs_isblock(fs, blksfree, fragno)) { 10963 cgp->cg_cs.cs_nffree -= fs->fs_frag; 10964 ffs_clusteracct(fs, cgp, fragno, 1); 10965 cgp->cg_cs.cs_nbfree++; 10966 } 10967 } 10968 stat_jnewblk++; 10969 jnewblk->jn_state &= ~ATTACHED; 10970 jnewblk->jn_state |= UNDONE; 10971 10972 return (frags); 10973 } 10974 10975 static void 10976 initiate_write_bmsafemap(bmsafemap, bp) 10977 struct bmsafemap *bmsafemap; 10978 struct buf *bp; /* The cg block. */ 10979 { 10980 struct jaddref *jaddref; 10981 struct jnewblk *jnewblk; 10982 uint8_t *inosused; 10983 uint8_t *blksfree; 10984 struct cg *cgp; 10985 struct fs *fs; 10986 ino_t ino; 10987 10988 /* 10989 * If this is a background write, we did this at the time that 10990 * the copy was made, so do not need to do it again. 10991 */ 10992 if (bmsafemap->sm_state & IOSTARTED) 10993 return; 10994 bmsafemap->sm_state |= IOSTARTED; 10995 /* 10996 * Clear any inode allocations which are pending journal writes. 10997 */ 10998 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd) != NULL) { 10999 cgp = (struct cg *)bp->b_data; 11000 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 11001 inosused = cg_inosused(cgp); 11002 LIST_FOREACH(jaddref, &bmsafemap->sm_jaddrefhd, ja_bmdeps) { 11003 ino = jaddref->ja_ino % fs->fs_ipg; 11004 if (isset(inosused, ino)) { 11005 if ((jaddref->ja_mode & IFMT) == IFDIR) 11006 cgp->cg_cs.cs_ndir--; 11007 cgp->cg_cs.cs_nifree++; 11008 clrbit(inosused, ino); 11009 jaddref->ja_state &= ~ATTACHED; 11010 jaddref->ja_state |= UNDONE; 11011 stat_jaddref++; 11012 } else 11013 panic("initiate_write_bmsafemap: inode %ju " 11014 "marked free", (uintmax_t)jaddref->ja_ino); 11015 } 11016 } 11017 /* 11018 * Clear any block allocations which are pending journal writes. 11019 */ 11020 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 11021 cgp = (struct cg *)bp->b_data; 11022 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 11023 blksfree = cg_blksfree(cgp); 11024 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 11025 if (jnewblk_rollback(jnewblk, fs, cgp, blksfree)) 11026 continue; 11027 panic("initiate_write_bmsafemap: block %jd " 11028 "marked free", jnewblk->jn_blkno); 11029 } 11030 } 11031 /* 11032 * Move allocation lists to the written lists so they can be 11033 * cleared once the block write is complete. 11034 */ 11035 LIST_SWAP(&bmsafemap->sm_inodedephd, &bmsafemap->sm_inodedepwr, 11036 inodedep, id_deps); 11037 LIST_SWAP(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr, 11038 newblk, nb_deps); 11039 LIST_SWAP(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, worklist, 11040 wk_list); 11041 } 11042 11043 /* 11044 * This routine is called during the completion interrupt 11045 * service routine for a disk write (from the procedure called 11046 * by the device driver to inform the filesystem caches of 11047 * a request completion). It should be called early in this 11048 * procedure, before the block is made available to other 11049 * processes or other routines are called. 11050 * 11051 */ 11052 static void 11053 softdep_disk_write_complete(bp) 11054 struct buf *bp; /* describes the completed disk write */ 11055 { 11056 struct worklist *wk; 11057 struct worklist *owk; 11058 struct ufsmount *ump; 11059 struct workhead reattach; 11060 struct freeblks *freeblks; 11061 struct buf *sbp; 11062 11063 ump = softdep_bp_to_mp(bp); 11064 KASSERT(LIST_EMPTY(&bp->b_dep) || ump != NULL, 11065 ("softdep_disk_write_complete: softdep_bp_to_mp returned NULL " 11066 "with outstanding dependencies for buffer %p", bp)); 11067 if (ump == NULL) 11068 return; 11069 /* 11070 * If an error occurred while doing the write, then the data 11071 * has not hit the disk and the dependencies cannot be processed. 11072 * But we do have to go through and roll forward any dependencies 11073 * that were rolled back before the disk write. 11074 */ 11075 sbp = NULL; 11076 ACQUIRE_LOCK(ump); 11077 if ((bp->b_ioflags & BIO_ERROR) != 0 && (bp->b_flags & B_INVAL) == 0) { 11078 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 11079 switch (wk->wk_type) { 11080 11081 case D_PAGEDEP: 11082 handle_written_filepage(WK_PAGEDEP(wk), bp, 0); 11083 continue; 11084 11085 case D_INODEDEP: 11086 handle_written_inodeblock(WK_INODEDEP(wk), 11087 bp, 0); 11088 continue; 11089 11090 case D_BMSAFEMAP: 11091 handle_written_bmsafemap(WK_BMSAFEMAP(wk), 11092 bp, 0); 11093 continue; 11094 11095 case D_INDIRDEP: 11096 handle_written_indirdep(WK_INDIRDEP(wk), 11097 bp, &sbp, 0); 11098 continue; 11099 default: 11100 /* nothing to roll forward */ 11101 continue; 11102 } 11103 } 11104 FREE_LOCK(ump); 11105 if (sbp) 11106 brelse(sbp); 11107 return; 11108 } 11109 LIST_INIT(&reattach); 11110 11111 /* 11112 * Ump SU lock must not be released anywhere in this code segment. 11113 */ 11114 owk = NULL; 11115 while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) { 11116 WORKLIST_REMOVE(wk); 11117 atomic_add_long(&dep_write[wk->wk_type], 1); 11118 if (wk == owk) 11119 panic("duplicate worklist: %p\n", wk); 11120 owk = wk; 11121 switch (wk->wk_type) { 11122 11123 case D_PAGEDEP: 11124 if (handle_written_filepage(WK_PAGEDEP(wk), bp, 11125 WRITESUCCEEDED)) 11126 WORKLIST_INSERT(&reattach, wk); 11127 continue; 11128 11129 case D_INODEDEP: 11130 if (handle_written_inodeblock(WK_INODEDEP(wk), bp, 11131 WRITESUCCEEDED)) 11132 WORKLIST_INSERT(&reattach, wk); 11133 continue; 11134 11135 case D_BMSAFEMAP: 11136 if (handle_written_bmsafemap(WK_BMSAFEMAP(wk), bp, 11137 WRITESUCCEEDED)) 11138 WORKLIST_INSERT(&reattach, wk); 11139 continue; 11140 11141 case D_MKDIR: 11142 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 11143 continue; 11144 11145 case D_ALLOCDIRECT: 11146 wk->wk_state |= COMPLETE; 11147 handle_allocdirect_partdone(WK_ALLOCDIRECT(wk), NULL); 11148 continue; 11149 11150 case D_ALLOCINDIR: 11151 wk->wk_state |= COMPLETE; 11152 handle_allocindir_partdone(WK_ALLOCINDIR(wk)); 11153 continue; 11154 11155 case D_INDIRDEP: 11156 if (handle_written_indirdep(WK_INDIRDEP(wk), bp, &sbp, 11157 WRITESUCCEEDED)) 11158 WORKLIST_INSERT(&reattach, wk); 11159 continue; 11160 11161 case D_FREEBLKS: 11162 wk->wk_state |= COMPLETE; 11163 freeblks = WK_FREEBLKS(wk); 11164 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE && 11165 LIST_EMPTY(&freeblks->fb_jblkdephd)) 11166 add_to_worklist(wk, WK_NODELAY); 11167 continue; 11168 11169 case D_FREEWORK: 11170 handle_written_freework(WK_FREEWORK(wk)); 11171 break; 11172 11173 case D_JSEGDEP: 11174 free_jsegdep(WK_JSEGDEP(wk)); 11175 continue; 11176 11177 case D_JSEG: 11178 handle_written_jseg(WK_JSEG(wk), bp); 11179 continue; 11180 11181 case D_SBDEP: 11182 if (handle_written_sbdep(WK_SBDEP(wk), bp)) 11183 WORKLIST_INSERT(&reattach, wk); 11184 continue; 11185 11186 case D_FREEDEP: 11187 free_freedep(WK_FREEDEP(wk)); 11188 continue; 11189 11190 default: 11191 panic("handle_disk_write_complete: Unknown type %s", 11192 TYPENAME(wk->wk_type)); 11193 /* NOTREACHED */ 11194 } 11195 } 11196 /* 11197 * Reattach any requests that must be redone. 11198 */ 11199 while ((wk = LIST_FIRST(&reattach)) != NULL) { 11200 WORKLIST_REMOVE(wk); 11201 WORKLIST_INSERT(&bp->b_dep, wk); 11202 } 11203 FREE_LOCK(ump); 11204 if (sbp) 11205 brelse(sbp); 11206 } 11207 11208 /* 11209 * Called from within softdep_disk_write_complete above. 11210 */ 11211 static void 11212 handle_allocdirect_partdone(adp, wkhd) 11213 struct allocdirect *adp; /* the completed allocdirect */ 11214 struct workhead *wkhd; /* Work to do when inode is writtne. */ 11215 { 11216 struct allocdirectlst *listhead; 11217 struct allocdirect *listadp; 11218 struct inodedep *inodedep; 11219 long bsize; 11220 11221 LOCK_OWNED(VFSTOUFS(adp->ad_block.nb_list.wk_mp)); 11222 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 11223 return; 11224 /* 11225 * The on-disk inode cannot claim to be any larger than the last 11226 * fragment that has been written. Otherwise, the on-disk inode 11227 * might have fragments that were not the last block in the file 11228 * which would corrupt the filesystem. Thus, we cannot free any 11229 * allocdirects after one whose ad_oldblkno claims a fragment as 11230 * these blocks must be rolled back to zero before writing the inode. 11231 * We check the currently active set of allocdirects in id_inoupdt 11232 * or id_extupdt as appropriate. 11233 */ 11234 inodedep = adp->ad_inodedep; 11235 bsize = inodedep->id_fs->fs_bsize; 11236 if (adp->ad_state & EXTDATA) 11237 listhead = &inodedep->id_extupdt; 11238 else 11239 listhead = &inodedep->id_inoupdt; 11240 TAILQ_FOREACH(listadp, listhead, ad_next) { 11241 /* found our block */ 11242 if (listadp == adp) 11243 break; 11244 /* continue if ad_oldlbn is not a fragment */ 11245 if (listadp->ad_oldsize == 0 || 11246 listadp->ad_oldsize == bsize) 11247 continue; 11248 /* hit a fragment */ 11249 return; 11250 } 11251 /* 11252 * If we have reached the end of the current list without 11253 * finding the just finished dependency, then it must be 11254 * on the future dependency list. Future dependencies cannot 11255 * be freed until they are moved to the current list. 11256 */ 11257 if (listadp == NULL) { 11258 #ifdef INVARIANTS 11259 if (adp->ad_state & EXTDATA) 11260 listhead = &inodedep->id_newextupdt; 11261 else 11262 listhead = &inodedep->id_newinoupdt; 11263 TAILQ_FOREACH(listadp, listhead, ad_next) 11264 /* found our block */ 11265 if (listadp == adp) 11266 break; 11267 if (listadp == NULL) 11268 panic("handle_allocdirect_partdone: lost dep"); 11269 #endif /* INVARIANTS */ 11270 return; 11271 } 11272 /* 11273 * If we have found the just finished dependency, then queue 11274 * it along with anything that follows it that is complete. 11275 * Since the pointer has not yet been written in the inode 11276 * as the dependency prevents it, place the allocdirect on the 11277 * bufwait list where it will be freed once the pointer is 11278 * valid. 11279 */ 11280 if (wkhd == NULL) 11281 wkhd = &inodedep->id_bufwait; 11282 for (; adp; adp = listadp) { 11283 listadp = TAILQ_NEXT(adp, ad_next); 11284 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 11285 return; 11286 TAILQ_REMOVE(listhead, adp, ad_next); 11287 WORKLIST_INSERT(wkhd, &adp->ad_block.nb_list); 11288 } 11289 } 11290 11291 /* 11292 * Called from within softdep_disk_write_complete above. This routine 11293 * completes successfully written allocindirs. 11294 */ 11295 static void 11296 handle_allocindir_partdone(aip) 11297 struct allocindir *aip; /* the completed allocindir */ 11298 { 11299 struct indirdep *indirdep; 11300 11301 if ((aip->ai_state & ALLCOMPLETE) != ALLCOMPLETE) 11302 return; 11303 indirdep = aip->ai_indirdep; 11304 LIST_REMOVE(aip, ai_next); 11305 /* 11306 * Don't set a pointer while the buffer is undergoing IO or while 11307 * we have active truncations. 11308 */ 11309 if (indirdep->ir_state & UNDONE || !TAILQ_EMPTY(&indirdep->ir_trunc)) { 11310 LIST_INSERT_HEAD(&indirdep->ir_donehd, aip, ai_next); 11311 return; 11312 } 11313 if (indirdep->ir_state & UFS1FMT) 11314 ((ufs1_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 11315 aip->ai_newblkno; 11316 else 11317 ((ufs2_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 11318 aip->ai_newblkno; 11319 /* 11320 * Await the pointer write before freeing the allocindir. 11321 */ 11322 LIST_INSERT_HEAD(&indirdep->ir_writehd, aip, ai_next); 11323 } 11324 11325 /* 11326 * Release segments held on a jwork list. 11327 */ 11328 static void 11329 handle_jwork(wkhd) 11330 struct workhead *wkhd; 11331 { 11332 struct worklist *wk; 11333 11334 while ((wk = LIST_FIRST(wkhd)) != NULL) { 11335 WORKLIST_REMOVE(wk); 11336 switch (wk->wk_type) { 11337 case D_JSEGDEP: 11338 free_jsegdep(WK_JSEGDEP(wk)); 11339 continue; 11340 case D_FREEDEP: 11341 free_freedep(WK_FREEDEP(wk)); 11342 continue; 11343 case D_FREEFRAG: 11344 rele_jseg(WK_JSEG(WK_FREEFRAG(wk)->ff_jdep)); 11345 WORKITEM_FREE(wk, D_FREEFRAG); 11346 continue; 11347 case D_FREEWORK: 11348 handle_written_freework(WK_FREEWORK(wk)); 11349 continue; 11350 default: 11351 panic("handle_jwork: Unknown type %s\n", 11352 TYPENAME(wk->wk_type)); 11353 } 11354 } 11355 } 11356 11357 /* 11358 * Handle the bufwait list on an inode when it is safe to release items 11359 * held there. This normally happens after an inode block is written but 11360 * may be delayed and handled later if there are pending journal items that 11361 * are not yet safe to be released. 11362 */ 11363 static struct freefile * 11364 handle_bufwait(inodedep, refhd) 11365 struct inodedep *inodedep; 11366 struct workhead *refhd; 11367 { 11368 struct jaddref *jaddref; 11369 struct freefile *freefile; 11370 struct worklist *wk; 11371 11372 freefile = NULL; 11373 while ((wk = LIST_FIRST(&inodedep->id_bufwait)) != NULL) { 11374 WORKLIST_REMOVE(wk); 11375 switch (wk->wk_type) { 11376 case D_FREEFILE: 11377 /* 11378 * We defer adding freefile to the worklist 11379 * until all other additions have been made to 11380 * ensure that it will be done after all the 11381 * old blocks have been freed. 11382 */ 11383 if (freefile != NULL) 11384 panic("handle_bufwait: freefile"); 11385 freefile = WK_FREEFILE(wk); 11386 continue; 11387 11388 case D_MKDIR: 11389 handle_written_mkdir(WK_MKDIR(wk), MKDIR_PARENT); 11390 continue; 11391 11392 case D_DIRADD: 11393 diradd_inode_written(WK_DIRADD(wk), inodedep); 11394 continue; 11395 11396 case D_FREEFRAG: 11397 wk->wk_state |= COMPLETE; 11398 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE) 11399 add_to_worklist(wk, 0); 11400 continue; 11401 11402 case D_DIRREM: 11403 wk->wk_state |= COMPLETE; 11404 add_to_worklist(wk, 0); 11405 continue; 11406 11407 case D_ALLOCDIRECT: 11408 case D_ALLOCINDIR: 11409 free_newblk(WK_NEWBLK(wk)); 11410 continue; 11411 11412 case D_JNEWBLK: 11413 wk->wk_state |= COMPLETE; 11414 free_jnewblk(WK_JNEWBLK(wk)); 11415 continue; 11416 11417 /* 11418 * Save freed journal segments and add references on 11419 * the supplied list which will delay their release 11420 * until the cg bitmap is cleared on disk. 11421 */ 11422 case D_JSEGDEP: 11423 if (refhd == NULL) 11424 free_jsegdep(WK_JSEGDEP(wk)); 11425 else 11426 WORKLIST_INSERT(refhd, wk); 11427 continue; 11428 11429 case D_JADDREF: 11430 jaddref = WK_JADDREF(wk); 11431 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 11432 if_deps); 11433 /* 11434 * Transfer any jaddrefs to the list to be freed with 11435 * the bitmap if we're handling a removed file. 11436 */ 11437 if (refhd == NULL) { 11438 wk->wk_state |= COMPLETE; 11439 free_jaddref(jaddref); 11440 } else 11441 WORKLIST_INSERT(refhd, wk); 11442 continue; 11443 11444 default: 11445 panic("handle_bufwait: Unknown type %p(%s)", 11446 wk, TYPENAME(wk->wk_type)); 11447 /* NOTREACHED */ 11448 } 11449 } 11450 return (freefile); 11451 } 11452 /* 11453 * Called from within softdep_disk_write_complete above to restore 11454 * in-memory inode block contents to their most up-to-date state. Note 11455 * that this routine is always called from interrupt level with further 11456 * interrupts from this device blocked. 11457 * 11458 * If the write did not succeed, we will do all the roll-forward 11459 * operations, but we will not take the actions that will allow its 11460 * dependencies to be processed. 11461 */ 11462 static int 11463 handle_written_inodeblock(inodedep, bp, flags) 11464 struct inodedep *inodedep; 11465 struct buf *bp; /* buffer containing the inode block */ 11466 int flags; 11467 { 11468 struct freefile *freefile; 11469 struct allocdirect *adp, *nextadp; 11470 struct ufs1_dinode *dp1 = NULL; 11471 struct ufs2_dinode *dp2 = NULL; 11472 struct workhead wkhd; 11473 int hadchanges, fstype; 11474 ino_t freelink; 11475 11476 LIST_INIT(&wkhd); 11477 hadchanges = 0; 11478 freefile = NULL; 11479 if ((inodedep->id_state & IOSTARTED) == 0) 11480 panic("handle_written_inodeblock: not started"); 11481 inodedep->id_state &= ~IOSTARTED; 11482 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) { 11483 fstype = UFS1; 11484 dp1 = (struct ufs1_dinode *)bp->b_data + 11485 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 11486 freelink = dp1->di_freelink; 11487 } else { 11488 fstype = UFS2; 11489 dp2 = (struct ufs2_dinode *)bp->b_data + 11490 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 11491 freelink = dp2->di_freelink; 11492 } 11493 /* 11494 * Leave this inodeblock dirty until it's in the list. 11495 */ 11496 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) == UNLINKED && 11497 (flags & WRITESUCCEEDED)) { 11498 struct inodedep *inon; 11499 11500 inon = TAILQ_NEXT(inodedep, id_unlinked); 11501 if ((inon == NULL && freelink == 0) || 11502 (inon && inon->id_ino == freelink)) { 11503 if (inon) 11504 inon->id_state |= UNLINKPREV; 11505 inodedep->id_state |= UNLINKNEXT; 11506 } 11507 hadchanges = 1; 11508 } 11509 /* 11510 * If we had to rollback the inode allocation because of 11511 * bitmaps being incomplete, then simply restore it. 11512 * Keep the block dirty so that it will not be reclaimed until 11513 * all associated dependencies have been cleared and the 11514 * corresponding updates written to disk. 11515 */ 11516 if (inodedep->id_savedino1 != NULL) { 11517 hadchanges = 1; 11518 if (fstype == UFS1) 11519 *dp1 = *inodedep->id_savedino1; 11520 else 11521 *dp2 = *inodedep->id_savedino2; 11522 free(inodedep->id_savedino1, M_SAVEDINO); 11523 inodedep->id_savedino1 = NULL; 11524 if ((bp->b_flags & B_DELWRI) == 0) 11525 stat_inode_bitmap++; 11526 bdirty(bp); 11527 /* 11528 * If the inode is clear here and GOINGAWAY it will never 11529 * be written. Process the bufwait and clear any pending 11530 * work which may include the freefile. 11531 */ 11532 if (inodedep->id_state & GOINGAWAY) 11533 goto bufwait; 11534 return (1); 11535 } 11536 if (flags & WRITESUCCEEDED) 11537 inodedep->id_state |= COMPLETE; 11538 /* 11539 * Roll forward anything that had to be rolled back before 11540 * the inode could be updated. 11541 */ 11542 for (adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; adp = nextadp) { 11543 nextadp = TAILQ_NEXT(adp, ad_next); 11544 if (adp->ad_state & ATTACHED) 11545 panic("handle_written_inodeblock: new entry"); 11546 if (fstype == UFS1) { 11547 if (adp->ad_offset < UFS_NDADDR) { 11548 if (dp1->di_db[adp->ad_offset]!=adp->ad_oldblkno) 11549 panic("%s %s #%jd mismatch %d != %jd", 11550 "handle_written_inodeblock:", 11551 "direct pointer", 11552 (intmax_t)adp->ad_offset, 11553 dp1->di_db[adp->ad_offset], 11554 (intmax_t)adp->ad_oldblkno); 11555 dp1->di_db[adp->ad_offset] = adp->ad_newblkno; 11556 } else { 11557 if (dp1->di_ib[adp->ad_offset - UFS_NDADDR] != 11558 0) 11559 panic("%s: %s #%jd allocated as %d", 11560 "handle_written_inodeblock", 11561 "indirect pointer", 11562 (intmax_t)adp->ad_offset - 11563 UFS_NDADDR, 11564 dp1->di_ib[adp->ad_offset - 11565 UFS_NDADDR]); 11566 dp1->di_ib[adp->ad_offset - UFS_NDADDR] = 11567 adp->ad_newblkno; 11568 } 11569 } else { 11570 if (adp->ad_offset < UFS_NDADDR) { 11571 if (dp2->di_db[adp->ad_offset]!=adp->ad_oldblkno) 11572 panic("%s: %s #%jd %s %jd != %jd", 11573 "handle_written_inodeblock", 11574 "direct pointer", 11575 (intmax_t)adp->ad_offset, "mismatch", 11576 (intmax_t)dp2->di_db[adp->ad_offset], 11577 (intmax_t)adp->ad_oldblkno); 11578 dp2->di_db[adp->ad_offset] = adp->ad_newblkno; 11579 } else { 11580 if (dp2->di_ib[adp->ad_offset - UFS_NDADDR] != 11581 0) 11582 panic("%s: %s #%jd allocated as %jd", 11583 "handle_written_inodeblock", 11584 "indirect pointer", 11585 (intmax_t)adp->ad_offset - 11586 UFS_NDADDR, 11587 (intmax_t) 11588 dp2->di_ib[adp->ad_offset - 11589 UFS_NDADDR]); 11590 dp2->di_ib[adp->ad_offset - UFS_NDADDR] = 11591 adp->ad_newblkno; 11592 } 11593 } 11594 adp->ad_state &= ~UNDONE; 11595 adp->ad_state |= ATTACHED; 11596 hadchanges = 1; 11597 } 11598 for (adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; adp = nextadp) { 11599 nextadp = TAILQ_NEXT(adp, ad_next); 11600 if (adp->ad_state & ATTACHED) 11601 panic("handle_written_inodeblock: new entry"); 11602 if (dp2->di_extb[adp->ad_offset] != adp->ad_oldblkno) 11603 panic("%s: direct pointers #%jd %s %jd != %jd", 11604 "handle_written_inodeblock", 11605 (intmax_t)adp->ad_offset, "mismatch", 11606 (intmax_t)dp2->di_extb[adp->ad_offset], 11607 (intmax_t)adp->ad_oldblkno); 11608 dp2->di_extb[adp->ad_offset] = adp->ad_newblkno; 11609 adp->ad_state &= ~UNDONE; 11610 adp->ad_state |= ATTACHED; 11611 hadchanges = 1; 11612 } 11613 if (hadchanges && (bp->b_flags & B_DELWRI) == 0) 11614 stat_direct_blk_ptrs++; 11615 /* 11616 * Reset the file size to its most up-to-date value. 11617 */ 11618 if (inodedep->id_savedsize == -1 || inodedep->id_savedextsize == -1) 11619 panic("handle_written_inodeblock: bad size"); 11620 if (inodedep->id_savednlink > UFS_LINK_MAX) 11621 panic("handle_written_inodeblock: Invalid link count " 11622 "%jd for inodedep %p", (uintmax_t)inodedep->id_savednlink, 11623 inodedep); 11624 if (fstype == UFS1) { 11625 if (dp1->di_nlink != inodedep->id_savednlink) { 11626 dp1->di_nlink = inodedep->id_savednlink; 11627 hadchanges = 1; 11628 } 11629 if (dp1->di_size != inodedep->id_savedsize) { 11630 dp1->di_size = inodedep->id_savedsize; 11631 hadchanges = 1; 11632 } 11633 } else { 11634 if (dp2->di_nlink != inodedep->id_savednlink) { 11635 dp2->di_nlink = inodedep->id_savednlink; 11636 hadchanges = 1; 11637 } 11638 if (dp2->di_size != inodedep->id_savedsize) { 11639 dp2->di_size = inodedep->id_savedsize; 11640 hadchanges = 1; 11641 } 11642 if (dp2->di_extsize != inodedep->id_savedextsize) { 11643 dp2->di_extsize = inodedep->id_savedextsize; 11644 hadchanges = 1; 11645 } 11646 } 11647 inodedep->id_savedsize = -1; 11648 inodedep->id_savedextsize = -1; 11649 inodedep->id_savednlink = -1; 11650 /* 11651 * If there were any rollbacks in the inode block, then it must be 11652 * marked dirty so that its will eventually get written back in 11653 * its correct form. 11654 */ 11655 if (hadchanges) { 11656 if (fstype == UFS2) 11657 ffs_update_dinode_ckhash(inodedep->id_fs, dp2); 11658 bdirty(bp); 11659 } 11660 bufwait: 11661 /* 11662 * If the write did not succeed, we have done all the roll-forward 11663 * operations, but we cannot take the actions that will allow its 11664 * dependencies to be processed. 11665 */ 11666 if ((flags & WRITESUCCEEDED) == 0) 11667 return (hadchanges); 11668 /* 11669 * Process any allocdirects that completed during the update. 11670 */ 11671 if ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL) 11672 handle_allocdirect_partdone(adp, &wkhd); 11673 if ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 11674 handle_allocdirect_partdone(adp, &wkhd); 11675 /* 11676 * Process deallocations that were held pending until the 11677 * inode had been written to disk. Freeing of the inode 11678 * is delayed until after all blocks have been freed to 11679 * avoid creation of new <vfsid, inum, lbn> triples 11680 * before the old ones have been deleted. Completely 11681 * unlinked inodes are not processed until the unlinked 11682 * inode list is written or the last reference is removed. 11683 */ 11684 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) != UNLINKED) { 11685 freefile = handle_bufwait(inodedep, NULL); 11686 if (freefile && !LIST_EMPTY(&wkhd)) { 11687 WORKLIST_INSERT(&wkhd, &freefile->fx_list); 11688 freefile = NULL; 11689 } 11690 } 11691 /* 11692 * Move rolled forward dependency completions to the bufwait list 11693 * now that those that were already written have been processed. 11694 */ 11695 if (!LIST_EMPTY(&wkhd) && hadchanges == 0) 11696 panic("handle_written_inodeblock: bufwait but no changes"); 11697 jwork_move(&inodedep->id_bufwait, &wkhd); 11698 11699 if (freefile != NULL) { 11700 /* 11701 * If the inode is goingaway it was never written. Fake up 11702 * the state here so free_inodedep() can succeed. 11703 */ 11704 if (inodedep->id_state & GOINGAWAY) 11705 inodedep->id_state |= COMPLETE | DEPCOMPLETE; 11706 if (free_inodedep(inodedep) == 0) 11707 panic("handle_written_inodeblock: live inodedep %p", 11708 inodedep); 11709 add_to_worklist(&freefile->fx_list, 0); 11710 return (0); 11711 } 11712 11713 /* 11714 * If no outstanding dependencies, free it. 11715 */ 11716 if (free_inodedep(inodedep) || 11717 (TAILQ_FIRST(&inodedep->id_inoreflst) == 0 && 11718 TAILQ_FIRST(&inodedep->id_inoupdt) == 0 && 11719 TAILQ_FIRST(&inodedep->id_extupdt) == 0 && 11720 LIST_FIRST(&inodedep->id_bufwait) == 0)) 11721 return (0); 11722 return (hadchanges); 11723 } 11724 11725 /* 11726 * Perform needed roll-forwards and kick off any dependencies that 11727 * can now be processed. 11728 * 11729 * If the write did not succeed, we will do all the roll-forward 11730 * operations, but we will not take the actions that will allow its 11731 * dependencies to be processed. 11732 */ 11733 static int 11734 handle_written_indirdep(indirdep, bp, bpp, flags) 11735 struct indirdep *indirdep; 11736 struct buf *bp; 11737 struct buf **bpp; 11738 int flags; 11739 { 11740 struct allocindir *aip; 11741 struct buf *sbp; 11742 int chgs; 11743 11744 if (indirdep->ir_state & GOINGAWAY) 11745 panic("handle_written_indirdep: indirdep gone"); 11746 if ((indirdep->ir_state & IOSTARTED) == 0) 11747 panic("handle_written_indirdep: IO not started"); 11748 chgs = 0; 11749 /* 11750 * If there were rollbacks revert them here. 11751 */ 11752 if (indirdep->ir_saveddata) { 11753 bcopy(indirdep->ir_saveddata, bp->b_data, bp->b_bcount); 11754 if (TAILQ_EMPTY(&indirdep->ir_trunc)) { 11755 free(indirdep->ir_saveddata, M_INDIRDEP); 11756 indirdep->ir_saveddata = NULL; 11757 } 11758 chgs = 1; 11759 } 11760 indirdep->ir_state &= ~(UNDONE | IOSTARTED); 11761 indirdep->ir_state |= ATTACHED; 11762 /* 11763 * If the write did not succeed, we have done all the roll-forward 11764 * operations, but we cannot take the actions that will allow its 11765 * dependencies to be processed. 11766 */ 11767 if ((flags & WRITESUCCEEDED) == 0) { 11768 stat_indir_blk_ptrs++; 11769 bdirty(bp); 11770 return (1); 11771 } 11772 /* 11773 * Move allocindirs with written pointers to the completehd if 11774 * the indirdep's pointer is not yet written. Otherwise 11775 * free them here. 11776 */ 11777 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != NULL) { 11778 LIST_REMOVE(aip, ai_next); 11779 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 11780 LIST_INSERT_HEAD(&indirdep->ir_completehd, aip, 11781 ai_next); 11782 newblk_freefrag(&aip->ai_block); 11783 continue; 11784 } 11785 free_newblk(&aip->ai_block); 11786 } 11787 /* 11788 * Move allocindirs that have finished dependency processing from 11789 * the done list to the write list after updating the pointers. 11790 */ 11791 if (TAILQ_EMPTY(&indirdep->ir_trunc)) { 11792 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != NULL) { 11793 handle_allocindir_partdone(aip); 11794 if (aip == LIST_FIRST(&indirdep->ir_donehd)) 11795 panic("disk_write_complete: not gone"); 11796 chgs = 1; 11797 } 11798 } 11799 /* 11800 * Preserve the indirdep if there were any changes or if it is not 11801 * yet valid on disk. 11802 */ 11803 if (chgs) { 11804 stat_indir_blk_ptrs++; 11805 bdirty(bp); 11806 return (1); 11807 } 11808 /* 11809 * If there were no changes we can discard the savedbp and detach 11810 * ourselves from the buf. We are only carrying completed pointers 11811 * in this case. 11812 */ 11813 sbp = indirdep->ir_savebp; 11814 sbp->b_flags |= B_INVAL | B_NOCACHE; 11815 indirdep->ir_savebp = NULL; 11816 indirdep->ir_bp = NULL; 11817 if (*bpp != NULL) 11818 panic("handle_written_indirdep: bp already exists."); 11819 *bpp = sbp; 11820 /* 11821 * The indirdep may not be freed until its parent points at it. 11822 */ 11823 if (indirdep->ir_state & DEPCOMPLETE) 11824 free_indirdep(indirdep); 11825 11826 return (0); 11827 } 11828 11829 /* 11830 * Process a diradd entry after its dependent inode has been written. 11831 */ 11832 static void 11833 diradd_inode_written(dap, inodedep) 11834 struct diradd *dap; 11835 struct inodedep *inodedep; 11836 { 11837 11838 LOCK_OWNED(VFSTOUFS(dap->da_list.wk_mp)); 11839 dap->da_state |= COMPLETE; 11840 complete_diradd(dap); 11841 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 11842 } 11843 11844 /* 11845 * Returns true if the bmsafemap will have rollbacks when written. Must only 11846 * be called with the per-filesystem lock and the buf lock on the cg held. 11847 */ 11848 static int 11849 bmsafemap_backgroundwrite(bmsafemap, bp) 11850 struct bmsafemap *bmsafemap; 11851 struct buf *bp; 11852 { 11853 int dirty; 11854 11855 LOCK_OWNED(VFSTOUFS(bmsafemap->sm_list.wk_mp)); 11856 dirty = !LIST_EMPTY(&bmsafemap->sm_jaddrefhd) | 11857 !LIST_EMPTY(&bmsafemap->sm_jnewblkhd); 11858 /* 11859 * If we're initiating a background write we need to process the 11860 * rollbacks as they exist now, not as they exist when IO starts. 11861 * No other consumers will look at the contents of the shadowed 11862 * buf so this is safe to do here. 11863 */ 11864 if (bp->b_xflags & BX_BKGRDMARKER) 11865 initiate_write_bmsafemap(bmsafemap, bp); 11866 11867 return (dirty); 11868 } 11869 11870 /* 11871 * Re-apply an allocation when a cg write is complete. 11872 */ 11873 static int 11874 jnewblk_rollforward(jnewblk, fs, cgp, blksfree) 11875 struct jnewblk *jnewblk; 11876 struct fs *fs; 11877 struct cg *cgp; 11878 uint8_t *blksfree; 11879 { 11880 ufs1_daddr_t fragno; 11881 ufs2_daddr_t blkno; 11882 long cgbno, bbase; 11883 int frags, blk; 11884 int i; 11885 11886 frags = 0; 11887 cgbno = dtogd(fs, jnewblk->jn_blkno); 11888 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++) { 11889 if (isclr(blksfree, cgbno + i)) 11890 panic("jnewblk_rollforward: re-allocated fragment"); 11891 frags++; 11892 } 11893 if (frags == fs->fs_frag) { 11894 blkno = fragstoblks(fs, cgbno); 11895 ffs_clrblock(fs, blksfree, (long)blkno); 11896 ffs_clusteracct(fs, cgp, blkno, -1); 11897 cgp->cg_cs.cs_nbfree--; 11898 } else { 11899 bbase = cgbno - fragnum(fs, cgbno); 11900 cgbno += jnewblk->jn_oldfrags; 11901 /* If a complete block had been reassembled, account for it. */ 11902 fragno = fragstoblks(fs, bbase); 11903 if (ffs_isblock(fs, blksfree, fragno)) { 11904 cgp->cg_cs.cs_nffree += fs->fs_frag; 11905 ffs_clusteracct(fs, cgp, fragno, -1); 11906 cgp->cg_cs.cs_nbfree--; 11907 } 11908 /* Decrement the old frags. */ 11909 blk = blkmap(fs, blksfree, bbase); 11910 ffs_fragacct(fs, blk, cgp->cg_frsum, -1); 11911 /* Allocate the fragment */ 11912 for (i = 0; i < frags; i++) 11913 clrbit(blksfree, cgbno + i); 11914 cgp->cg_cs.cs_nffree -= frags; 11915 /* Add back in counts associated with the new frags */ 11916 blk = blkmap(fs, blksfree, bbase); 11917 ffs_fragacct(fs, blk, cgp->cg_frsum, 1); 11918 } 11919 return (frags); 11920 } 11921 11922 /* 11923 * Complete a write to a bmsafemap structure. Roll forward any bitmap 11924 * changes if it's not a background write. Set all written dependencies 11925 * to DEPCOMPLETE and free the structure if possible. 11926 * 11927 * If the write did not succeed, we will do all the roll-forward 11928 * operations, but we will not take the actions that will allow its 11929 * dependencies to be processed. 11930 */ 11931 static int 11932 handle_written_bmsafemap(bmsafemap, bp, flags) 11933 struct bmsafemap *bmsafemap; 11934 struct buf *bp; 11935 int flags; 11936 { 11937 struct newblk *newblk; 11938 struct inodedep *inodedep; 11939 struct jaddref *jaddref, *jatmp; 11940 struct jnewblk *jnewblk, *jntmp; 11941 struct ufsmount *ump; 11942 uint8_t *inosused; 11943 uint8_t *blksfree; 11944 struct cg *cgp; 11945 struct fs *fs; 11946 ino_t ino; 11947 int foreground; 11948 int chgs; 11949 11950 if ((bmsafemap->sm_state & IOSTARTED) == 0) 11951 panic("handle_written_bmsafemap: Not started\n"); 11952 ump = VFSTOUFS(bmsafemap->sm_list.wk_mp); 11953 chgs = 0; 11954 bmsafemap->sm_state &= ~IOSTARTED; 11955 foreground = (bp->b_xflags & BX_BKGRDMARKER) == 0; 11956 /* 11957 * If write was successful, release journal work that was waiting 11958 * on the write. Otherwise move the work back. 11959 */ 11960 if (flags & WRITESUCCEEDED) 11961 handle_jwork(&bmsafemap->sm_freewr); 11962 else 11963 LIST_CONCAT(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, 11964 worklist, wk_list); 11965 11966 /* 11967 * Restore unwritten inode allocation pending jaddref writes. 11968 */ 11969 if (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd)) { 11970 cgp = (struct cg *)bp->b_data; 11971 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 11972 inosused = cg_inosused(cgp); 11973 LIST_FOREACH_SAFE(jaddref, &bmsafemap->sm_jaddrefhd, 11974 ja_bmdeps, jatmp) { 11975 if ((jaddref->ja_state & UNDONE) == 0) 11976 continue; 11977 ino = jaddref->ja_ino % fs->fs_ipg; 11978 if (isset(inosused, ino)) 11979 panic("handle_written_bmsafemap: " 11980 "re-allocated inode"); 11981 /* Do the roll-forward only if it's a real copy. */ 11982 if (foreground) { 11983 if ((jaddref->ja_mode & IFMT) == IFDIR) 11984 cgp->cg_cs.cs_ndir++; 11985 cgp->cg_cs.cs_nifree--; 11986 setbit(inosused, ino); 11987 chgs = 1; 11988 } 11989 jaddref->ja_state &= ~UNDONE; 11990 jaddref->ja_state |= ATTACHED; 11991 free_jaddref(jaddref); 11992 } 11993 } 11994 /* 11995 * Restore any block allocations which are pending journal writes. 11996 */ 11997 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 11998 cgp = (struct cg *)bp->b_data; 11999 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 12000 blksfree = cg_blksfree(cgp); 12001 LIST_FOREACH_SAFE(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps, 12002 jntmp) { 12003 if ((jnewblk->jn_state & UNDONE) == 0) 12004 continue; 12005 /* Do the roll-forward only if it's a real copy. */ 12006 if (foreground && 12007 jnewblk_rollforward(jnewblk, fs, cgp, blksfree)) 12008 chgs = 1; 12009 jnewblk->jn_state &= ~(UNDONE | NEWBLOCK); 12010 jnewblk->jn_state |= ATTACHED; 12011 free_jnewblk(jnewblk); 12012 } 12013 } 12014 /* 12015 * If the write did not succeed, we have done all the roll-forward 12016 * operations, but we cannot take the actions that will allow its 12017 * dependencies to be processed. 12018 */ 12019 if ((flags & WRITESUCCEEDED) == 0) { 12020 LIST_CONCAT(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr, 12021 newblk, nb_deps); 12022 LIST_CONCAT(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, 12023 worklist, wk_list); 12024 if (foreground) 12025 bdirty(bp); 12026 return (1); 12027 } 12028 while ((newblk = LIST_FIRST(&bmsafemap->sm_newblkwr))) { 12029 newblk->nb_state |= DEPCOMPLETE; 12030 newblk->nb_state &= ~ONDEPLIST; 12031 newblk->nb_bmsafemap = NULL; 12032 LIST_REMOVE(newblk, nb_deps); 12033 if (newblk->nb_list.wk_type == D_ALLOCDIRECT) 12034 handle_allocdirect_partdone( 12035 WK_ALLOCDIRECT(&newblk->nb_list), NULL); 12036 else if (newblk->nb_list.wk_type == D_ALLOCINDIR) 12037 handle_allocindir_partdone( 12038 WK_ALLOCINDIR(&newblk->nb_list)); 12039 else if (newblk->nb_list.wk_type != D_NEWBLK) 12040 panic("handle_written_bmsafemap: Unexpected type: %s", 12041 TYPENAME(newblk->nb_list.wk_type)); 12042 } 12043 while ((inodedep = LIST_FIRST(&bmsafemap->sm_inodedepwr)) != NULL) { 12044 inodedep->id_state |= DEPCOMPLETE; 12045 inodedep->id_state &= ~ONDEPLIST; 12046 LIST_REMOVE(inodedep, id_deps); 12047 inodedep->id_bmsafemap = NULL; 12048 } 12049 LIST_REMOVE(bmsafemap, sm_next); 12050 if (chgs == 0 && LIST_EMPTY(&bmsafemap->sm_jaddrefhd) && 12051 LIST_EMPTY(&bmsafemap->sm_jnewblkhd) && 12052 LIST_EMPTY(&bmsafemap->sm_newblkhd) && 12053 LIST_EMPTY(&bmsafemap->sm_inodedephd) && 12054 LIST_EMPTY(&bmsafemap->sm_freehd)) { 12055 LIST_REMOVE(bmsafemap, sm_hash); 12056 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 12057 return (0); 12058 } 12059 LIST_INSERT_HEAD(&ump->softdep_dirtycg, bmsafemap, sm_next); 12060 if (foreground) 12061 bdirty(bp); 12062 return (1); 12063 } 12064 12065 /* 12066 * Try to free a mkdir dependency. 12067 */ 12068 static void 12069 complete_mkdir(mkdir) 12070 struct mkdir *mkdir; 12071 { 12072 struct diradd *dap; 12073 12074 if ((mkdir->md_state & ALLCOMPLETE) != ALLCOMPLETE) 12075 return; 12076 LIST_REMOVE(mkdir, md_mkdirs); 12077 dap = mkdir->md_diradd; 12078 dap->da_state &= ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 12079 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) { 12080 dap->da_state |= DEPCOMPLETE; 12081 complete_diradd(dap); 12082 } 12083 WORKITEM_FREE(mkdir, D_MKDIR); 12084 } 12085 12086 /* 12087 * Handle the completion of a mkdir dependency. 12088 */ 12089 static void 12090 handle_written_mkdir(mkdir, type) 12091 struct mkdir *mkdir; 12092 int type; 12093 { 12094 12095 if ((mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)) != type) 12096 panic("handle_written_mkdir: bad type"); 12097 mkdir->md_state |= COMPLETE; 12098 complete_mkdir(mkdir); 12099 } 12100 12101 static int 12102 free_pagedep(pagedep) 12103 struct pagedep *pagedep; 12104 { 12105 int i; 12106 12107 if (pagedep->pd_state & NEWBLOCK) 12108 return (0); 12109 if (!LIST_EMPTY(&pagedep->pd_dirremhd)) 12110 return (0); 12111 for (i = 0; i < DAHASHSZ; i++) 12112 if (!LIST_EMPTY(&pagedep->pd_diraddhd[i])) 12113 return (0); 12114 if (!LIST_EMPTY(&pagedep->pd_pendinghd)) 12115 return (0); 12116 if (!LIST_EMPTY(&pagedep->pd_jmvrefhd)) 12117 return (0); 12118 if (pagedep->pd_state & ONWORKLIST) 12119 WORKLIST_REMOVE(&pagedep->pd_list); 12120 LIST_REMOVE(pagedep, pd_hash); 12121 WORKITEM_FREE(pagedep, D_PAGEDEP); 12122 12123 return (1); 12124 } 12125 12126 /* 12127 * Called from within softdep_disk_write_complete above. 12128 * A write operation was just completed. Removed inodes can 12129 * now be freed and associated block pointers may be committed. 12130 * Note that this routine is always called from interrupt level 12131 * with further interrupts from this device blocked. 12132 * 12133 * If the write did not succeed, we will do all the roll-forward 12134 * operations, but we will not take the actions that will allow its 12135 * dependencies to be processed. 12136 */ 12137 static int 12138 handle_written_filepage(pagedep, bp, flags) 12139 struct pagedep *pagedep; 12140 struct buf *bp; /* buffer containing the written page */ 12141 int flags; 12142 { 12143 struct dirrem *dirrem; 12144 struct diradd *dap, *nextdap; 12145 struct direct *ep; 12146 int i, chgs; 12147 12148 if ((pagedep->pd_state & IOSTARTED) == 0) 12149 panic("handle_written_filepage: not started"); 12150 pagedep->pd_state &= ~IOSTARTED; 12151 if ((flags & WRITESUCCEEDED) == 0) 12152 goto rollforward; 12153 /* 12154 * Process any directory removals that have been committed. 12155 */ 12156 while ((dirrem = LIST_FIRST(&pagedep->pd_dirremhd)) != NULL) { 12157 LIST_REMOVE(dirrem, dm_next); 12158 dirrem->dm_state |= COMPLETE; 12159 dirrem->dm_dirinum = pagedep->pd_ino; 12160 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 12161 ("handle_written_filepage: Journal entries not written.")); 12162 add_to_worklist(&dirrem->dm_list, 0); 12163 } 12164 /* 12165 * Free any directory additions that have been committed. 12166 * If it is a newly allocated block, we have to wait until 12167 * the on-disk directory inode claims the new block. 12168 */ 12169 if ((pagedep->pd_state & NEWBLOCK) == 0) 12170 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 12171 free_diradd(dap, NULL); 12172 rollforward: 12173 /* 12174 * Uncommitted directory entries must be restored. 12175 */ 12176 for (chgs = 0, i = 0; i < DAHASHSZ; i++) { 12177 for (dap = LIST_FIRST(&pagedep->pd_diraddhd[i]); dap; 12178 dap = nextdap) { 12179 nextdap = LIST_NEXT(dap, da_pdlist); 12180 if (dap->da_state & ATTACHED) 12181 panic("handle_written_filepage: attached"); 12182 ep = (struct direct *) 12183 ((char *)bp->b_data + dap->da_offset); 12184 ep->d_ino = dap->da_newinum; 12185 dap->da_state &= ~UNDONE; 12186 dap->da_state |= ATTACHED; 12187 chgs = 1; 12188 /* 12189 * If the inode referenced by the directory has 12190 * been written out, then the dependency can be 12191 * moved to the pending list. 12192 */ 12193 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 12194 LIST_REMOVE(dap, da_pdlist); 12195 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, 12196 da_pdlist); 12197 } 12198 } 12199 } 12200 /* 12201 * If there were any rollbacks in the directory, then it must be 12202 * marked dirty so that its will eventually get written back in 12203 * its correct form. 12204 */ 12205 if (chgs || (flags & WRITESUCCEEDED) == 0) { 12206 if ((bp->b_flags & B_DELWRI) == 0) 12207 stat_dir_entry++; 12208 bdirty(bp); 12209 return (1); 12210 } 12211 /* 12212 * If we are not waiting for a new directory block to be 12213 * claimed by its inode, then the pagedep will be freed. 12214 * Otherwise it will remain to track any new entries on 12215 * the page in case they are fsync'ed. 12216 */ 12217 free_pagedep(pagedep); 12218 return (0); 12219 } 12220 12221 /* 12222 * Writing back in-core inode structures. 12223 * 12224 * The filesystem only accesses an inode's contents when it occupies an 12225 * "in-core" inode structure. These "in-core" structures are separate from 12226 * the page frames used to cache inode blocks. Only the latter are 12227 * transferred to/from the disk. So, when the updated contents of the 12228 * "in-core" inode structure are copied to the corresponding in-memory inode 12229 * block, the dependencies are also transferred. The following procedure is 12230 * called when copying a dirty "in-core" inode to a cached inode block. 12231 */ 12232 12233 /* 12234 * Called when an inode is loaded from disk. If the effective link count 12235 * differed from the actual link count when it was last flushed, then we 12236 * need to ensure that the correct effective link count is put back. 12237 */ 12238 void 12239 softdep_load_inodeblock(ip) 12240 struct inode *ip; /* the "in_core" copy of the inode */ 12241 { 12242 struct inodedep *inodedep; 12243 struct ufsmount *ump; 12244 12245 ump = ITOUMP(ip); 12246 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 12247 ("softdep_load_inodeblock called on non-softdep filesystem")); 12248 /* 12249 * Check for alternate nlink count. 12250 */ 12251 ip->i_effnlink = ip->i_nlink; 12252 ACQUIRE_LOCK(ump); 12253 if (inodedep_lookup(UFSTOVFS(ump), ip->i_number, 0, &inodedep) == 0) { 12254 FREE_LOCK(ump); 12255 return; 12256 } 12257 ip->i_effnlink -= inodedep->id_nlinkdelta; 12258 KASSERT(ip->i_effnlink >= 0, 12259 ("softdep_load_inodeblock: negative i_effnlink")); 12260 FREE_LOCK(ump); 12261 } 12262 12263 /* 12264 * This routine is called just before the "in-core" inode 12265 * information is to be copied to the in-memory inode block. 12266 * Recall that an inode block contains several inodes. If 12267 * the force flag is set, then the dependencies will be 12268 * cleared so that the update can always be made. Note that 12269 * the buffer is locked when this routine is called, so we 12270 * will never be in the middle of writing the inode block 12271 * to disk. 12272 */ 12273 void 12274 softdep_update_inodeblock(ip, bp, waitfor) 12275 struct inode *ip; /* the "in_core" copy of the inode */ 12276 struct buf *bp; /* the buffer containing the inode block */ 12277 int waitfor; /* nonzero => update must be allowed */ 12278 { 12279 struct inodedep *inodedep; 12280 struct inoref *inoref; 12281 struct ufsmount *ump; 12282 struct worklist *wk; 12283 struct mount *mp; 12284 struct buf *ibp; 12285 struct fs *fs; 12286 int error; 12287 12288 ump = ITOUMP(ip); 12289 mp = UFSTOVFS(ump); 12290 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 12291 ("softdep_update_inodeblock called on non-softdep filesystem")); 12292 fs = ump->um_fs; 12293 /* 12294 * Preserve the freelink that is on disk. clear_unlinked_inodedep() 12295 * does not have access to the in-core ip so must write directly into 12296 * the inode block buffer when setting freelink. 12297 */ 12298 if (fs->fs_magic == FS_UFS1_MAGIC) 12299 DIP_SET(ip, i_freelink, ((struct ufs1_dinode *)bp->b_data + 12300 ino_to_fsbo(fs, ip->i_number))->di_freelink); 12301 else 12302 DIP_SET(ip, i_freelink, ((struct ufs2_dinode *)bp->b_data + 12303 ino_to_fsbo(fs, ip->i_number))->di_freelink); 12304 /* 12305 * If the effective link count is not equal to the actual link 12306 * count, then we must track the difference in an inodedep while 12307 * the inode is (potentially) tossed out of the cache. Otherwise, 12308 * if there is no existing inodedep, then there are no dependencies 12309 * to track. 12310 */ 12311 ACQUIRE_LOCK(ump); 12312 again: 12313 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 12314 FREE_LOCK(ump); 12315 if (ip->i_effnlink != ip->i_nlink) 12316 panic("softdep_update_inodeblock: bad link count"); 12317 return; 12318 } 12319 if (inodedep->id_nlinkdelta != ip->i_nlink - ip->i_effnlink) 12320 panic("softdep_update_inodeblock: bad delta"); 12321 /* 12322 * If we're flushing all dependencies we must also move any waiting 12323 * for journal writes onto the bufwait list prior to I/O. 12324 */ 12325 if (waitfor) { 12326 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 12327 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 12328 == DEPCOMPLETE) { 12329 jwait(&inoref->if_list, MNT_WAIT); 12330 goto again; 12331 } 12332 } 12333 } 12334 /* 12335 * Changes have been initiated. Anything depending on these 12336 * changes cannot occur until this inode has been written. 12337 */ 12338 inodedep->id_state &= ~COMPLETE; 12339 if ((inodedep->id_state & ONWORKLIST) == 0) 12340 WORKLIST_INSERT(&bp->b_dep, &inodedep->id_list); 12341 /* 12342 * Any new dependencies associated with the incore inode must 12343 * now be moved to the list associated with the buffer holding 12344 * the in-memory copy of the inode. Once merged process any 12345 * allocdirects that are completed by the merger. 12346 */ 12347 merge_inode_lists(&inodedep->id_newinoupdt, &inodedep->id_inoupdt); 12348 if (!TAILQ_EMPTY(&inodedep->id_inoupdt)) 12349 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_inoupdt), 12350 NULL); 12351 merge_inode_lists(&inodedep->id_newextupdt, &inodedep->id_extupdt); 12352 if (!TAILQ_EMPTY(&inodedep->id_extupdt)) 12353 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_extupdt), 12354 NULL); 12355 /* 12356 * Now that the inode has been pushed into the buffer, the 12357 * operations dependent on the inode being written to disk 12358 * can be moved to the id_bufwait so that they will be 12359 * processed when the buffer I/O completes. 12360 */ 12361 while ((wk = LIST_FIRST(&inodedep->id_inowait)) != NULL) { 12362 WORKLIST_REMOVE(wk); 12363 WORKLIST_INSERT(&inodedep->id_bufwait, wk); 12364 } 12365 /* 12366 * Newly allocated inodes cannot be written until the bitmap 12367 * that allocates them have been written (indicated by 12368 * DEPCOMPLETE being set in id_state). If we are doing a 12369 * forced sync (e.g., an fsync on a file), we force the bitmap 12370 * to be written so that the update can be done. 12371 */ 12372 if (waitfor == 0) { 12373 FREE_LOCK(ump); 12374 return; 12375 } 12376 retry: 12377 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) != 0) { 12378 FREE_LOCK(ump); 12379 return; 12380 } 12381 ibp = inodedep->id_bmsafemap->sm_buf; 12382 ibp = getdirtybuf(ibp, LOCK_PTR(ump), MNT_WAIT); 12383 if (ibp == NULL) { 12384 /* 12385 * If ibp came back as NULL, the dependency could have been 12386 * freed while we slept. Look it up again, and check to see 12387 * that it has completed. 12388 */ 12389 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 12390 goto retry; 12391 FREE_LOCK(ump); 12392 return; 12393 } 12394 FREE_LOCK(ump); 12395 if ((error = bwrite(ibp)) != 0) 12396 softdep_error("softdep_update_inodeblock: bwrite", error); 12397 } 12398 12399 /* 12400 * Merge the a new inode dependency list (such as id_newinoupdt) into an 12401 * old inode dependency list (such as id_inoupdt). 12402 */ 12403 static void 12404 merge_inode_lists(newlisthead, oldlisthead) 12405 struct allocdirectlst *newlisthead; 12406 struct allocdirectlst *oldlisthead; 12407 { 12408 struct allocdirect *listadp, *newadp; 12409 12410 newadp = TAILQ_FIRST(newlisthead); 12411 if (newadp != NULL) 12412 LOCK_OWNED(VFSTOUFS(newadp->ad_block.nb_list.wk_mp)); 12413 for (listadp = TAILQ_FIRST(oldlisthead); listadp && newadp;) { 12414 if (listadp->ad_offset < newadp->ad_offset) { 12415 listadp = TAILQ_NEXT(listadp, ad_next); 12416 continue; 12417 } 12418 TAILQ_REMOVE(newlisthead, newadp, ad_next); 12419 TAILQ_INSERT_BEFORE(listadp, newadp, ad_next); 12420 if (listadp->ad_offset == newadp->ad_offset) { 12421 allocdirect_merge(oldlisthead, newadp, 12422 listadp); 12423 listadp = newadp; 12424 } 12425 newadp = TAILQ_FIRST(newlisthead); 12426 } 12427 while ((newadp = TAILQ_FIRST(newlisthead)) != NULL) { 12428 TAILQ_REMOVE(newlisthead, newadp, ad_next); 12429 TAILQ_INSERT_TAIL(oldlisthead, newadp, ad_next); 12430 } 12431 } 12432 12433 /* 12434 * If we are doing an fsync, then we must ensure that any directory 12435 * entries for the inode have been written after the inode gets to disk. 12436 */ 12437 int 12438 softdep_fsync(vp) 12439 struct vnode *vp; /* the "in_core" copy of the inode */ 12440 { 12441 struct inodedep *inodedep; 12442 struct pagedep *pagedep; 12443 struct inoref *inoref; 12444 struct ufsmount *ump; 12445 struct worklist *wk; 12446 struct diradd *dap; 12447 struct mount *mp; 12448 struct vnode *pvp; 12449 struct inode *ip; 12450 struct buf *bp; 12451 struct fs *fs; 12452 struct thread *td = curthread; 12453 int error, flushparent, pagedep_new_block; 12454 ino_t parentino; 12455 ufs_lbn_t lbn; 12456 12457 ip = VTOI(vp); 12458 mp = vp->v_mount; 12459 ump = VFSTOUFS(mp); 12460 fs = ump->um_fs; 12461 if (MOUNTEDSOFTDEP(mp) == 0) 12462 return (0); 12463 ACQUIRE_LOCK(ump); 12464 restart: 12465 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 12466 FREE_LOCK(ump); 12467 return (0); 12468 } 12469 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 12470 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 12471 == DEPCOMPLETE) { 12472 jwait(&inoref->if_list, MNT_WAIT); 12473 goto restart; 12474 } 12475 } 12476 if (!LIST_EMPTY(&inodedep->id_inowait) || 12477 !TAILQ_EMPTY(&inodedep->id_extupdt) || 12478 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 12479 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 12480 !TAILQ_EMPTY(&inodedep->id_newinoupdt)) 12481 panic("softdep_fsync: pending ops %p", inodedep); 12482 for (error = 0, flushparent = 0; ; ) { 12483 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) == NULL) 12484 break; 12485 if (wk->wk_type != D_DIRADD) 12486 panic("softdep_fsync: Unexpected type %s", 12487 TYPENAME(wk->wk_type)); 12488 dap = WK_DIRADD(wk); 12489 /* 12490 * Flush our parent if this directory entry has a MKDIR_PARENT 12491 * dependency or is contained in a newly allocated block. 12492 */ 12493 if (dap->da_state & DIRCHG) 12494 pagedep = dap->da_previous->dm_pagedep; 12495 else 12496 pagedep = dap->da_pagedep; 12497 parentino = pagedep->pd_ino; 12498 lbn = pagedep->pd_lbn; 12499 if ((dap->da_state & (MKDIR_BODY | COMPLETE)) != COMPLETE) 12500 panic("softdep_fsync: dirty"); 12501 if ((dap->da_state & MKDIR_PARENT) || 12502 (pagedep->pd_state & NEWBLOCK)) 12503 flushparent = 1; 12504 else 12505 flushparent = 0; 12506 /* 12507 * If we are being fsync'ed as part of vgone'ing this vnode, 12508 * then we will not be able to release and recover the 12509 * vnode below, so we just have to give up on writing its 12510 * directory entry out. It will eventually be written, just 12511 * not now, but then the user was not asking to have it 12512 * written, so we are not breaking any promises. 12513 */ 12514 if (VN_IS_DOOMED(vp)) 12515 break; 12516 /* 12517 * We prevent deadlock by always fetching inodes from the 12518 * root, moving down the directory tree. Thus, when fetching 12519 * our parent directory, we first try to get the lock. If 12520 * that fails, we must unlock ourselves before requesting 12521 * the lock on our parent. See the comment in ufs_lookup 12522 * for details on possible races. 12523 */ 12524 FREE_LOCK(ump); 12525 if (ffs_vgetf(mp, parentino, LK_NOWAIT | LK_EXCLUSIVE, &pvp, 12526 FFSV_FORCEINSMQ)) { 12527 /* 12528 * Unmount cannot proceed after unlock because 12529 * caller must have called vn_start_write(). 12530 */ 12531 VOP_UNLOCK(vp, 0); 12532 error = ffs_vgetf(mp, parentino, LK_EXCLUSIVE, 12533 &pvp, FFSV_FORCEINSMQ); 12534 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 12535 if (VN_IS_DOOMED(vp)) { 12536 if (error == 0) 12537 vput(pvp); 12538 error = ENOENT; 12539 } 12540 if (error != 0) 12541 return (error); 12542 } 12543 /* 12544 * All MKDIR_PARENT dependencies and all the NEWBLOCK pagedeps 12545 * that are contained in direct blocks will be resolved by 12546 * doing a ffs_update. Pagedeps contained in indirect blocks 12547 * may require a complete sync'ing of the directory. So, we 12548 * try the cheap and fast ffs_update first, and if that fails, 12549 * then we do the slower ffs_syncvnode of the directory. 12550 */ 12551 if (flushparent) { 12552 int locked; 12553 12554 if ((error = ffs_update(pvp, 1)) != 0) { 12555 vput(pvp); 12556 return (error); 12557 } 12558 ACQUIRE_LOCK(ump); 12559 locked = 1; 12560 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) { 12561 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) != NULL) { 12562 if (wk->wk_type != D_DIRADD) 12563 panic("softdep_fsync: Unexpected type %s", 12564 TYPENAME(wk->wk_type)); 12565 dap = WK_DIRADD(wk); 12566 if (dap->da_state & DIRCHG) 12567 pagedep = dap->da_previous->dm_pagedep; 12568 else 12569 pagedep = dap->da_pagedep; 12570 pagedep_new_block = pagedep->pd_state & NEWBLOCK; 12571 FREE_LOCK(ump); 12572 locked = 0; 12573 if (pagedep_new_block && (error = 12574 ffs_syncvnode(pvp, MNT_WAIT, 0))) { 12575 vput(pvp); 12576 return (error); 12577 } 12578 } 12579 } 12580 if (locked) 12581 FREE_LOCK(ump); 12582 } 12583 /* 12584 * Flush directory page containing the inode's name. 12585 */ 12586 error = bread(pvp, lbn, blksize(fs, VTOI(pvp), lbn), td->td_ucred, 12587 &bp); 12588 if (error == 0) 12589 error = bwrite(bp); 12590 else 12591 brelse(bp); 12592 vput(pvp); 12593 if (error != 0) 12594 return (error); 12595 ACQUIRE_LOCK(ump); 12596 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 12597 break; 12598 } 12599 FREE_LOCK(ump); 12600 return (0); 12601 } 12602 12603 /* 12604 * Flush all the dirty bitmaps associated with the block device 12605 * before flushing the rest of the dirty blocks so as to reduce 12606 * the number of dependencies that will have to be rolled back. 12607 * 12608 * XXX Unused? 12609 */ 12610 void 12611 softdep_fsync_mountdev(vp) 12612 struct vnode *vp; 12613 { 12614 struct buf *bp, *nbp; 12615 struct worklist *wk; 12616 struct bufobj *bo; 12617 12618 if (!vn_isdisk(vp, NULL)) 12619 panic("softdep_fsync_mountdev: vnode not a disk"); 12620 bo = &vp->v_bufobj; 12621 restart: 12622 BO_LOCK(bo); 12623 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 12624 /* 12625 * If it is already scheduled, skip to the next buffer. 12626 */ 12627 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) 12628 continue; 12629 12630 if ((bp->b_flags & B_DELWRI) == 0) 12631 panic("softdep_fsync_mountdev: not dirty"); 12632 /* 12633 * We are only interested in bitmaps with outstanding 12634 * dependencies. 12635 */ 12636 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL || 12637 wk->wk_type != D_BMSAFEMAP || 12638 (bp->b_vflags & BV_BKGRDINPROG)) { 12639 BUF_UNLOCK(bp); 12640 continue; 12641 } 12642 BO_UNLOCK(bo); 12643 bremfree(bp); 12644 (void) bawrite(bp); 12645 goto restart; 12646 } 12647 drain_output(vp); 12648 BO_UNLOCK(bo); 12649 } 12650 12651 /* 12652 * Sync all cylinder groups that were dirty at the time this function is 12653 * called. Newly dirtied cgs will be inserted before the sentinel. This 12654 * is used to flush freedep activity that may be holding up writes to a 12655 * indirect block. 12656 */ 12657 static int 12658 sync_cgs(mp, waitfor) 12659 struct mount *mp; 12660 int waitfor; 12661 { 12662 struct bmsafemap *bmsafemap; 12663 struct bmsafemap *sentinel; 12664 struct ufsmount *ump; 12665 struct buf *bp; 12666 int error; 12667 12668 sentinel = malloc(sizeof(*sentinel), M_BMSAFEMAP, M_ZERO | M_WAITOK); 12669 sentinel->sm_cg = -1; 12670 ump = VFSTOUFS(mp); 12671 error = 0; 12672 ACQUIRE_LOCK(ump); 12673 LIST_INSERT_HEAD(&ump->softdep_dirtycg, sentinel, sm_next); 12674 for (bmsafemap = LIST_NEXT(sentinel, sm_next); bmsafemap != NULL; 12675 bmsafemap = LIST_NEXT(sentinel, sm_next)) { 12676 /* Skip sentinels and cgs with no work to release. */ 12677 if (bmsafemap->sm_cg == -1 || 12678 (LIST_EMPTY(&bmsafemap->sm_freehd) && 12679 LIST_EMPTY(&bmsafemap->sm_freewr))) { 12680 LIST_REMOVE(sentinel, sm_next); 12681 LIST_INSERT_AFTER(bmsafemap, sentinel, sm_next); 12682 continue; 12683 } 12684 /* 12685 * If we don't get the lock and we're waiting try again, if 12686 * not move on to the next buf and try to sync it. 12687 */ 12688 bp = getdirtybuf(bmsafemap->sm_buf, LOCK_PTR(ump), waitfor); 12689 if (bp == NULL && waitfor == MNT_WAIT) 12690 continue; 12691 LIST_REMOVE(sentinel, sm_next); 12692 LIST_INSERT_AFTER(bmsafemap, sentinel, sm_next); 12693 if (bp == NULL) 12694 continue; 12695 FREE_LOCK(ump); 12696 if (waitfor == MNT_NOWAIT) 12697 bawrite(bp); 12698 else 12699 error = bwrite(bp); 12700 ACQUIRE_LOCK(ump); 12701 if (error) 12702 break; 12703 } 12704 LIST_REMOVE(sentinel, sm_next); 12705 FREE_LOCK(ump); 12706 free(sentinel, M_BMSAFEMAP); 12707 return (error); 12708 } 12709 12710 /* 12711 * This routine is called when we are trying to synchronously flush a 12712 * file. This routine must eliminate any filesystem metadata dependencies 12713 * so that the syncing routine can succeed. 12714 */ 12715 int 12716 softdep_sync_metadata(struct vnode *vp) 12717 { 12718 struct inode *ip; 12719 int error; 12720 12721 ip = VTOI(vp); 12722 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 12723 ("softdep_sync_metadata called on non-softdep filesystem")); 12724 /* 12725 * Ensure that any direct block dependencies have been cleared, 12726 * truncations are started, and inode references are journaled. 12727 */ 12728 ACQUIRE_LOCK(VFSTOUFS(vp->v_mount)); 12729 /* 12730 * Write all journal records to prevent rollbacks on devvp. 12731 */ 12732 if (vp->v_type == VCHR) 12733 softdep_flushjournal(vp->v_mount); 12734 error = flush_inodedep_deps(vp, vp->v_mount, ip->i_number); 12735 /* 12736 * Ensure that all truncates are written so we won't find deps on 12737 * indirect blocks. 12738 */ 12739 process_truncates(vp); 12740 FREE_LOCK(VFSTOUFS(vp->v_mount)); 12741 12742 return (error); 12743 } 12744 12745 /* 12746 * This routine is called when we are attempting to sync a buf with 12747 * dependencies. If waitfor is MNT_NOWAIT it attempts to schedule any 12748 * other IO it can but returns EBUSY if the buffer is not yet able to 12749 * be written. Dependencies which will not cause rollbacks will always 12750 * return 0. 12751 */ 12752 int 12753 softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor) 12754 { 12755 struct indirdep *indirdep; 12756 struct pagedep *pagedep; 12757 struct allocindir *aip; 12758 struct newblk *newblk; 12759 struct ufsmount *ump; 12760 struct buf *nbp; 12761 struct worklist *wk; 12762 int i, error; 12763 12764 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 12765 ("softdep_sync_buf called on non-softdep filesystem")); 12766 /* 12767 * For VCHR we just don't want to force flush any dependencies that 12768 * will cause rollbacks. 12769 */ 12770 if (vp->v_type == VCHR) { 12771 if (waitfor == MNT_NOWAIT && softdep_count_dependencies(bp, 0)) 12772 return (EBUSY); 12773 return (0); 12774 } 12775 ump = VFSTOUFS(vp->v_mount); 12776 ACQUIRE_LOCK(ump); 12777 /* 12778 * As we hold the buffer locked, none of its dependencies 12779 * will disappear. 12780 */ 12781 error = 0; 12782 top: 12783 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 12784 switch (wk->wk_type) { 12785 12786 case D_ALLOCDIRECT: 12787 case D_ALLOCINDIR: 12788 newblk = WK_NEWBLK(wk); 12789 if (newblk->nb_jnewblk != NULL) { 12790 if (waitfor == MNT_NOWAIT) { 12791 error = EBUSY; 12792 goto out_unlock; 12793 } 12794 jwait(&newblk->nb_jnewblk->jn_list, waitfor); 12795 goto top; 12796 } 12797 if (newblk->nb_state & DEPCOMPLETE || 12798 waitfor == MNT_NOWAIT) 12799 continue; 12800 nbp = newblk->nb_bmsafemap->sm_buf; 12801 nbp = getdirtybuf(nbp, LOCK_PTR(ump), waitfor); 12802 if (nbp == NULL) 12803 goto top; 12804 FREE_LOCK(ump); 12805 if ((error = bwrite(nbp)) != 0) 12806 goto out; 12807 ACQUIRE_LOCK(ump); 12808 continue; 12809 12810 case D_INDIRDEP: 12811 indirdep = WK_INDIRDEP(wk); 12812 if (waitfor == MNT_NOWAIT) { 12813 if (!TAILQ_EMPTY(&indirdep->ir_trunc) || 12814 !LIST_EMPTY(&indirdep->ir_deplisthd)) { 12815 error = EBUSY; 12816 goto out_unlock; 12817 } 12818 } 12819 if (!TAILQ_EMPTY(&indirdep->ir_trunc)) 12820 panic("softdep_sync_buf: truncation pending."); 12821 restart: 12822 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) { 12823 newblk = (struct newblk *)aip; 12824 if (newblk->nb_jnewblk != NULL) { 12825 jwait(&newblk->nb_jnewblk->jn_list, 12826 waitfor); 12827 goto restart; 12828 } 12829 if (newblk->nb_state & DEPCOMPLETE) 12830 continue; 12831 nbp = newblk->nb_bmsafemap->sm_buf; 12832 nbp = getdirtybuf(nbp, LOCK_PTR(ump), waitfor); 12833 if (nbp == NULL) 12834 goto restart; 12835 FREE_LOCK(ump); 12836 if ((error = bwrite(nbp)) != 0) 12837 goto out; 12838 ACQUIRE_LOCK(ump); 12839 goto restart; 12840 } 12841 continue; 12842 12843 case D_PAGEDEP: 12844 /* 12845 * Only flush directory entries in synchronous passes. 12846 */ 12847 if (waitfor != MNT_WAIT) { 12848 error = EBUSY; 12849 goto out_unlock; 12850 } 12851 /* 12852 * While syncing snapshots, we must allow recursive 12853 * lookups. 12854 */ 12855 BUF_AREC(bp); 12856 /* 12857 * We are trying to sync a directory that may 12858 * have dependencies on both its own metadata 12859 * and/or dependencies on the inodes of any 12860 * recently allocated files. We walk its diradd 12861 * lists pushing out the associated inode. 12862 */ 12863 pagedep = WK_PAGEDEP(wk); 12864 for (i = 0; i < DAHASHSZ; i++) { 12865 if (LIST_FIRST(&pagedep->pd_diraddhd[i]) == 0) 12866 continue; 12867 if ((error = flush_pagedep_deps(vp, wk->wk_mp, 12868 &pagedep->pd_diraddhd[i]))) { 12869 BUF_NOREC(bp); 12870 goto out_unlock; 12871 } 12872 } 12873 BUF_NOREC(bp); 12874 continue; 12875 12876 case D_FREEWORK: 12877 case D_FREEDEP: 12878 case D_JSEGDEP: 12879 case D_JNEWBLK: 12880 continue; 12881 12882 default: 12883 panic("softdep_sync_buf: Unknown type %s", 12884 TYPENAME(wk->wk_type)); 12885 /* NOTREACHED */ 12886 } 12887 } 12888 out_unlock: 12889 FREE_LOCK(ump); 12890 out: 12891 return (error); 12892 } 12893 12894 /* 12895 * Flush the dependencies associated with an inodedep. 12896 */ 12897 static int 12898 flush_inodedep_deps(vp, mp, ino) 12899 struct vnode *vp; 12900 struct mount *mp; 12901 ino_t ino; 12902 { 12903 struct inodedep *inodedep; 12904 struct inoref *inoref; 12905 struct ufsmount *ump; 12906 int error, waitfor; 12907 12908 /* 12909 * This work is done in two passes. The first pass grabs most 12910 * of the buffers and begins asynchronously writing them. The 12911 * only way to wait for these asynchronous writes is to sleep 12912 * on the filesystem vnode which may stay busy for a long time 12913 * if the filesystem is active. So, instead, we make a second 12914 * pass over the dependencies blocking on each write. In the 12915 * usual case we will be blocking against a write that we 12916 * initiated, so when it is done the dependency will have been 12917 * resolved. Thus the second pass is expected to end quickly. 12918 * We give a brief window at the top of the loop to allow 12919 * any pending I/O to complete. 12920 */ 12921 ump = VFSTOUFS(mp); 12922 LOCK_OWNED(ump); 12923 for (error = 0, waitfor = MNT_NOWAIT; ; ) { 12924 if (error) 12925 return (error); 12926 FREE_LOCK(ump); 12927 ACQUIRE_LOCK(ump); 12928 restart: 12929 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 12930 return (0); 12931 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 12932 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 12933 == DEPCOMPLETE) { 12934 jwait(&inoref->if_list, MNT_WAIT); 12935 goto restart; 12936 } 12937 } 12938 if (flush_deplist(&inodedep->id_inoupdt, waitfor, &error) || 12939 flush_deplist(&inodedep->id_newinoupdt, waitfor, &error) || 12940 flush_deplist(&inodedep->id_extupdt, waitfor, &error) || 12941 flush_deplist(&inodedep->id_newextupdt, waitfor, &error)) 12942 continue; 12943 /* 12944 * If pass2, we are done, otherwise do pass 2. 12945 */ 12946 if (waitfor == MNT_WAIT) 12947 break; 12948 waitfor = MNT_WAIT; 12949 } 12950 /* 12951 * Try freeing inodedep in case all dependencies have been removed. 12952 */ 12953 if (inodedep_lookup(mp, ino, 0, &inodedep) != 0) 12954 (void) free_inodedep(inodedep); 12955 return (0); 12956 } 12957 12958 /* 12959 * Flush an inode dependency list. 12960 */ 12961 static int 12962 flush_deplist(listhead, waitfor, errorp) 12963 struct allocdirectlst *listhead; 12964 int waitfor; 12965 int *errorp; 12966 { 12967 struct allocdirect *adp; 12968 struct newblk *newblk; 12969 struct ufsmount *ump; 12970 struct buf *bp; 12971 12972 if ((adp = TAILQ_FIRST(listhead)) == NULL) 12973 return (0); 12974 ump = VFSTOUFS(adp->ad_list.wk_mp); 12975 LOCK_OWNED(ump); 12976 TAILQ_FOREACH(adp, listhead, ad_next) { 12977 newblk = (struct newblk *)adp; 12978 if (newblk->nb_jnewblk != NULL) { 12979 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT); 12980 return (1); 12981 } 12982 if (newblk->nb_state & DEPCOMPLETE) 12983 continue; 12984 bp = newblk->nb_bmsafemap->sm_buf; 12985 bp = getdirtybuf(bp, LOCK_PTR(ump), waitfor); 12986 if (bp == NULL) { 12987 if (waitfor == MNT_NOWAIT) 12988 continue; 12989 return (1); 12990 } 12991 FREE_LOCK(ump); 12992 if (waitfor == MNT_NOWAIT) 12993 bawrite(bp); 12994 else 12995 *errorp = bwrite(bp); 12996 ACQUIRE_LOCK(ump); 12997 return (1); 12998 } 12999 return (0); 13000 } 13001 13002 /* 13003 * Flush dependencies associated with an allocdirect block. 13004 */ 13005 static int 13006 flush_newblk_dep(vp, mp, lbn) 13007 struct vnode *vp; 13008 struct mount *mp; 13009 ufs_lbn_t lbn; 13010 { 13011 struct newblk *newblk; 13012 struct ufsmount *ump; 13013 struct bufobj *bo; 13014 struct inode *ip; 13015 struct buf *bp; 13016 ufs2_daddr_t blkno; 13017 int error; 13018 13019 error = 0; 13020 bo = &vp->v_bufobj; 13021 ip = VTOI(vp); 13022 blkno = DIP(ip, i_db[lbn]); 13023 if (blkno == 0) 13024 panic("flush_newblk_dep: Missing block"); 13025 ump = VFSTOUFS(mp); 13026 ACQUIRE_LOCK(ump); 13027 /* 13028 * Loop until all dependencies related to this block are satisfied. 13029 * We must be careful to restart after each sleep in case a write 13030 * completes some part of this process for us. 13031 */ 13032 for (;;) { 13033 if (newblk_lookup(mp, blkno, 0, &newblk) == 0) { 13034 FREE_LOCK(ump); 13035 break; 13036 } 13037 if (newblk->nb_list.wk_type != D_ALLOCDIRECT) 13038 panic("flush_newblk_dep: Bad newblk %p", newblk); 13039 /* 13040 * Flush the journal. 13041 */ 13042 if (newblk->nb_jnewblk != NULL) { 13043 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT); 13044 continue; 13045 } 13046 /* 13047 * Write the bitmap dependency. 13048 */ 13049 if ((newblk->nb_state & DEPCOMPLETE) == 0) { 13050 bp = newblk->nb_bmsafemap->sm_buf; 13051 bp = getdirtybuf(bp, LOCK_PTR(ump), MNT_WAIT); 13052 if (bp == NULL) 13053 continue; 13054 FREE_LOCK(ump); 13055 error = bwrite(bp); 13056 if (error) 13057 break; 13058 ACQUIRE_LOCK(ump); 13059 continue; 13060 } 13061 /* 13062 * Write the buffer. 13063 */ 13064 FREE_LOCK(ump); 13065 BO_LOCK(bo); 13066 bp = gbincore(bo, lbn); 13067 if (bp != NULL) { 13068 error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 13069 LK_INTERLOCK, BO_LOCKPTR(bo)); 13070 if (error == ENOLCK) { 13071 ACQUIRE_LOCK(ump); 13072 error = 0; 13073 continue; /* Slept, retry */ 13074 } 13075 if (error != 0) 13076 break; /* Failed */ 13077 if (bp->b_flags & B_DELWRI) { 13078 bremfree(bp); 13079 error = bwrite(bp); 13080 if (error) 13081 break; 13082 } else 13083 BUF_UNLOCK(bp); 13084 } else 13085 BO_UNLOCK(bo); 13086 /* 13087 * We have to wait for the direct pointers to 13088 * point at the newdirblk before the dependency 13089 * will go away. 13090 */ 13091 error = ffs_update(vp, 1); 13092 if (error) 13093 break; 13094 ACQUIRE_LOCK(ump); 13095 } 13096 return (error); 13097 } 13098 13099 /* 13100 * Eliminate a pagedep dependency by flushing out all its diradd dependencies. 13101 */ 13102 static int 13103 flush_pagedep_deps(pvp, mp, diraddhdp) 13104 struct vnode *pvp; 13105 struct mount *mp; 13106 struct diraddhd *diraddhdp; 13107 { 13108 struct inodedep *inodedep; 13109 struct inoref *inoref; 13110 struct ufsmount *ump; 13111 struct diradd *dap; 13112 struct vnode *vp; 13113 int error = 0; 13114 struct buf *bp; 13115 ino_t inum; 13116 struct diraddhd unfinished; 13117 13118 LIST_INIT(&unfinished); 13119 ump = VFSTOUFS(mp); 13120 LOCK_OWNED(ump); 13121 restart: 13122 while ((dap = LIST_FIRST(diraddhdp)) != NULL) { 13123 /* 13124 * Flush ourselves if this directory entry 13125 * has a MKDIR_PARENT dependency. 13126 */ 13127 if (dap->da_state & MKDIR_PARENT) { 13128 FREE_LOCK(ump); 13129 if ((error = ffs_update(pvp, 1)) != 0) 13130 break; 13131 ACQUIRE_LOCK(ump); 13132 /* 13133 * If that cleared dependencies, go on to next. 13134 */ 13135 if (dap != LIST_FIRST(diraddhdp)) 13136 continue; 13137 /* 13138 * All MKDIR_PARENT dependencies and all the 13139 * NEWBLOCK pagedeps that are contained in direct 13140 * blocks were resolved by doing above ffs_update. 13141 * Pagedeps contained in indirect blocks may 13142 * require a complete sync'ing of the directory. 13143 * We are in the midst of doing a complete sync, 13144 * so if they are not resolved in this pass we 13145 * defer them for now as they will be sync'ed by 13146 * our caller shortly. 13147 */ 13148 LIST_REMOVE(dap, da_pdlist); 13149 LIST_INSERT_HEAD(&unfinished, dap, da_pdlist); 13150 continue; 13151 } 13152 /* 13153 * A newly allocated directory must have its "." and 13154 * ".." entries written out before its name can be 13155 * committed in its parent. 13156 */ 13157 inum = dap->da_newinum; 13158 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 13159 panic("flush_pagedep_deps: lost inode1"); 13160 /* 13161 * Wait for any pending journal adds to complete so we don't 13162 * cause rollbacks while syncing. 13163 */ 13164 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 13165 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 13166 == DEPCOMPLETE) { 13167 jwait(&inoref->if_list, MNT_WAIT); 13168 goto restart; 13169 } 13170 } 13171 if (dap->da_state & MKDIR_BODY) { 13172 FREE_LOCK(ump); 13173 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 13174 FFSV_FORCEINSMQ))) 13175 break; 13176 error = flush_newblk_dep(vp, mp, 0); 13177 /* 13178 * If we still have the dependency we might need to 13179 * update the vnode to sync the new link count to 13180 * disk. 13181 */ 13182 if (error == 0 && dap == LIST_FIRST(diraddhdp)) 13183 error = ffs_update(vp, 1); 13184 vput(vp); 13185 if (error != 0) 13186 break; 13187 ACQUIRE_LOCK(ump); 13188 /* 13189 * If that cleared dependencies, go on to next. 13190 */ 13191 if (dap != LIST_FIRST(diraddhdp)) 13192 continue; 13193 if (dap->da_state & MKDIR_BODY) { 13194 inodedep_lookup(UFSTOVFS(ump), inum, 0, 13195 &inodedep); 13196 panic("flush_pagedep_deps: MKDIR_BODY " 13197 "inodedep %p dap %p vp %p", 13198 inodedep, dap, vp); 13199 } 13200 } 13201 /* 13202 * Flush the inode on which the directory entry depends. 13203 * Having accounted for MKDIR_PARENT and MKDIR_BODY above, 13204 * the only remaining dependency is that the updated inode 13205 * count must get pushed to disk. The inode has already 13206 * been pushed into its inode buffer (via VOP_UPDATE) at 13207 * the time of the reference count change. So we need only 13208 * locate that buffer, ensure that there will be no rollback 13209 * caused by a bitmap dependency, then write the inode buffer. 13210 */ 13211 retry: 13212 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 13213 panic("flush_pagedep_deps: lost inode"); 13214 /* 13215 * If the inode still has bitmap dependencies, 13216 * push them to disk. 13217 */ 13218 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) == 0) { 13219 bp = inodedep->id_bmsafemap->sm_buf; 13220 bp = getdirtybuf(bp, LOCK_PTR(ump), MNT_WAIT); 13221 if (bp == NULL) 13222 goto retry; 13223 FREE_LOCK(ump); 13224 if ((error = bwrite(bp)) != 0) 13225 break; 13226 ACQUIRE_LOCK(ump); 13227 if (dap != LIST_FIRST(diraddhdp)) 13228 continue; 13229 } 13230 /* 13231 * If the inode is still sitting in a buffer waiting 13232 * to be written or waiting for the link count to be 13233 * adjusted update it here to flush it to disk. 13234 */ 13235 if (dap == LIST_FIRST(diraddhdp)) { 13236 FREE_LOCK(ump); 13237 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 13238 FFSV_FORCEINSMQ))) 13239 break; 13240 error = ffs_update(vp, 1); 13241 vput(vp); 13242 if (error) 13243 break; 13244 ACQUIRE_LOCK(ump); 13245 } 13246 /* 13247 * If we have failed to get rid of all the dependencies 13248 * then something is seriously wrong. 13249 */ 13250 if (dap == LIST_FIRST(diraddhdp)) { 13251 inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep); 13252 panic("flush_pagedep_deps: failed to flush " 13253 "inodedep %p ino %ju dap %p", 13254 inodedep, (uintmax_t)inum, dap); 13255 } 13256 } 13257 if (error) 13258 ACQUIRE_LOCK(ump); 13259 while ((dap = LIST_FIRST(&unfinished)) != NULL) { 13260 LIST_REMOVE(dap, da_pdlist); 13261 LIST_INSERT_HEAD(diraddhdp, dap, da_pdlist); 13262 } 13263 return (error); 13264 } 13265 13266 /* 13267 * A large burst of file addition or deletion activity can drive the 13268 * memory load excessively high. First attempt to slow things down 13269 * using the techniques below. If that fails, this routine requests 13270 * the offending operations to fall back to running synchronously 13271 * until the memory load returns to a reasonable level. 13272 */ 13273 int 13274 softdep_slowdown(vp) 13275 struct vnode *vp; 13276 { 13277 struct ufsmount *ump; 13278 int jlow; 13279 int max_softdeps_hard; 13280 13281 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 13282 ("softdep_slowdown called on non-softdep filesystem")); 13283 ump = VFSTOUFS(vp->v_mount); 13284 ACQUIRE_LOCK(ump); 13285 jlow = 0; 13286 /* 13287 * Check for journal space if needed. 13288 */ 13289 if (DOINGSUJ(vp)) { 13290 if (journal_space(ump, 0) == 0) 13291 jlow = 1; 13292 } 13293 /* 13294 * If the system is under its limits and our filesystem is 13295 * not responsible for more than our share of the usage and 13296 * we are not low on journal space, then no need to slow down. 13297 */ 13298 max_softdeps_hard = max_softdeps * 11 / 10; 13299 if (dep_current[D_DIRREM] < max_softdeps_hard / 2 && 13300 dep_current[D_INODEDEP] < max_softdeps_hard && 13301 dep_current[D_INDIRDEP] < max_softdeps_hard / 1000 && 13302 dep_current[D_FREEBLKS] < max_softdeps_hard && jlow == 0 && 13303 ump->softdep_curdeps[D_DIRREM] < 13304 (max_softdeps_hard / 2) / stat_flush_threads && 13305 ump->softdep_curdeps[D_INODEDEP] < 13306 max_softdeps_hard / stat_flush_threads && 13307 ump->softdep_curdeps[D_INDIRDEP] < 13308 (max_softdeps_hard / 1000) / stat_flush_threads && 13309 ump->softdep_curdeps[D_FREEBLKS] < 13310 max_softdeps_hard / stat_flush_threads) { 13311 FREE_LOCK(ump); 13312 return (0); 13313 } 13314 /* 13315 * If the journal is low or our filesystem is over its limit 13316 * then speedup the cleanup. 13317 */ 13318 if (ump->softdep_curdeps[D_INDIRDEP] < 13319 (max_softdeps_hard / 1000) / stat_flush_threads || jlow) 13320 softdep_speedup(ump); 13321 stat_sync_limit_hit += 1; 13322 FREE_LOCK(ump); 13323 /* 13324 * We only slow down the rate at which new dependencies are 13325 * generated if we are not using journaling. With journaling, 13326 * the cleanup should always be sufficient to keep things 13327 * under control. 13328 */ 13329 if (DOINGSUJ(vp)) 13330 return (0); 13331 return (1); 13332 } 13333 13334 /* 13335 * Called by the allocation routines when they are about to fail 13336 * in the hope that we can free up the requested resource (inodes 13337 * or disk space). 13338 * 13339 * First check to see if the work list has anything on it. If it has, 13340 * clean up entries until we successfully free the requested resource. 13341 * Because this process holds inodes locked, we cannot handle any remove 13342 * requests that might block on a locked inode as that could lead to 13343 * deadlock. If the worklist yields none of the requested resource, 13344 * start syncing out vnodes to free up the needed space. 13345 */ 13346 int 13347 softdep_request_cleanup(fs, vp, cred, resource) 13348 struct fs *fs; 13349 struct vnode *vp; 13350 struct ucred *cred; 13351 int resource; 13352 { 13353 struct ufsmount *ump; 13354 struct mount *mp; 13355 long starttime; 13356 ufs2_daddr_t needed; 13357 int error, failed_vnode; 13358 13359 /* 13360 * If we are being called because of a process doing a 13361 * copy-on-write, then it is not safe to process any 13362 * worklist items as we will recurse into the copyonwrite 13363 * routine. This will result in an incoherent snapshot. 13364 * If the vnode that we hold is a snapshot, we must avoid 13365 * handling other resources that could cause deadlock. 13366 */ 13367 if ((curthread->td_pflags & TDP_COWINPROGRESS) || IS_SNAPSHOT(VTOI(vp))) 13368 return (0); 13369 13370 if (resource == FLUSH_BLOCKS_WAIT) 13371 stat_cleanup_blkrequests += 1; 13372 else 13373 stat_cleanup_inorequests += 1; 13374 13375 mp = vp->v_mount; 13376 ump = VFSTOUFS(mp); 13377 mtx_assert(UFS_MTX(ump), MA_OWNED); 13378 UFS_UNLOCK(ump); 13379 error = ffs_update(vp, 1); 13380 if (error != 0 || MOUNTEDSOFTDEP(mp) == 0) { 13381 UFS_LOCK(ump); 13382 return (0); 13383 } 13384 /* 13385 * If we are in need of resources, start by cleaning up 13386 * any block removals associated with our inode. 13387 */ 13388 ACQUIRE_LOCK(ump); 13389 process_removes(vp); 13390 process_truncates(vp); 13391 FREE_LOCK(ump); 13392 /* 13393 * Now clean up at least as many resources as we will need. 13394 * 13395 * When requested to clean up inodes, the number that are needed 13396 * is set by the number of simultaneous writers (mnt_writeopcount) 13397 * plus a bit of slop (2) in case some more writers show up while 13398 * we are cleaning. 13399 * 13400 * When requested to free up space, the amount of space that 13401 * we need is enough blocks to allocate a full-sized segment 13402 * (fs_contigsumsize). The number of such segments that will 13403 * be needed is set by the number of simultaneous writers 13404 * (mnt_writeopcount) plus a bit of slop (2) in case some more 13405 * writers show up while we are cleaning. 13406 * 13407 * Additionally, if we are unpriviledged and allocating space, 13408 * we need to ensure that we clean up enough blocks to get the 13409 * needed number of blocks over the threshold of the minimum 13410 * number of blocks required to be kept free by the filesystem 13411 * (fs_minfree). 13412 */ 13413 if (resource == FLUSH_INODES_WAIT) { 13414 needed = vfs_mount_fetch_counter(vp->v_mount, 13415 MNT_COUNT_WRITEOPCOUNT) + 2; 13416 } else if (resource == FLUSH_BLOCKS_WAIT) { 13417 needed = (vfs_mount_fetch_counter(vp->v_mount, 13418 MNT_COUNT_WRITEOPCOUNT) + 2) * fs->fs_contigsumsize; 13419 if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE)) 13420 needed += fragstoblks(fs, 13421 roundup((fs->fs_dsize * fs->fs_minfree / 100) - 13422 fs->fs_cstotal.cs_nffree, fs->fs_frag)); 13423 } else { 13424 printf("softdep_request_cleanup: Unknown resource type %d\n", 13425 resource); 13426 UFS_LOCK(ump); 13427 return (0); 13428 } 13429 starttime = time_second; 13430 retry: 13431 if ((resource == FLUSH_BLOCKS_WAIT && ump->softdep_on_worklist > 0 && 13432 fs->fs_cstotal.cs_nbfree <= needed) || 13433 (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 && 13434 fs->fs_cstotal.cs_nifree <= needed)) { 13435 ACQUIRE_LOCK(ump); 13436 if (ump->softdep_on_worklist > 0 && 13437 process_worklist_item(UFSTOVFS(ump), 13438 ump->softdep_on_worklist, LK_NOWAIT) != 0) 13439 stat_worklist_push += 1; 13440 FREE_LOCK(ump); 13441 } 13442 /* 13443 * If we still need resources and there are no more worklist 13444 * entries to process to obtain them, we have to start flushing 13445 * the dirty vnodes to force the release of additional requests 13446 * to the worklist that we can then process to reap addition 13447 * resources. We walk the vnodes associated with the mount point 13448 * until we get the needed worklist requests that we can reap. 13449 * 13450 * If there are several threads all needing to clean the same 13451 * mount point, only one is allowed to walk the mount list. 13452 * When several threads all try to walk the same mount list, 13453 * they end up competing with each other and often end up in 13454 * livelock. This approach ensures that forward progress is 13455 * made at the cost of occational ENOSPC errors being returned 13456 * that might otherwise have been avoided. 13457 */ 13458 error = 1; 13459 if ((resource == FLUSH_BLOCKS_WAIT && 13460 fs->fs_cstotal.cs_nbfree <= needed) || 13461 (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 && 13462 fs->fs_cstotal.cs_nifree <= needed)) { 13463 ACQUIRE_LOCK(ump); 13464 if ((ump->um_softdep->sd_flags & FLUSH_RC_ACTIVE) == 0) { 13465 ump->um_softdep->sd_flags |= FLUSH_RC_ACTIVE; 13466 FREE_LOCK(ump); 13467 failed_vnode = softdep_request_cleanup_flush(mp, ump); 13468 ACQUIRE_LOCK(ump); 13469 ump->um_softdep->sd_flags &= ~FLUSH_RC_ACTIVE; 13470 FREE_LOCK(ump); 13471 if (ump->softdep_on_worklist > 0) { 13472 stat_cleanup_retries += 1; 13473 if (!failed_vnode) 13474 goto retry; 13475 } 13476 } else { 13477 FREE_LOCK(ump); 13478 error = 0; 13479 } 13480 stat_cleanup_failures += 1; 13481 } 13482 if (time_second - starttime > stat_cleanup_high_delay) 13483 stat_cleanup_high_delay = time_second - starttime; 13484 UFS_LOCK(ump); 13485 return (error); 13486 } 13487 13488 /* 13489 * Scan the vnodes for the specified mount point flushing out any 13490 * vnodes that can be locked without waiting. Finally, try to flush 13491 * the device associated with the mount point if it can be locked 13492 * without waiting. 13493 * 13494 * We return 0 if we were able to lock every vnode in our scan. 13495 * If we had to skip one or more vnodes, we return 1. 13496 */ 13497 static int 13498 softdep_request_cleanup_flush(mp, ump) 13499 struct mount *mp; 13500 struct ufsmount *ump; 13501 { 13502 struct thread *td; 13503 struct vnode *lvp, *mvp; 13504 int failed_vnode; 13505 13506 failed_vnode = 0; 13507 td = curthread; 13508 MNT_VNODE_FOREACH_ALL(lvp, mp, mvp) { 13509 if (TAILQ_FIRST(&lvp->v_bufobj.bo_dirty.bv_hd) == 0) { 13510 VI_UNLOCK(lvp); 13511 continue; 13512 } 13513 if (vget(lvp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT, 13514 td) != 0) { 13515 failed_vnode = 1; 13516 continue; 13517 } 13518 if (lvp->v_vflag & VV_NOSYNC) { /* unlinked */ 13519 vput(lvp); 13520 continue; 13521 } 13522 (void) ffs_syncvnode(lvp, MNT_NOWAIT, 0); 13523 vput(lvp); 13524 } 13525 lvp = ump->um_devvp; 13526 if (vn_lock(lvp, LK_EXCLUSIVE | LK_NOWAIT) == 0) { 13527 VOP_FSYNC(lvp, MNT_NOWAIT, td); 13528 VOP_UNLOCK(lvp, 0); 13529 } 13530 return (failed_vnode); 13531 } 13532 13533 static bool 13534 softdep_excess_items(struct ufsmount *ump, int item) 13535 { 13536 13537 KASSERT(item >= 0 && item < D_LAST, ("item %d", item)); 13538 return (dep_current[item] > max_softdeps && 13539 ump->softdep_curdeps[item] > max_softdeps / 13540 stat_flush_threads); 13541 } 13542 13543 static void 13544 schedule_cleanup(struct mount *mp) 13545 { 13546 struct ufsmount *ump; 13547 struct thread *td; 13548 13549 ump = VFSTOUFS(mp); 13550 LOCK_OWNED(ump); 13551 FREE_LOCK(ump); 13552 td = curthread; 13553 if ((td->td_pflags & TDP_KTHREAD) != 0 && 13554 (td->td_proc->p_flag2 & P2_AST_SU) == 0) { 13555 /* 13556 * No ast is delivered to kernel threads, so nobody 13557 * would deref the mp. Some kernel threads 13558 * explicitely check for AST, e.g. NFS daemon does 13559 * this in the serving loop. 13560 */ 13561 return; 13562 } 13563 if (td->td_su != NULL) 13564 vfs_rel(td->td_su); 13565 vfs_ref(mp); 13566 td->td_su = mp; 13567 thread_lock(td); 13568 td->td_flags |= TDF_ASTPENDING; 13569 thread_unlock(td); 13570 } 13571 13572 static void 13573 softdep_ast_cleanup_proc(struct thread *td) 13574 { 13575 struct mount *mp; 13576 struct ufsmount *ump; 13577 int error; 13578 bool req; 13579 13580 while ((mp = td->td_su) != NULL) { 13581 td->td_su = NULL; 13582 error = vfs_busy(mp, MBF_NOWAIT); 13583 vfs_rel(mp); 13584 if (error != 0) 13585 return; 13586 if (ffs_own_mount(mp) && MOUNTEDSOFTDEP(mp)) { 13587 ump = VFSTOUFS(mp); 13588 for (;;) { 13589 req = false; 13590 ACQUIRE_LOCK(ump); 13591 if (softdep_excess_items(ump, D_INODEDEP)) { 13592 req = true; 13593 request_cleanup(mp, FLUSH_INODES); 13594 } 13595 if (softdep_excess_items(ump, D_DIRREM)) { 13596 req = true; 13597 request_cleanup(mp, FLUSH_BLOCKS); 13598 } 13599 FREE_LOCK(ump); 13600 if (softdep_excess_items(ump, D_NEWBLK) || 13601 softdep_excess_items(ump, D_ALLOCDIRECT) || 13602 softdep_excess_items(ump, D_ALLOCINDIR)) { 13603 error = vn_start_write(NULL, &mp, 13604 V_WAIT); 13605 if (error == 0) { 13606 req = true; 13607 VFS_SYNC(mp, MNT_WAIT); 13608 vn_finished_write(mp); 13609 } 13610 } 13611 if ((td->td_pflags & TDP_KTHREAD) != 0 || !req) 13612 break; 13613 } 13614 } 13615 vfs_unbusy(mp); 13616 } 13617 if ((mp = td->td_su) != NULL) { 13618 td->td_su = NULL; 13619 vfs_rel(mp); 13620 } 13621 } 13622 13623 /* 13624 * If memory utilization has gotten too high, deliberately slow things 13625 * down and speed up the I/O processing. 13626 */ 13627 static int 13628 request_cleanup(mp, resource) 13629 struct mount *mp; 13630 int resource; 13631 { 13632 struct thread *td = curthread; 13633 struct ufsmount *ump; 13634 13635 ump = VFSTOUFS(mp); 13636 LOCK_OWNED(ump); 13637 /* 13638 * We never hold up the filesystem syncer or buf daemon. 13639 */ 13640 if (td->td_pflags & (TDP_SOFTDEP|TDP_NORUNNINGBUF)) 13641 return (0); 13642 /* 13643 * First check to see if the work list has gotten backlogged. 13644 * If it has, co-opt this process to help clean up two entries. 13645 * Because this process may hold inodes locked, we cannot 13646 * handle any remove requests that might block on a locked 13647 * inode as that could lead to deadlock. We set TDP_SOFTDEP 13648 * to avoid recursively processing the worklist. 13649 */ 13650 if (ump->softdep_on_worklist > max_softdeps / 10) { 13651 td->td_pflags |= TDP_SOFTDEP; 13652 process_worklist_item(mp, 2, LK_NOWAIT); 13653 td->td_pflags &= ~TDP_SOFTDEP; 13654 stat_worklist_push += 2; 13655 return(1); 13656 } 13657 /* 13658 * Next, we attempt to speed up the syncer process. If that 13659 * is successful, then we allow the process to continue. 13660 */ 13661 if (softdep_speedup(ump) && 13662 resource != FLUSH_BLOCKS_WAIT && 13663 resource != FLUSH_INODES_WAIT) 13664 return(0); 13665 /* 13666 * If we are resource constrained on inode dependencies, try 13667 * flushing some dirty inodes. Otherwise, we are constrained 13668 * by file deletions, so try accelerating flushes of directories 13669 * with removal dependencies. We would like to do the cleanup 13670 * here, but we probably hold an inode locked at this point and 13671 * that might deadlock against one that we try to clean. So, 13672 * the best that we can do is request the syncer daemon to do 13673 * the cleanup for us. 13674 */ 13675 switch (resource) { 13676 13677 case FLUSH_INODES: 13678 case FLUSH_INODES_WAIT: 13679 ACQUIRE_GBLLOCK(&lk); 13680 stat_ino_limit_push += 1; 13681 req_clear_inodedeps += 1; 13682 FREE_GBLLOCK(&lk); 13683 stat_countp = &stat_ino_limit_hit; 13684 break; 13685 13686 case FLUSH_BLOCKS: 13687 case FLUSH_BLOCKS_WAIT: 13688 ACQUIRE_GBLLOCK(&lk); 13689 stat_blk_limit_push += 1; 13690 req_clear_remove += 1; 13691 FREE_GBLLOCK(&lk); 13692 stat_countp = &stat_blk_limit_hit; 13693 break; 13694 13695 default: 13696 panic("request_cleanup: unknown type"); 13697 } 13698 /* 13699 * Hopefully the syncer daemon will catch up and awaken us. 13700 * We wait at most tickdelay before proceeding in any case. 13701 */ 13702 ACQUIRE_GBLLOCK(&lk); 13703 FREE_LOCK(ump); 13704 proc_waiting += 1; 13705 if (callout_pending(&softdep_callout) == FALSE) 13706 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2, 13707 pause_timer, 0); 13708 13709 if ((td->td_pflags & TDP_KTHREAD) == 0) 13710 msleep((caddr_t)&proc_waiting, &lk, PPAUSE, "softupdate", 0); 13711 proc_waiting -= 1; 13712 FREE_GBLLOCK(&lk); 13713 ACQUIRE_LOCK(ump); 13714 return (1); 13715 } 13716 13717 /* 13718 * Awaken processes pausing in request_cleanup and clear proc_waiting 13719 * to indicate that there is no longer a timer running. Pause_timer 13720 * will be called with the global softdep mutex (&lk) locked. 13721 */ 13722 static void 13723 pause_timer(arg) 13724 void *arg; 13725 { 13726 13727 GBLLOCK_OWNED(&lk); 13728 /* 13729 * The callout_ API has acquired mtx and will hold it around this 13730 * function call. 13731 */ 13732 *stat_countp += proc_waiting; 13733 wakeup(&proc_waiting); 13734 } 13735 13736 /* 13737 * If requested, try removing inode or removal dependencies. 13738 */ 13739 static void 13740 check_clear_deps(mp) 13741 struct mount *mp; 13742 { 13743 13744 /* 13745 * If we are suspended, it may be because of our using 13746 * too many inodedeps, so help clear them out. 13747 */ 13748 if (MOUNTEDSUJ(mp) && VFSTOUFS(mp)->softdep_jblocks->jb_suspended) 13749 clear_inodedeps(mp); 13750 /* 13751 * General requests for cleanup of backed up dependencies 13752 */ 13753 ACQUIRE_GBLLOCK(&lk); 13754 if (req_clear_inodedeps) { 13755 req_clear_inodedeps -= 1; 13756 FREE_GBLLOCK(&lk); 13757 clear_inodedeps(mp); 13758 ACQUIRE_GBLLOCK(&lk); 13759 wakeup(&proc_waiting); 13760 } 13761 if (req_clear_remove) { 13762 req_clear_remove -= 1; 13763 FREE_GBLLOCK(&lk); 13764 clear_remove(mp); 13765 ACQUIRE_GBLLOCK(&lk); 13766 wakeup(&proc_waiting); 13767 } 13768 FREE_GBLLOCK(&lk); 13769 } 13770 13771 /* 13772 * Flush out a directory with at least one removal dependency in an effort to 13773 * reduce the number of dirrem, freefile, and freeblks dependency structures. 13774 */ 13775 static void 13776 clear_remove(mp) 13777 struct mount *mp; 13778 { 13779 struct pagedep_hashhead *pagedephd; 13780 struct pagedep *pagedep; 13781 struct ufsmount *ump; 13782 struct vnode *vp; 13783 struct bufobj *bo; 13784 int error, cnt; 13785 ino_t ino; 13786 13787 ump = VFSTOUFS(mp); 13788 LOCK_OWNED(ump); 13789 13790 for (cnt = 0; cnt <= ump->pagedep_hash_size; cnt++) { 13791 pagedephd = &ump->pagedep_hashtbl[ump->pagedep_nextclean++]; 13792 if (ump->pagedep_nextclean > ump->pagedep_hash_size) 13793 ump->pagedep_nextclean = 0; 13794 LIST_FOREACH(pagedep, pagedephd, pd_hash) { 13795 if (LIST_EMPTY(&pagedep->pd_dirremhd)) 13796 continue; 13797 ino = pagedep->pd_ino; 13798 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 13799 continue; 13800 FREE_LOCK(ump); 13801 13802 /* 13803 * Let unmount clear deps 13804 */ 13805 error = vfs_busy(mp, MBF_NOWAIT); 13806 if (error != 0) 13807 goto finish_write; 13808 error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 13809 FFSV_FORCEINSMQ); 13810 vfs_unbusy(mp); 13811 if (error != 0) { 13812 softdep_error("clear_remove: vget", error); 13813 goto finish_write; 13814 } 13815 if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0))) 13816 softdep_error("clear_remove: fsync", error); 13817 bo = &vp->v_bufobj; 13818 BO_LOCK(bo); 13819 drain_output(vp); 13820 BO_UNLOCK(bo); 13821 vput(vp); 13822 finish_write: 13823 vn_finished_write(mp); 13824 ACQUIRE_LOCK(ump); 13825 return; 13826 } 13827 } 13828 } 13829 13830 /* 13831 * Clear out a block of dirty inodes in an effort to reduce 13832 * the number of inodedep dependency structures. 13833 */ 13834 static void 13835 clear_inodedeps(mp) 13836 struct mount *mp; 13837 { 13838 struct inodedep_hashhead *inodedephd; 13839 struct inodedep *inodedep; 13840 struct ufsmount *ump; 13841 struct vnode *vp; 13842 struct fs *fs; 13843 int error, cnt; 13844 ino_t firstino, lastino, ino; 13845 13846 ump = VFSTOUFS(mp); 13847 fs = ump->um_fs; 13848 LOCK_OWNED(ump); 13849 /* 13850 * Pick a random inode dependency to be cleared. 13851 * We will then gather up all the inodes in its block 13852 * that have dependencies and flush them out. 13853 */ 13854 for (cnt = 0; cnt <= ump->inodedep_hash_size; cnt++) { 13855 inodedephd = &ump->inodedep_hashtbl[ump->inodedep_nextclean++]; 13856 if (ump->inodedep_nextclean > ump->inodedep_hash_size) 13857 ump->inodedep_nextclean = 0; 13858 if ((inodedep = LIST_FIRST(inodedephd)) != NULL) 13859 break; 13860 } 13861 if (inodedep == NULL) 13862 return; 13863 /* 13864 * Find the last inode in the block with dependencies. 13865 */ 13866 firstino = rounddown2(inodedep->id_ino, INOPB(fs)); 13867 for (lastino = firstino + INOPB(fs) - 1; lastino > firstino; lastino--) 13868 if (inodedep_lookup(mp, lastino, 0, &inodedep) != 0) 13869 break; 13870 /* 13871 * Asynchronously push all but the last inode with dependencies. 13872 * Synchronously push the last inode with dependencies to ensure 13873 * that the inode block gets written to free up the inodedeps. 13874 */ 13875 for (ino = firstino; ino <= lastino; ino++) { 13876 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 13877 continue; 13878 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 13879 continue; 13880 FREE_LOCK(ump); 13881 error = vfs_busy(mp, MBF_NOWAIT); /* Let unmount clear deps */ 13882 if (error != 0) { 13883 vn_finished_write(mp); 13884 ACQUIRE_LOCK(ump); 13885 return; 13886 } 13887 if ((error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 13888 FFSV_FORCEINSMQ)) != 0) { 13889 softdep_error("clear_inodedeps: vget", error); 13890 vfs_unbusy(mp); 13891 vn_finished_write(mp); 13892 ACQUIRE_LOCK(ump); 13893 return; 13894 } 13895 vfs_unbusy(mp); 13896 if (ino == lastino) { 13897 if ((error = ffs_syncvnode(vp, MNT_WAIT, 0))) 13898 softdep_error("clear_inodedeps: fsync1", error); 13899 } else { 13900 if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0))) 13901 softdep_error("clear_inodedeps: fsync2", error); 13902 BO_LOCK(&vp->v_bufobj); 13903 drain_output(vp); 13904 BO_UNLOCK(&vp->v_bufobj); 13905 } 13906 vput(vp); 13907 vn_finished_write(mp); 13908 ACQUIRE_LOCK(ump); 13909 } 13910 } 13911 13912 void 13913 softdep_buf_append(bp, wkhd) 13914 struct buf *bp; 13915 struct workhead *wkhd; 13916 { 13917 struct worklist *wk; 13918 struct ufsmount *ump; 13919 13920 if ((wk = LIST_FIRST(wkhd)) == NULL) 13921 return; 13922 KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0, 13923 ("softdep_buf_append called on non-softdep filesystem")); 13924 ump = VFSTOUFS(wk->wk_mp); 13925 ACQUIRE_LOCK(ump); 13926 while ((wk = LIST_FIRST(wkhd)) != NULL) { 13927 WORKLIST_REMOVE(wk); 13928 WORKLIST_INSERT(&bp->b_dep, wk); 13929 } 13930 FREE_LOCK(ump); 13931 13932 } 13933 13934 void 13935 softdep_inode_append(ip, cred, wkhd) 13936 struct inode *ip; 13937 struct ucred *cred; 13938 struct workhead *wkhd; 13939 { 13940 struct buf *bp; 13941 struct fs *fs; 13942 struct ufsmount *ump; 13943 int error; 13944 13945 ump = ITOUMP(ip); 13946 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 13947 ("softdep_inode_append called on non-softdep filesystem")); 13948 fs = ump->um_fs; 13949 error = bread(ump->um_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 13950 (int)fs->fs_bsize, cred, &bp); 13951 if (error) { 13952 bqrelse(bp); 13953 softdep_freework(wkhd); 13954 return; 13955 } 13956 softdep_buf_append(bp, wkhd); 13957 bqrelse(bp); 13958 } 13959 13960 void 13961 softdep_freework(wkhd) 13962 struct workhead *wkhd; 13963 { 13964 struct worklist *wk; 13965 struct ufsmount *ump; 13966 13967 if ((wk = LIST_FIRST(wkhd)) == NULL) 13968 return; 13969 KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0, 13970 ("softdep_freework called on non-softdep filesystem")); 13971 ump = VFSTOUFS(wk->wk_mp); 13972 ACQUIRE_LOCK(ump); 13973 handle_jwork(wkhd); 13974 FREE_LOCK(ump); 13975 } 13976 13977 static struct ufsmount * 13978 softdep_bp_to_mp(bp) 13979 struct buf *bp; 13980 { 13981 struct mount *mp; 13982 struct vnode *vp; 13983 13984 if (LIST_EMPTY(&bp->b_dep)) 13985 return (NULL); 13986 vp = bp->b_vp; 13987 KASSERT(vp != NULL, 13988 ("%s, buffer with dependencies lacks vnode", __func__)); 13989 13990 /* 13991 * The ump mount point is stable after we get a correct 13992 * pointer, since bp is locked and this prevents unmount from 13993 * proceeding. But to get to it, we cannot dereference bp->b_dep 13994 * head wk_mp, because we do not yet own SU ump lock and 13995 * workitem might be freed while dereferenced. 13996 */ 13997 retry: 13998 switch (vp->v_type) { 13999 case VCHR: 14000 VI_LOCK(vp); 14001 mp = vp->v_type == VCHR ? vp->v_rdev->si_mountpt : NULL; 14002 VI_UNLOCK(vp); 14003 if (mp == NULL) 14004 goto retry; 14005 break; 14006 case VREG: 14007 case VDIR: 14008 case VLNK: 14009 case VFIFO: 14010 case VSOCK: 14011 mp = vp->v_mount; 14012 break; 14013 case VBLK: 14014 vn_printf(vp, "softdep_bp_to_mp: unexpected block device\n"); 14015 /* FALLTHROUGH */ 14016 case VNON: 14017 case VBAD: 14018 case VMARKER: 14019 mp = NULL; 14020 break; 14021 default: 14022 vn_printf(vp, "unknown vnode type"); 14023 mp = NULL; 14024 break; 14025 } 14026 return (VFSTOUFS(mp)); 14027 } 14028 14029 /* 14030 * Function to determine if the buffer has outstanding dependencies 14031 * that will cause a roll-back if the buffer is written. If wantcount 14032 * is set, return number of dependencies, otherwise just yes or no. 14033 */ 14034 static int 14035 softdep_count_dependencies(bp, wantcount) 14036 struct buf *bp; 14037 int wantcount; 14038 { 14039 struct worklist *wk; 14040 struct ufsmount *ump; 14041 struct bmsafemap *bmsafemap; 14042 struct freework *freework; 14043 struct inodedep *inodedep; 14044 struct indirdep *indirdep; 14045 struct freeblks *freeblks; 14046 struct allocindir *aip; 14047 struct pagedep *pagedep; 14048 struct dirrem *dirrem; 14049 struct newblk *newblk; 14050 struct mkdir *mkdir; 14051 struct diradd *dap; 14052 int i, retval; 14053 14054 ump = softdep_bp_to_mp(bp); 14055 if (ump == NULL) 14056 return (0); 14057 retval = 0; 14058 ACQUIRE_LOCK(ump); 14059 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 14060 switch (wk->wk_type) { 14061 14062 case D_INODEDEP: 14063 inodedep = WK_INODEDEP(wk); 14064 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 14065 /* bitmap allocation dependency */ 14066 retval += 1; 14067 if (!wantcount) 14068 goto out; 14069 } 14070 if (TAILQ_FIRST(&inodedep->id_inoupdt)) { 14071 /* direct block pointer dependency */ 14072 retval += 1; 14073 if (!wantcount) 14074 goto out; 14075 } 14076 if (TAILQ_FIRST(&inodedep->id_extupdt)) { 14077 /* direct block pointer dependency */ 14078 retval += 1; 14079 if (!wantcount) 14080 goto out; 14081 } 14082 if (TAILQ_FIRST(&inodedep->id_inoreflst)) { 14083 /* Add reference dependency. */ 14084 retval += 1; 14085 if (!wantcount) 14086 goto out; 14087 } 14088 continue; 14089 14090 case D_INDIRDEP: 14091 indirdep = WK_INDIRDEP(wk); 14092 14093 TAILQ_FOREACH(freework, &indirdep->ir_trunc, fw_next) { 14094 /* indirect truncation dependency */ 14095 retval += 1; 14096 if (!wantcount) 14097 goto out; 14098 } 14099 14100 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) { 14101 /* indirect block pointer dependency */ 14102 retval += 1; 14103 if (!wantcount) 14104 goto out; 14105 } 14106 continue; 14107 14108 case D_PAGEDEP: 14109 pagedep = WK_PAGEDEP(wk); 14110 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) { 14111 if (LIST_FIRST(&dirrem->dm_jremrefhd)) { 14112 /* Journal remove ref dependency. */ 14113 retval += 1; 14114 if (!wantcount) 14115 goto out; 14116 } 14117 } 14118 for (i = 0; i < DAHASHSZ; i++) { 14119 14120 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 14121 /* directory entry dependency */ 14122 retval += 1; 14123 if (!wantcount) 14124 goto out; 14125 } 14126 } 14127 continue; 14128 14129 case D_BMSAFEMAP: 14130 bmsafemap = WK_BMSAFEMAP(wk); 14131 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd)) { 14132 /* Add reference dependency. */ 14133 retval += 1; 14134 if (!wantcount) 14135 goto out; 14136 } 14137 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd)) { 14138 /* Allocate block dependency. */ 14139 retval += 1; 14140 if (!wantcount) 14141 goto out; 14142 } 14143 continue; 14144 14145 case D_FREEBLKS: 14146 freeblks = WK_FREEBLKS(wk); 14147 if (LIST_FIRST(&freeblks->fb_jblkdephd)) { 14148 /* Freeblk journal dependency. */ 14149 retval += 1; 14150 if (!wantcount) 14151 goto out; 14152 } 14153 continue; 14154 14155 case D_ALLOCDIRECT: 14156 case D_ALLOCINDIR: 14157 newblk = WK_NEWBLK(wk); 14158 if (newblk->nb_jnewblk) { 14159 /* Journal allocate dependency. */ 14160 retval += 1; 14161 if (!wantcount) 14162 goto out; 14163 } 14164 continue; 14165 14166 case D_MKDIR: 14167 mkdir = WK_MKDIR(wk); 14168 if (mkdir->md_jaddref) { 14169 /* Journal reference dependency. */ 14170 retval += 1; 14171 if (!wantcount) 14172 goto out; 14173 } 14174 continue; 14175 14176 case D_FREEWORK: 14177 case D_FREEDEP: 14178 case D_JSEGDEP: 14179 case D_JSEG: 14180 case D_SBDEP: 14181 /* never a dependency on these blocks */ 14182 continue; 14183 14184 default: 14185 panic("softdep_count_dependencies: Unexpected type %s", 14186 TYPENAME(wk->wk_type)); 14187 /* NOTREACHED */ 14188 } 14189 } 14190 out: 14191 FREE_LOCK(ump); 14192 return (retval); 14193 } 14194 14195 /* 14196 * Acquire exclusive access to a buffer. 14197 * Must be called with a locked mtx parameter. 14198 * Return acquired buffer or NULL on failure. 14199 */ 14200 static struct buf * 14201 getdirtybuf(bp, lock, waitfor) 14202 struct buf *bp; 14203 struct rwlock *lock; 14204 int waitfor; 14205 { 14206 int error; 14207 14208 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) { 14209 if (waitfor != MNT_WAIT) 14210 return (NULL); 14211 error = BUF_LOCK(bp, 14212 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, lock); 14213 /* 14214 * Even if we successfully acquire bp here, we have dropped 14215 * lock, which may violates our guarantee. 14216 */ 14217 if (error == 0) 14218 BUF_UNLOCK(bp); 14219 else if (error != ENOLCK) 14220 panic("getdirtybuf: inconsistent lock: %d", error); 14221 rw_wlock(lock); 14222 return (NULL); 14223 } 14224 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 14225 if (lock != BO_LOCKPTR(bp->b_bufobj) && waitfor == MNT_WAIT) { 14226 rw_wunlock(lock); 14227 BO_LOCK(bp->b_bufobj); 14228 BUF_UNLOCK(bp); 14229 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 14230 bp->b_vflags |= BV_BKGRDWAIT; 14231 msleep(&bp->b_xflags, BO_LOCKPTR(bp->b_bufobj), 14232 PRIBIO | PDROP, "getbuf", 0); 14233 } else 14234 BO_UNLOCK(bp->b_bufobj); 14235 rw_wlock(lock); 14236 return (NULL); 14237 } 14238 BUF_UNLOCK(bp); 14239 if (waitfor != MNT_WAIT) 14240 return (NULL); 14241 #ifdef DEBUG_VFS_LOCKS 14242 if (bp->b_vp->v_type != VCHR) 14243 ASSERT_BO_WLOCKED(bp->b_bufobj); 14244 #endif 14245 bp->b_vflags |= BV_BKGRDWAIT; 14246 rw_sleep(&bp->b_xflags, lock, PRIBIO, "getbuf", 0); 14247 return (NULL); 14248 } 14249 if ((bp->b_flags & B_DELWRI) == 0) { 14250 BUF_UNLOCK(bp); 14251 return (NULL); 14252 } 14253 bremfree(bp); 14254 return (bp); 14255 } 14256 14257 14258 /* 14259 * Check if it is safe to suspend the file system now. On entry, 14260 * the vnode interlock for devvp should be held. Return 0 with 14261 * the mount interlock held if the file system can be suspended now, 14262 * otherwise return EAGAIN with the mount interlock held. 14263 */ 14264 int 14265 softdep_check_suspend(struct mount *mp, 14266 struct vnode *devvp, 14267 int softdep_depcnt, 14268 int softdep_accdepcnt, 14269 int secondary_writes, 14270 int secondary_accwrites) 14271 { 14272 struct bufobj *bo; 14273 struct ufsmount *ump; 14274 struct inodedep *inodedep; 14275 int error, unlinked; 14276 14277 bo = &devvp->v_bufobj; 14278 ASSERT_BO_WLOCKED(bo); 14279 14280 /* 14281 * If we are not running with soft updates, then we need only 14282 * deal with secondary writes as we try to suspend. 14283 */ 14284 if (MOUNTEDSOFTDEP(mp) == 0) { 14285 MNT_ILOCK(mp); 14286 while (mp->mnt_secondary_writes != 0) { 14287 BO_UNLOCK(bo); 14288 msleep(&mp->mnt_secondary_writes, MNT_MTX(mp), 14289 (PUSER - 1) | PDROP, "secwr", 0); 14290 BO_LOCK(bo); 14291 MNT_ILOCK(mp); 14292 } 14293 14294 /* 14295 * Reasons for needing more work before suspend: 14296 * - Dirty buffers on devvp. 14297 * - Secondary writes occurred after start of vnode sync loop 14298 */ 14299 error = 0; 14300 if (bo->bo_numoutput > 0 || 14301 bo->bo_dirty.bv_cnt > 0 || 14302 secondary_writes != 0 || 14303 mp->mnt_secondary_writes != 0 || 14304 secondary_accwrites != mp->mnt_secondary_accwrites) 14305 error = EAGAIN; 14306 BO_UNLOCK(bo); 14307 return (error); 14308 } 14309 14310 /* 14311 * If we are running with soft updates, then we need to coordinate 14312 * with them as we try to suspend. 14313 */ 14314 ump = VFSTOUFS(mp); 14315 for (;;) { 14316 if (!TRY_ACQUIRE_LOCK(ump)) { 14317 BO_UNLOCK(bo); 14318 ACQUIRE_LOCK(ump); 14319 FREE_LOCK(ump); 14320 BO_LOCK(bo); 14321 continue; 14322 } 14323 MNT_ILOCK(mp); 14324 if (mp->mnt_secondary_writes != 0) { 14325 FREE_LOCK(ump); 14326 BO_UNLOCK(bo); 14327 msleep(&mp->mnt_secondary_writes, 14328 MNT_MTX(mp), 14329 (PUSER - 1) | PDROP, "secwr", 0); 14330 BO_LOCK(bo); 14331 continue; 14332 } 14333 break; 14334 } 14335 14336 unlinked = 0; 14337 if (MOUNTEDSUJ(mp)) { 14338 for (inodedep = TAILQ_FIRST(&ump->softdep_unlinked); 14339 inodedep != NULL; 14340 inodedep = TAILQ_NEXT(inodedep, id_unlinked)) { 14341 if ((inodedep->id_state & (UNLINKED | UNLINKLINKS | 14342 UNLINKONLIST)) != (UNLINKED | UNLINKLINKS | 14343 UNLINKONLIST) || 14344 !check_inodedep_free(inodedep)) 14345 continue; 14346 unlinked++; 14347 } 14348 } 14349 14350 /* 14351 * Reasons for needing more work before suspend: 14352 * - Dirty buffers on devvp. 14353 * - Softdep activity occurred after start of vnode sync loop 14354 * - Secondary writes occurred after start of vnode sync loop 14355 */ 14356 error = 0; 14357 if (bo->bo_numoutput > 0 || 14358 bo->bo_dirty.bv_cnt > 0 || 14359 softdep_depcnt != unlinked || 14360 ump->softdep_deps != unlinked || 14361 softdep_accdepcnt != ump->softdep_accdeps || 14362 secondary_writes != 0 || 14363 mp->mnt_secondary_writes != 0 || 14364 secondary_accwrites != mp->mnt_secondary_accwrites) 14365 error = EAGAIN; 14366 FREE_LOCK(ump); 14367 BO_UNLOCK(bo); 14368 return (error); 14369 } 14370 14371 14372 /* 14373 * Get the number of dependency structures for the file system, both 14374 * the current number and the total number allocated. These will 14375 * later be used to detect that softdep processing has occurred. 14376 */ 14377 void 14378 softdep_get_depcounts(struct mount *mp, 14379 int *softdep_depsp, 14380 int *softdep_accdepsp) 14381 { 14382 struct ufsmount *ump; 14383 14384 if (MOUNTEDSOFTDEP(mp) == 0) { 14385 *softdep_depsp = 0; 14386 *softdep_accdepsp = 0; 14387 return; 14388 } 14389 ump = VFSTOUFS(mp); 14390 ACQUIRE_LOCK(ump); 14391 *softdep_depsp = ump->softdep_deps; 14392 *softdep_accdepsp = ump->softdep_accdeps; 14393 FREE_LOCK(ump); 14394 } 14395 14396 /* 14397 * Wait for pending output on a vnode to complete. 14398 */ 14399 static void 14400 drain_output(vp) 14401 struct vnode *vp; 14402 { 14403 14404 ASSERT_VOP_LOCKED(vp, "drain_output"); 14405 (void)bufobj_wwait(&vp->v_bufobj, 0, 0); 14406 } 14407 14408 /* 14409 * Called whenever a buffer that is being invalidated or reallocated 14410 * contains dependencies. This should only happen if an I/O error has 14411 * occurred. The routine is called with the buffer locked. 14412 */ 14413 static void 14414 softdep_deallocate_dependencies(bp) 14415 struct buf *bp; 14416 { 14417 14418 if ((bp->b_ioflags & BIO_ERROR) == 0) 14419 panic("softdep_deallocate_dependencies: dangling deps"); 14420 if (bp->b_vp != NULL && bp->b_vp->v_mount != NULL) 14421 softdep_error(bp->b_vp->v_mount->mnt_stat.f_mntonname, bp->b_error); 14422 else 14423 printf("softdep_deallocate_dependencies: " 14424 "got error %d while accessing filesystem\n", bp->b_error); 14425 if (bp->b_error != ENXIO) 14426 panic("softdep_deallocate_dependencies: unrecovered I/O error"); 14427 } 14428 14429 /* 14430 * Function to handle asynchronous write errors in the filesystem. 14431 */ 14432 static void 14433 softdep_error(func, error) 14434 char *func; 14435 int error; 14436 { 14437 14438 /* XXX should do something better! */ 14439 printf("%s: got error %d while accessing filesystem\n", func, error); 14440 } 14441 14442 #ifdef DDB 14443 14444 /* exported to ffs_vfsops.c */ 14445 extern void db_print_ffs(struct ufsmount *ump); 14446 void 14447 db_print_ffs(struct ufsmount *ump) 14448 { 14449 db_printf("mp %p (%s) devvp %p\n", ump->um_mountp, 14450 ump->um_mountp->mnt_stat.f_mntonname, ump->um_devvp); 14451 db_printf(" fs %p su_wl %d su_deps %d su_req %d\n", 14452 ump->um_fs, ump->softdep_on_worklist, 14453 ump->softdep_deps, ump->softdep_req); 14454 } 14455 14456 static void 14457 worklist_print(struct worklist *wk, int verbose) 14458 { 14459 14460 if (!verbose) { 14461 db_printf("%s: %p state 0x%b\n", TYPENAME(wk->wk_type), wk, 14462 (u_int)wk->wk_state, PRINT_SOFTDEP_FLAGS); 14463 return; 14464 } 14465 db_printf("worklist: %p type %s state 0x%b next %p\n ", wk, 14466 TYPENAME(wk->wk_type), (u_int)wk->wk_state, PRINT_SOFTDEP_FLAGS, 14467 LIST_NEXT(wk, wk_list)); 14468 db_print_ffs(VFSTOUFS(wk->wk_mp)); 14469 } 14470 14471 static void 14472 inodedep_print(struct inodedep *inodedep, int verbose) 14473 { 14474 14475 worklist_print(&inodedep->id_list, 0); 14476 db_printf(" fs %p ino %jd inoblk %jd delta %jd nlink %jd\n", 14477 inodedep->id_fs, 14478 (intmax_t)inodedep->id_ino, 14479 (intmax_t)fsbtodb(inodedep->id_fs, 14480 ino_to_fsba(inodedep->id_fs, inodedep->id_ino)), 14481 (intmax_t)inodedep->id_nlinkdelta, 14482 (intmax_t)inodedep->id_savednlink); 14483 14484 if (verbose == 0) 14485 return; 14486 14487 db_printf(" bmsafemap %p, mkdiradd %p, inoreflst %p\n", 14488 inodedep->id_bmsafemap, 14489 inodedep->id_mkdiradd, 14490 TAILQ_FIRST(&inodedep->id_inoreflst)); 14491 db_printf(" dirremhd %p, pendinghd %p, bufwait %p\n", 14492 LIST_FIRST(&inodedep->id_dirremhd), 14493 LIST_FIRST(&inodedep->id_pendinghd), 14494 LIST_FIRST(&inodedep->id_bufwait)); 14495 db_printf(" inowait %p, inoupdt %p, newinoupdt %p\n", 14496 LIST_FIRST(&inodedep->id_inowait), 14497 TAILQ_FIRST(&inodedep->id_inoupdt), 14498 TAILQ_FIRST(&inodedep->id_newinoupdt)); 14499 db_printf(" extupdt %p, newextupdt %p, freeblklst %p\n", 14500 TAILQ_FIRST(&inodedep->id_extupdt), 14501 TAILQ_FIRST(&inodedep->id_newextupdt), 14502 TAILQ_FIRST(&inodedep->id_freeblklst)); 14503 db_printf(" saveino %p, savedsize %jd, savedextsize %jd\n", 14504 inodedep->id_savedino1, 14505 (intmax_t)inodedep->id_savedsize, 14506 (intmax_t)inodedep->id_savedextsize); 14507 } 14508 14509 static void 14510 newblk_print(struct newblk *nbp) 14511 { 14512 14513 worklist_print(&nbp->nb_list, 0); 14514 db_printf(" newblkno %jd\n", (intmax_t)nbp->nb_newblkno); 14515 db_printf(" jnewblk %p, bmsafemap %p, freefrag %p\n", 14516 &nbp->nb_jnewblk, 14517 &nbp->nb_bmsafemap, 14518 &nbp->nb_freefrag); 14519 db_printf(" indirdeps %p, newdirblk %p, jwork %p\n", 14520 LIST_FIRST(&nbp->nb_indirdeps), 14521 LIST_FIRST(&nbp->nb_newdirblk), 14522 LIST_FIRST(&nbp->nb_jwork)); 14523 } 14524 14525 static void 14526 allocdirect_print(struct allocdirect *adp) 14527 { 14528 14529 newblk_print(&adp->ad_block); 14530 db_printf(" oldblkno %jd, oldsize %ld, newsize %ld\n", 14531 adp->ad_oldblkno, adp->ad_oldsize, adp->ad_newsize); 14532 db_printf(" offset %d, inodedep %p\n", 14533 adp->ad_offset, adp->ad_inodedep); 14534 } 14535 14536 static void 14537 allocindir_print(struct allocindir *aip) 14538 { 14539 14540 newblk_print(&aip->ai_block); 14541 db_printf(" oldblkno %jd, lbn %jd\n", 14542 (intmax_t)aip->ai_oldblkno, (intmax_t)aip->ai_lbn); 14543 db_printf(" offset %d, indirdep %p\n", 14544 aip->ai_offset, aip->ai_indirdep); 14545 } 14546 14547 static void 14548 mkdir_print(struct mkdir *mkdir) 14549 { 14550 14551 worklist_print(&mkdir->md_list, 0); 14552 db_printf(" diradd %p, jaddref %p, buf %p\n", 14553 mkdir->md_diradd, mkdir->md_jaddref, mkdir->md_buf); 14554 } 14555 14556 DB_SHOW_COMMAND(sd_inodedep, db_show_sd_inodedep) 14557 { 14558 14559 if (have_addr == 0) { 14560 db_printf("inodedep address required\n"); 14561 return; 14562 } 14563 inodedep_print((struct inodedep*)addr, 1); 14564 } 14565 14566 DB_SHOW_COMMAND(sd_allinodedeps, db_show_sd_allinodedeps) 14567 { 14568 struct inodedep_hashhead *inodedephd; 14569 struct inodedep *inodedep; 14570 struct ufsmount *ump; 14571 int cnt; 14572 14573 if (have_addr == 0) { 14574 db_printf("ufsmount address required\n"); 14575 return; 14576 } 14577 ump = (struct ufsmount *)addr; 14578 for (cnt = 0; cnt < ump->inodedep_hash_size; cnt++) { 14579 inodedephd = &ump->inodedep_hashtbl[cnt]; 14580 LIST_FOREACH(inodedep, inodedephd, id_hash) { 14581 inodedep_print(inodedep, 0); 14582 } 14583 } 14584 } 14585 14586 DB_SHOW_COMMAND(sd_worklist, db_show_sd_worklist) 14587 { 14588 14589 if (have_addr == 0) { 14590 db_printf("worklist address required\n"); 14591 return; 14592 } 14593 worklist_print((struct worklist *)addr, 1); 14594 } 14595 14596 DB_SHOW_COMMAND(sd_workhead, db_show_sd_workhead) 14597 { 14598 struct worklist *wk; 14599 struct workhead *wkhd; 14600 14601 if (have_addr == 0) { 14602 db_printf("worklist address required " 14603 "(for example value in bp->b_dep)\n"); 14604 return; 14605 } 14606 /* 14607 * We often do not have the address of the worklist head but 14608 * instead a pointer to its first entry (e.g., we have the 14609 * contents of bp->b_dep rather than &bp->b_dep). But the back 14610 * pointer of bp->b_dep will point at the head of the list, so 14611 * we cheat and use that instead. If we are in the middle of 14612 * a list we will still get the same result, so nothing 14613 * unexpected will result. 14614 */ 14615 wk = (struct worklist *)addr; 14616 if (wk == NULL) 14617 return; 14618 wkhd = (struct workhead *)wk->wk_list.le_prev; 14619 LIST_FOREACH(wk, wkhd, wk_list) { 14620 switch(wk->wk_type) { 14621 case D_INODEDEP: 14622 inodedep_print(WK_INODEDEP(wk), 0); 14623 continue; 14624 case D_ALLOCDIRECT: 14625 allocdirect_print(WK_ALLOCDIRECT(wk)); 14626 continue; 14627 case D_ALLOCINDIR: 14628 allocindir_print(WK_ALLOCINDIR(wk)); 14629 continue; 14630 case D_MKDIR: 14631 mkdir_print(WK_MKDIR(wk)); 14632 continue; 14633 default: 14634 worklist_print(wk, 0); 14635 continue; 14636 } 14637 } 14638 } 14639 14640 DB_SHOW_COMMAND(sd_mkdir, db_show_sd_mkdir) 14641 { 14642 if (have_addr == 0) { 14643 db_printf("mkdir address required\n"); 14644 return; 14645 } 14646 mkdir_print((struct mkdir *)addr); 14647 } 14648 14649 DB_SHOW_COMMAND(sd_mkdir_list, db_show_sd_mkdir_list) 14650 { 14651 struct mkdirlist *mkdirlisthd; 14652 struct mkdir *mkdir; 14653 14654 if (have_addr == 0) { 14655 db_printf("mkdir listhead address required\n"); 14656 return; 14657 } 14658 mkdirlisthd = (struct mkdirlist *)addr; 14659 LIST_FOREACH(mkdir, mkdirlisthd, md_mkdirs) { 14660 mkdir_print(mkdir); 14661 if (mkdir->md_diradd != NULL) { 14662 db_printf(" "); 14663 worklist_print(&mkdir->md_diradd->da_list, 0); 14664 } 14665 if (mkdir->md_jaddref != NULL) { 14666 db_printf(" "); 14667 worklist_print(&mkdir->md_jaddref->ja_list, 0); 14668 } 14669 } 14670 } 14671 14672 DB_SHOW_COMMAND(sd_allocdirect, db_show_sd_allocdirect) 14673 { 14674 if (have_addr == 0) { 14675 db_printf("allocdirect address required\n"); 14676 return; 14677 } 14678 allocdirect_print((struct allocdirect *)addr); 14679 } 14680 14681 DB_SHOW_COMMAND(sd_allocindir, db_show_sd_allocindir) 14682 { 14683 if (have_addr == 0) { 14684 db_printf("allocindir address required\n"); 14685 return; 14686 } 14687 allocindir_print((struct allocindir *)addr); 14688 } 14689 14690 #endif /* DDB */ 14691 14692 #endif /* SOFTUPDATES */ 14693