1 /*- 2 * Copyright 1998, 2000 Marshall Kirk McKusick. 3 * Copyright 2009, 2010 Jeffrey W. Roberson <jeff@FreeBSD.org> 4 * All rights reserved. 5 * 6 * The soft updates code is derived from the appendix of a University 7 * of Michigan technical report (Gregory R. Ganger and Yale N. Patt, 8 * "Soft Updates: A Solution to the Metadata Update Problem in File 9 * Systems", CSE-TR-254-95, August 1995). 10 * 11 * Further information about soft updates can be obtained from: 12 * 13 * Marshall Kirk McKusick http://www.mckusick.com/softdep/ 14 * 1614 Oxford Street mckusick@mckusick.com 15 * Berkeley, CA 94709-1608 +1-510-843-9542 16 * USA 17 * 18 * Redistribution and use in source and binary forms, with or without 19 * modification, are permitted provided that the following conditions 20 * are met: 21 * 22 * 1. Redistributions of source code must retain the above copyright 23 * notice, this list of conditions and the following disclaimer. 24 * 2. Redistributions in binary form must reproduce the above copyright 25 * notice, this list of conditions and the following disclaimer in the 26 * documentation and/or other materials provided with the distribution. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 34 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 37 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * from: @(#)ffs_softdep.c 9.59 (McKusick) 6/21/00 40 */ 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include "opt_ffs.h" 46 #include "opt_quota.h" 47 #include "opt_ddb.h" 48 49 /* 50 * For now we want the safety net that the DEBUG flag provides. 51 */ 52 #ifndef DEBUG 53 #define DEBUG 54 #endif 55 56 #include <sys/param.h> 57 #include <sys/kernel.h> 58 #include <sys/systm.h> 59 #include <sys/bio.h> 60 #include <sys/buf.h> 61 #include <sys/kdb.h> 62 #include <sys/kthread.h> 63 #include <sys/ktr.h> 64 #include <sys/limits.h> 65 #include <sys/lock.h> 66 #include <sys/malloc.h> 67 #include <sys/mount.h> 68 #include <sys/mutex.h> 69 #include <sys/namei.h> 70 #include <sys/priv.h> 71 #include <sys/proc.h> 72 #include <sys/racct.h> 73 #include <sys/rwlock.h> 74 #include <sys/stat.h> 75 #include <sys/sysctl.h> 76 #include <sys/syslog.h> 77 #include <sys/vnode.h> 78 #include <sys/conf.h> 79 80 #include <ufs/ufs/dir.h> 81 #include <ufs/ufs/extattr.h> 82 #include <ufs/ufs/quota.h> 83 #include <ufs/ufs/inode.h> 84 #include <ufs/ufs/ufsmount.h> 85 #include <ufs/ffs/fs.h> 86 #include <ufs/ffs/softdep.h> 87 #include <ufs/ffs/ffs_extern.h> 88 #include <ufs/ufs/ufs_extern.h> 89 90 #include <vm/vm.h> 91 #include <vm/vm_extern.h> 92 #include <vm/vm_object.h> 93 94 #include <geom/geom.h> 95 96 #include <ddb/ddb.h> 97 98 #define KTR_SUJ 0 /* Define to KTR_SPARE. */ 99 100 #ifndef SOFTUPDATES 101 102 int 103 softdep_flushfiles(oldmnt, flags, td) 104 struct mount *oldmnt; 105 int flags; 106 struct thread *td; 107 { 108 109 panic("softdep_flushfiles called"); 110 } 111 112 int 113 softdep_mount(devvp, mp, fs, cred) 114 struct vnode *devvp; 115 struct mount *mp; 116 struct fs *fs; 117 struct ucred *cred; 118 { 119 120 return (0); 121 } 122 123 void 124 softdep_initialize() 125 { 126 127 return; 128 } 129 130 void 131 softdep_uninitialize() 132 { 133 134 return; 135 } 136 137 void 138 softdep_unmount(mp) 139 struct mount *mp; 140 { 141 142 panic("softdep_unmount called"); 143 } 144 145 void 146 softdep_setup_sbupdate(ump, fs, bp) 147 struct ufsmount *ump; 148 struct fs *fs; 149 struct buf *bp; 150 { 151 152 panic("softdep_setup_sbupdate called"); 153 } 154 155 void 156 softdep_setup_inomapdep(bp, ip, newinum, mode) 157 struct buf *bp; 158 struct inode *ip; 159 ino_t newinum; 160 int mode; 161 { 162 163 panic("softdep_setup_inomapdep called"); 164 } 165 166 void 167 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 168 struct buf *bp; 169 struct mount *mp; 170 ufs2_daddr_t newblkno; 171 int frags; 172 int oldfrags; 173 { 174 175 panic("softdep_setup_blkmapdep called"); 176 } 177 178 void 179 softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 180 struct inode *ip; 181 ufs_lbn_t lbn; 182 ufs2_daddr_t newblkno; 183 ufs2_daddr_t oldblkno; 184 long newsize; 185 long oldsize; 186 struct buf *bp; 187 { 188 189 panic("softdep_setup_allocdirect called"); 190 } 191 192 void 193 softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 194 struct inode *ip; 195 ufs_lbn_t lbn; 196 ufs2_daddr_t newblkno; 197 ufs2_daddr_t oldblkno; 198 long newsize; 199 long oldsize; 200 struct buf *bp; 201 { 202 203 panic("softdep_setup_allocext called"); 204 } 205 206 void 207 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 208 struct inode *ip; 209 ufs_lbn_t lbn; 210 struct buf *bp; 211 int ptrno; 212 ufs2_daddr_t newblkno; 213 ufs2_daddr_t oldblkno; 214 struct buf *nbp; 215 { 216 217 panic("softdep_setup_allocindir_page called"); 218 } 219 220 void 221 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 222 struct buf *nbp; 223 struct inode *ip; 224 struct buf *bp; 225 int ptrno; 226 ufs2_daddr_t newblkno; 227 { 228 229 panic("softdep_setup_allocindir_meta called"); 230 } 231 232 void 233 softdep_journal_freeblocks(ip, cred, length, flags) 234 struct inode *ip; 235 struct ucred *cred; 236 off_t length; 237 int flags; 238 { 239 240 panic("softdep_journal_freeblocks called"); 241 } 242 243 void 244 softdep_journal_fsync(ip) 245 struct inode *ip; 246 { 247 248 panic("softdep_journal_fsync called"); 249 } 250 251 void 252 softdep_setup_freeblocks(ip, length, flags) 253 struct inode *ip; 254 off_t length; 255 int flags; 256 { 257 258 panic("softdep_setup_freeblocks called"); 259 } 260 261 void 262 softdep_freefile(pvp, ino, mode) 263 struct vnode *pvp; 264 ino_t ino; 265 int mode; 266 { 267 268 panic("softdep_freefile called"); 269 } 270 271 int 272 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 273 struct buf *bp; 274 struct inode *dp; 275 off_t diroffset; 276 ino_t newinum; 277 struct buf *newdirbp; 278 int isnewblk; 279 { 280 281 panic("softdep_setup_directory_add called"); 282 } 283 284 void 285 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 286 struct buf *bp; 287 struct inode *dp; 288 caddr_t base; 289 caddr_t oldloc; 290 caddr_t newloc; 291 int entrysize; 292 { 293 294 panic("softdep_change_directoryentry_offset called"); 295 } 296 297 void 298 softdep_setup_remove(bp, dp, ip, isrmdir) 299 struct buf *bp; 300 struct inode *dp; 301 struct inode *ip; 302 int isrmdir; 303 { 304 305 panic("softdep_setup_remove called"); 306 } 307 308 void 309 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 310 struct buf *bp; 311 struct inode *dp; 312 struct inode *ip; 313 ino_t newinum; 314 int isrmdir; 315 { 316 317 panic("softdep_setup_directory_change called"); 318 } 319 320 void 321 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 322 struct mount *mp; 323 struct buf *bp; 324 ufs2_daddr_t blkno; 325 int frags; 326 struct workhead *wkhd; 327 { 328 329 panic("%s called", __FUNCTION__); 330 } 331 332 void 333 softdep_setup_inofree(mp, bp, ino, wkhd) 334 struct mount *mp; 335 struct buf *bp; 336 ino_t ino; 337 struct workhead *wkhd; 338 { 339 340 panic("%s called", __FUNCTION__); 341 } 342 343 void 344 softdep_setup_unlink(dp, ip) 345 struct inode *dp; 346 struct inode *ip; 347 { 348 349 panic("%s called", __FUNCTION__); 350 } 351 352 void 353 softdep_setup_link(dp, ip) 354 struct inode *dp; 355 struct inode *ip; 356 { 357 358 panic("%s called", __FUNCTION__); 359 } 360 361 void 362 softdep_revert_link(dp, ip) 363 struct inode *dp; 364 struct inode *ip; 365 { 366 367 panic("%s called", __FUNCTION__); 368 } 369 370 void 371 softdep_setup_rmdir(dp, ip) 372 struct inode *dp; 373 struct inode *ip; 374 { 375 376 panic("%s called", __FUNCTION__); 377 } 378 379 void 380 softdep_revert_rmdir(dp, ip) 381 struct inode *dp; 382 struct inode *ip; 383 { 384 385 panic("%s called", __FUNCTION__); 386 } 387 388 void 389 softdep_setup_create(dp, ip) 390 struct inode *dp; 391 struct inode *ip; 392 { 393 394 panic("%s called", __FUNCTION__); 395 } 396 397 void 398 softdep_revert_create(dp, ip) 399 struct inode *dp; 400 struct inode *ip; 401 { 402 403 panic("%s called", __FUNCTION__); 404 } 405 406 void 407 softdep_setup_mkdir(dp, ip) 408 struct inode *dp; 409 struct inode *ip; 410 { 411 412 panic("%s called", __FUNCTION__); 413 } 414 415 void 416 softdep_revert_mkdir(dp, ip) 417 struct inode *dp; 418 struct inode *ip; 419 { 420 421 panic("%s called", __FUNCTION__); 422 } 423 424 void 425 softdep_setup_dotdot_link(dp, ip) 426 struct inode *dp; 427 struct inode *ip; 428 { 429 430 panic("%s called", __FUNCTION__); 431 } 432 433 int 434 softdep_prealloc(vp, waitok) 435 struct vnode *vp; 436 int waitok; 437 { 438 439 panic("%s called", __FUNCTION__); 440 } 441 442 int 443 softdep_journal_lookup(mp, vpp) 444 struct mount *mp; 445 struct vnode **vpp; 446 { 447 448 return (ENOENT); 449 } 450 451 void 452 softdep_change_linkcnt(ip) 453 struct inode *ip; 454 { 455 456 panic("softdep_change_linkcnt called"); 457 } 458 459 void 460 softdep_load_inodeblock(ip) 461 struct inode *ip; 462 { 463 464 panic("softdep_load_inodeblock called"); 465 } 466 467 void 468 softdep_update_inodeblock(ip, bp, waitfor) 469 struct inode *ip; 470 struct buf *bp; 471 int waitfor; 472 { 473 474 panic("softdep_update_inodeblock called"); 475 } 476 477 int 478 softdep_fsync(vp) 479 struct vnode *vp; /* the "in_core" copy of the inode */ 480 { 481 482 return (0); 483 } 484 485 void 486 softdep_fsync_mountdev(vp) 487 struct vnode *vp; 488 { 489 490 return; 491 } 492 493 int 494 softdep_flushworklist(oldmnt, countp, td) 495 struct mount *oldmnt; 496 int *countp; 497 struct thread *td; 498 { 499 500 *countp = 0; 501 return (0); 502 } 503 504 int 505 softdep_sync_metadata(struct vnode *vp) 506 { 507 508 panic("softdep_sync_metadata called"); 509 } 510 511 int 512 softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor) 513 { 514 515 panic("softdep_sync_buf called"); 516 } 517 518 int 519 softdep_slowdown(vp) 520 struct vnode *vp; 521 { 522 523 panic("softdep_slowdown called"); 524 } 525 526 int 527 softdep_request_cleanup(fs, vp, cred, resource) 528 struct fs *fs; 529 struct vnode *vp; 530 struct ucred *cred; 531 int resource; 532 { 533 534 return (0); 535 } 536 537 int 538 softdep_check_suspend(struct mount *mp, 539 struct vnode *devvp, 540 int softdep_depcnt, 541 int softdep_accdepcnt, 542 int secondary_writes, 543 int secondary_accwrites) 544 { 545 struct bufobj *bo; 546 int error; 547 548 (void) softdep_depcnt, 549 (void) softdep_accdepcnt; 550 551 bo = &devvp->v_bufobj; 552 ASSERT_BO_WLOCKED(bo); 553 554 MNT_ILOCK(mp); 555 while (mp->mnt_secondary_writes != 0) { 556 BO_UNLOCK(bo); 557 msleep(&mp->mnt_secondary_writes, MNT_MTX(mp), 558 (PUSER - 1) | PDROP, "secwr", 0); 559 BO_LOCK(bo); 560 MNT_ILOCK(mp); 561 } 562 563 /* 564 * Reasons for needing more work before suspend: 565 * - Dirty buffers on devvp. 566 * - Secondary writes occurred after start of vnode sync loop 567 */ 568 error = 0; 569 if (bo->bo_numoutput > 0 || 570 bo->bo_dirty.bv_cnt > 0 || 571 secondary_writes != 0 || 572 mp->mnt_secondary_writes != 0 || 573 secondary_accwrites != mp->mnt_secondary_accwrites) 574 error = EAGAIN; 575 BO_UNLOCK(bo); 576 return (error); 577 } 578 579 void 580 softdep_get_depcounts(struct mount *mp, 581 int *softdepactivep, 582 int *softdepactiveaccp) 583 { 584 (void) mp; 585 *softdepactivep = 0; 586 *softdepactiveaccp = 0; 587 } 588 589 void 590 softdep_buf_append(bp, wkhd) 591 struct buf *bp; 592 struct workhead *wkhd; 593 { 594 595 panic("softdep_buf_appendwork called"); 596 } 597 598 void 599 softdep_inode_append(ip, cred, wkhd) 600 struct inode *ip; 601 struct ucred *cred; 602 struct workhead *wkhd; 603 { 604 605 panic("softdep_inode_appendwork called"); 606 } 607 608 void 609 softdep_freework(wkhd) 610 struct workhead *wkhd; 611 { 612 613 panic("softdep_freework called"); 614 } 615 616 #else 617 618 FEATURE(softupdates, "FFS soft-updates support"); 619 620 static SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW, 0, 621 "soft updates stats"); 622 static SYSCTL_NODE(_debug_softdep, OID_AUTO, total, CTLFLAG_RW, 0, 623 "total dependencies allocated"); 624 static SYSCTL_NODE(_debug_softdep, OID_AUTO, highuse, CTLFLAG_RW, 0, 625 "high use dependencies allocated"); 626 static SYSCTL_NODE(_debug_softdep, OID_AUTO, current, CTLFLAG_RW, 0, 627 "current dependencies allocated"); 628 static SYSCTL_NODE(_debug_softdep, OID_AUTO, write, CTLFLAG_RW, 0, 629 "current dependencies written"); 630 631 unsigned long dep_current[D_LAST + 1]; 632 unsigned long dep_highuse[D_LAST + 1]; 633 unsigned long dep_total[D_LAST + 1]; 634 unsigned long dep_write[D_LAST + 1]; 635 636 #define SOFTDEP_TYPE(type, str, long) \ 637 static MALLOC_DEFINE(M_ ## type, #str, long); \ 638 SYSCTL_ULONG(_debug_softdep_total, OID_AUTO, str, CTLFLAG_RD, \ 639 &dep_total[D_ ## type], 0, ""); \ 640 SYSCTL_ULONG(_debug_softdep_current, OID_AUTO, str, CTLFLAG_RD, \ 641 &dep_current[D_ ## type], 0, ""); \ 642 SYSCTL_ULONG(_debug_softdep_highuse, OID_AUTO, str, CTLFLAG_RD, \ 643 &dep_highuse[D_ ## type], 0, ""); \ 644 SYSCTL_ULONG(_debug_softdep_write, OID_AUTO, str, CTLFLAG_RD, \ 645 &dep_write[D_ ## type], 0, ""); 646 647 SOFTDEP_TYPE(PAGEDEP, pagedep, "File page dependencies"); 648 SOFTDEP_TYPE(INODEDEP, inodedep, "Inode dependencies"); 649 SOFTDEP_TYPE(BMSAFEMAP, bmsafemap, 650 "Block or frag allocated from cyl group map"); 651 SOFTDEP_TYPE(NEWBLK, newblk, "New block or frag allocation dependency"); 652 SOFTDEP_TYPE(ALLOCDIRECT, allocdirect, "Block or frag dependency for an inode"); 653 SOFTDEP_TYPE(INDIRDEP, indirdep, "Indirect block dependencies"); 654 SOFTDEP_TYPE(ALLOCINDIR, allocindir, "Block dependency for an indirect block"); 655 SOFTDEP_TYPE(FREEFRAG, freefrag, "Previously used frag for an inode"); 656 SOFTDEP_TYPE(FREEBLKS, freeblks, "Blocks freed from an inode"); 657 SOFTDEP_TYPE(FREEFILE, freefile, "Inode deallocated"); 658 SOFTDEP_TYPE(DIRADD, diradd, "New directory entry"); 659 SOFTDEP_TYPE(MKDIR, mkdir, "New directory"); 660 SOFTDEP_TYPE(DIRREM, dirrem, "Directory entry deleted"); 661 SOFTDEP_TYPE(NEWDIRBLK, newdirblk, "Unclaimed new directory block"); 662 SOFTDEP_TYPE(FREEWORK, freework, "free an inode block"); 663 SOFTDEP_TYPE(FREEDEP, freedep, "track a block free"); 664 SOFTDEP_TYPE(JADDREF, jaddref, "Journal inode ref add"); 665 SOFTDEP_TYPE(JREMREF, jremref, "Journal inode ref remove"); 666 SOFTDEP_TYPE(JMVREF, jmvref, "Journal inode ref move"); 667 SOFTDEP_TYPE(JNEWBLK, jnewblk, "Journal new block"); 668 SOFTDEP_TYPE(JFREEBLK, jfreeblk, "Journal free block"); 669 SOFTDEP_TYPE(JFREEFRAG, jfreefrag, "Journal free frag"); 670 SOFTDEP_TYPE(JSEG, jseg, "Journal segment"); 671 SOFTDEP_TYPE(JSEGDEP, jsegdep, "Journal segment complete"); 672 SOFTDEP_TYPE(SBDEP, sbdep, "Superblock write dependency"); 673 SOFTDEP_TYPE(JTRUNC, jtrunc, "Journal inode truncation"); 674 SOFTDEP_TYPE(JFSYNC, jfsync, "Journal fsync complete"); 675 676 static MALLOC_DEFINE(M_SENTINEL, "sentinel", "Worklist sentinel"); 677 678 static MALLOC_DEFINE(M_SAVEDINO, "savedino", "Saved inodes"); 679 static MALLOC_DEFINE(M_JBLOCKS, "jblocks", "Journal block locations"); 680 static MALLOC_DEFINE(M_MOUNTDATA, "softdep", "Softdep per-mount data"); 681 682 #define M_SOFTDEP_FLAGS (M_WAITOK) 683 684 /* 685 * translate from workitem type to memory type 686 * MUST match the defines above, such that memtype[D_XXX] == M_XXX 687 */ 688 static struct malloc_type *memtype[] = { 689 M_PAGEDEP, 690 M_INODEDEP, 691 M_BMSAFEMAP, 692 M_NEWBLK, 693 M_ALLOCDIRECT, 694 M_INDIRDEP, 695 M_ALLOCINDIR, 696 M_FREEFRAG, 697 M_FREEBLKS, 698 M_FREEFILE, 699 M_DIRADD, 700 M_MKDIR, 701 M_DIRREM, 702 M_NEWDIRBLK, 703 M_FREEWORK, 704 M_FREEDEP, 705 M_JADDREF, 706 M_JREMREF, 707 M_JMVREF, 708 M_JNEWBLK, 709 M_JFREEBLK, 710 M_JFREEFRAG, 711 M_JSEG, 712 M_JSEGDEP, 713 M_SBDEP, 714 M_JTRUNC, 715 M_JFSYNC, 716 M_SENTINEL 717 }; 718 719 #define DtoM(type) (memtype[type]) 720 721 /* 722 * Names of malloc types. 723 */ 724 #define TYPENAME(type) \ 725 ((unsigned)(type) <= D_LAST ? memtype[type]->ks_shortdesc : "???") 726 /* 727 * End system adaptation definitions. 728 */ 729 730 #define DOTDOT_OFFSET offsetof(struct dirtemplate, dotdot_ino) 731 #define DOT_OFFSET offsetof(struct dirtemplate, dot_ino) 732 733 /* 734 * Internal function prototypes. 735 */ 736 static void check_clear_deps(struct mount *); 737 static void softdep_error(char *, int); 738 static int softdep_process_worklist(struct mount *, int); 739 static int softdep_waitidle(struct mount *, int); 740 static void drain_output(struct vnode *); 741 static struct buf *getdirtybuf(struct buf *, struct rwlock *, int); 742 static int check_inodedep_free(struct inodedep *); 743 static void clear_remove(struct mount *); 744 static void clear_inodedeps(struct mount *); 745 static void unlinked_inodedep(struct mount *, struct inodedep *); 746 static void clear_unlinked_inodedep(struct inodedep *); 747 static struct inodedep *first_unlinked_inodedep(struct ufsmount *); 748 static int flush_pagedep_deps(struct vnode *, struct mount *, 749 struct diraddhd *); 750 static int free_pagedep(struct pagedep *); 751 static int flush_newblk_dep(struct vnode *, struct mount *, ufs_lbn_t); 752 static int flush_inodedep_deps(struct vnode *, struct mount *, ino_t); 753 static int flush_deplist(struct allocdirectlst *, int, int *); 754 static int sync_cgs(struct mount *, int); 755 static int handle_written_filepage(struct pagedep *, struct buf *, int); 756 static int handle_written_sbdep(struct sbdep *, struct buf *); 757 static void initiate_write_sbdep(struct sbdep *); 758 static void diradd_inode_written(struct diradd *, struct inodedep *); 759 static int handle_written_indirdep(struct indirdep *, struct buf *, 760 struct buf**, int); 761 static int handle_written_inodeblock(struct inodedep *, struct buf *, int); 762 static int jnewblk_rollforward(struct jnewblk *, struct fs *, struct cg *, 763 uint8_t *); 764 static int handle_written_bmsafemap(struct bmsafemap *, struct buf *, int); 765 static void handle_written_jaddref(struct jaddref *); 766 static void handle_written_jremref(struct jremref *); 767 static void handle_written_jseg(struct jseg *, struct buf *); 768 static void handle_written_jnewblk(struct jnewblk *); 769 static void handle_written_jblkdep(struct jblkdep *); 770 static void handle_written_jfreefrag(struct jfreefrag *); 771 static void complete_jseg(struct jseg *); 772 static void complete_jsegs(struct jseg *); 773 static void jseg_write(struct ufsmount *ump, struct jseg *, uint8_t *); 774 static void jaddref_write(struct jaddref *, struct jseg *, uint8_t *); 775 static void jremref_write(struct jremref *, struct jseg *, uint8_t *); 776 static void jmvref_write(struct jmvref *, struct jseg *, uint8_t *); 777 static void jtrunc_write(struct jtrunc *, struct jseg *, uint8_t *); 778 static void jfsync_write(struct jfsync *, struct jseg *, uint8_t *data); 779 static void jnewblk_write(struct jnewblk *, struct jseg *, uint8_t *); 780 static void jfreeblk_write(struct jfreeblk *, struct jseg *, uint8_t *); 781 static void jfreefrag_write(struct jfreefrag *, struct jseg *, uint8_t *); 782 static inline void inoref_write(struct inoref *, struct jseg *, 783 struct jrefrec *); 784 static void handle_allocdirect_partdone(struct allocdirect *, 785 struct workhead *); 786 static struct jnewblk *cancel_newblk(struct newblk *, struct worklist *, 787 struct workhead *); 788 static void indirdep_complete(struct indirdep *); 789 static int indirblk_lookup(struct mount *, ufs2_daddr_t); 790 static void indirblk_insert(struct freework *); 791 static void indirblk_remove(struct freework *); 792 static void handle_allocindir_partdone(struct allocindir *); 793 static void initiate_write_filepage(struct pagedep *, struct buf *); 794 static void initiate_write_indirdep(struct indirdep*, struct buf *); 795 static void handle_written_mkdir(struct mkdir *, int); 796 static int jnewblk_rollback(struct jnewblk *, struct fs *, struct cg *, 797 uint8_t *); 798 static void initiate_write_bmsafemap(struct bmsafemap *, struct buf *); 799 static void initiate_write_inodeblock_ufs1(struct inodedep *, struct buf *); 800 static void initiate_write_inodeblock_ufs2(struct inodedep *, struct buf *); 801 static void handle_workitem_freefile(struct freefile *); 802 static int handle_workitem_remove(struct dirrem *, int); 803 static struct dirrem *newdirrem(struct buf *, struct inode *, 804 struct inode *, int, struct dirrem **); 805 static struct indirdep *indirdep_lookup(struct mount *, struct inode *, 806 struct buf *); 807 static void cancel_indirdep(struct indirdep *, struct buf *, 808 struct freeblks *); 809 static void free_indirdep(struct indirdep *); 810 static void free_diradd(struct diradd *, struct workhead *); 811 static void merge_diradd(struct inodedep *, struct diradd *); 812 static void complete_diradd(struct diradd *); 813 static struct diradd *diradd_lookup(struct pagedep *, int); 814 static struct jremref *cancel_diradd_dotdot(struct inode *, struct dirrem *, 815 struct jremref *); 816 static struct jremref *cancel_mkdir_dotdot(struct inode *, struct dirrem *, 817 struct jremref *); 818 static void cancel_diradd(struct diradd *, struct dirrem *, struct jremref *, 819 struct jremref *, struct jremref *); 820 static void dirrem_journal(struct dirrem *, struct jremref *, struct jremref *, 821 struct jremref *); 822 static void cancel_allocindir(struct allocindir *, struct buf *bp, 823 struct freeblks *, int); 824 static int setup_trunc_indir(struct freeblks *, struct inode *, 825 ufs_lbn_t, ufs_lbn_t, ufs2_daddr_t); 826 static void complete_trunc_indir(struct freework *); 827 static void trunc_indirdep(struct indirdep *, struct freeblks *, struct buf *, 828 int); 829 static void complete_mkdir(struct mkdir *); 830 static void free_newdirblk(struct newdirblk *); 831 static void free_jremref(struct jremref *); 832 static void free_jaddref(struct jaddref *); 833 static void free_jsegdep(struct jsegdep *); 834 static void free_jsegs(struct jblocks *); 835 static void rele_jseg(struct jseg *); 836 static void free_jseg(struct jseg *, struct jblocks *); 837 static void free_jnewblk(struct jnewblk *); 838 static void free_jblkdep(struct jblkdep *); 839 static void free_jfreefrag(struct jfreefrag *); 840 static void free_freedep(struct freedep *); 841 static void journal_jremref(struct dirrem *, struct jremref *, 842 struct inodedep *); 843 static void cancel_jnewblk(struct jnewblk *, struct workhead *); 844 static int cancel_jaddref(struct jaddref *, struct inodedep *, 845 struct workhead *); 846 static void cancel_jfreefrag(struct jfreefrag *); 847 static inline void setup_freedirect(struct freeblks *, struct inode *, 848 int, int); 849 static inline void setup_freeext(struct freeblks *, struct inode *, int, int); 850 static inline void setup_freeindir(struct freeblks *, struct inode *, int, 851 ufs_lbn_t, int); 852 static inline struct freeblks *newfreeblks(struct mount *, struct inode *); 853 static void freeblks_free(struct ufsmount *, struct freeblks *, int); 854 static void indir_trunc(struct freework *, ufs2_daddr_t, ufs_lbn_t); 855 static ufs2_daddr_t blkcount(struct fs *, ufs2_daddr_t, off_t); 856 static int trunc_check_buf(struct buf *, int *, ufs_lbn_t, int, int); 857 static void trunc_dependencies(struct inode *, struct freeblks *, ufs_lbn_t, 858 int, int); 859 static void trunc_pages(struct inode *, off_t, ufs2_daddr_t, int); 860 static int cancel_pagedep(struct pagedep *, struct freeblks *, int); 861 static int deallocate_dependencies(struct buf *, struct freeblks *, int); 862 static void newblk_freefrag(struct newblk*); 863 static void free_newblk(struct newblk *); 864 static void cancel_allocdirect(struct allocdirectlst *, 865 struct allocdirect *, struct freeblks *); 866 static int check_inode_unwritten(struct inodedep *); 867 static int free_inodedep(struct inodedep *); 868 static void freework_freeblock(struct freework *); 869 static void freework_enqueue(struct freework *); 870 static int handle_workitem_freeblocks(struct freeblks *, int); 871 static int handle_complete_freeblocks(struct freeblks *, int); 872 static void handle_workitem_indirblk(struct freework *); 873 static void handle_written_freework(struct freework *); 874 static void merge_inode_lists(struct allocdirectlst *,struct allocdirectlst *); 875 static struct worklist *jnewblk_merge(struct worklist *, struct worklist *, 876 struct workhead *); 877 static struct freefrag *setup_allocindir_phase2(struct buf *, struct inode *, 878 struct inodedep *, struct allocindir *, ufs_lbn_t); 879 static struct allocindir *newallocindir(struct inode *, int, ufs2_daddr_t, 880 ufs2_daddr_t, ufs_lbn_t); 881 static void handle_workitem_freefrag(struct freefrag *); 882 static struct freefrag *newfreefrag(struct inode *, ufs2_daddr_t, long, 883 ufs_lbn_t); 884 static void allocdirect_merge(struct allocdirectlst *, 885 struct allocdirect *, struct allocdirect *); 886 static struct freefrag *allocindir_merge(struct allocindir *, 887 struct allocindir *); 888 static int bmsafemap_find(struct bmsafemap_hashhead *, int, 889 struct bmsafemap **); 890 static struct bmsafemap *bmsafemap_lookup(struct mount *, struct buf *, 891 int cg, struct bmsafemap *); 892 static int newblk_find(struct newblk_hashhead *, ufs2_daddr_t, int, 893 struct newblk **); 894 static int newblk_lookup(struct mount *, ufs2_daddr_t, int, struct newblk **); 895 static int inodedep_find(struct inodedep_hashhead *, ino_t, 896 struct inodedep **); 897 static int inodedep_lookup(struct mount *, ino_t, int, struct inodedep **); 898 static int pagedep_lookup(struct mount *, struct buf *bp, ino_t, ufs_lbn_t, 899 int, struct pagedep **); 900 static int pagedep_find(struct pagedep_hashhead *, ino_t, ufs_lbn_t, 901 struct pagedep **); 902 static void pause_timer(void *); 903 static int request_cleanup(struct mount *, int); 904 static void schedule_cleanup(struct mount *); 905 static void softdep_ast_cleanup_proc(void); 906 static int process_worklist_item(struct mount *, int, int); 907 static void process_removes(struct vnode *); 908 static void process_truncates(struct vnode *); 909 static void jwork_move(struct workhead *, struct workhead *); 910 static void jwork_insert(struct workhead *, struct jsegdep *); 911 static void add_to_worklist(struct worklist *, int); 912 static void wake_worklist(struct worklist *); 913 static void wait_worklist(struct worklist *, char *); 914 static void remove_from_worklist(struct worklist *); 915 static void softdep_flush(void *); 916 static void softdep_flushjournal(struct mount *); 917 static int softdep_speedup(struct ufsmount *); 918 static void worklist_speedup(struct mount *); 919 static int journal_mount(struct mount *, struct fs *, struct ucred *); 920 static void journal_unmount(struct ufsmount *); 921 static int journal_space(struct ufsmount *, int); 922 static void journal_suspend(struct ufsmount *); 923 static int journal_unsuspend(struct ufsmount *ump); 924 static void softdep_prelink(struct vnode *, struct vnode *); 925 static void add_to_journal(struct worklist *); 926 static void remove_from_journal(struct worklist *); 927 static bool softdep_excess_items(struct ufsmount *, int); 928 static void softdep_process_journal(struct mount *, struct worklist *, int); 929 static struct jremref *newjremref(struct dirrem *, struct inode *, 930 struct inode *ip, off_t, nlink_t); 931 static struct jaddref *newjaddref(struct inode *, ino_t, off_t, int16_t, 932 uint16_t); 933 static inline void newinoref(struct inoref *, ino_t, ino_t, off_t, nlink_t, 934 uint16_t); 935 static inline struct jsegdep *inoref_jseg(struct inoref *); 936 static struct jmvref *newjmvref(struct inode *, ino_t, off_t, off_t); 937 static struct jfreeblk *newjfreeblk(struct freeblks *, ufs_lbn_t, 938 ufs2_daddr_t, int); 939 static void adjust_newfreework(struct freeblks *, int); 940 static struct jtrunc *newjtrunc(struct freeblks *, off_t, int); 941 static void move_newblock_dep(struct jaddref *, struct inodedep *); 942 static void cancel_jfreeblk(struct freeblks *, ufs2_daddr_t); 943 static struct jfreefrag *newjfreefrag(struct freefrag *, struct inode *, 944 ufs2_daddr_t, long, ufs_lbn_t); 945 static struct freework *newfreework(struct ufsmount *, struct freeblks *, 946 struct freework *, ufs_lbn_t, ufs2_daddr_t, int, int, int); 947 static int jwait(struct worklist *, int); 948 static struct inodedep *inodedep_lookup_ip(struct inode *); 949 static int bmsafemap_backgroundwrite(struct bmsafemap *, struct buf *); 950 static struct freefile *handle_bufwait(struct inodedep *, struct workhead *); 951 static void handle_jwork(struct workhead *); 952 static struct mkdir *setup_newdir(struct diradd *, ino_t, ino_t, struct buf *, 953 struct mkdir **); 954 static struct jblocks *jblocks_create(void); 955 static ufs2_daddr_t jblocks_alloc(struct jblocks *, int, int *); 956 static void jblocks_free(struct jblocks *, struct mount *, int); 957 static void jblocks_destroy(struct jblocks *); 958 static void jblocks_add(struct jblocks *, ufs2_daddr_t, int); 959 960 /* 961 * Exported softdep operations. 962 */ 963 static void softdep_disk_io_initiation(struct buf *); 964 static void softdep_disk_write_complete(struct buf *); 965 static void softdep_deallocate_dependencies(struct buf *); 966 static int softdep_count_dependencies(struct buf *bp, int); 967 968 /* 969 * Global lock over all of soft updates. 970 */ 971 static struct mtx lk; 972 MTX_SYSINIT(softdep_lock, &lk, "Global Softdep Lock", MTX_DEF); 973 974 #define ACQUIRE_GBLLOCK(lk) mtx_lock(lk) 975 #define FREE_GBLLOCK(lk) mtx_unlock(lk) 976 #define GBLLOCK_OWNED(lk) mtx_assert((lk), MA_OWNED) 977 978 /* 979 * Per-filesystem soft-updates locking. 980 */ 981 #define LOCK_PTR(ump) (&(ump)->um_softdep->sd_fslock) 982 #define TRY_ACQUIRE_LOCK(ump) rw_try_wlock(&(ump)->um_softdep->sd_fslock) 983 #define ACQUIRE_LOCK(ump) rw_wlock(&(ump)->um_softdep->sd_fslock) 984 #define FREE_LOCK(ump) rw_wunlock(&(ump)->um_softdep->sd_fslock) 985 #define LOCK_OWNED(ump) rw_assert(&(ump)->um_softdep->sd_fslock, \ 986 RA_WLOCKED) 987 988 #define BUF_AREC(bp) lockallowrecurse(&(bp)->b_lock) 989 #define BUF_NOREC(bp) lockdisablerecurse(&(bp)->b_lock) 990 991 /* 992 * Worklist queue management. 993 * These routines require that the lock be held. 994 */ 995 #ifndef /* NOT */ DEBUG 996 #define WORKLIST_INSERT(head, item) do { \ 997 (item)->wk_state |= ONWORKLIST; \ 998 LIST_INSERT_HEAD(head, item, wk_list); \ 999 } while (0) 1000 #define WORKLIST_REMOVE(item) do { \ 1001 (item)->wk_state &= ~ONWORKLIST; \ 1002 LIST_REMOVE(item, wk_list); \ 1003 } while (0) 1004 #define WORKLIST_INSERT_UNLOCKED WORKLIST_INSERT 1005 #define WORKLIST_REMOVE_UNLOCKED WORKLIST_REMOVE 1006 1007 #else /* DEBUG */ 1008 static void worklist_insert(struct workhead *, struct worklist *, int); 1009 static void worklist_remove(struct worklist *, int); 1010 1011 #define WORKLIST_INSERT(head, item) worklist_insert(head, item, 1) 1012 #define WORKLIST_INSERT_UNLOCKED(head, item) worklist_insert(head, item, 0) 1013 #define WORKLIST_REMOVE(item) worklist_remove(item, 1) 1014 #define WORKLIST_REMOVE_UNLOCKED(item) worklist_remove(item, 0) 1015 1016 static void 1017 worklist_insert(head, item, locked) 1018 struct workhead *head; 1019 struct worklist *item; 1020 int locked; 1021 { 1022 1023 if (locked) 1024 LOCK_OWNED(VFSTOUFS(item->wk_mp)); 1025 if (item->wk_state & ONWORKLIST) 1026 panic("worklist_insert: %p %s(0x%X) already on list", 1027 item, TYPENAME(item->wk_type), item->wk_state); 1028 item->wk_state |= ONWORKLIST; 1029 LIST_INSERT_HEAD(head, item, wk_list); 1030 } 1031 1032 static void 1033 worklist_remove(item, locked) 1034 struct worklist *item; 1035 int locked; 1036 { 1037 1038 if (locked) 1039 LOCK_OWNED(VFSTOUFS(item->wk_mp)); 1040 if ((item->wk_state & ONWORKLIST) == 0) 1041 panic("worklist_remove: %p %s(0x%X) not on list", 1042 item, TYPENAME(item->wk_type), item->wk_state); 1043 item->wk_state &= ~ONWORKLIST; 1044 LIST_REMOVE(item, wk_list); 1045 } 1046 #endif /* DEBUG */ 1047 1048 /* 1049 * Merge two jsegdeps keeping only the oldest one as newer references 1050 * can't be discarded until after older references. 1051 */ 1052 static inline struct jsegdep * 1053 jsegdep_merge(struct jsegdep *one, struct jsegdep *two) 1054 { 1055 struct jsegdep *swp; 1056 1057 if (two == NULL) 1058 return (one); 1059 1060 if (one->jd_seg->js_seq > two->jd_seg->js_seq) { 1061 swp = one; 1062 one = two; 1063 two = swp; 1064 } 1065 WORKLIST_REMOVE(&two->jd_list); 1066 free_jsegdep(two); 1067 1068 return (one); 1069 } 1070 1071 /* 1072 * If two freedeps are compatible free one to reduce list size. 1073 */ 1074 static inline struct freedep * 1075 freedep_merge(struct freedep *one, struct freedep *two) 1076 { 1077 if (two == NULL) 1078 return (one); 1079 1080 if (one->fd_freework == two->fd_freework) { 1081 WORKLIST_REMOVE(&two->fd_list); 1082 free_freedep(two); 1083 } 1084 return (one); 1085 } 1086 1087 /* 1088 * Move journal work from one list to another. Duplicate freedeps and 1089 * jsegdeps are coalesced to keep the lists as small as possible. 1090 */ 1091 static void 1092 jwork_move(dst, src) 1093 struct workhead *dst; 1094 struct workhead *src; 1095 { 1096 struct freedep *freedep; 1097 struct jsegdep *jsegdep; 1098 struct worklist *wkn; 1099 struct worklist *wk; 1100 1101 KASSERT(dst != src, 1102 ("jwork_move: dst == src")); 1103 freedep = NULL; 1104 jsegdep = NULL; 1105 LIST_FOREACH_SAFE(wk, dst, wk_list, wkn) { 1106 if (wk->wk_type == D_JSEGDEP) 1107 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1108 else if (wk->wk_type == D_FREEDEP) 1109 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1110 } 1111 1112 while ((wk = LIST_FIRST(src)) != NULL) { 1113 WORKLIST_REMOVE(wk); 1114 WORKLIST_INSERT(dst, wk); 1115 if (wk->wk_type == D_JSEGDEP) { 1116 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1117 continue; 1118 } 1119 if (wk->wk_type == D_FREEDEP) 1120 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1121 } 1122 } 1123 1124 static void 1125 jwork_insert(dst, jsegdep) 1126 struct workhead *dst; 1127 struct jsegdep *jsegdep; 1128 { 1129 struct jsegdep *jsegdepn; 1130 struct worklist *wk; 1131 1132 LIST_FOREACH(wk, dst, wk_list) 1133 if (wk->wk_type == D_JSEGDEP) 1134 break; 1135 if (wk == NULL) { 1136 WORKLIST_INSERT(dst, &jsegdep->jd_list); 1137 return; 1138 } 1139 jsegdepn = WK_JSEGDEP(wk); 1140 if (jsegdep->jd_seg->js_seq < jsegdepn->jd_seg->js_seq) { 1141 WORKLIST_REMOVE(wk); 1142 free_jsegdep(jsegdepn); 1143 WORKLIST_INSERT(dst, &jsegdep->jd_list); 1144 } else 1145 free_jsegdep(jsegdep); 1146 } 1147 1148 /* 1149 * Routines for tracking and managing workitems. 1150 */ 1151 static void workitem_free(struct worklist *, int); 1152 static void workitem_alloc(struct worklist *, int, struct mount *); 1153 static void workitem_reassign(struct worklist *, int); 1154 1155 #define WORKITEM_FREE(item, type) \ 1156 workitem_free((struct worklist *)(item), (type)) 1157 #define WORKITEM_REASSIGN(item, type) \ 1158 workitem_reassign((struct worklist *)(item), (type)) 1159 1160 static void 1161 workitem_free(item, type) 1162 struct worklist *item; 1163 int type; 1164 { 1165 struct ufsmount *ump; 1166 1167 #ifdef DEBUG 1168 if (item->wk_state & ONWORKLIST) 1169 panic("workitem_free: %s(0x%X) still on list", 1170 TYPENAME(item->wk_type), item->wk_state); 1171 if (item->wk_type != type && type != D_NEWBLK) 1172 panic("workitem_free: type mismatch %s != %s", 1173 TYPENAME(item->wk_type), TYPENAME(type)); 1174 #endif 1175 if (item->wk_state & IOWAITING) 1176 wakeup(item); 1177 ump = VFSTOUFS(item->wk_mp); 1178 LOCK_OWNED(ump); 1179 KASSERT(ump->softdep_deps > 0, 1180 ("workitem_free: %s: softdep_deps going negative", 1181 ump->um_fs->fs_fsmnt)); 1182 if (--ump->softdep_deps == 0 && ump->softdep_req) 1183 wakeup(&ump->softdep_deps); 1184 KASSERT(dep_current[item->wk_type] > 0, 1185 ("workitem_free: %s: dep_current[%s] going negative", 1186 ump->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1187 KASSERT(ump->softdep_curdeps[item->wk_type] > 0, 1188 ("workitem_free: %s: softdep_curdeps[%s] going negative", 1189 ump->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1190 atomic_subtract_long(&dep_current[item->wk_type], 1); 1191 ump->softdep_curdeps[item->wk_type] -= 1; 1192 free(item, DtoM(type)); 1193 } 1194 1195 static void 1196 workitem_alloc(item, type, mp) 1197 struct worklist *item; 1198 int type; 1199 struct mount *mp; 1200 { 1201 struct ufsmount *ump; 1202 1203 item->wk_type = type; 1204 item->wk_mp = mp; 1205 item->wk_state = 0; 1206 1207 ump = VFSTOUFS(mp); 1208 ACQUIRE_GBLLOCK(&lk); 1209 dep_current[type]++; 1210 if (dep_current[type] > dep_highuse[type]) 1211 dep_highuse[type] = dep_current[type]; 1212 dep_total[type]++; 1213 FREE_GBLLOCK(&lk); 1214 ACQUIRE_LOCK(ump); 1215 ump->softdep_curdeps[type] += 1; 1216 ump->softdep_deps++; 1217 ump->softdep_accdeps++; 1218 FREE_LOCK(ump); 1219 } 1220 1221 static void 1222 workitem_reassign(item, newtype) 1223 struct worklist *item; 1224 int newtype; 1225 { 1226 struct ufsmount *ump; 1227 1228 ump = VFSTOUFS(item->wk_mp); 1229 LOCK_OWNED(ump); 1230 KASSERT(ump->softdep_curdeps[item->wk_type] > 0, 1231 ("workitem_reassign: %s: softdep_curdeps[%s] going negative", 1232 VFSTOUFS(item->wk_mp)->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1233 ump->softdep_curdeps[item->wk_type] -= 1; 1234 ump->softdep_curdeps[newtype] += 1; 1235 KASSERT(dep_current[item->wk_type] > 0, 1236 ("workitem_reassign: %s: dep_current[%s] going negative", 1237 VFSTOUFS(item->wk_mp)->um_fs->fs_fsmnt, TYPENAME(item->wk_type))); 1238 ACQUIRE_GBLLOCK(&lk); 1239 dep_current[newtype]++; 1240 dep_current[item->wk_type]--; 1241 if (dep_current[newtype] > dep_highuse[newtype]) 1242 dep_highuse[newtype] = dep_current[newtype]; 1243 dep_total[newtype]++; 1244 FREE_GBLLOCK(&lk); 1245 item->wk_type = newtype; 1246 } 1247 1248 /* 1249 * Workitem queue management 1250 */ 1251 static int max_softdeps; /* maximum number of structs before slowdown */ 1252 static int tickdelay = 2; /* number of ticks to pause during slowdown */ 1253 static int proc_waiting; /* tracks whether we have a timeout posted */ 1254 static int *stat_countp; /* statistic to count in proc_waiting timeout */ 1255 static struct callout softdep_callout; 1256 static int req_clear_inodedeps; /* syncer process flush some inodedeps */ 1257 static int req_clear_remove; /* syncer process flush some freeblks */ 1258 static int softdep_flushcache = 0; /* Should we do BIO_FLUSH? */ 1259 1260 /* 1261 * runtime statistics 1262 */ 1263 static int stat_flush_threads; /* number of softdep flushing threads */ 1264 static int stat_worklist_push; /* number of worklist cleanups */ 1265 static int stat_blk_limit_push; /* number of times block limit neared */ 1266 static int stat_ino_limit_push; /* number of times inode limit neared */ 1267 static int stat_blk_limit_hit; /* number of times block slowdown imposed */ 1268 static int stat_ino_limit_hit; /* number of times inode slowdown imposed */ 1269 static int stat_sync_limit_hit; /* number of synchronous slowdowns imposed */ 1270 static int stat_indir_blk_ptrs; /* bufs redirtied as indir ptrs not written */ 1271 static int stat_inode_bitmap; /* bufs redirtied as inode bitmap not written */ 1272 static int stat_direct_blk_ptrs;/* bufs redirtied as direct ptrs not written */ 1273 static int stat_dir_entry; /* bufs redirtied as dir entry cannot write */ 1274 static int stat_jaddref; /* bufs redirtied as ino bitmap can not write */ 1275 static int stat_jnewblk; /* bufs redirtied as blk bitmap can not write */ 1276 static int stat_journal_min; /* Times hit journal min threshold */ 1277 static int stat_journal_low; /* Times hit journal low threshold */ 1278 static int stat_journal_wait; /* Times blocked in jwait(). */ 1279 static int stat_jwait_filepage; /* Times blocked in jwait() for filepage. */ 1280 static int stat_jwait_freeblks; /* Times blocked in jwait() for freeblks. */ 1281 static int stat_jwait_inode; /* Times blocked in jwait() for inodes. */ 1282 static int stat_jwait_newblk; /* Times blocked in jwait() for newblks. */ 1283 static int stat_cleanup_high_delay; /* Maximum cleanup delay (in ticks) */ 1284 static int stat_cleanup_blkrequests; /* Number of block cleanup requests */ 1285 static int stat_cleanup_inorequests; /* Number of inode cleanup requests */ 1286 static int stat_cleanup_retries; /* Number of cleanups that needed to flush */ 1287 static int stat_cleanup_failures; /* Number of cleanup requests that failed */ 1288 static int stat_emptyjblocks; /* Number of potentially empty journal blocks */ 1289 1290 SYSCTL_INT(_debug_softdep, OID_AUTO, max_softdeps, CTLFLAG_RW, 1291 &max_softdeps, 0, ""); 1292 SYSCTL_INT(_debug_softdep, OID_AUTO, tickdelay, CTLFLAG_RW, 1293 &tickdelay, 0, ""); 1294 SYSCTL_INT(_debug_softdep, OID_AUTO, flush_threads, CTLFLAG_RD, 1295 &stat_flush_threads, 0, ""); 1296 SYSCTL_INT(_debug_softdep, OID_AUTO, worklist_push, CTLFLAG_RW, 1297 &stat_worklist_push, 0,""); 1298 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_push, CTLFLAG_RW, 1299 &stat_blk_limit_push, 0,""); 1300 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_push, CTLFLAG_RW, 1301 &stat_ino_limit_push, 0,""); 1302 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_hit, CTLFLAG_RW, 1303 &stat_blk_limit_hit, 0, ""); 1304 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_hit, CTLFLAG_RW, 1305 &stat_ino_limit_hit, 0, ""); 1306 SYSCTL_INT(_debug_softdep, OID_AUTO, sync_limit_hit, CTLFLAG_RW, 1307 &stat_sync_limit_hit, 0, ""); 1308 SYSCTL_INT(_debug_softdep, OID_AUTO, indir_blk_ptrs, CTLFLAG_RW, 1309 &stat_indir_blk_ptrs, 0, ""); 1310 SYSCTL_INT(_debug_softdep, OID_AUTO, inode_bitmap, CTLFLAG_RW, 1311 &stat_inode_bitmap, 0, ""); 1312 SYSCTL_INT(_debug_softdep, OID_AUTO, direct_blk_ptrs, CTLFLAG_RW, 1313 &stat_direct_blk_ptrs, 0, ""); 1314 SYSCTL_INT(_debug_softdep, OID_AUTO, dir_entry, CTLFLAG_RW, 1315 &stat_dir_entry, 0, ""); 1316 SYSCTL_INT(_debug_softdep, OID_AUTO, jaddref_rollback, CTLFLAG_RW, 1317 &stat_jaddref, 0, ""); 1318 SYSCTL_INT(_debug_softdep, OID_AUTO, jnewblk_rollback, CTLFLAG_RW, 1319 &stat_jnewblk, 0, ""); 1320 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_low, CTLFLAG_RW, 1321 &stat_journal_low, 0, ""); 1322 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_min, CTLFLAG_RW, 1323 &stat_journal_min, 0, ""); 1324 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_wait, CTLFLAG_RW, 1325 &stat_journal_wait, 0, ""); 1326 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_filepage, CTLFLAG_RW, 1327 &stat_jwait_filepage, 0, ""); 1328 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_freeblks, CTLFLAG_RW, 1329 &stat_jwait_freeblks, 0, ""); 1330 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_inode, CTLFLAG_RW, 1331 &stat_jwait_inode, 0, ""); 1332 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_newblk, CTLFLAG_RW, 1333 &stat_jwait_newblk, 0, ""); 1334 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_blkrequests, CTLFLAG_RW, 1335 &stat_cleanup_blkrequests, 0, ""); 1336 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_inorequests, CTLFLAG_RW, 1337 &stat_cleanup_inorequests, 0, ""); 1338 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_high_delay, CTLFLAG_RW, 1339 &stat_cleanup_high_delay, 0, ""); 1340 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_retries, CTLFLAG_RW, 1341 &stat_cleanup_retries, 0, ""); 1342 SYSCTL_INT(_debug_softdep, OID_AUTO, cleanup_failures, CTLFLAG_RW, 1343 &stat_cleanup_failures, 0, ""); 1344 SYSCTL_INT(_debug_softdep, OID_AUTO, flushcache, CTLFLAG_RW, 1345 &softdep_flushcache, 0, ""); 1346 SYSCTL_INT(_debug_softdep, OID_AUTO, emptyjblocks, CTLFLAG_RD, 1347 &stat_emptyjblocks, 0, ""); 1348 1349 SYSCTL_DECL(_vfs_ffs); 1350 1351 /* Whether to recompute the summary at mount time */ 1352 static int compute_summary_at_mount = 0; 1353 SYSCTL_INT(_vfs_ffs, OID_AUTO, compute_summary_at_mount, CTLFLAG_RW, 1354 &compute_summary_at_mount, 0, "Recompute summary at mount"); 1355 static int print_threads = 0; 1356 SYSCTL_INT(_debug_softdep, OID_AUTO, print_threads, CTLFLAG_RW, 1357 &print_threads, 0, "Notify flusher thread start/stop"); 1358 1359 /* List of all filesystems mounted with soft updates */ 1360 static TAILQ_HEAD(, mount_softdeps) softdepmounts; 1361 1362 /* 1363 * This function cleans the worklist for a filesystem. 1364 * Each filesystem running with soft dependencies gets its own 1365 * thread to run in this function. The thread is started up in 1366 * softdep_mount and shutdown in softdep_unmount. They show up 1367 * as part of the kernel "bufdaemon" process whose process 1368 * entry is available in bufdaemonproc. 1369 */ 1370 static int searchfailed; 1371 extern struct proc *bufdaemonproc; 1372 static void 1373 softdep_flush(addr) 1374 void *addr; 1375 { 1376 struct mount *mp; 1377 struct thread *td; 1378 struct ufsmount *ump; 1379 1380 td = curthread; 1381 td->td_pflags |= TDP_NORUNNINGBUF; 1382 mp = (struct mount *)addr; 1383 ump = VFSTOUFS(mp); 1384 atomic_add_int(&stat_flush_threads, 1); 1385 ACQUIRE_LOCK(ump); 1386 ump->softdep_flags &= ~FLUSH_STARTING; 1387 wakeup(&ump->softdep_flushtd); 1388 FREE_LOCK(ump); 1389 if (print_threads) { 1390 if (stat_flush_threads == 1) 1391 printf("Running %s at pid %d\n", bufdaemonproc->p_comm, 1392 bufdaemonproc->p_pid); 1393 printf("Start thread %s\n", td->td_name); 1394 } 1395 for (;;) { 1396 while (softdep_process_worklist(mp, 0) > 0 || 1397 (MOUNTEDSUJ(mp) && 1398 VFSTOUFS(mp)->softdep_jblocks->jb_suspended)) 1399 kthread_suspend_check(); 1400 ACQUIRE_LOCK(ump); 1401 if ((ump->softdep_flags & (FLUSH_CLEANUP | FLUSH_EXIT)) == 0) 1402 msleep(&ump->softdep_flushtd, LOCK_PTR(ump), PVM, 1403 "sdflush", hz / 2); 1404 ump->softdep_flags &= ~FLUSH_CLEANUP; 1405 /* 1406 * Check to see if we are done and need to exit. 1407 */ 1408 if ((ump->softdep_flags & FLUSH_EXIT) == 0) { 1409 FREE_LOCK(ump); 1410 continue; 1411 } 1412 ump->softdep_flags &= ~FLUSH_EXIT; 1413 FREE_LOCK(ump); 1414 wakeup(&ump->softdep_flags); 1415 if (print_threads) 1416 printf("Stop thread %s: searchfailed %d, did cleanups %d\n", td->td_name, searchfailed, ump->um_softdep->sd_cleanups); 1417 atomic_subtract_int(&stat_flush_threads, 1); 1418 kthread_exit(); 1419 panic("kthread_exit failed\n"); 1420 } 1421 } 1422 1423 static void 1424 worklist_speedup(mp) 1425 struct mount *mp; 1426 { 1427 struct ufsmount *ump; 1428 1429 ump = VFSTOUFS(mp); 1430 LOCK_OWNED(ump); 1431 if ((ump->softdep_flags & (FLUSH_CLEANUP | FLUSH_EXIT)) == 0) 1432 ump->softdep_flags |= FLUSH_CLEANUP; 1433 wakeup(&ump->softdep_flushtd); 1434 } 1435 1436 static int 1437 softdep_speedup(ump) 1438 struct ufsmount *ump; 1439 { 1440 struct ufsmount *altump; 1441 struct mount_softdeps *sdp; 1442 1443 LOCK_OWNED(ump); 1444 worklist_speedup(ump->um_mountp); 1445 bd_speedup(); 1446 /* 1447 * If we have global shortages, then we need other 1448 * filesystems to help with the cleanup. Here we wakeup a 1449 * flusher thread for a filesystem that is over its fair 1450 * share of resources. 1451 */ 1452 if (req_clear_inodedeps || req_clear_remove) { 1453 ACQUIRE_GBLLOCK(&lk); 1454 TAILQ_FOREACH(sdp, &softdepmounts, sd_next) { 1455 if ((altump = sdp->sd_ump) == ump) 1456 continue; 1457 if (((req_clear_inodedeps && 1458 altump->softdep_curdeps[D_INODEDEP] > 1459 max_softdeps / stat_flush_threads) || 1460 (req_clear_remove && 1461 altump->softdep_curdeps[D_DIRREM] > 1462 (max_softdeps / 2) / stat_flush_threads)) && 1463 TRY_ACQUIRE_LOCK(altump)) 1464 break; 1465 } 1466 if (sdp == NULL) { 1467 searchfailed++; 1468 FREE_GBLLOCK(&lk); 1469 } else { 1470 /* 1471 * Move to the end of the list so we pick a 1472 * different one on out next try. 1473 */ 1474 TAILQ_REMOVE(&softdepmounts, sdp, sd_next); 1475 TAILQ_INSERT_TAIL(&softdepmounts, sdp, sd_next); 1476 FREE_GBLLOCK(&lk); 1477 if ((altump->softdep_flags & 1478 (FLUSH_CLEANUP | FLUSH_EXIT)) == 0) 1479 altump->softdep_flags |= FLUSH_CLEANUP; 1480 altump->um_softdep->sd_cleanups++; 1481 wakeup(&altump->softdep_flushtd); 1482 FREE_LOCK(altump); 1483 } 1484 } 1485 return (speedup_syncer()); 1486 } 1487 1488 /* 1489 * Add an item to the end of the work queue. 1490 * This routine requires that the lock be held. 1491 * This is the only routine that adds items to the list. 1492 * The following routine is the only one that removes items 1493 * and does so in order from first to last. 1494 */ 1495 1496 #define WK_HEAD 0x0001 /* Add to HEAD. */ 1497 #define WK_NODELAY 0x0002 /* Process immediately. */ 1498 1499 static void 1500 add_to_worklist(wk, flags) 1501 struct worklist *wk; 1502 int flags; 1503 { 1504 struct ufsmount *ump; 1505 1506 ump = VFSTOUFS(wk->wk_mp); 1507 LOCK_OWNED(ump); 1508 if (wk->wk_state & ONWORKLIST) 1509 panic("add_to_worklist: %s(0x%X) already on list", 1510 TYPENAME(wk->wk_type), wk->wk_state); 1511 wk->wk_state |= ONWORKLIST; 1512 if (ump->softdep_on_worklist == 0) { 1513 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list); 1514 ump->softdep_worklist_tail = wk; 1515 } else if (flags & WK_HEAD) { 1516 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list); 1517 } else { 1518 LIST_INSERT_AFTER(ump->softdep_worklist_tail, wk, wk_list); 1519 ump->softdep_worklist_tail = wk; 1520 } 1521 ump->softdep_on_worklist += 1; 1522 if (flags & WK_NODELAY) 1523 worklist_speedup(wk->wk_mp); 1524 } 1525 1526 /* 1527 * Remove the item to be processed. If we are removing the last 1528 * item on the list, we need to recalculate the tail pointer. 1529 */ 1530 static void 1531 remove_from_worklist(wk) 1532 struct worklist *wk; 1533 { 1534 struct ufsmount *ump; 1535 1536 ump = VFSTOUFS(wk->wk_mp); 1537 WORKLIST_REMOVE(wk); 1538 if (ump->softdep_worklist_tail == wk) 1539 ump->softdep_worklist_tail = 1540 (struct worklist *)wk->wk_list.le_prev; 1541 ump->softdep_on_worklist -= 1; 1542 } 1543 1544 static void 1545 wake_worklist(wk) 1546 struct worklist *wk; 1547 { 1548 if (wk->wk_state & IOWAITING) { 1549 wk->wk_state &= ~IOWAITING; 1550 wakeup(wk); 1551 } 1552 } 1553 1554 static void 1555 wait_worklist(wk, wmesg) 1556 struct worklist *wk; 1557 char *wmesg; 1558 { 1559 struct ufsmount *ump; 1560 1561 ump = VFSTOUFS(wk->wk_mp); 1562 wk->wk_state |= IOWAITING; 1563 msleep(wk, LOCK_PTR(ump), PVM, wmesg, 0); 1564 } 1565 1566 /* 1567 * Process that runs once per second to handle items in the background queue. 1568 * 1569 * Note that we ensure that everything is done in the order in which they 1570 * appear in the queue. The code below depends on this property to ensure 1571 * that blocks of a file are freed before the inode itself is freed. This 1572 * ordering ensures that no new <vfsid, inum, lbn> triples will be generated 1573 * until all the old ones have been purged from the dependency lists. 1574 */ 1575 static int 1576 softdep_process_worklist(mp, full) 1577 struct mount *mp; 1578 int full; 1579 { 1580 int cnt, matchcnt; 1581 struct ufsmount *ump; 1582 long starttime; 1583 1584 KASSERT(mp != NULL, ("softdep_process_worklist: NULL mp")); 1585 if (MOUNTEDSOFTDEP(mp) == 0) 1586 return (0); 1587 matchcnt = 0; 1588 ump = VFSTOUFS(mp); 1589 ACQUIRE_LOCK(ump); 1590 starttime = time_second; 1591 softdep_process_journal(mp, NULL, full ? MNT_WAIT : 0); 1592 check_clear_deps(mp); 1593 while (ump->softdep_on_worklist > 0) { 1594 if ((cnt = process_worklist_item(mp, 10, LK_NOWAIT)) == 0) 1595 break; 1596 else 1597 matchcnt += cnt; 1598 check_clear_deps(mp); 1599 /* 1600 * We do not generally want to stop for buffer space, but if 1601 * we are really being a buffer hog, we will stop and wait. 1602 */ 1603 if (should_yield()) { 1604 FREE_LOCK(ump); 1605 kern_yield(PRI_USER); 1606 bwillwrite(); 1607 ACQUIRE_LOCK(ump); 1608 } 1609 /* 1610 * Never allow processing to run for more than one 1611 * second. This gives the syncer thread the opportunity 1612 * to pause if appropriate. 1613 */ 1614 if (!full && starttime != time_second) 1615 break; 1616 } 1617 if (full == 0) 1618 journal_unsuspend(ump); 1619 FREE_LOCK(ump); 1620 return (matchcnt); 1621 } 1622 1623 /* 1624 * Process all removes associated with a vnode if we are running out of 1625 * journal space. Any other process which attempts to flush these will 1626 * be unable as we have the vnodes locked. 1627 */ 1628 static void 1629 process_removes(vp) 1630 struct vnode *vp; 1631 { 1632 struct inodedep *inodedep; 1633 struct dirrem *dirrem; 1634 struct ufsmount *ump; 1635 struct mount *mp; 1636 ino_t inum; 1637 1638 mp = vp->v_mount; 1639 ump = VFSTOUFS(mp); 1640 LOCK_OWNED(ump); 1641 inum = VTOI(vp)->i_number; 1642 for (;;) { 1643 top: 1644 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0) 1645 return; 1646 LIST_FOREACH(dirrem, &inodedep->id_dirremhd, dm_inonext) { 1647 /* 1648 * If another thread is trying to lock this vnode 1649 * it will fail but we must wait for it to do so 1650 * before we can proceed. 1651 */ 1652 if (dirrem->dm_state & INPROGRESS) { 1653 wait_worklist(&dirrem->dm_list, "pwrwait"); 1654 goto top; 1655 } 1656 if ((dirrem->dm_state & (COMPLETE | ONWORKLIST)) == 1657 (COMPLETE | ONWORKLIST)) 1658 break; 1659 } 1660 if (dirrem == NULL) 1661 return; 1662 remove_from_worklist(&dirrem->dm_list); 1663 FREE_LOCK(ump); 1664 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1665 panic("process_removes: suspended filesystem"); 1666 handle_workitem_remove(dirrem, 0); 1667 vn_finished_secondary_write(mp); 1668 ACQUIRE_LOCK(ump); 1669 } 1670 } 1671 1672 /* 1673 * Process all truncations associated with a vnode if we are running out 1674 * of journal space. This is called when the vnode lock is already held 1675 * and no other process can clear the truncation. This function returns 1676 * a value greater than zero if it did any work. 1677 */ 1678 static void 1679 process_truncates(vp) 1680 struct vnode *vp; 1681 { 1682 struct inodedep *inodedep; 1683 struct freeblks *freeblks; 1684 struct ufsmount *ump; 1685 struct mount *mp; 1686 ino_t inum; 1687 int cgwait; 1688 1689 mp = vp->v_mount; 1690 ump = VFSTOUFS(mp); 1691 LOCK_OWNED(ump); 1692 inum = VTOI(vp)->i_number; 1693 for (;;) { 1694 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0) 1695 return; 1696 cgwait = 0; 1697 TAILQ_FOREACH(freeblks, &inodedep->id_freeblklst, fb_next) { 1698 /* Journal entries not yet written. */ 1699 if (!LIST_EMPTY(&freeblks->fb_jblkdephd)) { 1700 jwait(&LIST_FIRST( 1701 &freeblks->fb_jblkdephd)->jb_list, 1702 MNT_WAIT); 1703 break; 1704 } 1705 /* Another thread is executing this item. */ 1706 if (freeblks->fb_state & INPROGRESS) { 1707 wait_worklist(&freeblks->fb_list, "ptrwait"); 1708 break; 1709 } 1710 /* Freeblks is waiting on a inode write. */ 1711 if ((freeblks->fb_state & COMPLETE) == 0) { 1712 FREE_LOCK(ump); 1713 ffs_update(vp, 1); 1714 ACQUIRE_LOCK(ump); 1715 break; 1716 } 1717 if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST)) == 1718 (ALLCOMPLETE | ONWORKLIST)) { 1719 remove_from_worklist(&freeblks->fb_list); 1720 freeblks->fb_state |= INPROGRESS; 1721 FREE_LOCK(ump); 1722 if (vn_start_secondary_write(NULL, &mp, 1723 V_NOWAIT)) 1724 panic("process_truncates: " 1725 "suspended filesystem"); 1726 handle_workitem_freeblocks(freeblks, 0); 1727 vn_finished_secondary_write(mp); 1728 ACQUIRE_LOCK(ump); 1729 break; 1730 } 1731 if (freeblks->fb_cgwait) 1732 cgwait++; 1733 } 1734 if (cgwait) { 1735 FREE_LOCK(ump); 1736 sync_cgs(mp, MNT_WAIT); 1737 ffs_sync_snap(mp, MNT_WAIT); 1738 ACQUIRE_LOCK(ump); 1739 continue; 1740 } 1741 if (freeblks == NULL) 1742 break; 1743 } 1744 return; 1745 } 1746 1747 /* 1748 * Process one item on the worklist. 1749 */ 1750 static int 1751 process_worklist_item(mp, target, flags) 1752 struct mount *mp; 1753 int target; 1754 int flags; 1755 { 1756 struct worklist sentinel; 1757 struct worklist *wk; 1758 struct ufsmount *ump; 1759 int matchcnt; 1760 int error; 1761 1762 KASSERT(mp != NULL, ("process_worklist_item: NULL mp")); 1763 /* 1764 * If we are being called because of a process doing a 1765 * copy-on-write, then it is not safe to write as we may 1766 * recurse into the copy-on-write routine. 1767 */ 1768 if (curthread->td_pflags & TDP_COWINPROGRESS) 1769 return (-1); 1770 PHOLD(curproc); /* Don't let the stack go away. */ 1771 ump = VFSTOUFS(mp); 1772 LOCK_OWNED(ump); 1773 matchcnt = 0; 1774 sentinel.wk_mp = NULL; 1775 sentinel.wk_type = D_SENTINEL; 1776 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, &sentinel, wk_list); 1777 for (wk = LIST_NEXT(&sentinel, wk_list); wk != NULL; 1778 wk = LIST_NEXT(&sentinel, wk_list)) { 1779 if (wk->wk_type == D_SENTINEL) { 1780 LIST_REMOVE(&sentinel, wk_list); 1781 LIST_INSERT_AFTER(wk, &sentinel, wk_list); 1782 continue; 1783 } 1784 if (wk->wk_state & INPROGRESS) 1785 panic("process_worklist_item: %p already in progress.", 1786 wk); 1787 wk->wk_state |= INPROGRESS; 1788 remove_from_worklist(wk); 1789 FREE_LOCK(ump); 1790 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1791 panic("process_worklist_item: suspended filesystem"); 1792 switch (wk->wk_type) { 1793 case D_DIRREM: 1794 /* removal of a directory entry */ 1795 error = handle_workitem_remove(WK_DIRREM(wk), flags); 1796 break; 1797 1798 case D_FREEBLKS: 1799 /* releasing blocks and/or fragments from a file */ 1800 error = handle_workitem_freeblocks(WK_FREEBLKS(wk), 1801 flags); 1802 break; 1803 1804 case D_FREEFRAG: 1805 /* releasing a fragment when replaced as a file grows */ 1806 handle_workitem_freefrag(WK_FREEFRAG(wk)); 1807 error = 0; 1808 break; 1809 1810 case D_FREEFILE: 1811 /* releasing an inode when its link count drops to 0 */ 1812 handle_workitem_freefile(WK_FREEFILE(wk)); 1813 error = 0; 1814 break; 1815 1816 default: 1817 panic("%s_process_worklist: Unknown type %s", 1818 "softdep", TYPENAME(wk->wk_type)); 1819 /* NOTREACHED */ 1820 } 1821 vn_finished_secondary_write(mp); 1822 ACQUIRE_LOCK(ump); 1823 if (error == 0) { 1824 if (++matchcnt == target) 1825 break; 1826 continue; 1827 } 1828 /* 1829 * We have to retry the worklist item later. Wake up any 1830 * waiters who may be able to complete it immediately and 1831 * add the item back to the head so we don't try to execute 1832 * it again. 1833 */ 1834 wk->wk_state &= ~INPROGRESS; 1835 wake_worklist(wk); 1836 add_to_worklist(wk, WK_HEAD); 1837 } 1838 LIST_REMOVE(&sentinel, wk_list); 1839 /* Sentinal could've become the tail from remove_from_worklist. */ 1840 if (ump->softdep_worklist_tail == &sentinel) 1841 ump->softdep_worklist_tail = 1842 (struct worklist *)sentinel.wk_list.le_prev; 1843 PRELE(curproc); 1844 return (matchcnt); 1845 } 1846 1847 /* 1848 * Move dependencies from one buffer to another. 1849 */ 1850 int 1851 softdep_move_dependencies(oldbp, newbp) 1852 struct buf *oldbp; 1853 struct buf *newbp; 1854 { 1855 struct worklist *wk, *wktail; 1856 struct ufsmount *ump; 1857 int dirty; 1858 1859 if ((wk = LIST_FIRST(&oldbp->b_dep)) == NULL) 1860 return (0); 1861 KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0, 1862 ("softdep_move_dependencies called on non-softdep filesystem")); 1863 dirty = 0; 1864 wktail = NULL; 1865 ump = VFSTOUFS(wk->wk_mp); 1866 ACQUIRE_LOCK(ump); 1867 while ((wk = LIST_FIRST(&oldbp->b_dep)) != NULL) { 1868 LIST_REMOVE(wk, wk_list); 1869 if (wk->wk_type == D_BMSAFEMAP && 1870 bmsafemap_backgroundwrite(WK_BMSAFEMAP(wk), newbp)) 1871 dirty = 1; 1872 if (wktail == NULL) 1873 LIST_INSERT_HEAD(&newbp->b_dep, wk, wk_list); 1874 else 1875 LIST_INSERT_AFTER(wktail, wk, wk_list); 1876 wktail = wk; 1877 } 1878 FREE_LOCK(ump); 1879 1880 return (dirty); 1881 } 1882 1883 /* 1884 * Purge the work list of all items associated with a particular mount point. 1885 */ 1886 int 1887 softdep_flushworklist(oldmnt, countp, td) 1888 struct mount *oldmnt; 1889 int *countp; 1890 struct thread *td; 1891 { 1892 struct vnode *devvp; 1893 struct ufsmount *ump; 1894 int count, error; 1895 1896 /* 1897 * Alternately flush the block device associated with the mount 1898 * point and process any dependencies that the flushing 1899 * creates. We continue until no more worklist dependencies 1900 * are found. 1901 */ 1902 *countp = 0; 1903 error = 0; 1904 ump = VFSTOUFS(oldmnt); 1905 devvp = ump->um_devvp; 1906 while ((count = softdep_process_worklist(oldmnt, 1)) > 0) { 1907 *countp += count; 1908 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY); 1909 error = VOP_FSYNC(devvp, MNT_WAIT, td); 1910 VOP_UNLOCK(devvp, 0); 1911 if (error != 0) 1912 break; 1913 } 1914 return (error); 1915 } 1916 1917 #define SU_WAITIDLE_RETRIES 20 1918 static int 1919 softdep_waitidle(struct mount *mp, int flags __unused) 1920 { 1921 struct ufsmount *ump; 1922 struct vnode *devvp; 1923 struct thread *td; 1924 int error, i; 1925 1926 ump = VFSTOUFS(mp); 1927 devvp = ump->um_devvp; 1928 td = curthread; 1929 error = 0; 1930 ACQUIRE_LOCK(ump); 1931 for (i = 0; i < SU_WAITIDLE_RETRIES && ump->softdep_deps != 0; i++) { 1932 ump->softdep_req = 1; 1933 KASSERT((flags & FORCECLOSE) == 0 || 1934 ump->softdep_on_worklist == 0, 1935 ("softdep_waitidle: work added after flush")); 1936 msleep(&ump->softdep_deps, LOCK_PTR(ump), PVM | PDROP, 1937 "softdeps", 10 * hz); 1938 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY); 1939 error = VOP_FSYNC(devvp, MNT_WAIT, td); 1940 VOP_UNLOCK(devvp, 0); 1941 ACQUIRE_LOCK(ump); 1942 if (error != 0) 1943 break; 1944 } 1945 ump->softdep_req = 0; 1946 if (i == SU_WAITIDLE_RETRIES && error == 0 && ump->softdep_deps != 0) { 1947 error = EBUSY; 1948 printf("softdep_waitidle: Failed to flush worklist for %p\n", 1949 mp); 1950 } 1951 FREE_LOCK(ump); 1952 return (error); 1953 } 1954 1955 /* 1956 * Flush all vnodes and worklist items associated with a specified mount point. 1957 */ 1958 int 1959 softdep_flushfiles(oldmnt, flags, td) 1960 struct mount *oldmnt; 1961 int flags; 1962 struct thread *td; 1963 { 1964 #ifdef QUOTA 1965 struct ufsmount *ump; 1966 int i; 1967 #endif 1968 int error, early, depcount, loopcnt, retry_flush_count, retry; 1969 int morework; 1970 1971 KASSERT(MOUNTEDSOFTDEP(oldmnt) != 0, 1972 ("softdep_flushfiles called on non-softdep filesystem")); 1973 loopcnt = 10; 1974 retry_flush_count = 3; 1975 retry_flush: 1976 error = 0; 1977 1978 /* 1979 * Alternately flush the vnodes associated with the mount 1980 * point and process any dependencies that the flushing 1981 * creates. In theory, this loop can happen at most twice, 1982 * but we give it a few extra just to be sure. 1983 */ 1984 for (; loopcnt > 0; loopcnt--) { 1985 /* 1986 * Do another flush in case any vnodes were brought in 1987 * as part of the cleanup operations. 1988 */ 1989 early = retry_flush_count == 1 || (oldmnt->mnt_kern_flag & 1990 MNTK_UNMOUNT) == 0 ? 0 : EARLYFLUSH; 1991 if ((error = ffs_flushfiles(oldmnt, flags | early, td)) != 0) 1992 break; 1993 if ((error = softdep_flushworklist(oldmnt, &depcount, td)) != 0 || 1994 depcount == 0) 1995 break; 1996 } 1997 /* 1998 * If we are unmounting then it is an error to fail. If we 1999 * are simply trying to downgrade to read-only, then filesystem 2000 * activity can keep us busy forever, so we just fail with EBUSY. 2001 */ 2002 if (loopcnt == 0) { 2003 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) 2004 panic("softdep_flushfiles: looping"); 2005 error = EBUSY; 2006 } 2007 if (!error) 2008 error = softdep_waitidle(oldmnt, flags); 2009 if (!error) { 2010 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) { 2011 retry = 0; 2012 MNT_ILOCK(oldmnt); 2013 KASSERT((oldmnt->mnt_kern_flag & MNTK_NOINSMNTQ) != 0, 2014 ("softdep_flushfiles: !MNTK_NOINSMNTQ")); 2015 morework = oldmnt->mnt_nvnodelistsize > 0; 2016 #ifdef QUOTA 2017 ump = VFSTOUFS(oldmnt); 2018 UFS_LOCK(ump); 2019 for (i = 0; i < MAXQUOTAS; i++) { 2020 if (ump->um_quotas[i] != NULLVP) 2021 morework = 1; 2022 } 2023 UFS_UNLOCK(ump); 2024 #endif 2025 if (morework) { 2026 if (--retry_flush_count > 0) { 2027 retry = 1; 2028 loopcnt = 3; 2029 } else 2030 error = EBUSY; 2031 } 2032 MNT_IUNLOCK(oldmnt); 2033 if (retry) 2034 goto retry_flush; 2035 } 2036 } 2037 return (error); 2038 } 2039 2040 /* 2041 * Structure hashing. 2042 * 2043 * There are four types of structures that can be looked up: 2044 * 1) pagedep structures identified by mount point, inode number, 2045 * and logical block. 2046 * 2) inodedep structures identified by mount point and inode number. 2047 * 3) newblk structures identified by mount point and 2048 * physical block number. 2049 * 4) bmsafemap structures identified by mount point and 2050 * cylinder group number. 2051 * 2052 * The "pagedep" and "inodedep" dependency structures are hashed 2053 * separately from the file blocks and inodes to which they correspond. 2054 * This separation helps when the in-memory copy of an inode or 2055 * file block must be replaced. It also obviates the need to access 2056 * an inode or file page when simply updating (or de-allocating) 2057 * dependency structures. Lookup of newblk structures is needed to 2058 * find newly allocated blocks when trying to associate them with 2059 * their allocdirect or allocindir structure. 2060 * 2061 * The lookup routines optionally create and hash a new instance when 2062 * an existing entry is not found. The bmsafemap lookup routine always 2063 * allocates a new structure if an existing one is not found. 2064 */ 2065 #define DEPALLOC 0x0001 /* allocate structure if lookup fails */ 2066 2067 /* 2068 * Structures and routines associated with pagedep caching. 2069 */ 2070 #define PAGEDEP_HASH(ump, inum, lbn) \ 2071 (&(ump)->pagedep_hashtbl[((inum) + (lbn)) & (ump)->pagedep_hash_size]) 2072 2073 static int 2074 pagedep_find(pagedephd, ino, lbn, pagedeppp) 2075 struct pagedep_hashhead *pagedephd; 2076 ino_t ino; 2077 ufs_lbn_t lbn; 2078 struct pagedep **pagedeppp; 2079 { 2080 struct pagedep *pagedep; 2081 2082 LIST_FOREACH(pagedep, pagedephd, pd_hash) { 2083 if (ino == pagedep->pd_ino && lbn == pagedep->pd_lbn) { 2084 *pagedeppp = pagedep; 2085 return (1); 2086 } 2087 } 2088 *pagedeppp = NULL; 2089 return (0); 2090 } 2091 /* 2092 * Look up a pagedep. Return 1 if found, 0 otherwise. 2093 * If not found, allocate if DEPALLOC flag is passed. 2094 * Found or allocated entry is returned in pagedeppp. 2095 * This routine must be called with splbio interrupts blocked. 2096 */ 2097 static int 2098 pagedep_lookup(mp, bp, ino, lbn, flags, pagedeppp) 2099 struct mount *mp; 2100 struct buf *bp; 2101 ino_t ino; 2102 ufs_lbn_t lbn; 2103 int flags; 2104 struct pagedep **pagedeppp; 2105 { 2106 struct pagedep *pagedep; 2107 struct pagedep_hashhead *pagedephd; 2108 struct worklist *wk; 2109 struct ufsmount *ump; 2110 int ret; 2111 int i; 2112 2113 ump = VFSTOUFS(mp); 2114 LOCK_OWNED(ump); 2115 if (bp) { 2116 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 2117 if (wk->wk_type == D_PAGEDEP) { 2118 *pagedeppp = WK_PAGEDEP(wk); 2119 return (1); 2120 } 2121 } 2122 } 2123 pagedephd = PAGEDEP_HASH(ump, ino, lbn); 2124 ret = pagedep_find(pagedephd, ino, lbn, pagedeppp); 2125 if (ret) { 2126 if (((*pagedeppp)->pd_state & ONWORKLIST) == 0 && bp) 2127 WORKLIST_INSERT(&bp->b_dep, &(*pagedeppp)->pd_list); 2128 return (1); 2129 } 2130 if ((flags & DEPALLOC) == 0) 2131 return (0); 2132 FREE_LOCK(ump); 2133 pagedep = malloc(sizeof(struct pagedep), 2134 M_PAGEDEP, M_SOFTDEP_FLAGS|M_ZERO); 2135 workitem_alloc(&pagedep->pd_list, D_PAGEDEP, mp); 2136 ACQUIRE_LOCK(ump); 2137 ret = pagedep_find(pagedephd, ino, lbn, pagedeppp); 2138 if (*pagedeppp) { 2139 /* 2140 * This should never happen since we only create pagedeps 2141 * with the vnode lock held. Could be an assert. 2142 */ 2143 WORKITEM_FREE(pagedep, D_PAGEDEP); 2144 return (ret); 2145 } 2146 pagedep->pd_ino = ino; 2147 pagedep->pd_lbn = lbn; 2148 LIST_INIT(&pagedep->pd_dirremhd); 2149 LIST_INIT(&pagedep->pd_pendinghd); 2150 for (i = 0; i < DAHASHSZ; i++) 2151 LIST_INIT(&pagedep->pd_diraddhd[i]); 2152 LIST_INSERT_HEAD(pagedephd, pagedep, pd_hash); 2153 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 2154 *pagedeppp = pagedep; 2155 return (0); 2156 } 2157 2158 /* 2159 * Structures and routines associated with inodedep caching. 2160 */ 2161 #define INODEDEP_HASH(ump, inum) \ 2162 (&(ump)->inodedep_hashtbl[(inum) & (ump)->inodedep_hash_size]) 2163 2164 static int 2165 inodedep_find(inodedephd, inum, inodedeppp) 2166 struct inodedep_hashhead *inodedephd; 2167 ino_t inum; 2168 struct inodedep **inodedeppp; 2169 { 2170 struct inodedep *inodedep; 2171 2172 LIST_FOREACH(inodedep, inodedephd, id_hash) 2173 if (inum == inodedep->id_ino) 2174 break; 2175 if (inodedep) { 2176 *inodedeppp = inodedep; 2177 return (1); 2178 } 2179 *inodedeppp = NULL; 2180 2181 return (0); 2182 } 2183 /* 2184 * Look up an inodedep. Return 1 if found, 0 if not found. 2185 * If not found, allocate if DEPALLOC flag is passed. 2186 * Found or allocated entry is returned in inodedeppp. 2187 * This routine must be called with splbio interrupts blocked. 2188 */ 2189 static int 2190 inodedep_lookup(mp, inum, flags, inodedeppp) 2191 struct mount *mp; 2192 ino_t inum; 2193 int flags; 2194 struct inodedep **inodedeppp; 2195 { 2196 struct inodedep *inodedep; 2197 struct inodedep_hashhead *inodedephd; 2198 struct ufsmount *ump; 2199 struct fs *fs; 2200 2201 ump = VFSTOUFS(mp); 2202 LOCK_OWNED(ump); 2203 fs = ump->um_fs; 2204 inodedephd = INODEDEP_HASH(ump, inum); 2205 2206 if (inodedep_find(inodedephd, inum, inodedeppp)) 2207 return (1); 2208 if ((flags & DEPALLOC) == 0) 2209 return (0); 2210 /* 2211 * If the system is over its limit and our filesystem is 2212 * responsible for more than our share of that usage and 2213 * we are not in a rush, request some inodedep cleanup. 2214 */ 2215 if (softdep_excess_items(ump, D_INODEDEP)) 2216 schedule_cleanup(mp); 2217 else 2218 FREE_LOCK(ump); 2219 inodedep = malloc(sizeof(struct inodedep), 2220 M_INODEDEP, M_SOFTDEP_FLAGS); 2221 workitem_alloc(&inodedep->id_list, D_INODEDEP, mp); 2222 ACQUIRE_LOCK(ump); 2223 if (inodedep_find(inodedephd, inum, inodedeppp)) { 2224 WORKITEM_FREE(inodedep, D_INODEDEP); 2225 return (1); 2226 } 2227 inodedep->id_fs = fs; 2228 inodedep->id_ino = inum; 2229 inodedep->id_state = ALLCOMPLETE; 2230 inodedep->id_nlinkdelta = 0; 2231 inodedep->id_savedino1 = NULL; 2232 inodedep->id_savedsize = -1; 2233 inodedep->id_savedextsize = -1; 2234 inodedep->id_savednlink = -1; 2235 inodedep->id_bmsafemap = NULL; 2236 inodedep->id_mkdiradd = NULL; 2237 LIST_INIT(&inodedep->id_dirremhd); 2238 LIST_INIT(&inodedep->id_pendinghd); 2239 LIST_INIT(&inodedep->id_inowait); 2240 LIST_INIT(&inodedep->id_bufwait); 2241 TAILQ_INIT(&inodedep->id_inoreflst); 2242 TAILQ_INIT(&inodedep->id_inoupdt); 2243 TAILQ_INIT(&inodedep->id_newinoupdt); 2244 TAILQ_INIT(&inodedep->id_extupdt); 2245 TAILQ_INIT(&inodedep->id_newextupdt); 2246 TAILQ_INIT(&inodedep->id_freeblklst); 2247 LIST_INSERT_HEAD(inodedephd, inodedep, id_hash); 2248 *inodedeppp = inodedep; 2249 return (0); 2250 } 2251 2252 /* 2253 * Structures and routines associated with newblk caching. 2254 */ 2255 #define NEWBLK_HASH(ump, inum) \ 2256 (&(ump)->newblk_hashtbl[(inum) & (ump)->newblk_hash_size]) 2257 2258 static int 2259 newblk_find(newblkhd, newblkno, flags, newblkpp) 2260 struct newblk_hashhead *newblkhd; 2261 ufs2_daddr_t newblkno; 2262 int flags; 2263 struct newblk **newblkpp; 2264 { 2265 struct newblk *newblk; 2266 2267 LIST_FOREACH(newblk, newblkhd, nb_hash) { 2268 if (newblkno != newblk->nb_newblkno) 2269 continue; 2270 /* 2271 * If we're creating a new dependency don't match those that 2272 * have already been converted to allocdirects. This is for 2273 * a frag extend. 2274 */ 2275 if ((flags & DEPALLOC) && newblk->nb_list.wk_type != D_NEWBLK) 2276 continue; 2277 break; 2278 } 2279 if (newblk) { 2280 *newblkpp = newblk; 2281 return (1); 2282 } 2283 *newblkpp = NULL; 2284 return (0); 2285 } 2286 2287 /* 2288 * Look up a newblk. Return 1 if found, 0 if not found. 2289 * If not found, allocate if DEPALLOC flag is passed. 2290 * Found or allocated entry is returned in newblkpp. 2291 */ 2292 static int 2293 newblk_lookup(mp, newblkno, flags, newblkpp) 2294 struct mount *mp; 2295 ufs2_daddr_t newblkno; 2296 int flags; 2297 struct newblk **newblkpp; 2298 { 2299 struct newblk *newblk; 2300 struct newblk_hashhead *newblkhd; 2301 struct ufsmount *ump; 2302 2303 ump = VFSTOUFS(mp); 2304 LOCK_OWNED(ump); 2305 newblkhd = NEWBLK_HASH(ump, newblkno); 2306 if (newblk_find(newblkhd, newblkno, flags, newblkpp)) 2307 return (1); 2308 if ((flags & DEPALLOC) == 0) 2309 return (0); 2310 if (softdep_excess_items(ump, D_NEWBLK) || 2311 softdep_excess_items(ump, D_ALLOCDIRECT) || 2312 softdep_excess_items(ump, D_ALLOCINDIR)) 2313 schedule_cleanup(mp); 2314 else 2315 FREE_LOCK(ump); 2316 newblk = malloc(sizeof(union allblk), M_NEWBLK, 2317 M_SOFTDEP_FLAGS | M_ZERO); 2318 workitem_alloc(&newblk->nb_list, D_NEWBLK, mp); 2319 ACQUIRE_LOCK(ump); 2320 if (newblk_find(newblkhd, newblkno, flags, newblkpp)) { 2321 WORKITEM_FREE(newblk, D_NEWBLK); 2322 return (1); 2323 } 2324 newblk->nb_freefrag = NULL; 2325 LIST_INIT(&newblk->nb_indirdeps); 2326 LIST_INIT(&newblk->nb_newdirblk); 2327 LIST_INIT(&newblk->nb_jwork); 2328 newblk->nb_state = ATTACHED; 2329 newblk->nb_newblkno = newblkno; 2330 LIST_INSERT_HEAD(newblkhd, newblk, nb_hash); 2331 *newblkpp = newblk; 2332 return (0); 2333 } 2334 2335 /* 2336 * Structures and routines associated with freed indirect block caching. 2337 */ 2338 #define INDIR_HASH(ump, blkno) \ 2339 (&(ump)->indir_hashtbl[(blkno) & (ump)->indir_hash_size]) 2340 2341 /* 2342 * Lookup an indirect block in the indir hash table. The freework is 2343 * removed and potentially freed. The caller must do a blocking journal 2344 * write before writing to the blkno. 2345 */ 2346 static int 2347 indirblk_lookup(mp, blkno) 2348 struct mount *mp; 2349 ufs2_daddr_t blkno; 2350 { 2351 struct freework *freework; 2352 struct indir_hashhead *wkhd; 2353 struct ufsmount *ump; 2354 2355 ump = VFSTOUFS(mp); 2356 wkhd = INDIR_HASH(ump, blkno); 2357 TAILQ_FOREACH(freework, wkhd, fw_next) { 2358 if (freework->fw_blkno != blkno) 2359 continue; 2360 indirblk_remove(freework); 2361 return (1); 2362 } 2363 return (0); 2364 } 2365 2366 /* 2367 * Insert an indirect block represented by freework into the indirblk 2368 * hash table so that it may prevent the block from being re-used prior 2369 * to the journal being written. 2370 */ 2371 static void 2372 indirblk_insert(freework) 2373 struct freework *freework; 2374 { 2375 struct jblocks *jblocks; 2376 struct jseg *jseg; 2377 struct ufsmount *ump; 2378 2379 ump = VFSTOUFS(freework->fw_list.wk_mp); 2380 jblocks = ump->softdep_jblocks; 2381 jseg = TAILQ_LAST(&jblocks->jb_segs, jseglst); 2382 if (jseg == NULL) 2383 return; 2384 2385 LIST_INSERT_HEAD(&jseg->js_indirs, freework, fw_segs); 2386 TAILQ_INSERT_HEAD(INDIR_HASH(ump, freework->fw_blkno), freework, 2387 fw_next); 2388 freework->fw_state &= ~DEPCOMPLETE; 2389 } 2390 2391 static void 2392 indirblk_remove(freework) 2393 struct freework *freework; 2394 { 2395 struct ufsmount *ump; 2396 2397 ump = VFSTOUFS(freework->fw_list.wk_mp); 2398 LIST_REMOVE(freework, fw_segs); 2399 TAILQ_REMOVE(INDIR_HASH(ump, freework->fw_blkno), freework, fw_next); 2400 freework->fw_state |= DEPCOMPLETE; 2401 if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE) 2402 WORKITEM_FREE(freework, D_FREEWORK); 2403 } 2404 2405 /* 2406 * Executed during filesystem system initialization before 2407 * mounting any filesystems. 2408 */ 2409 void 2410 softdep_initialize() 2411 { 2412 2413 TAILQ_INIT(&softdepmounts); 2414 #ifdef __LP64__ 2415 max_softdeps = desiredvnodes * 4; 2416 #else 2417 max_softdeps = desiredvnodes * 2; 2418 #endif 2419 2420 /* initialise bioops hack */ 2421 bioops.io_start = softdep_disk_io_initiation; 2422 bioops.io_complete = softdep_disk_write_complete; 2423 bioops.io_deallocate = softdep_deallocate_dependencies; 2424 bioops.io_countdeps = softdep_count_dependencies; 2425 softdep_ast_cleanup = softdep_ast_cleanup_proc; 2426 2427 /* Initialize the callout with an mtx. */ 2428 callout_init_mtx(&softdep_callout, &lk, 0); 2429 } 2430 2431 /* 2432 * Executed after all filesystems have been unmounted during 2433 * filesystem module unload. 2434 */ 2435 void 2436 softdep_uninitialize() 2437 { 2438 2439 /* clear bioops hack */ 2440 bioops.io_start = NULL; 2441 bioops.io_complete = NULL; 2442 bioops.io_deallocate = NULL; 2443 bioops.io_countdeps = NULL; 2444 softdep_ast_cleanup = NULL; 2445 2446 callout_drain(&softdep_callout); 2447 } 2448 2449 /* 2450 * Called at mount time to notify the dependency code that a 2451 * filesystem wishes to use it. 2452 */ 2453 int 2454 softdep_mount(devvp, mp, fs, cred) 2455 struct vnode *devvp; 2456 struct mount *mp; 2457 struct fs *fs; 2458 struct ucred *cred; 2459 { 2460 struct csum_total cstotal; 2461 struct mount_softdeps *sdp; 2462 struct ufsmount *ump; 2463 struct cg *cgp; 2464 struct buf *bp; 2465 int i, error, cyl; 2466 2467 sdp = malloc(sizeof(struct mount_softdeps), M_MOUNTDATA, 2468 M_WAITOK | M_ZERO); 2469 MNT_ILOCK(mp); 2470 mp->mnt_flag = (mp->mnt_flag & ~MNT_ASYNC) | MNT_SOFTDEP; 2471 if ((mp->mnt_kern_flag & MNTK_SOFTDEP) == 0) { 2472 mp->mnt_kern_flag = (mp->mnt_kern_flag & ~MNTK_ASYNC) | 2473 MNTK_SOFTDEP | MNTK_NOASYNC; 2474 } 2475 ump = VFSTOUFS(mp); 2476 ump->um_softdep = sdp; 2477 MNT_IUNLOCK(mp); 2478 rw_init(LOCK_PTR(ump), "Per-Filesystem Softdep Lock"); 2479 sdp->sd_ump = ump; 2480 LIST_INIT(&ump->softdep_workitem_pending); 2481 LIST_INIT(&ump->softdep_journal_pending); 2482 TAILQ_INIT(&ump->softdep_unlinked); 2483 LIST_INIT(&ump->softdep_dirtycg); 2484 ump->softdep_worklist_tail = NULL; 2485 ump->softdep_on_worklist = 0; 2486 ump->softdep_deps = 0; 2487 LIST_INIT(&ump->softdep_mkdirlisthd); 2488 ump->pagedep_hashtbl = hashinit(desiredvnodes / 5, M_PAGEDEP, 2489 &ump->pagedep_hash_size); 2490 ump->pagedep_nextclean = 0; 2491 ump->inodedep_hashtbl = hashinit(desiredvnodes, M_INODEDEP, 2492 &ump->inodedep_hash_size); 2493 ump->inodedep_nextclean = 0; 2494 ump->newblk_hashtbl = hashinit(max_softdeps / 2, M_NEWBLK, 2495 &ump->newblk_hash_size); 2496 ump->bmsafemap_hashtbl = hashinit(1024, M_BMSAFEMAP, 2497 &ump->bmsafemap_hash_size); 2498 i = 1 << (ffs(desiredvnodes / 10) - 1); 2499 ump->indir_hashtbl = malloc(i * sizeof(struct indir_hashhead), 2500 M_FREEWORK, M_WAITOK); 2501 ump->indir_hash_size = i - 1; 2502 for (i = 0; i <= ump->indir_hash_size; i++) 2503 TAILQ_INIT(&ump->indir_hashtbl[i]); 2504 ACQUIRE_GBLLOCK(&lk); 2505 TAILQ_INSERT_TAIL(&softdepmounts, sdp, sd_next); 2506 FREE_GBLLOCK(&lk); 2507 if ((fs->fs_flags & FS_SUJ) && 2508 (error = journal_mount(mp, fs, cred)) != 0) { 2509 printf("Failed to start journal: %d\n", error); 2510 softdep_unmount(mp); 2511 return (error); 2512 } 2513 /* 2514 * Start our flushing thread in the bufdaemon process. 2515 */ 2516 ACQUIRE_LOCK(ump); 2517 ump->softdep_flags |= FLUSH_STARTING; 2518 FREE_LOCK(ump); 2519 kproc_kthread_add(&softdep_flush, mp, &bufdaemonproc, 2520 &ump->softdep_flushtd, 0, 0, "softdepflush", "%s worker", 2521 mp->mnt_stat.f_mntonname); 2522 ACQUIRE_LOCK(ump); 2523 while ((ump->softdep_flags & FLUSH_STARTING) != 0) { 2524 msleep(&ump->softdep_flushtd, LOCK_PTR(ump), PVM, "sdstart", 2525 hz / 2); 2526 } 2527 FREE_LOCK(ump); 2528 /* 2529 * When doing soft updates, the counters in the 2530 * superblock may have gotten out of sync. Recomputation 2531 * can take a long time and can be deferred for background 2532 * fsck. However, the old behavior of scanning the cylinder 2533 * groups and recalculating them at mount time is available 2534 * by setting vfs.ffs.compute_summary_at_mount to one. 2535 */ 2536 if (compute_summary_at_mount == 0 || fs->fs_clean != 0) 2537 return (0); 2538 bzero(&cstotal, sizeof cstotal); 2539 for (cyl = 0; cyl < fs->fs_ncg; cyl++) { 2540 if ((error = bread(devvp, fsbtodb(fs, cgtod(fs, cyl)), 2541 fs->fs_cgsize, cred, &bp)) != 0) { 2542 brelse(bp); 2543 softdep_unmount(mp); 2544 return (error); 2545 } 2546 cgp = (struct cg *)bp->b_data; 2547 cstotal.cs_nffree += cgp->cg_cs.cs_nffree; 2548 cstotal.cs_nbfree += cgp->cg_cs.cs_nbfree; 2549 cstotal.cs_nifree += cgp->cg_cs.cs_nifree; 2550 cstotal.cs_ndir += cgp->cg_cs.cs_ndir; 2551 fs->fs_cs(fs, cyl) = cgp->cg_cs; 2552 brelse(bp); 2553 } 2554 #ifdef DEBUG 2555 if (bcmp(&cstotal, &fs->fs_cstotal, sizeof cstotal)) 2556 printf("%s: superblock summary recomputed\n", fs->fs_fsmnt); 2557 #endif 2558 bcopy(&cstotal, &fs->fs_cstotal, sizeof cstotal); 2559 return (0); 2560 } 2561 2562 void 2563 softdep_unmount(mp) 2564 struct mount *mp; 2565 { 2566 struct ufsmount *ump; 2567 #ifdef INVARIANTS 2568 int i; 2569 #endif 2570 2571 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 2572 ("softdep_unmount called on non-softdep filesystem")); 2573 ump = VFSTOUFS(mp); 2574 MNT_ILOCK(mp); 2575 mp->mnt_flag &= ~MNT_SOFTDEP; 2576 if (MOUNTEDSUJ(mp) == 0) { 2577 MNT_IUNLOCK(mp); 2578 } else { 2579 mp->mnt_flag &= ~MNT_SUJ; 2580 MNT_IUNLOCK(mp); 2581 journal_unmount(ump); 2582 } 2583 /* 2584 * Shut down our flushing thread. Check for NULL is if 2585 * softdep_mount errors out before the thread has been created. 2586 */ 2587 if (ump->softdep_flushtd != NULL) { 2588 ACQUIRE_LOCK(ump); 2589 ump->softdep_flags |= FLUSH_EXIT; 2590 wakeup(&ump->softdep_flushtd); 2591 msleep(&ump->softdep_flags, LOCK_PTR(ump), PVM | PDROP, 2592 "sdwait", 0); 2593 KASSERT((ump->softdep_flags & FLUSH_EXIT) == 0, 2594 ("Thread shutdown failed")); 2595 } 2596 /* 2597 * Free up our resources. 2598 */ 2599 ACQUIRE_GBLLOCK(&lk); 2600 TAILQ_REMOVE(&softdepmounts, ump->um_softdep, sd_next); 2601 FREE_GBLLOCK(&lk); 2602 rw_destroy(LOCK_PTR(ump)); 2603 hashdestroy(ump->pagedep_hashtbl, M_PAGEDEP, ump->pagedep_hash_size); 2604 hashdestroy(ump->inodedep_hashtbl, M_INODEDEP, ump->inodedep_hash_size); 2605 hashdestroy(ump->newblk_hashtbl, M_NEWBLK, ump->newblk_hash_size); 2606 hashdestroy(ump->bmsafemap_hashtbl, M_BMSAFEMAP, 2607 ump->bmsafemap_hash_size); 2608 free(ump->indir_hashtbl, M_FREEWORK); 2609 #ifdef INVARIANTS 2610 for (i = 0; i <= D_LAST; i++) 2611 KASSERT(ump->softdep_curdeps[i] == 0, 2612 ("Unmount %s: Dep type %s != 0 (%ld)", ump->um_fs->fs_fsmnt, 2613 TYPENAME(i), ump->softdep_curdeps[i])); 2614 #endif 2615 free(ump->um_softdep, M_MOUNTDATA); 2616 } 2617 2618 static struct jblocks * 2619 jblocks_create(void) 2620 { 2621 struct jblocks *jblocks; 2622 2623 jblocks = malloc(sizeof(*jblocks), M_JBLOCKS, M_WAITOK | M_ZERO); 2624 TAILQ_INIT(&jblocks->jb_segs); 2625 jblocks->jb_avail = 10; 2626 jblocks->jb_extent = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2627 M_JBLOCKS, M_WAITOK | M_ZERO); 2628 2629 return (jblocks); 2630 } 2631 2632 static ufs2_daddr_t 2633 jblocks_alloc(jblocks, bytes, actual) 2634 struct jblocks *jblocks; 2635 int bytes; 2636 int *actual; 2637 { 2638 ufs2_daddr_t daddr; 2639 struct jextent *jext; 2640 int freecnt; 2641 int blocks; 2642 2643 blocks = bytes / DEV_BSIZE; 2644 jext = &jblocks->jb_extent[jblocks->jb_head]; 2645 freecnt = jext->je_blocks - jblocks->jb_off; 2646 if (freecnt == 0) { 2647 jblocks->jb_off = 0; 2648 if (++jblocks->jb_head > jblocks->jb_used) 2649 jblocks->jb_head = 0; 2650 jext = &jblocks->jb_extent[jblocks->jb_head]; 2651 freecnt = jext->je_blocks; 2652 } 2653 if (freecnt > blocks) 2654 freecnt = blocks; 2655 *actual = freecnt * DEV_BSIZE; 2656 daddr = jext->je_daddr + jblocks->jb_off; 2657 jblocks->jb_off += freecnt; 2658 jblocks->jb_free -= freecnt; 2659 2660 return (daddr); 2661 } 2662 2663 static void 2664 jblocks_free(jblocks, mp, bytes) 2665 struct jblocks *jblocks; 2666 struct mount *mp; 2667 int bytes; 2668 { 2669 2670 LOCK_OWNED(VFSTOUFS(mp)); 2671 jblocks->jb_free += bytes / DEV_BSIZE; 2672 if (jblocks->jb_suspended) 2673 worklist_speedup(mp); 2674 wakeup(jblocks); 2675 } 2676 2677 static void 2678 jblocks_destroy(jblocks) 2679 struct jblocks *jblocks; 2680 { 2681 2682 if (jblocks->jb_extent) 2683 free(jblocks->jb_extent, M_JBLOCKS); 2684 free(jblocks, M_JBLOCKS); 2685 } 2686 2687 static void 2688 jblocks_add(jblocks, daddr, blocks) 2689 struct jblocks *jblocks; 2690 ufs2_daddr_t daddr; 2691 int blocks; 2692 { 2693 struct jextent *jext; 2694 2695 jblocks->jb_blocks += blocks; 2696 jblocks->jb_free += blocks; 2697 jext = &jblocks->jb_extent[jblocks->jb_used]; 2698 /* Adding the first block. */ 2699 if (jext->je_daddr == 0) { 2700 jext->je_daddr = daddr; 2701 jext->je_blocks = blocks; 2702 return; 2703 } 2704 /* Extending the last extent. */ 2705 if (jext->je_daddr + jext->je_blocks == daddr) { 2706 jext->je_blocks += blocks; 2707 return; 2708 } 2709 /* Adding a new extent. */ 2710 if (++jblocks->jb_used == jblocks->jb_avail) { 2711 jblocks->jb_avail *= 2; 2712 jext = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2713 M_JBLOCKS, M_WAITOK | M_ZERO); 2714 memcpy(jext, jblocks->jb_extent, 2715 sizeof(struct jextent) * jblocks->jb_used); 2716 free(jblocks->jb_extent, M_JBLOCKS); 2717 jblocks->jb_extent = jext; 2718 } 2719 jext = &jblocks->jb_extent[jblocks->jb_used]; 2720 jext->je_daddr = daddr; 2721 jext->je_blocks = blocks; 2722 return; 2723 } 2724 2725 int 2726 softdep_journal_lookup(mp, vpp) 2727 struct mount *mp; 2728 struct vnode **vpp; 2729 { 2730 struct componentname cnp; 2731 struct vnode *dvp; 2732 ino_t sujournal; 2733 int error; 2734 2735 error = VFS_VGET(mp, ROOTINO, LK_EXCLUSIVE, &dvp); 2736 if (error) 2737 return (error); 2738 bzero(&cnp, sizeof(cnp)); 2739 cnp.cn_nameiop = LOOKUP; 2740 cnp.cn_flags = ISLASTCN; 2741 cnp.cn_thread = curthread; 2742 cnp.cn_cred = curthread->td_ucred; 2743 cnp.cn_pnbuf = SUJ_FILE; 2744 cnp.cn_nameptr = SUJ_FILE; 2745 cnp.cn_namelen = strlen(SUJ_FILE); 2746 error = ufs_lookup_ino(dvp, NULL, &cnp, &sujournal); 2747 vput(dvp); 2748 if (error != 0) 2749 return (error); 2750 error = VFS_VGET(mp, sujournal, LK_EXCLUSIVE, vpp); 2751 return (error); 2752 } 2753 2754 /* 2755 * Open and verify the journal file. 2756 */ 2757 static int 2758 journal_mount(mp, fs, cred) 2759 struct mount *mp; 2760 struct fs *fs; 2761 struct ucred *cred; 2762 { 2763 struct jblocks *jblocks; 2764 struct ufsmount *ump; 2765 struct vnode *vp; 2766 struct inode *ip; 2767 ufs2_daddr_t blkno; 2768 int bcount; 2769 int error; 2770 int i; 2771 2772 ump = VFSTOUFS(mp); 2773 ump->softdep_journal_tail = NULL; 2774 ump->softdep_on_journal = 0; 2775 ump->softdep_accdeps = 0; 2776 ump->softdep_req = 0; 2777 ump->softdep_jblocks = NULL; 2778 error = softdep_journal_lookup(mp, &vp); 2779 if (error != 0) { 2780 printf("Failed to find journal. Use tunefs to create one\n"); 2781 return (error); 2782 } 2783 ip = VTOI(vp); 2784 if (ip->i_size < SUJ_MIN) { 2785 error = ENOSPC; 2786 goto out; 2787 } 2788 bcount = lblkno(fs, ip->i_size); /* Only use whole blocks. */ 2789 jblocks = jblocks_create(); 2790 for (i = 0; i < bcount; i++) { 2791 error = ufs_bmaparray(vp, i, &blkno, NULL, NULL, NULL); 2792 if (error) 2793 break; 2794 jblocks_add(jblocks, blkno, fsbtodb(fs, fs->fs_frag)); 2795 } 2796 if (error) { 2797 jblocks_destroy(jblocks); 2798 goto out; 2799 } 2800 jblocks->jb_low = jblocks->jb_free / 3; /* Reserve 33%. */ 2801 jblocks->jb_min = jblocks->jb_free / 10; /* Suspend at 10%. */ 2802 ump->softdep_jblocks = jblocks; 2803 out: 2804 if (error == 0) { 2805 MNT_ILOCK(mp); 2806 mp->mnt_flag |= MNT_SUJ; 2807 mp->mnt_flag &= ~MNT_SOFTDEP; 2808 MNT_IUNLOCK(mp); 2809 /* 2810 * Only validate the journal contents if the 2811 * filesystem is clean, otherwise we write the logs 2812 * but they'll never be used. If the filesystem was 2813 * still dirty when we mounted it the journal is 2814 * invalid and a new journal can only be valid if it 2815 * starts from a clean mount. 2816 */ 2817 if (fs->fs_clean) { 2818 DIP_SET(ip, i_modrev, fs->fs_mtime); 2819 ip->i_flags |= IN_MODIFIED; 2820 ffs_update(vp, 1); 2821 } 2822 } 2823 vput(vp); 2824 return (error); 2825 } 2826 2827 static void 2828 journal_unmount(ump) 2829 struct ufsmount *ump; 2830 { 2831 2832 if (ump->softdep_jblocks) 2833 jblocks_destroy(ump->softdep_jblocks); 2834 ump->softdep_jblocks = NULL; 2835 } 2836 2837 /* 2838 * Called when a journal record is ready to be written. Space is allocated 2839 * and the journal entry is created when the journal is flushed to stable 2840 * store. 2841 */ 2842 static void 2843 add_to_journal(wk) 2844 struct worklist *wk; 2845 { 2846 struct ufsmount *ump; 2847 2848 ump = VFSTOUFS(wk->wk_mp); 2849 LOCK_OWNED(ump); 2850 if (wk->wk_state & ONWORKLIST) 2851 panic("add_to_journal: %s(0x%X) already on list", 2852 TYPENAME(wk->wk_type), wk->wk_state); 2853 wk->wk_state |= ONWORKLIST | DEPCOMPLETE; 2854 if (LIST_EMPTY(&ump->softdep_journal_pending)) { 2855 ump->softdep_jblocks->jb_age = ticks; 2856 LIST_INSERT_HEAD(&ump->softdep_journal_pending, wk, wk_list); 2857 } else 2858 LIST_INSERT_AFTER(ump->softdep_journal_tail, wk, wk_list); 2859 ump->softdep_journal_tail = wk; 2860 ump->softdep_on_journal += 1; 2861 } 2862 2863 /* 2864 * Remove an arbitrary item for the journal worklist maintain the tail 2865 * pointer. This happens when a new operation obviates the need to 2866 * journal an old operation. 2867 */ 2868 static void 2869 remove_from_journal(wk) 2870 struct worklist *wk; 2871 { 2872 struct ufsmount *ump; 2873 2874 ump = VFSTOUFS(wk->wk_mp); 2875 LOCK_OWNED(ump); 2876 #ifdef SUJ_DEBUG 2877 { 2878 struct worklist *wkn; 2879 2880 LIST_FOREACH(wkn, &ump->softdep_journal_pending, wk_list) 2881 if (wkn == wk) 2882 break; 2883 if (wkn == NULL) 2884 panic("remove_from_journal: %p is not in journal", wk); 2885 } 2886 #endif 2887 /* 2888 * We emulate a TAILQ to save space in most structures which do not 2889 * require TAILQ semantics. Here we must update the tail position 2890 * when removing the tail which is not the final entry. This works 2891 * only if the worklist linkage are at the beginning of the structure. 2892 */ 2893 if (ump->softdep_journal_tail == wk) 2894 ump->softdep_journal_tail = 2895 (struct worklist *)wk->wk_list.le_prev; 2896 2897 WORKLIST_REMOVE(wk); 2898 ump->softdep_on_journal -= 1; 2899 } 2900 2901 /* 2902 * Check for journal space as well as dependency limits so the prelink 2903 * code can throttle both journaled and non-journaled filesystems. 2904 * Threshold is 0 for low and 1 for min. 2905 */ 2906 static int 2907 journal_space(ump, thresh) 2908 struct ufsmount *ump; 2909 int thresh; 2910 { 2911 struct jblocks *jblocks; 2912 int limit, avail; 2913 2914 jblocks = ump->softdep_jblocks; 2915 if (jblocks == NULL) 2916 return (1); 2917 /* 2918 * We use a tighter restriction here to prevent request_cleanup() 2919 * running in threads from running into locks we currently hold. 2920 * We have to be over the limit and our filesystem has to be 2921 * responsible for more than our share of that usage. 2922 */ 2923 limit = (max_softdeps / 10) * 9; 2924 if (dep_current[D_INODEDEP] > limit && 2925 ump->softdep_curdeps[D_INODEDEP] > limit / stat_flush_threads) 2926 return (0); 2927 if (thresh) 2928 thresh = jblocks->jb_min; 2929 else 2930 thresh = jblocks->jb_low; 2931 avail = (ump->softdep_on_journal * JREC_SIZE) / DEV_BSIZE; 2932 avail = jblocks->jb_free - avail; 2933 2934 return (avail > thresh); 2935 } 2936 2937 static void 2938 journal_suspend(ump) 2939 struct ufsmount *ump; 2940 { 2941 struct jblocks *jblocks; 2942 struct mount *mp; 2943 2944 mp = UFSTOVFS(ump); 2945 jblocks = ump->softdep_jblocks; 2946 MNT_ILOCK(mp); 2947 if ((mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 2948 stat_journal_min++; 2949 mp->mnt_kern_flag |= MNTK_SUSPEND; 2950 mp->mnt_susp_owner = ump->softdep_flushtd; 2951 } 2952 jblocks->jb_suspended = 1; 2953 MNT_IUNLOCK(mp); 2954 } 2955 2956 static int 2957 journal_unsuspend(struct ufsmount *ump) 2958 { 2959 struct jblocks *jblocks; 2960 struct mount *mp; 2961 2962 mp = UFSTOVFS(ump); 2963 jblocks = ump->softdep_jblocks; 2964 2965 if (jblocks != NULL && jblocks->jb_suspended && 2966 journal_space(ump, jblocks->jb_min)) { 2967 jblocks->jb_suspended = 0; 2968 FREE_LOCK(ump); 2969 mp->mnt_susp_owner = curthread; 2970 vfs_write_resume(mp, 0); 2971 ACQUIRE_LOCK(ump); 2972 return (1); 2973 } 2974 return (0); 2975 } 2976 2977 /* 2978 * Called before any allocation function to be certain that there is 2979 * sufficient space in the journal prior to creating any new records. 2980 * Since in the case of block allocation we may have multiple locked 2981 * buffers at the time of the actual allocation we can not block 2982 * when the journal records are created. Doing so would create a deadlock 2983 * if any of these buffers needed to be flushed to reclaim space. Instead 2984 * we require a sufficiently large amount of available space such that 2985 * each thread in the system could have passed this allocation check and 2986 * still have sufficient free space. With 20% of a minimum journal size 2987 * of 1MB we have 6553 records available. 2988 */ 2989 int 2990 softdep_prealloc(vp, waitok) 2991 struct vnode *vp; 2992 int waitok; 2993 { 2994 struct ufsmount *ump; 2995 2996 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 2997 ("softdep_prealloc called on non-softdep filesystem")); 2998 /* 2999 * Nothing to do if we are not running journaled soft updates. 3000 * If we currently hold the snapshot lock, we must avoid 3001 * handling other resources that could cause deadlock. Do not 3002 * touch quotas vnode since it is typically recursed with 3003 * other vnode locks held. 3004 */ 3005 if (DOINGSUJ(vp) == 0 || IS_SNAPSHOT(VTOI(vp)) || 3006 (vp->v_vflag & VV_SYSTEM) != 0) 3007 return (0); 3008 ump = VFSTOUFS(vp->v_mount); 3009 ACQUIRE_LOCK(ump); 3010 if (journal_space(ump, 0)) { 3011 FREE_LOCK(ump); 3012 return (0); 3013 } 3014 stat_journal_low++; 3015 FREE_LOCK(ump); 3016 if (waitok == MNT_NOWAIT) 3017 return (ENOSPC); 3018 /* 3019 * Attempt to sync this vnode once to flush any journal 3020 * work attached to it. 3021 */ 3022 if ((curthread->td_pflags & TDP_COWINPROGRESS) == 0) 3023 ffs_syncvnode(vp, waitok, 0); 3024 ACQUIRE_LOCK(ump); 3025 process_removes(vp); 3026 process_truncates(vp); 3027 if (journal_space(ump, 0) == 0) { 3028 softdep_speedup(ump); 3029 if (journal_space(ump, 1) == 0) 3030 journal_suspend(ump); 3031 } 3032 FREE_LOCK(ump); 3033 3034 return (0); 3035 } 3036 3037 /* 3038 * Before adjusting a link count on a vnode verify that we have sufficient 3039 * journal space. If not, process operations that depend on the currently 3040 * locked pair of vnodes to try to flush space as the syncer, buf daemon, 3041 * and softdep flush threads can not acquire these locks to reclaim space. 3042 */ 3043 static void 3044 softdep_prelink(dvp, vp) 3045 struct vnode *dvp; 3046 struct vnode *vp; 3047 { 3048 struct ufsmount *ump; 3049 3050 ump = VFSTOUFS(dvp->v_mount); 3051 LOCK_OWNED(ump); 3052 /* 3053 * Nothing to do if we have sufficient journal space. 3054 * If we currently hold the snapshot lock, we must avoid 3055 * handling other resources that could cause deadlock. 3056 */ 3057 if (journal_space(ump, 0) || (vp && IS_SNAPSHOT(VTOI(vp)))) 3058 return; 3059 stat_journal_low++; 3060 FREE_LOCK(ump); 3061 if (vp) 3062 ffs_syncvnode(vp, MNT_NOWAIT, 0); 3063 ffs_syncvnode(dvp, MNT_WAIT, 0); 3064 ACQUIRE_LOCK(ump); 3065 /* Process vp before dvp as it may create .. removes. */ 3066 if (vp) { 3067 process_removes(vp); 3068 process_truncates(vp); 3069 } 3070 process_removes(dvp); 3071 process_truncates(dvp); 3072 softdep_speedup(ump); 3073 process_worklist_item(UFSTOVFS(ump), 2, LK_NOWAIT); 3074 if (journal_space(ump, 0) == 0) { 3075 softdep_speedup(ump); 3076 if (journal_space(ump, 1) == 0) 3077 journal_suspend(ump); 3078 } 3079 } 3080 3081 static void 3082 jseg_write(ump, jseg, data) 3083 struct ufsmount *ump; 3084 struct jseg *jseg; 3085 uint8_t *data; 3086 { 3087 struct jsegrec *rec; 3088 3089 rec = (struct jsegrec *)data; 3090 rec->jsr_seq = jseg->js_seq; 3091 rec->jsr_oldest = jseg->js_oldseq; 3092 rec->jsr_cnt = jseg->js_cnt; 3093 rec->jsr_blocks = jseg->js_size / ump->um_devvp->v_bufobj.bo_bsize; 3094 rec->jsr_crc = 0; 3095 rec->jsr_time = ump->um_fs->fs_mtime; 3096 } 3097 3098 static inline void 3099 inoref_write(inoref, jseg, rec) 3100 struct inoref *inoref; 3101 struct jseg *jseg; 3102 struct jrefrec *rec; 3103 { 3104 3105 inoref->if_jsegdep->jd_seg = jseg; 3106 rec->jr_ino = inoref->if_ino; 3107 rec->jr_parent = inoref->if_parent; 3108 rec->jr_nlink = inoref->if_nlink; 3109 rec->jr_mode = inoref->if_mode; 3110 rec->jr_diroff = inoref->if_diroff; 3111 } 3112 3113 static void 3114 jaddref_write(jaddref, jseg, data) 3115 struct jaddref *jaddref; 3116 struct jseg *jseg; 3117 uint8_t *data; 3118 { 3119 struct jrefrec *rec; 3120 3121 rec = (struct jrefrec *)data; 3122 rec->jr_op = JOP_ADDREF; 3123 inoref_write(&jaddref->ja_ref, jseg, rec); 3124 } 3125 3126 static void 3127 jremref_write(jremref, jseg, data) 3128 struct jremref *jremref; 3129 struct jseg *jseg; 3130 uint8_t *data; 3131 { 3132 struct jrefrec *rec; 3133 3134 rec = (struct jrefrec *)data; 3135 rec->jr_op = JOP_REMREF; 3136 inoref_write(&jremref->jr_ref, jseg, rec); 3137 } 3138 3139 static void 3140 jmvref_write(jmvref, jseg, data) 3141 struct jmvref *jmvref; 3142 struct jseg *jseg; 3143 uint8_t *data; 3144 { 3145 struct jmvrec *rec; 3146 3147 rec = (struct jmvrec *)data; 3148 rec->jm_op = JOP_MVREF; 3149 rec->jm_ino = jmvref->jm_ino; 3150 rec->jm_parent = jmvref->jm_parent; 3151 rec->jm_oldoff = jmvref->jm_oldoff; 3152 rec->jm_newoff = jmvref->jm_newoff; 3153 } 3154 3155 static void 3156 jnewblk_write(jnewblk, jseg, data) 3157 struct jnewblk *jnewblk; 3158 struct jseg *jseg; 3159 uint8_t *data; 3160 { 3161 struct jblkrec *rec; 3162 3163 jnewblk->jn_jsegdep->jd_seg = jseg; 3164 rec = (struct jblkrec *)data; 3165 rec->jb_op = JOP_NEWBLK; 3166 rec->jb_ino = jnewblk->jn_ino; 3167 rec->jb_blkno = jnewblk->jn_blkno; 3168 rec->jb_lbn = jnewblk->jn_lbn; 3169 rec->jb_frags = jnewblk->jn_frags; 3170 rec->jb_oldfrags = jnewblk->jn_oldfrags; 3171 } 3172 3173 static void 3174 jfreeblk_write(jfreeblk, jseg, data) 3175 struct jfreeblk *jfreeblk; 3176 struct jseg *jseg; 3177 uint8_t *data; 3178 { 3179 struct jblkrec *rec; 3180 3181 jfreeblk->jf_dep.jb_jsegdep->jd_seg = jseg; 3182 rec = (struct jblkrec *)data; 3183 rec->jb_op = JOP_FREEBLK; 3184 rec->jb_ino = jfreeblk->jf_ino; 3185 rec->jb_blkno = jfreeblk->jf_blkno; 3186 rec->jb_lbn = jfreeblk->jf_lbn; 3187 rec->jb_frags = jfreeblk->jf_frags; 3188 rec->jb_oldfrags = 0; 3189 } 3190 3191 static void 3192 jfreefrag_write(jfreefrag, jseg, data) 3193 struct jfreefrag *jfreefrag; 3194 struct jseg *jseg; 3195 uint8_t *data; 3196 { 3197 struct jblkrec *rec; 3198 3199 jfreefrag->fr_jsegdep->jd_seg = jseg; 3200 rec = (struct jblkrec *)data; 3201 rec->jb_op = JOP_FREEBLK; 3202 rec->jb_ino = jfreefrag->fr_ino; 3203 rec->jb_blkno = jfreefrag->fr_blkno; 3204 rec->jb_lbn = jfreefrag->fr_lbn; 3205 rec->jb_frags = jfreefrag->fr_frags; 3206 rec->jb_oldfrags = 0; 3207 } 3208 3209 static void 3210 jtrunc_write(jtrunc, jseg, data) 3211 struct jtrunc *jtrunc; 3212 struct jseg *jseg; 3213 uint8_t *data; 3214 { 3215 struct jtrncrec *rec; 3216 3217 jtrunc->jt_dep.jb_jsegdep->jd_seg = jseg; 3218 rec = (struct jtrncrec *)data; 3219 rec->jt_op = JOP_TRUNC; 3220 rec->jt_ino = jtrunc->jt_ino; 3221 rec->jt_size = jtrunc->jt_size; 3222 rec->jt_extsize = jtrunc->jt_extsize; 3223 } 3224 3225 static void 3226 jfsync_write(jfsync, jseg, data) 3227 struct jfsync *jfsync; 3228 struct jseg *jseg; 3229 uint8_t *data; 3230 { 3231 struct jtrncrec *rec; 3232 3233 rec = (struct jtrncrec *)data; 3234 rec->jt_op = JOP_SYNC; 3235 rec->jt_ino = jfsync->jfs_ino; 3236 rec->jt_size = jfsync->jfs_size; 3237 rec->jt_extsize = jfsync->jfs_extsize; 3238 } 3239 3240 static void 3241 softdep_flushjournal(mp) 3242 struct mount *mp; 3243 { 3244 struct jblocks *jblocks; 3245 struct ufsmount *ump; 3246 3247 if (MOUNTEDSUJ(mp) == 0) 3248 return; 3249 ump = VFSTOUFS(mp); 3250 jblocks = ump->softdep_jblocks; 3251 ACQUIRE_LOCK(ump); 3252 while (ump->softdep_on_journal) { 3253 jblocks->jb_needseg = 1; 3254 softdep_process_journal(mp, NULL, MNT_WAIT); 3255 } 3256 FREE_LOCK(ump); 3257 } 3258 3259 static void softdep_synchronize_completed(struct bio *); 3260 static void softdep_synchronize(struct bio *, struct ufsmount *, void *); 3261 3262 static void 3263 softdep_synchronize_completed(bp) 3264 struct bio *bp; 3265 { 3266 struct jseg *oldest; 3267 struct jseg *jseg; 3268 struct ufsmount *ump; 3269 3270 /* 3271 * caller1 marks the last segment written before we issued the 3272 * synchronize cache. 3273 */ 3274 jseg = bp->bio_caller1; 3275 if (jseg == NULL) { 3276 g_destroy_bio(bp); 3277 return; 3278 } 3279 ump = VFSTOUFS(jseg->js_list.wk_mp); 3280 ACQUIRE_LOCK(ump); 3281 oldest = NULL; 3282 /* 3283 * Mark all the journal entries waiting on the synchronize cache 3284 * as completed so they may continue on. 3285 */ 3286 while (jseg != NULL && (jseg->js_state & COMPLETE) == 0) { 3287 jseg->js_state |= COMPLETE; 3288 oldest = jseg; 3289 jseg = TAILQ_PREV(jseg, jseglst, js_next); 3290 } 3291 /* 3292 * Restart deferred journal entry processing from the oldest 3293 * completed jseg. 3294 */ 3295 if (oldest) 3296 complete_jsegs(oldest); 3297 3298 FREE_LOCK(ump); 3299 g_destroy_bio(bp); 3300 } 3301 3302 /* 3303 * Send BIO_FLUSH/SYNCHRONIZE CACHE to the device to enforce write ordering 3304 * barriers. The journal must be written prior to any blocks that depend 3305 * on it and the journal can not be released until the blocks have be 3306 * written. This code handles both barriers simultaneously. 3307 */ 3308 static void 3309 softdep_synchronize(bp, ump, caller1) 3310 struct bio *bp; 3311 struct ufsmount *ump; 3312 void *caller1; 3313 { 3314 3315 bp->bio_cmd = BIO_FLUSH; 3316 bp->bio_flags |= BIO_ORDERED; 3317 bp->bio_data = NULL; 3318 bp->bio_offset = ump->um_cp->provider->mediasize; 3319 bp->bio_length = 0; 3320 bp->bio_done = softdep_synchronize_completed; 3321 bp->bio_caller1 = caller1; 3322 g_io_request(bp, 3323 (struct g_consumer *)ump->um_devvp->v_bufobj.bo_private); 3324 } 3325 3326 /* 3327 * Flush some journal records to disk. 3328 */ 3329 static void 3330 softdep_process_journal(mp, needwk, flags) 3331 struct mount *mp; 3332 struct worklist *needwk; 3333 int flags; 3334 { 3335 struct jblocks *jblocks; 3336 struct ufsmount *ump; 3337 struct worklist *wk; 3338 struct jseg *jseg; 3339 struct buf *bp; 3340 struct bio *bio; 3341 uint8_t *data; 3342 struct fs *fs; 3343 int shouldflush; 3344 int segwritten; 3345 int jrecmin; /* Minimum records per block. */ 3346 int jrecmax; /* Maximum records per block. */ 3347 int size; 3348 int cnt; 3349 int off; 3350 int devbsize; 3351 3352 if (MOUNTEDSUJ(mp) == 0) 3353 return; 3354 shouldflush = softdep_flushcache; 3355 bio = NULL; 3356 jseg = NULL; 3357 ump = VFSTOUFS(mp); 3358 LOCK_OWNED(ump); 3359 fs = ump->um_fs; 3360 jblocks = ump->softdep_jblocks; 3361 devbsize = ump->um_devvp->v_bufobj.bo_bsize; 3362 /* 3363 * We write anywhere between a disk block and fs block. The upper 3364 * bound is picked to prevent buffer cache fragmentation and limit 3365 * processing time per I/O. 3366 */ 3367 jrecmin = (devbsize / JREC_SIZE) - 1; /* -1 for seg header */ 3368 jrecmax = (fs->fs_bsize / devbsize) * jrecmin; 3369 segwritten = 0; 3370 for (;;) { 3371 cnt = ump->softdep_on_journal; 3372 /* 3373 * Criteria for writing a segment: 3374 * 1) We have a full block. 3375 * 2) We're called from jwait() and haven't found the 3376 * journal item yet. 3377 * 3) Always write if needseg is set. 3378 * 4) If we are called from process_worklist and have 3379 * not yet written anything we write a partial block 3380 * to enforce a 1 second maximum latency on journal 3381 * entries. 3382 */ 3383 if (cnt < (jrecmax - 1) && needwk == NULL && 3384 jblocks->jb_needseg == 0 && (segwritten || cnt == 0)) 3385 break; 3386 cnt++; 3387 /* 3388 * Verify some free journal space. softdep_prealloc() should 3389 * guarantee that we don't run out so this is indicative of 3390 * a problem with the flow control. Try to recover 3391 * gracefully in any event. 3392 */ 3393 while (jblocks->jb_free == 0) { 3394 if (flags != MNT_WAIT) 3395 break; 3396 printf("softdep: Out of journal space!\n"); 3397 softdep_speedup(ump); 3398 msleep(jblocks, LOCK_PTR(ump), PRIBIO, "jblocks", hz); 3399 } 3400 FREE_LOCK(ump); 3401 jseg = malloc(sizeof(*jseg), M_JSEG, M_SOFTDEP_FLAGS); 3402 workitem_alloc(&jseg->js_list, D_JSEG, mp); 3403 LIST_INIT(&jseg->js_entries); 3404 LIST_INIT(&jseg->js_indirs); 3405 jseg->js_state = ATTACHED; 3406 if (shouldflush == 0) 3407 jseg->js_state |= COMPLETE; 3408 else if (bio == NULL) 3409 bio = g_alloc_bio(); 3410 jseg->js_jblocks = jblocks; 3411 bp = geteblk(fs->fs_bsize, 0); 3412 ACQUIRE_LOCK(ump); 3413 /* 3414 * If there was a race while we were allocating the block 3415 * and jseg the entry we care about was likely written. 3416 * We bail out in both the WAIT and NOWAIT case and assume 3417 * the caller will loop if the entry it cares about is 3418 * not written. 3419 */ 3420 cnt = ump->softdep_on_journal; 3421 if (cnt + jblocks->jb_needseg == 0 || jblocks->jb_free == 0) { 3422 bp->b_flags |= B_INVAL | B_NOCACHE; 3423 WORKITEM_FREE(jseg, D_JSEG); 3424 FREE_LOCK(ump); 3425 brelse(bp); 3426 ACQUIRE_LOCK(ump); 3427 break; 3428 } 3429 /* 3430 * Calculate the disk block size required for the available 3431 * records rounded to the min size. 3432 */ 3433 if (cnt == 0) 3434 size = devbsize; 3435 else if (cnt < jrecmax) 3436 size = howmany(cnt, jrecmin) * devbsize; 3437 else 3438 size = fs->fs_bsize; 3439 /* 3440 * Allocate a disk block for this journal data and account 3441 * for truncation of the requested size if enough contiguous 3442 * space was not available. 3443 */ 3444 bp->b_blkno = jblocks_alloc(jblocks, size, &size); 3445 bp->b_lblkno = bp->b_blkno; 3446 bp->b_offset = bp->b_blkno * DEV_BSIZE; 3447 bp->b_bcount = size; 3448 bp->b_flags &= ~B_INVAL; 3449 bp->b_flags |= B_VALIDSUSPWRT | B_NOCOPY; 3450 /* 3451 * Initialize our jseg with cnt records. Assign the next 3452 * sequence number to it and link it in-order. 3453 */ 3454 cnt = MIN(cnt, (size / devbsize) * jrecmin); 3455 jseg->js_buf = bp; 3456 jseg->js_cnt = cnt; 3457 jseg->js_refs = cnt + 1; /* Self ref. */ 3458 jseg->js_size = size; 3459 jseg->js_seq = jblocks->jb_nextseq++; 3460 if (jblocks->jb_oldestseg == NULL) 3461 jblocks->jb_oldestseg = jseg; 3462 jseg->js_oldseq = jblocks->jb_oldestseg->js_seq; 3463 TAILQ_INSERT_TAIL(&jblocks->jb_segs, jseg, js_next); 3464 if (jblocks->jb_writeseg == NULL) 3465 jblocks->jb_writeseg = jseg; 3466 /* 3467 * Start filling in records from the pending list. 3468 */ 3469 data = bp->b_data; 3470 off = 0; 3471 3472 /* 3473 * Always put a header on the first block. 3474 * XXX As with below, there might not be a chance to get 3475 * into the loop. Ensure that something valid is written. 3476 */ 3477 jseg_write(ump, jseg, data); 3478 off += JREC_SIZE; 3479 data = bp->b_data + off; 3480 3481 /* 3482 * XXX Something is wrong here. There's no work to do, 3483 * but we need to perform and I/O and allow it to complete 3484 * anyways. 3485 */ 3486 if (LIST_EMPTY(&ump->softdep_journal_pending)) 3487 stat_emptyjblocks++; 3488 3489 while ((wk = LIST_FIRST(&ump->softdep_journal_pending)) 3490 != NULL) { 3491 if (cnt == 0) 3492 break; 3493 /* Place a segment header on every device block. */ 3494 if ((off % devbsize) == 0) { 3495 jseg_write(ump, jseg, data); 3496 off += JREC_SIZE; 3497 data = bp->b_data + off; 3498 } 3499 if (wk == needwk) 3500 needwk = NULL; 3501 remove_from_journal(wk); 3502 wk->wk_state |= INPROGRESS; 3503 WORKLIST_INSERT(&jseg->js_entries, wk); 3504 switch (wk->wk_type) { 3505 case D_JADDREF: 3506 jaddref_write(WK_JADDREF(wk), jseg, data); 3507 break; 3508 case D_JREMREF: 3509 jremref_write(WK_JREMREF(wk), jseg, data); 3510 break; 3511 case D_JMVREF: 3512 jmvref_write(WK_JMVREF(wk), jseg, data); 3513 break; 3514 case D_JNEWBLK: 3515 jnewblk_write(WK_JNEWBLK(wk), jseg, data); 3516 break; 3517 case D_JFREEBLK: 3518 jfreeblk_write(WK_JFREEBLK(wk), jseg, data); 3519 break; 3520 case D_JFREEFRAG: 3521 jfreefrag_write(WK_JFREEFRAG(wk), jseg, data); 3522 break; 3523 case D_JTRUNC: 3524 jtrunc_write(WK_JTRUNC(wk), jseg, data); 3525 break; 3526 case D_JFSYNC: 3527 jfsync_write(WK_JFSYNC(wk), jseg, data); 3528 break; 3529 default: 3530 panic("process_journal: Unknown type %s", 3531 TYPENAME(wk->wk_type)); 3532 /* NOTREACHED */ 3533 } 3534 off += JREC_SIZE; 3535 data = bp->b_data + off; 3536 cnt--; 3537 } 3538 3539 /* Clear any remaining space so we don't leak kernel data */ 3540 if (size > off) 3541 bzero(data, size - off); 3542 3543 /* 3544 * Write this one buffer and continue. 3545 */ 3546 segwritten = 1; 3547 jblocks->jb_needseg = 0; 3548 WORKLIST_INSERT(&bp->b_dep, &jseg->js_list); 3549 FREE_LOCK(ump); 3550 pbgetvp(ump->um_devvp, bp); 3551 /* 3552 * We only do the blocking wait once we find the journal 3553 * entry we're looking for. 3554 */ 3555 if (needwk == NULL && flags == MNT_WAIT) 3556 bwrite(bp); 3557 else 3558 bawrite(bp); 3559 ACQUIRE_LOCK(ump); 3560 } 3561 /* 3562 * If we wrote a segment issue a synchronize cache so the journal 3563 * is reflected on disk before the data is written. Since reclaiming 3564 * journal space also requires writing a journal record this 3565 * process also enforces a barrier before reclamation. 3566 */ 3567 if (segwritten && shouldflush) { 3568 softdep_synchronize(bio, ump, 3569 TAILQ_LAST(&jblocks->jb_segs, jseglst)); 3570 } else if (bio) 3571 g_destroy_bio(bio); 3572 /* 3573 * If we've suspended the filesystem because we ran out of journal 3574 * space either try to sync it here to make some progress or 3575 * unsuspend it if we already have. 3576 */ 3577 if (flags == 0 && jblocks->jb_suspended) { 3578 if (journal_unsuspend(ump)) 3579 return; 3580 FREE_LOCK(ump); 3581 VFS_SYNC(mp, MNT_NOWAIT); 3582 ffs_sbupdate(ump, MNT_WAIT, 0); 3583 ACQUIRE_LOCK(ump); 3584 } 3585 } 3586 3587 /* 3588 * Complete a jseg, allowing all dependencies awaiting journal writes 3589 * to proceed. Each journal dependency also attaches a jsegdep to dependent 3590 * structures so that the journal segment can be freed to reclaim space. 3591 */ 3592 static void 3593 complete_jseg(jseg) 3594 struct jseg *jseg; 3595 { 3596 struct worklist *wk; 3597 struct jmvref *jmvref; 3598 int waiting; 3599 #ifdef INVARIANTS 3600 int i = 0; 3601 #endif 3602 3603 while ((wk = LIST_FIRST(&jseg->js_entries)) != NULL) { 3604 WORKLIST_REMOVE(wk); 3605 waiting = wk->wk_state & IOWAITING; 3606 wk->wk_state &= ~(INPROGRESS | IOWAITING); 3607 wk->wk_state |= COMPLETE; 3608 KASSERT(i++ < jseg->js_cnt, 3609 ("handle_written_jseg: overflow %d >= %d", 3610 i - 1, jseg->js_cnt)); 3611 switch (wk->wk_type) { 3612 case D_JADDREF: 3613 handle_written_jaddref(WK_JADDREF(wk)); 3614 break; 3615 case D_JREMREF: 3616 handle_written_jremref(WK_JREMREF(wk)); 3617 break; 3618 case D_JMVREF: 3619 rele_jseg(jseg); /* No jsegdep. */ 3620 jmvref = WK_JMVREF(wk); 3621 LIST_REMOVE(jmvref, jm_deps); 3622 if ((jmvref->jm_pagedep->pd_state & ONWORKLIST) == 0) 3623 free_pagedep(jmvref->jm_pagedep); 3624 WORKITEM_FREE(jmvref, D_JMVREF); 3625 break; 3626 case D_JNEWBLK: 3627 handle_written_jnewblk(WK_JNEWBLK(wk)); 3628 break; 3629 case D_JFREEBLK: 3630 handle_written_jblkdep(&WK_JFREEBLK(wk)->jf_dep); 3631 break; 3632 case D_JTRUNC: 3633 handle_written_jblkdep(&WK_JTRUNC(wk)->jt_dep); 3634 break; 3635 case D_JFSYNC: 3636 rele_jseg(jseg); /* No jsegdep. */ 3637 WORKITEM_FREE(wk, D_JFSYNC); 3638 break; 3639 case D_JFREEFRAG: 3640 handle_written_jfreefrag(WK_JFREEFRAG(wk)); 3641 break; 3642 default: 3643 panic("handle_written_jseg: Unknown type %s", 3644 TYPENAME(wk->wk_type)); 3645 /* NOTREACHED */ 3646 } 3647 if (waiting) 3648 wakeup(wk); 3649 } 3650 /* Release the self reference so the structure may be freed. */ 3651 rele_jseg(jseg); 3652 } 3653 3654 /* 3655 * Determine which jsegs are ready for completion processing. Waits for 3656 * synchronize cache to complete as well as forcing in-order completion 3657 * of journal entries. 3658 */ 3659 static void 3660 complete_jsegs(jseg) 3661 struct jseg *jseg; 3662 { 3663 struct jblocks *jblocks; 3664 struct jseg *jsegn; 3665 3666 jblocks = jseg->js_jblocks; 3667 /* 3668 * Don't allow out of order completions. If this isn't the first 3669 * block wait for it to write before we're done. 3670 */ 3671 if (jseg != jblocks->jb_writeseg) 3672 return; 3673 /* Iterate through available jsegs processing their entries. */ 3674 while (jseg && (jseg->js_state & ALLCOMPLETE) == ALLCOMPLETE) { 3675 jblocks->jb_oldestwrseq = jseg->js_oldseq; 3676 jsegn = TAILQ_NEXT(jseg, js_next); 3677 complete_jseg(jseg); 3678 jseg = jsegn; 3679 } 3680 jblocks->jb_writeseg = jseg; 3681 /* 3682 * Attempt to free jsegs now that oldestwrseq may have advanced. 3683 */ 3684 free_jsegs(jblocks); 3685 } 3686 3687 /* 3688 * Mark a jseg as DEPCOMPLETE and throw away the buffer. Attempt to handle 3689 * the final completions. 3690 */ 3691 static void 3692 handle_written_jseg(jseg, bp) 3693 struct jseg *jseg; 3694 struct buf *bp; 3695 { 3696 3697 if (jseg->js_refs == 0) 3698 panic("handle_written_jseg: No self-reference on %p", jseg); 3699 jseg->js_state |= DEPCOMPLETE; 3700 /* 3701 * We'll never need this buffer again, set flags so it will be 3702 * discarded. 3703 */ 3704 bp->b_flags |= B_INVAL | B_NOCACHE; 3705 pbrelvp(bp); 3706 complete_jsegs(jseg); 3707 } 3708 3709 static inline struct jsegdep * 3710 inoref_jseg(inoref) 3711 struct inoref *inoref; 3712 { 3713 struct jsegdep *jsegdep; 3714 3715 jsegdep = inoref->if_jsegdep; 3716 inoref->if_jsegdep = NULL; 3717 3718 return (jsegdep); 3719 } 3720 3721 /* 3722 * Called once a jremref has made it to stable store. The jremref is marked 3723 * complete and we attempt to free it. Any pagedeps writes sleeping waiting 3724 * for the jremref to complete will be awoken by free_jremref. 3725 */ 3726 static void 3727 handle_written_jremref(jremref) 3728 struct jremref *jremref; 3729 { 3730 struct inodedep *inodedep; 3731 struct jsegdep *jsegdep; 3732 struct dirrem *dirrem; 3733 3734 /* Grab the jsegdep. */ 3735 jsegdep = inoref_jseg(&jremref->jr_ref); 3736 /* 3737 * Remove us from the inoref list. 3738 */ 3739 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 3740 0, &inodedep) == 0) 3741 panic("handle_written_jremref: Lost inodedep"); 3742 TAILQ_REMOVE(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 3743 /* 3744 * Complete the dirrem. 3745 */ 3746 dirrem = jremref->jr_dirrem; 3747 jremref->jr_dirrem = NULL; 3748 LIST_REMOVE(jremref, jr_deps); 3749 jsegdep->jd_state |= jremref->jr_state & MKDIR_PARENT; 3750 jwork_insert(&dirrem->dm_jwork, jsegdep); 3751 if (LIST_EMPTY(&dirrem->dm_jremrefhd) && 3752 (dirrem->dm_state & COMPLETE) != 0) 3753 add_to_worklist(&dirrem->dm_list, 0); 3754 free_jremref(jremref); 3755 } 3756 3757 /* 3758 * Called once a jaddref has made it to stable store. The dependency is 3759 * marked complete and any dependent structures are added to the inode 3760 * bufwait list to be completed as soon as it is written. If a bitmap write 3761 * depends on this entry we move the inode into the inodedephd of the 3762 * bmsafemap dependency and attempt to remove the jaddref from the bmsafemap. 3763 */ 3764 static void 3765 handle_written_jaddref(jaddref) 3766 struct jaddref *jaddref; 3767 { 3768 struct jsegdep *jsegdep; 3769 struct inodedep *inodedep; 3770 struct diradd *diradd; 3771 struct mkdir *mkdir; 3772 3773 /* Grab the jsegdep. */ 3774 jsegdep = inoref_jseg(&jaddref->ja_ref); 3775 mkdir = NULL; 3776 diradd = NULL; 3777 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 3778 0, &inodedep) == 0) 3779 panic("handle_written_jaddref: Lost inodedep."); 3780 if (jaddref->ja_diradd == NULL) 3781 panic("handle_written_jaddref: No dependency"); 3782 if (jaddref->ja_diradd->da_list.wk_type == D_DIRADD) { 3783 diradd = jaddref->ja_diradd; 3784 WORKLIST_INSERT(&inodedep->id_bufwait, &diradd->da_list); 3785 } else if (jaddref->ja_state & MKDIR_PARENT) { 3786 mkdir = jaddref->ja_mkdir; 3787 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir->md_list); 3788 } else if (jaddref->ja_state & MKDIR_BODY) 3789 mkdir = jaddref->ja_mkdir; 3790 else 3791 panic("handle_written_jaddref: Unknown dependency %p", 3792 jaddref->ja_diradd); 3793 jaddref->ja_diradd = NULL; /* also clears ja_mkdir */ 3794 /* 3795 * Remove us from the inode list. 3796 */ 3797 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, if_deps); 3798 /* 3799 * The mkdir may be waiting on the jaddref to clear before freeing. 3800 */ 3801 if (mkdir) { 3802 KASSERT(mkdir->md_list.wk_type == D_MKDIR, 3803 ("handle_written_jaddref: Incorrect type for mkdir %s", 3804 TYPENAME(mkdir->md_list.wk_type))); 3805 mkdir->md_jaddref = NULL; 3806 diradd = mkdir->md_diradd; 3807 mkdir->md_state |= DEPCOMPLETE; 3808 complete_mkdir(mkdir); 3809 } 3810 jwork_insert(&diradd->da_jwork, jsegdep); 3811 if (jaddref->ja_state & NEWBLOCK) { 3812 inodedep->id_state |= ONDEPLIST; 3813 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_inodedephd, 3814 inodedep, id_deps); 3815 } 3816 free_jaddref(jaddref); 3817 } 3818 3819 /* 3820 * Called once a jnewblk journal is written. The allocdirect or allocindir 3821 * is placed in the bmsafemap to await notification of a written bitmap. If 3822 * the operation was canceled we add the segdep to the appropriate 3823 * dependency to free the journal space once the canceling operation 3824 * completes. 3825 */ 3826 static void 3827 handle_written_jnewblk(jnewblk) 3828 struct jnewblk *jnewblk; 3829 { 3830 struct bmsafemap *bmsafemap; 3831 struct freefrag *freefrag; 3832 struct freework *freework; 3833 struct jsegdep *jsegdep; 3834 struct newblk *newblk; 3835 3836 /* Grab the jsegdep. */ 3837 jsegdep = jnewblk->jn_jsegdep; 3838 jnewblk->jn_jsegdep = NULL; 3839 if (jnewblk->jn_dep == NULL) 3840 panic("handle_written_jnewblk: No dependency for the segdep."); 3841 switch (jnewblk->jn_dep->wk_type) { 3842 case D_NEWBLK: 3843 case D_ALLOCDIRECT: 3844 case D_ALLOCINDIR: 3845 /* 3846 * Add the written block to the bmsafemap so it can 3847 * be notified when the bitmap is on disk. 3848 */ 3849 newblk = WK_NEWBLK(jnewblk->jn_dep); 3850 newblk->nb_jnewblk = NULL; 3851 if ((newblk->nb_state & GOINGAWAY) == 0) { 3852 bmsafemap = newblk->nb_bmsafemap; 3853 newblk->nb_state |= ONDEPLIST; 3854 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, 3855 nb_deps); 3856 } 3857 jwork_insert(&newblk->nb_jwork, jsegdep); 3858 break; 3859 case D_FREEFRAG: 3860 /* 3861 * A newblock being removed by a freefrag when replaced by 3862 * frag extension. 3863 */ 3864 freefrag = WK_FREEFRAG(jnewblk->jn_dep); 3865 freefrag->ff_jdep = NULL; 3866 jwork_insert(&freefrag->ff_jwork, jsegdep); 3867 break; 3868 case D_FREEWORK: 3869 /* 3870 * A direct block was removed by truncate. 3871 */ 3872 freework = WK_FREEWORK(jnewblk->jn_dep); 3873 freework->fw_jnewblk = NULL; 3874 jwork_insert(&freework->fw_freeblks->fb_jwork, jsegdep); 3875 break; 3876 default: 3877 panic("handle_written_jnewblk: Unknown type %d.", 3878 jnewblk->jn_dep->wk_type); 3879 } 3880 jnewblk->jn_dep = NULL; 3881 free_jnewblk(jnewblk); 3882 } 3883 3884 /* 3885 * Cancel a jfreefrag that won't be needed, probably due to colliding with 3886 * an in-flight allocation that has not yet been committed. Divorce us 3887 * from the freefrag and mark it DEPCOMPLETE so that it may be added 3888 * to the worklist. 3889 */ 3890 static void 3891 cancel_jfreefrag(jfreefrag) 3892 struct jfreefrag *jfreefrag; 3893 { 3894 struct freefrag *freefrag; 3895 3896 if (jfreefrag->fr_jsegdep) { 3897 free_jsegdep(jfreefrag->fr_jsegdep); 3898 jfreefrag->fr_jsegdep = NULL; 3899 } 3900 freefrag = jfreefrag->fr_freefrag; 3901 jfreefrag->fr_freefrag = NULL; 3902 free_jfreefrag(jfreefrag); 3903 freefrag->ff_state |= DEPCOMPLETE; 3904 CTR1(KTR_SUJ, "cancel_jfreefrag: blkno %jd", freefrag->ff_blkno); 3905 } 3906 3907 /* 3908 * Free a jfreefrag when the parent freefrag is rendered obsolete. 3909 */ 3910 static void 3911 free_jfreefrag(jfreefrag) 3912 struct jfreefrag *jfreefrag; 3913 { 3914 3915 if (jfreefrag->fr_state & INPROGRESS) 3916 WORKLIST_REMOVE(&jfreefrag->fr_list); 3917 else if (jfreefrag->fr_state & ONWORKLIST) 3918 remove_from_journal(&jfreefrag->fr_list); 3919 if (jfreefrag->fr_freefrag != NULL) 3920 panic("free_jfreefrag: Still attached to a freefrag."); 3921 WORKITEM_FREE(jfreefrag, D_JFREEFRAG); 3922 } 3923 3924 /* 3925 * Called when the journal write for a jfreefrag completes. The parent 3926 * freefrag is added to the worklist if this completes its dependencies. 3927 */ 3928 static void 3929 handle_written_jfreefrag(jfreefrag) 3930 struct jfreefrag *jfreefrag; 3931 { 3932 struct jsegdep *jsegdep; 3933 struct freefrag *freefrag; 3934 3935 /* Grab the jsegdep. */ 3936 jsegdep = jfreefrag->fr_jsegdep; 3937 jfreefrag->fr_jsegdep = NULL; 3938 freefrag = jfreefrag->fr_freefrag; 3939 if (freefrag == NULL) 3940 panic("handle_written_jfreefrag: No freefrag."); 3941 freefrag->ff_state |= DEPCOMPLETE; 3942 freefrag->ff_jdep = NULL; 3943 jwork_insert(&freefrag->ff_jwork, jsegdep); 3944 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 3945 add_to_worklist(&freefrag->ff_list, 0); 3946 jfreefrag->fr_freefrag = NULL; 3947 free_jfreefrag(jfreefrag); 3948 } 3949 3950 /* 3951 * Called when the journal write for a jfreeblk completes. The jfreeblk 3952 * is removed from the freeblks list of pending journal writes and the 3953 * jsegdep is moved to the freeblks jwork to be completed when all blocks 3954 * have been reclaimed. 3955 */ 3956 static void 3957 handle_written_jblkdep(jblkdep) 3958 struct jblkdep *jblkdep; 3959 { 3960 struct freeblks *freeblks; 3961 struct jsegdep *jsegdep; 3962 3963 /* Grab the jsegdep. */ 3964 jsegdep = jblkdep->jb_jsegdep; 3965 jblkdep->jb_jsegdep = NULL; 3966 freeblks = jblkdep->jb_freeblks; 3967 LIST_REMOVE(jblkdep, jb_deps); 3968 jwork_insert(&freeblks->fb_jwork, jsegdep); 3969 /* 3970 * If the freeblks is all journaled, we can add it to the worklist. 3971 */ 3972 if (LIST_EMPTY(&freeblks->fb_jblkdephd) && 3973 (freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) 3974 add_to_worklist(&freeblks->fb_list, WK_NODELAY); 3975 3976 free_jblkdep(jblkdep); 3977 } 3978 3979 static struct jsegdep * 3980 newjsegdep(struct worklist *wk) 3981 { 3982 struct jsegdep *jsegdep; 3983 3984 jsegdep = malloc(sizeof(*jsegdep), M_JSEGDEP, M_SOFTDEP_FLAGS); 3985 workitem_alloc(&jsegdep->jd_list, D_JSEGDEP, wk->wk_mp); 3986 jsegdep->jd_seg = NULL; 3987 3988 return (jsegdep); 3989 } 3990 3991 static struct jmvref * 3992 newjmvref(dp, ino, oldoff, newoff) 3993 struct inode *dp; 3994 ino_t ino; 3995 off_t oldoff; 3996 off_t newoff; 3997 { 3998 struct jmvref *jmvref; 3999 4000 jmvref = malloc(sizeof(*jmvref), M_JMVREF, M_SOFTDEP_FLAGS); 4001 workitem_alloc(&jmvref->jm_list, D_JMVREF, ITOVFS(dp)); 4002 jmvref->jm_list.wk_state = ATTACHED | DEPCOMPLETE; 4003 jmvref->jm_parent = dp->i_number; 4004 jmvref->jm_ino = ino; 4005 jmvref->jm_oldoff = oldoff; 4006 jmvref->jm_newoff = newoff; 4007 4008 return (jmvref); 4009 } 4010 4011 /* 4012 * Allocate a new jremref that tracks the removal of ip from dp with the 4013 * directory entry offset of diroff. Mark the entry as ATTACHED and 4014 * DEPCOMPLETE as we have all the information required for the journal write 4015 * and the directory has already been removed from the buffer. The caller 4016 * is responsible for linking the jremref into the pagedep and adding it 4017 * to the journal to write. The MKDIR_PARENT flag is set if we're doing 4018 * a DOTDOT addition so handle_workitem_remove() can properly assign 4019 * the jsegdep when we're done. 4020 */ 4021 static struct jremref * 4022 newjremref(struct dirrem *dirrem, struct inode *dp, struct inode *ip, 4023 off_t diroff, nlink_t nlink) 4024 { 4025 struct jremref *jremref; 4026 4027 jremref = malloc(sizeof(*jremref), M_JREMREF, M_SOFTDEP_FLAGS); 4028 workitem_alloc(&jremref->jr_list, D_JREMREF, ITOVFS(dp)); 4029 jremref->jr_state = ATTACHED; 4030 newinoref(&jremref->jr_ref, ip->i_number, dp->i_number, diroff, 4031 nlink, ip->i_mode); 4032 jremref->jr_dirrem = dirrem; 4033 4034 return (jremref); 4035 } 4036 4037 static inline void 4038 newinoref(struct inoref *inoref, ino_t ino, ino_t parent, off_t diroff, 4039 nlink_t nlink, uint16_t mode) 4040 { 4041 4042 inoref->if_jsegdep = newjsegdep(&inoref->if_list); 4043 inoref->if_diroff = diroff; 4044 inoref->if_ino = ino; 4045 inoref->if_parent = parent; 4046 inoref->if_nlink = nlink; 4047 inoref->if_mode = mode; 4048 } 4049 4050 /* 4051 * Allocate a new jaddref to track the addition of ino to dp at diroff. The 4052 * directory offset may not be known until later. The caller is responsible 4053 * adding the entry to the journal when this information is available. nlink 4054 * should be the link count prior to the addition and mode is only required 4055 * to have the correct FMT. 4056 */ 4057 static struct jaddref * 4058 newjaddref(struct inode *dp, ino_t ino, off_t diroff, int16_t nlink, 4059 uint16_t mode) 4060 { 4061 struct jaddref *jaddref; 4062 4063 jaddref = malloc(sizeof(*jaddref), M_JADDREF, M_SOFTDEP_FLAGS); 4064 workitem_alloc(&jaddref->ja_list, D_JADDREF, ITOVFS(dp)); 4065 jaddref->ja_state = ATTACHED; 4066 jaddref->ja_mkdir = NULL; 4067 newinoref(&jaddref->ja_ref, ino, dp->i_number, diroff, nlink, mode); 4068 4069 return (jaddref); 4070 } 4071 4072 /* 4073 * Create a new free dependency for a freework. The caller is responsible 4074 * for adjusting the reference count when it has the lock held. The freedep 4075 * will track an outstanding bitmap write that will ultimately clear the 4076 * freework to continue. 4077 */ 4078 static struct freedep * 4079 newfreedep(struct freework *freework) 4080 { 4081 struct freedep *freedep; 4082 4083 freedep = malloc(sizeof(*freedep), M_FREEDEP, M_SOFTDEP_FLAGS); 4084 workitem_alloc(&freedep->fd_list, D_FREEDEP, freework->fw_list.wk_mp); 4085 freedep->fd_freework = freework; 4086 4087 return (freedep); 4088 } 4089 4090 /* 4091 * Free a freedep structure once the buffer it is linked to is written. If 4092 * this is the last reference to the freework schedule it for completion. 4093 */ 4094 static void 4095 free_freedep(freedep) 4096 struct freedep *freedep; 4097 { 4098 struct freework *freework; 4099 4100 freework = freedep->fd_freework; 4101 freework->fw_freeblks->fb_cgwait--; 4102 if (--freework->fw_ref == 0) 4103 freework_enqueue(freework); 4104 WORKITEM_FREE(freedep, D_FREEDEP); 4105 } 4106 4107 /* 4108 * Allocate a new freework structure that may be a level in an indirect 4109 * when parent is not NULL or a top level block when it is. The top level 4110 * freework structures are allocated without the per-filesystem lock held 4111 * and before the freeblks is visible outside of softdep_setup_freeblocks(). 4112 */ 4113 static struct freework * 4114 newfreework(ump, freeblks, parent, lbn, nb, frags, off, journal) 4115 struct ufsmount *ump; 4116 struct freeblks *freeblks; 4117 struct freework *parent; 4118 ufs_lbn_t lbn; 4119 ufs2_daddr_t nb; 4120 int frags; 4121 int off; 4122 int journal; 4123 { 4124 struct freework *freework; 4125 4126 freework = malloc(sizeof(*freework), M_FREEWORK, M_SOFTDEP_FLAGS); 4127 workitem_alloc(&freework->fw_list, D_FREEWORK, freeblks->fb_list.wk_mp); 4128 freework->fw_state = ATTACHED; 4129 freework->fw_jnewblk = NULL; 4130 freework->fw_freeblks = freeblks; 4131 freework->fw_parent = parent; 4132 freework->fw_lbn = lbn; 4133 freework->fw_blkno = nb; 4134 freework->fw_frags = frags; 4135 freework->fw_indir = NULL; 4136 freework->fw_ref = (MOUNTEDSUJ(UFSTOVFS(ump)) == 0 || lbn >= -NXADDR) 4137 ? 0 : NINDIR(ump->um_fs) + 1; 4138 freework->fw_start = freework->fw_off = off; 4139 if (journal) 4140 newjfreeblk(freeblks, lbn, nb, frags); 4141 if (parent == NULL) { 4142 ACQUIRE_LOCK(ump); 4143 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list); 4144 freeblks->fb_ref++; 4145 FREE_LOCK(ump); 4146 } 4147 4148 return (freework); 4149 } 4150 4151 /* 4152 * Eliminate a jfreeblk for a block that does not need journaling. 4153 */ 4154 static void 4155 cancel_jfreeblk(freeblks, blkno) 4156 struct freeblks *freeblks; 4157 ufs2_daddr_t blkno; 4158 { 4159 struct jfreeblk *jfreeblk; 4160 struct jblkdep *jblkdep; 4161 4162 LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps) { 4163 if (jblkdep->jb_list.wk_type != D_JFREEBLK) 4164 continue; 4165 jfreeblk = WK_JFREEBLK(&jblkdep->jb_list); 4166 if (jfreeblk->jf_blkno == blkno) 4167 break; 4168 } 4169 if (jblkdep == NULL) 4170 return; 4171 CTR1(KTR_SUJ, "cancel_jfreeblk: blkno %jd", blkno); 4172 free_jsegdep(jblkdep->jb_jsegdep); 4173 LIST_REMOVE(jblkdep, jb_deps); 4174 WORKITEM_FREE(jfreeblk, D_JFREEBLK); 4175 } 4176 4177 /* 4178 * Allocate a new jfreeblk to journal top level block pointer when truncating 4179 * a file. The caller must add this to the worklist when the per-filesystem 4180 * lock is held. 4181 */ 4182 static struct jfreeblk * 4183 newjfreeblk(freeblks, lbn, blkno, frags) 4184 struct freeblks *freeblks; 4185 ufs_lbn_t lbn; 4186 ufs2_daddr_t blkno; 4187 int frags; 4188 { 4189 struct jfreeblk *jfreeblk; 4190 4191 jfreeblk = malloc(sizeof(*jfreeblk), M_JFREEBLK, M_SOFTDEP_FLAGS); 4192 workitem_alloc(&jfreeblk->jf_dep.jb_list, D_JFREEBLK, 4193 freeblks->fb_list.wk_mp); 4194 jfreeblk->jf_dep.jb_jsegdep = newjsegdep(&jfreeblk->jf_dep.jb_list); 4195 jfreeblk->jf_dep.jb_freeblks = freeblks; 4196 jfreeblk->jf_ino = freeblks->fb_inum; 4197 jfreeblk->jf_lbn = lbn; 4198 jfreeblk->jf_blkno = blkno; 4199 jfreeblk->jf_frags = frags; 4200 LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jfreeblk->jf_dep, jb_deps); 4201 4202 return (jfreeblk); 4203 } 4204 4205 /* 4206 * The journal is only prepared to handle full-size block numbers, so we 4207 * have to adjust the record to reflect the change to a full-size block. 4208 * For example, suppose we have a block made up of fragments 8-15 and 4209 * want to free its last two fragments. We are given a request that says: 4210 * FREEBLK ino=5, blkno=14, lbn=0, frags=2, oldfrags=0 4211 * where frags are the number of fragments to free and oldfrags are the 4212 * number of fragments to keep. To block align it, we have to change it to 4213 * have a valid full-size blkno, so it becomes: 4214 * FREEBLK ino=5, blkno=8, lbn=0, frags=2, oldfrags=6 4215 */ 4216 static void 4217 adjust_newfreework(freeblks, frag_offset) 4218 struct freeblks *freeblks; 4219 int frag_offset; 4220 { 4221 struct jfreeblk *jfreeblk; 4222 4223 KASSERT((LIST_FIRST(&freeblks->fb_jblkdephd) != NULL && 4224 LIST_FIRST(&freeblks->fb_jblkdephd)->jb_list.wk_type == D_JFREEBLK), 4225 ("adjust_newfreework: Missing freeblks dependency")); 4226 4227 jfreeblk = WK_JFREEBLK(LIST_FIRST(&freeblks->fb_jblkdephd)); 4228 jfreeblk->jf_blkno -= frag_offset; 4229 jfreeblk->jf_frags += frag_offset; 4230 } 4231 4232 /* 4233 * Allocate a new jtrunc to track a partial truncation. 4234 */ 4235 static struct jtrunc * 4236 newjtrunc(freeblks, size, extsize) 4237 struct freeblks *freeblks; 4238 off_t size; 4239 int extsize; 4240 { 4241 struct jtrunc *jtrunc; 4242 4243 jtrunc = malloc(sizeof(*jtrunc), M_JTRUNC, M_SOFTDEP_FLAGS); 4244 workitem_alloc(&jtrunc->jt_dep.jb_list, D_JTRUNC, 4245 freeblks->fb_list.wk_mp); 4246 jtrunc->jt_dep.jb_jsegdep = newjsegdep(&jtrunc->jt_dep.jb_list); 4247 jtrunc->jt_dep.jb_freeblks = freeblks; 4248 jtrunc->jt_ino = freeblks->fb_inum; 4249 jtrunc->jt_size = size; 4250 jtrunc->jt_extsize = extsize; 4251 LIST_INSERT_HEAD(&freeblks->fb_jblkdephd, &jtrunc->jt_dep, jb_deps); 4252 4253 return (jtrunc); 4254 } 4255 4256 /* 4257 * If we're canceling a new bitmap we have to search for another ref 4258 * to move into the bmsafemap dep. This might be better expressed 4259 * with another structure. 4260 */ 4261 static void 4262 move_newblock_dep(jaddref, inodedep) 4263 struct jaddref *jaddref; 4264 struct inodedep *inodedep; 4265 { 4266 struct inoref *inoref; 4267 struct jaddref *jaddrefn; 4268 4269 jaddrefn = NULL; 4270 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 4271 inoref = TAILQ_NEXT(inoref, if_deps)) { 4272 if ((jaddref->ja_state & NEWBLOCK) && 4273 inoref->if_list.wk_type == D_JADDREF) { 4274 jaddrefn = (struct jaddref *)inoref; 4275 break; 4276 } 4277 } 4278 if (jaddrefn == NULL) 4279 return; 4280 jaddrefn->ja_state &= ~(ATTACHED | UNDONE); 4281 jaddrefn->ja_state |= jaddref->ja_state & 4282 (ATTACHED | UNDONE | NEWBLOCK); 4283 jaddref->ja_state &= ~(ATTACHED | UNDONE | NEWBLOCK); 4284 jaddref->ja_state |= ATTACHED; 4285 LIST_REMOVE(jaddref, ja_bmdeps); 4286 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_jaddrefhd, jaddrefn, 4287 ja_bmdeps); 4288 } 4289 4290 /* 4291 * Cancel a jaddref either before it has been written or while it is being 4292 * written. This happens when a link is removed before the add reaches 4293 * the disk. The jaddref dependency is kept linked into the bmsafemap 4294 * and inode to prevent the link count or bitmap from reaching the disk 4295 * until handle_workitem_remove() re-adjusts the counts and bitmaps as 4296 * required. 4297 * 4298 * Returns 1 if the canceled addref requires journaling of the remove and 4299 * 0 otherwise. 4300 */ 4301 static int 4302 cancel_jaddref(jaddref, inodedep, wkhd) 4303 struct jaddref *jaddref; 4304 struct inodedep *inodedep; 4305 struct workhead *wkhd; 4306 { 4307 struct inoref *inoref; 4308 struct jsegdep *jsegdep; 4309 int needsj; 4310 4311 KASSERT((jaddref->ja_state & COMPLETE) == 0, 4312 ("cancel_jaddref: Canceling complete jaddref")); 4313 if (jaddref->ja_state & (INPROGRESS | COMPLETE)) 4314 needsj = 1; 4315 else 4316 needsj = 0; 4317 if (inodedep == NULL) 4318 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 4319 0, &inodedep) == 0) 4320 panic("cancel_jaddref: Lost inodedep"); 4321 /* 4322 * We must adjust the nlink of any reference operation that follows 4323 * us so that it is consistent with the in-memory reference. This 4324 * ensures that inode nlink rollbacks always have the correct link. 4325 */ 4326 if (needsj == 0) { 4327 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 4328 inoref = TAILQ_NEXT(inoref, if_deps)) { 4329 if (inoref->if_state & GOINGAWAY) 4330 break; 4331 inoref->if_nlink--; 4332 } 4333 } 4334 jsegdep = inoref_jseg(&jaddref->ja_ref); 4335 if (jaddref->ja_state & NEWBLOCK) 4336 move_newblock_dep(jaddref, inodedep); 4337 wake_worklist(&jaddref->ja_list); 4338 jaddref->ja_mkdir = NULL; 4339 if (jaddref->ja_state & INPROGRESS) { 4340 jaddref->ja_state &= ~INPROGRESS; 4341 WORKLIST_REMOVE(&jaddref->ja_list); 4342 jwork_insert(wkhd, jsegdep); 4343 } else { 4344 free_jsegdep(jsegdep); 4345 if (jaddref->ja_state & DEPCOMPLETE) 4346 remove_from_journal(&jaddref->ja_list); 4347 } 4348 jaddref->ja_state |= (GOINGAWAY | DEPCOMPLETE); 4349 /* 4350 * Leave NEWBLOCK jaddrefs on the inodedep so handle_workitem_remove 4351 * can arrange for them to be freed with the bitmap. Otherwise we 4352 * no longer need this addref attached to the inoreflst and it 4353 * will incorrectly adjust nlink if we leave it. 4354 */ 4355 if ((jaddref->ja_state & NEWBLOCK) == 0) { 4356 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 4357 if_deps); 4358 jaddref->ja_state |= COMPLETE; 4359 free_jaddref(jaddref); 4360 return (needsj); 4361 } 4362 /* 4363 * Leave the head of the list for jsegdeps for fast merging. 4364 */ 4365 if (LIST_FIRST(wkhd) != NULL) { 4366 jaddref->ja_state |= ONWORKLIST; 4367 LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jaddref->ja_list, wk_list); 4368 } else 4369 WORKLIST_INSERT(wkhd, &jaddref->ja_list); 4370 4371 return (needsj); 4372 } 4373 4374 /* 4375 * Attempt to free a jaddref structure when some work completes. This 4376 * should only succeed once the entry is written and all dependencies have 4377 * been notified. 4378 */ 4379 static void 4380 free_jaddref(jaddref) 4381 struct jaddref *jaddref; 4382 { 4383 4384 if ((jaddref->ja_state & ALLCOMPLETE) != ALLCOMPLETE) 4385 return; 4386 if (jaddref->ja_ref.if_jsegdep) 4387 panic("free_jaddref: segdep attached to jaddref %p(0x%X)\n", 4388 jaddref, jaddref->ja_state); 4389 if (jaddref->ja_state & NEWBLOCK) 4390 LIST_REMOVE(jaddref, ja_bmdeps); 4391 if (jaddref->ja_state & (INPROGRESS | ONWORKLIST)) 4392 panic("free_jaddref: Bad state %p(0x%X)", 4393 jaddref, jaddref->ja_state); 4394 if (jaddref->ja_mkdir != NULL) 4395 panic("free_jaddref: Work pending, 0x%X\n", jaddref->ja_state); 4396 WORKITEM_FREE(jaddref, D_JADDREF); 4397 } 4398 4399 /* 4400 * Free a jremref structure once it has been written or discarded. 4401 */ 4402 static void 4403 free_jremref(jremref) 4404 struct jremref *jremref; 4405 { 4406 4407 if (jremref->jr_ref.if_jsegdep) 4408 free_jsegdep(jremref->jr_ref.if_jsegdep); 4409 if (jremref->jr_state & INPROGRESS) 4410 panic("free_jremref: IO still pending"); 4411 WORKITEM_FREE(jremref, D_JREMREF); 4412 } 4413 4414 /* 4415 * Free a jnewblk structure. 4416 */ 4417 static void 4418 free_jnewblk(jnewblk) 4419 struct jnewblk *jnewblk; 4420 { 4421 4422 if ((jnewblk->jn_state & ALLCOMPLETE) != ALLCOMPLETE) 4423 return; 4424 LIST_REMOVE(jnewblk, jn_deps); 4425 if (jnewblk->jn_dep != NULL) 4426 panic("free_jnewblk: Dependency still attached."); 4427 WORKITEM_FREE(jnewblk, D_JNEWBLK); 4428 } 4429 4430 /* 4431 * Cancel a jnewblk which has been been made redundant by frag extension. 4432 */ 4433 static void 4434 cancel_jnewblk(jnewblk, wkhd) 4435 struct jnewblk *jnewblk; 4436 struct workhead *wkhd; 4437 { 4438 struct jsegdep *jsegdep; 4439 4440 CTR1(KTR_SUJ, "cancel_jnewblk: blkno %jd", jnewblk->jn_blkno); 4441 jsegdep = jnewblk->jn_jsegdep; 4442 if (jnewblk->jn_jsegdep == NULL || jnewblk->jn_dep == NULL) 4443 panic("cancel_jnewblk: Invalid state"); 4444 jnewblk->jn_jsegdep = NULL; 4445 jnewblk->jn_dep = NULL; 4446 jnewblk->jn_state |= GOINGAWAY; 4447 if (jnewblk->jn_state & INPROGRESS) { 4448 jnewblk->jn_state &= ~INPROGRESS; 4449 WORKLIST_REMOVE(&jnewblk->jn_list); 4450 jwork_insert(wkhd, jsegdep); 4451 } else { 4452 free_jsegdep(jsegdep); 4453 remove_from_journal(&jnewblk->jn_list); 4454 } 4455 wake_worklist(&jnewblk->jn_list); 4456 WORKLIST_INSERT(wkhd, &jnewblk->jn_list); 4457 } 4458 4459 static void 4460 free_jblkdep(jblkdep) 4461 struct jblkdep *jblkdep; 4462 { 4463 4464 if (jblkdep->jb_list.wk_type == D_JFREEBLK) 4465 WORKITEM_FREE(jblkdep, D_JFREEBLK); 4466 else if (jblkdep->jb_list.wk_type == D_JTRUNC) 4467 WORKITEM_FREE(jblkdep, D_JTRUNC); 4468 else 4469 panic("free_jblkdep: Unexpected type %s", 4470 TYPENAME(jblkdep->jb_list.wk_type)); 4471 } 4472 4473 /* 4474 * Free a single jseg once it is no longer referenced in memory or on 4475 * disk. Reclaim journal blocks and dependencies waiting for the segment 4476 * to disappear. 4477 */ 4478 static void 4479 free_jseg(jseg, jblocks) 4480 struct jseg *jseg; 4481 struct jblocks *jblocks; 4482 { 4483 struct freework *freework; 4484 4485 /* 4486 * Free freework structures that were lingering to indicate freed 4487 * indirect blocks that forced journal write ordering on reallocate. 4488 */ 4489 while ((freework = LIST_FIRST(&jseg->js_indirs)) != NULL) 4490 indirblk_remove(freework); 4491 if (jblocks->jb_oldestseg == jseg) 4492 jblocks->jb_oldestseg = TAILQ_NEXT(jseg, js_next); 4493 TAILQ_REMOVE(&jblocks->jb_segs, jseg, js_next); 4494 jblocks_free(jblocks, jseg->js_list.wk_mp, jseg->js_size); 4495 KASSERT(LIST_EMPTY(&jseg->js_entries), 4496 ("free_jseg: Freed jseg has valid entries.")); 4497 WORKITEM_FREE(jseg, D_JSEG); 4498 } 4499 4500 /* 4501 * Free all jsegs that meet the criteria for being reclaimed and update 4502 * oldestseg. 4503 */ 4504 static void 4505 free_jsegs(jblocks) 4506 struct jblocks *jblocks; 4507 { 4508 struct jseg *jseg; 4509 4510 /* 4511 * Free only those jsegs which have none allocated before them to 4512 * preserve the journal space ordering. 4513 */ 4514 while ((jseg = TAILQ_FIRST(&jblocks->jb_segs)) != NULL) { 4515 /* 4516 * Only reclaim space when nothing depends on this journal 4517 * set and another set has written that it is no longer 4518 * valid. 4519 */ 4520 if (jseg->js_refs != 0) { 4521 jblocks->jb_oldestseg = jseg; 4522 return; 4523 } 4524 if ((jseg->js_state & ALLCOMPLETE) != ALLCOMPLETE) 4525 break; 4526 if (jseg->js_seq > jblocks->jb_oldestwrseq) 4527 break; 4528 /* 4529 * We can free jsegs that didn't write entries when 4530 * oldestwrseq == js_seq. 4531 */ 4532 if (jseg->js_seq == jblocks->jb_oldestwrseq && 4533 jseg->js_cnt != 0) 4534 break; 4535 free_jseg(jseg, jblocks); 4536 } 4537 /* 4538 * If we exited the loop above we still must discover the 4539 * oldest valid segment. 4540 */ 4541 if (jseg) 4542 for (jseg = jblocks->jb_oldestseg; jseg != NULL; 4543 jseg = TAILQ_NEXT(jseg, js_next)) 4544 if (jseg->js_refs != 0) 4545 break; 4546 jblocks->jb_oldestseg = jseg; 4547 /* 4548 * The journal has no valid records but some jsegs may still be 4549 * waiting on oldestwrseq to advance. We force a small record 4550 * out to permit these lingering records to be reclaimed. 4551 */ 4552 if (jblocks->jb_oldestseg == NULL && !TAILQ_EMPTY(&jblocks->jb_segs)) 4553 jblocks->jb_needseg = 1; 4554 } 4555 4556 /* 4557 * Release one reference to a jseg and free it if the count reaches 0. This 4558 * should eventually reclaim journal space as well. 4559 */ 4560 static void 4561 rele_jseg(jseg) 4562 struct jseg *jseg; 4563 { 4564 4565 KASSERT(jseg->js_refs > 0, 4566 ("free_jseg: Invalid refcnt %d", jseg->js_refs)); 4567 if (--jseg->js_refs != 0) 4568 return; 4569 free_jsegs(jseg->js_jblocks); 4570 } 4571 4572 /* 4573 * Release a jsegdep and decrement the jseg count. 4574 */ 4575 static void 4576 free_jsegdep(jsegdep) 4577 struct jsegdep *jsegdep; 4578 { 4579 4580 if (jsegdep->jd_seg) 4581 rele_jseg(jsegdep->jd_seg); 4582 WORKITEM_FREE(jsegdep, D_JSEGDEP); 4583 } 4584 4585 /* 4586 * Wait for a journal item to make it to disk. Initiate journal processing 4587 * if required. 4588 */ 4589 static int 4590 jwait(wk, waitfor) 4591 struct worklist *wk; 4592 int waitfor; 4593 { 4594 4595 LOCK_OWNED(VFSTOUFS(wk->wk_mp)); 4596 /* 4597 * Blocking journal waits cause slow synchronous behavior. Record 4598 * stats on the frequency of these blocking operations. 4599 */ 4600 if (waitfor == MNT_WAIT) { 4601 stat_journal_wait++; 4602 switch (wk->wk_type) { 4603 case D_JREMREF: 4604 case D_JMVREF: 4605 stat_jwait_filepage++; 4606 break; 4607 case D_JTRUNC: 4608 case D_JFREEBLK: 4609 stat_jwait_freeblks++; 4610 break; 4611 case D_JNEWBLK: 4612 stat_jwait_newblk++; 4613 break; 4614 case D_JADDREF: 4615 stat_jwait_inode++; 4616 break; 4617 default: 4618 break; 4619 } 4620 } 4621 /* 4622 * If IO has not started we process the journal. We can't mark the 4623 * worklist item as IOWAITING because we drop the lock while 4624 * processing the journal and the worklist entry may be freed after 4625 * this point. The caller may call back in and re-issue the request. 4626 */ 4627 if ((wk->wk_state & INPROGRESS) == 0) { 4628 softdep_process_journal(wk->wk_mp, wk, waitfor); 4629 if (waitfor != MNT_WAIT) 4630 return (EBUSY); 4631 return (0); 4632 } 4633 if (waitfor != MNT_WAIT) 4634 return (EBUSY); 4635 wait_worklist(wk, "jwait"); 4636 return (0); 4637 } 4638 4639 /* 4640 * Lookup an inodedep based on an inode pointer and set the nlinkdelta as 4641 * appropriate. This is a convenience function to reduce duplicate code 4642 * for the setup and revert functions below. 4643 */ 4644 static struct inodedep * 4645 inodedep_lookup_ip(ip) 4646 struct inode *ip; 4647 { 4648 struct inodedep *inodedep; 4649 4650 KASSERT(ip->i_nlink >= ip->i_effnlink, 4651 ("inodedep_lookup_ip: bad delta")); 4652 (void) inodedep_lookup(ITOVFS(ip), ip->i_number, DEPALLOC, 4653 &inodedep); 4654 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 4655 KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked")); 4656 4657 return (inodedep); 4658 } 4659 4660 /* 4661 * Called prior to creating a new inode and linking it to a directory. The 4662 * jaddref structure must already be allocated by softdep_setup_inomapdep 4663 * and it is discovered here so we can initialize the mode and update 4664 * nlinkdelta. 4665 */ 4666 void 4667 softdep_setup_create(dp, ip) 4668 struct inode *dp; 4669 struct inode *ip; 4670 { 4671 struct inodedep *inodedep; 4672 struct jaddref *jaddref; 4673 struct vnode *dvp; 4674 4675 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4676 ("softdep_setup_create called on non-softdep filesystem")); 4677 KASSERT(ip->i_nlink == 1, 4678 ("softdep_setup_create: Invalid link count.")); 4679 dvp = ITOV(dp); 4680 ACQUIRE_LOCK(ITOUMP(dp)); 4681 inodedep = inodedep_lookup_ip(ip); 4682 if (DOINGSUJ(dvp)) { 4683 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4684 inoreflst); 4685 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 4686 ("softdep_setup_create: No addref structure present.")); 4687 } 4688 softdep_prelink(dvp, NULL); 4689 FREE_LOCK(ITOUMP(dp)); 4690 } 4691 4692 /* 4693 * Create a jaddref structure to track the addition of a DOTDOT link when 4694 * we are reparenting an inode as part of a rename. This jaddref will be 4695 * found by softdep_setup_directory_change. Adjusts nlinkdelta for 4696 * non-journaling softdep. 4697 */ 4698 void 4699 softdep_setup_dotdot_link(dp, ip) 4700 struct inode *dp; 4701 struct inode *ip; 4702 { 4703 struct inodedep *inodedep; 4704 struct jaddref *jaddref; 4705 struct vnode *dvp; 4706 4707 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4708 ("softdep_setup_dotdot_link called on non-softdep filesystem")); 4709 dvp = ITOV(dp); 4710 jaddref = NULL; 4711 /* 4712 * We don't set MKDIR_PARENT as this is not tied to a mkdir and 4713 * is used as a normal link would be. 4714 */ 4715 if (DOINGSUJ(dvp)) 4716 jaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 4717 dp->i_effnlink - 1, dp->i_mode); 4718 ACQUIRE_LOCK(ITOUMP(dp)); 4719 inodedep = inodedep_lookup_ip(dp); 4720 if (jaddref) 4721 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 4722 if_deps); 4723 softdep_prelink(dvp, ITOV(ip)); 4724 FREE_LOCK(ITOUMP(dp)); 4725 } 4726 4727 /* 4728 * Create a jaddref structure to track a new link to an inode. The directory 4729 * offset is not known until softdep_setup_directory_add or 4730 * softdep_setup_directory_change. Adjusts nlinkdelta for non-journaling 4731 * softdep. 4732 */ 4733 void 4734 softdep_setup_link(dp, ip) 4735 struct inode *dp; 4736 struct inode *ip; 4737 { 4738 struct inodedep *inodedep; 4739 struct jaddref *jaddref; 4740 struct vnode *dvp; 4741 4742 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4743 ("softdep_setup_link called on non-softdep filesystem")); 4744 dvp = ITOV(dp); 4745 jaddref = NULL; 4746 if (DOINGSUJ(dvp)) 4747 jaddref = newjaddref(dp, ip->i_number, 0, ip->i_effnlink - 1, 4748 ip->i_mode); 4749 ACQUIRE_LOCK(ITOUMP(dp)); 4750 inodedep = inodedep_lookup_ip(ip); 4751 if (jaddref) 4752 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 4753 if_deps); 4754 softdep_prelink(dvp, ITOV(ip)); 4755 FREE_LOCK(ITOUMP(dp)); 4756 } 4757 4758 /* 4759 * Called to create the jaddref structures to track . and .. references as 4760 * well as lookup and further initialize the incomplete jaddref created 4761 * by softdep_setup_inomapdep when the inode was allocated. Adjusts 4762 * nlinkdelta for non-journaling softdep. 4763 */ 4764 void 4765 softdep_setup_mkdir(dp, ip) 4766 struct inode *dp; 4767 struct inode *ip; 4768 { 4769 struct inodedep *inodedep; 4770 struct jaddref *dotdotaddref; 4771 struct jaddref *dotaddref; 4772 struct jaddref *jaddref; 4773 struct vnode *dvp; 4774 4775 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4776 ("softdep_setup_mkdir called on non-softdep filesystem")); 4777 dvp = ITOV(dp); 4778 dotaddref = dotdotaddref = NULL; 4779 if (DOINGSUJ(dvp)) { 4780 dotaddref = newjaddref(ip, ip->i_number, DOT_OFFSET, 1, 4781 ip->i_mode); 4782 dotaddref->ja_state |= MKDIR_BODY; 4783 dotdotaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 4784 dp->i_effnlink - 1, dp->i_mode); 4785 dotdotaddref->ja_state |= MKDIR_PARENT; 4786 } 4787 ACQUIRE_LOCK(ITOUMP(dp)); 4788 inodedep = inodedep_lookup_ip(ip); 4789 if (DOINGSUJ(dvp)) { 4790 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4791 inoreflst); 4792 KASSERT(jaddref != NULL, 4793 ("softdep_setup_mkdir: No addref structure present.")); 4794 KASSERT(jaddref->ja_parent == dp->i_number, 4795 ("softdep_setup_mkdir: bad parent %ju", 4796 (uintmax_t)jaddref->ja_parent)); 4797 TAILQ_INSERT_BEFORE(&jaddref->ja_ref, &dotaddref->ja_ref, 4798 if_deps); 4799 } 4800 inodedep = inodedep_lookup_ip(dp); 4801 if (DOINGSUJ(dvp)) 4802 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, 4803 &dotdotaddref->ja_ref, if_deps); 4804 softdep_prelink(ITOV(dp), NULL); 4805 FREE_LOCK(ITOUMP(dp)); 4806 } 4807 4808 /* 4809 * Called to track nlinkdelta of the inode and parent directories prior to 4810 * unlinking a directory. 4811 */ 4812 void 4813 softdep_setup_rmdir(dp, ip) 4814 struct inode *dp; 4815 struct inode *ip; 4816 { 4817 struct vnode *dvp; 4818 4819 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4820 ("softdep_setup_rmdir called on non-softdep filesystem")); 4821 dvp = ITOV(dp); 4822 ACQUIRE_LOCK(ITOUMP(dp)); 4823 (void) inodedep_lookup_ip(ip); 4824 (void) inodedep_lookup_ip(dp); 4825 softdep_prelink(dvp, ITOV(ip)); 4826 FREE_LOCK(ITOUMP(dp)); 4827 } 4828 4829 /* 4830 * Called to track nlinkdelta of the inode and parent directories prior to 4831 * unlink. 4832 */ 4833 void 4834 softdep_setup_unlink(dp, ip) 4835 struct inode *dp; 4836 struct inode *ip; 4837 { 4838 struct vnode *dvp; 4839 4840 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4841 ("softdep_setup_unlink called on non-softdep filesystem")); 4842 dvp = ITOV(dp); 4843 ACQUIRE_LOCK(ITOUMP(dp)); 4844 (void) inodedep_lookup_ip(ip); 4845 (void) inodedep_lookup_ip(dp); 4846 softdep_prelink(dvp, ITOV(ip)); 4847 FREE_LOCK(ITOUMP(dp)); 4848 } 4849 4850 /* 4851 * Called to release the journal structures created by a failed non-directory 4852 * creation. Adjusts nlinkdelta for non-journaling softdep. 4853 */ 4854 void 4855 softdep_revert_create(dp, ip) 4856 struct inode *dp; 4857 struct inode *ip; 4858 { 4859 struct inodedep *inodedep; 4860 struct jaddref *jaddref; 4861 struct vnode *dvp; 4862 4863 KASSERT(MOUNTEDSOFTDEP(ITOVFS((dp))) != 0, 4864 ("softdep_revert_create called on non-softdep filesystem")); 4865 dvp = ITOV(dp); 4866 ACQUIRE_LOCK(ITOUMP(dp)); 4867 inodedep = inodedep_lookup_ip(ip); 4868 if (DOINGSUJ(dvp)) { 4869 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4870 inoreflst); 4871 KASSERT(jaddref->ja_parent == dp->i_number, 4872 ("softdep_revert_create: addref parent mismatch")); 4873 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4874 } 4875 FREE_LOCK(ITOUMP(dp)); 4876 } 4877 4878 /* 4879 * Called to release the journal structures created by a failed link 4880 * addition. Adjusts nlinkdelta for non-journaling softdep. 4881 */ 4882 void 4883 softdep_revert_link(dp, ip) 4884 struct inode *dp; 4885 struct inode *ip; 4886 { 4887 struct inodedep *inodedep; 4888 struct jaddref *jaddref; 4889 struct vnode *dvp; 4890 4891 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4892 ("softdep_revert_link called on non-softdep filesystem")); 4893 dvp = ITOV(dp); 4894 ACQUIRE_LOCK(ITOUMP(dp)); 4895 inodedep = inodedep_lookup_ip(ip); 4896 if (DOINGSUJ(dvp)) { 4897 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4898 inoreflst); 4899 KASSERT(jaddref->ja_parent == dp->i_number, 4900 ("softdep_revert_link: addref parent mismatch")); 4901 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4902 } 4903 FREE_LOCK(ITOUMP(dp)); 4904 } 4905 4906 /* 4907 * Called to release the journal structures created by a failed mkdir 4908 * attempt. Adjusts nlinkdelta for non-journaling softdep. 4909 */ 4910 void 4911 softdep_revert_mkdir(dp, ip) 4912 struct inode *dp; 4913 struct inode *ip; 4914 { 4915 struct inodedep *inodedep; 4916 struct jaddref *jaddref; 4917 struct jaddref *dotaddref; 4918 struct vnode *dvp; 4919 4920 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4921 ("softdep_revert_mkdir called on non-softdep filesystem")); 4922 dvp = ITOV(dp); 4923 4924 ACQUIRE_LOCK(ITOUMP(dp)); 4925 inodedep = inodedep_lookup_ip(dp); 4926 if (DOINGSUJ(dvp)) { 4927 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4928 inoreflst); 4929 KASSERT(jaddref->ja_parent == ip->i_number, 4930 ("softdep_revert_mkdir: dotdot addref parent mismatch")); 4931 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4932 } 4933 inodedep = inodedep_lookup_ip(ip); 4934 if (DOINGSUJ(dvp)) { 4935 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4936 inoreflst); 4937 KASSERT(jaddref->ja_parent == dp->i_number, 4938 ("softdep_revert_mkdir: addref parent mismatch")); 4939 dotaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref, 4940 inoreflst, if_deps); 4941 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4942 KASSERT(dotaddref->ja_parent == ip->i_number, 4943 ("softdep_revert_mkdir: dot addref parent mismatch")); 4944 cancel_jaddref(dotaddref, inodedep, &inodedep->id_inowait); 4945 } 4946 FREE_LOCK(ITOUMP(dp)); 4947 } 4948 4949 /* 4950 * Called to correct nlinkdelta after a failed rmdir. 4951 */ 4952 void 4953 softdep_revert_rmdir(dp, ip) 4954 struct inode *dp; 4955 struct inode *ip; 4956 { 4957 4958 KASSERT(MOUNTEDSOFTDEP(ITOVFS(dp)) != 0, 4959 ("softdep_revert_rmdir called on non-softdep filesystem")); 4960 ACQUIRE_LOCK(ITOUMP(dp)); 4961 (void) inodedep_lookup_ip(ip); 4962 (void) inodedep_lookup_ip(dp); 4963 FREE_LOCK(ITOUMP(dp)); 4964 } 4965 4966 /* 4967 * Protecting the freemaps (or bitmaps). 4968 * 4969 * To eliminate the need to execute fsck before mounting a filesystem 4970 * after a power failure, one must (conservatively) guarantee that the 4971 * on-disk copy of the bitmaps never indicate that a live inode or block is 4972 * free. So, when a block or inode is allocated, the bitmap should be 4973 * updated (on disk) before any new pointers. When a block or inode is 4974 * freed, the bitmap should not be updated until all pointers have been 4975 * reset. The latter dependency is handled by the delayed de-allocation 4976 * approach described below for block and inode de-allocation. The former 4977 * dependency is handled by calling the following procedure when a block or 4978 * inode is allocated. When an inode is allocated an "inodedep" is created 4979 * with its DEPCOMPLETE flag cleared until its bitmap is written to disk. 4980 * Each "inodedep" is also inserted into the hash indexing structure so 4981 * that any additional link additions can be made dependent on the inode 4982 * allocation. 4983 * 4984 * The ufs filesystem maintains a number of free block counts (e.g., per 4985 * cylinder group, per cylinder and per <cylinder, rotational position> pair) 4986 * in addition to the bitmaps. These counts are used to improve efficiency 4987 * during allocation and therefore must be consistent with the bitmaps. 4988 * There is no convenient way to guarantee post-crash consistency of these 4989 * counts with simple update ordering, for two main reasons: (1) The counts 4990 * and bitmaps for a single cylinder group block are not in the same disk 4991 * sector. If a disk write is interrupted (e.g., by power failure), one may 4992 * be written and the other not. (2) Some of the counts are located in the 4993 * superblock rather than the cylinder group block. So, we focus our soft 4994 * updates implementation on protecting the bitmaps. When mounting a 4995 * filesystem, we recompute the auxiliary counts from the bitmaps. 4996 */ 4997 4998 /* 4999 * Called just after updating the cylinder group block to allocate an inode. 5000 */ 5001 void 5002 softdep_setup_inomapdep(bp, ip, newinum, mode) 5003 struct buf *bp; /* buffer for cylgroup block with inode map */ 5004 struct inode *ip; /* inode related to allocation */ 5005 ino_t newinum; /* new inode number being allocated */ 5006 int mode; 5007 { 5008 struct inodedep *inodedep; 5009 struct bmsafemap *bmsafemap; 5010 struct jaddref *jaddref; 5011 struct mount *mp; 5012 struct fs *fs; 5013 5014 mp = ITOVFS(ip); 5015 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5016 ("softdep_setup_inomapdep called on non-softdep filesystem")); 5017 fs = VFSTOUFS(mp)->um_fs; 5018 jaddref = NULL; 5019 5020 /* 5021 * Allocate the journal reference add structure so that the bitmap 5022 * can be dependent on it. 5023 */ 5024 if (MOUNTEDSUJ(mp)) { 5025 jaddref = newjaddref(ip, newinum, 0, 0, mode); 5026 jaddref->ja_state |= NEWBLOCK; 5027 } 5028 5029 /* 5030 * Create a dependency for the newly allocated inode. 5031 * Panic if it already exists as something is seriously wrong. 5032 * Otherwise add it to the dependency list for the buffer holding 5033 * the cylinder group map from which it was allocated. 5034 * 5035 * We have to preallocate a bmsafemap entry in case it is needed 5036 * in bmsafemap_lookup since once we allocate the inodedep, we 5037 * have to finish initializing it before we can FREE_LOCK(). 5038 * By preallocating, we avoid FREE_LOCK() while doing a malloc 5039 * in bmsafemap_lookup. We cannot call bmsafemap_lookup before 5040 * creating the inodedep as it can be freed during the time 5041 * that we FREE_LOCK() while allocating the inodedep. We must 5042 * call workitem_alloc() before entering the locked section as 5043 * it also acquires the lock and we must avoid trying doing so 5044 * recursively. 5045 */ 5046 bmsafemap = malloc(sizeof(struct bmsafemap), 5047 M_BMSAFEMAP, M_SOFTDEP_FLAGS); 5048 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); 5049 ACQUIRE_LOCK(ITOUMP(ip)); 5050 if ((inodedep_lookup(mp, newinum, DEPALLOC, &inodedep))) 5051 panic("softdep_setup_inomapdep: dependency %p for new" 5052 "inode already exists", inodedep); 5053 bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum), bmsafemap); 5054 if (jaddref) { 5055 LIST_INSERT_HEAD(&bmsafemap->sm_jaddrefhd, jaddref, ja_bmdeps); 5056 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 5057 if_deps); 5058 } else { 5059 inodedep->id_state |= ONDEPLIST; 5060 LIST_INSERT_HEAD(&bmsafemap->sm_inodedephd, inodedep, id_deps); 5061 } 5062 inodedep->id_bmsafemap = bmsafemap; 5063 inodedep->id_state &= ~DEPCOMPLETE; 5064 FREE_LOCK(ITOUMP(ip)); 5065 } 5066 5067 /* 5068 * Called just after updating the cylinder group block to 5069 * allocate block or fragment. 5070 */ 5071 void 5072 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 5073 struct buf *bp; /* buffer for cylgroup block with block map */ 5074 struct mount *mp; /* filesystem doing allocation */ 5075 ufs2_daddr_t newblkno; /* number of newly allocated block */ 5076 int frags; /* Number of fragments. */ 5077 int oldfrags; /* Previous number of fragments for extend. */ 5078 { 5079 struct newblk *newblk; 5080 struct bmsafemap *bmsafemap; 5081 struct jnewblk *jnewblk; 5082 struct ufsmount *ump; 5083 struct fs *fs; 5084 5085 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5086 ("softdep_setup_blkmapdep called on non-softdep filesystem")); 5087 ump = VFSTOUFS(mp); 5088 fs = ump->um_fs; 5089 jnewblk = NULL; 5090 /* 5091 * Create a dependency for the newly allocated block. 5092 * Add it to the dependency list for the buffer holding 5093 * the cylinder group map from which it was allocated. 5094 */ 5095 if (MOUNTEDSUJ(mp)) { 5096 jnewblk = malloc(sizeof(*jnewblk), M_JNEWBLK, M_SOFTDEP_FLAGS); 5097 workitem_alloc(&jnewblk->jn_list, D_JNEWBLK, mp); 5098 jnewblk->jn_jsegdep = newjsegdep(&jnewblk->jn_list); 5099 jnewblk->jn_state = ATTACHED; 5100 jnewblk->jn_blkno = newblkno; 5101 jnewblk->jn_frags = frags; 5102 jnewblk->jn_oldfrags = oldfrags; 5103 #ifdef SUJ_DEBUG 5104 { 5105 struct cg *cgp; 5106 uint8_t *blksfree; 5107 long bno; 5108 int i; 5109 5110 cgp = (struct cg *)bp->b_data; 5111 blksfree = cg_blksfree(cgp); 5112 bno = dtogd(fs, jnewblk->jn_blkno); 5113 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 5114 i++) { 5115 if (isset(blksfree, bno + i)) 5116 panic("softdep_setup_blkmapdep: " 5117 "free fragment %d from %d-%d " 5118 "state 0x%X dep %p", i, 5119 jnewblk->jn_oldfrags, 5120 jnewblk->jn_frags, 5121 jnewblk->jn_state, 5122 jnewblk->jn_dep); 5123 } 5124 } 5125 #endif 5126 } 5127 5128 CTR3(KTR_SUJ, 5129 "softdep_setup_blkmapdep: blkno %jd frags %d oldfrags %d", 5130 newblkno, frags, oldfrags); 5131 ACQUIRE_LOCK(ump); 5132 if (newblk_lookup(mp, newblkno, DEPALLOC, &newblk) != 0) 5133 panic("softdep_setup_blkmapdep: found block"); 5134 newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(mp, bp, 5135 dtog(fs, newblkno), NULL); 5136 if (jnewblk) { 5137 jnewblk->jn_dep = (struct worklist *)newblk; 5138 LIST_INSERT_HEAD(&bmsafemap->sm_jnewblkhd, jnewblk, jn_deps); 5139 } else { 5140 newblk->nb_state |= ONDEPLIST; 5141 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps); 5142 } 5143 newblk->nb_bmsafemap = bmsafemap; 5144 newblk->nb_jnewblk = jnewblk; 5145 FREE_LOCK(ump); 5146 } 5147 5148 #define BMSAFEMAP_HASH(ump, cg) \ 5149 (&(ump)->bmsafemap_hashtbl[(cg) & (ump)->bmsafemap_hash_size]) 5150 5151 static int 5152 bmsafemap_find(bmsafemaphd, cg, bmsafemapp) 5153 struct bmsafemap_hashhead *bmsafemaphd; 5154 int cg; 5155 struct bmsafemap **bmsafemapp; 5156 { 5157 struct bmsafemap *bmsafemap; 5158 5159 LIST_FOREACH(bmsafemap, bmsafemaphd, sm_hash) 5160 if (bmsafemap->sm_cg == cg) 5161 break; 5162 if (bmsafemap) { 5163 *bmsafemapp = bmsafemap; 5164 return (1); 5165 } 5166 *bmsafemapp = NULL; 5167 5168 return (0); 5169 } 5170 5171 /* 5172 * Find the bmsafemap associated with a cylinder group buffer. 5173 * If none exists, create one. The buffer must be locked when 5174 * this routine is called and this routine must be called with 5175 * the softdep lock held. To avoid giving up the lock while 5176 * allocating a new bmsafemap, a preallocated bmsafemap may be 5177 * provided. If it is provided but not needed, it is freed. 5178 */ 5179 static struct bmsafemap * 5180 bmsafemap_lookup(mp, bp, cg, newbmsafemap) 5181 struct mount *mp; 5182 struct buf *bp; 5183 int cg; 5184 struct bmsafemap *newbmsafemap; 5185 { 5186 struct bmsafemap_hashhead *bmsafemaphd; 5187 struct bmsafemap *bmsafemap, *collision; 5188 struct worklist *wk; 5189 struct ufsmount *ump; 5190 5191 ump = VFSTOUFS(mp); 5192 LOCK_OWNED(ump); 5193 KASSERT(bp != NULL, ("bmsafemap_lookup: missing buffer")); 5194 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 5195 if (wk->wk_type == D_BMSAFEMAP) { 5196 if (newbmsafemap) 5197 WORKITEM_FREE(newbmsafemap, D_BMSAFEMAP); 5198 return (WK_BMSAFEMAP(wk)); 5199 } 5200 } 5201 bmsafemaphd = BMSAFEMAP_HASH(ump, cg); 5202 if (bmsafemap_find(bmsafemaphd, cg, &bmsafemap) == 1) { 5203 if (newbmsafemap) 5204 WORKITEM_FREE(newbmsafemap, D_BMSAFEMAP); 5205 return (bmsafemap); 5206 } 5207 if (newbmsafemap) { 5208 bmsafemap = newbmsafemap; 5209 } else { 5210 FREE_LOCK(ump); 5211 bmsafemap = malloc(sizeof(struct bmsafemap), 5212 M_BMSAFEMAP, M_SOFTDEP_FLAGS); 5213 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); 5214 ACQUIRE_LOCK(ump); 5215 } 5216 bmsafemap->sm_buf = bp; 5217 LIST_INIT(&bmsafemap->sm_inodedephd); 5218 LIST_INIT(&bmsafemap->sm_inodedepwr); 5219 LIST_INIT(&bmsafemap->sm_newblkhd); 5220 LIST_INIT(&bmsafemap->sm_newblkwr); 5221 LIST_INIT(&bmsafemap->sm_jaddrefhd); 5222 LIST_INIT(&bmsafemap->sm_jnewblkhd); 5223 LIST_INIT(&bmsafemap->sm_freehd); 5224 LIST_INIT(&bmsafemap->sm_freewr); 5225 if (bmsafemap_find(bmsafemaphd, cg, &collision) == 1) { 5226 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 5227 return (collision); 5228 } 5229 bmsafemap->sm_cg = cg; 5230 LIST_INSERT_HEAD(bmsafemaphd, bmsafemap, sm_hash); 5231 LIST_INSERT_HEAD(&ump->softdep_dirtycg, bmsafemap, sm_next); 5232 WORKLIST_INSERT(&bp->b_dep, &bmsafemap->sm_list); 5233 return (bmsafemap); 5234 } 5235 5236 /* 5237 * Direct block allocation dependencies. 5238 * 5239 * When a new block is allocated, the corresponding disk locations must be 5240 * initialized (with zeros or new data) before the on-disk inode points to 5241 * them. Also, the freemap from which the block was allocated must be 5242 * updated (on disk) before the inode's pointer. These two dependencies are 5243 * independent of each other and are needed for all file blocks and indirect 5244 * blocks that are pointed to directly by the inode. Just before the 5245 * "in-core" version of the inode is updated with a newly allocated block 5246 * number, a procedure (below) is called to setup allocation dependency 5247 * structures. These structures are removed when the corresponding 5248 * dependencies are satisfied or when the block allocation becomes obsolete 5249 * (i.e., the file is deleted, the block is de-allocated, or the block is a 5250 * fragment that gets upgraded). All of these cases are handled in 5251 * procedures described later. 5252 * 5253 * When a file extension causes a fragment to be upgraded, either to a larger 5254 * fragment or to a full block, the on-disk location may change (if the 5255 * previous fragment could not simply be extended). In this case, the old 5256 * fragment must be de-allocated, but not until after the inode's pointer has 5257 * been updated. In most cases, this is handled by later procedures, which 5258 * will construct a "freefrag" structure to be added to the workitem queue 5259 * when the inode update is complete (or obsolete). The main exception to 5260 * this is when an allocation occurs while a pending allocation dependency 5261 * (for the same block pointer) remains. This case is handled in the main 5262 * allocation dependency setup procedure by immediately freeing the 5263 * unreferenced fragments. 5264 */ 5265 void 5266 softdep_setup_allocdirect(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 5267 struct inode *ip; /* inode to which block is being added */ 5268 ufs_lbn_t off; /* block pointer within inode */ 5269 ufs2_daddr_t newblkno; /* disk block number being added */ 5270 ufs2_daddr_t oldblkno; /* previous block number, 0 unless frag */ 5271 long newsize; /* size of new block */ 5272 long oldsize; /* size of new block */ 5273 struct buf *bp; /* bp for allocated block */ 5274 { 5275 struct allocdirect *adp, *oldadp; 5276 struct allocdirectlst *adphead; 5277 struct freefrag *freefrag; 5278 struct inodedep *inodedep; 5279 struct pagedep *pagedep; 5280 struct jnewblk *jnewblk; 5281 struct newblk *newblk; 5282 struct mount *mp; 5283 ufs_lbn_t lbn; 5284 5285 lbn = bp->b_lblkno; 5286 mp = ITOVFS(ip); 5287 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5288 ("softdep_setup_allocdirect called on non-softdep filesystem")); 5289 if (oldblkno && oldblkno != newblkno) 5290 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn); 5291 else 5292 freefrag = NULL; 5293 5294 CTR6(KTR_SUJ, 5295 "softdep_setup_allocdirect: ino %d blkno %jd oldblkno %jd " 5296 "off %jd newsize %ld oldsize %d", 5297 ip->i_number, newblkno, oldblkno, off, newsize, oldsize); 5298 ACQUIRE_LOCK(ITOUMP(ip)); 5299 if (off >= NDADDR) { 5300 if (lbn > 0) 5301 panic("softdep_setup_allocdirect: bad lbn %jd, off %jd", 5302 lbn, off); 5303 /* allocating an indirect block */ 5304 if (oldblkno != 0) 5305 panic("softdep_setup_allocdirect: non-zero indir"); 5306 } else { 5307 if (off != lbn) 5308 panic("softdep_setup_allocdirect: lbn %jd != off %jd", 5309 lbn, off); 5310 /* 5311 * Allocating a direct block. 5312 * 5313 * If we are allocating a directory block, then we must 5314 * allocate an associated pagedep to track additions and 5315 * deletions. 5316 */ 5317 if ((ip->i_mode & IFMT) == IFDIR) 5318 pagedep_lookup(mp, bp, ip->i_number, off, DEPALLOC, 5319 &pagedep); 5320 } 5321 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 5322 panic("softdep_setup_allocdirect: lost block"); 5323 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 5324 ("softdep_setup_allocdirect: newblk already initialized")); 5325 /* 5326 * Convert the newblk to an allocdirect. 5327 */ 5328 WORKITEM_REASSIGN(newblk, D_ALLOCDIRECT); 5329 adp = (struct allocdirect *)newblk; 5330 newblk->nb_freefrag = freefrag; 5331 adp->ad_offset = off; 5332 adp->ad_oldblkno = oldblkno; 5333 adp->ad_newsize = newsize; 5334 adp->ad_oldsize = oldsize; 5335 5336 /* 5337 * Finish initializing the journal. 5338 */ 5339 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 5340 jnewblk->jn_ino = ip->i_number; 5341 jnewblk->jn_lbn = lbn; 5342 add_to_journal(&jnewblk->jn_list); 5343 } 5344 if (freefrag && freefrag->ff_jdep != NULL && 5345 freefrag->ff_jdep->wk_type == D_JFREEFRAG) 5346 add_to_journal(freefrag->ff_jdep); 5347 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5348 adp->ad_inodedep = inodedep; 5349 5350 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 5351 /* 5352 * The list of allocdirects must be kept in sorted and ascending 5353 * order so that the rollback routines can quickly determine the 5354 * first uncommitted block (the size of the file stored on disk 5355 * ends at the end of the lowest committed fragment, or if there 5356 * are no fragments, at the end of the highest committed block). 5357 * Since files generally grow, the typical case is that the new 5358 * block is to be added at the end of the list. We speed this 5359 * special case by checking against the last allocdirect in the 5360 * list before laboriously traversing the list looking for the 5361 * insertion point. 5362 */ 5363 adphead = &inodedep->id_newinoupdt; 5364 oldadp = TAILQ_LAST(adphead, allocdirectlst); 5365 if (oldadp == NULL || oldadp->ad_offset <= off) { 5366 /* insert at end of list */ 5367 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 5368 if (oldadp != NULL && oldadp->ad_offset == off) 5369 allocdirect_merge(adphead, adp, oldadp); 5370 FREE_LOCK(ITOUMP(ip)); 5371 return; 5372 } 5373 TAILQ_FOREACH(oldadp, adphead, ad_next) { 5374 if (oldadp->ad_offset >= off) 5375 break; 5376 } 5377 if (oldadp == NULL) 5378 panic("softdep_setup_allocdirect: lost entry"); 5379 /* insert in middle of list */ 5380 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 5381 if (oldadp->ad_offset == off) 5382 allocdirect_merge(adphead, adp, oldadp); 5383 5384 FREE_LOCK(ITOUMP(ip)); 5385 } 5386 5387 /* 5388 * Merge a newer and older journal record to be stored either in a 5389 * newblock or freefrag. This handles aggregating journal records for 5390 * fragment allocation into a second record as well as replacing a 5391 * journal free with an aborted journal allocation. A segment for the 5392 * oldest record will be placed on wkhd if it has been written. If not 5393 * the segment for the newer record will suffice. 5394 */ 5395 static struct worklist * 5396 jnewblk_merge(new, old, wkhd) 5397 struct worklist *new; 5398 struct worklist *old; 5399 struct workhead *wkhd; 5400 { 5401 struct jnewblk *njnewblk; 5402 struct jnewblk *jnewblk; 5403 5404 /* Handle NULLs to simplify callers. */ 5405 if (new == NULL) 5406 return (old); 5407 if (old == NULL) 5408 return (new); 5409 /* Replace a jfreefrag with a jnewblk. */ 5410 if (new->wk_type == D_JFREEFRAG) { 5411 if (WK_JNEWBLK(old)->jn_blkno != WK_JFREEFRAG(new)->fr_blkno) 5412 panic("jnewblk_merge: blkno mismatch: %p, %p", 5413 old, new); 5414 cancel_jfreefrag(WK_JFREEFRAG(new)); 5415 return (old); 5416 } 5417 if (old->wk_type != D_JNEWBLK || new->wk_type != D_JNEWBLK) 5418 panic("jnewblk_merge: Bad type: old %d new %d\n", 5419 old->wk_type, new->wk_type); 5420 /* 5421 * Handle merging of two jnewblk records that describe 5422 * different sets of fragments in the same block. 5423 */ 5424 jnewblk = WK_JNEWBLK(old); 5425 njnewblk = WK_JNEWBLK(new); 5426 if (jnewblk->jn_blkno != njnewblk->jn_blkno) 5427 panic("jnewblk_merge: Merging disparate blocks."); 5428 /* 5429 * The record may be rolled back in the cg. 5430 */ 5431 if (jnewblk->jn_state & UNDONE) { 5432 jnewblk->jn_state &= ~UNDONE; 5433 njnewblk->jn_state |= UNDONE; 5434 njnewblk->jn_state &= ~ATTACHED; 5435 } 5436 /* 5437 * We modify the newer addref and free the older so that if neither 5438 * has been written the most up-to-date copy will be on disk. If 5439 * both have been written but rolled back we only temporarily need 5440 * one of them to fix the bits when the cg write completes. 5441 */ 5442 jnewblk->jn_state |= ATTACHED | COMPLETE; 5443 njnewblk->jn_oldfrags = jnewblk->jn_oldfrags; 5444 cancel_jnewblk(jnewblk, wkhd); 5445 WORKLIST_REMOVE(&jnewblk->jn_list); 5446 free_jnewblk(jnewblk); 5447 return (new); 5448 } 5449 5450 /* 5451 * Replace an old allocdirect dependency with a newer one. 5452 * This routine must be called with splbio interrupts blocked. 5453 */ 5454 static void 5455 allocdirect_merge(adphead, newadp, oldadp) 5456 struct allocdirectlst *adphead; /* head of list holding allocdirects */ 5457 struct allocdirect *newadp; /* allocdirect being added */ 5458 struct allocdirect *oldadp; /* existing allocdirect being checked */ 5459 { 5460 struct worklist *wk; 5461 struct freefrag *freefrag; 5462 5463 freefrag = NULL; 5464 LOCK_OWNED(VFSTOUFS(newadp->ad_list.wk_mp)); 5465 if (newadp->ad_oldblkno != oldadp->ad_newblkno || 5466 newadp->ad_oldsize != oldadp->ad_newsize || 5467 newadp->ad_offset >= NDADDR) 5468 panic("%s %jd != new %jd || old size %ld != new %ld", 5469 "allocdirect_merge: old blkno", 5470 (intmax_t)newadp->ad_oldblkno, 5471 (intmax_t)oldadp->ad_newblkno, 5472 newadp->ad_oldsize, oldadp->ad_newsize); 5473 newadp->ad_oldblkno = oldadp->ad_oldblkno; 5474 newadp->ad_oldsize = oldadp->ad_oldsize; 5475 /* 5476 * If the old dependency had a fragment to free or had never 5477 * previously had a block allocated, then the new dependency 5478 * can immediately post its freefrag and adopt the old freefrag. 5479 * This action is done by swapping the freefrag dependencies. 5480 * The new dependency gains the old one's freefrag, and the 5481 * old one gets the new one and then immediately puts it on 5482 * the worklist when it is freed by free_newblk. It is 5483 * not possible to do this swap when the old dependency had a 5484 * non-zero size but no previous fragment to free. This condition 5485 * arises when the new block is an extension of the old block. 5486 * Here, the first part of the fragment allocated to the new 5487 * dependency is part of the block currently claimed on disk by 5488 * the old dependency, so cannot legitimately be freed until the 5489 * conditions for the new dependency are fulfilled. 5490 */ 5491 freefrag = newadp->ad_freefrag; 5492 if (oldadp->ad_freefrag != NULL || oldadp->ad_oldblkno == 0) { 5493 newadp->ad_freefrag = oldadp->ad_freefrag; 5494 oldadp->ad_freefrag = freefrag; 5495 } 5496 /* 5497 * If we are tracking a new directory-block allocation, 5498 * move it from the old allocdirect to the new allocdirect. 5499 */ 5500 if ((wk = LIST_FIRST(&oldadp->ad_newdirblk)) != NULL) { 5501 WORKLIST_REMOVE(wk); 5502 if (!LIST_EMPTY(&oldadp->ad_newdirblk)) 5503 panic("allocdirect_merge: extra newdirblk"); 5504 WORKLIST_INSERT(&newadp->ad_newdirblk, wk); 5505 } 5506 TAILQ_REMOVE(adphead, oldadp, ad_next); 5507 /* 5508 * We need to move any journal dependencies over to the freefrag 5509 * that releases this block if it exists. Otherwise we are 5510 * extending an existing block and we'll wait until that is 5511 * complete to release the journal space and extend the 5512 * new journal to cover this old space as well. 5513 */ 5514 if (freefrag == NULL) { 5515 if (oldadp->ad_newblkno != newadp->ad_newblkno) 5516 panic("allocdirect_merge: %jd != %jd", 5517 oldadp->ad_newblkno, newadp->ad_newblkno); 5518 newadp->ad_block.nb_jnewblk = (struct jnewblk *) 5519 jnewblk_merge(&newadp->ad_block.nb_jnewblk->jn_list, 5520 &oldadp->ad_block.nb_jnewblk->jn_list, 5521 &newadp->ad_block.nb_jwork); 5522 oldadp->ad_block.nb_jnewblk = NULL; 5523 cancel_newblk(&oldadp->ad_block, NULL, 5524 &newadp->ad_block.nb_jwork); 5525 } else { 5526 wk = (struct worklist *) cancel_newblk(&oldadp->ad_block, 5527 &freefrag->ff_list, &freefrag->ff_jwork); 5528 freefrag->ff_jdep = jnewblk_merge(freefrag->ff_jdep, wk, 5529 &freefrag->ff_jwork); 5530 } 5531 free_newblk(&oldadp->ad_block); 5532 } 5533 5534 /* 5535 * Allocate a jfreefrag structure to journal a single block free. 5536 */ 5537 static struct jfreefrag * 5538 newjfreefrag(freefrag, ip, blkno, size, lbn) 5539 struct freefrag *freefrag; 5540 struct inode *ip; 5541 ufs2_daddr_t blkno; 5542 long size; 5543 ufs_lbn_t lbn; 5544 { 5545 struct jfreefrag *jfreefrag; 5546 struct fs *fs; 5547 5548 fs = ITOFS(ip); 5549 jfreefrag = malloc(sizeof(struct jfreefrag), M_JFREEFRAG, 5550 M_SOFTDEP_FLAGS); 5551 workitem_alloc(&jfreefrag->fr_list, D_JFREEFRAG, ITOVFS(ip)); 5552 jfreefrag->fr_jsegdep = newjsegdep(&jfreefrag->fr_list); 5553 jfreefrag->fr_state = ATTACHED | DEPCOMPLETE; 5554 jfreefrag->fr_ino = ip->i_number; 5555 jfreefrag->fr_lbn = lbn; 5556 jfreefrag->fr_blkno = blkno; 5557 jfreefrag->fr_frags = numfrags(fs, size); 5558 jfreefrag->fr_freefrag = freefrag; 5559 5560 return (jfreefrag); 5561 } 5562 5563 /* 5564 * Allocate a new freefrag structure. 5565 */ 5566 static struct freefrag * 5567 newfreefrag(ip, blkno, size, lbn) 5568 struct inode *ip; 5569 ufs2_daddr_t blkno; 5570 long size; 5571 ufs_lbn_t lbn; 5572 { 5573 struct freefrag *freefrag; 5574 struct ufsmount *ump; 5575 struct fs *fs; 5576 5577 CTR4(KTR_SUJ, "newfreefrag: ino %d blkno %jd size %ld lbn %jd", 5578 ip->i_number, blkno, size, lbn); 5579 ump = ITOUMP(ip); 5580 fs = ump->um_fs; 5581 if (fragnum(fs, blkno) + numfrags(fs, size) > fs->fs_frag) 5582 panic("newfreefrag: frag size"); 5583 freefrag = malloc(sizeof(struct freefrag), 5584 M_FREEFRAG, M_SOFTDEP_FLAGS); 5585 workitem_alloc(&freefrag->ff_list, D_FREEFRAG, UFSTOVFS(ump)); 5586 freefrag->ff_state = ATTACHED; 5587 LIST_INIT(&freefrag->ff_jwork); 5588 freefrag->ff_inum = ip->i_number; 5589 freefrag->ff_vtype = ITOV(ip)->v_type; 5590 freefrag->ff_blkno = blkno; 5591 freefrag->ff_fragsize = size; 5592 5593 if (MOUNTEDSUJ(UFSTOVFS(ump))) { 5594 freefrag->ff_jdep = (struct worklist *) 5595 newjfreefrag(freefrag, ip, blkno, size, lbn); 5596 } else { 5597 freefrag->ff_state |= DEPCOMPLETE; 5598 freefrag->ff_jdep = NULL; 5599 } 5600 5601 return (freefrag); 5602 } 5603 5604 /* 5605 * This workitem de-allocates fragments that were replaced during 5606 * file block allocation. 5607 */ 5608 static void 5609 handle_workitem_freefrag(freefrag) 5610 struct freefrag *freefrag; 5611 { 5612 struct ufsmount *ump = VFSTOUFS(freefrag->ff_list.wk_mp); 5613 struct workhead wkhd; 5614 5615 CTR3(KTR_SUJ, 5616 "handle_workitem_freefrag: ino %d blkno %jd size %ld", 5617 freefrag->ff_inum, freefrag->ff_blkno, freefrag->ff_fragsize); 5618 /* 5619 * It would be illegal to add new completion items to the 5620 * freefrag after it was schedule to be done so it must be 5621 * safe to modify the list head here. 5622 */ 5623 LIST_INIT(&wkhd); 5624 ACQUIRE_LOCK(ump); 5625 LIST_SWAP(&freefrag->ff_jwork, &wkhd, worklist, wk_list); 5626 /* 5627 * If the journal has not been written we must cancel it here. 5628 */ 5629 if (freefrag->ff_jdep) { 5630 if (freefrag->ff_jdep->wk_type != D_JNEWBLK) 5631 panic("handle_workitem_freefrag: Unexpected type %d\n", 5632 freefrag->ff_jdep->wk_type); 5633 cancel_jnewblk(WK_JNEWBLK(freefrag->ff_jdep), &wkhd); 5634 } 5635 FREE_LOCK(ump); 5636 ffs_blkfree(ump, ump->um_fs, ump->um_devvp, freefrag->ff_blkno, 5637 freefrag->ff_fragsize, freefrag->ff_inum, freefrag->ff_vtype, &wkhd); 5638 ACQUIRE_LOCK(ump); 5639 WORKITEM_FREE(freefrag, D_FREEFRAG); 5640 FREE_LOCK(ump); 5641 } 5642 5643 /* 5644 * Set up a dependency structure for an external attributes data block. 5645 * This routine follows much of the structure of softdep_setup_allocdirect. 5646 * See the description of softdep_setup_allocdirect above for details. 5647 */ 5648 void 5649 softdep_setup_allocext(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 5650 struct inode *ip; 5651 ufs_lbn_t off; 5652 ufs2_daddr_t newblkno; 5653 ufs2_daddr_t oldblkno; 5654 long newsize; 5655 long oldsize; 5656 struct buf *bp; 5657 { 5658 struct allocdirect *adp, *oldadp; 5659 struct allocdirectlst *adphead; 5660 struct freefrag *freefrag; 5661 struct inodedep *inodedep; 5662 struct jnewblk *jnewblk; 5663 struct newblk *newblk; 5664 struct mount *mp; 5665 struct ufsmount *ump; 5666 ufs_lbn_t lbn; 5667 5668 mp = ITOVFS(ip); 5669 ump = VFSTOUFS(mp); 5670 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5671 ("softdep_setup_allocext called on non-softdep filesystem")); 5672 KASSERT(off < NXADDR, ("softdep_setup_allocext: lbn %lld > NXADDR", 5673 (long long)off)); 5674 5675 lbn = bp->b_lblkno; 5676 if (oldblkno && oldblkno != newblkno) 5677 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn); 5678 else 5679 freefrag = NULL; 5680 5681 ACQUIRE_LOCK(ump); 5682 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 5683 panic("softdep_setup_allocext: lost block"); 5684 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 5685 ("softdep_setup_allocext: newblk already initialized")); 5686 /* 5687 * Convert the newblk to an allocdirect. 5688 */ 5689 WORKITEM_REASSIGN(newblk, D_ALLOCDIRECT); 5690 adp = (struct allocdirect *)newblk; 5691 newblk->nb_freefrag = freefrag; 5692 adp->ad_offset = off; 5693 adp->ad_oldblkno = oldblkno; 5694 adp->ad_newsize = newsize; 5695 adp->ad_oldsize = oldsize; 5696 adp->ad_state |= EXTDATA; 5697 5698 /* 5699 * Finish initializing the journal. 5700 */ 5701 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 5702 jnewblk->jn_ino = ip->i_number; 5703 jnewblk->jn_lbn = lbn; 5704 add_to_journal(&jnewblk->jn_list); 5705 } 5706 if (freefrag && freefrag->ff_jdep != NULL && 5707 freefrag->ff_jdep->wk_type == D_JFREEFRAG) 5708 add_to_journal(freefrag->ff_jdep); 5709 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5710 adp->ad_inodedep = inodedep; 5711 5712 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 5713 /* 5714 * The list of allocdirects must be kept in sorted and ascending 5715 * order so that the rollback routines can quickly determine the 5716 * first uncommitted block (the size of the file stored on disk 5717 * ends at the end of the lowest committed fragment, or if there 5718 * are no fragments, at the end of the highest committed block). 5719 * Since files generally grow, the typical case is that the new 5720 * block is to be added at the end of the list. We speed this 5721 * special case by checking against the last allocdirect in the 5722 * list before laboriously traversing the list looking for the 5723 * insertion point. 5724 */ 5725 adphead = &inodedep->id_newextupdt; 5726 oldadp = TAILQ_LAST(adphead, allocdirectlst); 5727 if (oldadp == NULL || oldadp->ad_offset <= off) { 5728 /* insert at end of list */ 5729 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 5730 if (oldadp != NULL && oldadp->ad_offset == off) 5731 allocdirect_merge(adphead, adp, oldadp); 5732 FREE_LOCK(ump); 5733 return; 5734 } 5735 TAILQ_FOREACH(oldadp, adphead, ad_next) { 5736 if (oldadp->ad_offset >= off) 5737 break; 5738 } 5739 if (oldadp == NULL) 5740 panic("softdep_setup_allocext: lost entry"); 5741 /* insert in middle of list */ 5742 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 5743 if (oldadp->ad_offset == off) 5744 allocdirect_merge(adphead, adp, oldadp); 5745 FREE_LOCK(ump); 5746 } 5747 5748 /* 5749 * Indirect block allocation dependencies. 5750 * 5751 * The same dependencies that exist for a direct block also exist when 5752 * a new block is allocated and pointed to by an entry in a block of 5753 * indirect pointers. The undo/redo states described above are also 5754 * used here. Because an indirect block contains many pointers that 5755 * may have dependencies, a second copy of the entire in-memory indirect 5756 * block is kept. The buffer cache copy is always completely up-to-date. 5757 * The second copy, which is used only as a source for disk writes, 5758 * contains only the safe pointers (i.e., those that have no remaining 5759 * update dependencies). The second copy is freed when all pointers 5760 * are safe. The cache is not allowed to replace indirect blocks with 5761 * pending update dependencies. If a buffer containing an indirect 5762 * block with dependencies is written, these routines will mark it 5763 * dirty again. It can only be successfully written once all the 5764 * dependencies are removed. The ffs_fsync routine in conjunction with 5765 * softdep_sync_metadata work together to get all the dependencies 5766 * removed so that a file can be successfully written to disk. Three 5767 * procedures are used when setting up indirect block pointer 5768 * dependencies. The division is necessary because of the organization 5769 * of the "balloc" routine and because of the distinction between file 5770 * pages and file metadata blocks. 5771 */ 5772 5773 /* 5774 * Allocate a new allocindir structure. 5775 */ 5776 static struct allocindir * 5777 newallocindir(ip, ptrno, newblkno, oldblkno, lbn) 5778 struct inode *ip; /* inode for file being extended */ 5779 int ptrno; /* offset of pointer in indirect block */ 5780 ufs2_daddr_t newblkno; /* disk block number being added */ 5781 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 5782 ufs_lbn_t lbn; 5783 { 5784 struct newblk *newblk; 5785 struct allocindir *aip; 5786 struct freefrag *freefrag; 5787 struct jnewblk *jnewblk; 5788 5789 if (oldblkno) 5790 freefrag = newfreefrag(ip, oldblkno, ITOFS(ip)->fs_bsize, lbn); 5791 else 5792 freefrag = NULL; 5793 ACQUIRE_LOCK(ITOUMP(ip)); 5794 if (newblk_lookup(ITOVFS(ip), newblkno, 0, &newblk) == 0) 5795 panic("new_allocindir: lost block"); 5796 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 5797 ("newallocindir: newblk already initialized")); 5798 WORKITEM_REASSIGN(newblk, D_ALLOCINDIR); 5799 newblk->nb_freefrag = freefrag; 5800 aip = (struct allocindir *)newblk; 5801 aip->ai_offset = ptrno; 5802 aip->ai_oldblkno = oldblkno; 5803 aip->ai_lbn = lbn; 5804 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 5805 jnewblk->jn_ino = ip->i_number; 5806 jnewblk->jn_lbn = lbn; 5807 add_to_journal(&jnewblk->jn_list); 5808 } 5809 if (freefrag && freefrag->ff_jdep != NULL && 5810 freefrag->ff_jdep->wk_type == D_JFREEFRAG) 5811 add_to_journal(freefrag->ff_jdep); 5812 return (aip); 5813 } 5814 5815 /* 5816 * Called just before setting an indirect block pointer 5817 * to a newly allocated file page. 5818 */ 5819 void 5820 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 5821 struct inode *ip; /* inode for file being extended */ 5822 ufs_lbn_t lbn; /* allocated block number within file */ 5823 struct buf *bp; /* buffer with indirect blk referencing page */ 5824 int ptrno; /* offset of pointer in indirect block */ 5825 ufs2_daddr_t newblkno; /* disk block number being added */ 5826 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 5827 struct buf *nbp; /* buffer holding allocated page */ 5828 { 5829 struct inodedep *inodedep; 5830 struct freefrag *freefrag; 5831 struct allocindir *aip; 5832 struct pagedep *pagedep; 5833 struct mount *mp; 5834 struct ufsmount *ump; 5835 5836 mp = ITOVFS(ip); 5837 ump = VFSTOUFS(mp); 5838 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 5839 ("softdep_setup_allocindir_page called on non-softdep filesystem")); 5840 KASSERT(lbn == nbp->b_lblkno, 5841 ("softdep_setup_allocindir_page: lbn %jd != lblkno %jd", 5842 lbn, bp->b_lblkno)); 5843 CTR4(KTR_SUJ, 5844 "softdep_setup_allocindir_page: ino %d blkno %jd oldblkno %jd " 5845 "lbn %jd", ip->i_number, newblkno, oldblkno, lbn); 5846 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_page"); 5847 aip = newallocindir(ip, ptrno, newblkno, oldblkno, lbn); 5848 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5849 /* 5850 * If we are allocating a directory page, then we must 5851 * allocate an associated pagedep to track additions and 5852 * deletions. 5853 */ 5854 if ((ip->i_mode & IFMT) == IFDIR) 5855 pagedep_lookup(mp, nbp, ip->i_number, lbn, DEPALLOC, &pagedep); 5856 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 5857 freefrag = setup_allocindir_phase2(bp, ip, inodedep, aip, lbn); 5858 FREE_LOCK(ump); 5859 if (freefrag) 5860 handle_workitem_freefrag(freefrag); 5861 } 5862 5863 /* 5864 * Called just before setting an indirect block pointer to a 5865 * newly allocated indirect block. 5866 */ 5867 void 5868 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 5869 struct buf *nbp; /* newly allocated indirect block */ 5870 struct inode *ip; /* inode for file being extended */ 5871 struct buf *bp; /* indirect block referencing allocated block */ 5872 int ptrno; /* offset of pointer in indirect block */ 5873 ufs2_daddr_t newblkno; /* disk block number being added */ 5874 { 5875 struct inodedep *inodedep; 5876 struct allocindir *aip; 5877 struct ufsmount *ump; 5878 ufs_lbn_t lbn; 5879 5880 ump = ITOUMP(ip); 5881 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 5882 ("softdep_setup_allocindir_meta called on non-softdep filesystem")); 5883 CTR3(KTR_SUJ, 5884 "softdep_setup_allocindir_meta: ino %d blkno %jd ptrno %d", 5885 ip->i_number, newblkno, ptrno); 5886 lbn = nbp->b_lblkno; 5887 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_meta"); 5888 aip = newallocindir(ip, ptrno, newblkno, 0, lbn); 5889 inodedep_lookup(UFSTOVFS(ump), ip->i_number, DEPALLOC, &inodedep); 5890 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 5891 if (setup_allocindir_phase2(bp, ip, inodedep, aip, lbn)) 5892 panic("softdep_setup_allocindir_meta: Block already existed"); 5893 FREE_LOCK(ump); 5894 } 5895 5896 static void 5897 indirdep_complete(indirdep) 5898 struct indirdep *indirdep; 5899 { 5900 struct allocindir *aip; 5901 5902 LIST_REMOVE(indirdep, ir_next); 5903 indirdep->ir_state |= DEPCOMPLETE; 5904 5905 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) { 5906 LIST_REMOVE(aip, ai_next); 5907 free_newblk(&aip->ai_block); 5908 } 5909 /* 5910 * If this indirdep is not attached to a buf it was simply waiting 5911 * on completion to clear completehd. free_indirdep() asserts 5912 * that nothing is dangling. 5913 */ 5914 if ((indirdep->ir_state & ONWORKLIST) == 0) 5915 free_indirdep(indirdep); 5916 } 5917 5918 static struct indirdep * 5919 indirdep_lookup(mp, ip, bp) 5920 struct mount *mp; 5921 struct inode *ip; 5922 struct buf *bp; 5923 { 5924 struct indirdep *indirdep, *newindirdep; 5925 struct newblk *newblk; 5926 struct ufsmount *ump; 5927 struct worklist *wk; 5928 struct fs *fs; 5929 ufs2_daddr_t blkno; 5930 5931 ump = VFSTOUFS(mp); 5932 LOCK_OWNED(ump); 5933 indirdep = NULL; 5934 newindirdep = NULL; 5935 fs = ump->um_fs; 5936 for (;;) { 5937 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 5938 if (wk->wk_type != D_INDIRDEP) 5939 continue; 5940 indirdep = WK_INDIRDEP(wk); 5941 break; 5942 } 5943 /* Found on the buffer worklist, no new structure to free. */ 5944 if (indirdep != NULL && newindirdep == NULL) 5945 return (indirdep); 5946 if (indirdep != NULL && newindirdep != NULL) 5947 panic("indirdep_lookup: simultaneous create"); 5948 /* None found on the buffer and a new structure is ready. */ 5949 if (indirdep == NULL && newindirdep != NULL) 5950 break; 5951 /* None found and no new structure available. */ 5952 FREE_LOCK(ump); 5953 newindirdep = malloc(sizeof(struct indirdep), 5954 M_INDIRDEP, M_SOFTDEP_FLAGS); 5955 workitem_alloc(&newindirdep->ir_list, D_INDIRDEP, mp); 5956 newindirdep->ir_state = ATTACHED; 5957 if (I_IS_UFS1(ip)) 5958 newindirdep->ir_state |= UFS1FMT; 5959 TAILQ_INIT(&newindirdep->ir_trunc); 5960 newindirdep->ir_saveddata = NULL; 5961 LIST_INIT(&newindirdep->ir_deplisthd); 5962 LIST_INIT(&newindirdep->ir_donehd); 5963 LIST_INIT(&newindirdep->ir_writehd); 5964 LIST_INIT(&newindirdep->ir_completehd); 5965 if (bp->b_blkno == bp->b_lblkno) { 5966 ufs_bmaparray(bp->b_vp, bp->b_lblkno, &blkno, bp, 5967 NULL, NULL); 5968 bp->b_blkno = blkno; 5969 } 5970 newindirdep->ir_freeblks = NULL; 5971 newindirdep->ir_savebp = 5972 getblk(ump->um_devvp, bp->b_blkno, bp->b_bcount, 0, 0, 0); 5973 newindirdep->ir_bp = bp; 5974 BUF_KERNPROC(newindirdep->ir_savebp); 5975 bcopy(bp->b_data, newindirdep->ir_savebp->b_data, bp->b_bcount); 5976 ACQUIRE_LOCK(ump); 5977 } 5978 indirdep = newindirdep; 5979 WORKLIST_INSERT(&bp->b_dep, &indirdep->ir_list); 5980 /* 5981 * If the block is not yet allocated we don't set DEPCOMPLETE so 5982 * that we don't free dependencies until the pointers are valid. 5983 * This could search b_dep for D_ALLOCDIRECT/D_ALLOCINDIR rather 5984 * than using the hash. 5985 */ 5986 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk)) 5987 LIST_INSERT_HEAD(&newblk->nb_indirdeps, indirdep, ir_next); 5988 else 5989 indirdep->ir_state |= DEPCOMPLETE; 5990 return (indirdep); 5991 } 5992 5993 /* 5994 * Called to finish the allocation of the "aip" allocated 5995 * by one of the two routines above. 5996 */ 5997 static struct freefrag * 5998 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn) 5999 struct buf *bp; /* in-memory copy of the indirect block */ 6000 struct inode *ip; /* inode for file being extended */ 6001 struct inodedep *inodedep; /* Inodedep for ip */ 6002 struct allocindir *aip; /* allocindir allocated by the above routines */ 6003 ufs_lbn_t lbn; /* Logical block number for this block. */ 6004 { 6005 struct fs *fs; 6006 struct indirdep *indirdep; 6007 struct allocindir *oldaip; 6008 struct freefrag *freefrag; 6009 struct mount *mp; 6010 struct ufsmount *ump; 6011 6012 mp = ITOVFS(ip); 6013 ump = VFSTOUFS(mp); 6014 LOCK_OWNED(ump); 6015 fs = ump->um_fs; 6016 if (bp->b_lblkno >= 0) 6017 panic("setup_allocindir_phase2: not indir blk"); 6018 KASSERT(aip->ai_offset >= 0 && aip->ai_offset < NINDIR(fs), 6019 ("setup_allocindir_phase2: Bad offset %d", aip->ai_offset)); 6020 indirdep = indirdep_lookup(mp, ip, bp); 6021 KASSERT(indirdep->ir_savebp != NULL, 6022 ("setup_allocindir_phase2 NULL ir_savebp")); 6023 aip->ai_indirdep = indirdep; 6024 /* 6025 * Check for an unwritten dependency for this indirect offset. If 6026 * there is, merge the old dependency into the new one. This happens 6027 * as a result of reallocblk only. 6028 */ 6029 freefrag = NULL; 6030 if (aip->ai_oldblkno != 0) { 6031 LIST_FOREACH(oldaip, &indirdep->ir_deplisthd, ai_next) { 6032 if (oldaip->ai_offset == aip->ai_offset) { 6033 freefrag = allocindir_merge(aip, oldaip); 6034 goto done; 6035 } 6036 } 6037 LIST_FOREACH(oldaip, &indirdep->ir_donehd, ai_next) { 6038 if (oldaip->ai_offset == aip->ai_offset) { 6039 freefrag = allocindir_merge(aip, oldaip); 6040 goto done; 6041 } 6042 } 6043 } 6044 done: 6045 LIST_INSERT_HEAD(&indirdep->ir_deplisthd, aip, ai_next); 6046 return (freefrag); 6047 } 6048 6049 /* 6050 * Merge two allocindirs which refer to the same block. Move newblock 6051 * dependencies and setup the freefrags appropriately. 6052 */ 6053 static struct freefrag * 6054 allocindir_merge(aip, oldaip) 6055 struct allocindir *aip; 6056 struct allocindir *oldaip; 6057 { 6058 struct freefrag *freefrag; 6059 struct worklist *wk; 6060 6061 if (oldaip->ai_newblkno != aip->ai_oldblkno) 6062 panic("allocindir_merge: blkno"); 6063 aip->ai_oldblkno = oldaip->ai_oldblkno; 6064 freefrag = aip->ai_freefrag; 6065 aip->ai_freefrag = oldaip->ai_freefrag; 6066 oldaip->ai_freefrag = NULL; 6067 KASSERT(freefrag != NULL, ("setup_allocindir_phase2: No freefrag")); 6068 /* 6069 * If we are tracking a new directory-block allocation, 6070 * move it from the old allocindir to the new allocindir. 6071 */ 6072 if ((wk = LIST_FIRST(&oldaip->ai_newdirblk)) != NULL) { 6073 WORKLIST_REMOVE(wk); 6074 if (!LIST_EMPTY(&oldaip->ai_newdirblk)) 6075 panic("allocindir_merge: extra newdirblk"); 6076 WORKLIST_INSERT(&aip->ai_newdirblk, wk); 6077 } 6078 /* 6079 * We can skip journaling for this freefrag and just complete 6080 * any pending journal work for the allocindir that is being 6081 * removed after the freefrag completes. 6082 */ 6083 if (freefrag->ff_jdep) 6084 cancel_jfreefrag(WK_JFREEFRAG(freefrag->ff_jdep)); 6085 LIST_REMOVE(oldaip, ai_next); 6086 freefrag->ff_jdep = (struct worklist *)cancel_newblk(&oldaip->ai_block, 6087 &freefrag->ff_list, &freefrag->ff_jwork); 6088 free_newblk(&oldaip->ai_block); 6089 6090 return (freefrag); 6091 } 6092 6093 static inline void 6094 setup_freedirect(freeblks, ip, i, needj) 6095 struct freeblks *freeblks; 6096 struct inode *ip; 6097 int i; 6098 int needj; 6099 { 6100 struct ufsmount *ump; 6101 ufs2_daddr_t blkno; 6102 int frags; 6103 6104 blkno = DIP(ip, i_db[i]); 6105 if (blkno == 0) 6106 return; 6107 DIP_SET(ip, i_db[i], 0); 6108 ump = ITOUMP(ip); 6109 frags = sblksize(ump->um_fs, ip->i_size, i); 6110 frags = numfrags(ump->um_fs, frags); 6111 newfreework(ump, freeblks, NULL, i, blkno, frags, 0, needj); 6112 } 6113 6114 static inline void 6115 setup_freeext(freeblks, ip, i, needj) 6116 struct freeblks *freeblks; 6117 struct inode *ip; 6118 int i; 6119 int needj; 6120 { 6121 struct ufsmount *ump; 6122 ufs2_daddr_t blkno; 6123 int frags; 6124 6125 blkno = ip->i_din2->di_extb[i]; 6126 if (blkno == 0) 6127 return; 6128 ip->i_din2->di_extb[i] = 0; 6129 ump = ITOUMP(ip); 6130 frags = sblksize(ump->um_fs, ip->i_din2->di_extsize, i); 6131 frags = numfrags(ump->um_fs, frags); 6132 newfreework(ump, freeblks, NULL, -1 - i, blkno, frags, 0, needj); 6133 } 6134 6135 static inline void 6136 setup_freeindir(freeblks, ip, i, lbn, needj) 6137 struct freeblks *freeblks; 6138 struct inode *ip; 6139 int i; 6140 ufs_lbn_t lbn; 6141 int needj; 6142 { 6143 struct ufsmount *ump; 6144 ufs2_daddr_t blkno; 6145 6146 blkno = DIP(ip, i_ib[i]); 6147 if (blkno == 0) 6148 return; 6149 DIP_SET(ip, i_ib[i], 0); 6150 ump = ITOUMP(ip); 6151 newfreework(ump, freeblks, NULL, lbn, blkno, ump->um_fs->fs_frag, 6152 0, needj); 6153 } 6154 6155 static inline struct freeblks * 6156 newfreeblks(mp, ip) 6157 struct mount *mp; 6158 struct inode *ip; 6159 { 6160 struct freeblks *freeblks; 6161 6162 freeblks = malloc(sizeof(struct freeblks), 6163 M_FREEBLKS, M_SOFTDEP_FLAGS|M_ZERO); 6164 workitem_alloc(&freeblks->fb_list, D_FREEBLKS, mp); 6165 LIST_INIT(&freeblks->fb_jblkdephd); 6166 LIST_INIT(&freeblks->fb_jwork); 6167 freeblks->fb_ref = 0; 6168 freeblks->fb_cgwait = 0; 6169 freeblks->fb_state = ATTACHED; 6170 freeblks->fb_uid = ip->i_uid; 6171 freeblks->fb_inum = ip->i_number; 6172 freeblks->fb_vtype = ITOV(ip)->v_type; 6173 freeblks->fb_modrev = DIP(ip, i_modrev); 6174 freeblks->fb_devvp = ITODEVVP(ip); 6175 freeblks->fb_chkcnt = 0; 6176 freeblks->fb_len = 0; 6177 6178 return (freeblks); 6179 } 6180 6181 static void 6182 trunc_indirdep(indirdep, freeblks, bp, off) 6183 struct indirdep *indirdep; 6184 struct freeblks *freeblks; 6185 struct buf *bp; 6186 int off; 6187 { 6188 struct allocindir *aip, *aipn; 6189 6190 /* 6191 * The first set of allocindirs won't be in savedbp. 6192 */ 6193 LIST_FOREACH_SAFE(aip, &indirdep->ir_deplisthd, ai_next, aipn) 6194 if (aip->ai_offset > off) 6195 cancel_allocindir(aip, bp, freeblks, 1); 6196 LIST_FOREACH_SAFE(aip, &indirdep->ir_donehd, ai_next, aipn) 6197 if (aip->ai_offset > off) 6198 cancel_allocindir(aip, bp, freeblks, 1); 6199 /* 6200 * These will exist in savedbp. 6201 */ 6202 LIST_FOREACH_SAFE(aip, &indirdep->ir_writehd, ai_next, aipn) 6203 if (aip->ai_offset > off) 6204 cancel_allocindir(aip, NULL, freeblks, 0); 6205 LIST_FOREACH_SAFE(aip, &indirdep->ir_completehd, ai_next, aipn) 6206 if (aip->ai_offset > off) 6207 cancel_allocindir(aip, NULL, freeblks, 0); 6208 } 6209 6210 /* 6211 * Follow the chain of indirects down to lastlbn creating a freework 6212 * structure for each. This will be used to start indir_trunc() at 6213 * the right offset and create the journal records for the parrtial 6214 * truncation. A second step will handle the truncated dependencies. 6215 */ 6216 static int 6217 setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno) 6218 struct freeblks *freeblks; 6219 struct inode *ip; 6220 ufs_lbn_t lbn; 6221 ufs_lbn_t lastlbn; 6222 ufs2_daddr_t blkno; 6223 { 6224 struct indirdep *indirdep; 6225 struct indirdep *indirn; 6226 struct freework *freework; 6227 struct newblk *newblk; 6228 struct mount *mp; 6229 struct ufsmount *ump; 6230 struct buf *bp; 6231 uint8_t *start; 6232 uint8_t *end; 6233 ufs_lbn_t lbnadd; 6234 int level; 6235 int error; 6236 int off; 6237 6238 6239 freework = NULL; 6240 if (blkno == 0) 6241 return (0); 6242 mp = freeblks->fb_list.wk_mp; 6243 ump = VFSTOUFS(mp); 6244 bp = getblk(ITOV(ip), lbn, mp->mnt_stat.f_iosize, 0, 0, 0); 6245 if ((bp->b_flags & B_CACHE) == 0) { 6246 bp->b_blkno = blkptrtodb(VFSTOUFS(mp), blkno); 6247 bp->b_iocmd = BIO_READ; 6248 bp->b_flags &= ~B_INVAL; 6249 bp->b_ioflags &= ~BIO_ERROR; 6250 vfs_busy_pages(bp, 0); 6251 bp->b_iooffset = dbtob(bp->b_blkno); 6252 bstrategy(bp); 6253 #ifdef RACCT 6254 if (racct_enable) { 6255 PROC_LOCK(curproc); 6256 racct_add_buf(curproc, bp, 0); 6257 PROC_UNLOCK(curproc); 6258 } 6259 #endif /* RACCT */ 6260 curthread->td_ru.ru_inblock++; 6261 error = bufwait(bp); 6262 if (error) { 6263 brelse(bp); 6264 return (error); 6265 } 6266 } 6267 level = lbn_level(lbn); 6268 lbnadd = lbn_offset(ump->um_fs, level); 6269 /* 6270 * Compute the offset of the last block we want to keep. Store 6271 * in the freework the first block we want to completely free. 6272 */ 6273 off = (lastlbn - -(lbn + level)) / lbnadd; 6274 if (off + 1 == NINDIR(ump->um_fs)) 6275 goto nowork; 6276 freework = newfreework(ump, freeblks, NULL, lbn, blkno, 0, off + 1, 0); 6277 /* 6278 * Link the freework into the indirdep. This will prevent any new 6279 * allocations from proceeding until we are finished with the 6280 * truncate and the block is written. 6281 */ 6282 ACQUIRE_LOCK(ump); 6283 indirdep = indirdep_lookup(mp, ip, bp); 6284 if (indirdep->ir_freeblks) 6285 panic("setup_trunc_indir: indirdep already truncated."); 6286 TAILQ_INSERT_TAIL(&indirdep->ir_trunc, freework, fw_next); 6287 freework->fw_indir = indirdep; 6288 /* 6289 * Cancel any allocindirs that will not make it to disk. 6290 * We have to do this for all copies of the indirdep that 6291 * live on this newblk. 6292 */ 6293 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 6294 newblk_lookup(mp, dbtofsb(ump->um_fs, bp->b_blkno), 0, &newblk); 6295 LIST_FOREACH(indirn, &newblk->nb_indirdeps, ir_next) 6296 trunc_indirdep(indirn, freeblks, bp, off); 6297 } else 6298 trunc_indirdep(indirdep, freeblks, bp, off); 6299 FREE_LOCK(ump); 6300 /* 6301 * Creation is protected by the buf lock. The saveddata is only 6302 * needed if a full truncation follows a partial truncation but it 6303 * is difficult to allocate in that case so we fetch it anyway. 6304 */ 6305 if (indirdep->ir_saveddata == NULL) 6306 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP, 6307 M_SOFTDEP_FLAGS); 6308 nowork: 6309 /* Fetch the blkno of the child and the zero start offset. */ 6310 if (I_IS_UFS1(ip)) { 6311 blkno = ((ufs1_daddr_t *)bp->b_data)[off]; 6312 start = (uint8_t *)&((ufs1_daddr_t *)bp->b_data)[off+1]; 6313 } else { 6314 blkno = ((ufs2_daddr_t *)bp->b_data)[off]; 6315 start = (uint8_t *)&((ufs2_daddr_t *)bp->b_data)[off+1]; 6316 } 6317 if (freework) { 6318 /* Zero the truncated pointers. */ 6319 end = bp->b_data + bp->b_bcount; 6320 bzero(start, end - start); 6321 bdwrite(bp); 6322 } else 6323 bqrelse(bp); 6324 if (level == 0) 6325 return (0); 6326 lbn++; /* adjust level */ 6327 lbn -= (off * lbnadd); 6328 return setup_trunc_indir(freeblks, ip, lbn, lastlbn, blkno); 6329 } 6330 6331 /* 6332 * Complete the partial truncation of an indirect block setup by 6333 * setup_trunc_indir(). This zeros the truncated pointers in the saved 6334 * copy and writes them to disk before the freeblks is allowed to complete. 6335 */ 6336 static void 6337 complete_trunc_indir(freework) 6338 struct freework *freework; 6339 { 6340 struct freework *fwn; 6341 struct indirdep *indirdep; 6342 struct ufsmount *ump; 6343 struct buf *bp; 6344 uintptr_t start; 6345 int count; 6346 6347 ump = VFSTOUFS(freework->fw_list.wk_mp); 6348 LOCK_OWNED(ump); 6349 indirdep = freework->fw_indir; 6350 for (;;) { 6351 bp = indirdep->ir_bp; 6352 /* See if the block was discarded. */ 6353 if (bp == NULL) 6354 break; 6355 /* Inline part of getdirtybuf(). We dont want bremfree. */ 6356 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) == 0) 6357 break; 6358 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 6359 LOCK_PTR(ump)) == 0) 6360 BUF_UNLOCK(bp); 6361 ACQUIRE_LOCK(ump); 6362 } 6363 freework->fw_state |= DEPCOMPLETE; 6364 TAILQ_REMOVE(&indirdep->ir_trunc, freework, fw_next); 6365 /* 6366 * Zero the pointers in the saved copy. 6367 */ 6368 if (indirdep->ir_state & UFS1FMT) 6369 start = sizeof(ufs1_daddr_t); 6370 else 6371 start = sizeof(ufs2_daddr_t); 6372 start *= freework->fw_start; 6373 count = indirdep->ir_savebp->b_bcount - start; 6374 start += (uintptr_t)indirdep->ir_savebp->b_data; 6375 bzero((char *)start, count); 6376 /* 6377 * We need to start the next truncation in the list if it has not 6378 * been started yet. 6379 */ 6380 fwn = TAILQ_FIRST(&indirdep->ir_trunc); 6381 if (fwn != NULL) { 6382 if (fwn->fw_freeblks == indirdep->ir_freeblks) 6383 TAILQ_REMOVE(&indirdep->ir_trunc, fwn, fw_next); 6384 if ((fwn->fw_state & ONWORKLIST) == 0) 6385 freework_enqueue(fwn); 6386 } 6387 /* 6388 * If bp is NULL the block was fully truncated, restore 6389 * the saved block list otherwise free it if it is no 6390 * longer needed. 6391 */ 6392 if (TAILQ_EMPTY(&indirdep->ir_trunc)) { 6393 if (bp == NULL) 6394 bcopy(indirdep->ir_saveddata, 6395 indirdep->ir_savebp->b_data, 6396 indirdep->ir_savebp->b_bcount); 6397 free(indirdep->ir_saveddata, M_INDIRDEP); 6398 indirdep->ir_saveddata = NULL; 6399 } 6400 /* 6401 * When bp is NULL there is a full truncation pending. We 6402 * must wait for this full truncation to be journaled before 6403 * we can release this freework because the disk pointers will 6404 * never be written as zero. 6405 */ 6406 if (bp == NULL) { 6407 if (LIST_EMPTY(&indirdep->ir_freeblks->fb_jblkdephd)) 6408 handle_written_freework(freework); 6409 else 6410 WORKLIST_INSERT(&indirdep->ir_freeblks->fb_freeworkhd, 6411 &freework->fw_list); 6412 } else { 6413 /* Complete when the real copy is written. */ 6414 WORKLIST_INSERT(&bp->b_dep, &freework->fw_list); 6415 BUF_UNLOCK(bp); 6416 } 6417 } 6418 6419 /* 6420 * Calculate the number of blocks we are going to release where datablocks 6421 * is the current total and length is the new file size. 6422 */ 6423 static ufs2_daddr_t 6424 blkcount(fs, datablocks, length) 6425 struct fs *fs; 6426 ufs2_daddr_t datablocks; 6427 off_t length; 6428 { 6429 off_t totblks, numblks; 6430 6431 totblks = 0; 6432 numblks = howmany(length, fs->fs_bsize); 6433 if (numblks <= NDADDR) { 6434 totblks = howmany(length, fs->fs_fsize); 6435 goto out; 6436 } 6437 totblks = blkstofrags(fs, numblks); 6438 numblks -= NDADDR; 6439 /* 6440 * Count all single, then double, then triple indirects required. 6441 * Subtracting one indirects worth of blocks for each pass 6442 * acknowledges one of each pointed to by the inode. 6443 */ 6444 for (;;) { 6445 totblks += blkstofrags(fs, howmany(numblks, NINDIR(fs))); 6446 numblks -= NINDIR(fs); 6447 if (numblks <= 0) 6448 break; 6449 numblks = howmany(numblks, NINDIR(fs)); 6450 } 6451 out: 6452 totblks = fsbtodb(fs, totblks); 6453 /* 6454 * Handle sparse files. We can't reclaim more blocks than the inode 6455 * references. We will correct it later in handle_complete_freeblks() 6456 * when we know the real count. 6457 */ 6458 if (totblks > datablocks) 6459 return (0); 6460 return (datablocks - totblks); 6461 } 6462 6463 /* 6464 * Handle freeblocks for journaled softupdate filesystems. 6465 * 6466 * Contrary to normal softupdates, we must preserve the block pointers in 6467 * indirects until their subordinates are free. This is to avoid journaling 6468 * every block that is freed which may consume more space than the journal 6469 * itself. The recovery program will see the free block journals at the 6470 * base of the truncated area and traverse them to reclaim space. The 6471 * pointers in the inode may be cleared immediately after the journal 6472 * records are written because each direct and indirect pointer in the 6473 * inode is recorded in a journal. This permits full truncation to proceed 6474 * asynchronously. The write order is journal -> inode -> cgs -> indirects. 6475 * 6476 * The algorithm is as follows: 6477 * 1) Traverse the in-memory state and create journal entries to release 6478 * the relevant blocks and full indirect trees. 6479 * 2) Traverse the indirect block chain adding partial truncation freework 6480 * records to indirects in the path to lastlbn. The freework will 6481 * prevent new allocation dependencies from being satisfied in this 6482 * indirect until the truncation completes. 6483 * 3) Read and lock the inode block, performing an update with the new size 6484 * and pointers. This prevents truncated data from becoming valid on 6485 * disk through step 4. 6486 * 4) Reap unsatisfied dependencies that are beyond the truncated area, 6487 * eliminate journal work for those records that do not require it. 6488 * 5) Schedule the journal records to be written followed by the inode block. 6489 * 6) Allocate any necessary frags for the end of file. 6490 * 7) Zero any partially truncated blocks. 6491 * 6492 * From this truncation proceeds asynchronously using the freework and 6493 * indir_trunc machinery. The file will not be extended again into a 6494 * partially truncated indirect block until all work is completed but 6495 * the normal dependency mechanism ensures that it is rolled back/forward 6496 * as appropriate. Further truncation may occur without delay and is 6497 * serialized in indir_trunc(). 6498 */ 6499 void 6500 softdep_journal_freeblocks(ip, cred, length, flags) 6501 struct inode *ip; /* The inode whose length is to be reduced */ 6502 struct ucred *cred; 6503 off_t length; /* The new length for the file */ 6504 int flags; /* IO_EXT and/or IO_NORMAL */ 6505 { 6506 struct freeblks *freeblks, *fbn; 6507 struct worklist *wk, *wkn; 6508 struct inodedep *inodedep; 6509 struct jblkdep *jblkdep; 6510 struct allocdirect *adp, *adpn; 6511 struct ufsmount *ump; 6512 struct fs *fs; 6513 struct buf *bp; 6514 struct vnode *vp; 6515 struct mount *mp; 6516 ufs2_daddr_t extblocks, datablocks; 6517 ufs_lbn_t tmpval, lbn, lastlbn; 6518 int frags, lastoff, iboff, allocblock, needj, error, i; 6519 6520 ump = ITOUMP(ip); 6521 mp = UFSTOVFS(ump); 6522 fs = ump->um_fs; 6523 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 6524 ("softdep_journal_freeblocks called on non-softdep filesystem")); 6525 vp = ITOV(ip); 6526 needj = 1; 6527 iboff = -1; 6528 allocblock = 0; 6529 extblocks = 0; 6530 datablocks = 0; 6531 frags = 0; 6532 freeblks = newfreeblks(mp, ip); 6533 ACQUIRE_LOCK(ump); 6534 /* 6535 * If we're truncating a removed file that will never be written 6536 * we don't need to journal the block frees. The canceled journals 6537 * for the allocations will suffice. 6538 */ 6539 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6540 if ((inodedep->id_state & (UNLINKED | DEPCOMPLETE)) == UNLINKED && 6541 length == 0) 6542 needj = 0; 6543 CTR3(KTR_SUJ, "softdep_journal_freeblks: ip %d length %ld needj %d", 6544 ip->i_number, length, needj); 6545 FREE_LOCK(ump); 6546 /* 6547 * Calculate the lbn that we are truncating to. This results in -1 6548 * if we're truncating the 0 bytes. So it is the last lbn we want 6549 * to keep, not the first lbn we want to truncate. 6550 */ 6551 lastlbn = lblkno(fs, length + fs->fs_bsize - 1) - 1; 6552 lastoff = blkoff(fs, length); 6553 /* 6554 * Compute frags we are keeping in lastlbn. 0 means all. 6555 */ 6556 if (lastlbn >= 0 && lastlbn < NDADDR) { 6557 frags = fragroundup(fs, lastoff); 6558 /* adp offset of last valid allocdirect. */ 6559 iboff = lastlbn; 6560 } else if (lastlbn > 0) 6561 iboff = NDADDR; 6562 if (fs->fs_magic == FS_UFS2_MAGIC) 6563 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize)); 6564 /* 6565 * Handle normal data blocks and indirects. This section saves 6566 * values used after the inode update to complete frag and indirect 6567 * truncation. 6568 */ 6569 if ((flags & IO_NORMAL) != 0) { 6570 /* 6571 * Handle truncation of whole direct and indirect blocks. 6572 */ 6573 for (i = iboff + 1; i < NDADDR; i++) 6574 setup_freedirect(freeblks, ip, i, needj); 6575 for (i = 0, tmpval = NINDIR(fs), lbn = NDADDR; i < NIADDR; 6576 i++, lbn += tmpval, tmpval *= NINDIR(fs)) { 6577 /* Release a whole indirect tree. */ 6578 if (lbn > lastlbn) { 6579 setup_freeindir(freeblks, ip, i, -lbn -i, 6580 needj); 6581 continue; 6582 } 6583 iboff = i + NDADDR; 6584 /* 6585 * Traverse partially truncated indirect tree. 6586 */ 6587 if (lbn <= lastlbn && lbn + tmpval - 1 > lastlbn) 6588 setup_trunc_indir(freeblks, ip, -lbn - i, 6589 lastlbn, DIP(ip, i_ib[i])); 6590 } 6591 /* 6592 * Handle partial truncation to a frag boundary. 6593 */ 6594 if (frags) { 6595 ufs2_daddr_t blkno; 6596 long oldfrags; 6597 6598 oldfrags = blksize(fs, ip, lastlbn); 6599 blkno = DIP(ip, i_db[lastlbn]); 6600 if (blkno && oldfrags != frags) { 6601 oldfrags -= frags; 6602 oldfrags = numfrags(fs, oldfrags); 6603 blkno += numfrags(fs, frags); 6604 newfreework(ump, freeblks, NULL, lastlbn, 6605 blkno, oldfrags, 0, needj); 6606 if (needj) 6607 adjust_newfreework(freeblks, 6608 numfrags(fs, frags)); 6609 } else if (blkno == 0) 6610 allocblock = 1; 6611 } 6612 /* 6613 * Add a journal record for partial truncate if we are 6614 * handling indirect blocks. Non-indirects need no extra 6615 * journaling. 6616 */ 6617 if (length != 0 && lastlbn >= NDADDR) { 6618 ip->i_flag |= IN_TRUNCATED; 6619 newjtrunc(freeblks, length, 0); 6620 } 6621 ip->i_size = length; 6622 DIP_SET(ip, i_size, ip->i_size); 6623 datablocks = DIP(ip, i_blocks) - extblocks; 6624 if (length != 0) 6625 datablocks = blkcount(fs, datablocks, length); 6626 freeblks->fb_len = length; 6627 } 6628 if ((flags & IO_EXT) != 0) { 6629 for (i = 0; i < NXADDR; i++) 6630 setup_freeext(freeblks, ip, i, needj); 6631 ip->i_din2->di_extsize = 0; 6632 datablocks += extblocks; 6633 } 6634 #ifdef QUOTA 6635 /* Reference the quotas in case the block count is wrong in the end. */ 6636 quotaref(vp, freeblks->fb_quota); 6637 (void) chkdq(ip, -datablocks, NOCRED, 0); 6638 #endif 6639 freeblks->fb_chkcnt = -datablocks; 6640 UFS_LOCK(ump); 6641 fs->fs_pendingblocks += datablocks; 6642 UFS_UNLOCK(ump); 6643 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks); 6644 /* 6645 * Handle truncation of incomplete alloc direct dependencies. We 6646 * hold the inode block locked to prevent incomplete dependencies 6647 * from reaching the disk while we are eliminating those that 6648 * have been truncated. This is a partially inlined ffs_update(). 6649 */ 6650 ufs_itimes(vp); 6651 ip->i_flag &= ~(IN_LAZYACCESS | IN_LAZYMOD | IN_MODIFIED); 6652 error = bread(ump->um_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 6653 (int)fs->fs_bsize, cred, &bp); 6654 if (error) { 6655 brelse(bp); 6656 softdep_error("softdep_journal_freeblocks", error); 6657 return; 6658 } 6659 if (bp->b_bufsize == fs->fs_bsize) 6660 bp->b_flags |= B_CLUSTEROK; 6661 softdep_update_inodeblock(ip, bp, 0); 6662 if (ump->um_fstype == UFS1) 6663 *((struct ufs1_dinode *)bp->b_data + 6664 ino_to_fsbo(fs, ip->i_number)) = *ip->i_din1; 6665 else 6666 *((struct ufs2_dinode *)bp->b_data + 6667 ino_to_fsbo(fs, ip->i_number)) = *ip->i_din2; 6668 ACQUIRE_LOCK(ump); 6669 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6670 if ((inodedep->id_state & IOSTARTED) != 0) 6671 panic("softdep_setup_freeblocks: inode busy"); 6672 /* 6673 * Add the freeblks structure to the list of operations that 6674 * must await the zero'ed inode being written to disk. If we 6675 * still have a bitmap dependency (needj), then the inode 6676 * has never been written to disk, so we can process the 6677 * freeblks below once we have deleted the dependencies. 6678 */ 6679 if (needj) 6680 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list); 6681 else 6682 freeblks->fb_state |= COMPLETE; 6683 if ((flags & IO_NORMAL) != 0) { 6684 TAILQ_FOREACH_SAFE(adp, &inodedep->id_inoupdt, ad_next, adpn) { 6685 if (adp->ad_offset > iboff) 6686 cancel_allocdirect(&inodedep->id_inoupdt, adp, 6687 freeblks); 6688 /* 6689 * Truncate the allocdirect. We could eliminate 6690 * or modify journal records as well. 6691 */ 6692 else if (adp->ad_offset == iboff && frags) 6693 adp->ad_newsize = frags; 6694 } 6695 } 6696 if ((flags & IO_EXT) != 0) 6697 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 6698 cancel_allocdirect(&inodedep->id_extupdt, adp, 6699 freeblks); 6700 /* 6701 * Scan the bufwait list for newblock dependencies that will never 6702 * make it to disk. 6703 */ 6704 LIST_FOREACH_SAFE(wk, &inodedep->id_bufwait, wk_list, wkn) { 6705 if (wk->wk_type != D_ALLOCDIRECT) 6706 continue; 6707 adp = WK_ALLOCDIRECT(wk); 6708 if (((flags & IO_NORMAL) != 0 && (adp->ad_offset > iboff)) || 6709 ((flags & IO_EXT) != 0 && (adp->ad_state & EXTDATA))) { 6710 cancel_jfreeblk(freeblks, adp->ad_newblkno); 6711 cancel_newblk(WK_NEWBLK(wk), NULL, &freeblks->fb_jwork); 6712 WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk); 6713 } 6714 } 6715 /* 6716 * Add journal work. 6717 */ 6718 LIST_FOREACH(jblkdep, &freeblks->fb_jblkdephd, jb_deps) 6719 add_to_journal(&jblkdep->jb_list); 6720 FREE_LOCK(ump); 6721 bdwrite(bp); 6722 /* 6723 * Truncate dependency structures beyond length. 6724 */ 6725 trunc_dependencies(ip, freeblks, lastlbn, frags, flags); 6726 /* 6727 * This is only set when we need to allocate a fragment because 6728 * none existed at the end of a frag-sized file. It handles only 6729 * allocating a new, zero filled block. 6730 */ 6731 if (allocblock) { 6732 ip->i_size = length - lastoff; 6733 DIP_SET(ip, i_size, ip->i_size); 6734 error = UFS_BALLOC(vp, length - 1, 1, cred, BA_CLRBUF, &bp); 6735 if (error != 0) { 6736 softdep_error("softdep_journal_freeblks", error); 6737 return; 6738 } 6739 ip->i_size = length; 6740 DIP_SET(ip, i_size, length); 6741 ip->i_flag |= IN_CHANGE | IN_UPDATE; 6742 allocbuf(bp, frags); 6743 ffs_update(vp, 0); 6744 bawrite(bp); 6745 } else if (lastoff != 0 && vp->v_type != VDIR) { 6746 int size; 6747 6748 /* 6749 * Zero the end of a truncated frag or block. 6750 */ 6751 size = sblksize(fs, length, lastlbn); 6752 error = bread(vp, lastlbn, size, cred, &bp); 6753 if (error) { 6754 softdep_error("softdep_journal_freeblks", error); 6755 return; 6756 } 6757 bzero((char *)bp->b_data + lastoff, size - lastoff); 6758 bawrite(bp); 6759 6760 } 6761 ACQUIRE_LOCK(ump); 6762 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6763 TAILQ_INSERT_TAIL(&inodedep->id_freeblklst, freeblks, fb_next); 6764 freeblks->fb_state |= DEPCOMPLETE | ONDEPLIST; 6765 /* 6766 * We zero earlier truncations so they don't erroneously 6767 * update i_blocks. 6768 */ 6769 if (freeblks->fb_len == 0 && (flags & IO_NORMAL) != 0) 6770 TAILQ_FOREACH(fbn, &inodedep->id_freeblklst, fb_next) 6771 fbn->fb_len = 0; 6772 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE && 6773 LIST_EMPTY(&freeblks->fb_jblkdephd)) 6774 freeblks->fb_state |= INPROGRESS; 6775 else 6776 freeblks = NULL; 6777 FREE_LOCK(ump); 6778 if (freeblks) 6779 handle_workitem_freeblocks(freeblks, 0); 6780 trunc_pages(ip, length, extblocks, flags); 6781 6782 } 6783 6784 /* 6785 * Flush a JOP_SYNC to the journal. 6786 */ 6787 void 6788 softdep_journal_fsync(ip) 6789 struct inode *ip; 6790 { 6791 struct jfsync *jfsync; 6792 struct ufsmount *ump; 6793 6794 ump = ITOUMP(ip); 6795 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 6796 ("softdep_journal_fsync called on non-softdep filesystem")); 6797 if ((ip->i_flag & IN_TRUNCATED) == 0) 6798 return; 6799 ip->i_flag &= ~IN_TRUNCATED; 6800 jfsync = malloc(sizeof(*jfsync), M_JFSYNC, M_SOFTDEP_FLAGS | M_ZERO); 6801 workitem_alloc(&jfsync->jfs_list, D_JFSYNC, UFSTOVFS(ump)); 6802 jfsync->jfs_size = ip->i_size; 6803 jfsync->jfs_ino = ip->i_number; 6804 ACQUIRE_LOCK(ump); 6805 add_to_journal(&jfsync->jfs_list); 6806 jwait(&jfsync->jfs_list, MNT_WAIT); 6807 FREE_LOCK(ump); 6808 } 6809 6810 /* 6811 * Block de-allocation dependencies. 6812 * 6813 * When blocks are de-allocated, the on-disk pointers must be nullified before 6814 * the blocks are made available for use by other files. (The true 6815 * requirement is that old pointers must be nullified before new on-disk 6816 * pointers are set. We chose this slightly more stringent requirement to 6817 * reduce complexity.) Our implementation handles this dependency by updating 6818 * the inode (or indirect block) appropriately but delaying the actual block 6819 * de-allocation (i.e., freemap and free space count manipulation) until 6820 * after the updated versions reach stable storage. After the disk is 6821 * updated, the blocks can be safely de-allocated whenever it is convenient. 6822 * This implementation handles only the common case of reducing a file's 6823 * length to zero. Other cases are handled by the conventional synchronous 6824 * write approach. 6825 * 6826 * The ffs implementation with which we worked double-checks 6827 * the state of the block pointers and file size as it reduces 6828 * a file's length. Some of this code is replicated here in our 6829 * soft updates implementation. The freeblks->fb_chkcnt field is 6830 * used to transfer a part of this information to the procedure 6831 * that eventually de-allocates the blocks. 6832 * 6833 * This routine should be called from the routine that shortens 6834 * a file's length, before the inode's size or block pointers 6835 * are modified. It will save the block pointer information for 6836 * later release and zero the inode so that the calling routine 6837 * can release it. 6838 */ 6839 void 6840 softdep_setup_freeblocks(ip, length, flags) 6841 struct inode *ip; /* The inode whose length is to be reduced */ 6842 off_t length; /* The new length for the file */ 6843 int flags; /* IO_EXT and/or IO_NORMAL */ 6844 { 6845 struct ufs1_dinode *dp1; 6846 struct ufs2_dinode *dp2; 6847 struct freeblks *freeblks; 6848 struct inodedep *inodedep; 6849 struct allocdirect *adp; 6850 struct ufsmount *ump; 6851 struct buf *bp; 6852 struct fs *fs; 6853 ufs2_daddr_t extblocks, datablocks; 6854 struct mount *mp; 6855 int i, delay, error; 6856 ufs_lbn_t tmpval; 6857 ufs_lbn_t lbn; 6858 6859 ump = ITOUMP(ip); 6860 mp = UFSTOVFS(ump); 6861 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 6862 ("softdep_setup_freeblocks called on non-softdep filesystem")); 6863 CTR2(KTR_SUJ, "softdep_setup_freeblks: ip %d length %ld", 6864 ip->i_number, length); 6865 KASSERT(length == 0, ("softdep_setup_freeblocks: non-zero length")); 6866 fs = ump->um_fs; 6867 if ((error = bread(ump->um_devvp, 6868 fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 6869 (int)fs->fs_bsize, NOCRED, &bp)) != 0) { 6870 brelse(bp); 6871 softdep_error("softdep_setup_freeblocks", error); 6872 return; 6873 } 6874 freeblks = newfreeblks(mp, ip); 6875 extblocks = 0; 6876 datablocks = 0; 6877 if (fs->fs_magic == FS_UFS2_MAGIC) 6878 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize)); 6879 if ((flags & IO_NORMAL) != 0) { 6880 for (i = 0; i < NDADDR; i++) 6881 setup_freedirect(freeblks, ip, i, 0); 6882 for (i = 0, tmpval = NINDIR(fs), lbn = NDADDR; i < NIADDR; 6883 i++, lbn += tmpval, tmpval *= NINDIR(fs)) 6884 setup_freeindir(freeblks, ip, i, -lbn -i, 0); 6885 ip->i_size = 0; 6886 DIP_SET(ip, i_size, 0); 6887 datablocks = DIP(ip, i_blocks) - extblocks; 6888 } 6889 if ((flags & IO_EXT) != 0) { 6890 for (i = 0; i < NXADDR; i++) 6891 setup_freeext(freeblks, ip, i, 0); 6892 ip->i_din2->di_extsize = 0; 6893 datablocks += extblocks; 6894 } 6895 #ifdef QUOTA 6896 /* Reference the quotas in case the block count is wrong in the end. */ 6897 quotaref(ITOV(ip), freeblks->fb_quota); 6898 (void) chkdq(ip, -datablocks, NOCRED, 0); 6899 #endif 6900 freeblks->fb_chkcnt = -datablocks; 6901 UFS_LOCK(ump); 6902 fs->fs_pendingblocks += datablocks; 6903 UFS_UNLOCK(ump); 6904 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - datablocks); 6905 /* 6906 * Push the zero'ed inode to to its disk buffer so that we are free 6907 * to delete its dependencies below. Once the dependencies are gone 6908 * the buffer can be safely released. 6909 */ 6910 if (ump->um_fstype == UFS1) { 6911 dp1 = ((struct ufs1_dinode *)bp->b_data + 6912 ino_to_fsbo(fs, ip->i_number)); 6913 ip->i_din1->di_freelink = dp1->di_freelink; 6914 *dp1 = *ip->i_din1; 6915 } else { 6916 dp2 = ((struct ufs2_dinode *)bp->b_data + 6917 ino_to_fsbo(fs, ip->i_number)); 6918 ip->i_din2->di_freelink = dp2->di_freelink; 6919 *dp2 = *ip->i_din2; 6920 } 6921 /* 6922 * Find and eliminate any inode dependencies. 6923 */ 6924 ACQUIRE_LOCK(ump); 6925 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 6926 if ((inodedep->id_state & IOSTARTED) != 0) 6927 panic("softdep_setup_freeblocks: inode busy"); 6928 /* 6929 * Add the freeblks structure to the list of operations that 6930 * must await the zero'ed inode being written to disk. If we 6931 * still have a bitmap dependency (delay == 0), then the inode 6932 * has never been written to disk, so we can process the 6933 * freeblks below once we have deleted the dependencies. 6934 */ 6935 delay = (inodedep->id_state & DEPCOMPLETE); 6936 if (delay) 6937 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list); 6938 else 6939 freeblks->fb_state |= COMPLETE; 6940 /* 6941 * Because the file length has been truncated to zero, any 6942 * pending block allocation dependency structures associated 6943 * with this inode are obsolete and can simply be de-allocated. 6944 * We must first merge the two dependency lists to get rid of 6945 * any duplicate freefrag structures, then purge the merged list. 6946 * If we still have a bitmap dependency, then the inode has never 6947 * been written to disk, so we can free any fragments without delay. 6948 */ 6949 if (flags & IO_NORMAL) { 6950 merge_inode_lists(&inodedep->id_newinoupdt, 6951 &inodedep->id_inoupdt); 6952 while ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL) 6953 cancel_allocdirect(&inodedep->id_inoupdt, adp, 6954 freeblks); 6955 } 6956 if (flags & IO_EXT) { 6957 merge_inode_lists(&inodedep->id_newextupdt, 6958 &inodedep->id_extupdt); 6959 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 6960 cancel_allocdirect(&inodedep->id_extupdt, adp, 6961 freeblks); 6962 } 6963 FREE_LOCK(ump); 6964 bdwrite(bp); 6965 trunc_dependencies(ip, freeblks, -1, 0, flags); 6966 ACQUIRE_LOCK(ump); 6967 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 6968 (void) free_inodedep(inodedep); 6969 freeblks->fb_state |= DEPCOMPLETE; 6970 /* 6971 * If the inode with zeroed block pointers is now on disk 6972 * we can start freeing blocks. 6973 */ 6974 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) 6975 freeblks->fb_state |= INPROGRESS; 6976 else 6977 freeblks = NULL; 6978 FREE_LOCK(ump); 6979 if (freeblks) 6980 handle_workitem_freeblocks(freeblks, 0); 6981 trunc_pages(ip, length, extblocks, flags); 6982 } 6983 6984 /* 6985 * Eliminate pages from the page cache that back parts of this inode and 6986 * adjust the vnode pager's idea of our size. This prevents stale data 6987 * from hanging around in the page cache. 6988 */ 6989 static void 6990 trunc_pages(ip, length, extblocks, flags) 6991 struct inode *ip; 6992 off_t length; 6993 ufs2_daddr_t extblocks; 6994 int flags; 6995 { 6996 struct vnode *vp; 6997 struct fs *fs; 6998 ufs_lbn_t lbn; 6999 off_t end, extend; 7000 7001 vp = ITOV(ip); 7002 fs = ITOFS(ip); 7003 extend = OFF_TO_IDX(lblktosize(fs, -extblocks)); 7004 if ((flags & IO_EXT) != 0) 7005 vn_pages_remove(vp, extend, 0); 7006 if ((flags & IO_NORMAL) == 0) 7007 return; 7008 BO_LOCK(&vp->v_bufobj); 7009 drain_output(vp); 7010 BO_UNLOCK(&vp->v_bufobj); 7011 /* 7012 * The vnode pager eliminates file pages we eliminate indirects 7013 * below. 7014 */ 7015 vnode_pager_setsize(vp, length); 7016 /* 7017 * Calculate the end based on the last indirect we want to keep. If 7018 * the block extends into indirects we can just use the negative of 7019 * its lbn. Doubles and triples exist at lower numbers so we must 7020 * be careful not to remove those, if they exist. double and triple 7021 * indirect lbns do not overlap with others so it is not important 7022 * to verify how many levels are required. 7023 */ 7024 lbn = lblkno(fs, length); 7025 if (lbn >= NDADDR) { 7026 /* Calculate the virtual lbn of the triple indirect. */ 7027 lbn = -lbn - (NIADDR - 1); 7028 end = OFF_TO_IDX(lblktosize(fs, lbn)); 7029 } else 7030 end = extend; 7031 vn_pages_remove(vp, OFF_TO_IDX(OFF_MAX), end); 7032 } 7033 7034 /* 7035 * See if the buf bp is in the range eliminated by truncation. 7036 */ 7037 static int 7038 trunc_check_buf(bp, blkoffp, lastlbn, lastoff, flags) 7039 struct buf *bp; 7040 int *blkoffp; 7041 ufs_lbn_t lastlbn; 7042 int lastoff; 7043 int flags; 7044 { 7045 ufs_lbn_t lbn; 7046 7047 *blkoffp = 0; 7048 /* Only match ext/normal blocks as appropriate. */ 7049 if (((flags & IO_EXT) == 0 && (bp->b_xflags & BX_ALTDATA)) || 7050 ((flags & IO_NORMAL) == 0 && (bp->b_xflags & BX_ALTDATA) == 0)) 7051 return (0); 7052 /* ALTDATA is always a full truncation. */ 7053 if ((bp->b_xflags & BX_ALTDATA) != 0) 7054 return (1); 7055 /* -1 is full truncation. */ 7056 if (lastlbn == -1) 7057 return (1); 7058 /* 7059 * If this is a partial truncate we only want those 7060 * blocks and indirect blocks that cover the range 7061 * we're after. 7062 */ 7063 lbn = bp->b_lblkno; 7064 if (lbn < 0) 7065 lbn = -(lbn + lbn_level(lbn)); 7066 if (lbn < lastlbn) 7067 return (0); 7068 /* Here we only truncate lblkno if it's partial. */ 7069 if (lbn == lastlbn) { 7070 if (lastoff == 0) 7071 return (0); 7072 *blkoffp = lastoff; 7073 } 7074 return (1); 7075 } 7076 7077 /* 7078 * Eliminate any dependencies that exist in memory beyond lblkno:off 7079 */ 7080 static void 7081 trunc_dependencies(ip, freeblks, lastlbn, lastoff, flags) 7082 struct inode *ip; 7083 struct freeblks *freeblks; 7084 ufs_lbn_t lastlbn; 7085 int lastoff; 7086 int flags; 7087 { 7088 struct bufobj *bo; 7089 struct vnode *vp; 7090 struct buf *bp; 7091 int blkoff; 7092 7093 /* 7094 * We must wait for any I/O in progress to finish so that 7095 * all potential buffers on the dirty list will be visible. 7096 * Once they are all there, walk the list and get rid of 7097 * any dependencies. 7098 */ 7099 vp = ITOV(ip); 7100 bo = &vp->v_bufobj; 7101 BO_LOCK(bo); 7102 drain_output(vp); 7103 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) 7104 bp->b_vflags &= ~BV_SCANNED; 7105 restart: 7106 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) { 7107 if (bp->b_vflags & BV_SCANNED) 7108 continue; 7109 if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) { 7110 bp->b_vflags |= BV_SCANNED; 7111 continue; 7112 } 7113 KASSERT(bp->b_bufobj == bo, ("Wrong object in buffer")); 7114 if ((bp = getdirtybuf(bp, BO_LOCKPTR(bo), MNT_WAIT)) == NULL) 7115 goto restart; 7116 BO_UNLOCK(bo); 7117 if (deallocate_dependencies(bp, freeblks, blkoff)) 7118 bqrelse(bp); 7119 else 7120 brelse(bp); 7121 BO_LOCK(bo); 7122 goto restart; 7123 } 7124 /* 7125 * Now do the work of vtruncbuf while also matching indirect blocks. 7126 */ 7127 TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs) 7128 bp->b_vflags &= ~BV_SCANNED; 7129 cleanrestart: 7130 TAILQ_FOREACH(bp, &bo->bo_clean.bv_hd, b_bobufs) { 7131 if (bp->b_vflags & BV_SCANNED) 7132 continue; 7133 if (!trunc_check_buf(bp, &blkoff, lastlbn, lastoff, flags)) { 7134 bp->b_vflags |= BV_SCANNED; 7135 continue; 7136 } 7137 if (BUF_LOCK(bp, 7138 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, 7139 BO_LOCKPTR(bo)) == ENOLCK) { 7140 BO_LOCK(bo); 7141 goto cleanrestart; 7142 } 7143 bp->b_vflags |= BV_SCANNED; 7144 bremfree(bp); 7145 if (blkoff != 0) { 7146 allocbuf(bp, blkoff); 7147 bqrelse(bp); 7148 } else { 7149 bp->b_flags |= B_INVAL | B_NOCACHE | B_RELBUF; 7150 brelse(bp); 7151 } 7152 BO_LOCK(bo); 7153 goto cleanrestart; 7154 } 7155 drain_output(vp); 7156 BO_UNLOCK(bo); 7157 } 7158 7159 static int 7160 cancel_pagedep(pagedep, freeblks, blkoff) 7161 struct pagedep *pagedep; 7162 struct freeblks *freeblks; 7163 int blkoff; 7164 { 7165 struct jremref *jremref; 7166 struct jmvref *jmvref; 7167 struct dirrem *dirrem, *tmp; 7168 int i; 7169 7170 /* 7171 * Copy any directory remove dependencies to the list 7172 * to be processed after the freeblks proceeds. If 7173 * directory entry never made it to disk they 7174 * can be dumped directly onto the work list. 7175 */ 7176 LIST_FOREACH_SAFE(dirrem, &pagedep->pd_dirremhd, dm_next, tmp) { 7177 /* Skip this directory removal if it is intended to remain. */ 7178 if (dirrem->dm_offset < blkoff) 7179 continue; 7180 /* 7181 * If there are any dirrems we wait for the journal write 7182 * to complete and then restart the buf scan as the lock 7183 * has been dropped. 7184 */ 7185 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) { 7186 jwait(&jremref->jr_list, MNT_WAIT); 7187 return (ERESTART); 7188 } 7189 LIST_REMOVE(dirrem, dm_next); 7190 dirrem->dm_dirinum = pagedep->pd_ino; 7191 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &dirrem->dm_list); 7192 } 7193 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) { 7194 jwait(&jmvref->jm_list, MNT_WAIT); 7195 return (ERESTART); 7196 } 7197 /* 7198 * When we're partially truncating a pagedep we just want to flush 7199 * journal entries and return. There can not be any adds in the 7200 * truncated portion of the directory and newblk must remain if 7201 * part of the block remains. 7202 */ 7203 if (blkoff != 0) { 7204 struct diradd *dap; 7205 7206 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist) 7207 if (dap->da_offset > blkoff) 7208 panic("cancel_pagedep: diradd %p off %d > %d", 7209 dap, dap->da_offset, blkoff); 7210 for (i = 0; i < DAHASHSZ; i++) 7211 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) 7212 if (dap->da_offset > blkoff) 7213 panic("cancel_pagedep: diradd %p off %d > %d", 7214 dap, dap->da_offset, blkoff); 7215 return (0); 7216 } 7217 /* 7218 * There should be no directory add dependencies present 7219 * as the directory could not be truncated until all 7220 * children were removed. 7221 */ 7222 KASSERT(LIST_FIRST(&pagedep->pd_pendinghd) == NULL, 7223 ("deallocate_dependencies: pendinghd != NULL")); 7224 for (i = 0; i < DAHASHSZ; i++) 7225 KASSERT(LIST_FIRST(&pagedep->pd_diraddhd[i]) == NULL, 7226 ("deallocate_dependencies: diraddhd != NULL")); 7227 if ((pagedep->pd_state & NEWBLOCK) != 0) 7228 free_newdirblk(pagedep->pd_newdirblk); 7229 if (free_pagedep(pagedep) == 0) 7230 panic("Failed to free pagedep %p", pagedep); 7231 return (0); 7232 } 7233 7234 /* 7235 * Reclaim any dependency structures from a buffer that is about to 7236 * be reallocated to a new vnode. The buffer must be locked, thus, 7237 * no I/O completion operations can occur while we are manipulating 7238 * its associated dependencies. The mutex is held so that other I/O's 7239 * associated with related dependencies do not occur. 7240 */ 7241 static int 7242 deallocate_dependencies(bp, freeblks, off) 7243 struct buf *bp; 7244 struct freeblks *freeblks; 7245 int off; 7246 { 7247 struct indirdep *indirdep; 7248 struct pagedep *pagedep; 7249 struct worklist *wk, *wkn; 7250 struct ufsmount *ump; 7251 7252 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL) 7253 goto done; 7254 ump = VFSTOUFS(wk->wk_mp); 7255 ACQUIRE_LOCK(ump); 7256 LIST_FOREACH_SAFE(wk, &bp->b_dep, wk_list, wkn) { 7257 switch (wk->wk_type) { 7258 case D_INDIRDEP: 7259 indirdep = WK_INDIRDEP(wk); 7260 if (bp->b_lblkno >= 0 || 7261 bp->b_blkno != indirdep->ir_savebp->b_lblkno) 7262 panic("deallocate_dependencies: not indir"); 7263 cancel_indirdep(indirdep, bp, freeblks); 7264 continue; 7265 7266 case D_PAGEDEP: 7267 pagedep = WK_PAGEDEP(wk); 7268 if (cancel_pagedep(pagedep, freeblks, off)) { 7269 FREE_LOCK(ump); 7270 return (ERESTART); 7271 } 7272 continue; 7273 7274 case D_ALLOCINDIR: 7275 /* 7276 * Simply remove the allocindir, we'll find it via 7277 * the indirdep where we can clear pointers if 7278 * needed. 7279 */ 7280 WORKLIST_REMOVE(wk); 7281 continue; 7282 7283 case D_FREEWORK: 7284 /* 7285 * A truncation is waiting for the zero'd pointers 7286 * to be written. It can be freed when the freeblks 7287 * is journaled. 7288 */ 7289 WORKLIST_REMOVE(wk); 7290 wk->wk_state |= ONDEPLIST; 7291 WORKLIST_INSERT(&freeblks->fb_freeworkhd, wk); 7292 break; 7293 7294 case D_ALLOCDIRECT: 7295 if (off != 0) 7296 continue; 7297 /* FALLTHROUGH */ 7298 default: 7299 panic("deallocate_dependencies: Unexpected type %s", 7300 TYPENAME(wk->wk_type)); 7301 /* NOTREACHED */ 7302 } 7303 } 7304 FREE_LOCK(ump); 7305 done: 7306 /* 7307 * Don't throw away this buf, we were partially truncating and 7308 * some deps may always remain. 7309 */ 7310 if (off) { 7311 allocbuf(bp, off); 7312 bp->b_vflags |= BV_SCANNED; 7313 return (EBUSY); 7314 } 7315 bp->b_flags |= B_INVAL | B_NOCACHE; 7316 7317 return (0); 7318 } 7319 7320 /* 7321 * An allocdirect is being canceled due to a truncate. We must make sure 7322 * the journal entry is released in concert with the blkfree that releases 7323 * the storage. Completed journal entries must not be released until the 7324 * space is no longer pointed to by the inode or in the bitmap. 7325 */ 7326 static void 7327 cancel_allocdirect(adphead, adp, freeblks) 7328 struct allocdirectlst *adphead; 7329 struct allocdirect *adp; 7330 struct freeblks *freeblks; 7331 { 7332 struct freework *freework; 7333 struct newblk *newblk; 7334 struct worklist *wk; 7335 7336 TAILQ_REMOVE(adphead, adp, ad_next); 7337 newblk = (struct newblk *)adp; 7338 freework = NULL; 7339 /* 7340 * Find the correct freework structure. 7341 */ 7342 LIST_FOREACH(wk, &freeblks->fb_freeworkhd, wk_list) { 7343 if (wk->wk_type != D_FREEWORK) 7344 continue; 7345 freework = WK_FREEWORK(wk); 7346 if (freework->fw_blkno == newblk->nb_newblkno) 7347 break; 7348 } 7349 if (freework == NULL) 7350 panic("cancel_allocdirect: Freework not found"); 7351 /* 7352 * If a newblk exists at all we still have the journal entry that 7353 * initiated the allocation so we do not need to journal the free. 7354 */ 7355 cancel_jfreeblk(freeblks, freework->fw_blkno); 7356 /* 7357 * If the journal hasn't been written the jnewblk must be passed 7358 * to the call to ffs_blkfree that reclaims the space. We accomplish 7359 * this by linking the journal dependency into the freework to be 7360 * freed when freework_freeblock() is called. If the journal has 7361 * been written we can simply reclaim the journal space when the 7362 * freeblks work is complete. 7363 */ 7364 freework->fw_jnewblk = cancel_newblk(newblk, &freework->fw_list, 7365 &freeblks->fb_jwork); 7366 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list); 7367 } 7368 7369 7370 /* 7371 * Cancel a new block allocation. May be an indirect or direct block. We 7372 * remove it from various lists and return any journal record that needs to 7373 * be resolved by the caller. 7374 * 7375 * A special consideration is made for indirects which were never pointed 7376 * at on disk and will never be found once this block is released. 7377 */ 7378 static struct jnewblk * 7379 cancel_newblk(newblk, wk, wkhd) 7380 struct newblk *newblk; 7381 struct worklist *wk; 7382 struct workhead *wkhd; 7383 { 7384 struct jnewblk *jnewblk; 7385 7386 CTR1(KTR_SUJ, "cancel_newblk: blkno %jd", newblk->nb_newblkno); 7387 7388 newblk->nb_state |= GOINGAWAY; 7389 /* 7390 * Previously we traversed the completedhd on each indirdep 7391 * attached to this newblk to cancel them and gather journal 7392 * work. Since we need only the oldest journal segment and 7393 * the lowest point on the tree will always have the oldest 7394 * journal segment we are free to release the segments 7395 * of any subordinates and may leave the indirdep list to 7396 * indirdep_complete() when this newblk is freed. 7397 */ 7398 if (newblk->nb_state & ONDEPLIST) { 7399 newblk->nb_state &= ~ONDEPLIST; 7400 LIST_REMOVE(newblk, nb_deps); 7401 } 7402 if (newblk->nb_state & ONWORKLIST) 7403 WORKLIST_REMOVE(&newblk->nb_list); 7404 /* 7405 * If the journal entry hasn't been written we save a pointer to 7406 * the dependency that frees it until it is written or the 7407 * superseding operation completes. 7408 */ 7409 jnewblk = newblk->nb_jnewblk; 7410 if (jnewblk != NULL && wk != NULL) { 7411 newblk->nb_jnewblk = NULL; 7412 jnewblk->jn_dep = wk; 7413 } 7414 if (!LIST_EMPTY(&newblk->nb_jwork)) 7415 jwork_move(wkhd, &newblk->nb_jwork); 7416 /* 7417 * When truncating we must free the newdirblk early to remove 7418 * the pagedep from the hash before returning. 7419 */ 7420 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL) 7421 free_newdirblk(WK_NEWDIRBLK(wk)); 7422 if (!LIST_EMPTY(&newblk->nb_newdirblk)) 7423 panic("cancel_newblk: extra newdirblk"); 7424 7425 return (jnewblk); 7426 } 7427 7428 /* 7429 * Schedule the freefrag associated with a newblk to be released once 7430 * the pointers are written and the previous block is no longer needed. 7431 */ 7432 static void 7433 newblk_freefrag(newblk) 7434 struct newblk *newblk; 7435 { 7436 struct freefrag *freefrag; 7437 7438 if (newblk->nb_freefrag == NULL) 7439 return; 7440 freefrag = newblk->nb_freefrag; 7441 newblk->nb_freefrag = NULL; 7442 freefrag->ff_state |= COMPLETE; 7443 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 7444 add_to_worklist(&freefrag->ff_list, 0); 7445 } 7446 7447 /* 7448 * Free a newblk. Generate a new freefrag work request if appropriate. 7449 * This must be called after the inode pointer and any direct block pointers 7450 * are valid or fully removed via truncate or frag extension. 7451 */ 7452 static void 7453 free_newblk(newblk) 7454 struct newblk *newblk; 7455 { 7456 struct indirdep *indirdep; 7457 struct worklist *wk; 7458 7459 KASSERT(newblk->nb_jnewblk == NULL, 7460 ("free_newblk: jnewblk %p still attached", newblk->nb_jnewblk)); 7461 KASSERT(newblk->nb_list.wk_type != D_NEWBLK, 7462 ("free_newblk: unclaimed newblk")); 7463 LOCK_OWNED(VFSTOUFS(newblk->nb_list.wk_mp)); 7464 newblk_freefrag(newblk); 7465 if (newblk->nb_state & ONDEPLIST) 7466 LIST_REMOVE(newblk, nb_deps); 7467 if (newblk->nb_state & ONWORKLIST) 7468 WORKLIST_REMOVE(&newblk->nb_list); 7469 LIST_REMOVE(newblk, nb_hash); 7470 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL) 7471 free_newdirblk(WK_NEWDIRBLK(wk)); 7472 if (!LIST_EMPTY(&newblk->nb_newdirblk)) 7473 panic("free_newblk: extra newdirblk"); 7474 while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL) 7475 indirdep_complete(indirdep); 7476 handle_jwork(&newblk->nb_jwork); 7477 WORKITEM_FREE(newblk, D_NEWBLK); 7478 } 7479 7480 /* 7481 * Free a newdirblk. Clear the NEWBLOCK flag on its associated pagedep. 7482 * This routine must be called with splbio interrupts blocked. 7483 */ 7484 static void 7485 free_newdirblk(newdirblk) 7486 struct newdirblk *newdirblk; 7487 { 7488 struct pagedep *pagedep; 7489 struct diradd *dap; 7490 struct worklist *wk; 7491 7492 LOCK_OWNED(VFSTOUFS(newdirblk->db_list.wk_mp)); 7493 WORKLIST_REMOVE(&newdirblk->db_list); 7494 /* 7495 * If the pagedep is still linked onto the directory buffer 7496 * dependency chain, then some of the entries on the 7497 * pd_pendinghd list may not be committed to disk yet. In 7498 * this case, we will simply clear the NEWBLOCK flag and 7499 * let the pd_pendinghd list be processed when the pagedep 7500 * is next written. If the pagedep is no longer on the buffer 7501 * dependency chain, then all the entries on the pd_pending 7502 * list are committed to disk and we can free them here. 7503 */ 7504 pagedep = newdirblk->db_pagedep; 7505 pagedep->pd_state &= ~NEWBLOCK; 7506 if ((pagedep->pd_state & ONWORKLIST) == 0) { 7507 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 7508 free_diradd(dap, NULL); 7509 /* 7510 * If no dependencies remain, the pagedep will be freed. 7511 */ 7512 free_pagedep(pagedep); 7513 } 7514 /* Should only ever be one item in the list. */ 7515 while ((wk = LIST_FIRST(&newdirblk->db_mkdir)) != NULL) { 7516 WORKLIST_REMOVE(wk); 7517 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 7518 } 7519 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 7520 } 7521 7522 /* 7523 * Prepare an inode to be freed. The actual free operation is not 7524 * done until the zero'ed inode has been written to disk. 7525 */ 7526 void 7527 softdep_freefile(pvp, ino, mode) 7528 struct vnode *pvp; 7529 ino_t ino; 7530 int mode; 7531 { 7532 struct inode *ip = VTOI(pvp); 7533 struct inodedep *inodedep; 7534 struct freefile *freefile; 7535 struct freeblks *freeblks; 7536 struct ufsmount *ump; 7537 7538 ump = ITOUMP(ip); 7539 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 7540 ("softdep_freefile called on non-softdep filesystem")); 7541 /* 7542 * This sets up the inode de-allocation dependency. 7543 */ 7544 freefile = malloc(sizeof(struct freefile), 7545 M_FREEFILE, M_SOFTDEP_FLAGS); 7546 workitem_alloc(&freefile->fx_list, D_FREEFILE, pvp->v_mount); 7547 freefile->fx_mode = mode; 7548 freefile->fx_oldinum = ino; 7549 freefile->fx_devvp = ump->um_devvp; 7550 LIST_INIT(&freefile->fx_jwork); 7551 UFS_LOCK(ump); 7552 ump->um_fs->fs_pendinginodes += 1; 7553 UFS_UNLOCK(ump); 7554 7555 /* 7556 * If the inodedep does not exist, then the zero'ed inode has 7557 * been written to disk. If the allocated inode has never been 7558 * written to disk, then the on-disk inode is zero'ed. In either 7559 * case we can free the file immediately. If the journal was 7560 * canceled before being written the inode will never make it to 7561 * disk and we must send the canceled journal entrys to 7562 * ffs_freefile() to be cleared in conjunction with the bitmap. 7563 * Any blocks waiting on the inode to write can be safely freed 7564 * here as it will never been written. 7565 */ 7566 ACQUIRE_LOCK(ump); 7567 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 7568 if (inodedep) { 7569 /* 7570 * Clear out freeblks that no longer need to reference 7571 * this inode. 7572 */ 7573 while ((freeblks = 7574 TAILQ_FIRST(&inodedep->id_freeblklst)) != NULL) { 7575 TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks, 7576 fb_next); 7577 freeblks->fb_state &= ~ONDEPLIST; 7578 } 7579 /* 7580 * Remove this inode from the unlinked list. 7581 */ 7582 if (inodedep->id_state & UNLINKED) { 7583 /* 7584 * Save the journal work to be freed with the bitmap 7585 * before we clear UNLINKED. Otherwise it can be lost 7586 * if the inode block is written. 7587 */ 7588 handle_bufwait(inodedep, &freefile->fx_jwork); 7589 clear_unlinked_inodedep(inodedep); 7590 /* 7591 * Re-acquire inodedep as we've dropped the 7592 * per-filesystem lock in clear_unlinked_inodedep(). 7593 */ 7594 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 7595 } 7596 } 7597 if (inodedep == NULL || check_inode_unwritten(inodedep)) { 7598 FREE_LOCK(ump); 7599 handle_workitem_freefile(freefile); 7600 return; 7601 } 7602 if ((inodedep->id_state & DEPCOMPLETE) == 0) 7603 inodedep->id_state |= GOINGAWAY; 7604 WORKLIST_INSERT(&inodedep->id_inowait, &freefile->fx_list); 7605 FREE_LOCK(ump); 7606 if (ip->i_number == ino) 7607 ip->i_flag |= IN_MODIFIED; 7608 } 7609 7610 /* 7611 * Check to see if an inode has never been written to disk. If 7612 * so free the inodedep and return success, otherwise return failure. 7613 * This routine must be called with splbio interrupts blocked. 7614 * 7615 * If we still have a bitmap dependency, then the inode has never 7616 * been written to disk. Drop the dependency as it is no longer 7617 * necessary since the inode is being deallocated. We set the 7618 * ALLCOMPLETE flags since the bitmap now properly shows that the 7619 * inode is not allocated. Even if the inode is actively being 7620 * written, it has been rolled back to its zero'ed state, so we 7621 * are ensured that a zero inode is what is on the disk. For short 7622 * lived files, this change will usually result in removing all the 7623 * dependencies from the inode so that it can be freed immediately. 7624 */ 7625 static int 7626 check_inode_unwritten(inodedep) 7627 struct inodedep *inodedep; 7628 { 7629 7630 LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp)); 7631 7632 if ((inodedep->id_state & (DEPCOMPLETE | UNLINKED)) != 0 || 7633 !LIST_EMPTY(&inodedep->id_dirremhd) || 7634 !LIST_EMPTY(&inodedep->id_pendinghd) || 7635 !LIST_EMPTY(&inodedep->id_bufwait) || 7636 !LIST_EMPTY(&inodedep->id_inowait) || 7637 !TAILQ_EMPTY(&inodedep->id_inoreflst) || 7638 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 7639 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 7640 !TAILQ_EMPTY(&inodedep->id_extupdt) || 7641 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 7642 !TAILQ_EMPTY(&inodedep->id_freeblklst) || 7643 inodedep->id_mkdiradd != NULL || 7644 inodedep->id_nlinkdelta != 0) 7645 return (0); 7646 /* 7647 * Another process might be in initiate_write_inodeblock_ufs[12] 7648 * trying to allocate memory without holding "Softdep Lock". 7649 */ 7650 if ((inodedep->id_state & IOSTARTED) != 0 && 7651 inodedep->id_savedino1 == NULL) 7652 return (0); 7653 7654 if (inodedep->id_state & ONDEPLIST) 7655 LIST_REMOVE(inodedep, id_deps); 7656 inodedep->id_state &= ~ONDEPLIST; 7657 inodedep->id_state |= ALLCOMPLETE; 7658 inodedep->id_bmsafemap = NULL; 7659 if (inodedep->id_state & ONWORKLIST) 7660 WORKLIST_REMOVE(&inodedep->id_list); 7661 if (inodedep->id_savedino1 != NULL) { 7662 free(inodedep->id_savedino1, M_SAVEDINO); 7663 inodedep->id_savedino1 = NULL; 7664 } 7665 if (free_inodedep(inodedep) == 0) 7666 panic("check_inode_unwritten: busy inode"); 7667 return (1); 7668 } 7669 7670 static int 7671 check_inodedep_free(inodedep) 7672 struct inodedep *inodedep; 7673 { 7674 7675 LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp)); 7676 if ((inodedep->id_state & ALLCOMPLETE) != ALLCOMPLETE || 7677 !LIST_EMPTY(&inodedep->id_dirremhd) || 7678 !LIST_EMPTY(&inodedep->id_pendinghd) || 7679 !LIST_EMPTY(&inodedep->id_bufwait) || 7680 !LIST_EMPTY(&inodedep->id_inowait) || 7681 !TAILQ_EMPTY(&inodedep->id_inoreflst) || 7682 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 7683 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 7684 !TAILQ_EMPTY(&inodedep->id_extupdt) || 7685 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 7686 !TAILQ_EMPTY(&inodedep->id_freeblklst) || 7687 inodedep->id_mkdiradd != NULL || 7688 inodedep->id_nlinkdelta != 0 || 7689 inodedep->id_savedino1 != NULL) 7690 return (0); 7691 return (1); 7692 } 7693 7694 /* 7695 * Try to free an inodedep structure. Return 1 if it could be freed. 7696 */ 7697 static int 7698 free_inodedep(inodedep) 7699 struct inodedep *inodedep; 7700 { 7701 7702 LOCK_OWNED(VFSTOUFS(inodedep->id_list.wk_mp)); 7703 if ((inodedep->id_state & (ONWORKLIST | UNLINKED)) != 0 || 7704 !check_inodedep_free(inodedep)) 7705 return (0); 7706 if (inodedep->id_state & ONDEPLIST) 7707 LIST_REMOVE(inodedep, id_deps); 7708 LIST_REMOVE(inodedep, id_hash); 7709 WORKITEM_FREE(inodedep, D_INODEDEP); 7710 return (1); 7711 } 7712 7713 /* 7714 * Free the block referenced by a freework structure. The parent freeblks 7715 * structure is released and completed when the final cg bitmap reaches 7716 * the disk. This routine may be freeing a jnewblk which never made it to 7717 * disk in which case we do not have to wait as the operation is undone 7718 * in memory immediately. 7719 */ 7720 static void 7721 freework_freeblock(freework) 7722 struct freework *freework; 7723 { 7724 struct freeblks *freeblks; 7725 struct jnewblk *jnewblk; 7726 struct ufsmount *ump; 7727 struct workhead wkhd; 7728 struct fs *fs; 7729 int bsize; 7730 int needj; 7731 7732 ump = VFSTOUFS(freework->fw_list.wk_mp); 7733 LOCK_OWNED(ump); 7734 /* 7735 * Handle partial truncate separately. 7736 */ 7737 if (freework->fw_indir) { 7738 complete_trunc_indir(freework); 7739 return; 7740 } 7741 freeblks = freework->fw_freeblks; 7742 fs = ump->um_fs; 7743 needj = MOUNTEDSUJ(freeblks->fb_list.wk_mp) != 0; 7744 bsize = lfragtosize(fs, freework->fw_frags); 7745 LIST_INIT(&wkhd); 7746 /* 7747 * DEPCOMPLETE is cleared in indirblk_insert() if the block lives 7748 * on the indirblk hashtable and prevents premature freeing. 7749 */ 7750 freework->fw_state |= DEPCOMPLETE; 7751 /* 7752 * SUJ needs to wait for the segment referencing freed indirect 7753 * blocks to expire so that we know the checker will not confuse 7754 * a re-allocated indirect block with its old contents. 7755 */ 7756 if (needj && freework->fw_lbn <= -NDADDR) 7757 indirblk_insert(freework); 7758 /* 7759 * If we are canceling an existing jnewblk pass it to the free 7760 * routine, otherwise pass the freeblk which will ultimately 7761 * release the freeblks. If we're not journaling, we can just 7762 * free the freeblks immediately. 7763 */ 7764 jnewblk = freework->fw_jnewblk; 7765 if (jnewblk != NULL) { 7766 cancel_jnewblk(jnewblk, &wkhd); 7767 needj = 0; 7768 } else if (needj) { 7769 freework->fw_state |= DELAYEDFREE; 7770 freeblks->fb_cgwait++; 7771 WORKLIST_INSERT(&wkhd, &freework->fw_list); 7772 } 7773 FREE_LOCK(ump); 7774 freeblks_free(ump, freeblks, btodb(bsize)); 7775 CTR4(KTR_SUJ, 7776 "freework_freeblock: ino %d blkno %jd lbn %jd size %ld", 7777 freeblks->fb_inum, freework->fw_blkno, freework->fw_lbn, bsize); 7778 ffs_blkfree(ump, fs, freeblks->fb_devvp, freework->fw_blkno, bsize, 7779 freeblks->fb_inum, freeblks->fb_vtype, &wkhd); 7780 ACQUIRE_LOCK(ump); 7781 /* 7782 * The jnewblk will be discarded and the bits in the map never 7783 * made it to disk. We can immediately free the freeblk. 7784 */ 7785 if (needj == 0) 7786 handle_written_freework(freework); 7787 } 7788 7789 /* 7790 * We enqueue freework items that need processing back on the freeblks and 7791 * add the freeblks to the worklist. This makes it easier to find all work 7792 * required to flush a truncation in process_truncates(). 7793 */ 7794 static void 7795 freework_enqueue(freework) 7796 struct freework *freework; 7797 { 7798 struct freeblks *freeblks; 7799 7800 freeblks = freework->fw_freeblks; 7801 if ((freework->fw_state & INPROGRESS) == 0) 7802 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &freework->fw_list); 7803 if ((freeblks->fb_state & 7804 (ONWORKLIST | INPROGRESS | ALLCOMPLETE)) == ALLCOMPLETE && 7805 LIST_EMPTY(&freeblks->fb_jblkdephd)) 7806 add_to_worklist(&freeblks->fb_list, WK_NODELAY); 7807 } 7808 7809 /* 7810 * Start, continue, or finish the process of freeing an indirect block tree. 7811 * The free operation may be paused at any point with fw_off containing the 7812 * offset to restart from. This enables us to implement some flow control 7813 * for large truncates which may fan out and generate a huge number of 7814 * dependencies. 7815 */ 7816 static void 7817 handle_workitem_indirblk(freework) 7818 struct freework *freework; 7819 { 7820 struct freeblks *freeblks; 7821 struct ufsmount *ump; 7822 struct fs *fs; 7823 7824 freeblks = freework->fw_freeblks; 7825 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 7826 fs = ump->um_fs; 7827 if (freework->fw_state & DEPCOMPLETE) { 7828 handle_written_freework(freework); 7829 return; 7830 } 7831 if (freework->fw_off == NINDIR(fs)) { 7832 freework_freeblock(freework); 7833 return; 7834 } 7835 freework->fw_state |= INPROGRESS; 7836 FREE_LOCK(ump); 7837 indir_trunc(freework, fsbtodb(fs, freework->fw_blkno), 7838 freework->fw_lbn); 7839 ACQUIRE_LOCK(ump); 7840 } 7841 7842 /* 7843 * Called when a freework structure attached to a cg buf is written. The 7844 * ref on either the parent or the freeblks structure is released and 7845 * the freeblks is added back to the worklist if there is more work to do. 7846 */ 7847 static void 7848 handle_written_freework(freework) 7849 struct freework *freework; 7850 { 7851 struct freeblks *freeblks; 7852 struct freework *parent; 7853 7854 freeblks = freework->fw_freeblks; 7855 parent = freework->fw_parent; 7856 if (freework->fw_state & DELAYEDFREE) 7857 freeblks->fb_cgwait--; 7858 freework->fw_state |= COMPLETE; 7859 if ((freework->fw_state & ALLCOMPLETE) == ALLCOMPLETE) 7860 WORKITEM_FREE(freework, D_FREEWORK); 7861 if (parent) { 7862 if (--parent->fw_ref == 0) 7863 freework_enqueue(parent); 7864 return; 7865 } 7866 if (--freeblks->fb_ref != 0) 7867 return; 7868 if ((freeblks->fb_state & (ALLCOMPLETE | ONWORKLIST | INPROGRESS)) == 7869 ALLCOMPLETE && LIST_EMPTY(&freeblks->fb_jblkdephd)) 7870 add_to_worklist(&freeblks->fb_list, WK_NODELAY); 7871 } 7872 7873 /* 7874 * This workitem routine performs the block de-allocation. 7875 * The workitem is added to the pending list after the updated 7876 * inode block has been written to disk. As mentioned above, 7877 * checks regarding the number of blocks de-allocated (compared 7878 * to the number of blocks allocated for the file) are also 7879 * performed in this function. 7880 */ 7881 static int 7882 handle_workitem_freeblocks(freeblks, flags) 7883 struct freeblks *freeblks; 7884 int flags; 7885 { 7886 struct freework *freework; 7887 struct newblk *newblk; 7888 struct allocindir *aip; 7889 struct ufsmount *ump; 7890 struct worklist *wk; 7891 7892 KASSERT(LIST_EMPTY(&freeblks->fb_jblkdephd), 7893 ("handle_workitem_freeblocks: Journal entries not written.")); 7894 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 7895 ACQUIRE_LOCK(ump); 7896 while ((wk = LIST_FIRST(&freeblks->fb_freeworkhd)) != NULL) { 7897 WORKLIST_REMOVE(wk); 7898 switch (wk->wk_type) { 7899 case D_DIRREM: 7900 wk->wk_state |= COMPLETE; 7901 add_to_worklist(wk, 0); 7902 continue; 7903 7904 case D_ALLOCDIRECT: 7905 free_newblk(WK_NEWBLK(wk)); 7906 continue; 7907 7908 case D_ALLOCINDIR: 7909 aip = WK_ALLOCINDIR(wk); 7910 freework = NULL; 7911 if (aip->ai_state & DELAYEDFREE) { 7912 FREE_LOCK(ump); 7913 freework = newfreework(ump, freeblks, NULL, 7914 aip->ai_lbn, aip->ai_newblkno, 7915 ump->um_fs->fs_frag, 0, 0); 7916 ACQUIRE_LOCK(ump); 7917 } 7918 newblk = WK_NEWBLK(wk); 7919 if (newblk->nb_jnewblk) { 7920 freework->fw_jnewblk = newblk->nb_jnewblk; 7921 newblk->nb_jnewblk->jn_dep = &freework->fw_list; 7922 newblk->nb_jnewblk = NULL; 7923 } 7924 free_newblk(newblk); 7925 continue; 7926 7927 case D_FREEWORK: 7928 freework = WK_FREEWORK(wk); 7929 if (freework->fw_lbn <= -NDADDR) 7930 handle_workitem_indirblk(freework); 7931 else 7932 freework_freeblock(freework); 7933 continue; 7934 default: 7935 panic("handle_workitem_freeblocks: Unknown type %s", 7936 TYPENAME(wk->wk_type)); 7937 } 7938 } 7939 if (freeblks->fb_ref != 0) { 7940 freeblks->fb_state &= ~INPROGRESS; 7941 wake_worklist(&freeblks->fb_list); 7942 freeblks = NULL; 7943 } 7944 FREE_LOCK(ump); 7945 if (freeblks) 7946 return handle_complete_freeblocks(freeblks, flags); 7947 return (0); 7948 } 7949 7950 /* 7951 * Handle completion of block free via truncate. This allows fs_pending 7952 * to track the actual free block count more closely than if we only updated 7953 * it at the end. We must be careful to handle cases where the block count 7954 * on free was incorrect. 7955 */ 7956 static void 7957 freeblks_free(ump, freeblks, blocks) 7958 struct ufsmount *ump; 7959 struct freeblks *freeblks; 7960 int blocks; 7961 { 7962 struct fs *fs; 7963 ufs2_daddr_t remain; 7964 7965 UFS_LOCK(ump); 7966 remain = -freeblks->fb_chkcnt; 7967 freeblks->fb_chkcnt += blocks; 7968 if (remain > 0) { 7969 if (remain < blocks) 7970 blocks = remain; 7971 fs = ump->um_fs; 7972 fs->fs_pendingblocks -= blocks; 7973 } 7974 UFS_UNLOCK(ump); 7975 } 7976 7977 /* 7978 * Once all of the freework workitems are complete we can retire the 7979 * freeblocks dependency and any journal work awaiting completion. This 7980 * can not be called until all other dependencies are stable on disk. 7981 */ 7982 static int 7983 handle_complete_freeblocks(freeblks, flags) 7984 struct freeblks *freeblks; 7985 int flags; 7986 { 7987 struct inodedep *inodedep; 7988 struct inode *ip; 7989 struct vnode *vp; 7990 struct fs *fs; 7991 struct ufsmount *ump; 7992 ufs2_daddr_t spare; 7993 7994 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 7995 fs = ump->um_fs; 7996 flags = LK_EXCLUSIVE | flags; 7997 spare = freeblks->fb_chkcnt; 7998 7999 /* 8000 * If we did not release the expected number of blocks we may have 8001 * to adjust the inode block count here. Only do so if it wasn't 8002 * a truncation to zero and the modrev still matches. 8003 */ 8004 if (spare && freeblks->fb_len != 0) { 8005 if (ffs_vgetf(freeblks->fb_list.wk_mp, freeblks->fb_inum, 8006 flags, &vp, FFSV_FORCEINSMQ) != 0) 8007 return (EBUSY); 8008 ip = VTOI(vp); 8009 if (DIP(ip, i_modrev) == freeblks->fb_modrev) { 8010 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - spare); 8011 ip->i_flag |= IN_CHANGE; 8012 /* 8013 * We must wait so this happens before the 8014 * journal is reclaimed. 8015 */ 8016 ffs_update(vp, 1); 8017 } 8018 vput(vp); 8019 } 8020 if (spare < 0) { 8021 UFS_LOCK(ump); 8022 fs->fs_pendingblocks += spare; 8023 UFS_UNLOCK(ump); 8024 } 8025 #ifdef QUOTA 8026 /* Handle spare. */ 8027 if (spare) 8028 quotaadj(freeblks->fb_quota, ump, -spare); 8029 quotarele(freeblks->fb_quota); 8030 #endif 8031 ACQUIRE_LOCK(ump); 8032 if (freeblks->fb_state & ONDEPLIST) { 8033 inodedep_lookup(freeblks->fb_list.wk_mp, freeblks->fb_inum, 8034 0, &inodedep); 8035 TAILQ_REMOVE(&inodedep->id_freeblklst, freeblks, fb_next); 8036 freeblks->fb_state &= ~ONDEPLIST; 8037 if (TAILQ_EMPTY(&inodedep->id_freeblklst)) 8038 free_inodedep(inodedep); 8039 } 8040 /* 8041 * All of the freeblock deps must be complete prior to this call 8042 * so it's now safe to complete earlier outstanding journal entries. 8043 */ 8044 handle_jwork(&freeblks->fb_jwork); 8045 WORKITEM_FREE(freeblks, D_FREEBLKS); 8046 FREE_LOCK(ump); 8047 return (0); 8048 } 8049 8050 /* 8051 * Release blocks associated with the freeblks and stored in the indirect 8052 * block dbn. If level is greater than SINGLE, the block is an indirect block 8053 * and recursive calls to indirtrunc must be used to cleanse other indirect 8054 * blocks. 8055 * 8056 * This handles partial and complete truncation of blocks. Partial is noted 8057 * with goingaway == 0. In this case the freework is completed after the 8058 * zero'd indirects are written to disk. For full truncation the freework 8059 * is completed after the block is freed. 8060 */ 8061 static void 8062 indir_trunc(freework, dbn, lbn) 8063 struct freework *freework; 8064 ufs2_daddr_t dbn; 8065 ufs_lbn_t lbn; 8066 { 8067 struct freework *nfreework; 8068 struct workhead wkhd; 8069 struct freeblks *freeblks; 8070 struct buf *bp; 8071 struct fs *fs; 8072 struct indirdep *indirdep; 8073 struct ufsmount *ump; 8074 ufs1_daddr_t *bap1; 8075 ufs2_daddr_t nb, nnb, *bap2; 8076 ufs_lbn_t lbnadd, nlbn; 8077 int i, nblocks, ufs1fmt; 8078 int freedblocks; 8079 int goingaway; 8080 int freedeps; 8081 int needj; 8082 int level; 8083 int cnt; 8084 8085 freeblks = freework->fw_freeblks; 8086 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 8087 fs = ump->um_fs; 8088 /* 8089 * Get buffer of block pointers to be freed. There are three cases: 8090 * 8091 * 1) Partial truncate caches the indirdep pointer in the freework 8092 * which provides us a back copy to the save bp which holds the 8093 * pointers we want to clear. When this completes the zero 8094 * pointers are written to the real copy. 8095 * 2) The indirect is being completely truncated, cancel_indirdep() 8096 * eliminated the real copy and placed the indirdep on the saved 8097 * copy. The indirdep and buf are discarded when this completes. 8098 * 3) The indirect was not in memory, we read a copy off of the disk 8099 * using the devvp and drop and invalidate the buffer when we're 8100 * done. 8101 */ 8102 goingaway = 1; 8103 indirdep = NULL; 8104 if (freework->fw_indir != NULL) { 8105 goingaway = 0; 8106 indirdep = freework->fw_indir; 8107 bp = indirdep->ir_savebp; 8108 if (bp == NULL || bp->b_blkno != dbn) 8109 panic("indir_trunc: Bad saved buf %p blkno %jd", 8110 bp, (intmax_t)dbn); 8111 } else if ((bp = incore(&freeblks->fb_devvp->v_bufobj, dbn)) != NULL) { 8112 /* 8113 * The lock prevents the buf dep list from changing and 8114 * indirects on devvp should only ever have one dependency. 8115 */ 8116 indirdep = WK_INDIRDEP(LIST_FIRST(&bp->b_dep)); 8117 if (indirdep == NULL || (indirdep->ir_state & GOINGAWAY) == 0) 8118 panic("indir_trunc: Bad indirdep %p from buf %p", 8119 indirdep, bp); 8120 } else if (bread(freeblks->fb_devvp, dbn, (int)fs->fs_bsize, 8121 NOCRED, &bp) != 0) { 8122 brelse(bp); 8123 return; 8124 } 8125 ACQUIRE_LOCK(ump); 8126 /* Protects against a race with complete_trunc_indir(). */ 8127 freework->fw_state &= ~INPROGRESS; 8128 /* 8129 * If we have an indirdep we need to enforce the truncation order 8130 * and discard it when it is complete. 8131 */ 8132 if (indirdep) { 8133 if (freework != TAILQ_FIRST(&indirdep->ir_trunc) && 8134 !TAILQ_EMPTY(&indirdep->ir_trunc)) { 8135 /* 8136 * Add the complete truncate to the list on the 8137 * indirdep to enforce in-order processing. 8138 */ 8139 if (freework->fw_indir == NULL) 8140 TAILQ_INSERT_TAIL(&indirdep->ir_trunc, 8141 freework, fw_next); 8142 FREE_LOCK(ump); 8143 return; 8144 } 8145 /* 8146 * If we're goingaway, free the indirdep. Otherwise it will 8147 * linger until the write completes. 8148 */ 8149 if (goingaway) 8150 free_indirdep(indirdep); 8151 } 8152 FREE_LOCK(ump); 8153 /* Initialize pointers depending on block size. */ 8154 if (ump->um_fstype == UFS1) { 8155 bap1 = (ufs1_daddr_t *)bp->b_data; 8156 nb = bap1[freework->fw_off]; 8157 ufs1fmt = 1; 8158 bap2 = NULL; 8159 } else { 8160 bap2 = (ufs2_daddr_t *)bp->b_data; 8161 nb = bap2[freework->fw_off]; 8162 ufs1fmt = 0; 8163 bap1 = NULL; 8164 } 8165 level = lbn_level(lbn); 8166 needj = MOUNTEDSUJ(UFSTOVFS(ump)) != 0; 8167 lbnadd = lbn_offset(fs, level); 8168 nblocks = btodb(fs->fs_bsize); 8169 nfreework = freework; 8170 freedeps = 0; 8171 cnt = 0; 8172 /* 8173 * Reclaim blocks. Traverses into nested indirect levels and 8174 * arranges for the current level to be freed when subordinates 8175 * are free when journaling. 8176 */ 8177 for (i = freework->fw_off; i < NINDIR(fs); i++, nb = nnb) { 8178 if (i != NINDIR(fs) - 1) { 8179 if (ufs1fmt) 8180 nnb = bap1[i+1]; 8181 else 8182 nnb = bap2[i+1]; 8183 } else 8184 nnb = 0; 8185 if (nb == 0) 8186 continue; 8187 cnt++; 8188 if (level != 0) { 8189 nlbn = (lbn + 1) - (i * lbnadd); 8190 if (needj != 0) { 8191 nfreework = newfreework(ump, freeblks, freework, 8192 nlbn, nb, fs->fs_frag, 0, 0); 8193 freedeps++; 8194 } 8195 indir_trunc(nfreework, fsbtodb(fs, nb), nlbn); 8196 } else { 8197 struct freedep *freedep; 8198 8199 /* 8200 * Attempt to aggregate freedep dependencies for 8201 * all blocks being released to the same CG. 8202 */ 8203 LIST_INIT(&wkhd); 8204 if (needj != 0 && 8205 (nnb == 0 || (dtog(fs, nb) != dtog(fs, nnb)))) { 8206 freedep = newfreedep(freework); 8207 WORKLIST_INSERT_UNLOCKED(&wkhd, 8208 &freedep->fd_list); 8209 freedeps++; 8210 } 8211 CTR3(KTR_SUJ, 8212 "indir_trunc: ino %d blkno %jd size %ld", 8213 freeblks->fb_inum, nb, fs->fs_bsize); 8214 ffs_blkfree(ump, fs, freeblks->fb_devvp, nb, 8215 fs->fs_bsize, freeblks->fb_inum, 8216 freeblks->fb_vtype, &wkhd); 8217 } 8218 } 8219 if (goingaway) { 8220 bp->b_flags |= B_INVAL | B_NOCACHE; 8221 brelse(bp); 8222 } 8223 freedblocks = 0; 8224 if (level == 0) 8225 freedblocks = (nblocks * cnt); 8226 if (needj == 0) 8227 freedblocks += nblocks; 8228 freeblks_free(ump, freeblks, freedblocks); 8229 /* 8230 * If we are journaling set up the ref counts and offset so this 8231 * indirect can be completed when its children are free. 8232 */ 8233 if (needj) { 8234 ACQUIRE_LOCK(ump); 8235 freework->fw_off = i; 8236 freework->fw_ref += freedeps; 8237 freework->fw_ref -= NINDIR(fs) + 1; 8238 if (level == 0) 8239 freeblks->fb_cgwait += freedeps; 8240 if (freework->fw_ref == 0) 8241 freework_freeblock(freework); 8242 FREE_LOCK(ump); 8243 return; 8244 } 8245 /* 8246 * If we're not journaling we can free the indirect now. 8247 */ 8248 dbn = dbtofsb(fs, dbn); 8249 CTR3(KTR_SUJ, 8250 "indir_trunc 2: ino %d blkno %jd size %ld", 8251 freeblks->fb_inum, dbn, fs->fs_bsize); 8252 ffs_blkfree(ump, fs, freeblks->fb_devvp, dbn, fs->fs_bsize, 8253 freeblks->fb_inum, freeblks->fb_vtype, NULL); 8254 /* Non SUJ softdep does single-threaded truncations. */ 8255 if (freework->fw_blkno == dbn) { 8256 freework->fw_state |= ALLCOMPLETE; 8257 ACQUIRE_LOCK(ump); 8258 handle_written_freework(freework); 8259 FREE_LOCK(ump); 8260 } 8261 return; 8262 } 8263 8264 /* 8265 * Cancel an allocindir when it is removed via truncation. When bp is not 8266 * NULL the indirect never appeared on disk and is scheduled to be freed 8267 * independently of the indir so we can more easily track journal work. 8268 */ 8269 static void 8270 cancel_allocindir(aip, bp, freeblks, trunc) 8271 struct allocindir *aip; 8272 struct buf *bp; 8273 struct freeblks *freeblks; 8274 int trunc; 8275 { 8276 struct indirdep *indirdep; 8277 struct freefrag *freefrag; 8278 struct newblk *newblk; 8279 8280 newblk = (struct newblk *)aip; 8281 LIST_REMOVE(aip, ai_next); 8282 /* 8283 * We must eliminate the pointer in bp if it must be freed on its 8284 * own due to partial truncate or pending journal work. 8285 */ 8286 if (bp && (trunc || newblk->nb_jnewblk)) { 8287 /* 8288 * Clear the pointer and mark the aip to be freed 8289 * directly if it never existed on disk. 8290 */ 8291 aip->ai_state |= DELAYEDFREE; 8292 indirdep = aip->ai_indirdep; 8293 if (indirdep->ir_state & UFS1FMT) 8294 ((ufs1_daddr_t *)bp->b_data)[aip->ai_offset] = 0; 8295 else 8296 ((ufs2_daddr_t *)bp->b_data)[aip->ai_offset] = 0; 8297 } 8298 /* 8299 * When truncating the previous pointer will be freed via 8300 * savedbp. Eliminate the freefrag which would dup free. 8301 */ 8302 if (trunc && (freefrag = newblk->nb_freefrag) != NULL) { 8303 newblk->nb_freefrag = NULL; 8304 if (freefrag->ff_jdep) 8305 cancel_jfreefrag( 8306 WK_JFREEFRAG(freefrag->ff_jdep)); 8307 jwork_move(&freeblks->fb_jwork, &freefrag->ff_jwork); 8308 WORKITEM_FREE(freefrag, D_FREEFRAG); 8309 } 8310 /* 8311 * If the journal hasn't been written the jnewblk must be passed 8312 * to the call to ffs_blkfree that reclaims the space. We accomplish 8313 * this by leaving the journal dependency on the newblk to be freed 8314 * when a freework is created in handle_workitem_freeblocks(). 8315 */ 8316 cancel_newblk(newblk, NULL, &freeblks->fb_jwork); 8317 WORKLIST_INSERT(&freeblks->fb_freeworkhd, &newblk->nb_list); 8318 } 8319 8320 /* 8321 * Create the mkdir dependencies for . and .. in a new directory. Link them 8322 * in to a newdirblk so any subsequent additions are tracked properly. The 8323 * caller is responsible for adding the mkdir1 dependency to the journal 8324 * and updating id_mkdiradd. This function returns with the per-filesystem 8325 * lock held. 8326 */ 8327 static struct mkdir * 8328 setup_newdir(dap, newinum, dinum, newdirbp, mkdirp) 8329 struct diradd *dap; 8330 ino_t newinum; 8331 ino_t dinum; 8332 struct buf *newdirbp; 8333 struct mkdir **mkdirp; 8334 { 8335 struct newblk *newblk; 8336 struct pagedep *pagedep; 8337 struct inodedep *inodedep; 8338 struct newdirblk *newdirblk; 8339 struct mkdir *mkdir1, *mkdir2; 8340 struct worklist *wk; 8341 struct jaddref *jaddref; 8342 struct ufsmount *ump; 8343 struct mount *mp; 8344 8345 mp = dap->da_list.wk_mp; 8346 ump = VFSTOUFS(mp); 8347 newdirblk = malloc(sizeof(struct newdirblk), M_NEWDIRBLK, 8348 M_SOFTDEP_FLAGS); 8349 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 8350 LIST_INIT(&newdirblk->db_mkdir); 8351 mkdir1 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 8352 workitem_alloc(&mkdir1->md_list, D_MKDIR, mp); 8353 mkdir1->md_state = ATTACHED | MKDIR_BODY; 8354 mkdir1->md_diradd = dap; 8355 mkdir1->md_jaddref = NULL; 8356 mkdir2 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 8357 workitem_alloc(&mkdir2->md_list, D_MKDIR, mp); 8358 mkdir2->md_state = ATTACHED | MKDIR_PARENT; 8359 mkdir2->md_diradd = dap; 8360 mkdir2->md_jaddref = NULL; 8361 if (MOUNTEDSUJ(mp) == 0) { 8362 mkdir1->md_state |= DEPCOMPLETE; 8363 mkdir2->md_state |= DEPCOMPLETE; 8364 } 8365 /* 8366 * Dependency on "." and ".." being written to disk. 8367 */ 8368 mkdir1->md_buf = newdirbp; 8369 ACQUIRE_LOCK(VFSTOUFS(mp)); 8370 LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir1, md_mkdirs); 8371 /* 8372 * We must link the pagedep, allocdirect, and newdirblk for 8373 * the initial file page so the pointer to the new directory 8374 * is not written until the directory contents are live and 8375 * any subsequent additions are not marked live until the 8376 * block is reachable via the inode. 8377 */ 8378 if (pagedep_lookup(mp, newdirbp, newinum, 0, 0, &pagedep) == 0) 8379 panic("setup_newdir: lost pagedep"); 8380 LIST_FOREACH(wk, &newdirbp->b_dep, wk_list) 8381 if (wk->wk_type == D_ALLOCDIRECT) 8382 break; 8383 if (wk == NULL) 8384 panic("setup_newdir: lost allocdirect"); 8385 if (pagedep->pd_state & NEWBLOCK) 8386 panic("setup_newdir: NEWBLOCK already set"); 8387 newblk = WK_NEWBLK(wk); 8388 pagedep->pd_state |= NEWBLOCK; 8389 pagedep->pd_newdirblk = newdirblk; 8390 newdirblk->db_pagedep = pagedep; 8391 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 8392 WORKLIST_INSERT(&newdirblk->db_mkdir, &mkdir1->md_list); 8393 /* 8394 * Look up the inodedep for the parent directory so that we 8395 * can link mkdir2 into the pending dotdot jaddref or 8396 * the inode write if there is none. If the inode is 8397 * ALLCOMPLETE and no jaddref is present all dependencies have 8398 * been satisfied and mkdir2 can be freed. 8399 */ 8400 inodedep_lookup(mp, dinum, 0, &inodedep); 8401 if (MOUNTEDSUJ(mp)) { 8402 if (inodedep == NULL) 8403 panic("setup_newdir: Lost parent."); 8404 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 8405 inoreflst); 8406 KASSERT(jaddref != NULL && jaddref->ja_parent == newinum && 8407 (jaddref->ja_state & MKDIR_PARENT), 8408 ("setup_newdir: bad dotdot jaddref %p", jaddref)); 8409 LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir2, md_mkdirs); 8410 mkdir2->md_jaddref = jaddref; 8411 jaddref->ja_mkdir = mkdir2; 8412 } else if (inodedep == NULL || 8413 (inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 8414 dap->da_state &= ~MKDIR_PARENT; 8415 WORKITEM_FREE(mkdir2, D_MKDIR); 8416 mkdir2 = NULL; 8417 } else { 8418 LIST_INSERT_HEAD(&ump->softdep_mkdirlisthd, mkdir2, md_mkdirs); 8419 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir2->md_list); 8420 } 8421 *mkdirp = mkdir2; 8422 8423 return (mkdir1); 8424 } 8425 8426 /* 8427 * Directory entry addition dependencies. 8428 * 8429 * When adding a new directory entry, the inode (with its incremented link 8430 * count) must be written to disk before the directory entry's pointer to it. 8431 * Also, if the inode is newly allocated, the corresponding freemap must be 8432 * updated (on disk) before the directory entry's pointer. These requirements 8433 * are met via undo/redo on the directory entry's pointer, which consists 8434 * simply of the inode number. 8435 * 8436 * As directory entries are added and deleted, the free space within a 8437 * directory block can become fragmented. The ufs filesystem will compact 8438 * a fragmented directory block to make space for a new entry. When this 8439 * occurs, the offsets of previously added entries change. Any "diradd" 8440 * dependency structures corresponding to these entries must be updated with 8441 * the new offsets. 8442 */ 8443 8444 /* 8445 * This routine is called after the in-memory inode's link 8446 * count has been incremented, but before the directory entry's 8447 * pointer to the inode has been set. 8448 */ 8449 int 8450 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 8451 struct buf *bp; /* buffer containing directory block */ 8452 struct inode *dp; /* inode for directory */ 8453 off_t diroffset; /* offset of new entry in directory */ 8454 ino_t newinum; /* inode referenced by new directory entry */ 8455 struct buf *newdirbp; /* non-NULL => contents of new mkdir */ 8456 int isnewblk; /* entry is in a newly allocated block */ 8457 { 8458 int offset; /* offset of new entry within directory block */ 8459 ufs_lbn_t lbn; /* block in directory containing new entry */ 8460 struct fs *fs; 8461 struct diradd *dap; 8462 struct newblk *newblk; 8463 struct pagedep *pagedep; 8464 struct inodedep *inodedep; 8465 struct newdirblk *newdirblk; 8466 struct mkdir *mkdir1, *mkdir2; 8467 struct jaddref *jaddref; 8468 struct ufsmount *ump; 8469 struct mount *mp; 8470 int isindir; 8471 8472 mp = ITOVFS(dp); 8473 ump = VFSTOUFS(mp); 8474 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 8475 ("softdep_setup_directory_add called on non-softdep filesystem")); 8476 /* 8477 * Whiteouts have no dependencies. 8478 */ 8479 if (newinum == WINO) { 8480 if (newdirbp != NULL) 8481 bdwrite(newdirbp); 8482 return (0); 8483 } 8484 jaddref = NULL; 8485 mkdir1 = mkdir2 = NULL; 8486 fs = ump->um_fs; 8487 lbn = lblkno(fs, diroffset); 8488 offset = blkoff(fs, diroffset); 8489 dap = malloc(sizeof(struct diradd), M_DIRADD, 8490 M_SOFTDEP_FLAGS|M_ZERO); 8491 workitem_alloc(&dap->da_list, D_DIRADD, mp); 8492 dap->da_offset = offset; 8493 dap->da_newinum = newinum; 8494 dap->da_state = ATTACHED; 8495 LIST_INIT(&dap->da_jwork); 8496 isindir = bp->b_lblkno >= NDADDR; 8497 newdirblk = NULL; 8498 if (isnewblk && 8499 (isindir ? blkoff(fs, diroffset) : fragoff(fs, diroffset)) == 0) { 8500 newdirblk = malloc(sizeof(struct newdirblk), 8501 M_NEWDIRBLK, M_SOFTDEP_FLAGS); 8502 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 8503 LIST_INIT(&newdirblk->db_mkdir); 8504 } 8505 /* 8506 * If we're creating a new directory setup the dependencies and set 8507 * the dap state to wait for them. Otherwise it's COMPLETE and 8508 * we can move on. 8509 */ 8510 if (newdirbp == NULL) { 8511 dap->da_state |= DEPCOMPLETE; 8512 ACQUIRE_LOCK(ump); 8513 } else { 8514 dap->da_state |= MKDIR_BODY | MKDIR_PARENT; 8515 mkdir1 = setup_newdir(dap, newinum, dp->i_number, newdirbp, 8516 &mkdir2); 8517 } 8518 /* 8519 * Link into parent directory pagedep to await its being written. 8520 */ 8521 pagedep_lookup(mp, bp, dp->i_number, lbn, DEPALLOC, &pagedep); 8522 #ifdef DEBUG 8523 if (diradd_lookup(pagedep, offset) != NULL) 8524 panic("softdep_setup_directory_add: %p already at off %d\n", 8525 diradd_lookup(pagedep, offset), offset); 8526 #endif 8527 dap->da_pagedep = pagedep; 8528 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], dap, 8529 da_pdlist); 8530 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 8531 /* 8532 * If we're journaling, link the diradd into the jaddref so it 8533 * may be completed after the journal entry is written. Otherwise, 8534 * link the diradd into its inodedep. If the inode is not yet 8535 * written place it on the bufwait list, otherwise do the post-inode 8536 * write processing to put it on the id_pendinghd list. 8537 */ 8538 if (MOUNTEDSUJ(mp)) { 8539 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 8540 inoreflst); 8541 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 8542 ("softdep_setup_directory_add: bad jaddref %p", jaddref)); 8543 jaddref->ja_diroff = diroffset; 8544 jaddref->ja_diradd = dap; 8545 add_to_journal(&jaddref->ja_list); 8546 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) 8547 diradd_inode_written(dap, inodedep); 8548 else 8549 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 8550 /* 8551 * Add the journal entries for . and .. links now that the primary 8552 * link is written. 8553 */ 8554 if (mkdir1 != NULL && MOUNTEDSUJ(mp)) { 8555 jaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref, 8556 inoreflst, if_deps); 8557 KASSERT(jaddref != NULL && 8558 jaddref->ja_ino == jaddref->ja_parent && 8559 (jaddref->ja_state & MKDIR_BODY), 8560 ("softdep_setup_directory_add: bad dot jaddref %p", 8561 jaddref)); 8562 mkdir1->md_jaddref = jaddref; 8563 jaddref->ja_mkdir = mkdir1; 8564 /* 8565 * It is important that the dotdot journal entry 8566 * is added prior to the dot entry since dot writes 8567 * both the dot and dotdot links. These both must 8568 * be added after the primary link for the journal 8569 * to remain consistent. 8570 */ 8571 add_to_journal(&mkdir2->md_jaddref->ja_list); 8572 add_to_journal(&jaddref->ja_list); 8573 } 8574 /* 8575 * If we are adding a new directory remember this diradd so that if 8576 * we rename it we can keep the dot and dotdot dependencies. If 8577 * we are adding a new name for an inode that has a mkdiradd we 8578 * must be in rename and we have to move the dot and dotdot 8579 * dependencies to this new name. The old name is being orphaned 8580 * soon. 8581 */ 8582 if (mkdir1 != NULL) { 8583 if (inodedep->id_mkdiradd != NULL) 8584 panic("softdep_setup_directory_add: Existing mkdir"); 8585 inodedep->id_mkdiradd = dap; 8586 } else if (inodedep->id_mkdiradd) 8587 merge_diradd(inodedep, dap); 8588 if (newdirblk != NULL) { 8589 /* 8590 * There is nothing to do if we are already tracking 8591 * this block. 8592 */ 8593 if ((pagedep->pd_state & NEWBLOCK) != 0) { 8594 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 8595 FREE_LOCK(ump); 8596 return (0); 8597 } 8598 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk) 8599 == 0) 8600 panic("softdep_setup_directory_add: lost entry"); 8601 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 8602 pagedep->pd_state |= NEWBLOCK; 8603 pagedep->pd_newdirblk = newdirblk; 8604 newdirblk->db_pagedep = pagedep; 8605 FREE_LOCK(ump); 8606 /* 8607 * If we extended into an indirect signal direnter to sync. 8608 */ 8609 if (isindir) 8610 return (1); 8611 return (0); 8612 } 8613 FREE_LOCK(ump); 8614 return (0); 8615 } 8616 8617 /* 8618 * This procedure is called to change the offset of a directory 8619 * entry when compacting a directory block which must be owned 8620 * exclusively by the caller. Note that the actual entry movement 8621 * must be done in this procedure to ensure that no I/O completions 8622 * occur while the move is in progress. 8623 */ 8624 void 8625 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 8626 struct buf *bp; /* Buffer holding directory block. */ 8627 struct inode *dp; /* inode for directory */ 8628 caddr_t base; /* address of dp->i_offset */ 8629 caddr_t oldloc; /* address of old directory location */ 8630 caddr_t newloc; /* address of new directory location */ 8631 int entrysize; /* size of directory entry */ 8632 { 8633 int offset, oldoffset, newoffset; 8634 struct pagedep *pagedep; 8635 struct jmvref *jmvref; 8636 struct diradd *dap; 8637 struct direct *de; 8638 struct mount *mp; 8639 struct ufsmount *ump; 8640 ufs_lbn_t lbn; 8641 int flags; 8642 8643 mp = ITOVFS(dp); 8644 ump = VFSTOUFS(mp); 8645 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 8646 ("softdep_change_directoryentry_offset called on " 8647 "non-softdep filesystem")); 8648 de = (struct direct *)oldloc; 8649 jmvref = NULL; 8650 flags = 0; 8651 /* 8652 * Moves are always journaled as it would be too complex to 8653 * determine if any affected adds or removes are present in the 8654 * journal. 8655 */ 8656 if (MOUNTEDSUJ(mp)) { 8657 flags = DEPALLOC; 8658 jmvref = newjmvref(dp, de->d_ino, 8659 dp->i_offset + (oldloc - base), 8660 dp->i_offset + (newloc - base)); 8661 } 8662 lbn = lblkno(ump->um_fs, dp->i_offset); 8663 offset = blkoff(ump->um_fs, dp->i_offset); 8664 oldoffset = offset + (oldloc - base); 8665 newoffset = offset + (newloc - base); 8666 ACQUIRE_LOCK(ump); 8667 if (pagedep_lookup(mp, bp, dp->i_number, lbn, flags, &pagedep) == 0) 8668 goto done; 8669 dap = diradd_lookup(pagedep, oldoffset); 8670 if (dap) { 8671 dap->da_offset = newoffset; 8672 newoffset = DIRADDHASH(newoffset); 8673 oldoffset = DIRADDHASH(oldoffset); 8674 if ((dap->da_state & ALLCOMPLETE) != ALLCOMPLETE && 8675 newoffset != oldoffset) { 8676 LIST_REMOVE(dap, da_pdlist); 8677 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[newoffset], 8678 dap, da_pdlist); 8679 } 8680 } 8681 done: 8682 if (jmvref) { 8683 jmvref->jm_pagedep = pagedep; 8684 LIST_INSERT_HEAD(&pagedep->pd_jmvrefhd, jmvref, jm_deps); 8685 add_to_journal(&jmvref->jm_list); 8686 } 8687 bcopy(oldloc, newloc, entrysize); 8688 FREE_LOCK(ump); 8689 } 8690 8691 /* 8692 * Move the mkdir dependencies and journal work from one diradd to another 8693 * when renaming a directory. The new name must depend on the mkdir deps 8694 * completing as the old name did. Directories can only have one valid link 8695 * at a time so one must be canonical. 8696 */ 8697 static void 8698 merge_diradd(inodedep, newdap) 8699 struct inodedep *inodedep; 8700 struct diradd *newdap; 8701 { 8702 struct diradd *olddap; 8703 struct mkdir *mkdir, *nextmd; 8704 struct ufsmount *ump; 8705 short state; 8706 8707 olddap = inodedep->id_mkdiradd; 8708 inodedep->id_mkdiradd = newdap; 8709 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 8710 newdap->da_state &= ~DEPCOMPLETE; 8711 ump = VFSTOUFS(inodedep->id_list.wk_mp); 8712 for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir; 8713 mkdir = nextmd) { 8714 nextmd = LIST_NEXT(mkdir, md_mkdirs); 8715 if (mkdir->md_diradd != olddap) 8716 continue; 8717 mkdir->md_diradd = newdap; 8718 state = mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY); 8719 newdap->da_state |= state; 8720 olddap->da_state &= ~state; 8721 if ((olddap->da_state & 8722 (MKDIR_PARENT | MKDIR_BODY)) == 0) 8723 break; 8724 } 8725 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 8726 panic("merge_diradd: unfound ref"); 8727 } 8728 /* 8729 * Any mkdir related journal items are not safe to be freed until 8730 * the new name is stable. 8731 */ 8732 jwork_move(&newdap->da_jwork, &olddap->da_jwork); 8733 olddap->da_state |= DEPCOMPLETE; 8734 complete_diradd(olddap); 8735 } 8736 8737 /* 8738 * Move the diradd to the pending list when all diradd dependencies are 8739 * complete. 8740 */ 8741 static void 8742 complete_diradd(dap) 8743 struct diradd *dap; 8744 { 8745 struct pagedep *pagedep; 8746 8747 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 8748 if (dap->da_state & DIRCHG) 8749 pagedep = dap->da_previous->dm_pagedep; 8750 else 8751 pagedep = dap->da_pagedep; 8752 LIST_REMOVE(dap, da_pdlist); 8753 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 8754 } 8755 } 8756 8757 /* 8758 * Cancel a diradd when a dirrem overlaps with it. We must cancel the journal 8759 * add entries and conditonally journal the remove. 8760 */ 8761 static void 8762 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref) 8763 struct diradd *dap; 8764 struct dirrem *dirrem; 8765 struct jremref *jremref; 8766 struct jremref *dotremref; 8767 struct jremref *dotdotremref; 8768 { 8769 struct inodedep *inodedep; 8770 struct jaddref *jaddref; 8771 struct inoref *inoref; 8772 struct ufsmount *ump; 8773 struct mkdir *mkdir; 8774 8775 /* 8776 * If no remove references were allocated we're on a non-journaled 8777 * filesystem and can skip the cancel step. 8778 */ 8779 if (jremref == NULL) { 8780 free_diradd(dap, NULL); 8781 return; 8782 } 8783 /* 8784 * Cancel the primary name an free it if it does not require 8785 * journaling. 8786 */ 8787 if (inodedep_lookup(dap->da_list.wk_mp, dap->da_newinum, 8788 0, &inodedep) != 0) { 8789 /* Abort the addref that reference this diradd. */ 8790 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 8791 if (inoref->if_list.wk_type != D_JADDREF) 8792 continue; 8793 jaddref = (struct jaddref *)inoref; 8794 if (jaddref->ja_diradd != dap) 8795 continue; 8796 if (cancel_jaddref(jaddref, inodedep, 8797 &dirrem->dm_jwork) == 0) { 8798 free_jremref(jremref); 8799 jremref = NULL; 8800 } 8801 break; 8802 } 8803 } 8804 /* 8805 * Cancel subordinate names and free them if they do not require 8806 * journaling. 8807 */ 8808 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 8809 ump = VFSTOUFS(dap->da_list.wk_mp); 8810 LIST_FOREACH(mkdir, &ump->softdep_mkdirlisthd, md_mkdirs) { 8811 if (mkdir->md_diradd != dap) 8812 continue; 8813 if ((jaddref = mkdir->md_jaddref) == NULL) 8814 continue; 8815 mkdir->md_jaddref = NULL; 8816 if (mkdir->md_state & MKDIR_PARENT) { 8817 if (cancel_jaddref(jaddref, NULL, 8818 &dirrem->dm_jwork) == 0) { 8819 free_jremref(dotdotremref); 8820 dotdotremref = NULL; 8821 } 8822 } else { 8823 if (cancel_jaddref(jaddref, inodedep, 8824 &dirrem->dm_jwork) == 0) { 8825 free_jremref(dotremref); 8826 dotremref = NULL; 8827 } 8828 } 8829 } 8830 } 8831 8832 if (jremref) 8833 journal_jremref(dirrem, jremref, inodedep); 8834 if (dotremref) 8835 journal_jremref(dirrem, dotremref, inodedep); 8836 if (dotdotremref) 8837 journal_jremref(dirrem, dotdotremref, NULL); 8838 jwork_move(&dirrem->dm_jwork, &dap->da_jwork); 8839 free_diradd(dap, &dirrem->dm_jwork); 8840 } 8841 8842 /* 8843 * Free a diradd dependency structure. This routine must be called 8844 * with splbio interrupts blocked. 8845 */ 8846 static void 8847 free_diradd(dap, wkhd) 8848 struct diradd *dap; 8849 struct workhead *wkhd; 8850 { 8851 struct dirrem *dirrem; 8852 struct pagedep *pagedep; 8853 struct inodedep *inodedep; 8854 struct mkdir *mkdir, *nextmd; 8855 struct ufsmount *ump; 8856 8857 ump = VFSTOUFS(dap->da_list.wk_mp); 8858 LOCK_OWNED(ump); 8859 LIST_REMOVE(dap, da_pdlist); 8860 if (dap->da_state & ONWORKLIST) 8861 WORKLIST_REMOVE(&dap->da_list); 8862 if ((dap->da_state & DIRCHG) == 0) { 8863 pagedep = dap->da_pagedep; 8864 } else { 8865 dirrem = dap->da_previous; 8866 pagedep = dirrem->dm_pagedep; 8867 dirrem->dm_dirinum = pagedep->pd_ino; 8868 dirrem->dm_state |= COMPLETE; 8869 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 8870 add_to_worklist(&dirrem->dm_list, 0); 8871 } 8872 if (inodedep_lookup(pagedep->pd_list.wk_mp, dap->da_newinum, 8873 0, &inodedep) != 0) 8874 if (inodedep->id_mkdiradd == dap) 8875 inodedep->id_mkdiradd = NULL; 8876 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 8877 for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir; 8878 mkdir = nextmd) { 8879 nextmd = LIST_NEXT(mkdir, md_mkdirs); 8880 if (mkdir->md_diradd != dap) 8881 continue; 8882 dap->da_state &= 8883 ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 8884 LIST_REMOVE(mkdir, md_mkdirs); 8885 if (mkdir->md_state & ONWORKLIST) 8886 WORKLIST_REMOVE(&mkdir->md_list); 8887 if (mkdir->md_jaddref != NULL) 8888 panic("free_diradd: Unexpected jaddref"); 8889 WORKITEM_FREE(mkdir, D_MKDIR); 8890 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) 8891 break; 8892 } 8893 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 8894 panic("free_diradd: unfound ref"); 8895 } 8896 if (inodedep) 8897 free_inodedep(inodedep); 8898 /* 8899 * Free any journal segments waiting for the directory write. 8900 */ 8901 handle_jwork(&dap->da_jwork); 8902 WORKITEM_FREE(dap, D_DIRADD); 8903 } 8904 8905 /* 8906 * Directory entry removal dependencies. 8907 * 8908 * When removing a directory entry, the entry's inode pointer must be 8909 * zero'ed on disk before the corresponding inode's link count is decremented 8910 * (possibly freeing the inode for re-use). This dependency is handled by 8911 * updating the directory entry but delaying the inode count reduction until 8912 * after the directory block has been written to disk. After this point, the 8913 * inode count can be decremented whenever it is convenient. 8914 */ 8915 8916 /* 8917 * This routine should be called immediately after removing 8918 * a directory entry. The inode's link count should not be 8919 * decremented by the calling procedure -- the soft updates 8920 * code will do this task when it is safe. 8921 */ 8922 void 8923 softdep_setup_remove(bp, dp, ip, isrmdir) 8924 struct buf *bp; /* buffer containing directory block */ 8925 struct inode *dp; /* inode for the directory being modified */ 8926 struct inode *ip; /* inode for directory entry being removed */ 8927 int isrmdir; /* indicates if doing RMDIR */ 8928 { 8929 struct dirrem *dirrem, *prevdirrem; 8930 struct inodedep *inodedep; 8931 struct ufsmount *ump; 8932 int direct; 8933 8934 ump = ITOUMP(ip); 8935 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 8936 ("softdep_setup_remove called on non-softdep filesystem")); 8937 /* 8938 * Allocate a new dirrem if appropriate and ACQUIRE_LOCK. We want 8939 * newdirrem() to setup the full directory remove which requires 8940 * isrmdir > 1. 8941 */ 8942 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem); 8943 /* 8944 * Add the dirrem to the inodedep's pending remove list for quick 8945 * discovery later. 8946 */ 8947 if (inodedep_lookup(UFSTOVFS(ump), ip->i_number, 0, &inodedep) == 0) 8948 panic("softdep_setup_remove: Lost inodedep."); 8949 KASSERT((inodedep->id_state & UNLINKED) == 0, ("inode unlinked")); 8950 dirrem->dm_state |= ONDEPLIST; 8951 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 8952 8953 /* 8954 * If the COMPLETE flag is clear, then there were no active 8955 * entries and we want to roll back to a zeroed entry until 8956 * the new inode is committed to disk. If the COMPLETE flag is 8957 * set then we have deleted an entry that never made it to 8958 * disk. If the entry we deleted resulted from a name change, 8959 * then the old name still resides on disk. We cannot delete 8960 * its inode (returned to us in prevdirrem) until the zeroed 8961 * directory entry gets to disk. The new inode has never been 8962 * referenced on the disk, so can be deleted immediately. 8963 */ 8964 if ((dirrem->dm_state & COMPLETE) == 0) { 8965 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, dirrem, 8966 dm_next); 8967 FREE_LOCK(ump); 8968 } else { 8969 if (prevdirrem != NULL) 8970 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, 8971 prevdirrem, dm_next); 8972 dirrem->dm_dirinum = dirrem->dm_pagedep->pd_ino; 8973 direct = LIST_EMPTY(&dirrem->dm_jremrefhd); 8974 FREE_LOCK(ump); 8975 if (direct) 8976 handle_workitem_remove(dirrem, 0); 8977 } 8978 } 8979 8980 /* 8981 * Check for an entry matching 'offset' on both the pd_dirraddhd list and the 8982 * pd_pendinghd list of a pagedep. 8983 */ 8984 static struct diradd * 8985 diradd_lookup(pagedep, offset) 8986 struct pagedep *pagedep; 8987 int offset; 8988 { 8989 struct diradd *dap; 8990 8991 LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(offset)], da_pdlist) 8992 if (dap->da_offset == offset) 8993 return (dap); 8994 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist) 8995 if (dap->da_offset == offset) 8996 return (dap); 8997 return (NULL); 8998 } 8999 9000 /* 9001 * Search for a .. diradd dependency in a directory that is being removed. 9002 * If the directory was renamed to a new parent we have a diradd rather 9003 * than a mkdir for the .. entry. We need to cancel it now before 9004 * it is found in truncate(). 9005 */ 9006 static struct jremref * 9007 cancel_diradd_dotdot(ip, dirrem, jremref) 9008 struct inode *ip; 9009 struct dirrem *dirrem; 9010 struct jremref *jremref; 9011 { 9012 struct pagedep *pagedep; 9013 struct diradd *dap; 9014 struct worklist *wk; 9015 9016 if (pagedep_lookup(ITOVFS(ip), NULL, ip->i_number, 0, 0, &pagedep) == 0) 9017 return (jremref); 9018 dap = diradd_lookup(pagedep, DOTDOT_OFFSET); 9019 if (dap == NULL) 9020 return (jremref); 9021 cancel_diradd(dap, dirrem, jremref, NULL, NULL); 9022 /* 9023 * Mark any journal work as belonging to the parent so it is freed 9024 * with the .. reference. 9025 */ 9026 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 9027 wk->wk_state |= MKDIR_PARENT; 9028 return (NULL); 9029 } 9030 9031 /* 9032 * Cancel the MKDIR_PARENT mkdir component of a diradd when we're going to 9033 * replace it with a dirrem/diradd pair as a result of re-parenting a 9034 * directory. This ensures that we don't simultaneously have a mkdir and 9035 * a diradd for the same .. entry. 9036 */ 9037 static struct jremref * 9038 cancel_mkdir_dotdot(ip, dirrem, jremref) 9039 struct inode *ip; 9040 struct dirrem *dirrem; 9041 struct jremref *jremref; 9042 { 9043 struct inodedep *inodedep; 9044 struct jaddref *jaddref; 9045 struct ufsmount *ump; 9046 struct mkdir *mkdir; 9047 struct diradd *dap; 9048 struct mount *mp; 9049 9050 mp = ITOVFS(ip); 9051 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 9052 return (jremref); 9053 dap = inodedep->id_mkdiradd; 9054 if (dap == NULL || (dap->da_state & MKDIR_PARENT) == 0) 9055 return (jremref); 9056 ump = VFSTOUFS(inodedep->id_list.wk_mp); 9057 for (mkdir = LIST_FIRST(&ump->softdep_mkdirlisthd); mkdir; 9058 mkdir = LIST_NEXT(mkdir, md_mkdirs)) 9059 if (mkdir->md_diradd == dap && mkdir->md_state & MKDIR_PARENT) 9060 break; 9061 if (mkdir == NULL) 9062 panic("cancel_mkdir_dotdot: Unable to find mkdir\n"); 9063 if ((jaddref = mkdir->md_jaddref) != NULL) { 9064 mkdir->md_jaddref = NULL; 9065 jaddref->ja_state &= ~MKDIR_PARENT; 9066 if (inodedep_lookup(mp, jaddref->ja_ino, 0, &inodedep) == 0) 9067 panic("cancel_mkdir_dotdot: Lost parent inodedep"); 9068 if (cancel_jaddref(jaddref, inodedep, &dirrem->dm_jwork)) { 9069 journal_jremref(dirrem, jremref, inodedep); 9070 jremref = NULL; 9071 } 9072 } 9073 if (mkdir->md_state & ONWORKLIST) 9074 WORKLIST_REMOVE(&mkdir->md_list); 9075 mkdir->md_state |= ALLCOMPLETE; 9076 complete_mkdir(mkdir); 9077 return (jremref); 9078 } 9079 9080 static void 9081 journal_jremref(dirrem, jremref, inodedep) 9082 struct dirrem *dirrem; 9083 struct jremref *jremref; 9084 struct inodedep *inodedep; 9085 { 9086 9087 if (inodedep == NULL) 9088 if (inodedep_lookup(jremref->jr_list.wk_mp, 9089 jremref->jr_ref.if_ino, 0, &inodedep) == 0) 9090 panic("journal_jremref: Lost inodedep"); 9091 LIST_INSERT_HEAD(&dirrem->dm_jremrefhd, jremref, jr_deps); 9092 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 9093 add_to_journal(&jremref->jr_list); 9094 } 9095 9096 static void 9097 dirrem_journal(dirrem, jremref, dotremref, dotdotremref) 9098 struct dirrem *dirrem; 9099 struct jremref *jremref; 9100 struct jremref *dotremref; 9101 struct jremref *dotdotremref; 9102 { 9103 struct inodedep *inodedep; 9104 9105 9106 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 0, 9107 &inodedep) == 0) 9108 panic("dirrem_journal: Lost inodedep"); 9109 journal_jremref(dirrem, jremref, inodedep); 9110 if (dotremref) 9111 journal_jremref(dirrem, dotremref, inodedep); 9112 if (dotdotremref) 9113 journal_jremref(dirrem, dotdotremref, NULL); 9114 } 9115 9116 /* 9117 * Allocate a new dirrem if appropriate and return it along with 9118 * its associated pagedep. Called without a lock, returns with lock. 9119 */ 9120 static struct dirrem * 9121 newdirrem(bp, dp, ip, isrmdir, prevdirremp) 9122 struct buf *bp; /* buffer containing directory block */ 9123 struct inode *dp; /* inode for the directory being modified */ 9124 struct inode *ip; /* inode for directory entry being removed */ 9125 int isrmdir; /* indicates if doing RMDIR */ 9126 struct dirrem **prevdirremp; /* previously referenced inode, if any */ 9127 { 9128 int offset; 9129 ufs_lbn_t lbn; 9130 struct diradd *dap; 9131 struct dirrem *dirrem; 9132 struct pagedep *pagedep; 9133 struct jremref *jremref; 9134 struct jremref *dotremref; 9135 struct jremref *dotdotremref; 9136 struct vnode *dvp; 9137 struct ufsmount *ump; 9138 9139 /* 9140 * Whiteouts have no deletion dependencies. 9141 */ 9142 if (ip == NULL) 9143 panic("newdirrem: whiteout"); 9144 dvp = ITOV(dp); 9145 ump = ITOUMP(dp); 9146 9147 /* 9148 * If the system is over its limit and our filesystem is 9149 * responsible for more than our share of that usage and 9150 * we are not a snapshot, request some inodedep cleanup. 9151 * Limiting the number of dirrem structures will also limit 9152 * the number of freefile and freeblks structures. 9153 */ 9154 ACQUIRE_LOCK(ump); 9155 if (!IS_SNAPSHOT(ip) && softdep_excess_items(ump, D_DIRREM)) 9156 schedule_cleanup(UFSTOVFS(ump)); 9157 else 9158 FREE_LOCK(ump); 9159 dirrem = malloc(sizeof(struct dirrem), M_DIRREM, M_SOFTDEP_FLAGS | 9160 M_ZERO); 9161 workitem_alloc(&dirrem->dm_list, D_DIRREM, dvp->v_mount); 9162 LIST_INIT(&dirrem->dm_jremrefhd); 9163 LIST_INIT(&dirrem->dm_jwork); 9164 dirrem->dm_state = isrmdir ? RMDIR : 0; 9165 dirrem->dm_oldinum = ip->i_number; 9166 *prevdirremp = NULL; 9167 /* 9168 * Allocate remove reference structures to track journal write 9169 * dependencies. We will always have one for the link and 9170 * when doing directories we will always have one more for dot. 9171 * When renaming a directory we skip the dotdot link change so 9172 * this is not needed. 9173 */ 9174 jremref = dotremref = dotdotremref = NULL; 9175 if (DOINGSUJ(dvp)) { 9176 if (isrmdir) { 9177 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 9178 ip->i_effnlink + 2); 9179 dotremref = newjremref(dirrem, ip, ip, DOT_OFFSET, 9180 ip->i_effnlink + 1); 9181 dotdotremref = newjremref(dirrem, ip, dp, DOTDOT_OFFSET, 9182 dp->i_effnlink + 1); 9183 dotdotremref->jr_state |= MKDIR_PARENT; 9184 } else 9185 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 9186 ip->i_effnlink + 1); 9187 } 9188 ACQUIRE_LOCK(ump); 9189 lbn = lblkno(ump->um_fs, dp->i_offset); 9190 offset = blkoff(ump->um_fs, dp->i_offset); 9191 pagedep_lookup(UFSTOVFS(ump), bp, dp->i_number, lbn, DEPALLOC, 9192 &pagedep); 9193 dirrem->dm_pagedep = pagedep; 9194 dirrem->dm_offset = offset; 9195 /* 9196 * If we're renaming a .. link to a new directory, cancel any 9197 * existing MKDIR_PARENT mkdir. If it has already been canceled 9198 * the jremref is preserved for any potential diradd in this 9199 * location. This can not coincide with a rmdir. 9200 */ 9201 if (dp->i_offset == DOTDOT_OFFSET) { 9202 if (isrmdir) 9203 panic("newdirrem: .. directory change during remove?"); 9204 jremref = cancel_mkdir_dotdot(dp, dirrem, jremref); 9205 } 9206 /* 9207 * If we're removing a directory search for the .. dependency now and 9208 * cancel it. Any pending journal work will be added to the dirrem 9209 * to be completed when the workitem remove completes. 9210 */ 9211 if (isrmdir) 9212 dotdotremref = cancel_diradd_dotdot(ip, dirrem, dotdotremref); 9213 /* 9214 * Check for a diradd dependency for the same directory entry. 9215 * If present, then both dependencies become obsolete and can 9216 * be de-allocated. 9217 */ 9218 dap = diradd_lookup(pagedep, offset); 9219 if (dap == NULL) { 9220 /* 9221 * Link the jremref structures into the dirrem so they are 9222 * written prior to the pagedep. 9223 */ 9224 if (jremref) 9225 dirrem_journal(dirrem, jremref, dotremref, 9226 dotdotremref); 9227 return (dirrem); 9228 } 9229 /* 9230 * Must be ATTACHED at this point. 9231 */ 9232 if ((dap->da_state & ATTACHED) == 0) 9233 panic("newdirrem: not ATTACHED"); 9234 if (dap->da_newinum != ip->i_number) 9235 panic("newdirrem: inum %ju should be %ju", 9236 (uintmax_t)ip->i_number, (uintmax_t)dap->da_newinum); 9237 /* 9238 * If we are deleting a changed name that never made it to disk, 9239 * then return the dirrem describing the previous inode (which 9240 * represents the inode currently referenced from this entry on disk). 9241 */ 9242 if ((dap->da_state & DIRCHG) != 0) { 9243 *prevdirremp = dap->da_previous; 9244 dap->da_state &= ~DIRCHG; 9245 dap->da_pagedep = pagedep; 9246 } 9247 /* 9248 * We are deleting an entry that never made it to disk. 9249 * Mark it COMPLETE so we can delete its inode immediately. 9250 */ 9251 dirrem->dm_state |= COMPLETE; 9252 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref); 9253 #ifdef SUJ_DEBUG 9254 if (isrmdir == 0) { 9255 struct worklist *wk; 9256 9257 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 9258 if (wk->wk_state & (MKDIR_BODY | MKDIR_PARENT)) 9259 panic("bad wk %p (0x%X)\n", wk, wk->wk_state); 9260 } 9261 #endif 9262 9263 return (dirrem); 9264 } 9265 9266 /* 9267 * Directory entry change dependencies. 9268 * 9269 * Changing an existing directory entry requires that an add operation 9270 * be completed first followed by a deletion. The semantics for the addition 9271 * are identical to the description of adding a new entry above except 9272 * that the rollback is to the old inode number rather than zero. Once 9273 * the addition dependency is completed, the removal is done as described 9274 * in the removal routine above. 9275 */ 9276 9277 /* 9278 * This routine should be called immediately after changing 9279 * a directory entry. The inode's link count should not be 9280 * decremented by the calling procedure -- the soft updates 9281 * code will perform this task when it is safe. 9282 */ 9283 void 9284 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 9285 struct buf *bp; /* buffer containing directory block */ 9286 struct inode *dp; /* inode for the directory being modified */ 9287 struct inode *ip; /* inode for directory entry being removed */ 9288 ino_t newinum; /* new inode number for changed entry */ 9289 int isrmdir; /* indicates if doing RMDIR */ 9290 { 9291 int offset; 9292 struct diradd *dap = NULL; 9293 struct dirrem *dirrem, *prevdirrem; 9294 struct pagedep *pagedep; 9295 struct inodedep *inodedep; 9296 struct jaddref *jaddref; 9297 struct mount *mp; 9298 struct ufsmount *ump; 9299 9300 mp = ITOVFS(dp); 9301 ump = VFSTOUFS(mp); 9302 offset = blkoff(ump->um_fs, dp->i_offset); 9303 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 9304 ("softdep_setup_directory_change called on non-softdep filesystem")); 9305 9306 /* 9307 * Whiteouts do not need diradd dependencies. 9308 */ 9309 if (newinum != WINO) { 9310 dap = malloc(sizeof(struct diradd), 9311 M_DIRADD, M_SOFTDEP_FLAGS|M_ZERO); 9312 workitem_alloc(&dap->da_list, D_DIRADD, mp); 9313 dap->da_state = DIRCHG | ATTACHED | DEPCOMPLETE; 9314 dap->da_offset = offset; 9315 dap->da_newinum = newinum; 9316 LIST_INIT(&dap->da_jwork); 9317 } 9318 9319 /* 9320 * Allocate a new dirrem and ACQUIRE_LOCK. 9321 */ 9322 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem); 9323 pagedep = dirrem->dm_pagedep; 9324 /* 9325 * The possible values for isrmdir: 9326 * 0 - non-directory file rename 9327 * 1 - directory rename within same directory 9328 * inum - directory rename to new directory of given inode number 9329 * When renaming to a new directory, we are both deleting and 9330 * creating a new directory entry, so the link count on the new 9331 * directory should not change. Thus we do not need the followup 9332 * dirrem which is usually done in handle_workitem_remove. We set 9333 * the DIRCHG flag to tell handle_workitem_remove to skip the 9334 * followup dirrem. 9335 */ 9336 if (isrmdir > 1) 9337 dirrem->dm_state |= DIRCHG; 9338 9339 /* 9340 * Whiteouts have no additional dependencies, 9341 * so just put the dirrem on the correct list. 9342 */ 9343 if (newinum == WINO) { 9344 if ((dirrem->dm_state & COMPLETE) == 0) { 9345 LIST_INSERT_HEAD(&pagedep->pd_dirremhd, dirrem, 9346 dm_next); 9347 } else { 9348 dirrem->dm_dirinum = pagedep->pd_ino; 9349 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 9350 add_to_worklist(&dirrem->dm_list, 0); 9351 } 9352 FREE_LOCK(ump); 9353 return; 9354 } 9355 /* 9356 * Add the dirrem to the inodedep's pending remove list for quick 9357 * discovery later. A valid nlinkdelta ensures that this lookup 9358 * will not fail. 9359 */ 9360 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 9361 panic("softdep_setup_directory_change: Lost inodedep."); 9362 dirrem->dm_state |= ONDEPLIST; 9363 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 9364 9365 /* 9366 * If the COMPLETE flag is clear, then there were no active 9367 * entries and we want to roll back to the previous inode until 9368 * the new inode is committed to disk. If the COMPLETE flag is 9369 * set, then we have deleted an entry that never made it to disk. 9370 * If the entry we deleted resulted from a name change, then the old 9371 * inode reference still resides on disk. Any rollback that we do 9372 * needs to be to that old inode (returned to us in prevdirrem). If 9373 * the entry we deleted resulted from a create, then there is 9374 * no entry on the disk, so we want to roll back to zero rather 9375 * than the uncommitted inode. In either of the COMPLETE cases we 9376 * want to immediately free the unwritten and unreferenced inode. 9377 */ 9378 if ((dirrem->dm_state & COMPLETE) == 0) { 9379 dap->da_previous = dirrem; 9380 } else { 9381 if (prevdirrem != NULL) { 9382 dap->da_previous = prevdirrem; 9383 } else { 9384 dap->da_state &= ~DIRCHG; 9385 dap->da_pagedep = pagedep; 9386 } 9387 dirrem->dm_dirinum = pagedep->pd_ino; 9388 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 9389 add_to_worklist(&dirrem->dm_list, 0); 9390 } 9391 /* 9392 * Lookup the jaddref for this journal entry. We must finish 9393 * initializing it and make the diradd write dependent on it. 9394 * If we're not journaling, put it on the id_bufwait list if the 9395 * inode is not yet written. If it is written, do the post-inode 9396 * write processing to put it on the id_pendinghd list. 9397 */ 9398 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 9399 if (MOUNTEDSUJ(mp)) { 9400 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 9401 inoreflst); 9402 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 9403 ("softdep_setup_directory_change: bad jaddref %p", 9404 jaddref)); 9405 jaddref->ja_diroff = dp->i_offset; 9406 jaddref->ja_diradd = dap; 9407 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 9408 dap, da_pdlist); 9409 add_to_journal(&jaddref->ja_list); 9410 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 9411 dap->da_state |= COMPLETE; 9412 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 9413 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 9414 } else { 9415 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 9416 dap, da_pdlist); 9417 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 9418 } 9419 /* 9420 * If we're making a new name for a directory that has not been 9421 * committed when need to move the dot and dotdot references to 9422 * this new name. 9423 */ 9424 if (inodedep->id_mkdiradd && dp->i_offset != DOTDOT_OFFSET) 9425 merge_diradd(inodedep, dap); 9426 FREE_LOCK(ump); 9427 } 9428 9429 /* 9430 * Called whenever the link count on an inode is changed. 9431 * It creates an inode dependency so that the new reference(s) 9432 * to the inode cannot be committed to disk until the updated 9433 * inode has been written. 9434 */ 9435 void 9436 softdep_change_linkcnt(ip) 9437 struct inode *ip; /* the inode with the increased link count */ 9438 { 9439 struct inodedep *inodedep; 9440 struct ufsmount *ump; 9441 9442 ump = ITOUMP(ip); 9443 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 9444 ("softdep_change_linkcnt called on non-softdep filesystem")); 9445 ACQUIRE_LOCK(ump); 9446 inodedep_lookup(UFSTOVFS(ump), ip->i_number, DEPALLOC, &inodedep); 9447 if (ip->i_nlink < ip->i_effnlink) 9448 panic("softdep_change_linkcnt: bad delta"); 9449 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 9450 FREE_LOCK(ump); 9451 } 9452 9453 /* 9454 * Attach a sbdep dependency to the superblock buf so that we can keep 9455 * track of the head of the linked list of referenced but unlinked inodes. 9456 */ 9457 void 9458 softdep_setup_sbupdate(ump, fs, bp) 9459 struct ufsmount *ump; 9460 struct fs *fs; 9461 struct buf *bp; 9462 { 9463 struct sbdep *sbdep; 9464 struct worklist *wk; 9465 9466 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 9467 ("softdep_setup_sbupdate called on non-softdep filesystem")); 9468 LIST_FOREACH(wk, &bp->b_dep, wk_list) 9469 if (wk->wk_type == D_SBDEP) 9470 break; 9471 if (wk != NULL) 9472 return; 9473 sbdep = malloc(sizeof(struct sbdep), M_SBDEP, M_SOFTDEP_FLAGS); 9474 workitem_alloc(&sbdep->sb_list, D_SBDEP, UFSTOVFS(ump)); 9475 sbdep->sb_fs = fs; 9476 sbdep->sb_ump = ump; 9477 ACQUIRE_LOCK(ump); 9478 WORKLIST_INSERT(&bp->b_dep, &sbdep->sb_list); 9479 FREE_LOCK(ump); 9480 } 9481 9482 /* 9483 * Return the first unlinked inodedep which is ready to be the head of the 9484 * list. The inodedep and all those after it must have valid next pointers. 9485 */ 9486 static struct inodedep * 9487 first_unlinked_inodedep(ump) 9488 struct ufsmount *ump; 9489 { 9490 struct inodedep *inodedep; 9491 struct inodedep *idp; 9492 9493 LOCK_OWNED(ump); 9494 for (inodedep = TAILQ_LAST(&ump->softdep_unlinked, inodedeplst); 9495 inodedep; inodedep = idp) { 9496 if ((inodedep->id_state & UNLINKNEXT) == 0) 9497 return (NULL); 9498 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 9499 if (idp == NULL || (idp->id_state & UNLINKNEXT) == 0) 9500 break; 9501 if ((inodedep->id_state & UNLINKPREV) == 0) 9502 break; 9503 } 9504 return (inodedep); 9505 } 9506 9507 /* 9508 * Set the sujfree unlinked head pointer prior to writing a superblock. 9509 */ 9510 static void 9511 initiate_write_sbdep(sbdep) 9512 struct sbdep *sbdep; 9513 { 9514 struct inodedep *inodedep; 9515 struct fs *bpfs; 9516 struct fs *fs; 9517 9518 bpfs = sbdep->sb_fs; 9519 fs = sbdep->sb_ump->um_fs; 9520 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 9521 if (inodedep) { 9522 fs->fs_sujfree = inodedep->id_ino; 9523 inodedep->id_state |= UNLINKPREV; 9524 } else 9525 fs->fs_sujfree = 0; 9526 bpfs->fs_sujfree = fs->fs_sujfree; 9527 } 9528 9529 /* 9530 * After a superblock is written determine whether it must be written again 9531 * due to a changing unlinked list head. 9532 */ 9533 static int 9534 handle_written_sbdep(sbdep, bp) 9535 struct sbdep *sbdep; 9536 struct buf *bp; 9537 { 9538 struct inodedep *inodedep; 9539 struct fs *fs; 9540 9541 LOCK_OWNED(sbdep->sb_ump); 9542 fs = sbdep->sb_fs; 9543 /* 9544 * If the superblock doesn't match the in-memory list start over. 9545 */ 9546 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 9547 if ((inodedep && fs->fs_sujfree != inodedep->id_ino) || 9548 (inodedep == NULL && fs->fs_sujfree != 0)) { 9549 bdirty(bp); 9550 return (1); 9551 } 9552 WORKITEM_FREE(sbdep, D_SBDEP); 9553 if (fs->fs_sujfree == 0) 9554 return (0); 9555 /* 9556 * Now that we have a record of this inode in stable store allow it 9557 * to be written to free up pending work. Inodes may see a lot of 9558 * write activity after they are unlinked which we must not hold up. 9559 */ 9560 for (; inodedep != NULL; inodedep = TAILQ_NEXT(inodedep, id_unlinked)) { 9561 if ((inodedep->id_state & UNLINKLINKS) != UNLINKLINKS) 9562 panic("handle_written_sbdep: Bad inodedep %p (0x%X)", 9563 inodedep, inodedep->id_state); 9564 if (inodedep->id_state & UNLINKONLIST) 9565 break; 9566 inodedep->id_state |= DEPCOMPLETE | UNLINKONLIST; 9567 } 9568 9569 return (0); 9570 } 9571 9572 /* 9573 * Mark an inodedep as unlinked and insert it into the in-memory unlinked list. 9574 */ 9575 static void 9576 unlinked_inodedep(mp, inodedep) 9577 struct mount *mp; 9578 struct inodedep *inodedep; 9579 { 9580 struct ufsmount *ump; 9581 9582 ump = VFSTOUFS(mp); 9583 LOCK_OWNED(ump); 9584 if (MOUNTEDSUJ(mp) == 0) 9585 return; 9586 ump->um_fs->fs_fmod = 1; 9587 if (inodedep->id_state & UNLINKED) 9588 panic("unlinked_inodedep: %p already unlinked\n", inodedep); 9589 inodedep->id_state |= UNLINKED; 9590 TAILQ_INSERT_HEAD(&ump->softdep_unlinked, inodedep, id_unlinked); 9591 } 9592 9593 /* 9594 * Remove an inodedep from the unlinked inodedep list. This may require 9595 * disk writes if the inode has made it that far. 9596 */ 9597 static void 9598 clear_unlinked_inodedep(inodedep) 9599 struct inodedep *inodedep; 9600 { 9601 struct ufsmount *ump; 9602 struct inodedep *idp; 9603 struct inodedep *idn; 9604 struct fs *fs; 9605 struct buf *bp; 9606 ino_t ino; 9607 ino_t nino; 9608 ino_t pino; 9609 int error; 9610 9611 ump = VFSTOUFS(inodedep->id_list.wk_mp); 9612 fs = ump->um_fs; 9613 ino = inodedep->id_ino; 9614 error = 0; 9615 for (;;) { 9616 LOCK_OWNED(ump); 9617 KASSERT((inodedep->id_state & UNLINKED) != 0, 9618 ("clear_unlinked_inodedep: inodedep %p not unlinked", 9619 inodedep)); 9620 /* 9621 * If nothing has yet been written simply remove us from 9622 * the in memory list and return. This is the most common 9623 * case where handle_workitem_remove() loses the final 9624 * reference. 9625 */ 9626 if ((inodedep->id_state & UNLINKLINKS) == 0) 9627 break; 9628 /* 9629 * If we have a NEXT pointer and no PREV pointer we can simply 9630 * clear NEXT's PREV and remove ourselves from the list. Be 9631 * careful not to clear PREV if the superblock points at 9632 * next as well. 9633 */ 9634 idn = TAILQ_NEXT(inodedep, id_unlinked); 9635 if ((inodedep->id_state & UNLINKLINKS) == UNLINKNEXT) { 9636 if (idn && fs->fs_sujfree != idn->id_ino) 9637 idn->id_state &= ~UNLINKPREV; 9638 break; 9639 } 9640 /* 9641 * Here we have an inodedep which is actually linked into 9642 * the list. We must remove it by forcing a write to the 9643 * link before us, whether it be the superblock or an inode. 9644 * Unfortunately the list may change while we're waiting 9645 * on the buf lock for either resource so we must loop until 9646 * we lock the right one. If both the superblock and an 9647 * inode point to this inode we must clear the inode first 9648 * followed by the superblock. 9649 */ 9650 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 9651 pino = 0; 9652 if (idp && (idp->id_state & UNLINKNEXT)) 9653 pino = idp->id_ino; 9654 FREE_LOCK(ump); 9655 if (pino == 0) { 9656 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 9657 (int)fs->fs_sbsize, 0, 0, 0); 9658 } else { 9659 error = bread(ump->um_devvp, 9660 fsbtodb(fs, ino_to_fsba(fs, pino)), 9661 (int)fs->fs_bsize, NOCRED, &bp); 9662 if (error) 9663 brelse(bp); 9664 } 9665 ACQUIRE_LOCK(ump); 9666 if (error) 9667 break; 9668 /* If the list has changed restart the loop. */ 9669 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 9670 nino = 0; 9671 if (idp && (idp->id_state & UNLINKNEXT)) 9672 nino = idp->id_ino; 9673 if (nino != pino || 9674 (inodedep->id_state & UNLINKPREV) != UNLINKPREV) { 9675 FREE_LOCK(ump); 9676 brelse(bp); 9677 ACQUIRE_LOCK(ump); 9678 continue; 9679 } 9680 nino = 0; 9681 idn = TAILQ_NEXT(inodedep, id_unlinked); 9682 if (idn) 9683 nino = idn->id_ino; 9684 /* 9685 * Remove us from the in memory list. After this we cannot 9686 * access the inodedep. 9687 */ 9688 KASSERT((inodedep->id_state & UNLINKED) != 0, 9689 ("clear_unlinked_inodedep: inodedep %p not unlinked", 9690 inodedep)); 9691 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST); 9692 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 9693 FREE_LOCK(ump); 9694 /* 9695 * The predecessor's next pointer is manually updated here 9696 * so that the NEXT flag is never cleared for an element 9697 * that is in the list. 9698 */ 9699 if (pino == 0) { 9700 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 9701 ffs_oldfscompat_write((struct fs *)bp->b_data, ump); 9702 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data, 9703 bp); 9704 } else if (fs->fs_magic == FS_UFS1_MAGIC) 9705 ((struct ufs1_dinode *)bp->b_data + 9706 ino_to_fsbo(fs, pino))->di_freelink = nino; 9707 else 9708 ((struct ufs2_dinode *)bp->b_data + 9709 ino_to_fsbo(fs, pino))->di_freelink = nino; 9710 /* 9711 * If the bwrite fails we have no recourse to recover. The 9712 * filesystem is corrupted already. 9713 */ 9714 bwrite(bp); 9715 ACQUIRE_LOCK(ump); 9716 /* 9717 * If the superblock pointer still needs to be cleared force 9718 * a write here. 9719 */ 9720 if (fs->fs_sujfree == ino) { 9721 FREE_LOCK(ump); 9722 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 9723 (int)fs->fs_sbsize, 0, 0, 0); 9724 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 9725 ffs_oldfscompat_write((struct fs *)bp->b_data, ump); 9726 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data, 9727 bp); 9728 bwrite(bp); 9729 ACQUIRE_LOCK(ump); 9730 } 9731 9732 if (fs->fs_sujfree != ino) 9733 return; 9734 panic("clear_unlinked_inodedep: Failed to clear free head"); 9735 } 9736 if (inodedep->id_ino == fs->fs_sujfree) 9737 panic("clear_unlinked_inodedep: Freeing head of free list"); 9738 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS | UNLINKONLIST); 9739 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 9740 return; 9741 } 9742 9743 /* 9744 * This workitem decrements the inode's link count. 9745 * If the link count reaches zero, the file is removed. 9746 */ 9747 static int 9748 handle_workitem_remove(dirrem, flags) 9749 struct dirrem *dirrem; 9750 int flags; 9751 { 9752 struct inodedep *inodedep; 9753 struct workhead dotdotwk; 9754 struct worklist *wk; 9755 struct ufsmount *ump; 9756 struct mount *mp; 9757 struct vnode *vp; 9758 struct inode *ip; 9759 ino_t oldinum; 9760 9761 if (dirrem->dm_state & ONWORKLIST) 9762 panic("handle_workitem_remove: dirrem %p still on worklist", 9763 dirrem); 9764 oldinum = dirrem->dm_oldinum; 9765 mp = dirrem->dm_list.wk_mp; 9766 ump = VFSTOUFS(mp); 9767 flags |= LK_EXCLUSIVE; 9768 if (ffs_vgetf(mp, oldinum, flags, &vp, FFSV_FORCEINSMQ) != 0) 9769 return (EBUSY); 9770 ip = VTOI(vp); 9771 ACQUIRE_LOCK(ump); 9772 if ((inodedep_lookup(mp, oldinum, 0, &inodedep)) == 0) 9773 panic("handle_workitem_remove: lost inodedep"); 9774 if (dirrem->dm_state & ONDEPLIST) 9775 LIST_REMOVE(dirrem, dm_inonext); 9776 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 9777 ("handle_workitem_remove: Journal entries not written.")); 9778 9779 /* 9780 * Move all dependencies waiting on the remove to complete 9781 * from the dirrem to the inode inowait list to be completed 9782 * after the inode has been updated and written to disk. Any 9783 * marked MKDIR_PARENT are saved to be completed when the .. ref 9784 * is removed. 9785 */ 9786 LIST_INIT(&dotdotwk); 9787 while ((wk = LIST_FIRST(&dirrem->dm_jwork)) != NULL) { 9788 WORKLIST_REMOVE(wk); 9789 if (wk->wk_state & MKDIR_PARENT) { 9790 wk->wk_state &= ~MKDIR_PARENT; 9791 WORKLIST_INSERT(&dotdotwk, wk); 9792 continue; 9793 } 9794 WORKLIST_INSERT(&inodedep->id_inowait, wk); 9795 } 9796 LIST_SWAP(&dirrem->dm_jwork, &dotdotwk, worklist, wk_list); 9797 /* 9798 * Normal file deletion. 9799 */ 9800 if ((dirrem->dm_state & RMDIR) == 0) { 9801 ip->i_nlink--; 9802 DIP_SET(ip, i_nlink, ip->i_nlink); 9803 ip->i_flag |= IN_CHANGE; 9804 if (ip->i_nlink < ip->i_effnlink) 9805 panic("handle_workitem_remove: bad file delta"); 9806 if (ip->i_nlink == 0) 9807 unlinked_inodedep(mp, inodedep); 9808 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 9809 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 9810 ("handle_workitem_remove: worklist not empty. %s", 9811 TYPENAME(LIST_FIRST(&dirrem->dm_jwork)->wk_type))); 9812 WORKITEM_FREE(dirrem, D_DIRREM); 9813 FREE_LOCK(ump); 9814 goto out; 9815 } 9816 /* 9817 * Directory deletion. Decrement reference count for both the 9818 * just deleted parent directory entry and the reference for ".". 9819 * Arrange to have the reference count on the parent decremented 9820 * to account for the loss of "..". 9821 */ 9822 ip->i_nlink -= 2; 9823 DIP_SET(ip, i_nlink, ip->i_nlink); 9824 ip->i_flag |= IN_CHANGE; 9825 if (ip->i_nlink < ip->i_effnlink) 9826 panic("handle_workitem_remove: bad dir delta"); 9827 if (ip->i_nlink == 0) 9828 unlinked_inodedep(mp, inodedep); 9829 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 9830 /* 9831 * Rename a directory to a new parent. Since, we are both deleting 9832 * and creating a new directory entry, the link count on the new 9833 * directory should not change. Thus we skip the followup dirrem. 9834 */ 9835 if (dirrem->dm_state & DIRCHG) { 9836 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 9837 ("handle_workitem_remove: DIRCHG and worklist not empty.")); 9838 WORKITEM_FREE(dirrem, D_DIRREM); 9839 FREE_LOCK(ump); 9840 goto out; 9841 } 9842 dirrem->dm_state = ONDEPLIST; 9843 dirrem->dm_oldinum = dirrem->dm_dirinum; 9844 /* 9845 * Place the dirrem on the parent's diremhd list. 9846 */ 9847 if (inodedep_lookup(mp, dirrem->dm_oldinum, 0, &inodedep) == 0) 9848 panic("handle_workitem_remove: lost dir inodedep"); 9849 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 9850 /* 9851 * If the allocated inode has never been written to disk, then 9852 * the on-disk inode is zero'ed and we can remove the file 9853 * immediately. When journaling if the inode has been marked 9854 * unlinked and not DEPCOMPLETE we know it can never be written. 9855 */ 9856 inodedep_lookup(mp, oldinum, 0, &inodedep); 9857 if (inodedep == NULL || 9858 (inodedep->id_state & (DEPCOMPLETE | UNLINKED)) == UNLINKED || 9859 check_inode_unwritten(inodedep)) { 9860 FREE_LOCK(ump); 9861 vput(vp); 9862 return handle_workitem_remove(dirrem, flags); 9863 } 9864 WORKLIST_INSERT(&inodedep->id_inowait, &dirrem->dm_list); 9865 FREE_LOCK(ump); 9866 ip->i_flag |= IN_CHANGE; 9867 out: 9868 ffs_update(vp, 0); 9869 vput(vp); 9870 return (0); 9871 } 9872 9873 /* 9874 * Inode de-allocation dependencies. 9875 * 9876 * When an inode's link count is reduced to zero, it can be de-allocated. We 9877 * found it convenient to postpone de-allocation until after the inode is 9878 * written to disk with its new link count (zero). At this point, all of the 9879 * on-disk inode's block pointers are nullified and, with careful dependency 9880 * list ordering, all dependencies related to the inode will be satisfied and 9881 * the corresponding dependency structures de-allocated. So, if/when the 9882 * inode is reused, there will be no mixing of old dependencies with new 9883 * ones. This artificial dependency is set up by the block de-allocation 9884 * procedure above (softdep_setup_freeblocks) and completed by the 9885 * following procedure. 9886 */ 9887 static void 9888 handle_workitem_freefile(freefile) 9889 struct freefile *freefile; 9890 { 9891 struct workhead wkhd; 9892 struct fs *fs; 9893 struct inodedep *idp; 9894 struct ufsmount *ump; 9895 int error; 9896 9897 ump = VFSTOUFS(freefile->fx_list.wk_mp); 9898 fs = ump->um_fs; 9899 #ifdef DEBUG 9900 ACQUIRE_LOCK(ump); 9901 error = inodedep_lookup(UFSTOVFS(ump), freefile->fx_oldinum, 0, &idp); 9902 FREE_LOCK(ump); 9903 if (error) 9904 panic("handle_workitem_freefile: inodedep %p survived", idp); 9905 #endif 9906 UFS_LOCK(ump); 9907 fs->fs_pendinginodes -= 1; 9908 UFS_UNLOCK(ump); 9909 LIST_INIT(&wkhd); 9910 LIST_SWAP(&freefile->fx_jwork, &wkhd, worklist, wk_list); 9911 if ((error = ffs_freefile(ump, fs, freefile->fx_devvp, 9912 freefile->fx_oldinum, freefile->fx_mode, &wkhd)) != 0) 9913 softdep_error("handle_workitem_freefile", error); 9914 ACQUIRE_LOCK(ump); 9915 WORKITEM_FREE(freefile, D_FREEFILE); 9916 FREE_LOCK(ump); 9917 } 9918 9919 9920 /* 9921 * Helper function which unlinks marker element from work list and returns 9922 * the next element on the list. 9923 */ 9924 static __inline struct worklist * 9925 markernext(struct worklist *marker) 9926 { 9927 struct worklist *next; 9928 9929 next = LIST_NEXT(marker, wk_list); 9930 LIST_REMOVE(marker, wk_list); 9931 return next; 9932 } 9933 9934 /* 9935 * Disk writes. 9936 * 9937 * The dependency structures constructed above are most actively used when file 9938 * system blocks are written to disk. No constraints are placed on when a 9939 * block can be written, but unsatisfied update dependencies are made safe by 9940 * modifying (or replacing) the source memory for the duration of the disk 9941 * write. When the disk write completes, the memory block is again brought 9942 * up-to-date. 9943 * 9944 * In-core inode structure reclamation. 9945 * 9946 * Because there are a finite number of "in-core" inode structures, they are 9947 * reused regularly. By transferring all inode-related dependencies to the 9948 * in-memory inode block and indexing them separately (via "inodedep"s), we 9949 * can allow "in-core" inode structures to be reused at any time and avoid 9950 * any increase in contention. 9951 * 9952 * Called just before entering the device driver to initiate a new disk I/O. 9953 * The buffer must be locked, thus, no I/O completion operations can occur 9954 * while we are manipulating its associated dependencies. 9955 */ 9956 static void 9957 softdep_disk_io_initiation(bp) 9958 struct buf *bp; /* structure describing disk write to occur */ 9959 { 9960 struct worklist *wk; 9961 struct worklist marker; 9962 struct inodedep *inodedep; 9963 struct freeblks *freeblks; 9964 struct jblkdep *jblkdep; 9965 struct newblk *newblk; 9966 struct ufsmount *ump; 9967 9968 /* 9969 * We only care about write operations. There should never 9970 * be dependencies for reads. 9971 */ 9972 if (bp->b_iocmd != BIO_WRITE) 9973 panic("softdep_disk_io_initiation: not write"); 9974 9975 if (bp->b_vflags & BV_BKGRDINPROG) 9976 panic("softdep_disk_io_initiation: Writing buffer with " 9977 "background write in progress: %p", bp); 9978 9979 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL) 9980 return; 9981 ump = VFSTOUFS(wk->wk_mp); 9982 9983 marker.wk_type = D_LAST + 1; /* Not a normal workitem */ 9984 PHOLD(curproc); /* Don't swap out kernel stack */ 9985 ACQUIRE_LOCK(ump); 9986 /* 9987 * Do any necessary pre-I/O processing. 9988 */ 9989 for (wk = LIST_FIRST(&bp->b_dep); wk != NULL; 9990 wk = markernext(&marker)) { 9991 LIST_INSERT_AFTER(wk, &marker, wk_list); 9992 switch (wk->wk_type) { 9993 9994 case D_PAGEDEP: 9995 initiate_write_filepage(WK_PAGEDEP(wk), bp); 9996 continue; 9997 9998 case D_INODEDEP: 9999 inodedep = WK_INODEDEP(wk); 10000 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) 10001 initiate_write_inodeblock_ufs1(inodedep, bp); 10002 else 10003 initiate_write_inodeblock_ufs2(inodedep, bp); 10004 continue; 10005 10006 case D_INDIRDEP: 10007 initiate_write_indirdep(WK_INDIRDEP(wk), bp); 10008 continue; 10009 10010 case D_BMSAFEMAP: 10011 initiate_write_bmsafemap(WK_BMSAFEMAP(wk), bp); 10012 continue; 10013 10014 case D_JSEG: 10015 WK_JSEG(wk)->js_buf = NULL; 10016 continue; 10017 10018 case D_FREEBLKS: 10019 freeblks = WK_FREEBLKS(wk); 10020 jblkdep = LIST_FIRST(&freeblks->fb_jblkdephd); 10021 /* 10022 * We have to wait for the freeblks to be journaled 10023 * before we can write an inodeblock with updated 10024 * pointers. Be careful to arrange the marker so 10025 * we revisit the freeblks if it's not removed by 10026 * the first jwait(). 10027 */ 10028 if (jblkdep != NULL) { 10029 LIST_REMOVE(&marker, wk_list); 10030 LIST_INSERT_BEFORE(wk, &marker, wk_list); 10031 jwait(&jblkdep->jb_list, MNT_WAIT); 10032 } 10033 continue; 10034 case D_ALLOCDIRECT: 10035 case D_ALLOCINDIR: 10036 /* 10037 * We have to wait for the jnewblk to be journaled 10038 * before we can write to a block if the contents 10039 * may be confused with an earlier file's indirect 10040 * at recovery time. Handle the marker as described 10041 * above. 10042 */ 10043 newblk = WK_NEWBLK(wk); 10044 if (newblk->nb_jnewblk != NULL && 10045 indirblk_lookup(newblk->nb_list.wk_mp, 10046 newblk->nb_newblkno)) { 10047 LIST_REMOVE(&marker, wk_list); 10048 LIST_INSERT_BEFORE(wk, &marker, wk_list); 10049 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT); 10050 } 10051 continue; 10052 10053 case D_SBDEP: 10054 initiate_write_sbdep(WK_SBDEP(wk)); 10055 continue; 10056 10057 case D_MKDIR: 10058 case D_FREEWORK: 10059 case D_FREEDEP: 10060 case D_JSEGDEP: 10061 continue; 10062 10063 default: 10064 panic("handle_disk_io_initiation: Unexpected type %s", 10065 TYPENAME(wk->wk_type)); 10066 /* NOTREACHED */ 10067 } 10068 } 10069 FREE_LOCK(ump); 10070 PRELE(curproc); /* Allow swapout of kernel stack */ 10071 } 10072 10073 /* 10074 * Called from within the procedure above to deal with unsatisfied 10075 * allocation dependencies in a directory. The buffer must be locked, 10076 * thus, no I/O completion operations can occur while we are 10077 * manipulating its associated dependencies. 10078 */ 10079 static void 10080 initiate_write_filepage(pagedep, bp) 10081 struct pagedep *pagedep; 10082 struct buf *bp; 10083 { 10084 struct jremref *jremref; 10085 struct jmvref *jmvref; 10086 struct dirrem *dirrem; 10087 struct diradd *dap; 10088 struct direct *ep; 10089 int i; 10090 10091 if (pagedep->pd_state & IOSTARTED) { 10092 /* 10093 * This can only happen if there is a driver that does not 10094 * understand chaining. Here biodone will reissue the call 10095 * to strategy for the incomplete buffers. 10096 */ 10097 printf("initiate_write_filepage: already started\n"); 10098 return; 10099 } 10100 pagedep->pd_state |= IOSTARTED; 10101 /* 10102 * Wait for all journal remove dependencies to hit the disk. 10103 * We can not allow any potentially conflicting directory adds 10104 * to be visible before removes and rollback is too difficult. 10105 * The per-filesystem lock may be dropped and re-acquired, however 10106 * we hold the buf locked so the dependency can not go away. 10107 */ 10108 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) 10109 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) 10110 jwait(&jremref->jr_list, MNT_WAIT); 10111 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) 10112 jwait(&jmvref->jm_list, MNT_WAIT); 10113 for (i = 0; i < DAHASHSZ; i++) { 10114 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 10115 ep = (struct direct *) 10116 ((char *)bp->b_data + dap->da_offset); 10117 if (ep->d_ino != dap->da_newinum) 10118 panic("%s: dir inum %ju != new %ju", 10119 "initiate_write_filepage", 10120 (uintmax_t)ep->d_ino, 10121 (uintmax_t)dap->da_newinum); 10122 if (dap->da_state & DIRCHG) 10123 ep->d_ino = dap->da_previous->dm_oldinum; 10124 else 10125 ep->d_ino = 0; 10126 dap->da_state &= ~ATTACHED; 10127 dap->da_state |= UNDONE; 10128 } 10129 } 10130 } 10131 10132 /* 10133 * Version of initiate_write_inodeblock that handles UFS1 dinodes. 10134 * Note that any bug fixes made to this routine must be done in the 10135 * version found below. 10136 * 10137 * Called from within the procedure above to deal with unsatisfied 10138 * allocation dependencies in an inodeblock. The buffer must be 10139 * locked, thus, no I/O completion operations can occur while we 10140 * are manipulating its associated dependencies. 10141 */ 10142 static void 10143 initiate_write_inodeblock_ufs1(inodedep, bp) 10144 struct inodedep *inodedep; 10145 struct buf *bp; /* The inode block */ 10146 { 10147 struct allocdirect *adp, *lastadp; 10148 struct ufs1_dinode *dp; 10149 struct ufs1_dinode *sip; 10150 struct inoref *inoref; 10151 struct ufsmount *ump; 10152 struct fs *fs; 10153 ufs_lbn_t i; 10154 #ifdef INVARIANTS 10155 ufs_lbn_t prevlbn = 0; 10156 #endif 10157 int deplist; 10158 10159 if (inodedep->id_state & IOSTARTED) 10160 panic("initiate_write_inodeblock_ufs1: already started"); 10161 inodedep->id_state |= IOSTARTED; 10162 fs = inodedep->id_fs; 10163 ump = VFSTOUFS(inodedep->id_list.wk_mp); 10164 LOCK_OWNED(ump); 10165 dp = (struct ufs1_dinode *)bp->b_data + 10166 ino_to_fsbo(fs, inodedep->id_ino); 10167 10168 /* 10169 * If we're on the unlinked list but have not yet written our 10170 * next pointer initialize it here. 10171 */ 10172 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 10173 struct inodedep *inon; 10174 10175 inon = TAILQ_NEXT(inodedep, id_unlinked); 10176 dp->di_freelink = inon ? inon->id_ino : 0; 10177 } 10178 /* 10179 * If the bitmap is not yet written, then the allocated 10180 * inode cannot be written to disk. 10181 */ 10182 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 10183 if (inodedep->id_savedino1 != NULL) 10184 panic("initiate_write_inodeblock_ufs1: I/O underway"); 10185 FREE_LOCK(ump); 10186 sip = malloc(sizeof(struct ufs1_dinode), 10187 M_SAVEDINO, M_SOFTDEP_FLAGS); 10188 ACQUIRE_LOCK(ump); 10189 inodedep->id_savedino1 = sip; 10190 *inodedep->id_savedino1 = *dp; 10191 bzero((caddr_t)dp, sizeof(struct ufs1_dinode)); 10192 dp->di_gen = inodedep->id_savedino1->di_gen; 10193 dp->di_freelink = inodedep->id_savedino1->di_freelink; 10194 return; 10195 } 10196 /* 10197 * If no dependencies, then there is nothing to roll back. 10198 */ 10199 inodedep->id_savedsize = dp->di_size; 10200 inodedep->id_savedextsize = 0; 10201 inodedep->id_savednlink = dp->di_nlink; 10202 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 10203 TAILQ_EMPTY(&inodedep->id_inoreflst)) 10204 return; 10205 /* 10206 * Revert the link count to that of the first unwritten journal entry. 10207 */ 10208 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 10209 if (inoref) 10210 dp->di_nlink = inoref->if_nlink; 10211 /* 10212 * Set the dependencies to busy. 10213 */ 10214 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10215 adp = TAILQ_NEXT(adp, ad_next)) { 10216 #ifdef INVARIANTS 10217 if (deplist != 0 && prevlbn >= adp->ad_offset) 10218 panic("softdep_write_inodeblock: lbn order"); 10219 prevlbn = adp->ad_offset; 10220 if (adp->ad_offset < NDADDR && 10221 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 10222 panic("%s: direct pointer #%jd mismatch %d != %jd", 10223 "softdep_write_inodeblock", 10224 (intmax_t)adp->ad_offset, 10225 dp->di_db[adp->ad_offset], 10226 (intmax_t)adp->ad_newblkno); 10227 if (adp->ad_offset >= NDADDR && 10228 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno) 10229 panic("%s: indirect pointer #%jd mismatch %d != %jd", 10230 "softdep_write_inodeblock", 10231 (intmax_t)adp->ad_offset - NDADDR, 10232 dp->di_ib[adp->ad_offset - NDADDR], 10233 (intmax_t)adp->ad_newblkno); 10234 deplist |= 1 << adp->ad_offset; 10235 if ((adp->ad_state & ATTACHED) == 0) 10236 panic("softdep_write_inodeblock: Unknown state 0x%x", 10237 adp->ad_state); 10238 #endif /* INVARIANTS */ 10239 adp->ad_state &= ~ATTACHED; 10240 adp->ad_state |= UNDONE; 10241 } 10242 /* 10243 * The on-disk inode cannot claim to be any larger than the last 10244 * fragment that has been written. Otherwise, the on-disk inode 10245 * might have fragments that were not the last block in the file 10246 * which would corrupt the filesystem. 10247 */ 10248 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10249 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 10250 if (adp->ad_offset >= NDADDR) 10251 break; 10252 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 10253 /* keep going until hitting a rollback to a frag */ 10254 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 10255 continue; 10256 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 10257 for (i = adp->ad_offset + 1; i < NDADDR; i++) { 10258 #ifdef INVARIANTS 10259 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 10260 panic("softdep_write_inodeblock: lost dep1"); 10261 #endif /* INVARIANTS */ 10262 dp->di_db[i] = 0; 10263 } 10264 for (i = 0; i < NIADDR; i++) { 10265 #ifdef INVARIANTS 10266 if (dp->di_ib[i] != 0 && 10267 (deplist & ((1 << NDADDR) << i)) == 0) 10268 panic("softdep_write_inodeblock: lost dep2"); 10269 #endif /* INVARIANTS */ 10270 dp->di_ib[i] = 0; 10271 } 10272 return; 10273 } 10274 /* 10275 * If we have zero'ed out the last allocated block of the file, 10276 * roll back the size to the last currently allocated block. 10277 * We know that this last allocated block is a full-sized as 10278 * we already checked for fragments in the loop above. 10279 */ 10280 if (lastadp != NULL && 10281 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 10282 for (i = lastadp->ad_offset; i >= 0; i--) 10283 if (dp->di_db[i] != 0) 10284 break; 10285 dp->di_size = (i + 1) * fs->fs_bsize; 10286 } 10287 /* 10288 * The only dependencies are for indirect blocks. 10289 * 10290 * The file size for indirect block additions is not guaranteed. 10291 * Such a guarantee would be non-trivial to achieve. The conventional 10292 * synchronous write implementation also does not make this guarantee. 10293 * Fsck should catch and fix discrepancies. Arguably, the file size 10294 * can be over-estimated without destroying integrity when the file 10295 * moves into the indirect blocks (i.e., is large). If we want to 10296 * postpone fsck, we are stuck with this argument. 10297 */ 10298 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 10299 dp->di_ib[adp->ad_offset - NDADDR] = 0; 10300 } 10301 10302 /* 10303 * Version of initiate_write_inodeblock that handles UFS2 dinodes. 10304 * Note that any bug fixes made to this routine must be done in the 10305 * version found above. 10306 * 10307 * Called from within the procedure above to deal with unsatisfied 10308 * allocation dependencies in an inodeblock. The buffer must be 10309 * locked, thus, no I/O completion operations can occur while we 10310 * are manipulating its associated dependencies. 10311 */ 10312 static void 10313 initiate_write_inodeblock_ufs2(inodedep, bp) 10314 struct inodedep *inodedep; 10315 struct buf *bp; /* The inode block */ 10316 { 10317 struct allocdirect *adp, *lastadp; 10318 struct ufs2_dinode *dp; 10319 struct ufs2_dinode *sip; 10320 struct inoref *inoref; 10321 struct ufsmount *ump; 10322 struct fs *fs; 10323 ufs_lbn_t i; 10324 #ifdef INVARIANTS 10325 ufs_lbn_t prevlbn = 0; 10326 #endif 10327 int deplist; 10328 10329 if (inodedep->id_state & IOSTARTED) 10330 panic("initiate_write_inodeblock_ufs2: already started"); 10331 inodedep->id_state |= IOSTARTED; 10332 fs = inodedep->id_fs; 10333 ump = VFSTOUFS(inodedep->id_list.wk_mp); 10334 LOCK_OWNED(ump); 10335 dp = (struct ufs2_dinode *)bp->b_data + 10336 ino_to_fsbo(fs, inodedep->id_ino); 10337 10338 /* 10339 * If we're on the unlinked list but have not yet written our 10340 * next pointer initialize it here. 10341 */ 10342 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 10343 struct inodedep *inon; 10344 10345 inon = TAILQ_NEXT(inodedep, id_unlinked); 10346 dp->di_freelink = inon ? inon->id_ino : 0; 10347 } 10348 /* 10349 * If the bitmap is not yet written, then the allocated 10350 * inode cannot be written to disk. 10351 */ 10352 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 10353 if (inodedep->id_savedino2 != NULL) 10354 panic("initiate_write_inodeblock_ufs2: I/O underway"); 10355 FREE_LOCK(ump); 10356 sip = malloc(sizeof(struct ufs2_dinode), 10357 M_SAVEDINO, M_SOFTDEP_FLAGS); 10358 ACQUIRE_LOCK(ump); 10359 inodedep->id_savedino2 = sip; 10360 *inodedep->id_savedino2 = *dp; 10361 bzero((caddr_t)dp, sizeof(struct ufs2_dinode)); 10362 dp->di_gen = inodedep->id_savedino2->di_gen; 10363 dp->di_freelink = inodedep->id_savedino2->di_freelink; 10364 return; 10365 } 10366 /* 10367 * If no dependencies, then there is nothing to roll back. 10368 */ 10369 inodedep->id_savedsize = dp->di_size; 10370 inodedep->id_savedextsize = dp->di_extsize; 10371 inodedep->id_savednlink = dp->di_nlink; 10372 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 10373 TAILQ_EMPTY(&inodedep->id_extupdt) && 10374 TAILQ_EMPTY(&inodedep->id_inoreflst)) 10375 return; 10376 /* 10377 * Revert the link count to that of the first unwritten journal entry. 10378 */ 10379 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 10380 if (inoref) 10381 dp->di_nlink = inoref->if_nlink; 10382 10383 /* 10384 * Set the ext data dependencies to busy. 10385 */ 10386 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 10387 adp = TAILQ_NEXT(adp, ad_next)) { 10388 #ifdef INVARIANTS 10389 if (deplist != 0 && prevlbn >= adp->ad_offset) 10390 panic("softdep_write_inodeblock: lbn order"); 10391 prevlbn = adp->ad_offset; 10392 if (dp->di_extb[adp->ad_offset] != adp->ad_newblkno) 10393 panic("%s: direct pointer #%jd mismatch %jd != %jd", 10394 "softdep_write_inodeblock", 10395 (intmax_t)adp->ad_offset, 10396 (intmax_t)dp->di_extb[adp->ad_offset], 10397 (intmax_t)adp->ad_newblkno); 10398 deplist |= 1 << adp->ad_offset; 10399 if ((adp->ad_state & ATTACHED) == 0) 10400 panic("softdep_write_inodeblock: Unknown state 0x%x", 10401 adp->ad_state); 10402 #endif /* INVARIANTS */ 10403 adp->ad_state &= ~ATTACHED; 10404 adp->ad_state |= UNDONE; 10405 } 10406 /* 10407 * The on-disk inode cannot claim to be any larger than the last 10408 * fragment that has been written. Otherwise, the on-disk inode 10409 * might have fragments that were not the last block in the ext 10410 * data which would corrupt the filesystem. 10411 */ 10412 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 10413 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 10414 dp->di_extb[adp->ad_offset] = adp->ad_oldblkno; 10415 /* keep going until hitting a rollback to a frag */ 10416 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 10417 continue; 10418 dp->di_extsize = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 10419 for (i = adp->ad_offset + 1; i < NXADDR; i++) { 10420 #ifdef INVARIANTS 10421 if (dp->di_extb[i] != 0 && (deplist & (1 << i)) == 0) 10422 panic("softdep_write_inodeblock: lost dep1"); 10423 #endif /* INVARIANTS */ 10424 dp->di_extb[i] = 0; 10425 } 10426 lastadp = NULL; 10427 break; 10428 } 10429 /* 10430 * If we have zero'ed out the last allocated block of the ext 10431 * data, roll back the size to the last currently allocated block. 10432 * We know that this last allocated block is a full-sized as 10433 * we already checked for fragments in the loop above. 10434 */ 10435 if (lastadp != NULL && 10436 dp->di_extsize <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 10437 for (i = lastadp->ad_offset; i >= 0; i--) 10438 if (dp->di_extb[i] != 0) 10439 break; 10440 dp->di_extsize = (i + 1) * fs->fs_bsize; 10441 } 10442 /* 10443 * Set the file data dependencies to busy. 10444 */ 10445 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10446 adp = TAILQ_NEXT(adp, ad_next)) { 10447 #ifdef INVARIANTS 10448 if (deplist != 0 && prevlbn >= adp->ad_offset) 10449 panic("softdep_write_inodeblock: lbn order"); 10450 if ((adp->ad_state & ATTACHED) == 0) 10451 panic("inodedep %p and adp %p not attached", inodedep, adp); 10452 prevlbn = adp->ad_offset; 10453 if (adp->ad_offset < NDADDR && 10454 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 10455 panic("%s: direct pointer #%jd mismatch %jd != %jd", 10456 "softdep_write_inodeblock", 10457 (intmax_t)adp->ad_offset, 10458 (intmax_t)dp->di_db[adp->ad_offset], 10459 (intmax_t)adp->ad_newblkno); 10460 if (adp->ad_offset >= NDADDR && 10461 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno) 10462 panic("%s indirect pointer #%jd mismatch %jd != %jd", 10463 "softdep_write_inodeblock:", 10464 (intmax_t)adp->ad_offset - NDADDR, 10465 (intmax_t)dp->di_ib[adp->ad_offset - NDADDR], 10466 (intmax_t)adp->ad_newblkno); 10467 deplist |= 1 << adp->ad_offset; 10468 if ((adp->ad_state & ATTACHED) == 0) 10469 panic("softdep_write_inodeblock: Unknown state 0x%x", 10470 adp->ad_state); 10471 #endif /* INVARIANTS */ 10472 adp->ad_state &= ~ATTACHED; 10473 adp->ad_state |= UNDONE; 10474 } 10475 /* 10476 * The on-disk inode cannot claim to be any larger than the last 10477 * fragment that has been written. Otherwise, the on-disk inode 10478 * might have fragments that were not the last block in the file 10479 * which would corrupt the filesystem. 10480 */ 10481 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 10482 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 10483 if (adp->ad_offset >= NDADDR) 10484 break; 10485 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 10486 /* keep going until hitting a rollback to a frag */ 10487 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 10488 continue; 10489 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 10490 for (i = adp->ad_offset + 1; i < NDADDR; i++) { 10491 #ifdef INVARIANTS 10492 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 10493 panic("softdep_write_inodeblock: lost dep2"); 10494 #endif /* INVARIANTS */ 10495 dp->di_db[i] = 0; 10496 } 10497 for (i = 0; i < NIADDR; i++) { 10498 #ifdef INVARIANTS 10499 if (dp->di_ib[i] != 0 && 10500 (deplist & ((1 << NDADDR) << i)) == 0) 10501 panic("softdep_write_inodeblock: lost dep3"); 10502 #endif /* INVARIANTS */ 10503 dp->di_ib[i] = 0; 10504 } 10505 return; 10506 } 10507 /* 10508 * If we have zero'ed out the last allocated block of the file, 10509 * roll back the size to the last currently allocated block. 10510 * We know that this last allocated block is a full-sized as 10511 * we already checked for fragments in the loop above. 10512 */ 10513 if (lastadp != NULL && 10514 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 10515 for (i = lastadp->ad_offset; i >= 0; i--) 10516 if (dp->di_db[i] != 0) 10517 break; 10518 dp->di_size = (i + 1) * fs->fs_bsize; 10519 } 10520 /* 10521 * The only dependencies are for indirect blocks. 10522 * 10523 * The file size for indirect block additions is not guaranteed. 10524 * Such a guarantee would be non-trivial to achieve. The conventional 10525 * synchronous write implementation also does not make this guarantee. 10526 * Fsck should catch and fix discrepancies. Arguably, the file size 10527 * can be over-estimated without destroying integrity when the file 10528 * moves into the indirect blocks (i.e., is large). If we want to 10529 * postpone fsck, we are stuck with this argument. 10530 */ 10531 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 10532 dp->di_ib[adp->ad_offset - NDADDR] = 0; 10533 } 10534 10535 /* 10536 * Cancel an indirdep as a result of truncation. Release all of the 10537 * children allocindirs and place their journal work on the appropriate 10538 * list. 10539 */ 10540 static void 10541 cancel_indirdep(indirdep, bp, freeblks) 10542 struct indirdep *indirdep; 10543 struct buf *bp; 10544 struct freeblks *freeblks; 10545 { 10546 struct allocindir *aip; 10547 10548 /* 10549 * None of the indirect pointers will ever be visible, 10550 * so they can simply be tossed. GOINGAWAY ensures 10551 * that allocated pointers will be saved in the buffer 10552 * cache until they are freed. Note that they will 10553 * only be able to be found by their physical address 10554 * since the inode mapping the logical address will 10555 * be gone. The save buffer used for the safe copy 10556 * was allocated in setup_allocindir_phase2 using 10557 * the physical address so it could be used for this 10558 * purpose. Hence we swap the safe copy with the real 10559 * copy, allowing the safe copy to be freed and holding 10560 * on to the real copy for later use in indir_trunc. 10561 */ 10562 if (indirdep->ir_state & GOINGAWAY) 10563 panic("cancel_indirdep: already gone"); 10564 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 10565 indirdep->ir_state |= DEPCOMPLETE; 10566 LIST_REMOVE(indirdep, ir_next); 10567 } 10568 indirdep->ir_state |= GOINGAWAY; 10569 /* 10570 * Pass in bp for blocks still have journal writes 10571 * pending so we can cancel them on their own. 10572 */ 10573 while ((aip = LIST_FIRST(&indirdep->ir_deplisthd)) != NULL) 10574 cancel_allocindir(aip, bp, freeblks, 0); 10575 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != NULL) 10576 cancel_allocindir(aip, NULL, freeblks, 0); 10577 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != NULL) 10578 cancel_allocindir(aip, NULL, freeblks, 0); 10579 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) 10580 cancel_allocindir(aip, NULL, freeblks, 0); 10581 /* 10582 * If there are pending partial truncations we need to keep the 10583 * old block copy around until they complete. This is because 10584 * the current b_data is not a perfect superset of the available 10585 * blocks. 10586 */ 10587 if (TAILQ_EMPTY(&indirdep->ir_trunc)) 10588 bcopy(bp->b_data, indirdep->ir_savebp->b_data, bp->b_bcount); 10589 else 10590 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount); 10591 WORKLIST_REMOVE(&indirdep->ir_list); 10592 WORKLIST_INSERT(&indirdep->ir_savebp->b_dep, &indirdep->ir_list); 10593 indirdep->ir_bp = NULL; 10594 indirdep->ir_freeblks = freeblks; 10595 } 10596 10597 /* 10598 * Free an indirdep once it no longer has new pointers to track. 10599 */ 10600 static void 10601 free_indirdep(indirdep) 10602 struct indirdep *indirdep; 10603 { 10604 10605 KASSERT(TAILQ_EMPTY(&indirdep->ir_trunc), 10606 ("free_indirdep: Indir trunc list not empty.")); 10607 KASSERT(LIST_EMPTY(&indirdep->ir_completehd), 10608 ("free_indirdep: Complete head not empty.")); 10609 KASSERT(LIST_EMPTY(&indirdep->ir_writehd), 10610 ("free_indirdep: write head not empty.")); 10611 KASSERT(LIST_EMPTY(&indirdep->ir_donehd), 10612 ("free_indirdep: done head not empty.")); 10613 KASSERT(LIST_EMPTY(&indirdep->ir_deplisthd), 10614 ("free_indirdep: deplist head not empty.")); 10615 KASSERT((indirdep->ir_state & DEPCOMPLETE), 10616 ("free_indirdep: %p still on newblk list.", indirdep)); 10617 KASSERT(indirdep->ir_saveddata == NULL, 10618 ("free_indirdep: %p still has saved data.", indirdep)); 10619 if (indirdep->ir_state & ONWORKLIST) 10620 WORKLIST_REMOVE(&indirdep->ir_list); 10621 WORKITEM_FREE(indirdep, D_INDIRDEP); 10622 } 10623 10624 /* 10625 * Called before a write to an indirdep. This routine is responsible for 10626 * rolling back pointers to a safe state which includes only those 10627 * allocindirs which have been completed. 10628 */ 10629 static void 10630 initiate_write_indirdep(indirdep, bp) 10631 struct indirdep *indirdep; 10632 struct buf *bp; 10633 { 10634 struct ufsmount *ump; 10635 10636 indirdep->ir_state |= IOSTARTED; 10637 if (indirdep->ir_state & GOINGAWAY) 10638 panic("disk_io_initiation: indirdep gone"); 10639 /* 10640 * If there are no remaining dependencies, this will be writing 10641 * the real pointers. 10642 */ 10643 if (LIST_EMPTY(&indirdep->ir_deplisthd) && 10644 TAILQ_EMPTY(&indirdep->ir_trunc)) 10645 return; 10646 /* 10647 * Replace up-to-date version with safe version. 10648 */ 10649 if (indirdep->ir_saveddata == NULL) { 10650 ump = VFSTOUFS(indirdep->ir_list.wk_mp); 10651 LOCK_OWNED(ump); 10652 FREE_LOCK(ump); 10653 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP, 10654 M_SOFTDEP_FLAGS); 10655 ACQUIRE_LOCK(ump); 10656 } 10657 indirdep->ir_state &= ~ATTACHED; 10658 indirdep->ir_state |= UNDONE; 10659 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount); 10660 bcopy(indirdep->ir_savebp->b_data, bp->b_data, 10661 bp->b_bcount); 10662 } 10663 10664 /* 10665 * Called when an inode has been cleared in a cg bitmap. This finally 10666 * eliminates any canceled jaddrefs 10667 */ 10668 void 10669 softdep_setup_inofree(mp, bp, ino, wkhd) 10670 struct mount *mp; 10671 struct buf *bp; 10672 ino_t ino; 10673 struct workhead *wkhd; 10674 { 10675 struct worklist *wk, *wkn; 10676 struct inodedep *inodedep; 10677 struct ufsmount *ump; 10678 uint8_t *inosused; 10679 struct cg *cgp; 10680 struct fs *fs; 10681 10682 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 10683 ("softdep_setup_inofree called on non-softdep filesystem")); 10684 ump = VFSTOUFS(mp); 10685 ACQUIRE_LOCK(ump); 10686 fs = ump->um_fs; 10687 cgp = (struct cg *)bp->b_data; 10688 inosused = cg_inosused(cgp); 10689 if (isset(inosused, ino % fs->fs_ipg)) 10690 panic("softdep_setup_inofree: inode %ju not freed.", 10691 (uintmax_t)ino); 10692 if (inodedep_lookup(mp, ino, 0, &inodedep)) 10693 panic("softdep_setup_inofree: ino %ju has existing inodedep %p", 10694 (uintmax_t)ino, inodedep); 10695 if (wkhd) { 10696 LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) { 10697 if (wk->wk_type != D_JADDREF) 10698 continue; 10699 WORKLIST_REMOVE(wk); 10700 /* 10701 * We can free immediately even if the jaddref 10702 * isn't attached in a background write as now 10703 * the bitmaps are reconciled. 10704 */ 10705 wk->wk_state |= COMPLETE | ATTACHED; 10706 free_jaddref(WK_JADDREF(wk)); 10707 } 10708 jwork_move(&bp->b_dep, wkhd); 10709 } 10710 FREE_LOCK(ump); 10711 } 10712 10713 10714 /* 10715 * Called via ffs_blkfree() after a set of frags has been cleared from a cg 10716 * map. Any dependencies waiting for the write to clear are added to the 10717 * buf's list and any jnewblks that are being canceled are discarded 10718 * immediately. 10719 */ 10720 void 10721 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 10722 struct mount *mp; 10723 struct buf *bp; 10724 ufs2_daddr_t blkno; 10725 int frags; 10726 struct workhead *wkhd; 10727 { 10728 struct bmsafemap *bmsafemap; 10729 struct jnewblk *jnewblk; 10730 struct ufsmount *ump; 10731 struct worklist *wk; 10732 struct fs *fs; 10733 #ifdef SUJ_DEBUG 10734 uint8_t *blksfree; 10735 struct cg *cgp; 10736 ufs2_daddr_t jstart; 10737 ufs2_daddr_t jend; 10738 ufs2_daddr_t end; 10739 long bno; 10740 int i; 10741 #endif 10742 10743 CTR3(KTR_SUJ, 10744 "softdep_setup_blkfree: blkno %jd frags %d wk head %p", 10745 blkno, frags, wkhd); 10746 10747 ump = VFSTOUFS(mp); 10748 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 10749 ("softdep_setup_blkfree called on non-softdep filesystem")); 10750 ACQUIRE_LOCK(ump); 10751 /* Lookup the bmsafemap so we track when it is dirty. */ 10752 fs = ump->um_fs; 10753 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL); 10754 /* 10755 * Detach any jnewblks which have been canceled. They must linger 10756 * until the bitmap is cleared again by ffs_blkfree() to prevent 10757 * an unjournaled allocation from hitting the disk. 10758 */ 10759 if (wkhd) { 10760 while ((wk = LIST_FIRST(wkhd)) != NULL) { 10761 CTR2(KTR_SUJ, 10762 "softdep_setup_blkfree: blkno %jd wk type %d", 10763 blkno, wk->wk_type); 10764 WORKLIST_REMOVE(wk); 10765 if (wk->wk_type != D_JNEWBLK) { 10766 WORKLIST_INSERT(&bmsafemap->sm_freehd, wk); 10767 continue; 10768 } 10769 jnewblk = WK_JNEWBLK(wk); 10770 KASSERT(jnewblk->jn_state & GOINGAWAY, 10771 ("softdep_setup_blkfree: jnewblk not canceled.")); 10772 #ifdef SUJ_DEBUG 10773 /* 10774 * Assert that this block is free in the bitmap 10775 * before we discard the jnewblk. 10776 */ 10777 cgp = (struct cg *)bp->b_data; 10778 blksfree = cg_blksfree(cgp); 10779 bno = dtogd(fs, jnewblk->jn_blkno); 10780 for (i = jnewblk->jn_oldfrags; 10781 i < jnewblk->jn_frags; i++) { 10782 if (isset(blksfree, bno + i)) 10783 continue; 10784 panic("softdep_setup_blkfree: not free"); 10785 } 10786 #endif 10787 /* 10788 * Even if it's not attached we can free immediately 10789 * as the new bitmap is correct. 10790 */ 10791 wk->wk_state |= COMPLETE | ATTACHED; 10792 free_jnewblk(jnewblk); 10793 } 10794 } 10795 10796 #ifdef SUJ_DEBUG 10797 /* 10798 * Assert that we are not freeing a block which has an outstanding 10799 * allocation dependency. 10800 */ 10801 fs = VFSTOUFS(mp)->um_fs; 10802 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno), NULL); 10803 end = blkno + frags; 10804 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 10805 /* 10806 * Don't match against blocks that will be freed when the 10807 * background write is done. 10808 */ 10809 if ((jnewblk->jn_state & (ATTACHED | COMPLETE | DEPCOMPLETE)) == 10810 (COMPLETE | DEPCOMPLETE)) 10811 continue; 10812 jstart = jnewblk->jn_blkno + jnewblk->jn_oldfrags; 10813 jend = jnewblk->jn_blkno + jnewblk->jn_frags; 10814 if ((blkno >= jstart && blkno < jend) || 10815 (end > jstart && end <= jend)) { 10816 printf("state 0x%X %jd - %d %d dep %p\n", 10817 jnewblk->jn_state, jnewblk->jn_blkno, 10818 jnewblk->jn_oldfrags, jnewblk->jn_frags, 10819 jnewblk->jn_dep); 10820 panic("softdep_setup_blkfree: " 10821 "%jd-%jd(%d) overlaps with %jd-%jd", 10822 blkno, end, frags, jstart, jend); 10823 } 10824 } 10825 #endif 10826 FREE_LOCK(ump); 10827 } 10828 10829 /* 10830 * Revert a block allocation when the journal record that describes it 10831 * is not yet written. 10832 */ 10833 static int 10834 jnewblk_rollback(jnewblk, fs, cgp, blksfree) 10835 struct jnewblk *jnewblk; 10836 struct fs *fs; 10837 struct cg *cgp; 10838 uint8_t *blksfree; 10839 { 10840 ufs1_daddr_t fragno; 10841 long cgbno, bbase; 10842 int frags, blk; 10843 int i; 10844 10845 frags = 0; 10846 cgbno = dtogd(fs, jnewblk->jn_blkno); 10847 /* 10848 * We have to test which frags need to be rolled back. We may 10849 * be operating on a stale copy when doing background writes. 10850 */ 10851 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++) 10852 if (isclr(blksfree, cgbno + i)) 10853 frags++; 10854 if (frags == 0) 10855 return (0); 10856 /* 10857 * This is mostly ffs_blkfree() sans some validation and 10858 * superblock updates. 10859 */ 10860 if (frags == fs->fs_frag) { 10861 fragno = fragstoblks(fs, cgbno); 10862 ffs_setblock(fs, blksfree, fragno); 10863 ffs_clusteracct(fs, cgp, fragno, 1); 10864 cgp->cg_cs.cs_nbfree++; 10865 } else { 10866 cgbno += jnewblk->jn_oldfrags; 10867 bbase = cgbno - fragnum(fs, cgbno); 10868 /* Decrement the old frags. */ 10869 blk = blkmap(fs, blksfree, bbase); 10870 ffs_fragacct(fs, blk, cgp->cg_frsum, -1); 10871 /* Deallocate the fragment */ 10872 for (i = 0; i < frags; i++) 10873 setbit(blksfree, cgbno + i); 10874 cgp->cg_cs.cs_nffree += frags; 10875 /* Add back in counts associated with the new frags */ 10876 blk = blkmap(fs, blksfree, bbase); 10877 ffs_fragacct(fs, blk, cgp->cg_frsum, 1); 10878 /* If a complete block has been reassembled, account for it. */ 10879 fragno = fragstoblks(fs, bbase); 10880 if (ffs_isblock(fs, blksfree, fragno)) { 10881 cgp->cg_cs.cs_nffree -= fs->fs_frag; 10882 ffs_clusteracct(fs, cgp, fragno, 1); 10883 cgp->cg_cs.cs_nbfree++; 10884 } 10885 } 10886 stat_jnewblk++; 10887 jnewblk->jn_state &= ~ATTACHED; 10888 jnewblk->jn_state |= UNDONE; 10889 10890 return (frags); 10891 } 10892 10893 static void 10894 initiate_write_bmsafemap(bmsafemap, bp) 10895 struct bmsafemap *bmsafemap; 10896 struct buf *bp; /* The cg block. */ 10897 { 10898 struct jaddref *jaddref; 10899 struct jnewblk *jnewblk; 10900 uint8_t *inosused; 10901 uint8_t *blksfree; 10902 struct cg *cgp; 10903 struct fs *fs; 10904 ino_t ino; 10905 10906 /* 10907 * If this is a background write, we did this at the time that 10908 * the copy was made, so do not need to do it again. 10909 */ 10910 if (bmsafemap->sm_state & IOSTARTED) 10911 return; 10912 bmsafemap->sm_state |= IOSTARTED; 10913 /* 10914 * Clear any inode allocations which are pending journal writes. 10915 */ 10916 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd) != NULL) { 10917 cgp = (struct cg *)bp->b_data; 10918 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 10919 inosused = cg_inosused(cgp); 10920 LIST_FOREACH(jaddref, &bmsafemap->sm_jaddrefhd, ja_bmdeps) { 10921 ino = jaddref->ja_ino % fs->fs_ipg; 10922 if (isset(inosused, ino)) { 10923 if ((jaddref->ja_mode & IFMT) == IFDIR) 10924 cgp->cg_cs.cs_ndir--; 10925 cgp->cg_cs.cs_nifree++; 10926 clrbit(inosused, ino); 10927 jaddref->ja_state &= ~ATTACHED; 10928 jaddref->ja_state |= UNDONE; 10929 stat_jaddref++; 10930 } else 10931 panic("initiate_write_bmsafemap: inode %ju " 10932 "marked free", (uintmax_t)jaddref->ja_ino); 10933 } 10934 } 10935 /* 10936 * Clear any block allocations which are pending journal writes. 10937 */ 10938 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 10939 cgp = (struct cg *)bp->b_data; 10940 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 10941 blksfree = cg_blksfree(cgp); 10942 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 10943 if (jnewblk_rollback(jnewblk, fs, cgp, blksfree)) 10944 continue; 10945 panic("initiate_write_bmsafemap: block %jd " 10946 "marked free", jnewblk->jn_blkno); 10947 } 10948 } 10949 /* 10950 * Move allocation lists to the written lists so they can be 10951 * cleared once the block write is complete. 10952 */ 10953 LIST_SWAP(&bmsafemap->sm_inodedephd, &bmsafemap->sm_inodedepwr, 10954 inodedep, id_deps); 10955 LIST_SWAP(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr, 10956 newblk, nb_deps); 10957 LIST_SWAP(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, worklist, 10958 wk_list); 10959 } 10960 10961 /* 10962 * This routine is called during the completion interrupt 10963 * service routine for a disk write (from the procedure called 10964 * by the device driver to inform the filesystem caches of 10965 * a request completion). It should be called early in this 10966 * procedure, before the block is made available to other 10967 * processes or other routines are called. 10968 * 10969 */ 10970 static void 10971 softdep_disk_write_complete(bp) 10972 struct buf *bp; /* describes the completed disk write */ 10973 { 10974 struct worklist *wk; 10975 struct worklist *owk; 10976 struct ufsmount *ump; 10977 struct workhead reattach; 10978 struct freeblks *freeblks; 10979 struct buf *sbp; 10980 10981 /* 10982 * If an error occurred while doing the write, then the data 10983 * has not hit the disk and the dependencies cannot be processed. 10984 * But we do have to go through and roll forward any dependencies 10985 * that were rolled back before the disk write. 10986 */ 10987 if ((bp->b_ioflags & BIO_ERROR) != 0 && (bp->b_flags & B_INVAL) == 0) { 10988 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 10989 switch (wk->wk_type) { 10990 10991 case D_PAGEDEP: 10992 handle_written_filepage(WK_PAGEDEP(wk), bp, 0); 10993 continue; 10994 10995 case D_INODEDEP: 10996 handle_written_inodeblock(WK_INODEDEP(wk), 10997 bp, 0); 10998 continue; 10999 11000 case D_BMSAFEMAP: 11001 handle_written_bmsafemap(WK_BMSAFEMAP(wk), 11002 bp, 0); 11003 continue; 11004 11005 case D_INDIRDEP: 11006 handle_written_indirdep(WK_INDIRDEP(wk), 11007 bp, &sbp, 0); 11008 continue; 11009 default: 11010 /* nothing to roll forward */ 11011 continue; 11012 } 11013 } 11014 return; 11015 } 11016 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL) 11017 return; 11018 ump = VFSTOUFS(wk->wk_mp); 11019 LIST_INIT(&reattach); 11020 /* 11021 * This lock must not be released anywhere in this code segment. 11022 */ 11023 sbp = NULL; 11024 owk = NULL; 11025 ACQUIRE_LOCK(ump); 11026 while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) { 11027 WORKLIST_REMOVE(wk); 11028 atomic_add_long(&dep_write[wk->wk_type], 1); 11029 if (wk == owk) 11030 panic("duplicate worklist: %p\n", wk); 11031 owk = wk; 11032 switch (wk->wk_type) { 11033 11034 case D_PAGEDEP: 11035 if (handle_written_filepage(WK_PAGEDEP(wk), bp, 11036 WRITESUCCEEDED)) 11037 WORKLIST_INSERT(&reattach, wk); 11038 continue; 11039 11040 case D_INODEDEP: 11041 if (handle_written_inodeblock(WK_INODEDEP(wk), bp, 11042 WRITESUCCEEDED)) 11043 WORKLIST_INSERT(&reattach, wk); 11044 continue; 11045 11046 case D_BMSAFEMAP: 11047 if (handle_written_bmsafemap(WK_BMSAFEMAP(wk), bp, 11048 WRITESUCCEEDED)) 11049 WORKLIST_INSERT(&reattach, wk); 11050 continue; 11051 11052 case D_MKDIR: 11053 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 11054 continue; 11055 11056 case D_ALLOCDIRECT: 11057 wk->wk_state |= COMPLETE; 11058 handle_allocdirect_partdone(WK_ALLOCDIRECT(wk), NULL); 11059 continue; 11060 11061 case D_ALLOCINDIR: 11062 wk->wk_state |= COMPLETE; 11063 handle_allocindir_partdone(WK_ALLOCINDIR(wk)); 11064 continue; 11065 11066 case D_INDIRDEP: 11067 if (handle_written_indirdep(WK_INDIRDEP(wk), bp, &sbp, 11068 WRITESUCCEEDED)) 11069 WORKLIST_INSERT(&reattach, wk); 11070 continue; 11071 11072 case D_FREEBLKS: 11073 wk->wk_state |= COMPLETE; 11074 freeblks = WK_FREEBLKS(wk); 11075 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE && 11076 LIST_EMPTY(&freeblks->fb_jblkdephd)) 11077 add_to_worklist(wk, WK_NODELAY); 11078 continue; 11079 11080 case D_FREEWORK: 11081 handle_written_freework(WK_FREEWORK(wk)); 11082 break; 11083 11084 case D_JSEGDEP: 11085 free_jsegdep(WK_JSEGDEP(wk)); 11086 continue; 11087 11088 case D_JSEG: 11089 handle_written_jseg(WK_JSEG(wk), bp); 11090 continue; 11091 11092 case D_SBDEP: 11093 if (handle_written_sbdep(WK_SBDEP(wk), bp)) 11094 WORKLIST_INSERT(&reattach, wk); 11095 continue; 11096 11097 case D_FREEDEP: 11098 free_freedep(WK_FREEDEP(wk)); 11099 continue; 11100 11101 default: 11102 panic("handle_disk_write_complete: Unknown type %s", 11103 TYPENAME(wk->wk_type)); 11104 /* NOTREACHED */ 11105 } 11106 } 11107 /* 11108 * Reattach any requests that must be redone. 11109 */ 11110 while ((wk = LIST_FIRST(&reattach)) != NULL) { 11111 WORKLIST_REMOVE(wk); 11112 WORKLIST_INSERT(&bp->b_dep, wk); 11113 } 11114 FREE_LOCK(ump); 11115 if (sbp) 11116 brelse(sbp); 11117 } 11118 11119 /* 11120 * Called from within softdep_disk_write_complete above. Note that 11121 * this routine is always called from interrupt level with further 11122 * splbio interrupts blocked. 11123 */ 11124 static void 11125 handle_allocdirect_partdone(adp, wkhd) 11126 struct allocdirect *adp; /* the completed allocdirect */ 11127 struct workhead *wkhd; /* Work to do when inode is writtne. */ 11128 { 11129 struct allocdirectlst *listhead; 11130 struct allocdirect *listadp; 11131 struct inodedep *inodedep; 11132 long bsize; 11133 11134 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 11135 return; 11136 /* 11137 * The on-disk inode cannot claim to be any larger than the last 11138 * fragment that has been written. Otherwise, the on-disk inode 11139 * might have fragments that were not the last block in the file 11140 * which would corrupt the filesystem. Thus, we cannot free any 11141 * allocdirects after one whose ad_oldblkno claims a fragment as 11142 * these blocks must be rolled back to zero before writing the inode. 11143 * We check the currently active set of allocdirects in id_inoupdt 11144 * or id_extupdt as appropriate. 11145 */ 11146 inodedep = adp->ad_inodedep; 11147 bsize = inodedep->id_fs->fs_bsize; 11148 if (adp->ad_state & EXTDATA) 11149 listhead = &inodedep->id_extupdt; 11150 else 11151 listhead = &inodedep->id_inoupdt; 11152 TAILQ_FOREACH(listadp, listhead, ad_next) { 11153 /* found our block */ 11154 if (listadp == adp) 11155 break; 11156 /* continue if ad_oldlbn is not a fragment */ 11157 if (listadp->ad_oldsize == 0 || 11158 listadp->ad_oldsize == bsize) 11159 continue; 11160 /* hit a fragment */ 11161 return; 11162 } 11163 /* 11164 * If we have reached the end of the current list without 11165 * finding the just finished dependency, then it must be 11166 * on the future dependency list. Future dependencies cannot 11167 * be freed until they are moved to the current list. 11168 */ 11169 if (listadp == NULL) { 11170 #ifdef DEBUG 11171 if (adp->ad_state & EXTDATA) 11172 listhead = &inodedep->id_newextupdt; 11173 else 11174 listhead = &inodedep->id_newinoupdt; 11175 TAILQ_FOREACH(listadp, listhead, ad_next) 11176 /* found our block */ 11177 if (listadp == adp) 11178 break; 11179 if (listadp == NULL) 11180 panic("handle_allocdirect_partdone: lost dep"); 11181 #endif /* DEBUG */ 11182 return; 11183 } 11184 /* 11185 * If we have found the just finished dependency, then queue 11186 * it along with anything that follows it that is complete. 11187 * Since the pointer has not yet been written in the inode 11188 * as the dependency prevents it, place the allocdirect on the 11189 * bufwait list where it will be freed once the pointer is 11190 * valid. 11191 */ 11192 if (wkhd == NULL) 11193 wkhd = &inodedep->id_bufwait; 11194 for (; adp; adp = listadp) { 11195 listadp = TAILQ_NEXT(adp, ad_next); 11196 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 11197 return; 11198 TAILQ_REMOVE(listhead, adp, ad_next); 11199 WORKLIST_INSERT(wkhd, &adp->ad_block.nb_list); 11200 } 11201 } 11202 11203 /* 11204 * Called from within softdep_disk_write_complete above. This routine 11205 * completes successfully written allocindirs. 11206 */ 11207 static void 11208 handle_allocindir_partdone(aip) 11209 struct allocindir *aip; /* the completed allocindir */ 11210 { 11211 struct indirdep *indirdep; 11212 11213 if ((aip->ai_state & ALLCOMPLETE) != ALLCOMPLETE) 11214 return; 11215 indirdep = aip->ai_indirdep; 11216 LIST_REMOVE(aip, ai_next); 11217 /* 11218 * Don't set a pointer while the buffer is undergoing IO or while 11219 * we have active truncations. 11220 */ 11221 if (indirdep->ir_state & UNDONE || !TAILQ_EMPTY(&indirdep->ir_trunc)) { 11222 LIST_INSERT_HEAD(&indirdep->ir_donehd, aip, ai_next); 11223 return; 11224 } 11225 if (indirdep->ir_state & UFS1FMT) 11226 ((ufs1_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 11227 aip->ai_newblkno; 11228 else 11229 ((ufs2_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 11230 aip->ai_newblkno; 11231 /* 11232 * Await the pointer write before freeing the allocindir. 11233 */ 11234 LIST_INSERT_HEAD(&indirdep->ir_writehd, aip, ai_next); 11235 } 11236 11237 /* 11238 * Release segments held on a jwork list. 11239 */ 11240 static void 11241 handle_jwork(wkhd) 11242 struct workhead *wkhd; 11243 { 11244 struct worklist *wk; 11245 11246 while ((wk = LIST_FIRST(wkhd)) != NULL) { 11247 WORKLIST_REMOVE(wk); 11248 switch (wk->wk_type) { 11249 case D_JSEGDEP: 11250 free_jsegdep(WK_JSEGDEP(wk)); 11251 continue; 11252 case D_FREEDEP: 11253 free_freedep(WK_FREEDEP(wk)); 11254 continue; 11255 case D_FREEFRAG: 11256 rele_jseg(WK_JSEG(WK_FREEFRAG(wk)->ff_jdep)); 11257 WORKITEM_FREE(wk, D_FREEFRAG); 11258 continue; 11259 case D_FREEWORK: 11260 handle_written_freework(WK_FREEWORK(wk)); 11261 continue; 11262 default: 11263 panic("handle_jwork: Unknown type %s\n", 11264 TYPENAME(wk->wk_type)); 11265 } 11266 } 11267 } 11268 11269 /* 11270 * Handle the bufwait list on an inode when it is safe to release items 11271 * held there. This normally happens after an inode block is written but 11272 * may be delayed and handled later if there are pending journal items that 11273 * are not yet safe to be released. 11274 */ 11275 static struct freefile * 11276 handle_bufwait(inodedep, refhd) 11277 struct inodedep *inodedep; 11278 struct workhead *refhd; 11279 { 11280 struct jaddref *jaddref; 11281 struct freefile *freefile; 11282 struct worklist *wk; 11283 11284 freefile = NULL; 11285 while ((wk = LIST_FIRST(&inodedep->id_bufwait)) != NULL) { 11286 WORKLIST_REMOVE(wk); 11287 switch (wk->wk_type) { 11288 case D_FREEFILE: 11289 /* 11290 * We defer adding freefile to the worklist 11291 * until all other additions have been made to 11292 * ensure that it will be done after all the 11293 * old blocks have been freed. 11294 */ 11295 if (freefile != NULL) 11296 panic("handle_bufwait: freefile"); 11297 freefile = WK_FREEFILE(wk); 11298 continue; 11299 11300 case D_MKDIR: 11301 handle_written_mkdir(WK_MKDIR(wk), MKDIR_PARENT); 11302 continue; 11303 11304 case D_DIRADD: 11305 diradd_inode_written(WK_DIRADD(wk), inodedep); 11306 continue; 11307 11308 case D_FREEFRAG: 11309 wk->wk_state |= COMPLETE; 11310 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE) 11311 add_to_worklist(wk, 0); 11312 continue; 11313 11314 case D_DIRREM: 11315 wk->wk_state |= COMPLETE; 11316 add_to_worklist(wk, 0); 11317 continue; 11318 11319 case D_ALLOCDIRECT: 11320 case D_ALLOCINDIR: 11321 free_newblk(WK_NEWBLK(wk)); 11322 continue; 11323 11324 case D_JNEWBLK: 11325 wk->wk_state |= COMPLETE; 11326 free_jnewblk(WK_JNEWBLK(wk)); 11327 continue; 11328 11329 /* 11330 * Save freed journal segments and add references on 11331 * the supplied list which will delay their release 11332 * until the cg bitmap is cleared on disk. 11333 */ 11334 case D_JSEGDEP: 11335 if (refhd == NULL) 11336 free_jsegdep(WK_JSEGDEP(wk)); 11337 else 11338 WORKLIST_INSERT(refhd, wk); 11339 continue; 11340 11341 case D_JADDREF: 11342 jaddref = WK_JADDREF(wk); 11343 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 11344 if_deps); 11345 /* 11346 * Transfer any jaddrefs to the list to be freed with 11347 * the bitmap if we're handling a removed file. 11348 */ 11349 if (refhd == NULL) { 11350 wk->wk_state |= COMPLETE; 11351 free_jaddref(jaddref); 11352 } else 11353 WORKLIST_INSERT(refhd, wk); 11354 continue; 11355 11356 default: 11357 panic("handle_bufwait: Unknown type %p(%s)", 11358 wk, TYPENAME(wk->wk_type)); 11359 /* NOTREACHED */ 11360 } 11361 } 11362 return (freefile); 11363 } 11364 /* 11365 * Called from within softdep_disk_write_complete above to restore 11366 * in-memory inode block contents to their most up-to-date state. Note 11367 * that this routine is always called from interrupt level with further 11368 * interrupts from this device blocked. 11369 * 11370 * If the write did not succeed, we will do all the roll-forward 11371 * operations, but we will not take the actions that will allow its 11372 * dependencies to be processed. 11373 */ 11374 static int 11375 handle_written_inodeblock(inodedep, bp, flags) 11376 struct inodedep *inodedep; 11377 struct buf *bp; /* buffer containing the inode block */ 11378 int flags; 11379 { 11380 struct freefile *freefile; 11381 struct allocdirect *adp, *nextadp; 11382 struct ufs1_dinode *dp1 = NULL; 11383 struct ufs2_dinode *dp2 = NULL; 11384 struct workhead wkhd; 11385 int hadchanges, fstype; 11386 ino_t freelink; 11387 11388 LIST_INIT(&wkhd); 11389 hadchanges = 0; 11390 freefile = NULL; 11391 if ((inodedep->id_state & IOSTARTED) == 0) 11392 panic("handle_written_inodeblock: not started"); 11393 inodedep->id_state &= ~IOSTARTED; 11394 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) { 11395 fstype = UFS1; 11396 dp1 = (struct ufs1_dinode *)bp->b_data + 11397 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 11398 freelink = dp1->di_freelink; 11399 } else { 11400 fstype = UFS2; 11401 dp2 = (struct ufs2_dinode *)bp->b_data + 11402 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 11403 freelink = dp2->di_freelink; 11404 } 11405 /* 11406 * Leave this inodeblock dirty until it's in the list. 11407 */ 11408 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) == UNLINKED && 11409 (flags & WRITESUCCEEDED)) { 11410 struct inodedep *inon; 11411 11412 inon = TAILQ_NEXT(inodedep, id_unlinked); 11413 if ((inon == NULL && freelink == 0) || 11414 (inon && inon->id_ino == freelink)) { 11415 if (inon) 11416 inon->id_state |= UNLINKPREV; 11417 inodedep->id_state |= UNLINKNEXT; 11418 } 11419 hadchanges = 1; 11420 } 11421 /* 11422 * If we had to rollback the inode allocation because of 11423 * bitmaps being incomplete, then simply restore it. 11424 * Keep the block dirty so that it will not be reclaimed until 11425 * all associated dependencies have been cleared and the 11426 * corresponding updates written to disk. 11427 */ 11428 if (inodedep->id_savedino1 != NULL) { 11429 hadchanges = 1; 11430 if (fstype == UFS1) 11431 *dp1 = *inodedep->id_savedino1; 11432 else 11433 *dp2 = *inodedep->id_savedino2; 11434 free(inodedep->id_savedino1, M_SAVEDINO); 11435 inodedep->id_savedino1 = NULL; 11436 if ((bp->b_flags & B_DELWRI) == 0) 11437 stat_inode_bitmap++; 11438 bdirty(bp); 11439 /* 11440 * If the inode is clear here and GOINGAWAY it will never 11441 * be written. Process the bufwait and clear any pending 11442 * work which may include the freefile. 11443 */ 11444 if (inodedep->id_state & GOINGAWAY) 11445 goto bufwait; 11446 return (1); 11447 } 11448 if (flags & WRITESUCCEEDED) 11449 inodedep->id_state |= COMPLETE; 11450 /* 11451 * Roll forward anything that had to be rolled back before 11452 * the inode could be updated. 11453 */ 11454 for (adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; adp = nextadp) { 11455 nextadp = TAILQ_NEXT(adp, ad_next); 11456 if (adp->ad_state & ATTACHED) 11457 panic("handle_written_inodeblock: new entry"); 11458 if (fstype == UFS1) { 11459 if (adp->ad_offset < NDADDR) { 11460 if (dp1->di_db[adp->ad_offset]!=adp->ad_oldblkno) 11461 panic("%s %s #%jd mismatch %d != %jd", 11462 "handle_written_inodeblock:", 11463 "direct pointer", 11464 (intmax_t)adp->ad_offset, 11465 dp1->di_db[adp->ad_offset], 11466 (intmax_t)adp->ad_oldblkno); 11467 dp1->di_db[adp->ad_offset] = adp->ad_newblkno; 11468 } else { 11469 if (dp1->di_ib[adp->ad_offset - NDADDR] != 0) 11470 panic("%s: %s #%jd allocated as %d", 11471 "handle_written_inodeblock", 11472 "indirect pointer", 11473 (intmax_t)adp->ad_offset - NDADDR, 11474 dp1->di_ib[adp->ad_offset - NDADDR]); 11475 dp1->di_ib[adp->ad_offset - NDADDR] = 11476 adp->ad_newblkno; 11477 } 11478 } else { 11479 if (adp->ad_offset < NDADDR) { 11480 if (dp2->di_db[adp->ad_offset]!=adp->ad_oldblkno) 11481 panic("%s: %s #%jd %s %jd != %jd", 11482 "handle_written_inodeblock", 11483 "direct pointer", 11484 (intmax_t)adp->ad_offset, "mismatch", 11485 (intmax_t)dp2->di_db[adp->ad_offset], 11486 (intmax_t)adp->ad_oldblkno); 11487 dp2->di_db[adp->ad_offset] = adp->ad_newblkno; 11488 } else { 11489 if (dp2->di_ib[adp->ad_offset - NDADDR] != 0) 11490 panic("%s: %s #%jd allocated as %jd", 11491 "handle_written_inodeblock", 11492 "indirect pointer", 11493 (intmax_t)adp->ad_offset - NDADDR, 11494 (intmax_t) 11495 dp2->di_ib[adp->ad_offset - NDADDR]); 11496 dp2->di_ib[adp->ad_offset - NDADDR] = 11497 adp->ad_newblkno; 11498 } 11499 } 11500 adp->ad_state &= ~UNDONE; 11501 adp->ad_state |= ATTACHED; 11502 hadchanges = 1; 11503 } 11504 for (adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; adp = nextadp) { 11505 nextadp = TAILQ_NEXT(adp, ad_next); 11506 if (adp->ad_state & ATTACHED) 11507 panic("handle_written_inodeblock: new entry"); 11508 if (dp2->di_extb[adp->ad_offset] != adp->ad_oldblkno) 11509 panic("%s: direct pointers #%jd %s %jd != %jd", 11510 "handle_written_inodeblock", 11511 (intmax_t)adp->ad_offset, "mismatch", 11512 (intmax_t)dp2->di_extb[adp->ad_offset], 11513 (intmax_t)adp->ad_oldblkno); 11514 dp2->di_extb[adp->ad_offset] = adp->ad_newblkno; 11515 adp->ad_state &= ~UNDONE; 11516 adp->ad_state |= ATTACHED; 11517 hadchanges = 1; 11518 } 11519 if (hadchanges && (bp->b_flags & B_DELWRI) == 0) 11520 stat_direct_blk_ptrs++; 11521 /* 11522 * Reset the file size to its most up-to-date value. 11523 */ 11524 if (inodedep->id_savedsize == -1 || inodedep->id_savedextsize == -1) 11525 panic("handle_written_inodeblock: bad size"); 11526 if (inodedep->id_savednlink > LINK_MAX) 11527 panic("handle_written_inodeblock: Invalid link count " 11528 "%d for inodedep %p", inodedep->id_savednlink, inodedep); 11529 if (fstype == UFS1) { 11530 if (dp1->di_nlink != inodedep->id_savednlink) { 11531 dp1->di_nlink = inodedep->id_savednlink; 11532 hadchanges = 1; 11533 } 11534 if (dp1->di_size != inodedep->id_savedsize) { 11535 dp1->di_size = inodedep->id_savedsize; 11536 hadchanges = 1; 11537 } 11538 } else { 11539 if (dp2->di_nlink != inodedep->id_savednlink) { 11540 dp2->di_nlink = inodedep->id_savednlink; 11541 hadchanges = 1; 11542 } 11543 if (dp2->di_size != inodedep->id_savedsize) { 11544 dp2->di_size = inodedep->id_savedsize; 11545 hadchanges = 1; 11546 } 11547 if (dp2->di_extsize != inodedep->id_savedextsize) { 11548 dp2->di_extsize = inodedep->id_savedextsize; 11549 hadchanges = 1; 11550 } 11551 } 11552 inodedep->id_savedsize = -1; 11553 inodedep->id_savedextsize = -1; 11554 inodedep->id_savednlink = -1; 11555 /* 11556 * If there were any rollbacks in the inode block, then it must be 11557 * marked dirty so that its will eventually get written back in 11558 * its correct form. 11559 */ 11560 if (hadchanges) 11561 bdirty(bp); 11562 bufwait: 11563 /* 11564 * If the write did not succeed, we have done all the roll-forward 11565 * operations, but we cannot take the actions that will allow its 11566 * dependencies to be processed. 11567 */ 11568 if ((flags & WRITESUCCEEDED) == 0) 11569 return (hadchanges); 11570 /* 11571 * Process any allocdirects that completed during the update. 11572 */ 11573 if ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL) 11574 handle_allocdirect_partdone(adp, &wkhd); 11575 if ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 11576 handle_allocdirect_partdone(adp, &wkhd); 11577 /* 11578 * Process deallocations that were held pending until the 11579 * inode had been written to disk. Freeing of the inode 11580 * is delayed until after all blocks have been freed to 11581 * avoid creation of new <vfsid, inum, lbn> triples 11582 * before the old ones have been deleted. Completely 11583 * unlinked inodes are not processed until the unlinked 11584 * inode list is written or the last reference is removed. 11585 */ 11586 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) != UNLINKED) { 11587 freefile = handle_bufwait(inodedep, NULL); 11588 if (freefile && !LIST_EMPTY(&wkhd)) { 11589 WORKLIST_INSERT(&wkhd, &freefile->fx_list); 11590 freefile = NULL; 11591 } 11592 } 11593 /* 11594 * Move rolled forward dependency completions to the bufwait list 11595 * now that those that were already written have been processed. 11596 */ 11597 if (!LIST_EMPTY(&wkhd) && hadchanges == 0) 11598 panic("handle_written_inodeblock: bufwait but no changes"); 11599 jwork_move(&inodedep->id_bufwait, &wkhd); 11600 11601 if (freefile != NULL) { 11602 /* 11603 * If the inode is goingaway it was never written. Fake up 11604 * the state here so free_inodedep() can succeed. 11605 */ 11606 if (inodedep->id_state & GOINGAWAY) 11607 inodedep->id_state |= COMPLETE | DEPCOMPLETE; 11608 if (free_inodedep(inodedep) == 0) 11609 panic("handle_written_inodeblock: live inodedep %p", 11610 inodedep); 11611 add_to_worklist(&freefile->fx_list, 0); 11612 return (0); 11613 } 11614 11615 /* 11616 * If no outstanding dependencies, free it. 11617 */ 11618 if (free_inodedep(inodedep) || 11619 (TAILQ_FIRST(&inodedep->id_inoreflst) == 0 && 11620 TAILQ_FIRST(&inodedep->id_inoupdt) == 0 && 11621 TAILQ_FIRST(&inodedep->id_extupdt) == 0 && 11622 LIST_FIRST(&inodedep->id_bufwait) == 0)) 11623 return (0); 11624 return (hadchanges); 11625 } 11626 11627 /* 11628 * Perform needed roll-forwards and kick off any dependencies that 11629 * can now be processed. 11630 * 11631 * If the write did not succeed, we will do all the roll-forward 11632 * operations, but we will not take the actions that will allow its 11633 * dependencies to be processed. 11634 */ 11635 static int 11636 handle_written_indirdep(indirdep, bp, bpp, flags) 11637 struct indirdep *indirdep; 11638 struct buf *bp; 11639 struct buf **bpp; 11640 int flags; 11641 { 11642 struct allocindir *aip; 11643 struct buf *sbp; 11644 int chgs; 11645 11646 if (indirdep->ir_state & GOINGAWAY) 11647 panic("handle_written_indirdep: indirdep gone"); 11648 if ((indirdep->ir_state & IOSTARTED) == 0) 11649 panic("handle_written_indirdep: IO not started"); 11650 chgs = 0; 11651 /* 11652 * If there were rollbacks revert them here. 11653 */ 11654 if (indirdep->ir_saveddata) { 11655 bcopy(indirdep->ir_saveddata, bp->b_data, bp->b_bcount); 11656 if (TAILQ_EMPTY(&indirdep->ir_trunc)) { 11657 free(indirdep->ir_saveddata, M_INDIRDEP); 11658 indirdep->ir_saveddata = NULL; 11659 } 11660 chgs = 1; 11661 } 11662 indirdep->ir_state &= ~(UNDONE | IOSTARTED); 11663 indirdep->ir_state |= ATTACHED; 11664 /* 11665 * If the write did not succeed, we have done all the roll-forward 11666 * operations, but we cannot take the actions that will allow its 11667 * dependencies to be processed. 11668 */ 11669 if ((flags & WRITESUCCEEDED) == 0) { 11670 stat_indir_blk_ptrs++; 11671 bdirty(bp); 11672 return (1); 11673 } 11674 /* 11675 * Move allocindirs with written pointers to the completehd if 11676 * the indirdep's pointer is not yet written. Otherwise 11677 * free them here. 11678 */ 11679 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != NULL) { 11680 LIST_REMOVE(aip, ai_next); 11681 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 11682 LIST_INSERT_HEAD(&indirdep->ir_completehd, aip, 11683 ai_next); 11684 newblk_freefrag(&aip->ai_block); 11685 continue; 11686 } 11687 free_newblk(&aip->ai_block); 11688 } 11689 /* 11690 * Move allocindirs that have finished dependency processing from 11691 * the done list to the write list after updating the pointers. 11692 */ 11693 if (TAILQ_EMPTY(&indirdep->ir_trunc)) { 11694 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != NULL) { 11695 handle_allocindir_partdone(aip); 11696 if (aip == LIST_FIRST(&indirdep->ir_donehd)) 11697 panic("disk_write_complete: not gone"); 11698 chgs = 1; 11699 } 11700 } 11701 /* 11702 * Preserve the indirdep if there were any changes or if it is not 11703 * yet valid on disk. 11704 */ 11705 if (chgs) { 11706 stat_indir_blk_ptrs++; 11707 bdirty(bp); 11708 return (1); 11709 } 11710 /* 11711 * If there were no changes we can discard the savedbp and detach 11712 * ourselves from the buf. We are only carrying completed pointers 11713 * in this case. 11714 */ 11715 sbp = indirdep->ir_savebp; 11716 sbp->b_flags |= B_INVAL | B_NOCACHE; 11717 indirdep->ir_savebp = NULL; 11718 indirdep->ir_bp = NULL; 11719 if (*bpp != NULL) 11720 panic("handle_written_indirdep: bp already exists."); 11721 *bpp = sbp; 11722 /* 11723 * The indirdep may not be freed until its parent points at it. 11724 */ 11725 if (indirdep->ir_state & DEPCOMPLETE) 11726 free_indirdep(indirdep); 11727 11728 return (0); 11729 } 11730 11731 /* 11732 * Process a diradd entry after its dependent inode has been written. 11733 * This routine must be called with splbio interrupts blocked. 11734 */ 11735 static void 11736 diradd_inode_written(dap, inodedep) 11737 struct diradd *dap; 11738 struct inodedep *inodedep; 11739 { 11740 11741 dap->da_state |= COMPLETE; 11742 complete_diradd(dap); 11743 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 11744 } 11745 11746 /* 11747 * Returns true if the bmsafemap will have rollbacks when written. Must only 11748 * be called with the per-filesystem lock and the buf lock on the cg held. 11749 */ 11750 static int 11751 bmsafemap_backgroundwrite(bmsafemap, bp) 11752 struct bmsafemap *bmsafemap; 11753 struct buf *bp; 11754 { 11755 int dirty; 11756 11757 LOCK_OWNED(VFSTOUFS(bmsafemap->sm_list.wk_mp)); 11758 dirty = !LIST_EMPTY(&bmsafemap->sm_jaddrefhd) | 11759 !LIST_EMPTY(&bmsafemap->sm_jnewblkhd); 11760 /* 11761 * If we're initiating a background write we need to process the 11762 * rollbacks as they exist now, not as they exist when IO starts. 11763 * No other consumers will look at the contents of the shadowed 11764 * buf so this is safe to do here. 11765 */ 11766 if (bp->b_xflags & BX_BKGRDMARKER) 11767 initiate_write_bmsafemap(bmsafemap, bp); 11768 11769 return (dirty); 11770 } 11771 11772 /* 11773 * Re-apply an allocation when a cg write is complete. 11774 */ 11775 static int 11776 jnewblk_rollforward(jnewblk, fs, cgp, blksfree) 11777 struct jnewblk *jnewblk; 11778 struct fs *fs; 11779 struct cg *cgp; 11780 uint8_t *blksfree; 11781 { 11782 ufs1_daddr_t fragno; 11783 ufs2_daddr_t blkno; 11784 long cgbno, bbase; 11785 int frags, blk; 11786 int i; 11787 11788 frags = 0; 11789 cgbno = dtogd(fs, jnewblk->jn_blkno); 11790 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; i++) { 11791 if (isclr(blksfree, cgbno + i)) 11792 panic("jnewblk_rollforward: re-allocated fragment"); 11793 frags++; 11794 } 11795 if (frags == fs->fs_frag) { 11796 blkno = fragstoblks(fs, cgbno); 11797 ffs_clrblock(fs, blksfree, (long)blkno); 11798 ffs_clusteracct(fs, cgp, blkno, -1); 11799 cgp->cg_cs.cs_nbfree--; 11800 } else { 11801 bbase = cgbno - fragnum(fs, cgbno); 11802 cgbno += jnewblk->jn_oldfrags; 11803 /* If a complete block had been reassembled, account for it. */ 11804 fragno = fragstoblks(fs, bbase); 11805 if (ffs_isblock(fs, blksfree, fragno)) { 11806 cgp->cg_cs.cs_nffree += fs->fs_frag; 11807 ffs_clusteracct(fs, cgp, fragno, -1); 11808 cgp->cg_cs.cs_nbfree--; 11809 } 11810 /* Decrement the old frags. */ 11811 blk = blkmap(fs, blksfree, bbase); 11812 ffs_fragacct(fs, blk, cgp->cg_frsum, -1); 11813 /* Allocate the fragment */ 11814 for (i = 0; i < frags; i++) 11815 clrbit(blksfree, cgbno + i); 11816 cgp->cg_cs.cs_nffree -= frags; 11817 /* Add back in counts associated with the new frags */ 11818 blk = blkmap(fs, blksfree, bbase); 11819 ffs_fragacct(fs, blk, cgp->cg_frsum, 1); 11820 } 11821 return (frags); 11822 } 11823 11824 /* 11825 * Complete a write to a bmsafemap structure. Roll forward any bitmap 11826 * changes if it's not a background write. Set all written dependencies 11827 * to DEPCOMPLETE and free the structure if possible. 11828 * 11829 * If the write did not succeed, we will do all the roll-forward 11830 * operations, but we will not take the actions that will allow its 11831 * dependencies to be processed. 11832 */ 11833 static int 11834 handle_written_bmsafemap(bmsafemap, bp, flags) 11835 struct bmsafemap *bmsafemap; 11836 struct buf *bp; 11837 int flags; 11838 { 11839 struct newblk *newblk; 11840 struct inodedep *inodedep; 11841 struct jaddref *jaddref, *jatmp; 11842 struct jnewblk *jnewblk, *jntmp; 11843 struct ufsmount *ump; 11844 uint8_t *inosused; 11845 uint8_t *blksfree; 11846 struct cg *cgp; 11847 struct fs *fs; 11848 ino_t ino; 11849 int foreground; 11850 int chgs; 11851 11852 if ((bmsafemap->sm_state & IOSTARTED) == 0) 11853 panic("handle_written_bmsafemap: Not started\n"); 11854 ump = VFSTOUFS(bmsafemap->sm_list.wk_mp); 11855 chgs = 0; 11856 bmsafemap->sm_state &= ~IOSTARTED; 11857 foreground = (bp->b_xflags & BX_BKGRDMARKER) == 0; 11858 /* 11859 * If write was successful, release journal work that was waiting 11860 * on the write. Otherwise move the work back. 11861 */ 11862 if (flags & WRITESUCCEEDED) 11863 handle_jwork(&bmsafemap->sm_freewr); 11864 else 11865 LIST_CONCAT(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, 11866 worklist, wk_list); 11867 11868 /* 11869 * Restore unwritten inode allocation pending jaddref writes. 11870 */ 11871 if (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd)) { 11872 cgp = (struct cg *)bp->b_data; 11873 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 11874 inosused = cg_inosused(cgp); 11875 LIST_FOREACH_SAFE(jaddref, &bmsafemap->sm_jaddrefhd, 11876 ja_bmdeps, jatmp) { 11877 if ((jaddref->ja_state & UNDONE) == 0) 11878 continue; 11879 ino = jaddref->ja_ino % fs->fs_ipg; 11880 if (isset(inosused, ino)) 11881 panic("handle_written_bmsafemap: " 11882 "re-allocated inode"); 11883 /* Do the roll-forward only if it's a real copy. */ 11884 if (foreground) { 11885 if ((jaddref->ja_mode & IFMT) == IFDIR) 11886 cgp->cg_cs.cs_ndir++; 11887 cgp->cg_cs.cs_nifree--; 11888 setbit(inosused, ino); 11889 chgs = 1; 11890 } 11891 jaddref->ja_state &= ~UNDONE; 11892 jaddref->ja_state |= ATTACHED; 11893 free_jaddref(jaddref); 11894 } 11895 } 11896 /* 11897 * Restore any block allocations which are pending journal writes. 11898 */ 11899 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 11900 cgp = (struct cg *)bp->b_data; 11901 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 11902 blksfree = cg_blksfree(cgp); 11903 LIST_FOREACH_SAFE(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps, 11904 jntmp) { 11905 if ((jnewblk->jn_state & UNDONE) == 0) 11906 continue; 11907 /* Do the roll-forward only if it's a real copy. */ 11908 if (foreground && 11909 jnewblk_rollforward(jnewblk, fs, cgp, blksfree)) 11910 chgs = 1; 11911 jnewblk->jn_state &= ~(UNDONE | NEWBLOCK); 11912 jnewblk->jn_state |= ATTACHED; 11913 free_jnewblk(jnewblk); 11914 } 11915 } 11916 /* 11917 * If the write did not succeed, we have done all the roll-forward 11918 * operations, but we cannot take the actions that will allow its 11919 * dependencies to be processed. 11920 */ 11921 if ((flags & WRITESUCCEEDED) == 0) { 11922 LIST_CONCAT(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr, 11923 newblk, nb_deps); 11924 LIST_CONCAT(&bmsafemap->sm_freehd, &bmsafemap->sm_freewr, 11925 worklist, wk_list); 11926 if (foreground) 11927 bdirty(bp); 11928 return (1); 11929 } 11930 while ((newblk = LIST_FIRST(&bmsafemap->sm_newblkwr))) { 11931 newblk->nb_state |= DEPCOMPLETE; 11932 newblk->nb_state &= ~ONDEPLIST; 11933 newblk->nb_bmsafemap = NULL; 11934 LIST_REMOVE(newblk, nb_deps); 11935 if (newblk->nb_list.wk_type == D_ALLOCDIRECT) 11936 handle_allocdirect_partdone( 11937 WK_ALLOCDIRECT(&newblk->nb_list), NULL); 11938 else if (newblk->nb_list.wk_type == D_ALLOCINDIR) 11939 handle_allocindir_partdone( 11940 WK_ALLOCINDIR(&newblk->nb_list)); 11941 else if (newblk->nb_list.wk_type != D_NEWBLK) 11942 panic("handle_written_bmsafemap: Unexpected type: %s", 11943 TYPENAME(newblk->nb_list.wk_type)); 11944 } 11945 while ((inodedep = LIST_FIRST(&bmsafemap->sm_inodedepwr)) != NULL) { 11946 inodedep->id_state |= DEPCOMPLETE; 11947 inodedep->id_state &= ~ONDEPLIST; 11948 LIST_REMOVE(inodedep, id_deps); 11949 inodedep->id_bmsafemap = NULL; 11950 } 11951 LIST_REMOVE(bmsafemap, sm_next); 11952 if (chgs == 0 && LIST_EMPTY(&bmsafemap->sm_jaddrefhd) && 11953 LIST_EMPTY(&bmsafemap->sm_jnewblkhd) && 11954 LIST_EMPTY(&bmsafemap->sm_newblkhd) && 11955 LIST_EMPTY(&bmsafemap->sm_inodedephd) && 11956 LIST_EMPTY(&bmsafemap->sm_freehd)) { 11957 LIST_REMOVE(bmsafemap, sm_hash); 11958 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 11959 return (0); 11960 } 11961 LIST_INSERT_HEAD(&ump->softdep_dirtycg, bmsafemap, sm_next); 11962 if (foreground) 11963 bdirty(bp); 11964 return (1); 11965 } 11966 11967 /* 11968 * Try to free a mkdir dependency. 11969 */ 11970 static void 11971 complete_mkdir(mkdir) 11972 struct mkdir *mkdir; 11973 { 11974 struct diradd *dap; 11975 11976 if ((mkdir->md_state & ALLCOMPLETE) != ALLCOMPLETE) 11977 return; 11978 LIST_REMOVE(mkdir, md_mkdirs); 11979 dap = mkdir->md_diradd; 11980 dap->da_state &= ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 11981 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) { 11982 dap->da_state |= DEPCOMPLETE; 11983 complete_diradd(dap); 11984 } 11985 WORKITEM_FREE(mkdir, D_MKDIR); 11986 } 11987 11988 /* 11989 * Handle the completion of a mkdir dependency. 11990 */ 11991 static void 11992 handle_written_mkdir(mkdir, type) 11993 struct mkdir *mkdir; 11994 int type; 11995 { 11996 11997 if ((mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)) != type) 11998 panic("handle_written_mkdir: bad type"); 11999 mkdir->md_state |= COMPLETE; 12000 complete_mkdir(mkdir); 12001 } 12002 12003 static int 12004 free_pagedep(pagedep) 12005 struct pagedep *pagedep; 12006 { 12007 int i; 12008 12009 if (pagedep->pd_state & NEWBLOCK) 12010 return (0); 12011 if (!LIST_EMPTY(&pagedep->pd_dirremhd)) 12012 return (0); 12013 for (i = 0; i < DAHASHSZ; i++) 12014 if (!LIST_EMPTY(&pagedep->pd_diraddhd[i])) 12015 return (0); 12016 if (!LIST_EMPTY(&pagedep->pd_pendinghd)) 12017 return (0); 12018 if (!LIST_EMPTY(&pagedep->pd_jmvrefhd)) 12019 return (0); 12020 if (pagedep->pd_state & ONWORKLIST) 12021 WORKLIST_REMOVE(&pagedep->pd_list); 12022 LIST_REMOVE(pagedep, pd_hash); 12023 WORKITEM_FREE(pagedep, D_PAGEDEP); 12024 12025 return (1); 12026 } 12027 12028 /* 12029 * Called from within softdep_disk_write_complete above. 12030 * A write operation was just completed. Removed inodes can 12031 * now be freed and associated block pointers may be committed. 12032 * Note that this routine is always called from interrupt level 12033 * with further interrupts from this device blocked. 12034 * 12035 * If the write did not succeed, we will do all the roll-forward 12036 * operations, but we will not take the actions that will allow its 12037 * dependencies to be processed. 12038 */ 12039 static int 12040 handle_written_filepage(pagedep, bp, flags) 12041 struct pagedep *pagedep; 12042 struct buf *bp; /* buffer containing the written page */ 12043 int flags; 12044 { 12045 struct dirrem *dirrem; 12046 struct diradd *dap, *nextdap; 12047 struct direct *ep; 12048 int i, chgs; 12049 12050 if ((pagedep->pd_state & IOSTARTED) == 0) 12051 panic("handle_written_filepage: not started"); 12052 pagedep->pd_state &= ~IOSTARTED; 12053 if ((flags & WRITESUCCEEDED) == 0) 12054 goto rollforward; 12055 /* 12056 * Process any directory removals that have been committed. 12057 */ 12058 while ((dirrem = LIST_FIRST(&pagedep->pd_dirremhd)) != NULL) { 12059 LIST_REMOVE(dirrem, dm_next); 12060 dirrem->dm_state |= COMPLETE; 12061 dirrem->dm_dirinum = pagedep->pd_ino; 12062 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 12063 ("handle_written_filepage: Journal entries not written.")); 12064 add_to_worklist(&dirrem->dm_list, 0); 12065 } 12066 /* 12067 * Free any directory additions that have been committed. 12068 * If it is a newly allocated block, we have to wait until 12069 * the on-disk directory inode claims the new block. 12070 */ 12071 if ((pagedep->pd_state & NEWBLOCK) == 0) 12072 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 12073 free_diradd(dap, NULL); 12074 rollforward: 12075 /* 12076 * Uncommitted directory entries must be restored. 12077 */ 12078 for (chgs = 0, i = 0; i < DAHASHSZ; i++) { 12079 for (dap = LIST_FIRST(&pagedep->pd_diraddhd[i]); dap; 12080 dap = nextdap) { 12081 nextdap = LIST_NEXT(dap, da_pdlist); 12082 if (dap->da_state & ATTACHED) 12083 panic("handle_written_filepage: attached"); 12084 ep = (struct direct *) 12085 ((char *)bp->b_data + dap->da_offset); 12086 ep->d_ino = dap->da_newinum; 12087 dap->da_state &= ~UNDONE; 12088 dap->da_state |= ATTACHED; 12089 chgs = 1; 12090 /* 12091 * If the inode referenced by the directory has 12092 * been written out, then the dependency can be 12093 * moved to the pending list. 12094 */ 12095 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 12096 LIST_REMOVE(dap, da_pdlist); 12097 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, 12098 da_pdlist); 12099 } 12100 } 12101 } 12102 /* 12103 * If there were any rollbacks in the directory, then it must be 12104 * marked dirty so that its will eventually get written back in 12105 * its correct form. 12106 */ 12107 if (chgs || (flags & WRITESUCCEEDED) == 0) { 12108 if ((bp->b_flags & B_DELWRI) == 0) 12109 stat_dir_entry++; 12110 bdirty(bp); 12111 return (1); 12112 } 12113 /* 12114 * If we are not waiting for a new directory block to be 12115 * claimed by its inode, then the pagedep will be freed. 12116 * Otherwise it will remain to track any new entries on 12117 * the page in case they are fsync'ed. 12118 */ 12119 free_pagedep(pagedep); 12120 return (0); 12121 } 12122 12123 /* 12124 * Writing back in-core inode structures. 12125 * 12126 * The filesystem only accesses an inode's contents when it occupies an 12127 * "in-core" inode structure. These "in-core" structures are separate from 12128 * the page frames used to cache inode blocks. Only the latter are 12129 * transferred to/from the disk. So, when the updated contents of the 12130 * "in-core" inode structure are copied to the corresponding in-memory inode 12131 * block, the dependencies are also transferred. The following procedure is 12132 * called when copying a dirty "in-core" inode to a cached inode block. 12133 */ 12134 12135 /* 12136 * Called when an inode is loaded from disk. If the effective link count 12137 * differed from the actual link count when it was last flushed, then we 12138 * need to ensure that the correct effective link count is put back. 12139 */ 12140 void 12141 softdep_load_inodeblock(ip) 12142 struct inode *ip; /* the "in_core" copy of the inode */ 12143 { 12144 struct inodedep *inodedep; 12145 struct ufsmount *ump; 12146 12147 ump = ITOUMP(ip); 12148 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 12149 ("softdep_load_inodeblock called on non-softdep filesystem")); 12150 /* 12151 * Check for alternate nlink count. 12152 */ 12153 ip->i_effnlink = ip->i_nlink; 12154 ACQUIRE_LOCK(ump); 12155 if (inodedep_lookup(UFSTOVFS(ump), ip->i_number, 0, &inodedep) == 0) { 12156 FREE_LOCK(ump); 12157 return; 12158 } 12159 ip->i_effnlink -= inodedep->id_nlinkdelta; 12160 FREE_LOCK(ump); 12161 } 12162 12163 /* 12164 * This routine is called just before the "in-core" inode 12165 * information is to be copied to the in-memory inode block. 12166 * Recall that an inode block contains several inodes. If 12167 * the force flag is set, then the dependencies will be 12168 * cleared so that the update can always be made. Note that 12169 * the buffer is locked when this routine is called, so we 12170 * will never be in the middle of writing the inode block 12171 * to disk. 12172 */ 12173 void 12174 softdep_update_inodeblock(ip, bp, waitfor) 12175 struct inode *ip; /* the "in_core" copy of the inode */ 12176 struct buf *bp; /* the buffer containing the inode block */ 12177 int waitfor; /* nonzero => update must be allowed */ 12178 { 12179 struct inodedep *inodedep; 12180 struct inoref *inoref; 12181 struct ufsmount *ump; 12182 struct worklist *wk; 12183 struct mount *mp; 12184 struct buf *ibp; 12185 struct fs *fs; 12186 int error; 12187 12188 ump = ITOUMP(ip); 12189 mp = UFSTOVFS(ump); 12190 KASSERT(MOUNTEDSOFTDEP(mp) != 0, 12191 ("softdep_update_inodeblock called on non-softdep filesystem")); 12192 fs = ump->um_fs; 12193 /* 12194 * Preserve the freelink that is on disk. clear_unlinked_inodedep() 12195 * does not have access to the in-core ip so must write directly into 12196 * the inode block buffer when setting freelink. 12197 */ 12198 if (fs->fs_magic == FS_UFS1_MAGIC) 12199 DIP_SET(ip, i_freelink, ((struct ufs1_dinode *)bp->b_data + 12200 ino_to_fsbo(fs, ip->i_number))->di_freelink); 12201 else 12202 DIP_SET(ip, i_freelink, ((struct ufs2_dinode *)bp->b_data + 12203 ino_to_fsbo(fs, ip->i_number))->di_freelink); 12204 /* 12205 * If the effective link count is not equal to the actual link 12206 * count, then we must track the difference in an inodedep while 12207 * the inode is (potentially) tossed out of the cache. Otherwise, 12208 * if there is no existing inodedep, then there are no dependencies 12209 * to track. 12210 */ 12211 ACQUIRE_LOCK(ump); 12212 again: 12213 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 12214 FREE_LOCK(ump); 12215 if (ip->i_effnlink != ip->i_nlink) 12216 panic("softdep_update_inodeblock: bad link count"); 12217 return; 12218 } 12219 if (inodedep->id_nlinkdelta != ip->i_nlink - ip->i_effnlink) 12220 panic("softdep_update_inodeblock: bad delta"); 12221 /* 12222 * If we're flushing all dependencies we must also move any waiting 12223 * for journal writes onto the bufwait list prior to I/O. 12224 */ 12225 if (waitfor) { 12226 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 12227 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 12228 == DEPCOMPLETE) { 12229 jwait(&inoref->if_list, MNT_WAIT); 12230 goto again; 12231 } 12232 } 12233 } 12234 /* 12235 * Changes have been initiated. Anything depending on these 12236 * changes cannot occur until this inode has been written. 12237 */ 12238 inodedep->id_state &= ~COMPLETE; 12239 if ((inodedep->id_state & ONWORKLIST) == 0) 12240 WORKLIST_INSERT(&bp->b_dep, &inodedep->id_list); 12241 /* 12242 * Any new dependencies associated with the incore inode must 12243 * now be moved to the list associated with the buffer holding 12244 * the in-memory copy of the inode. Once merged process any 12245 * allocdirects that are completed by the merger. 12246 */ 12247 merge_inode_lists(&inodedep->id_newinoupdt, &inodedep->id_inoupdt); 12248 if (!TAILQ_EMPTY(&inodedep->id_inoupdt)) 12249 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_inoupdt), 12250 NULL); 12251 merge_inode_lists(&inodedep->id_newextupdt, &inodedep->id_extupdt); 12252 if (!TAILQ_EMPTY(&inodedep->id_extupdt)) 12253 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_extupdt), 12254 NULL); 12255 /* 12256 * Now that the inode has been pushed into the buffer, the 12257 * operations dependent on the inode being written to disk 12258 * can be moved to the id_bufwait so that they will be 12259 * processed when the buffer I/O completes. 12260 */ 12261 while ((wk = LIST_FIRST(&inodedep->id_inowait)) != NULL) { 12262 WORKLIST_REMOVE(wk); 12263 WORKLIST_INSERT(&inodedep->id_bufwait, wk); 12264 } 12265 /* 12266 * Newly allocated inodes cannot be written until the bitmap 12267 * that allocates them have been written (indicated by 12268 * DEPCOMPLETE being set in id_state). If we are doing a 12269 * forced sync (e.g., an fsync on a file), we force the bitmap 12270 * to be written so that the update can be done. 12271 */ 12272 if (waitfor == 0) { 12273 FREE_LOCK(ump); 12274 return; 12275 } 12276 retry: 12277 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) != 0) { 12278 FREE_LOCK(ump); 12279 return; 12280 } 12281 ibp = inodedep->id_bmsafemap->sm_buf; 12282 ibp = getdirtybuf(ibp, LOCK_PTR(ump), MNT_WAIT); 12283 if (ibp == NULL) { 12284 /* 12285 * If ibp came back as NULL, the dependency could have been 12286 * freed while we slept. Look it up again, and check to see 12287 * that it has completed. 12288 */ 12289 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 12290 goto retry; 12291 FREE_LOCK(ump); 12292 return; 12293 } 12294 FREE_LOCK(ump); 12295 if ((error = bwrite(ibp)) != 0) 12296 softdep_error("softdep_update_inodeblock: bwrite", error); 12297 } 12298 12299 /* 12300 * Merge the a new inode dependency list (such as id_newinoupdt) into an 12301 * old inode dependency list (such as id_inoupdt). This routine must be 12302 * called with splbio interrupts blocked. 12303 */ 12304 static void 12305 merge_inode_lists(newlisthead, oldlisthead) 12306 struct allocdirectlst *newlisthead; 12307 struct allocdirectlst *oldlisthead; 12308 { 12309 struct allocdirect *listadp, *newadp; 12310 12311 newadp = TAILQ_FIRST(newlisthead); 12312 for (listadp = TAILQ_FIRST(oldlisthead); listadp && newadp;) { 12313 if (listadp->ad_offset < newadp->ad_offset) { 12314 listadp = TAILQ_NEXT(listadp, ad_next); 12315 continue; 12316 } 12317 TAILQ_REMOVE(newlisthead, newadp, ad_next); 12318 TAILQ_INSERT_BEFORE(listadp, newadp, ad_next); 12319 if (listadp->ad_offset == newadp->ad_offset) { 12320 allocdirect_merge(oldlisthead, newadp, 12321 listadp); 12322 listadp = newadp; 12323 } 12324 newadp = TAILQ_FIRST(newlisthead); 12325 } 12326 while ((newadp = TAILQ_FIRST(newlisthead)) != NULL) { 12327 TAILQ_REMOVE(newlisthead, newadp, ad_next); 12328 TAILQ_INSERT_TAIL(oldlisthead, newadp, ad_next); 12329 } 12330 } 12331 12332 /* 12333 * If we are doing an fsync, then we must ensure that any directory 12334 * entries for the inode have been written after the inode gets to disk. 12335 */ 12336 int 12337 softdep_fsync(vp) 12338 struct vnode *vp; /* the "in_core" copy of the inode */ 12339 { 12340 struct inodedep *inodedep; 12341 struct pagedep *pagedep; 12342 struct inoref *inoref; 12343 struct ufsmount *ump; 12344 struct worklist *wk; 12345 struct diradd *dap; 12346 struct mount *mp; 12347 struct vnode *pvp; 12348 struct inode *ip; 12349 struct buf *bp; 12350 struct fs *fs; 12351 struct thread *td = curthread; 12352 int error, flushparent, pagedep_new_block; 12353 ino_t parentino; 12354 ufs_lbn_t lbn; 12355 12356 ip = VTOI(vp); 12357 mp = vp->v_mount; 12358 ump = VFSTOUFS(mp); 12359 fs = ump->um_fs; 12360 if (MOUNTEDSOFTDEP(mp) == 0) 12361 return (0); 12362 ACQUIRE_LOCK(ump); 12363 restart: 12364 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 12365 FREE_LOCK(ump); 12366 return (0); 12367 } 12368 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 12369 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 12370 == DEPCOMPLETE) { 12371 jwait(&inoref->if_list, MNT_WAIT); 12372 goto restart; 12373 } 12374 } 12375 if (!LIST_EMPTY(&inodedep->id_inowait) || 12376 !TAILQ_EMPTY(&inodedep->id_extupdt) || 12377 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 12378 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 12379 !TAILQ_EMPTY(&inodedep->id_newinoupdt)) 12380 panic("softdep_fsync: pending ops %p", inodedep); 12381 for (error = 0, flushparent = 0; ; ) { 12382 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) == NULL) 12383 break; 12384 if (wk->wk_type != D_DIRADD) 12385 panic("softdep_fsync: Unexpected type %s", 12386 TYPENAME(wk->wk_type)); 12387 dap = WK_DIRADD(wk); 12388 /* 12389 * Flush our parent if this directory entry has a MKDIR_PARENT 12390 * dependency or is contained in a newly allocated block. 12391 */ 12392 if (dap->da_state & DIRCHG) 12393 pagedep = dap->da_previous->dm_pagedep; 12394 else 12395 pagedep = dap->da_pagedep; 12396 parentino = pagedep->pd_ino; 12397 lbn = pagedep->pd_lbn; 12398 if ((dap->da_state & (MKDIR_BODY | COMPLETE)) != COMPLETE) 12399 panic("softdep_fsync: dirty"); 12400 if ((dap->da_state & MKDIR_PARENT) || 12401 (pagedep->pd_state & NEWBLOCK)) 12402 flushparent = 1; 12403 else 12404 flushparent = 0; 12405 /* 12406 * If we are being fsync'ed as part of vgone'ing this vnode, 12407 * then we will not be able to release and recover the 12408 * vnode below, so we just have to give up on writing its 12409 * directory entry out. It will eventually be written, just 12410 * not now, but then the user was not asking to have it 12411 * written, so we are not breaking any promises. 12412 */ 12413 if (vp->v_iflag & VI_DOOMED) 12414 break; 12415 /* 12416 * We prevent deadlock by always fetching inodes from the 12417 * root, moving down the directory tree. Thus, when fetching 12418 * our parent directory, we first try to get the lock. If 12419 * that fails, we must unlock ourselves before requesting 12420 * the lock on our parent. See the comment in ufs_lookup 12421 * for details on possible races. 12422 */ 12423 FREE_LOCK(ump); 12424 if (ffs_vgetf(mp, parentino, LK_NOWAIT | LK_EXCLUSIVE, &pvp, 12425 FFSV_FORCEINSMQ)) { 12426 error = vfs_busy(mp, MBF_NOWAIT); 12427 if (error != 0) { 12428 vfs_ref(mp); 12429 VOP_UNLOCK(vp, 0); 12430 error = vfs_busy(mp, 0); 12431 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 12432 vfs_rel(mp); 12433 if (error != 0) 12434 return (ENOENT); 12435 if (vp->v_iflag & VI_DOOMED) { 12436 vfs_unbusy(mp); 12437 return (ENOENT); 12438 } 12439 } 12440 VOP_UNLOCK(vp, 0); 12441 error = ffs_vgetf(mp, parentino, LK_EXCLUSIVE, 12442 &pvp, FFSV_FORCEINSMQ); 12443 vfs_unbusy(mp); 12444 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 12445 if (vp->v_iflag & VI_DOOMED) { 12446 if (error == 0) 12447 vput(pvp); 12448 error = ENOENT; 12449 } 12450 if (error != 0) 12451 return (error); 12452 } 12453 /* 12454 * All MKDIR_PARENT dependencies and all the NEWBLOCK pagedeps 12455 * that are contained in direct blocks will be resolved by 12456 * doing a ffs_update. Pagedeps contained in indirect blocks 12457 * may require a complete sync'ing of the directory. So, we 12458 * try the cheap and fast ffs_update first, and if that fails, 12459 * then we do the slower ffs_syncvnode of the directory. 12460 */ 12461 if (flushparent) { 12462 int locked; 12463 12464 if ((error = ffs_update(pvp, 1)) != 0) { 12465 vput(pvp); 12466 return (error); 12467 } 12468 ACQUIRE_LOCK(ump); 12469 locked = 1; 12470 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) { 12471 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) != NULL) { 12472 if (wk->wk_type != D_DIRADD) 12473 panic("softdep_fsync: Unexpected type %s", 12474 TYPENAME(wk->wk_type)); 12475 dap = WK_DIRADD(wk); 12476 if (dap->da_state & DIRCHG) 12477 pagedep = dap->da_previous->dm_pagedep; 12478 else 12479 pagedep = dap->da_pagedep; 12480 pagedep_new_block = pagedep->pd_state & NEWBLOCK; 12481 FREE_LOCK(ump); 12482 locked = 0; 12483 if (pagedep_new_block && (error = 12484 ffs_syncvnode(pvp, MNT_WAIT, 0))) { 12485 vput(pvp); 12486 return (error); 12487 } 12488 } 12489 } 12490 if (locked) 12491 FREE_LOCK(ump); 12492 } 12493 /* 12494 * Flush directory page containing the inode's name. 12495 */ 12496 error = bread(pvp, lbn, blksize(fs, VTOI(pvp), lbn), td->td_ucred, 12497 &bp); 12498 if (error == 0) 12499 error = bwrite(bp); 12500 else 12501 brelse(bp); 12502 vput(pvp); 12503 if (error != 0) 12504 return (error); 12505 ACQUIRE_LOCK(ump); 12506 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 12507 break; 12508 } 12509 FREE_LOCK(ump); 12510 return (0); 12511 } 12512 12513 /* 12514 * Flush all the dirty bitmaps associated with the block device 12515 * before flushing the rest of the dirty blocks so as to reduce 12516 * the number of dependencies that will have to be rolled back. 12517 * 12518 * XXX Unused? 12519 */ 12520 void 12521 softdep_fsync_mountdev(vp) 12522 struct vnode *vp; 12523 { 12524 struct buf *bp, *nbp; 12525 struct worklist *wk; 12526 struct bufobj *bo; 12527 12528 if (!vn_isdisk(vp, NULL)) 12529 panic("softdep_fsync_mountdev: vnode not a disk"); 12530 bo = &vp->v_bufobj; 12531 restart: 12532 BO_LOCK(bo); 12533 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 12534 /* 12535 * If it is already scheduled, skip to the next buffer. 12536 */ 12537 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) 12538 continue; 12539 12540 if ((bp->b_flags & B_DELWRI) == 0) 12541 panic("softdep_fsync_mountdev: not dirty"); 12542 /* 12543 * We are only interested in bitmaps with outstanding 12544 * dependencies. 12545 */ 12546 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL || 12547 wk->wk_type != D_BMSAFEMAP || 12548 (bp->b_vflags & BV_BKGRDINPROG)) { 12549 BUF_UNLOCK(bp); 12550 continue; 12551 } 12552 BO_UNLOCK(bo); 12553 bremfree(bp); 12554 (void) bawrite(bp); 12555 goto restart; 12556 } 12557 drain_output(vp); 12558 BO_UNLOCK(bo); 12559 } 12560 12561 /* 12562 * Sync all cylinder groups that were dirty at the time this function is 12563 * called. Newly dirtied cgs will be inserted before the sentinel. This 12564 * is used to flush freedep activity that may be holding up writes to a 12565 * indirect block. 12566 */ 12567 static int 12568 sync_cgs(mp, waitfor) 12569 struct mount *mp; 12570 int waitfor; 12571 { 12572 struct bmsafemap *bmsafemap; 12573 struct bmsafemap *sentinel; 12574 struct ufsmount *ump; 12575 struct buf *bp; 12576 int error; 12577 12578 sentinel = malloc(sizeof(*sentinel), M_BMSAFEMAP, M_ZERO | M_WAITOK); 12579 sentinel->sm_cg = -1; 12580 ump = VFSTOUFS(mp); 12581 error = 0; 12582 ACQUIRE_LOCK(ump); 12583 LIST_INSERT_HEAD(&ump->softdep_dirtycg, sentinel, sm_next); 12584 for (bmsafemap = LIST_NEXT(sentinel, sm_next); bmsafemap != NULL; 12585 bmsafemap = LIST_NEXT(sentinel, sm_next)) { 12586 /* Skip sentinels and cgs with no work to release. */ 12587 if (bmsafemap->sm_cg == -1 || 12588 (LIST_EMPTY(&bmsafemap->sm_freehd) && 12589 LIST_EMPTY(&bmsafemap->sm_freewr))) { 12590 LIST_REMOVE(sentinel, sm_next); 12591 LIST_INSERT_AFTER(bmsafemap, sentinel, sm_next); 12592 continue; 12593 } 12594 /* 12595 * If we don't get the lock and we're waiting try again, if 12596 * not move on to the next buf and try to sync it. 12597 */ 12598 bp = getdirtybuf(bmsafemap->sm_buf, LOCK_PTR(ump), waitfor); 12599 if (bp == NULL && waitfor == MNT_WAIT) 12600 continue; 12601 LIST_REMOVE(sentinel, sm_next); 12602 LIST_INSERT_AFTER(bmsafemap, sentinel, sm_next); 12603 if (bp == NULL) 12604 continue; 12605 FREE_LOCK(ump); 12606 if (waitfor == MNT_NOWAIT) 12607 bawrite(bp); 12608 else 12609 error = bwrite(bp); 12610 ACQUIRE_LOCK(ump); 12611 if (error) 12612 break; 12613 } 12614 LIST_REMOVE(sentinel, sm_next); 12615 FREE_LOCK(ump); 12616 free(sentinel, M_BMSAFEMAP); 12617 return (error); 12618 } 12619 12620 /* 12621 * This routine is called when we are trying to synchronously flush a 12622 * file. This routine must eliminate any filesystem metadata dependencies 12623 * so that the syncing routine can succeed. 12624 */ 12625 int 12626 softdep_sync_metadata(struct vnode *vp) 12627 { 12628 struct inode *ip; 12629 int error; 12630 12631 ip = VTOI(vp); 12632 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 12633 ("softdep_sync_metadata called on non-softdep filesystem")); 12634 /* 12635 * Ensure that any direct block dependencies have been cleared, 12636 * truncations are started, and inode references are journaled. 12637 */ 12638 ACQUIRE_LOCK(VFSTOUFS(vp->v_mount)); 12639 /* 12640 * Write all journal records to prevent rollbacks on devvp. 12641 */ 12642 if (vp->v_type == VCHR) 12643 softdep_flushjournal(vp->v_mount); 12644 error = flush_inodedep_deps(vp, vp->v_mount, ip->i_number); 12645 /* 12646 * Ensure that all truncates are written so we won't find deps on 12647 * indirect blocks. 12648 */ 12649 process_truncates(vp); 12650 FREE_LOCK(VFSTOUFS(vp->v_mount)); 12651 12652 return (error); 12653 } 12654 12655 /* 12656 * This routine is called when we are attempting to sync a buf with 12657 * dependencies. If waitfor is MNT_NOWAIT it attempts to schedule any 12658 * other IO it can but returns EBUSY if the buffer is not yet able to 12659 * be written. Dependencies which will not cause rollbacks will always 12660 * return 0. 12661 */ 12662 int 12663 softdep_sync_buf(struct vnode *vp, struct buf *bp, int waitfor) 12664 { 12665 struct indirdep *indirdep; 12666 struct pagedep *pagedep; 12667 struct allocindir *aip; 12668 struct newblk *newblk; 12669 struct ufsmount *ump; 12670 struct buf *nbp; 12671 struct worklist *wk; 12672 int i, error; 12673 12674 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 12675 ("softdep_sync_buf called on non-softdep filesystem")); 12676 /* 12677 * For VCHR we just don't want to force flush any dependencies that 12678 * will cause rollbacks. 12679 */ 12680 if (vp->v_type == VCHR) { 12681 if (waitfor == MNT_NOWAIT && softdep_count_dependencies(bp, 0)) 12682 return (EBUSY); 12683 return (0); 12684 } 12685 ump = VFSTOUFS(vp->v_mount); 12686 ACQUIRE_LOCK(ump); 12687 /* 12688 * As we hold the buffer locked, none of its dependencies 12689 * will disappear. 12690 */ 12691 error = 0; 12692 top: 12693 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 12694 switch (wk->wk_type) { 12695 12696 case D_ALLOCDIRECT: 12697 case D_ALLOCINDIR: 12698 newblk = WK_NEWBLK(wk); 12699 if (newblk->nb_jnewblk != NULL) { 12700 if (waitfor == MNT_NOWAIT) { 12701 error = EBUSY; 12702 goto out_unlock; 12703 } 12704 jwait(&newblk->nb_jnewblk->jn_list, waitfor); 12705 goto top; 12706 } 12707 if (newblk->nb_state & DEPCOMPLETE || 12708 waitfor == MNT_NOWAIT) 12709 continue; 12710 nbp = newblk->nb_bmsafemap->sm_buf; 12711 nbp = getdirtybuf(nbp, LOCK_PTR(ump), waitfor); 12712 if (nbp == NULL) 12713 goto top; 12714 FREE_LOCK(ump); 12715 if ((error = bwrite(nbp)) != 0) 12716 goto out; 12717 ACQUIRE_LOCK(ump); 12718 continue; 12719 12720 case D_INDIRDEP: 12721 indirdep = WK_INDIRDEP(wk); 12722 if (waitfor == MNT_NOWAIT) { 12723 if (!TAILQ_EMPTY(&indirdep->ir_trunc) || 12724 !LIST_EMPTY(&indirdep->ir_deplisthd)) { 12725 error = EBUSY; 12726 goto out_unlock; 12727 } 12728 } 12729 if (!TAILQ_EMPTY(&indirdep->ir_trunc)) 12730 panic("softdep_sync_buf: truncation pending."); 12731 restart: 12732 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) { 12733 newblk = (struct newblk *)aip; 12734 if (newblk->nb_jnewblk != NULL) { 12735 jwait(&newblk->nb_jnewblk->jn_list, 12736 waitfor); 12737 goto restart; 12738 } 12739 if (newblk->nb_state & DEPCOMPLETE) 12740 continue; 12741 nbp = newblk->nb_bmsafemap->sm_buf; 12742 nbp = getdirtybuf(nbp, LOCK_PTR(ump), waitfor); 12743 if (nbp == NULL) 12744 goto restart; 12745 FREE_LOCK(ump); 12746 if ((error = bwrite(nbp)) != 0) 12747 goto out; 12748 ACQUIRE_LOCK(ump); 12749 goto restart; 12750 } 12751 continue; 12752 12753 case D_PAGEDEP: 12754 /* 12755 * Only flush directory entries in synchronous passes. 12756 */ 12757 if (waitfor != MNT_WAIT) { 12758 error = EBUSY; 12759 goto out_unlock; 12760 } 12761 /* 12762 * While syncing snapshots, we must allow recursive 12763 * lookups. 12764 */ 12765 BUF_AREC(bp); 12766 /* 12767 * We are trying to sync a directory that may 12768 * have dependencies on both its own metadata 12769 * and/or dependencies on the inodes of any 12770 * recently allocated files. We walk its diradd 12771 * lists pushing out the associated inode. 12772 */ 12773 pagedep = WK_PAGEDEP(wk); 12774 for (i = 0; i < DAHASHSZ; i++) { 12775 if (LIST_FIRST(&pagedep->pd_diraddhd[i]) == 0) 12776 continue; 12777 if ((error = flush_pagedep_deps(vp, wk->wk_mp, 12778 &pagedep->pd_diraddhd[i]))) { 12779 BUF_NOREC(bp); 12780 goto out_unlock; 12781 } 12782 } 12783 BUF_NOREC(bp); 12784 continue; 12785 12786 case D_FREEWORK: 12787 case D_FREEDEP: 12788 case D_JSEGDEP: 12789 case D_JNEWBLK: 12790 continue; 12791 12792 default: 12793 panic("softdep_sync_buf: Unknown type %s", 12794 TYPENAME(wk->wk_type)); 12795 /* NOTREACHED */ 12796 } 12797 } 12798 out_unlock: 12799 FREE_LOCK(ump); 12800 out: 12801 return (error); 12802 } 12803 12804 /* 12805 * Flush the dependencies associated with an inodedep. 12806 * Called with splbio blocked. 12807 */ 12808 static int 12809 flush_inodedep_deps(vp, mp, ino) 12810 struct vnode *vp; 12811 struct mount *mp; 12812 ino_t ino; 12813 { 12814 struct inodedep *inodedep; 12815 struct inoref *inoref; 12816 struct ufsmount *ump; 12817 int error, waitfor; 12818 12819 /* 12820 * This work is done in two passes. The first pass grabs most 12821 * of the buffers and begins asynchronously writing them. The 12822 * only way to wait for these asynchronous writes is to sleep 12823 * on the filesystem vnode which may stay busy for a long time 12824 * if the filesystem is active. So, instead, we make a second 12825 * pass over the dependencies blocking on each write. In the 12826 * usual case we will be blocking against a write that we 12827 * initiated, so when it is done the dependency will have been 12828 * resolved. Thus the second pass is expected to end quickly. 12829 * We give a brief window at the top of the loop to allow 12830 * any pending I/O to complete. 12831 */ 12832 ump = VFSTOUFS(mp); 12833 LOCK_OWNED(ump); 12834 for (error = 0, waitfor = MNT_NOWAIT; ; ) { 12835 if (error) 12836 return (error); 12837 FREE_LOCK(ump); 12838 ACQUIRE_LOCK(ump); 12839 restart: 12840 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 12841 return (0); 12842 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 12843 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 12844 == DEPCOMPLETE) { 12845 jwait(&inoref->if_list, MNT_WAIT); 12846 goto restart; 12847 } 12848 } 12849 if (flush_deplist(&inodedep->id_inoupdt, waitfor, &error) || 12850 flush_deplist(&inodedep->id_newinoupdt, waitfor, &error) || 12851 flush_deplist(&inodedep->id_extupdt, waitfor, &error) || 12852 flush_deplist(&inodedep->id_newextupdt, waitfor, &error)) 12853 continue; 12854 /* 12855 * If pass2, we are done, otherwise do pass 2. 12856 */ 12857 if (waitfor == MNT_WAIT) 12858 break; 12859 waitfor = MNT_WAIT; 12860 } 12861 /* 12862 * Try freeing inodedep in case all dependencies have been removed. 12863 */ 12864 if (inodedep_lookup(mp, ino, 0, &inodedep) != 0) 12865 (void) free_inodedep(inodedep); 12866 return (0); 12867 } 12868 12869 /* 12870 * Flush an inode dependency list. 12871 * Called with splbio blocked. 12872 */ 12873 static int 12874 flush_deplist(listhead, waitfor, errorp) 12875 struct allocdirectlst *listhead; 12876 int waitfor; 12877 int *errorp; 12878 { 12879 struct allocdirect *adp; 12880 struct newblk *newblk; 12881 struct ufsmount *ump; 12882 struct buf *bp; 12883 12884 if ((adp = TAILQ_FIRST(listhead)) == NULL) 12885 return (0); 12886 ump = VFSTOUFS(adp->ad_list.wk_mp); 12887 LOCK_OWNED(ump); 12888 TAILQ_FOREACH(adp, listhead, ad_next) { 12889 newblk = (struct newblk *)adp; 12890 if (newblk->nb_jnewblk != NULL) { 12891 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT); 12892 return (1); 12893 } 12894 if (newblk->nb_state & DEPCOMPLETE) 12895 continue; 12896 bp = newblk->nb_bmsafemap->sm_buf; 12897 bp = getdirtybuf(bp, LOCK_PTR(ump), waitfor); 12898 if (bp == NULL) { 12899 if (waitfor == MNT_NOWAIT) 12900 continue; 12901 return (1); 12902 } 12903 FREE_LOCK(ump); 12904 if (waitfor == MNT_NOWAIT) 12905 bawrite(bp); 12906 else 12907 *errorp = bwrite(bp); 12908 ACQUIRE_LOCK(ump); 12909 return (1); 12910 } 12911 return (0); 12912 } 12913 12914 /* 12915 * Flush dependencies associated with an allocdirect block. 12916 */ 12917 static int 12918 flush_newblk_dep(vp, mp, lbn) 12919 struct vnode *vp; 12920 struct mount *mp; 12921 ufs_lbn_t lbn; 12922 { 12923 struct newblk *newblk; 12924 struct ufsmount *ump; 12925 struct bufobj *bo; 12926 struct inode *ip; 12927 struct buf *bp; 12928 ufs2_daddr_t blkno; 12929 int error; 12930 12931 error = 0; 12932 bo = &vp->v_bufobj; 12933 ip = VTOI(vp); 12934 blkno = DIP(ip, i_db[lbn]); 12935 if (blkno == 0) 12936 panic("flush_newblk_dep: Missing block"); 12937 ump = VFSTOUFS(mp); 12938 ACQUIRE_LOCK(ump); 12939 /* 12940 * Loop until all dependencies related to this block are satisfied. 12941 * We must be careful to restart after each sleep in case a write 12942 * completes some part of this process for us. 12943 */ 12944 for (;;) { 12945 if (newblk_lookup(mp, blkno, 0, &newblk) == 0) { 12946 FREE_LOCK(ump); 12947 break; 12948 } 12949 if (newblk->nb_list.wk_type != D_ALLOCDIRECT) 12950 panic("flush_newblk_deps: Bad newblk %p", newblk); 12951 /* 12952 * Flush the journal. 12953 */ 12954 if (newblk->nb_jnewblk != NULL) { 12955 jwait(&newblk->nb_jnewblk->jn_list, MNT_WAIT); 12956 continue; 12957 } 12958 /* 12959 * Write the bitmap dependency. 12960 */ 12961 if ((newblk->nb_state & DEPCOMPLETE) == 0) { 12962 bp = newblk->nb_bmsafemap->sm_buf; 12963 bp = getdirtybuf(bp, LOCK_PTR(ump), MNT_WAIT); 12964 if (bp == NULL) 12965 continue; 12966 FREE_LOCK(ump); 12967 error = bwrite(bp); 12968 if (error) 12969 break; 12970 ACQUIRE_LOCK(ump); 12971 continue; 12972 } 12973 /* 12974 * Write the buffer. 12975 */ 12976 FREE_LOCK(ump); 12977 BO_LOCK(bo); 12978 bp = gbincore(bo, lbn); 12979 if (bp != NULL) { 12980 error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 12981 LK_INTERLOCK, BO_LOCKPTR(bo)); 12982 if (error == ENOLCK) { 12983 ACQUIRE_LOCK(ump); 12984 error = 0; 12985 continue; /* Slept, retry */ 12986 } 12987 if (error != 0) 12988 break; /* Failed */ 12989 if (bp->b_flags & B_DELWRI) { 12990 bremfree(bp); 12991 error = bwrite(bp); 12992 if (error) 12993 break; 12994 } else 12995 BUF_UNLOCK(bp); 12996 } else 12997 BO_UNLOCK(bo); 12998 /* 12999 * We have to wait for the direct pointers to 13000 * point at the newdirblk before the dependency 13001 * will go away. 13002 */ 13003 error = ffs_update(vp, 1); 13004 if (error) 13005 break; 13006 ACQUIRE_LOCK(ump); 13007 } 13008 return (error); 13009 } 13010 13011 /* 13012 * Eliminate a pagedep dependency by flushing out all its diradd dependencies. 13013 * Called with splbio blocked. 13014 */ 13015 static int 13016 flush_pagedep_deps(pvp, mp, diraddhdp) 13017 struct vnode *pvp; 13018 struct mount *mp; 13019 struct diraddhd *diraddhdp; 13020 { 13021 struct inodedep *inodedep; 13022 struct inoref *inoref; 13023 struct ufsmount *ump; 13024 struct diradd *dap; 13025 struct vnode *vp; 13026 int error = 0; 13027 struct buf *bp; 13028 ino_t inum; 13029 struct diraddhd unfinished; 13030 13031 LIST_INIT(&unfinished); 13032 ump = VFSTOUFS(mp); 13033 LOCK_OWNED(ump); 13034 restart: 13035 while ((dap = LIST_FIRST(diraddhdp)) != NULL) { 13036 /* 13037 * Flush ourselves if this directory entry 13038 * has a MKDIR_PARENT dependency. 13039 */ 13040 if (dap->da_state & MKDIR_PARENT) { 13041 FREE_LOCK(ump); 13042 if ((error = ffs_update(pvp, 1)) != 0) 13043 break; 13044 ACQUIRE_LOCK(ump); 13045 /* 13046 * If that cleared dependencies, go on to next. 13047 */ 13048 if (dap != LIST_FIRST(diraddhdp)) 13049 continue; 13050 /* 13051 * All MKDIR_PARENT dependencies and all the 13052 * NEWBLOCK pagedeps that are contained in direct 13053 * blocks were resolved by doing above ffs_update. 13054 * Pagedeps contained in indirect blocks may 13055 * require a complete sync'ing of the directory. 13056 * We are in the midst of doing a complete sync, 13057 * so if they are not resolved in this pass we 13058 * defer them for now as they will be sync'ed by 13059 * our caller shortly. 13060 */ 13061 LIST_REMOVE(dap, da_pdlist); 13062 LIST_INSERT_HEAD(&unfinished, dap, da_pdlist); 13063 continue; 13064 } 13065 /* 13066 * A newly allocated directory must have its "." and 13067 * ".." entries written out before its name can be 13068 * committed in its parent. 13069 */ 13070 inum = dap->da_newinum; 13071 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 13072 panic("flush_pagedep_deps: lost inode1"); 13073 /* 13074 * Wait for any pending journal adds to complete so we don't 13075 * cause rollbacks while syncing. 13076 */ 13077 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 13078 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 13079 == DEPCOMPLETE) { 13080 jwait(&inoref->if_list, MNT_WAIT); 13081 goto restart; 13082 } 13083 } 13084 if (dap->da_state & MKDIR_BODY) { 13085 FREE_LOCK(ump); 13086 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 13087 FFSV_FORCEINSMQ))) 13088 break; 13089 error = flush_newblk_dep(vp, mp, 0); 13090 /* 13091 * If we still have the dependency we might need to 13092 * update the vnode to sync the new link count to 13093 * disk. 13094 */ 13095 if (error == 0 && dap == LIST_FIRST(diraddhdp)) 13096 error = ffs_update(vp, 1); 13097 vput(vp); 13098 if (error != 0) 13099 break; 13100 ACQUIRE_LOCK(ump); 13101 /* 13102 * If that cleared dependencies, go on to next. 13103 */ 13104 if (dap != LIST_FIRST(diraddhdp)) 13105 continue; 13106 if (dap->da_state & MKDIR_BODY) { 13107 inodedep_lookup(UFSTOVFS(ump), inum, 0, 13108 &inodedep); 13109 panic("flush_pagedep_deps: MKDIR_BODY " 13110 "inodedep %p dap %p vp %p", 13111 inodedep, dap, vp); 13112 } 13113 } 13114 /* 13115 * Flush the inode on which the directory entry depends. 13116 * Having accounted for MKDIR_PARENT and MKDIR_BODY above, 13117 * the only remaining dependency is that the updated inode 13118 * count must get pushed to disk. The inode has already 13119 * been pushed into its inode buffer (via VOP_UPDATE) at 13120 * the time of the reference count change. So we need only 13121 * locate that buffer, ensure that there will be no rollback 13122 * caused by a bitmap dependency, then write the inode buffer. 13123 */ 13124 retry: 13125 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 13126 panic("flush_pagedep_deps: lost inode"); 13127 /* 13128 * If the inode still has bitmap dependencies, 13129 * push them to disk. 13130 */ 13131 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) == 0) { 13132 bp = inodedep->id_bmsafemap->sm_buf; 13133 bp = getdirtybuf(bp, LOCK_PTR(ump), MNT_WAIT); 13134 if (bp == NULL) 13135 goto retry; 13136 FREE_LOCK(ump); 13137 if ((error = bwrite(bp)) != 0) 13138 break; 13139 ACQUIRE_LOCK(ump); 13140 if (dap != LIST_FIRST(diraddhdp)) 13141 continue; 13142 } 13143 /* 13144 * If the inode is still sitting in a buffer waiting 13145 * to be written or waiting for the link count to be 13146 * adjusted update it here to flush it to disk. 13147 */ 13148 if (dap == LIST_FIRST(diraddhdp)) { 13149 FREE_LOCK(ump); 13150 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 13151 FFSV_FORCEINSMQ))) 13152 break; 13153 error = ffs_update(vp, 1); 13154 vput(vp); 13155 if (error) 13156 break; 13157 ACQUIRE_LOCK(ump); 13158 } 13159 /* 13160 * If we have failed to get rid of all the dependencies 13161 * then something is seriously wrong. 13162 */ 13163 if (dap == LIST_FIRST(diraddhdp)) { 13164 inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep); 13165 panic("flush_pagedep_deps: failed to flush " 13166 "inodedep %p ino %ju dap %p", 13167 inodedep, (uintmax_t)inum, dap); 13168 } 13169 } 13170 if (error) 13171 ACQUIRE_LOCK(ump); 13172 while ((dap = LIST_FIRST(&unfinished)) != NULL) { 13173 LIST_REMOVE(dap, da_pdlist); 13174 LIST_INSERT_HEAD(diraddhdp, dap, da_pdlist); 13175 } 13176 return (error); 13177 } 13178 13179 /* 13180 * A large burst of file addition or deletion activity can drive the 13181 * memory load excessively high. First attempt to slow things down 13182 * using the techniques below. If that fails, this routine requests 13183 * the offending operations to fall back to running synchronously 13184 * until the memory load returns to a reasonable level. 13185 */ 13186 int 13187 softdep_slowdown(vp) 13188 struct vnode *vp; 13189 { 13190 struct ufsmount *ump; 13191 int jlow; 13192 int max_softdeps_hard; 13193 13194 KASSERT(MOUNTEDSOFTDEP(vp->v_mount) != 0, 13195 ("softdep_slowdown called on non-softdep filesystem")); 13196 ump = VFSTOUFS(vp->v_mount); 13197 ACQUIRE_LOCK(ump); 13198 jlow = 0; 13199 /* 13200 * Check for journal space if needed. 13201 */ 13202 if (DOINGSUJ(vp)) { 13203 if (journal_space(ump, 0) == 0) 13204 jlow = 1; 13205 } 13206 /* 13207 * If the system is under its limits and our filesystem is 13208 * not responsible for more than our share of the usage and 13209 * we are not low on journal space, then no need to slow down. 13210 */ 13211 max_softdeps_hard = max_softdeps * 11 / 10; 13212 if (dep_current[D_DIRREM] < max_softdeps_hard / 2 && 13213 dep_current[D_INODEDEP] < max_softdeps_hard && 13214 dep_current[D_INDIRDEP] < max_softdeps_hard / 1000 && 13215 dep_current[D_FREEBLKS] < max_softdeps_hard && jlow == 0 && 13216 ump->softdep_curdeps[D_DIRREM] < 13217 (max_softdeps_hard / 2) / stat_flush_threads && 13218 ump->softdep_curdeps[D_INODEDEP] < 13219 max_softdeps_hard / stat_flush_threads && 13220 ump->softdep_curdeps[D_INDIRDEP] < 13221 (max_softdeps_hard / 1000) / stat_flush_threads && 13222 ump->softdep_curdeps[D_FREEBLKS] < 13223 max_softdeps_hard / stat_flush_threads) { 13224 FREE_LOCK(ump); 13225 return (0); 13226 } 13227 /* 13228 * If the journal is low or our filesystem is over its limit 13229 * then speedup the cleanup. 13230 */ 13231 if (ump->softdep_curdeps[D_INDIRDEP] < 13232 (max_softdeps_hard / 1000) / stat_flush_threads || jlow) 13233 softdep_speedup(ump); 13234 stat_sync_limit_hit += 1; 13235 FREE_LOCK(ump); 13236 /* 13237 * We only slow down the rate at which new dependencies are 13238 * generated if we are not using journaling. With journaling, 13239 * the cleanup should always be sufficient to keep things 13240 * under control. 13241 */ 13242 if (DOINGSUJ(vp)) 13243 return (0); 13244 return (1); 13245 } 13246 13247 /* 13248 * Called by the allocation routines when they are about to fail 13249 * in the hope that we can free up the requested resource (inodes 13250 * or disk space). 13251 * 13252 * First check to see if the work list has anything on it. If it has, 13253 * clean up entries until we successfully free the requested resource. 13254 * Because this process holds inodes locked, we cannot handle any remove 13255 * requests that might block on a locked inode as that could lead to 13256 * deadlock. If the worklist yields none of the requested resource, 13257 * start syncing out vnodes to free up the needed space. 13258 */ 13259 int 13260 softdep_request_cleanup(fs, vp, cred, resource) 13261 struct fs *fs; 13262 struct vnode *vp; 13263 struct ucred *cred; 13264 int resource; 13265 { 13266 struct ufsmount *ump; 13267 struct mount *mp; 13268 struct vnode *lvp, *mvp; 13269 long starttime; 13270 ufs2_daddr_t needed; 13271 int error; 13272 13273 /* 13274 * If we are being called because of a process doing a 13275 * copy-on-write, then it is not safe to process any 13276 * worklist items as we will recurse into the copyonwrite 13277 * routine. This will result in an incoherent snapshot. 13278 * If the vnode that we hold is a snapshot, we must avoid 13279 * handling other resources that could cause deadlock. 13280 */ 13281 if ((curthread->td_pflags & TDP_COWINPROGRESS) || IS_SNAPSHOT(VTOI(vp))) 13282 return (0); 13283 13284 if (resource == FLUSH_BLOCKS_WAIT) 13285 stat_cleanup_blkrequests += 1; 13286 else 13287 stat_cleanup_inorequests += 1; 13288 13289 mp = vp->v_mount; 13290 ump = VFSTOUFS(mp); 13291 mtx_assert(UFS_MTX(ump), MA_OWNED); 13292 UFS_UNLOCK(ump); 13293 error = ffs_update(vp, 1); 13294 if (error != 0 || MOUNTEDSOFTDEP(mp) == 0) { 13295 UFS_LOCK(ump); 13296 return (0); 13297 } 13298 /* 13299 * If we are in need of resources, start by cleaning up 13300 * any block removals associated with our inode. 13301 */ 13302 ACQUIRE_LOCK(ump); 13303 process_removes(vp); 13304 process_truncates(vp); 13305 FREE_LOCK(ump); 13306 /* 13307 * Now clean up at least as many resources as we will need. 13308 * 13309 * When requested to clean up inodes, the number that are needed 13310 * is set by the number of simultaneous writers (mnt_writeopcount) 13311 * plus a bit of slop (2) in case some more writers show up while 13312 * we are cleaning. 13313 * 13314 * When requested to free up space, the amount of space that 13315 * we need is enough blocks to allocate a full-sized segment 13316 * (fs_contigsumsize). The number of such segments that will 13317 * be needed is set by the number of simultaneous writers 13318 * (mnt_writeopcount) plus a bit of slop (2) in case some more 13319 * writers show up while we are cleaning. 13320 * 13321 * Additionally, if we are unpriviledged and allocating space, 13322 * we need to ensure that we clean up enough blocks to get the 13323 * needed number of blocks over the threshold of the minimum 13324 * number of blocks required to be kept free by the filesystem 13325 * (fs_minfree). 13326 */ 13327 if (resource == FLUSH_INODES_WAIT) { 13328 needed = vp->v_mount->mnt_writeopcount + 2; 13329 } else if (resource == FLUSH_BLOCKS_WAIT) { 13330 needed = (vp->v_mount->mnt_writeopcount + 2) * 13331 fs->fs_contigsumsize; 13332 if (priv_check_cred(cred, PRIV_VFS_BLOCKRESERVE, 0)) 13333 needed += fragstoblks(fs, 13334 roundup((fs->fs_dsize * fs->fs_minfree / 100) - 13335 fs->fs_cstotal.cs_nffree, fs->fs_frag)); 13336 } else { 13337 UFS_LOCK(ump); 13338 printf("softdep_request_cleanup: Unknown resource type %d\n", 13339 resource); 13340 return (0); 13341 } 13342 starttime = time_second; 13343 retry: 13344 if ((resource == FLUSH_BLOCKS_WAIT && ump->softdep_on_worklist > 0 && 13345 fs->fs_cstotal.cs_nbfree <= needed) || 13346 (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 && 13347 fs->fs_cstotal.cs_nifree <= needed)) { 13348 ACQUIRE_LOCK(ump); 13349 if (ump->softdep_on_worklist > 0 && 13350 process_worklist_item(UFSTOVFS(ump), 13351 ump->softdep_on_worklist, LK_NOWAIT) != 0) 13352 stat_worklist_push += 1; 13353 FREE_LOCK(ump); 13354 } 13355 /* 13356 * If we still need resources and there are no more worklist 13357 * entries to process to obtain them, we have to start flushing 13358 * the dirty vnodes to force the release of additional requests 13359 * to the worklist that we can then process to reap addition 13360 * resources. We walk the vnodes associated with the mount point 13361 * until we get the needed worklist requests that we can reap. 13362 */ 13363 if ((resource == FLUSH_BLOCKS_WAIT && 13364 fs->fs_cstotal.cs_nbfree <= needed) || 13365 (resource == FLUSH_INODES_WAIT && fs->fs_pendinginodes > 0 && 13366 fs->fs_cstotal.cs_nifree <= needed)) { 13367 MNT_VNODE_FOREACH_ALL(lvp, mp, mvp) { 13368 if (TAILQ_FIRST(&lvp->v_bufobj.bo_dirty.bv_hd) == 0) { 13369 VI_UNLOCK(lvp); 13370 continue; 13371 } 13372 if (vget(lvp, LK_EXCLUSIVE | LK_INTERLOCK | LK_NOWAIT, 13373 curthread)) 13374 continue; 13375 if (lvp->v_vflag & VV_NOSYNC) { /* unlinked */ 13376 vput(lvp); 13377 continue; 13378 } 13379 (void) ffs_syncvnode(lvp, MNT_NOWAIT, 0); 13380 vput(lvp); 13381 } 13382 lvp = ump->um_devvp; 13383 if (vn_lock(lvp, LK_EXCLUSIVE | LK_NOWAIT) == 0) { 13384 VOP_FSYNC(lvp, MNT_NOWAIT, curthread); 13385 VOP_UNLOCK(lvp, 0); 13386 } 13387 if (ump->softdep_on_worklist > 0) { 13388 stat_cleanup_retries += 1; 13389 goto retry; 13390 } 13391 stat_cleanup_failures += 1; 13392 } 13393 if (time_second - starttime > stat_cleanup_high_delay) 13394 stat_cleanup_high_delay = time_second - starttime; 13395 UFS_LOCK(ump); 13396 return (1); 13397 } 13398 13399 static bool 13400 softdep_excess_items(struct ufsmount *ump, int item) 13401 { 13402 13403 KASSERT(item >= 0 && item < D_LAST, ("item %d", item)); 13404 return (dep_current[item] > max_softdeps && 13405 ump->softdep_curdeps[item] > max_softdeps / 13406 stat_flush_threads); 13407 } 13408 13409 static void 13410 schedule_cleanup(struct mount *mp) 13411 { 13412 struct ufsmount *ump; 13413 struct thread *td; 13414 13415 ump = VFSTOUFS(mp); 13416 LOCK_OWNED(ump); 13417 FREE_LOCK(ump); 13418 td = curthread; 13419 if ((td->td_pflags & TDP_KTHREAD) != 0 && 13420 (td->td_proc->p_flag2 & P2_AST_SU) == 0) { 13421 /* 13422 * No ast is delivered to kernel threads, so nobody 13423 * would deref the mp. Some kernel threads 13424 * explicitely check for AST, e.g. NFS daemon does 13425 * this in the serving loop. 13426 */ 13427 return; 13428 } 13429 if (td->td_su != NULL) 13430 vfs_rel(td->td_su); 13431 vfs_ref(mp); 13432 td->td_su = mp; 13433 thread_lock(td); 13434 td->td_flags |= TDF_ASTPENDING; 13435 thread_unlock(td); 13436 } 13437 13438 static void 13439 softdep_ast_cleanup_proc(void) 13440 { 13441 struct thread *td; 13442 struct mount *mp; 13443 struct ufsmount *ump; 13444 int error; 13445 bool req; 13446 13447 td = curthread; 13448 while ((mp = td->td_su) != NULL) { 13449 td->td_su = NULL; 13450 error = vfs_busy(mp, MBF_NOWAIT); 13451 vfs_rel(mp); 13452 if (error != 0) 13453 return; 13454 if (ffs_own_mount(mp) && MOUNTEDSOFTDEP(mp)) { 13455 ump = VFSTOUFS(mp); 13456 for (;;) { 13457 req = false; 13458 ACQUIRE_LOCK(ump); 13459 if (softdep_excess_items(ump, D_INODEDEP)) { 13460 req = true; 13461 request_cleanup(mp, FLUSH_INODES); 13462 } 13463 if (softdep_excess_items(ump, D_DIRREM)) { 13464 req = true; 13465 request_cleanup(mp, FLUSH_BLOCKS); 13466 } 13467 FREE_LOCK(ump); 13468 if (softdep_excess_items(ump, D_NEWBLK) || 13469 softdep_excess_items(ump, D_ALLOCDIRECT) || 13470 softdep_excess_items(ump, D_ALLOCINDIR)) { 13471 error = vn_start_write(NULL, &mp, 13472 V_WAIT); 13473 if (error == 0) { 13474 req = true; 13475 VFS_SYNC(mp, MNT_WAIT); 13476 vn_finished_write(mp); 13477 } 13478 } 13479 if ((td->td_pflags & TDP_KTHREAD) != 0 || !req) 13480 break; 13481 } 13482 } 13483 vfs_unbusy(mp); 13484 } 13485 } 13486 13487 /* 13488 * If memory utilization has gotten too high, deliberately slow things 13489 * down and speed up the I/O processing. 13490 */ 13491 static int 13492 request_cleanup(mp, resource) 13493 struct mount *mp; 13494 int resource; 13495 { 13496 struct thread *td = curthread; 13497 struct ufsmount *ump; 13498 13499 ump = VFSTOUFS(mp); 13500 LOCK_OWNED(ump); 13501 /* 13502 * We never hold up the filesystem syncer or buf daemon. 13503 */ 13504 if (td->td_pflags & (TDP_SOFTDEP|TDP_NORUNNINGBUF)) 13505 return (0); 13506 /* 13507 * First check to see if the work list has gotten backlogged. 13508 * If it has, co-opt this process to help clean up two entries. 13509 * Because this process may hold inodes locked, we cannot 13510 * handle any remove requests that might block on a locked 13511 * inode as that could lead to deadlock. We set TDP_SOFTDEP 13512 * to avoid recursively processing the worklist. 13513 */ 13514 if (ump->softdep_on_worklist > max_softdeps / 10) { 13515 td->td_pflags |= TDP_SOFTDEP; 13516 process_worklist_item(mp, 2, LK_NOWAIT); 13517 td->td_pflags &= ~TDP_SOFTDEP; 13518 stat_worklist_push += 2; 13519 return(1); 13520 } 13521 /* 13522 * Next, we attempt to speed up the syncer process. If that 13523 * is successful, then we allow the process to continue. 13524 */ 13525 if (softdep_speedup(ump) && 13526 resource != FLUSH_BLOCKS_WAIT && 13527 resource != FLUSH_INODES_WAIT) 13528 return(0); 13529 /* 13530 * If we are resource constrained on inode dependencies, try 13531 * flushing some dirty inodes. Otherwise, we are constrained 13532 * by file deletions, so try accelerating flushes of directories 13533 * with removal dependencies. We would like to do the cleanup 13534 * here, but we probably hold an inode locked at this point and 13535 * that might deadlock against one that we try to clean. So, 13536 * the best that we can do is request the syncer daemon to do 13537 * the cleanup for us. 13538 */ 13539 switch (resource) { 13540 13541 case FLUSH_INODES: 13542 case FLUSH_INODES_WAIT: 13543 ACQUIRE_GBLLOCK(&lk); 13544 stat_ino_limit_push += 1; 13545 req_clear_inodedeps += 1; 13546 FREE_GBLLOCK(&lk); 13547 stat_countp = &stat_ino_limit_hit; 13548 break; 13549 13550 case FLUSH_BLOCKS: 13551 case FLUSH_BLOCKS_WAIT: 13552 ACQUIRE_GBLLOCK(&lk); 13553 stat_blk_limit_push += 1; 13554 req_clear_remove += 1; 13555 FREE_GBLLOCK(&lk); 13556 stat_countp = &stat_blk_limit_hit; 13557 break; 13558 13559 default: 13560 panic("request_cleanup: unknown type"); 13561 } 13562 /* 13563 * Hopefully the syncer daemon will catch up and awaken us. 13564 * We wait at most tickdelay before proceeding in any case. 13565 */ 13566 ACQUIRE_GBLLOCK(&lk); 13567 FREE_LOCK(ump); 13568 proc_waiting += 1; 13569 if (callout_pending(&softdep_callout) == FALSE) 13570 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2, 13571 pause_timer, 0); 13572 13573 if ((td->td_pflags & TDP_KTHREAD) == 0) 13574 msleep((caddr_t)&proc_waiting, &lk, PPAUSE, "softupdate", 0); 13575 proc_waiting -= 1; 13576 FREE_GBLLOCK(&lk); 13577 ACQUIRE_LOCK(ump); 13578 return (1); 13579 } 13580 13581 /* 13582 * Awaken processes pausing in request_cleanup and clear proc_waiting 13583 * to indicate that there is no longer a timer running. Pause_timer 13584 * will be called with the global softdep mutex (&lk) locked. 13585 */ 13586 static void 13587 pause_timer(arg) 13588 void *arg; 13589 { 13590 13591 GBLLOCK_OWNED(&lk); 13592 /* 13593 * The callout_ API has acquired mtx and will hold it around this 13594 * function call. 13595 */ 13596 *stat_countp += proc_waiting; 13597 wakeup(&proc_waiting); 13598 } 13599 13600 /* 13601 * If requested, try removing inode or removal dependencies. 13602 */ 13603 static void 13604 check_clear_deps(mp) 13605 struct mount *mp; 13606 { 13607 13608 /* 13609 * If we are suspended, it may be because of our using 13610 * too many inodedeps, so help clear them out. 13611 */ 13612 if (MOUNTEDSUJ(mp) && VFSTOUFS(mp)->softdep_jblocks->jb_suspended) 13613 clear_inodedeps(mp); 13614 /* 13615 * General requests for cleanup of backed up dependencies 13616 */ 13617 ACQUIRE_GBLLOCK(&lk); 13618 if (req_clear_inodedeps) { 13619 req_clear_inodedeps -= 1; 13620 FREE_GBLLOCK(&lk); 13621 clear_inodedeps(mp); 13622 ACQUIRE_GBLLOCK(&lk); 13623 wakeup(&proc_waiting); 13624 } 13625 if (req_clear_remove) { 13626 req_clear_remove -= 1; 13627 FREE_GBLLOCK(&lk); 13628 clear_remove(mp); 13629 ACQUIRE_GBLLOCK(&lk); 13630 wakeup(&proc_waiting); 13631 } 13632 FREE_GBLLOCK(&lk); 13633 } 13634 13635 /* 13636 * Flush out a directory with at least one removal dependency in an effort to 13637 * reduce the number of dirrem, freefile, and freeblks dependency structures. 13638 */ 13639 static void 13640 clear_remove(mp) 13641 struct mount *mp; 13642 { 13643 struct pagedep_hashhead *pagedephd; 13644 struct pagedep *pagedep; 13645 struct ufsmount *ump; 13646 struct vnode *vp; 13647 struct bufobj *bo; 13648 int error, cnt; 13649 ino_t ino; 13650 13651 ump = VFSTOUFS(mp); 13652 LOCK_OWNED(ump); 13653 13654 for (cnt = 0; cnt <= ump->pagedep_hash_size; cnt++) { 13655 pagedephd = &ump->pagedep_hashtbl[ump->pagedep_nextclean++]; 13656 if (ump->pagedep_nextclean > ump->pagedep_hash_size) 13657 ump->pagedep_nextclean = 0; 13658 LIST_FOREACH(pagedep, pagedephd, pd_hash) { 13659 if (LIST_EMPTY(&pagedep->pd_dirremhd)) 13660 continue; 13661 ino = pagedep->pd_ino; 13662 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 13663 continue; 13664 FREE_LOCK(ump); 13665 13666 /* 13667 * Let unmount clear deps 13668 */ 13669 error = vfs_busy(mp, MBF_NOWAIT); 13670 if (error != 0) 13671 goto finish_write; 13672 error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 13673 FFSV_FORCEINSMQ); 13674 vfs_unbusy(mp); 13675 if (error != 0) { 13676 softdep_error("clear_remove: vget", error); 13677 goto finish_write; 13678 } 13679 if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0))) 13680 softdep_error("clear_remove: fsync", error); 13681 bo = &vp->v_bufobj; 13682 BO_LOCK(bo); 13683 drain_output(vp); 13684 BO_UNLOCK(bo); 13685 vput(vp); 13686 finish_write: 13687 vn_finished_write(mp); 13688 ACQUIRE_LOCK(ump); 13689 return; 13690 } 13691 } 13692 } 13693 13694 /* 13695 * Clear out a block of dirty inodes in an effort to reduce 13696 * the number of inodedep dependency structures. 13697 */ 13698 static void 13699 clear_inodedeps(mp) 13700 struct mount *mp; 13701 { 13702 struct inodedep_hashhead *inodedephd; 13703 struct inodedep *inodedep; 13704 struct ufsmount *ump; 13705 struct vnode *vp; 13706 struct fs *fs; 13707 int error, cnt; 13708 ino_t firstino, lastino, ino; 13709 13710 ump = VFSTOUFS(mp); 13711 fs = ump->um_fs; 13712 LOCK_OWNED(ump); 13713 /* 13714 * Pick a random inode dependency to be cleared. 13715 * We will then gather up all the inodes in its block 13716 * that have dependencies and flush them out. 13717 */ 13718 for (cnt = 0; cnt <= ump->inodedep_hash_size; cnt++) { 13719 inodedephd = &ump->inodedep_hashtbl[ump->inodedep_nextclean++]; 13720 if (ump->inodedep_nextclean > ump->inodedep_hash_size) 13721 ump->inodedep_nextclean = 0; 13722 if ((inodedep = LIST_FIRST(inodedephd)) != NULL) 13723 break; 13724 } 13725 if (inodedep == NULL) 13726 return; 13727 /* 13728 * Find the last inode in the block with dependencies. 13729 */ 13730 firstino = rounddown2(inodedep->id_ino, INOPB(fs)); 13731 for (lastino = firstino + INOPB(fs) - 1; lastino > firstino; lastino--) 13732 if (inodedep_lookup(mp, lastino, 0, &inodedep) != 0) 13733 break; 13734 /* 13735 * Asynchronously push all but the last inode with dependencies. 13736 * Synchronously push the last inode with dependencies to ensure 13737 * that the inode block gets written to free up the inodedeps. 13738 */ 13739 for (ino = firstino; ino <= lastino; ino++) { 13740 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 13741 continue; 13742 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 13743 continue; 13744 FREE_LOCK(ump); 13745 error = vfs_busy(mp, MBF_NOWAIT); /* Let unmount clear deps */ 13746 if (error != 0) { 13747 vn_finished_write(mp); 13748 ACQUIRE_LOCK(ump); 13749 return; 13750 } 13751 if ((error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 13752 FFSV_FORCEINSMQ)) != 0) { 13753 softdep_error("clear_inodedeps: vget", error); 13754 vfs_unbusy(mp); 13755 vn_finished_write(mp); 13756 ACQUIRE_LOCK(ump); 13757 return; 13758 } 13759 vfs_unbusy(mp); 13760 if (ino == lastino) { 13761 if ((error = ffs_syncvnode(vp, MNT_WAIT, 0))) 13762 softdep_error("clear_inodedeps: fsync1", error); 13763 } else { 13764 if ((error = ffs_syncvnode(vp, MNT_NOWAIT, 0))) 13765 softdep_error("clear_inodedeps: fsync2", error); 13766 BO_LOCK(&vp->v_bufobj); 13767 drain_output(vp); 13768 BO_UNLOCK(&vp->v_bufobj); 13769 } 13770 vput(vp); 13771 vn_finished_write(mp); 13772 ACQUIRE_LOCK(ump); 13773 } 13774 } 13775 13776 void 13777 softdep_buf_append(bp, wkhd) 13778 struct buf *bp; 13779 struct workhead *wkhd; 13780 { 13781 struct worklist *wk; 13782 struct ufsmount *ump; 13783 13784 if ((wk = LIST_FIRST(wkhd)) == NULL) 13785 return; 13786 KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0, 13787 ("softdep_buf_append called on non-softdep filesystem")); 13788 ump = VFSTOUFS(wk->wk_mp); 13789 ACQUIRE_LOCK(ump); 13790 while ((wk = LIST_FIRST(wkhd)) != NULL) { 13791 WORKLIST_REMOVE(wk); 13792 WORKLIST_INSERT(&bp->b_dep, wk); 13793 } 13794 FREE_LOCK(ump); 13795 13796 } 13797 13798 void 13799 softdep_inode_append(ip, cred, wkhd) 13800 struct inode *ip; 13801 struct ucred *cred; 13802 struct workhead *wkhd; 13803 { 13804 struct buf *bp; 13805 struct fs *fs; 13806 struct ufsmount *ump; 13807 int error; 13808 13809 ump = ITOUMP(ip); 13810 KASSERT(MOUNTEDSOFTDEP(UFSTOVFS(ump)) != 0, 13811 ("softdep_inode_append called on non-softdep filesystem")); 13812 fs = ump->um_fs; 13813 error = bread(ump->um_devvp, fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 13814 (int)fs->fs_bsize, cred, &bp); 13815 if (error) { 13816 bqrelse(bp); 13817 softdep_freework(wkhd); 13818 return; 13819 } 13820 softdep_buf_append(bp, wkhd); 13821 bqrelse(bp); 13822 } 13823 13824 void 13825 softdep_freework(wkhd) 13826 struct workhead *wkhd; 13827 { 13828 struct worklist *wk; 13829 struct ufsmount *ump; 13830 13831 if ((wk = LIST_FIRST(wkhd)) == NULL) 13832 return; 13833 KASSERT(MOUNTEDSOFTDEP(wk->wk_mp) != 0, 13834 ("softdep_freework called on non-softdep filesystem")); 13835 ump = VFSTOUFS(wk->wk_mp); 13836 ACQUIRE_LOCK(ump); 13837 handle_jwork(wkhd); 13838 FREE_LOCK(ump); 13839 } 13840 13841 /* 13842 * Function to determine if the buffer has outstanding dependencies 13843 * that will cause a roll-back if the buffer is written. If wantcount 13844 * is set, return number of dependencies, otherwise just yes or no. 13845 */ 13846 static int 13847 softdep_count_dependencies(bp, wantcount) 13848 struct buf *bp; 13849 int wantcount; 13850 { 13851 struct worklist *wk; 13852 struct ufsmount *ump; 13853 struct bmsafemap *bmsafemap; 13854 struct freework *freework; 13855 struct inodedep *inodedep; 13856 struct indirdep *indirdep; 13857 struct freeblks *freeblks; 13858 struct allocindir *aip; 13859 struct pagedep *pagedep; 13860 struct dirrem *dirrem; 13861 struct newblk *newblk; 13862 struct mkdir *mkdir; 13863 struct diradd *dap; 13864 int i, retval; 13865 13866 retval = 0; 13867 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL) 13868 return (0); 13869 ump = VFSTOUFS(wk->wk_mp); 13870 ACQUIRE_LOCK(ump); 13871 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 13872 switch (wk->wk_type) { 13873 13874 case D_INODEDEP: 13875 inodedep = WK_INODEDEP(wk); 13876 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 13877 /* bitmap allocation dependency */ 13878 retval += 1; 13879 if (!wantcount) 13880 goto out; 13881 } 13882 if (TAILQ_FIRST(&inodedep->id_inoupdt)) { 13883 /* direct block pointer dependency */ 13884 retval += 1; 13885 if (!wantcount) 13886 goto out; 13887 } 13888 if (TAILQ_FIRST(&inodedep->id_extupdt)) { 13889 /* direct block pointer dependency */ 13890 retval += 1; 13891 if (!wantcount) 13892 goto out; 13893 } 13894 if (TAILQ_FIRST(&inodedep->id_inoreflst)) { 13895 /* Add reference dependency. */ 13896 retval += 1; 13897 if (!wantcount) 13898 goto out; 13899 } 13900 continue; 13901 13902 case D_INDIRDEP: 13903 indirdep = WK_INDIRDEP(wk); 13904 13905 TAILQ_FOREACH(freework, &indirdep->ir_trunc, fw_next) { 13906 /* indirect truncation dependency */ 13907 retval += 1; 13908 if (!wantcount) 13909 goto out; 13910 } 13911 13912 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) { 13913 /* indirect block pointer dependency */ 13914 retval += 1; 13915 if (!wantcount) 13916 goto out; 13917 } 13918 continue; 13919 13920 case D_PAGEDEP: 13921 pagedep = WK_PAGEDEP(wk); 13922 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) { 13923 if (LIST_FIRST(&dirrem->dm_jremrefhd)) { 13924 /* Journal remove ref dependency. */ 13925 retval += 1; 13926 if (!wantcount) 13927 goto out; 13928 } 13929 } 13930 for (i = 0; i < DAHASHSZ; i++) { 13931 13932 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 13933 /* directory entry dependency */ 13934 retval += 1; 13935 if (!wantcount) 13936 goto out; 13937 } 13938 } 13939 continue; 13940 13941 case D_BMSAFEMAP: 13942 bmsafemap = WK_BMSAFEMAP(wk); 13943 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd)) { 13944 /* Add reference dependency. */ 13945 retval += 1; 13946 if (!wantcount) 13947 goto out; 13948 } 13949 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd)) { 13950 /* Allocate block dependency. */ 13951 retval += 1; 13952 if (!wantcount) 13953 goto out; 13954 } 13955 continue; 13956 13957 case D_FREEBLKS: 13958 freeblks = WK_FREEBLKS(wk); 13959 if (LIST_FIRST(&freeblks->fb_jblkdephd)) { 13960 /* Freeblk journal dependency. */ 13961 retval += 1; 13962 if (!wantcount) 13963 goto out; 13964 } 13965 continue; 13966 13967 case D_ALLOCDIRECT: 13968 case D_ALLOCINDIR: 13969 newblk = WK_NEWBLK(wk); 13970 if (newblk->nb_jnewblk) { 13971 /* Journal allocate dependency. */ 13972 retval += 1; 13973 if (!wantcount) 13974 goto out; 13975 } 13976 continue; 13977 13978 case D_MKDIR: 13979 mkdir = WK_MKDIR(wk); 13980 if (mkdir->md_jaddref) { 13981 /* Journal reference dependency. */ 13982 retval += 1; 13983 if (!wantcount) 13984 goto out; 13985 } 13986 continue; 13987 13988 case D_FREEWORK: 13989 case D_FREEDEP: 13990 case D_JSEGDEP: 13991 case D_JSEG: 13992 case D_SBDEP: 13993 /* never a dependency on these blocks */ 13994 continue; 13995 13996 default: 13997 panic("softdep_count_dependencies: Unexpected type %s", 13998 TYPENAME(wk->wk_type)); 13999 /* NOTREACHED */ 14000 } 14001 } 14002 out: 14003 FREE_LOCK(ump); 14004 return retval; 14005 } 14006 14007 /* 14008 * Acquire exclusive access to a buffer. 14009 * Must be called with a locked mtx parameter. 14010 * Return acquired buffer or NULL on failure. 14011 */ 14012 static struct buf * 14013 getdirtybuf(bp, lock, waitfor) 14014 struct buf *bp; 14015 struct rwlock *lock; 14016 int waitfor; 14017 { 14018 int error; 14019 14020 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) { 14021 if (waitfor != MNT_WAIT) 14022 return (NULL); 14023 error = BUF_LOCK(bp, 14024 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, lock); 14025 /* 14026 * Even if we successfully acquire bp here, we have dropped 14027 * lock, which may violates our guarantee. 14028 */ 14029 if (error == 0) 14030 BUF_UNLOCK(bp); 14031 else if (error != ENOLCK) 14032 panic("getdirtybuf: inconsistent lock: %d", error); 14033 rw_wlock(lock); 14034 return (NULL); 14035 } 14036 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 14037 if (lock != BO_LOCKPTR(bp->b_bufobj) && waitfor == MNT_WAIT) { 14038 rw_wunlock(lock); 14039 BO_LOCK(bp->b_bufobj); 14040 BUF_UNLOCK(bp); 14041 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 14042 bp->b_vflags |= BV_BKGRDWAIT; 14043 msleep(&bp->b_xflags, BO_LOCKPTR(bp->b_bufobj), 14044 PRIBIO | PDROP, "getbuf", 0); 14045 } else 14046 BO_UNLOCK(bp->b_bufobj); 14047 rw_wlock(lock); 14048 return (NULL); 14049 } 14050 BUF_UNLOCK(bp); 14051 if (waitfor != MNT_WAIT) 14052 return (NULL); 14053 /* 14054 * The lock argument must be bp->b_vp's mutex in 14055 * this case. 14056 */ 14057 #ifdef DEBUG_VFS_LOCKS 14058 if (bp->b_vp->v_type != VCHR) 14059 ASSERT_BO_WLOCKED(bp->b_bufobj); 14060 #endif 14061 bp->b_vflags |= BV_BKGRDWAIT; 14062 rw_sleep(&bp->b_xflags, lock, PRIBIO, "getbuf", 0); 14063 return (NULL); 14064 } 14065 if ((bp->b_flags & B_DELWRI) == 0) { 14066 BUF_UNLOCK(bp); 14067 return (NULL); 14068 } 14069 bremfree(bp); 14070 return (bp); 14071 } 14072 14073 14074 /* 14075 * Check if it is safe to suspend the file system now. On entry, 14076 * the vnode interlock for devvp should be held. Return 0 with 14077 * the mount interlock held if the file system can be suspended now, 14078 * otherwise return EAGAIN with the mount interlock held. 14079 */ 14080 int 14081 softdep_check_suspend(struct mount *mp, 14082 struct vnode *devvp, 14083 int softdep_depcnt, 14084 int softdep_accdepcnt, 14085 int secondary_writes, 14086 int secondary_accwrites) 14087 { 14088 struct bufobj *bo; 14089 struct ufsmount *ump; 14090 struct inodedep *inodedep; 14091 int error, unlinked; 14092 14093 bo = &devvp->v_bufobj; 14094 ASSERT_BO_WLOCKED(bo); 14095 14096 /* 14097 * If we are not running with soft updates, then we need only 14098 * deal with secondary writes as we try to suspend. 14099 */ 14100 if (MOUNTEDSOFTDEP(mp) == 0) { 14101 MNT_ILOCK(mp); 14102 while (mp->mnt_secondary_writes != 0) { 14103 BO_UNLOCK(bo); 14104 msleep(&mp->mnt_secondary_writes, MNT_MTX(mp), 14105 (PUSER - 1) | PDROP, "secwr", 0); 14106 BO_LOCK(bo); 14107 MNT_ILOCK(mp); 14108 } 14109 14110 /* 14111 * Reasons for needing more work before suspend: 14112 * - Dirty buffers on devvp. 14113 * - Secondary writes occurred after start of vnode sync loop 14114 */ 14115 error = 0; 14116 if (bo->bo_numoutput > 0 || 14117 bo->bo_dirty.bv_cnt > 0 || 14118 secondary_writes != 0 || 14119 mp->mnt_secondary_writes != 0 || 14120 secondary_accwrites != mp->mnt_secondary_accwrites) 14121 error = EAGAIN; 14122 BO_UNLOCK(bo); 14123 return (error); 14124 } 14125 14126 /* 14127 * If we are running with soft updates, then we need to coordinate 14128 * with them as we try to suspend. 14129 */ 14130 ump = VFSTOUFS(mp); 14131 for (;;) { 14132 if (!TRY_ACQUIRE_LOCK(ump)) { 14133 BO_UNLOCK(bo); 14134 ACQUIRE_LOCK(ump); 14135 FREE_LOCK(ump); 14136 BO_LOCK(bo); 14137 continue; 14138 } 14139 MNT_ILOCK(mp); 14140 if (mp->mnt_secondary_writes != 0) { 14141 FREE_LOCK(ump); 14142 BO_UNLOCK(bo); 14143 msleep(&mp->mnt_secondary_writes, 14144 MNT_MTX(mp), 14145 (PUSER - 1) | PDROP, "secwr", 0); 14146 BO_LOCK(bo); 14147 continue; 14148 } 14149 break; 14150 } 14151 14152 unlinked = 0; 14153 if (MOUNTEDSUJ(mp)) { 14154 for (inodedep = TAILQ_FIRST(&ump->softdep_unlinked); 14155 inodedep != NULL; 14156 inodedep = TAILQ_NEXT(inodedep, id_unlinked)) { 14157 if ((inodedep->id_state & (UNLINKED | UNLINKLINKS | 14158 UNLINKONLIST)) != (UNLINKED | UNLINKLINKS | 14159 UNLINKONLIST) || 14160 !check_inodedep_free(inodedep)) 14161 continue; 14162 unlinked++; 14163 } 14164 } 14165 14166 /* 14167 * Reasons for needing more work before suspend: 14168 * - Dirty buffers on devvp. 14169 * - Softdep activity occurred after start of vnode sync loop 14170 * - Secondary writes occurred after start of vnode sync loop 14171 */ 14172 error = 0; 14173 if (bo->bo_numoutput > 0 || 14174 bo->bo_dirty.bv_cnt > 0 || 14175 softdep_depcnt != unlinked || 14176 ump->softdep_deps != unlinked || 14177 softdep_accdepcnt != ump->softdep_accdeps || 14178 secondary_writes != 0 || 14179 mp->mnt_secondary_writes != 0 || 14180 secondary_accwrites != mp->mnt_secondary_accwrites) 14181 error = EAGAIN; 14182 FREE_LOCK(ump); 14183 BO_UNLOCK(bo); 14184 return (error); 14185 } 14186 14187 14188 /* 14189 * Get the number of dependency structures for the file system, both 14190 * the current number and the total number allocated. These will 14191 * later be used to detect that softdep processing has occurred. 14192 */ 14193 void 14194 softdep_get_depcounts(struct mount *mp, 14195 int *softdep_depsp, 14196 int *softdep_accdepsp) 14197 { 14198 struct ufsmount *ump; 14199 14200 if (MOUNTEDSOFTDEP(mp) == 0) { 14201 *softdep_depsp = 0; 14202 *softdep_accdepsp = 0; 14203 return; 14204 } 14205 ump = VFSTOUFS(mp); 14206 ACQUIRE_LOCK(ump); 14207 *softdep_depsp = ump->softdep_deps; 14208 *softdep_accdepsp = ump->softdep_accdeps; 14209 FREE_LOCK(ump); 14210 } 14211 14212 /* 14213 * Wait for pending output on a vnode to complete. 14214 * Must be called with vnode lock and interlock locked. 14215 * 14216 * XXX: Should just be a call to bufobj_wwait(). 14217 */ 14218 static void 14219 drain_output(vp) 14220 struct vnode *vp; 14221 { 14222 struct bufobj *bo; 14223 14224 bo = &vp->v_bufobj; 14225 ASSERT_VOP_LOCKED(vp, "drain_output"); 14226 ASSERT_BO_WLOCKED(bo); 14227 14228 while (bo->bo_numoutput) { 14229 bo->bo_flag |= BO_WWAIT; 14230 msleep((caddr_t)&bo->bo_numoutput, 14231 BO_LOCKPTR(bo), PRIBIO + 1, "drainvp", 0); 14232 } 14233 } 14234 14235 /* 14236 * Called whenever a buffer that is being invalidated or reallocated 14237 * contains dependencies. This should only happen if an I/O error has 14238 * occurred. The routine is called with the buffer locked. 14239 */ 14240 static void 14241 softdep_deallocate_dependencies(bp) 14242 struct buf *bp; 14243 { 14244 14245 if ((bp->b_ioflags & BIO_ERROR) == 0) 14246 panic("softdep_deallocate_dependencies: dangling deps"); 14247 if (bp->b_vp != NULL && bp->b_vp->v_mount != NULL) 14248 softdep_error(bp->b_vp->v_mount->mnt_stat.f_mntonname, bp->b_error); 14249 else 14250 printf("softdep_deallocate_dependencies: " 14251 "got error %d while accessing filesystem\n", bp->b_error); 14252 if (bp->b_error != ENXIO) 14253 panic("softdep_deallocate_dependencies: unrecovered I/O error"); 14254 } 14255 14256 /* 14257 * Function to handle asynchronous write errors in the filesystem. 14258 */ 14259 static void 14260 softdep_error(func, error) 14261 char *func; 14262 int error; 14263 { 14264 14265 /* XXX should do something better! */ 14266 printf("%s: got error %d while accessing filesystem\n", func, error); 14267 } 14268 14269 #ifdef DDB 14270 14271 static void 14272 inodedep_print(struct inodedep *inodedep, int verbose) 14273 { 14274 db_printf("%p fs %p st %x ino %jd inoblk %jd delta %d nlink %d" 14275 " saveino %p\n", 14276 inodedep, inodedep->id_fs, inodedep->id_state, 14277 (intmax_t)inodedep->id_ino, 14278 (intmax_t)fsbtodb(inodedep->id_fs, 14279 ino_to_fsba(inodedep->id_fs, inodedep->id_ino)), 14280 inodedep->id_nlinkdelta, inodedep->id_savednlink, 14281 inodedep->id_savedino1); 14282 14283 if (verbose == 0) 14284 return; 14285 14286 db_printf("\tpendinghd %p, bufwait %p, inowait %p, inoreflst %p, " 14287 "mkdiradd %p\n", 14288 LIST_FIRST(&inodedep->id_pendinghd), 14289 LIST_FIRST(&inodedep->id_bufwait), 14290 LIST_FIRST(&inodedep->id_inowait), 14291 TAILQ_FIRST(&inodedep->id_inoreflst), 14292 inodedep->id_mkdiradd); 14293 db_printf("\tinoupdt %p, newinoupdt %p, extupdt %p, newextupdt %p\n", 14294 TAILQ_FIRST(&inodedep->id_inoupdt), 14295 TAILQ_FIRST(&inodedep->id_newinoupdt), 14296 TAILQ_FIRST(&inodedep->id_extupdt), 14297 TAILQ_FIRST(&inodedep->id_newextupdt)); 14298 } 14299 14300 DB_SHOW_COMMAND(inodedep, db_show_inodedep) 14301 { 14302 14303 if (have_addr == 0) { 14304 db_printf("Address required\n"); 14305 return; 14306 } 14307 inodedep_print((struct inodedep*)addr, 1); 14308 } 14309 14310 DB_SHOW_COMMAND(inodedeps, db_show_inodedeps) 14311 { 14312 struct inodedep_hashhead *inodedephd; 14313 struct inodedep *inodedep; 14314 struct ufsmount *ump; 14315 int cnt; 14316 14317 if (have_addr == 0) { 14318 db_printf("Address required\n"); 14319 return; 14320 } 14321 ump = (struct ufsmount *)addr; 14322 for (cnt = 0; cnt < ump->inodedep_hash_size; cnt++) { 14323 inodedephd = &ump->inodedep_hashtbl[cnt]; 14324 LIST_FOREACH(inodedep, inodedephd, id_hash) { 14325 inodedep_print(inodedep, 0); 14326 } 14327 } 14328 } 14329 14330 DB_SHOW_COMMAND(worklist, db_show_worklist) 14331 { 14332 struct worklist *wk; 14333 14334 if (have_addr == 0) { 14335 db_printf("Address required\n"); 14336 return; 14337 } 14338 wk = (struct worklist *)addr; 14339 printf("worklist: %p type %s state 0x%X\n", 14340 wk, TYPENAME(wk->wk_type), wk->wk_state); 14341 } 14342 14343 DB_SHOW_COMMAND(workhead, db_show_workhead) 14344 { 14345 struct workhead *wkhd; 14346 struct worklist *wk; 14347 int i; 14348 14349 if (have_addr == 0) { 14350 db_printf("Address required\n"); 14351 return; 14352 } 14353 wkhd = (struct workhead *)addr; 14354 wk = LIST_FIRST(wkhd); 14355 for (i = 0; i < 100 && wk != NULL; i++, wk = LIST_NEXT(wk, wk_list)) 14356 db_printf("worklist: %p type %s state 0x%X", 14357 wk, TYPENAME(wk->wk_type), wk->wk_state); 14358 if (i == 100) 14359 db_printf("workhead overflow"); 14360 printf("\n"); 14361 } 14362 14363 14364 DB_SHOW_COMMAND(mkdirs, db_show_mkdirs) 14365 { 14366 struct mkdirlist *mkdirlisthd; 14367 struct jaddref *jaddref; 14368 struct diradd *diradd; 14369 struct mkdir *mkdir; 14370 14371 if (have_addr == 0) { 14372 db_printf("Address required\n"); 14373 return; 14374 } 14375 mkdirlisthd = (struct mkdirlist *)addr; 14376 LIST_FOREACH(mkdir, mkdirlisthd, md_mkdirs) { 14377 diradd = mkdir->md_diradd; 14378 db_printf("mkdir: %p state 0x%X dap %p state 0x%X", 14379 mkdir, mkdir->md_state, diradd, diradd->da_state); 14380 if ((jaddref = mkdir->md_jaddref) != NULL) 14381 db_printf(" jaddref %p jaddref state 0x%X", 14382 jaddref, jaddref->ja_state); 14383 db_printf("\n"); 14384 } 14385 } 14386 14387 /* exported to ffs_vfsops.c */ 14388 extern void db_print_ffs(struct ufsmount *ump); 14389 void 14390 db_print_ffs(struct ufsmount *ump) 14391 { 14392 db_printf("mp %p %s devvp %p fs %p su_wl %d su_deps %d su_req %d\n", 14393 ump->um_mountp, ump->um_mountp->mnt_stat.f_mntonname, 14394 ump->um_devvp, ump->um_fs, ump->softdep_on_worklist, 14395 ump->softdep_deps, ump->softdep_req); 14396 } 14397 14398 #endif /* DDB */ 14399 14400 #endif /* SOFTUPDATES */ 14401