1 /*- 2 * Copyright 1998, 2000 Marshall Kirk McKusick. 3 * Copyright 2009, 2010 Jeffrey W. Roberson <jeff@FreeBSD.org> 4 * All rights reserved. 5 * 6 * The soft updates code is derived from the appendix of a University 7 * of Michigan technical report (Gregory R. Ganger and Yale N. Patt, 8 * "Soft Updates: A Solution to the Metadata Update Problem in File 9 * Systems", CSE-TR-254-95, August 1995). 10 * 11 * Further information about soft updates can be obtained from: 12 * 13 * Marshall Kirk McKusick http://www.mckusick.com/softdep/ 14 * 1614 Oxford Street mckusick@mckusick.com 15 * Berkeley, CA 94709-1608 +1-510-843-9542 16 * USA 17 * 18 * Redistribution and use in source and binary forms, with or without 19 * modification, are permitted provided that the following conditions 20 * are met: 21 * 22 * 1. Redistributions of source code must retain the above copyright 23 * notice, this list of conditions and the following disclaimer. 24 * 2. Redistributions in binary form must reproduce the above copyright 25 * notice, this list of conditions and the following disclaimer in the 26 * documentation and/or other materials provided with the distribution. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 34 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 37 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * from: @(#)ffs_softdep.c 9.59 (McKusick) 6/21/00 40 */ 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include "opt_ffs.h" 46 #include "opt_ddb.h" 47 48 /* 49 * For now we want the safety net that the DEBUG flag provides. 50 */ 51 #ifndef DEBUG 52 #define DEBUG 53 #endif 54 55 #include <sys/param.h> 56 #include <sys/kernel.h> 57 #include <sys/systm.h> 58 #include <sys/bio.h> 59 #include <sys/buf.h> 60 #include <sys/kdb.h> 61 #include <sys/kthread.h> 62 #include <sys/lock.h> 63 #include <sys/malloc.h> 64 #include <sys/mount.h> 65 #include <sys/mutex.h> 66 #include <sys/namei.h> 67 #include <sys/proc.h> 68 #include <sys/stat.h> 69 #include <sys/sysctl.h> 70 #include <sys/syslog.h> 71 #include <sys/vnode.h> 72 #include <sys/conf.h> 73 #include <ufs/ufs/dir.h> 74 #include <ufs/ufs/extattr.h> 75 #include <ufs/ufs/quota.h> 76 #include <ufs/ufs/inode.h> 77 #include <ufs/ufs/ufsmount.h> 78 #include <ufs/ffs/fs.h> 79 #include <ufs/ffs/softdep.h> 80 #include <ufs/ffs/ffs_extern.h> 81 #include <ufs/ufs/ufs_extern.h> 82 83 #include <vm/vm.h> 84 85 #include <ddb/ddb.h> 86 87 #ifndef SOFTUPDATES 88 89 int 90 softdep_flushfiles(oldmnt, flags, td) 91 struct mount *oldmnt; 92 int flags; 93 struct thread *td; 94 { 95 96 panic("softdep_flushfiles called"); 97 } 98 99 int 100 softdep_mount(devvp, mp, fs, cred) 101 struct vnode *devvp; 102 struct mount *mp; 103 struct fs *fs; 104 struct ucred *cred; 105 { 106 107 return (0); 108 } 109 110 void 111 softdep_initialize() 112 { 113 114 return; 115 } 116 117 void 118 softdep_uninitialize() 119 { 120 121 return; 122 } 123 124 void 125 softdep_unmount(mp) 126 struct mount *mp; 127 { 128 129 } 130 131 void 132 softdep_setup_sbupdate(ump, fs, bp) 133 struct ufsmount *ump; 134 struct fs *fs; 135 struct buf *bp; 136 { 137 } 138 139 void 140 softdep_setup_inomapdep(bp, ip, newinum) 141 struct buf *bp; 142 struct inode *ip; 143 ino_t newinum; 144 { 145 146 panic("softdep_setup_inomapdep called"); 147 } 148 149 void 150 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 151 struct buf *bp; 152 struct mount *mp; 153 ufs2_daddr_t newblkno; 154 int frags; 155 int oldfrags; 156 { 157 158 panic("softdep_setup_blkmapdep called"); 159 } 160 161 void 162 softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 163 struct inode *ip; 164 ufs_lbn_t lbn; 165 ufs2_daddr_t newblkno; 166 ufs2_daddr_t oldblkno; 167 long newsize; 168 long oldsize; 169 struct buf *bp; 170 { 171 172 panic("softdep_setup_allocdirect called"); 173 } 174 175 void 176 softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 177 struct inode *ip; 178 ufs_lbn_t lbn; 179 ufs2_daddr_t newblkno; 180 ufs2_daddr_t oldblkno; 181 long newsize; 182 long oldsize; 183 struct buf *bp; 184 { 185 186 panic("softdep_setup_allocext called"); 187 } 188 189 void 190 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 191 struct inode *ip; 192 ufs_lbn_t lbn; 193 struct buf *bp; 194 int ptrno; 195 ufs2_daddr_t newblkno; 196 ufs2_daddr_t oldblkno; 197 struct buf *nbp; 198 { 199 200 panic("softdep_setup_allocindir_page called"); 201 } 202 203 void 204 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 205 struct buf *nbp; 206 struct inode *ip; 207 struct buf *bp; 208 int ptrno; 209 ufs2_daddr_t newblkno; 210 { 211 212 panic("softdep_setup_allocindir_meta called"); 213 } 214 215 void 216 softdep_setup_freeblocks(ip, length, flags) 217 struct inode *ip; 218 off_t length; 219 int flags; 220 { 221 222 panic("softdep_setup_freeblocks called"); 223 } 224 225 void 226 softdep_freefile(pvp, ino, mode) 227 struct vnode *pvp; 228 ino_t ino; 229 int mode; 230 { 231 232 panic("softdep_freefile called"); 233 } 234 235 int 236 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 237 struct buf *bp; 238 struct inode *dp; 239 off_t diroffset; 240 ino_t newinum; 241 struct buf *newdirbp; 242 int isnewblk; 243 { 244 245 panic("softdep_setup_directory_add called"); 246 } 247 248 void 249 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 250 struct buf *bp; 251 struct inode *dp; 252 caddr_t base; 253 caddr_t oldloc; 254 caddr_t newloc; 255 int entrysize; 256 { 257 258 panic("softdep_change_directoryentry_offset called"); 259 } 260 261 void 262 softdep_setup_remove(bp, dp, ip, isrmdir) 263 struct buf *bp; 264 struct inode *dp; 265 struct inode *ip; 266 int isrmdir; 267 { 268 269 panic("softdep_setup_remove called"); 270 } 271 272 void 273 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 274 struct buf *bp; 275 struct inode *dp; 276 struct inode *ip; 277 ino_t newinum; 278 int isrmdir; 279 { 280 281 panic("softdep_setup_directory_change called"); 282 } 283 284 void * 285 softdep_setup_trunc(vp, length, flags) 286 struct vnode *vp; 287 off_t length; 288 int flags; 289 { 290 291 panic("%s called", __FUNCTION__); 292 293 return (NULL); 294 } 295 296 int 297 softdep_complete_trunc(vp, cookie) 298 struct vnode *vp; 299 void *cookie; 300 { 301 302 panic("%s called", __FUNCTION__); 303 304 return (0); 305 } 306 307 void 308 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 309 struct mount *mp; 310 struct buf *bp; 311 ufs2_daddr_t blkno; 312 int frags; 313 struct workhead *wkhd; 314 { 315 316 panic("%s called", __FUNCTION__); 317 } 318 319 void 320 softdep_setup_inofree(mp, bp, ino, wkhd) 321 struct mount *mp; 322 struct buf *bp; 323 ino_t ino; 324 struct workhead *wkhd; 325 { 326 327 panic("%s called", __FUNCTION__); 328 } 329 330 void 331 softdep_setup_unlink(dp, ip) 332 struct inode *dp; 333 struct inode *ip; 334 { 335 336 panic("%s called", __FUNCTION__); 337 } 338 339 void 340 softdep_setup_link(dp, ip) 341 struct inode *dp; 342 struct inode *ip; 343 { 344 345 panic("%s called", __FUNCTION__); 346 } 347 348 void 349 softdep_revert_link(dp, ip) 350 struct inode *dp; 351 struct inode *ip; 352 { 353 354 panic("%s called", __FUNCTION__); 355 } 356 357 void 358 softdep_setup_rmdir(dp, ip) 359 struct inode *dp; 360 struct inode *ip; 361 { 362 363 panic("%s called", __FUNCTION__); 364 } 365 366 void 367 softdep_revert_rmdir(dp, ip) 368 struct inode *dp; 369 struct inode *ip; 370 { 371 372 panic("%s called", __FUNCTION__); 373 } 374 375 void 376 softdep_setup_create(dp, ip) 377 struct inode *dp; 378 struct inode *ip; 379 { 380 381 panic("%s called", __FUNCTION__); 382 } 383 384 void 385 softdep_revert_create(dp, ip) 386 struct inode *dp; 387 struct inode *ip; 388 { 389 390 panic("%s called", __FUNCTION__); 391 } 392 393 void 394 softdep_setup_mkdir(dp, ip) 395 struct inode *dp; 396 struct inode *ip; 397 { 398 399 panic("%s called", __FUNCTION__); 400 } 401 402 void 403 softdep_revert_mkdir(dp, ip) 404 struct inode *dp; 405 struct inode *ip; 406 { 407 408 panic("%s called", __FUNCTION__); 409 } 410 411 void 412 softdep_setup_dotdot_link(dp, ip) 413 struct inode *dp; 414 struct inode *ip; 415 { 416 417 panic("%s called", __FUNCTION__); 418 } 419 420 int 421 softdep_prealloc(vp, waitok) 422 struct vnode *vp; 423 int waitok; 424 { 425 426 panic("%s called", __FUNCTION__); 427 428 return (0); 429 } 430 431 int 432 softdep_journal_lookup(mp, vpp) 433 struct mount *mp; 434 struct vnode **vpp; 435 { 436 437 return (ENOENT); 438 } 439 440 void 441 softdep_change_linkcnt(ip) 442 struct inode *ip; 443 { 444 445 panic("softdep_change_linkcnt called"); 446 } 447 448 void 449 softdep_load_inodeblock(ip) 450 struct inode *ip; 451 { 452 453 panic("softdep_load_inodeblock called"); 454 } 455 456 void 457 softdep_update_inodeblock(ip, bp, waitfor) 458 struct inode *ip; 459 struct buf *bp; 460 int waitfor; 461 { 462 463 panic("softdep_update_inodeblock called"); 464 } 465 466 int 467 softdep_fsync(vp) 468 struct vnode *vp; /* the "in_core" copy of the inode */ 469 { 470 471 return (0); 472 } 473 474 void 475 softdep_fsync_mountdev(vp) 476 struct vnode *vp; 477 { 478 479 return; 480 } 481 482 int 483 softdep_flushworklist(oldmnt, countp, td) 484 struct mount *oldmnt; 485 int *countp; 486 struct thread *td; 487 { 488 489 *countp = 0; 490 return (0); 491 } 492 493 int 494 softdep_sync_metadata(struct vnode *vp) 495 { 496 497 return (0); 498 } 499 500 int 501 softdep_slowdown(vp) 502 struct vnode *vp; 503 { 504 505 panic("softdep_slowdown called"); 506 } 507 508 void 509 softdep_releasefile(ip) 510 struct inode *ip; /* inode with the zero effective link count */ 511 { 512 513 panic("softdep_releasefile called"); 514 } 515 516 int 517 softdep_request_cleanup(fs, vp) 518 struct fs *fs; 519 struct vnode *vp; 520 { 521 522 return (0); 523 } 524 525 int 526 softdep_check_suspend(struct mount *mp, 527 struct vnode *devvp, 528 int softdep_deps, 529 int softdep_accdeps, 530 int secondary_writes, 531 int secondary_accwrites) 532 { 533 struct bufobj *bo; 534 int error; 535 536 (void) softdep_deps, 537 (void) softdep_accdeps; 538 539 bo = &devvp->v_bufobj; 540 ASSERT_BO_LOCKED(bo); 541 542 MNT_ILOCK(mp); 543 while (mp->mnt_secondary_writes != 0) { 544 BO_UNLOCK(bo); 545 msleep(&mp->mnt_secondary_writes, MNT_MTX(mp), 546 (PUSER - 1) | PDROP, "secwr", 0); 547 BO_LOCK(bo); 548 MNT_ILOCK(mp); 549 } 550 551 /* 552 * Reasons for needing more work before suspend: 553 * - Dirty buffers on devvp. 554 * - Secondary writes occurred after start of vnode sync loop 555 */ 556 error = 0; 557 if (bo->bo_numoutput > 0 || 558 bo->bo_dirty.bv_cnt > 0 || 559 secondary_writes != 0 || 560 mp->mnt_secondary_writes != 0 || 561 secondary_accwrites != mp->mnt_secondary_accwrites) 562 error = EAGAIN; 563 BO_UNLOCK(bo); 564 return (error); 565 } 566 567 void 568 softdep_get_depcounts(struct mount *mp, 569 int *softdepactivep, 570 int *softdepactiveaccp) 571 { 572 (void) mp; 573 *softdepactivep = 0; 574 *softdepactiveaccp = 0; 575 } 576 577 #else 578 /* 579 * These definitions need to be adapted to the system to which 580 * this file is being ported. 581 */ 582 583 #define M_SOFTDEP_FLAGS (M_WAITOK | M_USE_RESERVE) 584 585 #define D_PAGEDEP 0 586 #define D_INODEDEP 1 587 #define D_BMSAFEMAP 2 588 #define D_NEWBLK 3 589 #define D_ALLOCDIRECT 4 590 #define D_INDIRDEP 5 591 #define D_ALLOCINDIR 6 592 #define D_FREEFRAG 7 593 #define D_FREEBLKS 8 594 #define D_FREEFILE 9 595 #define D_DIRADD 10 596 #define D_MKDIR 11 597 #define D_DIRREM 12 598 #define D_NEWDIRBLK 13 599 #define D_FREEWORK 14 600 #define D_FREEDEP 15 601 #define D_JADDREF 16 602 #define D_JREMREF 17 603 #define D_JMVREF 18 604 #define D_JNEWBLK 19 605 #define D_JFREEBLK 20 606 #define D_JFREEFRAG 21 607 #define D_JSEG 22 608 #define D_JSEGDEP 23 609 #define D_SBDEP 24 610 #define D_JTRUNC 25 611 #define D_LAST D_JTRUNC 612 613 unsigned long dep_current[D_LAST + 1]; 614 unsigned long dep_total[D_LAST + 1]; 615 616 617 SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW, 0, "soft updates stats"); 618 SYSCTL_NODE(_debug_softdep, OID_AUTO, total, CTLFLAG_RW, 0, 619 "total dependencies allocated"); 620 SYSCTL_NODE(_debug_softdep, OID_AUTO, current, CTLFLAG_RW, 0, 621 "current dependencies allocated"); 622 623 #define SOFTDEP_TYPE(type, str, long) \ 624 static MALLOC_DEFINE(M_ ## type, #str, long); \ 625 SYSCTL_LONG(_debug_softdep_total, OID_AUTO, str, CTLFLAG_RD, \ 626 &dep_total[D_ ## type], 0, ""); \ 627 SYSCTL_LONG(_debug_softdep_current, OID_AUTO, str, CTLFLAG_RD, \ 628 &dep_current[D_ ## type], 0, ""); 629 630 SOFTDEP_TYPE(PAGEDEP, pagedep, "File page dependencies"); 631 SOFTDEP_TYPE(INODEDEP, inodedep, "Inode dependencies"); 632 SOFTDEP_TYPE(BMSAFEMAP, bmsafemap, 633 "Block or frag allocated from cyl group map"); 634 SOFTDEP_TYPE(NEWBLK, newblk, "New block or frag allocation dependency"); 635 SOFTDEP_TYPE(ALLOCDIRECT, allocdirect, "Block or frag dependency for an inode"); 636 SOFTDEP_TYPE(INDIRDEP, indirdep, "Indirect block dependencies"); 637 SOFTDEP_TYPE(ALLOCINDIR, allocindir, "Block dependency for an indirect block"); 638 SOFTDEP_TYPE(FREEFRAG, freefrag, "Previously used frag for an inode"); 639 SOFTDEP_TYPE(FREEBLKS, freeblks, "Blocks freed from an inode"); 640 SOFTDEP_TYPE(FREEFILE, freefile, "Inode deallocated"); 641 SOFTDEP_TYPE(DIRADD, diradd, "New directory entry"); 642 SOFTDEP_TYPE(MKDIR, mkdir, "New directory"); 643 SOFTDEP_TYPE(DIRREM, dirrem, "Directory entry deleted"); 644 SOFTDEP_TYPE(NEWDIRBLK, newdirblk, "Unclaimed new directory block"); 645 SOFTDEP_TYPE(FREEWORK, freework, "free an inode block"); 646 SOFTDEP_TYPE(FREEDEP, freedep, "track a block free"); 647 SOFTDEP_TYPE(JADDREF, jaddref, "Journal inode ref add"); 648 SOFTDEP_TYPE(JREMREF, jremref, "Journal inode ref remove"); 649 SOFTDEP_TYPE(JMVREF, jmvref, "Journal inode ref move"); 650 SOFTDEP_TYPE(JNEWBLK, jnewblk, "Journal new block"); 651 SOFTDEP_TYPE(JFREEBLK, jfreeblk, "Journal free block"); 652 SOFTDEP_TYPE(JFREEFRAG, jfreefrag, "Journal free frag"); 653 SOFTDEP_TYPE(JSEG, jseg, "Journal segment"); 654 SOFTDEP_TYPE(JSEGDEP, jsegdep, "Journal segment complete"); 655 SOFTDEP_TYPE(SBDEP, sbdep, "Superblock write dependency"); 656 SOFTDEP_TYPE(JTRUNC, jtrunc, "Journal inode truncation"); 657 658 static MALLOC_DEFINE(M_SAVEDINO, "savedino", "Saved inodes"); 659 static MALLOC_DEFINE(M_JBLOCKS, "jblocks", "Journal block locations"); 660 661 /* 662 * translate from workitem type to memory type 663 * MUST match the defines above, such that memtype[D_XXX] == M_XXX 664 */ 665 static struct malloc_type *memtype[] = { 666 M_PAGEDEP, 667 M_INODEDEP, 668 M_BMSAFEMAP, 669 M_NEWBLK, 670 M_ALLOCDIRECT, 671 M_INDIRDEP, 672 M_ALLOCINDIR, 673 M_FREEFRAG, 674 M_FREEBLKS, 675 M_FREEFILE, 676 M_DIRADD, 677 M_MKDIR, 678 M_DIRREM, 679 M_NEWDIRBLK, 680 M_FREEWORK, 681 M_FREEDEP, 682 M_JADDREF, 683 M_JREMREF, 684 M_JMVREF, 685 M_JNEWBLK, 686 M_JFREEBLK, 687 M_JFREEFRAG, 688 M_JSEG, 689 M_JSEGDEP, 690 M_SBDEP, 691 M_JTRUNC 692 }; 693 694 #define DtoM(type) (memtype[type]) 695 696 /* 697 * Names of malloc types. 698 */ 699 #define TYPENAME(type) \ 700 ((unsigned)(type) <= D_LAST ? memtype[type]->ks_shortdesc : "???") 701 /* 702 * End system adaptation definitions. 703 */ 704 705 #define DOTDOT_OFFSET offsetof(struct dirtemplate, dotdot_ino) 706 #define DOT_OFFSET offsetof(struct dirtemplate, dot_ino) 707 708 /* 709 * Forward declarations. 710 */ 711 struct inodedep_hashhead; 712 struct newblk_hashhead; 713 struct pagedep_hashhead; 714 struct bmsafemap_hashhead; 715 716 /* 717 * Internal function prototypes. 718 */ 719 static void softdep_error(char *, int); 720 static void drain_output(struct vnode *); 721 static struct buf *getdirtybuf(struct buf *, struct mtx *, int); 722 static void clear_remove(struct thread *); 723 static void clear_inodedeps(struct thread *); 724 static void unlinked_inodedep(struct mount *, struct inodedep *); 725 static void clear_unlinked_inodedep(struct inodedep *); 726 static struct inodedep *first_unlinked_inodedep(struct ufsmount *); 727 static int flush_pagedep_deps(struct vnode *, struct mount *, 728 struct diraddhd *); 729 static void free_pagedep(struct pagedep *); 730 static int flush_newblk_dep(struct vnode *, struct mount *, ufs_lbn_t); 731 static int flush_inodedep_deps(struct mount *, ino_t); 732 static int flush_deplist(struct allocdirectlst *, int, int *); 733 static int handle_written_filepage(struct pagedep *, struct buf *); 734 static int handle_written_sbdep(struct sbdep *, struct buf *); 735 static void initiate_write_sbdep(struct sbdep *); 736 static void diradd_inode_written(struct diradd *, struct inodedep *); 737 static int handle_written_indirdep(struct indirdep *, struct buf *, 738 struct buf**); 739 static int handle_written_inodeblock(struct inodedep *, struct buf *); 740 static int handle_written_bmsafemap(struct bmsafemap *, struct buf *); 741 static void handle_written_jaddref(struct jaddref *); 742 static void handle_written_jremref(struct jremref *); 743 static void handle_written_jseg(struct jseg *, struct buf *); 744 static void handle_written_jnewblk(struct jnewblk *); 745 static void handle_written_jfreeblk(struct jfreeblk *); 746 static void handle_written_jfreefrag(struct jfreefrag *); 747 static void complete_jseg(struct jseg *); 748 static void jseg_write(struct fs *, struct jblocks *, struct jseg *, 749 uint8_t *); 750 static void jaddref_write(struct jaddref *, struct jseg *, uint8_t *); 751 static void jremref_write(struct jremref *, struct jseg *, uint8_t *); 752 static void jmvref_write(struct jmvref *, struct jseg *, uint8_t *); 753 static void jtrunc_write(struct jtrunc *, struct jseg *, uint8_t *); 754 static void jnewblk_write(struct jnewblk *, struct jseg *, uint8_t *); 755 static void jfreeblk_write(struct jfreeblk *, struct jseg *, uint8_t *); 756 static void jfreefrag_write(struct jfreefrag *, struct jseg *, uint8_t *); 757 static inline void inoref_write(struct inoref *, struct jseg *, 758 struct jrefrec *); 759 static void handle_allocdirect_partdone(struct allocdirect *, 760 struct workhead *); 761 static void cancel_newblk(struct newblk *, struct workhead *); 762 static void indirdep_complete(struct indirdep *); 763 static void handle_allocindir_partdone(struct allocindir *); 764 static void initiate_write_filepage(struct pagedep *, struct buf *); 765 static void initiate_write_indirdep(struct indirdep*, struct buf *); 766 static void handle_written_mkdir(struct mkdir *, int); 767 static void initiate_write_bmsafemap(struct bmsafemap *, struct buf *); 768 static void initiate_write_inodeblock_ufs1(struct inodedep *, struct buf *); 769 static void initiate_write_inodeblock_ufs2(struct inodedep *, struct buf *); 770 static void handle_workitem_freefile(struct freefile *); 771 static void handle_workitem_remove(struct dirrem *, struct vnode *); 772 static struct dirrem *newdirrem(struct buf *, struct inode *, 773 struct inode *, int, struct dirrem **); 774 static void cancel_indirdep(struct indirdep *, struct buf *, struct inodedep *, 775 struct freeblks *); 776 static void free_indirdep(struct indirdep *); 777 static void free_diradd(struct diradd *, struct workhead *); 778 static void merge_diradd(struct inodedep *, struct diradd *); 779 static void complete_diradd(struct diradd *); 780 static struct diradd *diradd_lookup(struct pagedep *, int); 781 static struct jremref *cancel_diradd_dotdot(struct inode *, struct dirrem *, 782 struct jremref *); 783 static struct jremref *cancel_mkdir_dotdot(struct inode *, struct dirrem *, 784 struct jremref *); 785 static void cancel_diradd(struct diradd *, struct dirrem *, struct jremref *, 786 struct jremref *, struct jremref *); 787 static void dirrem_journal(struct dirrem *, struct jremref *, struct jremref *, 788 struct jremref *); 789 static void cancel_allocindir(struct allocindir *, struct inodedep *, 790 struct freeblks *); 791 static void complete_mkdir(struct mkdir *); 792 static void free_newdirblk(struct newdirblk *); 793 static void free_jremref(struct jremref *); 794 static void free_jaddref(struct jaddref *); 795 static void free_jsegdep(struct jsegdep *); 796 static void free_jseg(struct jseg *); 797 static void free_jnewblk(struct jnewblk *); 798 static void free_jfreeblk(struct jfreeblk *); 799 static void free_jfreefrag(struct jfreefrag *); 800 static void free_freedep(struct freedep *); 801 static void journal_jremref(struct dirrem *, struct jremref *, 802 struct inodedep *); 803 static void cancel_jnewblk(struct jnewblk *, struct workhead *); 804 static int cancel_jaddref(struct jaddref *, struct inodedep *, 805 struct workhead *); 806 static void cancel_jfreefrag(struct jfreefrag *); 807 static void indir_trunc(struct freework *, ufs2_daddr_t, ufs_lbn_t); 808 static int deallocate_dependencies(struct buf *, struct inodedep *, 809 struct freeblks *); 810 static void free_newblk(struct newblk *); 811 static void cancel_allocdirect(struct allocdirectlst *, 812 struct allocdirect *, struct freeblks *, int); 813 static int check_inode_unwritten(struct inodedep *); 814 static int free_inodedep(struct inodedep *); 815 static void freework_freeblock(struct freework *); 816 static void handle_workitem_freeblocks(struct freeblks *, int); 817 static void handle_complete_freeblocks(struct freeblks *); 818 static void handle_workitem_indirblk(struct freework *); 819 static void handle_written_freework(struct freework *); 820 static void merge_inode_lists(struct allocdirectlst *,struct allocdirectlst *); 821 static void setup_allocindir_phase2(struct buf *, struct inode *, 822 struct inodedep *, struct allocindir *, ufs_lbn_t); 823 static struct allocindir *newallocindir(struct inode *, int, ufs2_daddr_t, 824 ufs2_daddr_t, ufs_lbn_t); 825 static void handle_workitem_freefrag(struct freefrag *); 826 static struct freefrag *newfreefrag(struct inode *, ufs2_daddr_t, long, 827 ufs_lbn_t); 828 static void allocdirect_merge(struct allocdirectlst *, 829 struct allocdirect *, struct allocdirect *); 830 static struct freefrag *allocindir_merge(struct allocindir *, 831 struct allocindir *); 832 static int bmsafemap_find(struct bmsafemap_hashhead *, struct mount *, int, 833 struct bmsafemap **); 834 static struct bmsafemap *bmsafemap_lookup(struct mount *, struct buf *, 835 int cg); 836 static int newblk_find(struct newblk_hashhead *, struct mount *, ufs2_daddr_t, 837 int, struct newblk **); 838 static int newblk_lookup(struct mount *, ufs2_daddr_t, int, struct newblk **); 839 static int inodedep_find(struct inodedep_hashhead *, struct fs *, ino_t, 840 struct inodedep **); 841 static int inodedep_lookup(struct mount *, ino_t, int, struct inodedep **); 842 static int pagedep_lookup(struct mount *, ino_t, ufs_lbn_t, int, 843 struct pagedep **); 844 static int pagedep_find(struct pagedep_hashhead *, ino_t, ufs_lbn_t, 845 struct mount *mp, int, struct pagedep **); 846 static void pause_timer(void *); 847 static int request_cleanup(struct mount *, int); 848 static int process_worklist_item(struct mount *, int); 849 static void process_removes(struct vnode *); 850 static void jwork_move(struct workhead *, struct workhead *); 851 static void add_to_worklist(struct worklist *, int); 852 static void remove_from_worklist(struct worklist *); 853 static void softdep_flush(void); 854 static int softdep_speedup(void); 855 static void worklist_speedup(void); 856 static int journal_mount(struct mount *, struct fs *, struct ucred *); 857 static void journal_unmount(struct mount *); 858 static int journal_space(struct ufsmount *, int); 859 static void journal_suspend(struct ufsmount *); 860 static void softdep_prelink(struct vnode *, struct vnode *); 861 static void add_to_journal(struct worklist *); 862 static void remove_from_journal(struct worklist *); 863 static void softdep_process_journal(struct mount *, int); 864 static struct jremref *newjremref(struct dirrem *, struct inode *, 865 struct inode *ip, off_t, nlink_t); 866 static struct jaddref *newjaddref(struct inode *, ino_t, off_t, int16_t, 867 uint16_t); 868 static inline void newinoref(struct inoref *, ino_t, ino_t, off_t, nlink_t, 869 uint16_t); 870 static inline struct jsegdep *inoref_jseg(struct inoref *); 871 static struct jmvref *newjmvref(struct inode *, ino_t, off_t, off_t); 872 static struct jfreeblk *newjfreeblk(struct freeblks *, ufs_lbn_t, 873 ufs2_daddr_t, int); 874 static struct jfreefrag *newjfreefrag(struct freefrag *, struct inode *, 875 ufs2_daddr_t, long, ufs_lbn_t); 876 static struct freework *newfreework(struct freeblks *, struct freework *, 877 ufs_lbn_t, ufs2_daddr_t, int, int); 878 static void jwait(struct worklist *wk); 879 static struct inodedep *inodedep_lookup_ip(struct inode *); 880 static int bmsafemap_rollbacks(struct bmsafemap *); 881 static struct freefile *handle_bufwait(struct inodedep *, struct workhead *); 882 static void handle_jwork(struct workhead *); 883 static struct mkdir *setup_newdir(struct diradd *, ino_t, ino_t, struct buf *, 884 struct mkdir **); 885 static struct jblocks *jblocks_create(void); 886 static ufs2_daddr_t jblocks_alloc(struct jblocks *, int, int *); 887 static void jblocks_free(struct jblocks *, struct mount *, int); 888 static void jblocks_destroy(struct jblocks *); 889 static void jblocks_add(struct jblocks *, ufs2_daddr_t, int); 890 891 /* 892 * Exported softdep operations. 893 */ 894 static void softdep_disk_io_initiation(struct buf *); 895 static void softdep_disk_write_complete(struct buf *); 896 static void softdep_deallocate_dependencies(struct buf *); 897 static int softdep_count_dependencies(struct buf *bp, int); 898 899 static struct mtx lk; 900 MTX_SYSINIT(softdep_lock, &lk, "Softdep Lock", MTX_DEF); 901 902 #define TRY_ACQUIRE_LOCK(lk) mtx_trylock(lk) 903 #define ACQUIRE_LOCK(lk) mtx_lock(lk) 904 #define FREE_LOCK(lk) mtx_unlock(lk) 905 906 #define BUF_AREC(bp) ((bp)->b_lock.lock_object.lo_flags |= LO_RECURSABLE) 907 #define BUF_NOREC(bp) ((bp)->b_lock.lock_object.lo_flags &= ~LO_RECURSABLE) 908 909 /* 910 * Worklist queue management. 911 * These routines require that the lock be held. 912 */ 913 #ifndef /* NOT */ DEBUG 914 #define WORKLIST_INSERT(head, item) do { \ 915 (item)->wk_state |= ONWORKLIST; \ 916 LIST_INSERT_HEAD(head, item, wk_list); \ 917 } while (0) 918 #define WORKLIST_REMOVE(item) do { \ 919 (item)->wk_state &= ~ONWORKLIST; \ 920 LIST_REMOVE(item, wk_list); \ 921 } while (0) 922 #define WORKLIST_INSERT_UNLOCKED WORKLIST_INSERT 923 #define WORKLIST_REMOVE_UNLOCKED WORKLIST_REMOVE 924 925 #else /* DEBUG */ 926 static void worklist_insert(struct workhead *, struct worklist *, int); 927 static void worklist_remove(struct worklist *, int); 928 929 #define WORKLIST_INSERT(head, item) worklist_insert(head, item, 1) 930 #define WORKLIST_INSERT_UNLOCKED(head, item) worklist_insert(head, item, 0) 931 #define WORKLIST_REMOVE(item) worklist_remove(item, 1) 932 #define WORKLIST_REMOVE_UNLOCKED(item) worklist_remove(item, 0) 933 934 static void 935 worklist_insert(head, item, locked) 936 struct workhead *head; 937 struct worklist *item; 938 int locked; 939 { 940 941 if (locked) 942 mtx_assert(&lk, MA_OWNED); 943 if (item->wk_state & ONWORKLIST) 944 panic("worklist_insert: %p %s(0x%X) already on list", 945 item, TYPENAME(item->wk_type), item->wk_state); 946 item->wk_state |= ONWORKLIST; 947 LIST_INSERT_HEAD(head, item, wk_list); 948 } 949 950 static void 951 worklist_remove(item, locked) 952 struct worklist *item; 953 int locked; 954 { 955 956 if (locked) 957 mtx_assert(&lk, MA_OWNED); 958 if ((item->wk_state & ONWORKLIST) == 0) 959 panic("worklist_remove: %p %s(0x%X) not on list", 960 item, TYPENAME(item->wk_type), item->wk_state); 961 item->wk_state &= ~ONWORKLIST; 962 LIST_REMOVE(item, wk_list); 963 } 964 #endif /* DEBUG */ 965 966 /* 967 * Merge two jsegdeps keeping only the oldest one as newer references 968 * can't be discarded until after older references. 969 */ 970 static inline struct jsegdep * 971 jsegdep_merge(struct jsegdep *one, struct jsegdep *two) 972 { 973 struct jsegdep *swp; 974 975 if (two == NULL) 976 return (one); 977 978 if (one->jd_seg->js_seq > two->jd_seg->js_seq) { 979 swp = one; 980 one = two; 981 two = swp; 982 } 983 WORKLIST_REMOVE(&two->jd_list); 984 free_jsegdep(two); 985 986 return (one); 987 } 988 989 /* 990 * If two freedeps are compatible free one to reduce list size. 991 */ 992 static inline struct freedep * 993 freedep_merge(struct freedep *one, struct freedep *two) 994 { 995 if (two == NULL) 996 return (one); 997 998 if (one->fd_freework == two->fd_freework) { 999 WORKLIST_REMOVE(&two->fd_list); 1000 free_freedep(two); 1001 } 1002 return (one); 1003 } 1004 1005 /* 1006 * Move journal work from one list to another. Duplicate freedeps and 1007 * jsegdeps are coalesced to keep the lists as small as possible. 1008 */ 1009 static void 1010 jwork_move(dst, src) 1011 struct workhead *dst; 1012 struct workhead *src; 1013 { 1014 struct freedep *freedep; 1015 struct jsegdep *jsegdep; 1016 struct worklist *wkn; 1017 struct worklist *wk; 1018 1019 KASSERT(dst != src, 1020 ("jwork_move: dst == src")); 1021 freedep = NULL; 1022 jsegdep = NULL; 1023 LIST_FOREACH_SAFE(wk, dst, wk_list, wkn) { 1024 if (wk->wk_type == D_JSEGDEP) 1025 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1026 if (wk->wk_type == D_FREEDEP) 1027 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1028 } 1029 1030 mtx_assert(&lk, MA_OWNED); 1031 while ((wk = LIST_FIRST(src)) != NULL) { 1032 WORKLIST_REMOVE(wk); 1033 WORKLIST_INSERT(dst, wk); 1034 if (wk->wk_type == D_JSEGDEP) { 1035 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1036 continue; 1037 } 1038 if (wk->wk_type == D_FREEDEP) 1039 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1040 } 1041 } 1042 1043 /* 1044 * Routines for tracking and managing workitems. 1045 */ 1046 static void workitem_free(struct worklist *, int); 1047 static void workitem_alloc(struct worklist *, int, struct mount *); 1048 1049 #define WORKITEM_FREE(item, type) workitem_free((struct worklist *)(item), (type)) 1050 1051 static void 1052 workitem_free(item, type) 1053 struct worklist *item; 1054 int type; 1055 { 1056 struct ufsmount *ump; 1057 mtx_assert(&lk, MA_OWNED); 1058 1059 #ifdef DEBUG 1060 if (item->wk_state & ONWORKLIST) 1061 panic("workitem_free: %s(0x%X) still on list", 1062 TYPENAME(item->wk_type), item->wk_state); 1063 if (item->wk_type != type) 1064 panic("workitem_free: type mismatch %s != %s", 1065 TYPENAME(item->wk_type), TYPENAME(type)); 1066 #endif 1067 ump = VFSTOUFS(item->wk_mp); 1068 if (--ump->softdep_deps == 0 && ump->softdep_req) 1069 wakeup(&ump->softdep_deps); 1070 dep_current[type]--; 1071 free(item, DtoM(type)); 1072 } 1073 1074 static void 1075 workitem_alloc(item, type, mp) 1076 struct worklist *item; 1077 int type; 1078 struct mount *mp; 1079 { 1080 item->wk_type = type; 1081 item->wk_mp = mp; 1082 item->wk_state = 0; 1083 ACQUIRE_LOCK(&lk); 1084 dep_current[type]++; 1085 dep_total[type]++; 1086 VFSTOUFS(mp)->softdep_deps++; 1087 VFSTOUFS(mp)->softdep_accdeps++; 1088 FREE_LOCK(&lk); 1089 } 1090 1091 /* 1092 * Workitem queue management 1093 */ 1094 static int max_softdeps; /* maximum number of structs before slowdown */ 1095 static int maxindirdeps = 50; /* max number of indirdeps before slowdown */ 1096 static int tickdelay = 2; /* number of ticks to pause during slowdown */ 1097 static int proc_waiting; /* tracks whether we have a timeout posted */ 1098 static int *stat_countp; /* statistic to count in proc_waiting timeout */ 1099 static struct callout softdep_callout; 1100 static int req_pending; 1101 static int req_clear_inodedeps; /* syncer process flush some inodedeps */ 1102 #define FLUSH_INODES 1 1103 static int req_clear_remove; /* syncer process flush some freeblks */ 1104 #define FLUSH_REMOVE 2 1105 #define FLUSH_REMOVE_WAIT 3 1106 static long num_freeblkdep; /* number of freeblks workitems allocated */ 1107 1108 /* 1109 * runtime statistics 1110 */ 1111 static int stat_worklist_push; /* number of worklist cleanups */ 1112 static int stat_blk_limit_push; /* number of times block limit neared */ 1113 static int stat_ino_limit_push; /* number of times inode limit neared */ 1114 static int stat_blk_limit_hit; /* number of times block slowdown imposed */ 1115 static int stat_ino_limit_hit; /* number of times inode slowdown imposed */ 1116 static int stat_sync_limit_hit; /* number of synchronous slowdowns imposed */ 1117 static int stat_indir_blk_ptrs; /* bufs redirtied as indir ptrs not written */ 1118 static int stat_inode_bitmap; /* bufs redirtied as inode bitmap not written */ 1119 static int stat_direct_blk_ptrs;/* bufs redirtied as direct ptrs not written */ 1120 static int stat_dir_entry; /* bufs redirtied as dir entry cannot write */ 1121 static int stat_jaddref; /* bufs redirtied as ino bitmap can not write */ 1122 static int stat_jnewblk; /* bufs redirtied as blk bitmap can not write */ 1123 static int stat_journal_min; /* Times hit journal min threshold */ 1124 static int stat_journal_low; /* Times hit journal low threshold */ 1125 static int stat_journal_wait; /* Times blocked in jwait(). */ 1126 static int stat_jwait_filepage; /* Times blocked in jwait() for filepage. */ 1127 static int stat_jwait_freeblks; /* Times blocked in jwait() for freeblks. */ 1128 static int stat_jwait_inode; /* Times blocked in jwait() for inodes. */ 1129 static int stat_jwait_newblk; /* Times blocked in jwait() for newblks. */ 1130 1131 SYSCTL_INT(_debug_softdep, OID_AUTO, max_softdeps, CTLFLAG_RW, 1132 &max_softdeps, 0, ""); 1133 SYSCTL_INT(_debug_softdep, OID_AUTO, tickdelay, CTLFLAG_RW, 1134 &tickdelay, 0, ""); 1135 SYSCTL_INT(_debug_softdep, OID_AUTO, maxindirdeps, CTLFLAG_RW, 1136 &maxindirdeps, 0, ""); 1137 SYSCTL_INT(_debug_softdep, OID_AUTO, worklist_push, CTLFLAG_RW, 1138 &stat_worklist_push, 0,""); 1139 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_push, CTLFLAG_RW, 1140 &stat_blk_limit_push, 0,""); 1141 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_push, CTLFLAG_RW, 1142 &stat_ino_limit_push, 0,""); 1143 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_hit, CTLFLAG_RW, 1144 &stat_blk_limit_hit, 0, ""); 1145 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_hit, CTLFLAG_RW, 1146 &stat_ino_limit_hit, 0, ""); 1147 SYSCTL_INT(_debug_softdep, OID_AUTO, sync_limit_hit, CTLFLAG_RW, 1148 &stat_sync_limit_hit, 0, ""); 1149 SYSCTL_INT(_debug_softdep, OID_AUTO, indir_blk_ptrs, CTLFLAG_RW, 1150 &stat_indir_blk_ptrs, 0, ""); 1151 SYSCTL_INT(_debug_softdep, OID_AUTO, inode_bitmap, CTLFLAG_RW, 1152 &stat_inode_bitmap, 0, ""); 1153 SYSCTL_INT(_debug_softdep, OID_AUTO, direct_blk_ptrs, CTLFLAG_RW, 1154 &stat_direct_blk_ptrs, 0, ""); 1155 SYSCTL_INT(_debug_softdep, OID_AUTO, dir_entry, CTLFLAG_RW, 1156 &stat_dir_entry, 0, ""); 1157 SYSCTL_INT(_debug_softdep, OID_AUTO, jaddref_rollback, CTLFLAG_RW, 1158 &stat_jaddref, 0, ""); 1159 SYSCTL_INT(_debug_softdep, OID_AUTO, jnewblk_rollback, CTLFLAG_RW, 1160 &stat_jnewblk, 0, ""); 1161 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_low, CTLFLAG_RW, 1162 &stat_journal_low, 0, ""); 1163 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_min, CTLFLAG_RW, 1164 &stat_journal_min, 0, ""); 1165 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_wait, CTLFLAG_RW, 1166 &stat_journal_wait, 0, ""); 1167 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_filepage, CTLFLAG_RW, 1168 &stat_jwait_filepage, 0, ""); 1169 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_freeblks, CTLFLAG_RW, 1170 &stat_jwait_freeblks, 0, ""); 1171 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_inode, CTLFLAG_RW, 1172 &stat_jwait_inode, 0, ""); 1173 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_newblk, CTLFLAG_RW, 1174 &stat_jwait_newblk, 0, ""); 1175 1176 SYSCTL_DECL(_vfs_ffs); 1177 1178 LIST_HEAD(bmsafemap_hashhead, bmsafemap) *bmsafemap_hashtbl; 1179 static u_long bmsafemap_hash; /* size of hash table - 1 */ 1180 1181 static int compute_summary_at_mount = 0; /* Whether to recompute the summary at mount time */ 1182 SYSCTL_INT(_vfs_ffs, OID_AUTO, compute_summary_at_mount, CTLFLAG_RW, 1183 &compute_summary_at_mount, 0, "Recompute summary at mount"); 1184 1185 static struct proc *softdepproc; 1186 static struct kproc_desc softdep_kp = { 1187 "softdepflush", 1188 softdep_flush, 1189 &softdepproc 1190 }; 1191 SYSINIT(sdproc, SI_SUB_KTHREAD_UPDATE, SI_ORDER_ANY, kproc_start, 1192 &softdep_kp); 1193 1194 static void 1195 softdep_flush(void) 1196 { 1197 struct mount *nmp; 1198 struct mount *mp; 1199 struct ufsmount *ump; 1200 struct thread *td; 1201 int remaining; 1202 int progress; 1203 int vfslocked; 1204 1205 td = curthread; 1206 td->td_pflags |= TDP_NORUNNINGBUF; 1207 1208 for (;;) { 1209 kproc_suspend_check(softdepproc); 1210 vfslocked = VFS_LOCK_GIANT((struct mount *)NULL); 1211 ACQUIRE_LOCK(&lk); 1212 /* 1213 * If requested, try removing inode or removal dependencies. 1214 */ 1215 if (req_clear_inodedeps) { 1216 clear_inodedeps(td); 1217 req_clear_inodedeps -= 1; 1218 wakeup_one(&proc_waiting); 1219 } 1220 if (req_clear_remove) { 1221 clear_remove(td); 1222 req_clear_remove -= 1; 1223 wakeup_one(&proc_waiting); 1224 } 1225 FREE_LOCK(&lk); 1226 VFS_UNLOCK_GIANT(vfslocked); 1227 remaining = progress = 0; 1228 mtx_lock(&mountlist_mtx); 1229 for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) { 1230 nmp = TAILQ_NEXT(mp, mnt_list); 1231 if ((mp->mnt_flag & MNT_SOFTDEP) == 0) 1232 continue; 1233 if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) 1234 continue; 1235 vfslocked = VFS_LOCK_GIANT(mp); 1236 progress += softdep_process_worklist(mp, 0); 1237 ump = VFSTOUFS(mp); 1238 remaining += ump->softdep_on_worklist - 1239 ump->softdep_on_worklist_inprogress; 1240 VFS_UNLOCK_GIANT(vfslocked); 1241 mtx_lock(&mountlist_mtx); 1242 nmp = TAILQ_NEXT(mp, mnt_list); 1243 vfs_unbusy(mp); 1244 } 1245 mtx_unlock(&mountlist_mtx); 1246 if (remaining && progress) 1247 continue; 1248 ACQUIRE_LOCK(&lk); 1249 if (!req_pending) 1250 msleep(&req_pending, &lk, PVM, "sdflush", hz); 1251 req_pending = 0; 1252 FREE_LOCK(&lk); 1253 } 1254 } 1255 1256 static void 1257 worklist_speedup(void) 1258 { 1259 mtx_assert(&lk, MA_OWNED); 1260 if (req_pending == 0) { 1261 req_pending = 1; 1262 wakeup(&req_pending); 1263 } 1264 } 1265 1266 static int 1267 softdep_speedup(void) 1268 { 1269 1270 worklist_speedup(); 1271 bd_speedup(); 1272 return speedup_syncer(); 1273 } 1274 1275 /* 1276 * Add an item to the end of the work queue. 1277 * This routine requires that the lock be held. 1278 * This is the only routine that adds items to the list. 1279 * The following routine is the only one that removes items 1280 * and does so in order from first to last. 1281 */ 1282 static void 1283 add_to_worklist(wk, nodelay) 1284 struct worklist *wk; 1285 int nodelay; 1286 { 1287 struct ufsmount *ump; 1288 1289 mtx_assert(&lk, MA_OWNED); 1290 ump = VFSTOUFS(wk->wk_mp); 1291 if (wk->wk_state & ONWORKLIST) 1292 panic("add_to_worklist: %s(0x%X) already on list", 1293 TYPENAME(wk->wk_type), wk->wk_state); 1294 wk->wk_state |= ONWORKLIST; 1295 if (LIST_EMPTY(&ump->softdep_workitem_pending)) 1296 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list); 1297 else 1298 LIST_INSERT_AFTER(ump->softdep_worklist_tail, wk, wk_list); 1299 ump->softdep_worklist_tail = wk; 1300 ump->softdep_on_worklist += 1; 1301 if (nodelay) 1302 worklist_speedup(); 1303 } 1304 1305 /* 1306 * Remove the item to be processed. If we are removing the last 1307 * item on the list, we need to recalculate the tail pointer. 1308 */ 1309 static void 1310 remove_from_worklist(wk) 1311 struct worklist *wk; 1312 { 1313 struct ufsmount *ump; 1314 struct worklist *wkend; 1315 1316 ump = VFSTOUFS(wk->wk_mp); 1317 WORKLIST_REMOVE(wk); 1318 if (wk == ump->softdep_worklist_tail) { 1319 LIST_FOREACH(wkend, &ump->softdep_workitem_pending, wk_list) 1320 if (LIST_NEXT(wkend, wk_list) == NULL) 1321 break; 1322 ump->softdep_worklist_tail = wkend; 1323 } 1324 ump->softdep_on_worklist -= 1; 1325 } 1326 1327 /* 1328 * Process that runs once per second to handle items in the background queue. 1329 * 1330 * Note that we ensure that everything is done in the order in which they 1331 * appear in the queue. The code below depends on this property to ensure 1332 * that blocks of a file are freed before the inode itself is freed. This 1333 * ordering ensures that no new <vfsid, inum, lbn> triples will be generated 1334 * until all the old ones have been purged from the dependency lists. 1335 */ 1336 int 1337 softdep_process_worklist(mp, full) 1338 struct mount *mp; 1339 int full; 1340 { 1341 struct thread *td = curthread; 1342 int cnt, matchcnt, loopcount; 1343 struct ufsmount *ump; 1344 long starttime; 1345 1346 KASSERT(mp != NULL, ("softdep_process_worklist: NULL mp")); 1347 /* 1348 * Record the process identifier of our caller so that we can give 1349 * this process preferential treatment in request_cleanup below. 1350 */ 1351 matchcnt = 0; 1352 ump = VFSTOUFS(mp); 1353 ACQUIRE_LOCK(&lk); 1354 loopcount = 1; 1355 starttime = time_second; 1356 softdep_process_journal(mp, full?MNT_WAIT:0); 1357 while (ump->softdep_on_worklist > 0) { 1358 if ((cnt = process_worklist_item(mp, LK_NOWAIT)) == -1) 1359 break; 1360 else 1361 matchcnt += cnt; 1362 /* 1363 * If requested, try removing inode or removal dependencies. 1364 */ 1365 if (req_clear_inodedeps) { 1366 clear_inodedeps(td); 1367 req_clear_inodedeps -= 1; 1368 wakeup_one(&proc_waiting); 1369 } 1370 if (req_clear_remove) { 1371 clear_remove(td); 1372 req_clear_remove -= 1; 1373 wakeup_one(&proc_waiting); 1374 } 1375 /* 1376 * We do not generally want to stop for buffer space, but if 1377 * we are really being a buffer hog, we will stop and wait. 1378 */ 1379 if (loopcount++ % 128 == 0) { 1380 FREE_LOCK(&lk); 1381 uio_yield(); 1382 bwillwrite(); 1383 ACQUIRE_LOCK(&lk); 1384 } 1385 /* 1386 * Never allow processing to run for more than one 1387 * second. Otherwise the other mountpoints may get 1388 * excessively backlogged. 1389 */ 1390 if (!full && starttime != time_second) 1391 break; 1392 } 1393 FREE_LOCK(&lk); 1394 return (matchcnt); 1395 } 1396 1397 /* 1398 * Process all removes associated with a vnode if we are running out of 1399 * journal space. Any other process which attempts to flush these will 1400 * be unable as we have the vnodes locked. 1401 */ 1402 static void 1403 process_removes(vp) 1404 struct vnode *vp; 1405 { 1406 struct inodedep *inodedep; 1407 struct dirrem *dirrem; 1408 struct mount *mp; 1409 ino_t inum; 1410 1411 mtx_assert(&lk, MA_OWNED); 1412 1413 mp = vp->v_mount; 1414 inum = VTOI(vp)->i_number; 1415 for (;;) { 1416 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0) 1417 return; 1418 LIST_FOREACH(dirrem, &inodedep->id_dirremhd, dm_inonext) 1419 if ((dirrem->dm_state & (COMPLETE | ONWORKLIST)) == 1420 (COMPLETE | ONWORKLIST)) 1421 break; 1422 if (dirrem == NULL) 1423 return; 1424 /* 1425 * If another thread is trying to lock this vnode it will 1426 * fail but we must wait for it to do so before we can 1427 * proceed. 1428 */ 1429 if (dirrem->dm_state & INPROGRESS) { 1430 dirrem->dm_state |= IOWAITING; 1431 msleep(&dirrem->dm_list, &lk, PVM, "pwrwait", 0); 1432 continue; 1433 } 1434 remove_from_worklist(&dirrem->dm_list); 1435 FREE_LOCK(&lk); 1436 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1437 panic("process_removes: suspended filesystem"); 1438 handle_workitem_remove(dirrem, vp); 1439 vn_finished_secondary_write(mp); 1440 ACQUIRE_LOCK(&lk); 1441 } 1442 } 1443 1444 /* 1445 * Process one item on the worklist. 1446 */ 1447 static int 1448 process_worklist_item(mp, flags) 1449 struct mount *mp; 1450 int flags; 1451 { 1452 struct worklist *wk; 1453 struct ufsmount *ump; 1454 struct vnode *vp; 1455 int matchcnt = 0; 1456 1457 mtx_assert(&lk, MA_OWNED); 1458 KASSERT(mp != NULL, ("process_worklist_item: NULL mp")); 1459 /* 1460 * If we are being called because of a process doing a 1461 * copy-on-write, then it is not safe to write as we may 1462 * recurse into the copy-on-write routine. 1463 */ 1464 if (curthread->td_pflags & TDP_COWINPROGRESS) 1465 return (-1); 1466 /* 1467 * Normally we just process each item on the worklist in order. 1468 * However, if we are in a situation where we cannot lock any 1469 * inodes, we have to skip over any dirrem requests whose 1470 * vnodes are resident and locked. 1471 */ 1472 vp = NULL; 1473 ump = VFSTOUFS(mp); 1474 LIST_FOREACH(wk, &ump->softdep_workitem_pending, wk_list) { 1475 if (wk->wk_state & INPROGRESS) 1476 continue; 1477 if ((flags & LK_NOWAIT) == 0 || wk->wk_type != D_DIRREM) 1478 break; 1479 wk->wk_state |= INPROGRESS; 1480 ump->softdep_on_worklist_inprogress++; 1481 FREE_LOCK(&lk); 1482 ffs_vgetf(mp, WK_DIRREM(wk)->dm_oldinum, 1483 LK_NOWAIT | LK_EXCLUSIVE, &vp, FFSV_FORCEINSMQ); 1484 ACQUIRE_LOCK(&lk); 1485 if (wk->wk_state & IOWAITING) { 1486 wk->wk_state &= ~IOWAITING; 1487 wakeup(wk); 1488 } 1489 wk->wk_state &= ~INPROGRESS; 1490 ump->softdep_on_worklist_inprogress--; 1491 if (vp != NULL) 1492 break; 1493 } 1494 if (wk == 0) 1495 return (-1); 1496 remove_from_worklist(wk); 1497 FREE_LOCK(&lk); 1498 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1499 panic("process_worklist_item: suspended filesystem"); 1500 matchcnt++; 1501 switch (wk->wk_type) { 1502 1503 case D_DIRREM: 1504 /* removal of a directory entry */ 1505 handle_workitem_remove(WK_DIRREM(wk), vp); 1506 if (vp) 1507 vput(vp); 1508 break; 1509 1510 case D_FREEBLKS: 1511 /* releasing blocks and/or fragments from a file */ 1512 handle_workitem_freeblocks(WK_FREEBLKS(wk), flags & LK_NOWAIT); 1513 break; 1514 1515 case D_FREEFRAG: 1516 /* releasing a fragment when replaced as a file grows */ 1517 handle_workitem_freefrag(WK_FREEFRAG(wk)); 1518 break; 1519 1520 case D_FREEFILE: 1521 /* releasing an inode when its link count drops to 0 */ 1522 handle_workitem_freefile(WK_FREEFILE(wk)); 1523 break; 1524 1525 case D_FREEWORK: 1526 /* Final block in an indirect was freed. */ 1527 handle_workitem_indirblk(WK_FREEWORK(wk)); 1528 break; 1529 1530 default: 1531 panic("%s_process_worklist: Unknown type %s", 1532 "softdep", TYPENAME(wk->wk_type)); 1533 /* NOTREACHED */ 1534 } 1535 vn_finished_secondary_write(mp); 1536 ACQUIRE_LOCK(&lk); 1537 return (matchcnt); 1538 } 1539 1540 /* 1541 * Move dependencies from one buffer to another. 1542 */ 1543 int 1544 softdep_move_dependencies(oldbp, newbp) 1545 struct buf *oldbp; 1546 struct buf *newbp; 1547 { 1548 struct worklist *wk, *wktail; 1549 int dirty; 1550 1551 dirty = 0; 1552 wktail = NULL; 1553 ACQUIRE_LOCK(&lk); 1554 while ((wk = LIST_FIRST(&oldbp->b_dep)) != NULL) { 1555 LIST_REMOVE(wk, wk_list); 1556 if (wk->wk_type == D_BMSAFEMAP && 1557 bmsafemap_rollbacks(WK_BMSAFEMAP(wk))) 1558 dirty = 1; 1559 if (wktail == 0) 1560 LIST_INSERT_HEAD(&newbp->b_dep, wk, wk_list); 1561 else 1562 LIST_INSERT_AFTER(wktail, wk, wk_list); 1563 wktail = wk; 1564 } 1565 FREE_LOCK(&lk); 1566 1567 return (dirty); 1568 } 1569 1570 /* 1571 * Purge the work list of all items associated with a particular mount point. 1572 */ 1573 int 1574 softdep_flushworklist(oldmnt, countp, td) 1575 struct mount *oldmnt; 1576 int *countp; 1577 struct thread *td; 1578 { 1579 struct vnode *devvp; 1580 int count, error = 0; 1581 struct ufsmount *ump; 1582 1583 /* 1584 * Alternately flush the block device associated with the mount 1585 * point and process any dependencies that the flushing 1586 * creates. We continue until no more worklist dependencies 1587 * are found. 1588 */ 1589 *countp = 0; 1590 ump = VFSTOUFS(oldmnt); 1591 devvp = ump->um_devvp; 1592 while ((count = softdep_process_worklist(oldmnt, 1)) > 0) { 1593 *countp += count; 1594 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY); 1595 error = VOP_FSYNC(devvp, MNT_WAIT, td); 1596 VOP_UNLOCK(devvp, 0); 1597 if (error) 1598 break; 1599 } 1600 return (error); 1601 } 1602 1603 int 1604 softdep_waitidle(struct mount *mp) 1605 { 1606 struct ufsmount *ump; 1607 int error; 1608 int i; 1609 1610 ump = VFSTOUFS(mp); 1611 ACQUIRE_LOCK(&lk); 1612 for (i = 0; i < 10 && ump->softdep_deps; i++) { 1613 ump->softdep_req = 1; 1614 if (ump->softdep_on_worklist) 1615 panic("softdep_waitidle: work added after flush."); 1616 msleep(&ump->softdep_deps, &lk, PVM, "softdeps", 1); 1617 } 1618 ump->softdep_req = 0; 1619 FREE_LOCK(&lk); 1620 error = 0; 1621 if (i == 10) { 1622 error = EBUSY; 1623 printf("softdep_waitidle: Failed to flush worklist for %p\n", 1624 mp); 1625 } 1626 1627 return (error); 1628 } 1629 1630 /* 1631 * Flush all vnodes and worklist items associated with a specified mount point. 1632 */ 1633 int 1634 softdep_flushfiles(oldmnt, flags, td) 1635 struct mount *oldmnt; 1636 int flags; 1637 struct thread *td; 1638 { 1639 int error, depcount, loopcnt, retry_flush_count, retry; 1640 1641 loopcnt = 10; 1642 retry_flush_count = 3; 1643 retry_flush: 1644 error = 0; 1645 1646 /* 1647 * Alternately flush the vnodes associated with the mount 1648 * point and process any dependencies that the flushing 1649 * creates. In theory, this loop can happen at most twice, 1650 * but we give it a few extra just to be sure. 1651 */ 1652 for (; loopcnt > 0; loopcnt--) { 1653 /* 1654 * Do another flush in case any vnodes were brought in 1655 * as part of the cleanup operations. 1656 */ 1657 if ((error = ffs_flushfiles(oldmnt, flags, td)) != 0) 1658 break; 1659 if ((error = softdep_flushworklist(oldmnt, &depcount, td)) != 0 || 1660 depcount == 0) 1661 break; 1662 } 1663 /* 1664 * If we are unmounting then it is an error to fail. If we 1665 * are simply trying to downgrade to read-only, then filesystem 1666 * activity can keep us busy forever, so we just fail with EBUSY. 1667 */ 1668 if (loopcnt == 0) { 1669 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) 1670 panic("softdep_flushfiles: looping"); 1671 error = EBUSY; 1672 } 1673 if (!error) 1674 error = softdep_waitidle(oldmnt); 1675 if (!error) { 1676 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) { 1677 retry = 0; 1678 MNT_ILOCK(oldmnt); 1679 KASSERT((oldmnt->mnt_kern_flag & MNTK_NOINSMNTQ) != 0, 1680 ("softdep_flushfiles: !MNTK_NOINSMNTQ")); 1681 if (oldmnt->mnt_nvnodelistsize > 0) { 1682 if (--retry_flush_count > 0) { 1683 retry = 1; 1684 loopcnt = 3; 1685 } else 1686 error = EBUSY; 1687 } 1688 MNT_IUNLOCK(oldmnt); 1689 if (retry) 1690 goto retry_flush; 1691 } 1692 } 1693 return (error); 1694 } 1695 1696 /* 1697 * Structure hashing. 1698 * 1699 * There are three types of structures that can be looked up: 1700 * 1) pagedep structures identified by mount point, inode number, 1701 * and logical block. 1702 * 2) inodedep structures identified by mount point and inode number. 1703 * 3) newblk structures identified by mount point and 1704 * physical block number. 1705 * 1706 * The "pagedep" and "inodedep" dependency structures are hashed 1707 * separately from the file blocks and inodes to which they correspond. 1708 * This separation helps when the in-memory copy of an inode or 1709 * file block must be replaced. It also obviates the need to access 1710 * an inode or file page when simply updating (or de-allocating) 1711 * dependency structures. Lookup of newblk structures is needed to 1712 * find newly allocated blocks when trying to associate them with 1713 * their allocdirect or allocindir structure. 1714 * 1715 * The lookup routines optionally create and hash a new instance when 1716 * an existing entry is not found. 1717 */ 1718 #define DEPALLOC 0x0001 /* allocate structure if lookup fails */ 1719 #define NODELAY 0x0002 /* cannot do background work */ 1720 1721 /* 1722 * Structures and routines associated with pagedep caching. 1723 */ 1724 LIST_HEAD(pagedep_hashhead, pagedep) *pagedep_hashtbl; 1725 u_long pagedep_hash; /* size of hash table - 1 */ 1726 #define PAGEDEP_HASH(mp, inum, lbn) \ 1727 (&pagedep_hashtbl[((((register_t)(mp)) >> 13) + (inum) + (lbn)) & \ 1728 pagedep_hash]) 1729 1730 static int 1731 pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp) 1732 struct pagedep_hashhead *pagedephd; 1733 ino_t ino; 1734 ufs_lbn_t lbn; 1735 struct mount *mp; 1736 int flags; 1737 struct pagedep **pagedeppp; 1738 { 1739 struct pagedep *pagedep; 1740 1741 LIST_FOREACH(pagedep, pagedephd, pd_hash) 1742 if (ino == pagedep->pd_ino && 1743 lbn == pagedep->pd_lbn && 1744 mp == pagedep->pd_list.wk_mp) 1745 break; 1746 if (pagedep) { 1747 *pagedeppp = pagedep; 1748 if ((flags & DEPALLOC) != 0 && 1749 (pagedep->pd_state & ONWORKLIST) == 0) 1750 return (0); 1751 return (1); 1752 } 1753 *pagedeppp = NULL; 1754 return (0); 1755 } 1756 /* 1757 * Look up a pagedep. Return 1 if found, 0 if not found or found 1758 * when asked to allocate but not associated with any buffer. 1759 * If not found, allocate if DEPALLOC flag is passed. 1760 * Found or allocated entry is returned in pagedeppp. 1761 * This routine must be called with splbio interrupts blocked. 1762 */ 1763 static int 1764 pagedep_lookup(mp, ino, lbn, flags, pagedeppp) 1765 struct mount *mp; 1766 ino_t ino; 1767 ufs_lbn_t lbn; 1768 int flags; 1769 struct pagedep **pagedeppp; 1770 { 1771 struct pagedep *pagedep; 1772 struct pagedep_hashhead *pagedephd; 1773 int ret; 1774 int i; 1775 1776 mtx_assert(&lk, MA_OWNED); 1777 pagedephd = PAGEDEP_HASH(mp, ino, lbn); 1778 1779 ret = pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp); 1780 if (*pagedeppp || (flags & DEPALLOC) == 0) 1781 return (ret); 1782 FREE_LOCK(&lk); 1783 pagedep = malloc(sizeof(struct pagedep), 1784 M_PAGEDEP, M_SOFTDEP_FLAGS|M_ZERO); 1785 workitem_alloc(&pagedep->pd_list, D_PAGEDEP, mp); 1786 ACQUIRE_LOCK(&lk); 1787 ret = pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp); 1788 if (*pagedeppp) { 1789 WORKITEM_FREE(pagedep, D_PAGEDEP); 1790 return (ret); 1791 } 1792 pagedep->pd_ino = ino; 1793 pagedep->pd_lbn = lbn; 1794 LIST_INIT(&pagedep->pd_dirremhd); 1795 LIST_INIT(&pagedep->pd_pendinghd); 1796 for (i = 0; i < DAHASHSZ; i++) 1797 LIST_INIT(&pagedep->pd_diraddhd[i]); 1798 LIST_INSERT_HEAD(pagedephd, pagedep, pd_hash); 1799 *pagedeppp = pagedep; 1800 return (0); 1801 } 1802 1803 /* 1804 * Structures and routines associated with inodedep caching. 1805 */ 1806 LIST_HEAD(inodedep_hashhead, inodedep) *inodedep_hashtbl; 1807 static u_long inodedep_hash; /* size of hash table - 1 */ 1808 static long num_inodedep; /* number of inodedep allocated */ 1809 #define INODEDEP_HASH(fs, inum) \ 1810 (&inodedep_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & inodedep_hash]) 1811 1812 static int 1813 inodedep_find(inodedephd, fs, inum, inodedeppp) 1814 struct inodedep_hashhead *inodedephd; 1815 struct fs *fs; 1816 ino_t inum; 1817 struct inodedep **inodedeppp; 1818 { 1819 struct inodedep *inodedep; 1820 1821 LIST_FOREACH(inodedep, inodedephd, id_hash) 1822 if (inum == inodedep->id_ino && fs == inodedep->id_fs) 1823 break; 1824 if (inodedep) { 1825 *inodedeppp = inodedep; 1826 return (1); 1827 } 1828 *inodedeppp = NULL; 1829 1830 return (0); 1831 } 1832 /* 1833 * Look up an inodedep. Return 1 if found, 0 if not found. 1834 * If not found, allocate if DEPALLOC flag is passed. 1835 * Found or allocated entry is returned in inodedeppp. 1836 * This routine must be called with splbio interrupts blocked. 1837 */ 1838 static int 1839 inodedep_lookup(mp, inum, flags, inodedeppp) 1840 struct mount *mp; 1841 ino_t inum; 1842 int flags; 1843 struct inodedep **inodedeppp; 1844 { 1845 struct inodedep *inodedep; 1846 struct inodedep_hashhead *inodedephd; 1847 struct fs *fs; 1848 1849 mtx_assert(&lk, MA_OWNED); 1850 fs = VFSTOUFS(mp)->um_fs; 1851 inodedephd = INODEDEP_HASH(fs, inum); 1852 1853 if (inodedep_find(inodedephd, fs, inum, inodedeppp)) 1854 return (1); 1855 if ((flags & DEPALLOC) == 0) 1856 return (0); 1857 /* 1858 * If we are over our limit, try to improve the situation. 1859 */ 1860 if (num_inodedep > max_softdeps && (flags & NODELAY) == 0) 1861 request_cleanup(mp, FLUSH_INODES); 1862 FREE_LOCK(&lk); 1863 inodedep = malloc(sizeof(struct inodedep), 1864 M_INODEDEP, M_SOFTDEP_FLAGS); 1865 workitem_alloc(&inodedep->id_list, D_INODEDEP, mp); 1866 ACQUIRE_LOCK(&lk); 1867 if (inodedep_find(inodedephd, fs, inum, inodedeppp)) { 1868 WORKITEM_FREE(inodedep, D_INODEDEP); 1869 return (1); 1870 } 1871 num_inodedep += 1; 1872 inodedep->id_fs = fs; 1873 inodedep->id_ino = inum; 1874 inodedep->id_state = ALLCOMPLETE; 1875 inodedep->id_nlinkdelta = 0; 1876 inodedep->id_savedino1 = NULL; 1877 inodedep->id_savedsize = -1; 1878 inodedep->id_savedextsize = -1; 1879 inodedep->id_savednlink = -1; 1880 inodedep->id_bmsafemap = NULL; 1881 inodedep->id_mkdiradd = NULL; 1882 LIST_INIT(&inodedep->id_dirremhd); 1883 LIST_INIT(&inodedep->id_pendinghd); 1884 LIST_INIT(&inodedep->id_inowait); 1885 LIST_INIT(&inodedep->id_bufwait); 1886 TAILQ_INIT(&inodedep->id_inoreflst); 1887 TAILQ_INIT(&inodedep->id_inoupdt); 1888 TAILQ_INIT(&inodedep->id_newinoupdt); 1889 TAILQ_INIT(&inodedep->id_extupdt); 1890 TAILQ_INIT(&inodedep->id_newextupdt); 1891 LIST_INSERT_HEAD(inodedephd, inodedep, id_hash); 1892 *inodedeppp = inodedep; 1893 return (0); 1894 } 1895 1896 /* 1897 * Structures and routines associated with newblk caching. 1898 */ 1899 LIST_HEAD(newblk_hashhead, newblk) *newblk_hashtbl; 1900 u_long newblk_hash; /* size of hash table - 1 */ 1901 #define NEWBLK_HASH(fs, inum) \ 1902 (&newblk_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & newblk_hash]) 1903 1904 static int 1905 newblk_find(newblkhd, mp, newblkno, flags, newblkpp) 1906 struct newblk_hashhead *newblkhd; 1907 struct mount *mp; 1908 ufs2_daddr_t newblkno; 1909 int flags; 1910 struct newblk **newblkpp; 1911 { 1912 struct newblk *newblk; 1913 1914 LIST_FOREACH(newblk, newblkhd, nb_hash) { 1915 if (newblkno != newblk->nb_newblkno) 1916 continue; 1917 if (mp != newblk->nb_list.wk_mp) 1918 continue; 1919 /* 1920 * If we're creating a new dependency don't match those that 1921 * have already been converted to allocdirects. This is for 1922 * a frag extend. 1923 */ 1924 if ((flags & DEPALLOC) && newblk->nb_list.wk_type != D_NEWBLK) 1925 continue; 1926 break; 1927 } 1928 if (newblk) { 1929 *newblkpp = newblk; 1930 return (1); 1931 } 1932 *newblkpp = NULL; 1933 return (0); 1934 } 1935 1936 /* 1937 * Look up a newblk. Return 1 if found, 0 if not found. 1938 * If not found, allocate if DEPALLOC flag is passed. 1939 * Found or allocated entry is returned in newblkpp. 1940 */ 1941 static int 1942 newblk_lookup(mp, newblkno, flags, newblkpp) 1943 struct mount *mp; 1944 ufs2_daddr_t newblkno; 1945 int flags; 1946 struct newblk **newblkpp; 1947 { 1948 struct newblk *newblk; 1949 struct newblk_hashhead *newblkhd; 1950 1951 newblkhd = NEWBLK_HASH(VFSTOUFS(mp)->um_fs, newblkno); 1952 if (newblk_find(newblkhd, mp, newblkno, flags, newblkpp)) 1953 return (1); 1954 if ((flags & DEPALLOC) == 0) 1955 return (0); 1956 FREE_LOCK(&lk); 1957 newblk = malloc(sizeof(union allblk), M_NEWBLK, 1958 M_SOFTDEP_FLAGS | M_ZERO); 1959 workitem_alloc(&newblk->nb_list, D_NEWBLK, mp); 1960 ACQUIRE_LOCK(&lk); 1961 if (newblk_find(newblkhd, mp, newblkno, flags, newblkpp)) { 1962 WORKITEM_FREE(newblk, D_NEWBLK); 1963 return (1); 1964 } 1965 newblk->nb_freefrag = NULL; 1966 LIST_INIT(&newblk->nb_indirdeps); 1967 LIST_INIT(&newblk->nb_newdirblk); 1968 LIST_INIT(&newblk->nb_jwork); 1969 newblk->nb_state = ATTACHED; 1970 newblk->nb_newblkno = newblkno; 1971 LIST_INSERT_HEAD(newblkhd, newblk, nb_hash); 1972 *newblkpp = newblk; 1973 return (0); 1974 } 1975 1976 /* 1977 * Executed during filesystem system initialization before 1978 * mounting any filesystems. 1979 */ 1980 void 1981 softdep_initialize() 1982 { 1983 1984 LIST_INIT(&mkdirlisthd); 1985 max_softdeps = desiredvnodes * 4; 1986 pagedep_hashtbl = hashinit(desiredvnodes / 5, M_PAGEDEP, &pagedep_hash); 1987 inodedep_hashtbl = hashinit(desiredvnodes, M_INODEDEP, &inodedep_hash); 1988 newblk_hashtbl = hashinit(desiredvnodes / 5, M_NEWBLK, &newblk_hash); 1989 bmsafemap_hashtbl = hashinit(1024, M_BMSAFEMAP, &bmsafemap_hash); 1990 1991 /* initialise bioops hack */ 1992 bioops.io_start = softdep_disk_io_initiation; 1993 bioops.io_complete = softdep_disk_write_complete; 1994 bioops.io_deallocate = softdep_deallocate_dependencies; 1995 bioops.io_countdeps = softdep_count_dependencies; 1996 1997 /* Initialize the callout with an mtx. */ 1998 callout_init_mtx(&softdep_callout, &lk, 0); 1999 } 2000 2001 /* 2002 * Executed after all filesystems have been unmounted during 2003 * filesystem module unload. 2004 */ 2005 void 2006 softdep_uninitialize() 2007 { 2008 2009 callout_drain(&softdep_callout); 2010 hashdestroy(pagedep_hashtbl, M_PAGEDEP, pagedep_hash); 2011 hashdestroy(inodedep_hashtbl, M_INODEDEP, inodedep_hash); 2012 hashdestroy(newblk_hashtbl, M_NEWBLK, newblk_hash); 2013 hashdestroy(bmsafemap_hashtbl, M_BMSAFEMAP, bmsafemap_hash); 2014 } 2015 2016 /* 2017 * Called at mount time to notify the dependency code that a 2018 * filesystem wishes to use it. 2019 */ 2020 int 2021 softdep_mount(devvp, mp, fs, cred) 2022 struct vnode *devvp; 2023 struct mount *mp; 2024 struct fs *fs; 2025 struct ucred *cred; 2026 { 2027 struct csum_total cstotal; 2028 struct ufsmount *ump; 2029 struct cg *cgp; 2030 struct buf *bp; 2031 int error, cyl; 2032 2033 MNT_ILOCK(mp); 2034 mp->mnt_flag = (mp->mnt_flag & ~MNT_ASYNC) | MNT_SOFTDEP; 2035 if ((mp->mnt_kern_flag & MNTK_SOFTDEP) == 0) { 2036 mp->mnt_kern_flag = (mp->mnt_kern_flag & ~MNTK_ASYNC) | 2037 MNTK_SOFTDEP; 2038 mp->mnt_noasync++; 2039 } 2040 MNT_IUNLOCK(mp); 2041 ump = VFSTOUFS(mp); 2042 LIST_INIT(&ump->softdep_workitem_pending); 2043 LIST_INIT(&ump->softdep_journal_pending); 2044 TAILQ_INIT(&ump->softdep_unlinked); 2045 ump->softdep_worklist_tail = NULL; 2046 ump->softdep_on_worklist = 0; 2047 ump->softdep_deps = 0; 2048 if ((fs->fs_flags & FS_SUJ) && 2049 (error = journal_mount(mp, fs, cred)) != 0) { 2050 printf("Failed to start journal: %d\n", error); 2051 return (error); 2052 } 2053 /* 2054 * When doing soft updates, the counters in the 2055 * superblock may have gotten out of sync. Recomputation 2056 * can take a long time and can be deferred for background 2057 * fsck. However, the old behavior of scanning the cylinder 2058 * groups and recalculating them at mount time is available 2059 * by setting vfs.ffs.compute_summary_at_mount to one. 2060 */ 2061 if (compute_summary_at_mount == 0 || fs->fs_clean != 0) 2062 return (0); 2063 bzero(&cstotal, sizeof cstotal); 2064 for (cyl = 0; cyl < fs->fs_ncg; cyl++) { 2065 if ((error = bread(devvp, fsbtodb(fs, cgtod(fs, cyl)), 2066 fs->fs_cgsize, cred, &bp)) != 0) { 2067 brelse(bp); 2068 return (error); 2069 } 2070 cgp = (struct cg *)bp->b_data; 2071 cstotal.cs_nffree += cgp->cg_cs.cs_nffree; 2072 cstotal.cs_nbfree += cgp->cg_cs.cs_nbfree; 2073 cstotal.cs_nifree += cgp->cg_cs.cs_nifree; 2074 cstotal.cs_ndir += cgp->cg_cs.cs_ndir; 2075 fs->fs_cs(fs, cyl) = cgp->cg_cs; 2076 brelse(bp); 2077 } 2078 #ifdef DEBUG 2079 if (bcmp(&cstotal, &fs->fs_cstotal, sizeof cstotal)) 2080 printf("%s: superblock summary recomputed\n", fs->fs_fsmnt); 2081 #endif 2082 bcopy(&cstotal, &fs->fs_cstotal, sizeof cstotal); 2083 return (0); 2084 } 2085 2086 void 2087 softdep_unmount(mp) 2088 struct mount *mp; 2089 { 2090 2091 if (mp->mnt_kern_flag & MNTK_SUJ) 2092 journal_unmount(mp); 2093 } 2094 2095 struct jblocks { 2096 struct jseglst jb_segs; /* TAILQ of current segments. */ 2097 struct jseg *jb_writeseg; /* Next write to complete. */ 2098 struct jextent *jb_extent; /* Extent array. */ 2099 uint64_t jb_nextseq; /* Next sequence number. */ 2100 uint64_t jb_oldestseq; /* Oldest active sequence number. */ 2101 int jb_avail; /* Available extents. */ 2102 int jb_used; /* Last used extent. */ 2103 int jb_head; /* Allocator head. */ 2104 int jb_off; /* Allocator extent offset. */ 2105 int jb_blocks; /* Total disk blocks covered. */ 2106 int jb_free; /* Total disk blocks free. */ 2107 int jb_min; /* Minimum free space. */ 2108 int jb_low; /* Low on space. */ 2109 int jb_age; /* Insertion time of oldest rec. */ 2110 int jb_suspended; /* Did journal suspend writes? */ 2111 }; 2112 2113 struct jextent { 2114 ufs2_daddr_t je_daddr; /* Disk block address. */ 2115 int je_blocks; /* Disk block count. */ 2116 }; 2117 2118 static struct jblocks * 2119 jblocks_create(void) 2120 { 2121 struct jblocks *jblocks; 2122 2123 jblocks = malloc(sizeof(*jblocks), M_JBLOCKS, M_WAITOK | M_ZERO); 2124 TAILQ_INIT(&jblocks->jb_segs); 2125 jblocks->jb_avail = 10; 2126 jblocks->jb_extent = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2127 M_JBLOCKS, M_WAITOK | M_ZERO); 2128 2129 return (jblocks); 2130 } 2131 2132 static ufs2_daddr_t 2133 jblocks_alloc(jblocks, bytes, actual) 2134 struct jblocks *jblocks; 2135 int bytes; 2136 int *actual; 2137 { 2138 ufs2_daddr_t daddr; 2139 struct jextent *jext; 2140 int freecnt; 2141 int blocks; 2142 2143 blocks = bytes / DEV_BSIZE; 2144 jext = &jblocks->jb_extent[jblocks->jb_head]; 2145 freecnt = jext->je_blocks - jblocks->jb_off; 2146 if (freecnt == 0) { 2147 jblocks->jb_off = 0; 2148 if (++jblocks->jb_head > jblocks->jb_used) 2149 jblocks->jb_head = 0; 2150 jext = &jblocks->jb_extent[jblocks->jb_head]; 2151 freecnt = jext->je_blocks; 2152 } 2153 if (freecnt > blocks) 2154 freecnt = blocks; 2155 *actual = freecnt * DEV_BSIZE; 2156 daddr = jext->je_daddr + jblocks->jb_off; 2157 jblocks->jb_off += freecnt; 2158 jblocks->jb_free -= freecnt; 2159 2160 return (daddr); 2161 } 2162 2163 static void 2164 jblocks_free(jblocks, mp, bytes) 2165 struct jblocks *jblocks; 2166 struct mount *mp; 2167 int bytes; 2168 { 2169 2170 jblocks->jb_free += bytes / DEV_BSIZE; 2171 if (jblocks->jb_suspended) 2172 worklist_speedup(); 2173 wakeup(jblocks); 2174 } 2175 2176 static void 2177 jblocks_destroy(jblocks) 2178 struct jblocks *jblocks; 2179 { 2180 2181 if (jblocks->jb_extent) 2182 free(jblocks->jb_extent, M_JBLOCKS); 2183 free(jblocks, M_JBLOCKS); 2184 } 2185 2186 static void 2187 jblocks_add(jblocks, daddr, blocks) 2188 struct jblocks *jblocks; 2189 ufs2_daddr_t daddr; 2190 int blocks; 2191 { 2192 struct jextent *jext; 2193 2194 jblocks->jb_blocks += blocks; 2195 jblocks->jb_free += blocks; 2196 jext = &jblocks->jb_extent[jblocks->jb_used]; 2197 /* Adding the first block. */ 2198 if (jext->je_daddr == 0) { 2199 jext->je_daddr = daddr; 2200 jext->je_blocks = blocks; 2201 return; 2202 } 2203 /* Extending the last extent. */ 2204 if (jext->je_daddr + jext->je_blocks == daddr) { 2205 jext->je_blocks += blocks; 2206 return; 2207 } 2208 /* Adding a new extent. */ 2209 if (++jblocks->jb_used == jblocks->jb_avail) { 2210 jblocks->jb_avail *= 2; 2211 jext = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2212 M_JBLOCKS, M_WAITOK | M_ZERO); 2213 memcpy(jext, jblocks->jb_extent, 2214 sizeof(struct jextent) * jblocks->jb_used); 2215 free(jblocks->jb_extent, M_JBLOCKS); 2216 jblocks->jb_extent = jext; 2217 } 2218 jext = &jblocks->jb_extent[jblocks->jb_used]; 2219 jext->je_daddr = daddr; 2220 jext->je_blocks = blocks; 2221 return; 2222 } 2223 2224 int 2225 softdep_journal_lookup(mp, vpp) 2226 struct mount *mp; 2227 struct vnode **vpp; 2228 { 2229 struct componentname cnp; 2230 struct vnode *dvp; 2231 ino_t sujournal; 2232 int error; 2233 2234 error = VFS_VGET(mp, ROOTINO, LK_EXCLUSIVE, &dvp); 2235 if (error) 2236 return (error); 2237 bzero(&cnp, sizeof(cnp)); 2238 cnp.cn_nameiop = LOOKUP; 2239 cnp.cn_flags = ISLASTCN; 2240 cnp.cn_thread = curthread; 2241 cnp.cn_cred = curthread->td_ucred; 2242 cnp.cn_pnbuf = SUJ_FILE; 2243 cnp.cn_nameptr = SUJ_FILE; 2244 cnp.cn_namelen = strlen(SUJ_FILE); 2245 error = ufs_lookup_ino(dvp, NULL, &cnp, &sujournal); 2246 vput(dvp); 2247 if (error != 0) 2248 return (error); 2249 error = VFS_VGET(mp, sujournal, LK_EXCLUSIVE, vpp); 2250 return (error); 2251 } 2252 2253 /* 2254 * Open and verify the journal file. 2255 */ 2256 static int 2257 journal_mount(mp, fs, cred) 2258 struct mount *mp; 2259 struct fs *fs; 2260 struct ucred *cred; 2261 { 2262 struct jblocks *jblocks; 2263 struct vnode *vp; 2264 struct inode *ip; 2265 ufs2_daddr_t blkno; 2266 int bcount; 2267 int error; 2268 int i; 2269 2270 mp->mnt_kern_flag |= MNTK_SUJ; 2271 error = softdep_journal_lookup(mp, &vp); 2272 if (error != 0) { 2273 printf("Failed to find journal. Use tunefs to create one\n"); 2274 return (error); 2275 } 2276 ip = VTOI(vp); 2277 if (ip->i_size < SUJ_MIN) { 2278 error = ENOSPC; 2279 goto out; 2280 } 2281 bcount = lblkno(fs, ip->i_size); /* Only use whole blocks. */ 2282 jblocks = jblocks_create(); 2283 for (i = 0; i < bcount; i++) { 2284 error = ufs_bmaparray(vp, i, &blkno, NULL, NULL, NULL); 2285 if (error) 2286 break; 2287 jblocks_add(jblocks, blkno, fsbtodb(fs, fs->fs_frag)); 2288 } 2289 if (error) { 2290 jblocks_destroy(jblocks); 2291 goto out; 2292 } 2293 jblocks->jb_low = jblocks->jb_free / 3; /* Reserve 33%. */ 2294 jblocks->jb_min = jblocks->jb_free / 10; /* Suspend at 10%. */ 2295 /* 2296 * Only validate the journal contents if the filesystem is clean, 2297 * otherwise we write the logs but they'll never be used. If the 2298 * filesystem was still dirty when we mounted it the journal is 2299 * invalid and a new journal can only be valid if it starts from a 2300 * clean mount. 2301 */ 2302 if (fs->fs_clean) { 2303 DIP_SET(ip, i_modrev, fs->fs_mtime); 2304 ip->i_flags |= IN_MODIFIED; 2305 ffs_update(vp, 1); 2306 } 2307 VFSTOUFS(mp)->softdep_jblocks = jblocks; 2308 out: 2309 vput(vp); 2310 return (error); 2311 } 2312 2313 static void 2314 journal_unmount(mp) 2315 struct mount *mp; 2316 { 2317 struct ufsmount *ump; 2318 2319 ump = VFSTOUFS(mp); 2320 if (ump->softdep_jblocks) 2321 jblocks_destroy(ump->softdep_jblocks); 2322 ump->softdep_jblocks = NULL; 2323 } 2324 2325 /* 2326 * Called when a journal record is ready to be written. Space is allocated 2327 * and the journal entry is created when the journal is flushed to stable 2328 * store. 2329 */ 2330 static void 2331 add_to_journal(wk) 2332 struct worklist *wk; 2333 { 2334 struct ufsmount *ump; 2335 2336 mtx_assert(&lk, MA_OWNED); 2337 ump = VFSTOUFS(wk->wk_mp); 2338 if (wk->wk_state & ONWORKLIST) 2339 panic("add_to_journal: %s(0x%X) already on list", 2340 TYPENAME(wk->wk_type), wk->wk_state); 2341 wk->wk_state |= ONWORKLIST | DEPCOMPLETE; 2342 if (LIST_EMPTY(&ump->softdep_journal_pending)) { 2343 ump->softdep_jblocks->jb_age = ticks; 2344 LIST_INSERT_HEAD(&ump->softdep_journal_pending, wk, wk_list); 2345 } else 2346 LIST_INSERT_AFTER(ump->softdep_journal_tail, wk, wk_list); 2347 ump->softdep_journal_tail = wk; 2348 ump->softdep_on_journal += 1; 2349 } 2350 2351 /* 2352 * Remove an arbitrary item for the journal worklist maintain the tail 2353 * pointer. This happens when a new operation obviates the need to 2354 * journal an old operation. 2355 */ 2356 static void 2357 remove_from_journal(wk) 2358 struct worklist *wk; 2359 { 2360 struct ufsmount *ump; 2361 2362 mtx_assert(&lk, MA_OWNED); 2363 ump = VFSTOUFS(wk->wk_mp); 2364 #ifdef SUJ_DEBUG 2365 { 2366 struct worklist *wkn; 2367 2368 LIST_FOREACH(wkn, &ump->softdep_journal_pending, wk_list) 2369 if (wkn == wk) 2370 break; 2371 if (wkn == NULL) 2372 panic("remove_from_journal: %p is not in journal", wk); 2373 } 2374 #endif 2375 /* 2376 * We emulate a TAILQ to save space in most structures which do not 2377 * require TAILQ semantics. Here we must update the tail position 2378 * when removing the tail which is not the final entry. 2379 */ 2380 if (ump->softdep_journal_tail == wk) 2381 ump->softdep_journal_tail = 2382 (struct worklist *)wk->wk_list.le_prev; 2383 2384 WORKLIST_REMOVE(wk); 2385 ump->softdep_on_journal -= 1; 2386 } 2387 2388 /* 2389 * Check for journal space as well as dependency limits so the prelink 2390 * code can throttle both journaled and non-journaled filesystems. 2391 * Threshold is 0 for low and 1 for min. 2392 */ 2393 static int 2394 journal_space(ump, thresh) 2395 struct ufsmount *ump; 2396 int thresh; 2397 { 2398 struct jblocks *jblocks; 2399 int avail; 2400 2401 jblocks = ump->softdep_jblocks; 2402 if (jblocks == NULL) 2403 return (1); 2404 /* 2405 * We use a tighter restriction here to prevent request_cleanup() 2406 * running in threads from running into locks we currently hold. 2407 */ 2408 if (num_inodedep > (max_softdeps / 10) * 9) 2409 return (0); 2410 if (thresh) 2411 thresh = jblocks->jb_min; 2412 else 2413 thresh = jblocks->jb_low; 2414 avail = (ump->softdep_on_journal * JREC_SIZE) / DEV_BSIZE; 2415 avail = jblocks->jb_free - avail; 2416 2417 return (avail > thresh); 2418 } 2419 2420 static void 2421 journal_suspend(ump) 2422 struct ufsmount *ump; 2423 { 2424 struct jblocks *jblocks; 2425 struct mount *mp; 2426 2427 mp = UFSTOVFS(ump); 2428 jblocks = ump->softdep_jblocks; 2429 MNT_ILOCK(mp); 2430 if ((mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 2431 stat_journal_min++; 2432 mp->mnt_kern_flag |= MNTK_SUSPEND; 2433 mp->mnt_susp_owner = FIRST_THREAD_IN_PROC(softdepproc); 2434 } 2435 jblocks->jb_suspended = 1; 2436 MNT_IUNLOCK(mp); 2437 } 2438 2439 /* 2440 * Called before any allocation function to be certain that there is 2441 * sufficient space in the journal prior to creating any new records. 2442 * Since in the case of block allocation we may have multiple locked 2443 * buffers at the time of the actual allocation we can not block 2444 * when the journal records are created. Doing so would create a deadlock 2445 * if any of these buffers needed to be flushed to reclaim space. Instead 2446 * we require a sufficiently large amount of available space such that 2447 * each thread in the system could have passed this allocation check and 2448 * still have sufficient free space. With 20% of a minimum journal size 2449 * of 1MB we have 6553 records available. 2450 */ 2451 int 2452 softdep_prealloc(vp, waitok) 2453 struct vnode *vp; 2454 int waitok; 2455 { 2456 struct ufsmount *ump; 2457 2458 if (DOINGSUJ(vp) == 0) 2459 return (0); 2460 ump = VFSTOUFS(vp->v_mount); 2461 ACQUIRE_LOCK(&lk); 2462 if (journal_space(ump, 0)) { 2463 FREE_LOCK(&lk); 2464 return (0); 2465 } 2466 stat_journal_low++; 2467 FREE_LOCK(&lk); 2468 if (waitok == MNT_NOWAIT) 2469 return (ENOSPC); 2470 /* 2471 * Attempt to sync this vnode once to flush any journal 2472 * work attached to it. 2473 */ 2474 if ((curthread->td_pflags & TDP_COWINPROGRESS) == 0) 2475 ffs_syncvnode(vp, waitok); 2476 ACQUIRE_LOCK(&lk); 2477 process_removes(vp); 2478 if (journal_space(ump, 0) == 0) { 2479 softdep_speedup(); 2480 if (journal_space(ump, 1) == 0) 2481 journal_suspend(ump); 2482 } 2483 FREE_LOCK(&lk); 2484 2485 return (0); 2486 } 2487 2488 /* 2489 * Before adjusting a link count on a vnode verify that we have sufficient 2490 * journal space. If not, process operations that depend on the currently 2491 * locked pair of vnodes to try to flush space as the syncer, buf daemon, 2492 * and softdep flush threads can not acquire these locks to reclaim space. 2493 */ 2494 static void 2495 softdep_prelink(dvp, vp) 2496 struct vnode *dvp; 2497 struct vnode *vp; 2498 { 2499 struct ufsmount *ump; 2500 2501 ump = VFSTOUFS(dvp->v_mount); 2502 mtx_assert(&lk, MA_OWNED); 2503 if (journal_space(ump, 0)) 2504 return; 2505 stat_journal_low++; 2506 FREE_LOCK(&lk); 2507 if (vp) 2508 ffs_syncvnode(vp, MNT_NOWAIT); 2509 ffs_syncvnode(dvp, MNT_WAIT); 2510 ACQUIRE_LOCK(&lk); 2511 /* Process vp before dvp as it may create .. removes. */ 2512 if (vp) 2513 process_removes(vp); 2514 process_removes(dvp); 2515 softdep_speedup(); 2516 process_worklist_item(UFSTOVFS(ump), LK_NOWAIT); 2517 process_worklist_item(UFSTOVFS(ump), LK_NOWAIT); 2518 if (journal_space(ump, 0) == 0) { 2519 softdep_speedup(); 2520 if (journal_space(ump, 1) == 0) 2521 journal_suspend(ump); 2522 } 2523 } 2524 2525 static void 2526 jseg_write(fs, jblocks, jseg, data) 2527 struct fs *fs; 2528 struct jblocks *jblocks; 2529 struct jseg *jseg; 2530 uint8_t *data; 2531 { 2532 struct jsegrec *rec; 2533 2534 rec = (struct jsegrec *)data; 2535 rec->jsr_seq = jseg->js_seq; 2536 rec->jsr_oldest = jblocks->jb_oldestseq; 2537 rec->jsr_cnt = jseg->js_cnt; 2538 rec->jsr_blocks = jseg->js_size / DEV_BSIZE; 2539 rec->jsr_crc = 0; 2540 rec->jsr_time = fs->fs_mtime; 2541 } 2542 2543 static inline void 2544 inoref_write(inoref, jseg, rec) 2545 struct inoref *inoref; 2546 struct jseg *jseg; 2547 struct jrefrec *rec; 2548 { 2549 2550 inoref->if_jsegdep->jd_seg = jseg; 2551 rec->jr_ino = inoref->if_ino; 2552 rec->jr_parent = inoref->if_parent; 2553 rec->jr_nlink = inoref->if_nlink; 2554 rec->jr_mode = inoref->if_mode; 2555 rec->jr_diroff = inoref->if_diroff; 2556 } 2557 2558 static void 2559 jaddref_write(jaddref, jseg, data) 2560 struct jaddref *jaddref; 2561 struct jseg *jseg; 2562 uint8_t *data; 2563 { 2564 struct jrefrec *rec; 2565 2566 rec = (struct jrefrec *)data; 2567 rec->jr_op = JOP_ADDREF; 2568 inoref_write(&jaddref->ja_ref, jseg, rec); 2569 } 2570 2571 static void 2572 jremref_write(jremref, jseg, data) 2573 struct jremref *jremref; 2574 struct jseg *jseg; 2575 uint8_t *data; 2576 { 2577 struct jrefrec *rec; 2578 2579 rec = (struct jrefrec *)data; 2580 rec->jr_op = JOP_REMREF; 2581 inoref_write(&jremref->jr_ref, jseg, rec); 2582 } 2583 2584 static void 2585 jmvref_write(jmvref, jseg, data) 2586 struct jmvref *jmvref; 2587 struct jseg *jseg; 2588 uint8_t *data; 2589 { 2590 struct jmvrec *rec; 2591 2592 rec = (struct jmvrec *)data; 2593 rec->jm_op = JOP_MVREF; 2594 rec->jm_ino = jmvref->jm_ino; 2595 rec->jm_parent = jmvref->jm_parent; 2596 rec->jm_oldoff = jmvref->jm_oldoff; 2597 rec->jm_newoff = jmvref->jm_newoff; 2598 } 2599 2600 static void 2601 jnewblk_write(jnewblk, jseg, data) 2602 struct jnewblk *jnewblk; 2603 struct jseg *jseg; 2604 uint8_t *data; 2605 { 2606 struct jblkrec *rec; 2607 2608 jnewblk->jn_jsegdep->jd_seg = jseg; 2609 rec = (struct jblkrec *)data; 2610 rec->jb_op = JOP_NEWBLK; 2611 rec->jb_ino = jnewblk->jn_ino; 2612 rec->jb_blkno = jnewblk->jn_blkno; 2613 rec->jb_lbn = jnewblk->jn_lbn; 2614 rec->jb_frags = jnewblk->jn_frags; 2615 rec->jb_oldfrags = jnewblk->jn_oldfrags; 2616 } 2617 2618 static void 2619 jfreeblk_write(jfreeblk, jseg, data) 2620 struct jfreeblk *jfreeblk; 2621 struct jseg *jseg; 2622 uint8_t *data; 2623 { 2624 struct jblkrec *rec; 2625 2626 jfreeblk->jf_jsegdep->jd_seg = jseg; 2627 rec = (struct jblkrec *)data; 2628 rec->jb_op = JOP_FREEBLK; 2629 rec->jb_ino = jfreeblk->jf_ino; 2630 rec->jb_blkno = jfreeblk->jf_blkno; 2631 rec->jb_lbn = jfreeblk->jf_lbn; 2632 rec->jb_frags = jfreeblk->jf_frags; 2633 rec->jb_oldfrags = 0; 2634 } 2635 2636 static void 2637 jfreefrag_write(jfreefrag, jseg, data) 2638 struct jfreefrag *jfreefrag; 2639 struct jseg *jseg; 2640 uint8_t *data; 2641 { 2642 struct jblkrec *rec; 2643 2644 jfreefrag->fr_jsegdep->jd_seg = jseg; 2645 rec = (struct jblkrec *)data; 2646 rec->jb_op = JOP_FREEBLK; 2647 rec->jb_ino = jfreefrag->fr_ino; 2648 rec->jb_blkno = jfreefrag->fr_blkno; 2649 rec->jb_lbn = jfreefrag->fr_lbn; 2650 rec->jb_frags = jfreefrag->fr_frags; 2651 rec->jb_oldfrags = 0; 2652 } 2653 2654 static void 2655 jtrunc_write(jtrunc, jseg, data) 2656 struct jtrunc *jtrunc; 2657 struct jseg *jseg; 2658 uint8_t *data; 2659 { 2660 struct jtrncrec *rec; 2661 2662 rec = (struct jtrncrec *)data; 2663 rec->jt_op = JOP_TRUNC; 2664 rec->jt_ino = jtrunc->jt_ino; 2665 rec->jt_size = jtrunc->jt_size; 2666 rec->jt_extsize = jtrunc->jt_extsize; 2667 } 2668 2669 /* 2670 * Flush some journal records to disk. 2671 */ 2672 static void 2673 softdep_process_journal(mp, flags) 2674 struct mount *mp; 2675 int flags; 2676 { 2677 struct jblocks *jblocks; 2678 struct ufsmount *ump; 2679 struct worklist *wk; 2680 struct jseg *jseg; 2681 struct buf *bp; 2682 uint8_t *data; 2683 struct fs *fs; 2684 int segwritten; 2685 int jrecmin; /* Minimum records per block. */ 2686 int jrecmax; /* Maximum records per block. */ 2687 int size; 2688 int cnt; 2689 int off; 2690 2691 if ((mp->mnt_kern_flag & MNTK_SUJ) == 0) 2692 return; 2693 ump = VFSTOUFS(mp); 2694 fs = ump->um_fs; 2695 jblocks = ump->softdep_jblocks; 2696 /* 2697 * We write anywhere between a disk block and fs block. The upper 2698 * bound is picked to prevent buffer cache fragmentation and limit 2699 * processing time per I/O. 2700 */ 2701 jrecmin = (DEV_BSIZE / JREC_SIZE) - 1; /* -1 for seg header */ 2702 jrecmax = (fs->fs_bsize / DEV_BSIZE) * jrecmin; 2703 segwritten = 0; 2704 while ((cnt = ump->softdep_on_journal) != 0) { 2705 /* 2706 * Create a new segment to hold as many as 'cnt' journal 2707 * entries and add them to the segment. Notice cnt is 2708 * off by one to account for the space required by the 2709 * jsegrec. If we don't have a full block to log skip it 2710 * unless we haven't written anything. 2711 */ 2712 cnt++; 2713 if (cnt < jrecmax && segwritten) 2714 break; 2715 /* 2716 * Verify some free journal space. softdep_prealloc() should 2717 * guarantee that we don't run out so this is indicative of 2718 * a problem with the flow control. Try to recover 2719 * gracefully in any event. 2720 */ 2721 while (jblocks->jb_free == 0) { 2722 if (flags != MNT_WAIT) 2723 break; 2724 printf("softdep: Out of journal space!\n"); 2725 softdep_speedup(); 2726 msleep(jblocks, &lk, PRIBIO, "jblocks", hz); 2727 } 2728 FREE_LOCK(&lk); 2729 jseg = malloc(sizeof(*jseg), M_JSEG, M_SOFTDEP_FLAGS); 2730 workitem_alloc(&jseg->js_list, D_JSEG, mp); 2731 LIST_INIT(&jseg->js_entries); 2732 jseg->js_state = ATTACHED; 2733 jseg->js_jblocks = jblocks; 2734 bp = geteblk(fs->fs_bsize, 0); 2735 ACQUIRE_LOCK(&lk); 2736 /* 2737 * If there was a race while we were allocating the block 2738 * and jseg the entry we care about was likely written. 2739 * We bail out in both the WAIT and NOWAIT case and assume 2740 * the caller will loop if the entry it cares about is 2741 * not written. 2742 */ 2743 if (ump->softdep_on_journal == 0 || jblocks->jb_free == 0) { 2744 bp->b_flags |= B_INVAL | B_NOCACHE; 2745 WORKITEM_FREE(jseg, D_JSEG); 2746 FREE_LOCK(&lk); 2747 brelse(bp); 2748 ACQUIRE_LOCK(&lk); 2749 break; 2750 } 2751 /* 2752 * Calculate the disk block size required for the available 2753 * records rounded to the min size. 2754 */ 2755 cnt = ump->softdep_on_journal; 2756 if (cnt < jrecmax) 2757 size = howmany(cnt, jrecmin) * DEV_BSIZE; 2758 else 2759 size = fs->fs_bsize; 2760 /* 2761 * Allocate a disk block for this journal data and account 2762 * for truncation of the requested size if enough contiguous 2763 * space was not available. 2764 */ 2765 bp->b_blkno = jblocks_alloc(jblocks, size, &size); 2766 bp->b_lblkno = bp->b_blkno; 2767 bp->b_offset = bp->b_blkno * DEV_BSIZE; 2768 bp->b_bcount = size; 2769 bp->b_bufobj = &ump->um_devvp->v_bufobj; 2770 bp->b_flags &= ~B_INVAL; 2771 bp->b_flags |= B_VALIDSUSPWRT | B_NOCOPY; 2772 /* 2773 * Initialize our jseg with cnt records. Assign the next 2774 * sequence number to it and link it in-order. 2775 */ 2776 cnt = MIN(ump->softdep_on_journal, 2777 (size / DEV_BSIZE) * jrecmin); 2778 jseg->js_buf = bp; 2779 jseg->js_cnt = cnt; 2780 jseg->js_refs = cnt + 1; /* Self ref. */ 2781 jseg->js_size = size; 2782 jseg->js_seq = jblocks->jb_nextseq++; 2783 if (TAILQ_EMPTY(&jblocks->jb_segs)) 2784 jblocks->jb_oldestseq = jseg->js_seq; 2785 TAILQ_INSERT_TAIL(&jblocks->jb_segs, jseg, js_next); 2786 if (jblocks->jb_writeseg == NULL) 2787 jblocks->jb_writeseg = jseg; 2788 /* 2789 * Start filling in records from the pending list. 2790 */ 2791 data = bp->b_data; 2792 off = 0; 2793 while ((wk = LIST_FIRST(&ump->softdep_journal_pending)) 2794 != NULL) { 2795 /* Place a segment header on every device block. */ 2796 if ((off % DEV_BSIZE) == 0) { 2797 jseg_write(fs, jblocks, jseg, data); 2798 off += JREC_SIZE; 2799 data = bp->b_data + off; 2800 } 2801 remove_from_journal(wk); 2802 wk->wk_state |= IOSTARTED; 2803 WORKLIST_INSERT(&jseg->js_entries, wk); 2804 switch (wk->wk_type) { 2805 case D_JADDREF: 2806 jaddref_write(WK_JADDREF(wk), jseg, data); 2807 break; 2808 case D_JREMREF: 2809 jremref_write(WK_JREMREF(wk), jseg, data); 2810 break; 2811 case D_JMVREF: 2812 jmvref_write(WK_JMVREF(wk), jseg, data); 2813 break; 2814 case D_JNEWBLK: 2815 jnewblk_write(WK_JNEWBLK(wk), jseg, data); 2816 break; 2817 case D_JFREEBLK: 2818 jfreeblk_write(WK_JFREEBLK(wk), jseg, data); 2819 break; 2820 case D_JFREEFRAG: 2821 jfreefrag_write(WK_JFREEFRAG(wk), jseg, data); 2822 break; 2823 case D_JTRUNC: 2824 jtrunc_write(WK_JTRUNC(wk), jseg, data); 2825 break; 2826 default: 2827 panic("process_journal: Unknown type %s", 2828 TYPENAME(wk->wk_type)); 2829 /* NOTREACHED */ 2830 } 2831 if (--cnt == 0) 2832 break; 2833 off += JREC_SIZE; 2834 data = bp->b_data + off; 2835 } 2836 /* 2837 * Write this one buffer and continue. 2838 */ 2839 WORKLIST_INSERT(&bp->b_dep, &jseg->js_list); 2840 FREE_LOCK(&lk); 2841 BO_LOCK(bp->b_bufobj); 2842 bgetvp(ump->um_devvp, bp); 2843 BO_UNLOCK(bp->b_bufobj); 2844 if (flags == MNT_NOWAIT) 2845 bawrite(bp); 2846 else 2847 bwrite(bp); 2848 ACQUIRE_LOCK(&lk); 2849 } 2850 /* 2851 * If we've suspended the filesystem because we ran out of journal 2852 * space either try to sync it here to make some progress or 2853 * unsuspend it if we already have. 2854 */ 2855 if (flags == 0 && jblocks && jblocks->jb_suspended) { 2856 if (journal_space(ump, jblocks->jb_min)) { 2857 FREE_LOCK(&lk); 2858 jblocks->jb_suspended = 0; 2859 mp->mnt_susp_owner = curthread; 2860 vfs_write_resume(mp); 2861 ACQUIRE_LOCK(&lk); 2862 return; 2863 } 2864 FREE_LOCK(&lk); 2865 VFS_SYNC(mp, MNT_NOWAIT); 2866 ffs_sbupdate(ump, MNT_WAIT, 0); 2867 ACQUIRE_LOCK(&lk); 2868 } 2869 } 2870 2871 /* 2872 * Complete a jseg, allowing all dependencies awaiting journal writes 2873 * to proceed. Each journal dependency also attaches a jsegdep to dependent 2874 * structures so that the journal segment can be freed to reclaim space. 2875 */ 2876 static void 2877 complete_jseg(jseg) 2878 struct jseg *jseg; 2879 { 2880 struct worklist *wk; 2881 struct jmvref *jmvref; 2882 int waiting; 2883 int i; 2884 2885 i = 0; 2886 while ((wk = LIST_FIRST(&jseg->js_entries)) != NULL) { 2887 WORKLIST_REMOVE(wk); 2888 waiting = wk->wk_state & IOWAITING; 2889 wk->wk_state &= ~(IOSTARTED | IOWAITING); 2890 wk->wk_state |= COMPLETE; 2891 KASSERT(i < jseg->js_cnt, 2892 ("handle_written_jseg: overflow %d >= %d", 2893 i, jseg->js_cnt)); 2894 switch (wk->wk_type) { 2895 case D_JADDREF: 2896 handle_written_jaddref(WK_JADDREF(wk)); 2897 break; 2898 case D_JREMREF: 2899 handle_written_jremref(WK_JREMREF(wk)); 2900 break; 2901 case D_JMVREF: 2902 /* No jsegdep here. */ 2903 free_jseg(jseg); 2904 jmvref = WK_JMVREF(wk); 2905 LIST_REMOVE(jmvref, jm_deps); 2906 free_pagedep(jmvref->jm_pagedep); 2907 WORKITEM_FREE(jmvref, D_JMVREF); 2908 break; 2909 case D_JNEWBLK: 2910 handle_written_jnewblk(WK_JNEWBLK(wk)); 2911 break; 2912 case D_JFREEBLK: 2913 handle_written_jfreeblk(WK_JFREEBLK(wk)); 2914 break; 2915 case D_JFREEFRAG: 2916 handle_written_jfreefrag(WK_JFREEFRAG(wk)); 2917 break; 2918 case D_JTRUNC: 2919 WK_JTRUNC(wk)->jt_jsegdep->jd_seg = jseg; 2920 WORKITEM_FREE(wk, D_JTRUNC); 2921 break; 2922 default: 2923 panic("handle_written_jseg: Unknown type %s", 2924 TYPENAME(wk->wk_type)); 2925 /* NOTREACHED */ 2926 } 2927 if (waiting) 2928 wakeup(wk); 2929 } 2930 /* Release the self reference so the structure may be freed. */ 2931 free_jseg(jseg); 2932 } 2933 2934 /* 2935 * Mark a jseg as DEPCOMPLETE and throw away the buffer. Handle jseg 2936 * completions in order only. 2937 */ 2938 static void 2939 handle_written_jseg(jseg, bp) 2940 struct jseg *jseg; 2941 struct buf *bp; 2942 { 2943 struct jblocks *jblocks; 2944 struct jseg *jsegn; 2945 2946 if (jseg->js_refs == 0) 2947 panic("handle_written_jseg: No self-reference on %p", jseg); 2948 jseg->js_state |= DEPCOMPLETE; 2949 /* 2950 * We'll never need this buffer again, set flags so it will be 2951 * discarded. 2952 */ 2953 bp->b_flags |= B_INVAL | B_NOCACHE; 2954 jblocks = jseg->js_jblocks; 2955 /* 2956 * Don't allow out of order completions. If this isn't the first 2957 * block wait for it to write before we're done. 2958 */ 2959 if (jseg != jblocks->jb_writeseg) 2960 return; 2961 /* Iterate through available jsegs processing their entries. */ 2962 do { 2963 jsegn = TAILQ_NEXT(jseg, js_next); 2964 complete_jseg(jseg); 2965 jseg = jsegn; 2966 } while (jseg && jseg->js_state & DEPCOMPLETE); 2967 jblocks->jb_writeseg = jseg; 2968 } 2969 2970 static inline struct jsegdep * 2971 inoref_jseg(inoref) 2972 struct inoref *inoref; 2973 { 2974 struct jsegdep *jsegdep; 2975 2976 jsegdep = inoref->if_jsegdep; 2977 inoref->if_jsegdep = NULL; 2978 2979 return (jsegdep); 2980 } 2981 2982 /* 2983 * Called once a jremref has made it to stable store. The jremref is marked 2984 * complete and we attempt to free it. Any pagedeps writes sleeping waiting 2985 * for the jremref to complete will be awoken by free_jremref. 2986 */ 2987 static void 2988 handle_written_jremref(jremref) 2989 struct jremref *jremref; 2990 { 2991 struct inodedep *inodedep; 2992 struct jsegdep *jsegdep; 2993 struct dirrem *dirrem; 2994 2995 /* Grab the jsegdep. */ 2996 jsegdep = inoref_jseg(&jremref->jr_ref); 2997 /* 2998 * Remove us from the inoref list. 2999 */ 3000 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 3001 0, &inodedep) == 0) 3002 panic("handle_written_jremref: Lost inodedep"); 3003 TAILQ_REMOVE(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 3004 /* 3005 * Complete the dirrem. 3006 */ 3007 dirrem = jremref->jr_dirrem; 3008 jremref->jr_dirrem = NULL; 3009 LIST_REMOVE(jremref, jr_deps); 3010 jsegdep->jd_state |= jremref->jr_state & MKDIR_PARENT; 3011 WORKLIST_INSERT(&dirrem->dm_jwork, &jsegdep->jd_list); 3012 if (LIST_EMPTY(&dirrem->dm_jremrefhd) && 3013 (dirrem->dm_state & COMPLETE) != 0) 3014 add_to_worklist(&dirrem->dm_list, 0); 3015 free_jremref(jremref); 3016 } 3017 3018 /* 3019 * Called once a jaddref has made it to stable store. The dependency is 3020 * marked complete and any dependent structures are added to the inode 3021 * bufwait list to be completed as soon as it is written. If a bitmap write 3022 * depends on this entry we move the inode into the inodedephd of the 3023 * bmsafemap dependency and attempt to remove the jaddref from the bmsafemap. 3024 */ 3025 static void 3026 handle_written_jaddref(jaddref) 3027 struct jaddref *jaddref; 3028 { 3029 struct jsegdep *jsegdep; 3030 struct inodedep *inodedep; 3031 struct diradd *diradd; 3032 struct mkdir *mkdir; 3033 3034 /* Grab the jsegdep. */ 3035 jsegdep = inoref_jseg(&jaddref->ja_ref); 3036 mkdir = NULL; 3037 diradd = NULL; 3038 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 3039 0, &inodedep) == 0) 3040 panic("handle_written_jaddref: Lost inodedep."); 3041 if (jaddref->ja_diradd == NULL) 3042 panic("handle_written_jaddref: No dependency"); 3043 if (jaddref->ja_diradd->da_list.wk_type == D_DIRADD) { 3044 diradd = jaddref->ja_diradd; 3045 WORKLIST_INSERT(&inodedep->id_bufwait, &diradd->da_list); 3046 } else if (jaddref->ja_state & MKDIR_PARENT) { 3047 mkdir = jaddref->ja_mkdir; 3048 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir->md_list); 3049 } else if (jaddref->ja_state & MKDIR_BODY) 3050 mkdir = jaddref->ja_mkdir; 3051 else 3052 panic("handle_written_jaddref: Unknown dependency %p", 3053 jaddref->ja_diradd); 3054 jaddref->ja_diradd = NULL; /* also clears ja_mkdir */ 3055 /* 3056 * Remove us from the inode list. 3057 */ 3058 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, if_deps); 3059 /* 3060 * The mkdir may be waiting on the jaddref to clear before freeing. 3061 */ 3062 if (mkdir) { 3063 KASSERT(mkdir->md_list.wk_type == D_MKDIR, 3064 ("handle_written_jaddref: Incorrect type for mkdir %s", 3065 TYPENAME(mkdir->md_list.wk_type))); 3066 mkdir->md_jaddref = NULL; 3067 diradd = mkdir->md_diradd; 3068 mkdir->md_state |= DEPCOMPLETE; 3069 complete_mkdir(mkdir); 3070 } 3071 WORKLIST_INSERT(&diradd->da_jwork, &jsegdep->jd_list); 3072 if (jaddref->ja_state & NEWBLOCK) { 3073 inodedep->id_state |= ONDEPLIST; 3074 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_inodedephd, 3075 inodedep, id_deps); 3076 } 3077 free_jaddref(jaddref); 3078 } 3079 3080 /* 3081 * Called once a jnewblk journal is written. The allocdirect or allocindir 3082 * is placed in the bmsafemap to await notification of a written bitmap. 3083 */ 3084 static void 3085 handle_written_jnewblk(jnewblk) 3086 struct jnewblk *jnewblk; 3087 { 3088 struct bmsafemap *bmsafemap; 3089 struct jsegdep *jsegdep; 3090 struct newblk *newblk; 3091 3092 /* Grab the jsegdep. */ 3093 jsegdep = jnewblk->jn_jsegdep; 3094 jnewblk->jn_jsegdep = NULL; 3095 /* 3096 * Add the written block to the bmsafemap so it can be notified when 3097 * the bitmap is on disk. 3098 */ 3099 newblk = jnewblk->jn_newblk; 3100 jnewblk->jn_newblk = NULL; 3101 if (newblk == NULL) 3102 panic("handle_written_jnewblk: No dependency for the segdep."); 3103 3104 newblk->nb_jnewblk = NULL; 3105 bmsafemap = newblk->nb_bmsafemap; 3106 WORKLIST_INSERT(&newblk->nb_jwork, &jsegdep->jd_list); 3107 newblk->nb_state |= ONDEPLIST; 3108 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps); 3109 free_jnewblk(jnewblk); 3110 } 3111 3112 /* 3113 * Cancel a jfreefrag that won't be needed, probably due to colliding with 3114 * an in-flight allocation that has not yet been committed. Divorce us 3115 * from the freefrag and mark it DEPCOMPLETE so that it may be added 3116 * to the worklist. 3117 */ 3118 static void 3119 cancel_jfreefrag(jfreefrag) 3120 struct jfreefrag *jfreefrag; 3121 { 3122 struct freefrag *freefrag; 3123 3124 if (jfreefrag->fr_jsegdep) { 3125 free_jsegdep(jfreefrag->fr_jsegdep); 3126 jfreefrag->fr_jsegdep = NULL; 3127 } 3128 freefrag = jfreefrag->fr_freefrag; 3129 jfreefrag->fr_freefrag = NULL; 3130 freefrag->ff_jfreefrag = NULL; 3131 free_jfreefrag(jfreefrag); 3132 freefrag->ff_state |= DEPCOMPLETE; 3133 } 3134 3135 /* 3136 * Free a jfreefrag when the parent freefrag is rendered obsolete. 3137 */ 3138 static void 3139 free_jfreefrag(jfreefrag) 3140 struct jfreefrag *jfreefrag; 3141 { 3142 3143 if (jfreefrag->fr_state & IOSTARTED) 3144 WORKLIST_REMOVE(&jfreefrag->fr_list); 3145 else if (jfreefrag->fr_state & ONWORKLIST) 3146 remove_from_journal(&jfreefrag->fr_list); 3147 if (jfreefrag->fr_freefrag != NULL) 3148 panic("free_jfreefrag: Still attached to a freefrag."); 3149 WORKITEM_FREE(jfreefrag, D_JFREEFRAG); 3150 } 3151 3152 /* 3153 * Called when the journal write for a jfreefrag completes. The parent 3154 * freefrag is added to the worklist if this completes its dependencies. 3155 */ 3156 static void 3157 handle_written_jfreefrag(jfreefrag) 3158 struct jfreefrag *jfreefrag; 3159 { 3160 struct jsegdep *jsegdep; 3161 struct freefrag *freefrag; 3162 3163 /* Grab the jsegdep. */ 3164 jsegdep = jfreefrag->fr_jsegdep; 3165 jfreefrag->fr_jsegdep = NULL; 3166 freefrag = jfreefrag->fr_freefrag; 3167 if (freefrag == NULL) 3168 panic("handle_written_jfreefrag: No freefrag."); 3169 freefrag->ff_state |= DEPCOMPLETE; 3170 freefrag->ff_jfreefrag = NULL; 3171 WORKLIST_INSERT(&freefrag->ff_jwork, &jsegdep->jd_list); 3172 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 3173 add_to_worklist(&freefrag->ff_list, 0); 3174 jfreefrag->fr_freefrag = NULL; 3175 free_jfreefrag(jfreefrag); 3176 } 3177 3178 /* 3179 * Called when the journal write for a jfreeblk completes. The jfreeblk 3180 * is removed from the freeblks list of pending journal writes and the 3181 * jsegdep is moved to the freeblks jwork to be completed when all blocks 3182 * have been reclaimed. 3183 */ 3184 static void 3185 handle_written_jfreeblk(jfreeblk) 3186 struct jfreeblk *jfreeblk; 3187 { 3188 struct freeblks *freeblks; 3189 struct jsegdep *jsegdep; 3190 3191 /* Grab the jsegdep. */ 3192 jsegdep = jfreeblk->jf_jsegdep; 3193 jfreeblk->jf_jsegdep = NULL; 3194 freeblks = jfreeblk->jf_freeblks; 3195 LIST_REMOVE(jfreeblk, jf_deps); 3196 WORKLIST_INSERT(&freeblks->fb_jwork, &jsegdep->jd_list); 3197 /* 3198 * If the freeblks is all journaled, we can add it to the worklist. 3199 */ 3200 if (LIST_EMPTY(&freeblks->fb_jfreeblkhd) && 3201 (freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) { 3202 /* Remove from the b_dep that is waiting on this write. */ 3203 if (freeblks->fb_state & ONWORKLIST) 3204 WORKLIST_REMOVE(&freeblks->fb_list); 3205 add_to_worklist(&freeblks->fb_list, 1); 3206 } 3207 3208 free_jfreeblk(jfreeblk); 3209 } 3210 3211 static struct jsegdep * 3212 newjsegdep(struct worklist *wk) 3213 { 3214 struct jsegdep *jsegdep; 3215 3216 jsegdep = malloc(sizeof(*jsegdep), M_JSEGDEP, M_SOFTDEP_FLAGS); 3217 workitem_alloc(&jsegdep->jd_list, D_JSEGDEP, wk->wk_mp); 3218 jsegdep->jd_seg = NULL; 3219 3220 return (jsegdep); 3221 } 3222 3223 static struct jmvref * 3224 newjmvref(dp, ino, oldoff, newoff) 3225 struct inode *dp; 3226 ino_t ino; 3227 off_t oldoff; 3228 off_t newoff; 3229 { 3230 struct jmvref *jmvref; 3231 3232 jmvref = malloc(sizeof(*jmvref), M_JMVREF, M_SOFTDEP_FLAGS); 3233 workitem_alloc(&jmvref->jm_list, D_JMVREF, UFSTOVFS(dp->i_ump)); 3234 jmvref->jm_list.wk_state = ATTACHED | DEPCOMPLETE; 3235 jmvref->jm_parent = dp->i_number; 3236 jmvref->jm_ino = ino; 3237 jmvref->jm_oldoff = oldoff; 3238 jmvref->jm_newoff = newoff; 3239 3240 return (jmvref); 3241 } 3242 3243 /* 3244 * Allocate a new jremref that tracks the removal of ip from dp with the 3245 * directory entry offset of diroff. Mark the entry as ATTACHED and 3246 * DEPCOMPLETE as we have all the information required for the journal write 3247 * and the directory has already been removed from the buffer. The caller 3248 * is responsible for linking the jremref into the pagedep and adding it 3249 * to the journal to write. The MKDIR_PARENT flag is set if we're doing 3250 * a DOTDOT addition so handle_workitem_remove() can properly assign 3251 * the jsegdep when we're done. 3252 */ 3253 static struct jremref * 3254 newjremref(struct dirrem *dirrem, struct inode *dp, struct inode *ip, 3255 off_t diroff, nlink_t nlink) 3256 { 3257 struct jremref *jremref; 3258 3259 jremref = malloc(sizeof(*jremref), M_JREMREF, M_SOFTDEP_FLAGS); 3260 workitem_alloc(&jremref->jr_list, D_JREMREF, UFSTOVFS(dp->i_ump)); 3261 jremref->jr_state = ATTACHED; 3262 newinoref(&jremref->jr_ref, ip->i_number, dp->i_number, diroff, 3263 nlink, ip->i_mode); 3264 jremref->jr_dirrem = dirrem; 3265 3266 return (jremref); 3267 } 3268 3269 static inline void 3270 newinoref(struct inoref *inoref, ino_t ino, ino_t parent, off_t diroff, 3271 nlink_t nlink, uint16_t mode) 3272 { 3273 3274 inoref->if_jsegdep = newjsegdep(&inoref->if_list); 3275 inoref->if_diroff = diroff; 3276 inoref->if_ino = ino; 3277 inoref->if_parent = parent; 3278 inoref->if_nlink = nlink; 3279 inoref->if_mode = mode; 3280 } 3281 3282 /* 3283 * Allocate a new jaddref to track the addition of ino to dp at diroff. The 3284 * directory offset may not be known until later. The caller is responsible 3285 * adding the entry to the journal when this information is available. nlink 3286 * should be the link count prior to the addition and mode is only required 3287 * to have the correct FMT. 3288 */ 3289 static struct jaddref * 3290 newjaddref(struct inode *dp, ino_t ino, off_t diroff, int16_t nlink, 3291 uint16_t mode) 3292 { 3293 struct jaddref *jaddref; 3294 3295 jaddref = malloc(sizeof(*jaddref), M_JADDREF, M_SOFTDEP_FLAGS); 3296 workitem_alloc(&jaddref->ja_list, D_JADDREF, UFSTOVFS(dp->i_ump)); 3297 jaddref->ja_state = ATTACHED; 3298 jaddref->ja_mkdir = NULL; 3299 newinoref(&jaddref->ja_ref, ino, dp->i_number, diroff, nlink, mode); 3300 3301 return (jaddref); 3302 } 3303 3304 /* 3305 * Create a new free dependency for a freework. The caller is responsible 3306 * for adjusting the reference count when it has the lock held. The freedep 3307 * will track an outstanding bitmap write that will ultimately clear the 3308 * freework to continue. 3309 */ 3310 static struct freedep * 3311 newfreedep(struct freework *freework) 3312 { 3313 struct freedep *freedep; 3314 3315 freedep = malloc(sizeof(*freedep), M_FREEDEP, M_SOFTDEP_FLAGS); 3316 workitem_alloc(&freedep->fd_list, D_FREEDEP, freework->fw_list.wk_mp); 3317 freedep->fd_freework = freework; 3318 3319 return (freedep); 3320 } 3321 3322 /* 3323 * Free a freedep structure once the buffer it is linked to is written. If 3324 * this is the last reference to the freework schedule it for completion. 3325 */ 3326 static void 3327 free_freedep(freedep) 3328 struct freedep *freedep; 3329 { 3330 3331 if (--freedep->fd_freework->fw_ref == 0) 3332 add_to_worklist(&freedep->fd_freework->fw_list, 1); 3333 WORKITEM_FREE(freedep, D_FREEDEP); 3334 } 3335 3336 /* 3337 * Allocate a new freework structure that may be a level in an indirect 3338 * when parent is not NULL or a top level block when it is. The top level 3339 * freework structures are allocated without lk held and before the freeblks 3340 * is visible outside of softdep_setup_freeblocks(). 3341 */ 3342 static struct freework * 3343 newfreework(freeblks, parent, lbn, nb, frags, journal) 3344 struct freeblks *freeblks; 3345 struct freework *parent; 3346 ufs_lbn_t lbn; 3347 ufs2_daddr_t nb; 3348 int frags; 3349 int journal; 3350 { 3351 struct freework *freework; 3352 3353 freework = malloc(sizeof(*freework), M_FREEWORK, M_SOFTDEP_FLAGS); 3354 workitem_alloc(&freework->fw_list, D_FREEWORK, freeblks->fb_list.wk_mp); 3355 freework->fw_freeblks = freeblks; 3356 freework->fw_parent = parent; 3357 freework->fw_lbn = lbn; 3358 freework->fw_blkno = nb; 3359 freework->fw_frags = frags; 3360 freework->fw_ref = 0; 3361 freework->fw_off = 0; 3362 LIST_INIT(&freework->fw_jwork); 3363 3364 if (parent == NULL) { 3365 WORKLIST_INSERT_UNLOCKED(&freeblks->fb_freeworkhd, 3366 &freework->fw_list); 3367 freeblks->fb_ref++; 3368 } 3369 if (journal) 3370 newjfreeblk(freeblks, lbn, nb, frags); 3371 3372 return (freework); 3373 } 3374 3375 /* 3376 * Allocate a new jfreeblk to journal top level block pointer when truncating 3377 * a file. The caller must add this to the worklist when lk is held. 3378 */ 3379 static struct jfreeblk * 3380 newjfreeblk(freeblks, lbn, blkno, frags) 3381 struct freeblks *freeblks; 3382 ufs_lbn_t lbn; 3383 ufs2_daddr_t blkno; 3384 int frags; 3385 { 3386 struct jfreeblk *jfreeblk; 3387 3388 jfreeblk = malloc(sizeof(*jfreeblk), M_JFREEBLK, M_SOFTDEP_FLAGS); 3389 workitem_alloc(&jfreeblk->jf_list, D_JFREEBLK, freeblks->fb_list.wk_mp); 3390 jfreeblk->jf_jsegdep = newjsegdep(&jfreeblk->jf_list); 3391 jfreeblk->jf_state = ATTACHED | DEPCOMPLETE; 3392 jfreeblk->jf_ino = freeblks->fb_previousinum; 3393 jfreeblk->jf_lbn = lbn; 3394 jfreeblk->jf_blkno = blkno; 3395 jfreeblk->jf_frags = frags; 3396 jfreeblk->jf_freeblks = freeblks; 3397 LIST_INSERT_HEAD(&freeblks->fb_jfreeblkhd, jfreeblk, jf_deps); 3398 3399 return (jfreeblk); 3400 } 3401 3402 static void move_newblock_dep(struct jaddref *, struct inodedep *); 3403 /* 3404 * If we're canceling a new bitmap we have to search for another ref 3405 * to move into the bmsafemap dep. This might be better expressed 3406 * with another structure. 3407 */ 3408 static void 3409 move_newblock_dep(jaddref, inodedep) 3410 struct jaddref *jaddref; 3411 struct inodedep *inodedep; 3412 { 3413 struct inoref *inoref; 3414 struct jaddref *jaddrefn; 3415 3416 jaddrefn = NULL; 3417 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 3418 inoref = TAILQ_NEXT(inoref, if_deps)) { 3419 if ((jaddref->ja_state & NEWBLOCK) && 3420 inoref->if_list.wk_type == D_JADDREF) { 3421 jaddrefn = (struct jaddref *)inoref; 3422 break; 3423 } 3424 } 3425 if (jaddrefn == NULL) 3426 return; 3427 jaddrefn->ja_state &= ~(ATTACHED | UNDONE); 3428 jaddrefn->ja_state |= jaddref->ja_state & 3429 (ATTACHED | UNDONE | NEWBLOCK); 3430 jaddref->ja_state &= ~(ATTACHED | UNDONE | NEWBLOCK); 3431 jaddref->ja_state |= ATTACHED; 3432 LIST_REMOVE(jaddref, ja_bmdeps); 3433 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_jaddrefhd, jaddrefn, 3434 ja_bmdeps); 3435 } 3436 3437 /* 3438 * Cancel a jaddref either before it has been written or while it is being 3439 * written. This happens when a link is removed before the add reaches 3440 * the disk. The jaddref dependency is kept linked into the bmsafemap 3441 * and inode to prevent the link count or bitmap from reaching the disk 3442 * until handle_workitem_remove() re-adjusts the counts and bitmaps as 3443 * required. 3444 * 3445 * Returns 1 if the canceled addref requires journaling of the remove and 3446 * 0 otherwise. 3447 */ 3448 static int 3449 cancel_jaddref(jaddref, inodedep, wkhd) 3450 struct jaddref *jaddref; 3451 struct inodedep *inodedep; 3452 struct workhead *wkhd; 3453 { 3454 struct inoref *inoref; 3455 struct jsegdep *jsegdep; 3456 int needsj; 3457 3458 KASSERT((jaddref->ja_state & COMPLETE) == 0, 3459 ("cancel_jaddref: Canceling complete jaddref")); 3460 if (jaddref->ja_state & (IOSTARTED | COMPLETE)) 3461 needsj = 1; 3462 else 3463 needsj = 0; 3464 if (inodedep == NULL) 3465 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 3466 0, &inodedep) == 0) 3467 panic("cancel_jaddref: Lost inodedep"); 3468 /* 3469 * We must adjust the nlink of any reference operation that follows 3470 * us so that it is consistent with the in-memory reference. This 3471 * ensures that inode nlink rollbacks always have the correct link. 3472 */ 3473 if (needsj == 0) 3474 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 3475 inoref = TAILQ_NEXT(inoref, if_deps)) 3476 inoref->if_nlink--; 3477 jsegdep = inoref_jseg(&jaddref->ja_ref); 3478 if (jaddref->ja_state & NEWBLOCK) 3479 move_newblock_dep(jaddref, inodedep); 3480 if (jaddref->ja_state & IOWAITING) { 3481 jaddref->ja_state &= ~IOWAITING; 3482 wakeup(&jaddref->ja_list); 3483 } 3484 jaddref->ja_mkdir = NULL; 3485 if (jaddref->ja_state & IOSTARTED) { 3486 jaddref->ja_state &= ~IOSTARTED; 3487 WORKLIST_REMOVE(&jaddref->ja_list); 3488 WORKLIST_INSERT(wkhd, &jsegdep->jd_list); 3489 } else { 3490 free_jsegdep(jsegdep); 3491 if (jaddref->ja_state & DEPCOMPLETE) 3492 remove_from_journal(&jaddref->ja_list); 3493 } 3494 /* 3495 * Leave NEWBLOCK jaddrefs on the inodedep so handle_workitem_remove 3496 * can arrange for them to be freed with the bitmap. Otherwise we 3497 * no longer need this addref attached to the inoreflst and it 3498 * will incorrectly adjust nlink if we leave it. 3499 */ 3500 if ((jaddref->ja_state & NEWBLOCK) == 0) { 3501 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 3502 if_deps); 3503 jaddref->ja_state |= COMPLETE; 3504 free_jaddref(jaddref); 3505 return (needsj); 3506 } 3507 jaddref->ja_state |= GOINGAWAY; 3508 /* 3509 * Leave the head of the list for jsegdeps for fast merging. 3510 */ 3511 if (LIST_FIRST(wkhd) != NULL) { 3512 jaddref->ja_state |= ONWORKLIST; 3513 LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jaddref->ja_list, wk_list); 3514 } else 3515 WORKLIST_INSERT(wkhd, &jaddref->ja_list); 3516 3517 return (needsj); 3518 } 3519 3520 /* 3521 * Attempt to free a jaddref structure when some work completes. This 3522 * should only succeed once the entry is written and all dependencies have 3523 * been notified. 3524 */ 3525 static void 3526 free_jaddref(jaddref) 3527 struct jaddref *jaddref; 3528 { 3529 3530 if ((jaddref->ja_state & ALLCOMPLETE) != ALLCOMPLETE) 3531 return; 3532 if (jaddref->ja_ref.if_jsegdep) 3533 panic("free_jaddref: segdep attached to jaddref %p(0x%X)\n", 3534 jaddref, jaddref->ja_state); 3535 if (jaddref->ja_state & NEWBLOCK) 3536 LIST_REMOVE(jaddref, ja_bmdeps); 3537 if (jaddref->ja_state & (IOSTARTED | ONWORKLIST)) 3538 panic("free_jaddref: Bad state %p(0x%X)", 3539 jaddref, jaddref->ja_state); 3540 if (jaddref->ja_mkdir != NULL) 3541 panic("free_jaddref: Work pending, 0x%X\n", jaddref->ja_state); 3542 WORKITEM_FREE(jaddref, D_JADDREF); 3543 } 3544 3545 /* 3546 * Free a jremref structure once it has been written or discarded. 3547 */ 3548 static void 3549 free_jremref(jremref) 3550 struct jremref *jremref; 3551 { 3552 3553 if (jremref->jr_ref.if_jsegdep) 3554 free_jsegdep(jremref->jr_ref.if_jsegdep); 3555 if (jremref->jr_state & IOSTARTED) 3556 panic("free_jremref: IO still pending"); 3557 WORKITEM_FREE(jremref, D_JREMREF); 3558 } 3559 3560 /* 3561 * Free a jnewblk structure. 3562 */ 3563 static void 3564 free_jnewblk(jnewblk) 3565 struct jnewblk *jnewblk; 3566 { 3567 3568 if ((jnewblk->jn_state & ALLCOMPLETE) != ALLCOMPLETE) 3569 return; 3570 LIST_REMOVE(jnewblk, jn_deps); 3571 if (jnewblk->jn_newblk != NULL) 3572 panic("free_jnewblk: Dependency still attached."); 3573 WORKITEM_FREE(jnewblk, D_JNEWBLK); 3574 } 3575 3576 /* 3577 * Cancel a jnewblk which has been superseded by a freeblk. The jnewblk 3578 * is kept linked into the bmsafemap until the free completes, thus 3579 * preventing the modified state from ever reaching disk. The free 3580 * routine must pass this structure via ffs_blkfree() to 3581 * softdep_setup_freeblks() so there is no race in releasing the space. 3582 */ 3583 static void 3584 cancel_jnewblk(jnewblk, wkhd) 3585 struct jnewblk *jnewblk; 3586 struct workhead *wkhd; 3587 { 3588 struct jsegdep *jsegdep; 3589 3590 jsegdep = jnewblk->jn_jsegdep; 3591 jnewblk->jn_jsegdep = NULL; 3592 free_jsegdep(jsegdep); 3593 jnewblk->jn_newblk = NULL; 3594 jnewblk->jn_state |= GOINGAWAY; 3595 if (jnewblk->jn_state & IOSTARTED) { 3596 jnewblk->jn_state &= ~IOSTARTED; 3597 WORKLIST_REMOVE(&jnewblk->jn_list); 3598 } else 3599 remove_from_journal(&jnewblk->jn_list); 3600 /* 3601 * Leave the head of the list for jsegdeps for fast merging. 3602 */ 3603 if (LIST_FIRST(wkhd) != NULL) { 3604 jnewblk->jn_state |= ONWORKLIST; 3605 LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jnewblk->jn_list, wk_list); 3606 } else 3607 WORKLIST_INSERT(wkhd, &jnewblk->jn_list); 3608 if (jnewblk->jn_state & IOWAITING) { 3609 jnewblk->jn_state &= ~IOWAITING; 3610 wakeup(&jnewblk->jn_list); 3611 } 3612 } 3613 3614 static void 3615 free_jfreeblk(jfreeblk) 3616 struct jfreeblk *jfreeblk; 3617 { 3618 3619 WORKITEM_FREE(jfreeblk, D_JFREEBLK); 3620 } 3621 3622 /* 3623 * Release one reference to a jseg and free it if the count reaches 0. This 3624 * should eventually reclaim journal space as well. 3625 */ 3626 static void 3627 free_jseg(jseg) 3628 struct jseg *jseg; 3629 { 3630 struct jblocks *jblocks; 3631 3632 KASSERT(jseg->js_refs > 0, 3633 ("free_jseg: Invalid refcnt %d", jseg->js_refs)); 3634 if (--jseg->js_refs != 0) 3635 return; 3636 /* 3637 * Free only those jsegs which have none allocated before them to 3638 * preserve the journal space ordering. 3639 */ 3640 jblocks = jseg->js_jblocks; 3641 while ((jseg = TAILQ_FIRST(&jblocks->jb_segs)) != NULL) { 3642 jblocks->jb_oldestseq = jseg->js_seq; 3643 if (jseg->js_refs != 0) 3644 break; 3645 TAILQ_REMOVE(&jblocks->jb_segs, jseg, js_next); 3646 jblocks_free(jblocks, jseg->js_list.wk_mp, jseg->js_size); 3647 KASSERT(LIST_EMPTY(&jseg->js_entries), 3648 ("free_jseg: Freed jseg has valid entries.")); 3649 WORKITEM_FREE(jseg, D_JSEG); 3650 } 3651 } 3652 3653 /* 3654 * Release a jsegdep and decrement the jseg count. 3655 */ 3656 static void 3657 free_jsegdep(jsegdep) 3658 struct jsegdep *jsegdep; 3659 { 3660 3661 if (jsegdep->jd_seg) 3662 free_jseg(jsegdep->jd_seg); 3663 WORKITEM_FREE(jsegdep, D_JSEGDEP); 3664 } 3665 3666 /* 3667 * Wait for a journal item to make it to disk. Initiate journal processing 3668 * if required. 3669 */ 3670 static void 3671 jwait(wk) 3672 struct worklist *wk; 3673 { 3674 3675 stat_journal_wait++; 3676 /* 3677 * If IO has not started we process the journal. We can't mark the 3678 * worklist item as IOWAITING because we drop the lock while 3679 * processing the journal and the worklist entry may be freed after 3680 * this point. The caller may call back in and re-issue the request. 3681 */ 3682 if ((wk->wk_state & IOSTARTED) == 0) { 3683 softdep_process_journal(wk->wk_mp, MNT_WAIT); 3684 return; 3685 } 3686 wk->wk_state |= IOWAITING; 3687 msleep(wk, &lk, PRIBIO, "jwait", 0); 3688 } 3689 3690 /* 3691 * Lookup an inodedep based on an inode pointer and set the nlinkdelta as 3692 * appropriate. This is a convenience function to reduce duplicate code 3693 * for the setup and revert functions below. 3694 */ 3695 static struct inodedep * 3696 inodedep_lookup_ip(ip) 3697 struct inode *ip; 3698 { 3699 struct inodedep *inodedep; 3700 3701 KASSERT(ip->i_nlink >= ip->i_effnlink, 3702 ("inodedep_lookup_ip: bad delta")); 3703 (void) inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 3704 DEPALLOC, &inodedep); 3705 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 3706 3707 return (inodedep); 3708 } 3709 3710 /* 3711 * Create a journal entry that describes a truncate that we're about to 3712 * perform. The inode allocations and frees between here and the completion 3713 * of the operation are done asynchronously and without journaling. At 3714 * the end of the operation the vnode is sync'd and the journal space 3715 * is released. Recovery will discover the partially completed truncate 3716 * and complete it. 3717 */ 3718 void * 3719 softdep_setup_trunc(vp, length, flags) 3720 struct vnode *vp; 3721 off_t length; 3722 int flags; 3723 { 3724 struct jsegdep *jsegdep; 3725 struct jtrunc *jtrunc; 3726 struct ufsmount *ump; 3727 struct inode *ip; 3728 3729 softdep_prealloc(vp, MNT_WAIT); 3730 ip = VTOI(vp); 3731 ump = VFSTOUFS(vp->v_mount); 3732 jtrunc = malloc(sizeof(*jtrunc), M_JTRUNC, M_SOFTDEP_FLAGS); 3733 workitem_alloc(&jtrunc->jt_list, D_JTRUNC, vp->v_mount); 3734 jsegdep = jtrunc->jt_jsegdep = newjsegdep(&jtrunc->jt_list); 3735 jtrunc->jt_ino = ip->i_number; 3736 jtrunc->jt_extsize = 0; 3737 jtrunc->jt_size = length; 3738 if ((flags & IO_EXT) == 0 && ump->um_fstype == UFS2) 3739 jtrunc->jt_extsize = ip->i_din2->di_extsize; 3740 if ((flags & IO_NORMAL) == 0) 3741 jtrunc->jt_size = DIP(ip, i_size); 3742 ACQUIRE_LOCK(&lk); 3743 add_to_journal(&jtrunc->jt_list); 3744 while (jsegdep->jd_seg == NULL) { 3745 stat_jwait_freeblks++; 3746 jwait(&jtrunc->jt_list); 3747 } 3748 FREE_LOCK(&lk); 3749 3750 return (jsegdep); 3751 } 3752 3753 /* 3754 * After synchronous truncation is complete we free sync the vnode and 3755 * release the jsegdep so the journal space can be freed. 3756 */ 3757 int 3758 softdep_complete_trunc(vp, cookie) 3759 struct vnode *vp; 3760 void *cookie; 3761 { 3762 int error; 3763 3764 error = ffs_syncvnode(vp, MNT_WAIT); 3765 ACQUIRE_LOCK(&lk); 3766 free_jsegdep((struct jsegdep *)cookie); 3767 FREE_LOCK(&lk); 3768 3769 return (error); 3770 } 3771 3772 /* 3773 * Called prior to creating a new inode and linking it to a directory. The 3774 * jaddref structure must already be allocated by softdep_setup_inomapdep 3775 * and it is discovered here so we can initialize the mode and update 3776 * nlinkdelta. 3777 */ 3778 void 3779 softdep_setup_create(dp, ip) 3780 struct inode *dp; 3781 struct inode *ip; 3782 { 3783 struct inodedep *inodedep; 3784 struct jaddref *jaddref; 3785 struct vnode *dvp; 3786 3787 KASSERT(ip->i_nlink == 1, 3788 ("softdep_setup_create: Invalid link count.")); 3789 dvp = ITOV(dp); 3790 ACQUIRE_LOCK(&lk); 3791 inodedep = inodedep_lookup_ip(ip); 3792 if (DOINGSUJ(dvp)) { 3793 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 3794 inoreflst); 3795 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 3796 ("softdep_setup_create: No addref structure present.")); 3797 jaddref->ja_mode = ip->i_mode; 3798 } 3799 softdep_prelink(dvp, NULL); 3800 FREE_LOCK(&lk); 3801 } 3802 3803 /* 3804 * Create a jaddref structure to track the addition of a DOTDOT link when 3805 * we are reparenting an inode as part of a rename. This jaddref will be 3806 * found by softdep_setup_directory_change. Adjusts nlinkdelta for 3807 * non-journaling softdep. 3808 */ 3809 void 3810 softdep_setup_dotdot_link(dp, ip) 3811 struct inode *dp; 3812 struct inode *ip; 3813 { 3814 struct inodedep *inodedep; 3815 struct jaddref *jaddref; 3816 struct vnode *dvp; 3817 struct vnode *vp; 3818 3819 dvp = ITOV(dp); 3820 vp = ITOV(ip); 3821 jaddref = NULL; 3822 /* 3823 * We don't set MKDIR_PARENT as this is not tied to a mkdir and 3824 * is used as a normal link would be. 3825 */ 3826 if (DOINGSUJ(dvp)) 3827 jaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 3828 dp->i_effnlink - 1, dp->i_mode); 3829 ACQUIRE_LOCK(&lk); 3830 inodedep = inodedep_lookup_ip(dp); 3831 if (jaddref) 3832 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 3833 if_deps); 3834 softdep_prelink(dvp, ITOV(ip)); 3835 FREE_LOCK(&lk); 3836 } 3837 3838 /* 3839 * Create a jaddref structure to track a new link to an inode. The directory 3840 * offset is not known until softdep_setup_directory_add or 3841 * softdep_setup_directory_change. Adjusts nlinkdelta for non-journaling 3842 * softdep. 3843 */ 3844 void 3845 softdep_setup_link(dp, ip) 3846 struct inode *dp; 3847 struct inode *ip; 3848 { 3849 struct inodedep *inodedep; 3850 struct jaddref *jaddref; 3851 struct vnode *dvp; 3852 3853 dvp = ITOV(dp); 3854 jaddref = NULL; 3855 if (DOINGSUJ(dvp)) 3856 jaddref = newjaddref(dp, ip->i_number, 0, ip->i_effnlink - 1, 3857 ip->i_mode); 3858 ACQUIRE_LOCK(&lk); 3859 inodedep = inodedep_lookup_ip(ip); 3860 if (jaddref) 3861 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 3862 if_deps); 3863 softdep_prelink(dvp, ITOV(ip)); 3864 FREE_LOCK(&lk); 3865 } 3866 3867 /* 3868 * Called to create the jaddref structures to track . and .. references as 3869 * well as lookup and further initialize the incomplete jaddref created 3870 * by softdep_setup_inomapdep when the inode was allocated. Adjusts 3871 * nlinkdelta for non-journaling softdep. 3872 */ 3873 void 3874 softdep_setup_mkdir(dp, ip) 3875 struct inode *dp; 3876 struct inode *ip; 3877 { 3878 struct inodedep *inodedep; 3879 struct jaddref *dotdotaddref; 3880 struct jaddref *dotaddref; 3881 struct jaddref *jaddref; 3882 struct vnode *dvp; 3883 3884 dvp = ITOV(dp); 3885 dotaddref = dotdotaddref = NULL; 3886 if (DOINGSUJ(dvp)) { 3887 dotaddref = newjaddref(ip, ip->i_number, DOT_OFFSET, 1, 3888 ip->i_mode); 3889 dotaddref->ja_state |= MKDIR_BODY; 3890 dotdotaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 3891 dp->i_effnlink - 1, dp->i_mode); 3892 dotdotaddref->ja_state |= MKDIR_PARENT; 3893 } 3894 ACQUIRE_LOCK(&lk); 3895 inodedep = inodedep_lookup_ip(ip); 3896 if (DOINGSUJ(dvp)) { 3897 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 3898 inoreflst); 3899 KASSERT(jaddref != NULL, 3900 ("softdep_setup_mkdir: No addref structure present.")); 3901 KASSERT(jaddref->ja_parent == dp->i_number, 3902 ("softdep_setup_mkdir: bad parent %d", 3903 jaddref->ja_parent)); 3904 jaddref->ja_mode = ip->i_mode; 3905 TAILQ_INSERT_BEFORE(&jaddref->ja_ref, &dotaddref->ja_ref, 3906 if_deps); 3907 } 3908 inodedep = inodedep_lookup_ip(dp); 3909 if (DOINGSUJ(dvp)) 3910 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, 3911 &dotdotaddref->ja_ref, if_deps); 3912 softdep_prelink(ITOV(dp), NULL); 3913 FREE_LOCK(&lk); 3914 } 3915 3916 /* 3917 * Called to track nlinkdelta of the inode and parent directories prior to 3918 * unlinking a directory. 3919 */ 3920 void 3921 softdep_setup_rmdir(dp, ip) 3922 struct inode *dp; 3923 struct inode *ip; 3924 { 3925 struct vnode *dvp; 3926 3927 dvp = ITOV(dp); 3928 ACQUIRE_LOCK(&lk); 3929 (void) inodedep_lookup_ip(ip); 3930 (void) inodedep_lookup_ip(dp); 3931 softdep_prelink(dvp, ITOV(ip)); 3932 FREE_LOCK(&lk); 3933 } 3934 3935 /* 3936 * Called to track nlinkdelta of the inode and parent directories prior to 3937 * unlink. 3938 */ 3939 void 3940 softdep_setup_unlink(dp, ip) 3941 struct inode *dp; 3942 struct inode *ip; 3943 { 3944 struct vnode *dvp; 3945 3946 dvp = ITOV(dp); 3947 ACQUIRE_LOCK(&lk); 3948 (void) inodedep_lookup_ip(ip); 3949 (void) inodedep_lookup_ip(dp); 3950 softdep_prelink(dvp, ITOV(ip)); 3951 FREE_LOCK(&lk); 3952 } 3953 3954 /* 3955 * Called to release the journal structures created by a failed non-directory 3956 * creation. Adjusts nlinkdelta for non-journaling softdep. 3957 */ 3958 void 3959 softdep_revert_create(dp, ip) 3960 struct inode *dp; 3961 struct inode *ip; 3962 { 3963 struct inodedep *inodedep; 3964 struct jaddref *jaddref; 3965 struct vnode *dvp; 3966 3967 dvp = ITOV(dp); 3968 ACQUIRE_LOCK(&lk); 3969 inodedep = inodedep_lookup_ip(ip); 3970 if (DOINGSUJ(dvp)) { 3971 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 3972 inoreflst); 3973 KASSERT(jaddref->ja_parent == dp->i_number, 3974 ("softdep_revert_create: addref parent mismatch")); 3975 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 3976 } 3977 FREE_LOCK(&lk); 3978 } 3979 3980 /* 3981 * Called to release the journal structures created by a failed dotdot link 3982 * creation. Adjusts nlinkdelta for non-journaling softdep. 3983 */ 3984 void 3985 softdep_revert_dotdot_link(dp, ip) 3986 struct inode *dp; 3987 struct inode *ip; 3988 { 3989 struct inodedep *inodedep; 3990 struct jaddref *jaddref; 3991 struct vnode *dvp; 3992 3993 dvp = ITOV(dp); 3994 ACQUIRE_LOCK(&lk); 3995 inodedep = inodedep_lookup_ip(dp); 3996 if (DOINGSUJ(dvp)) { 3997 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 3998 inoreflst); 3999 KASSERT(jaddref->ja_parent == ip->i_number, 4000 ("softdep_revert_dotdot_link: addref parent mismatch")); 4001 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4002 } 4003 FREE_LOCK(&lk); 4004 } 4005 4006 /* 4007 * Called to release the journal structures created by a failed link 4008 * addition. Adjusts nlinkdelta for non-journaling softdep. 4009 */ 4010 void 4011 softdep_revert_link(dp, ip) 4012 struct inode *dp; 4013 struct inode *ip; 4014 { 4015 struct inodedep *inodedep; 4016 struct jaddref *jaddref; 4017 struct vnode *dvp; 4018 4019 dvp = ITOV(dp); 4020 ACQUIRE_LOCK(&lk); 4021 inodedep = inodedep_lookup_ip(ip); 4022 if (DOINGSUJ(dvp)) { 4023 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4024 inoreflst); 4025 KASSERT(jaddref->ja_parent == dp->i_number, 4026 ("softdep_revert_link: addref parent mismatch")); 4027 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4028 } 4029 FREE_LOCK(&lk); 4030 } 4031 4032 /* 4033 * Called to release the journal structures created by a failed mkdir 4034 * attempt. Adjusts nlinkdelta for non-journaling softdep. 4035 */ 4036 void 4037 softdep_revert_mkdir(dp, ip) 4038 struct inode *dp; 4039 struct inode *ip; 4040 { 4041 struct inodedep *inodedep; 4042 struct jaddref *jaddref; 4043 struct vnode *dvp; 4044 4045 dvp = ITOV(dp); 4046 4047 ACQUIRE_LOCK(&lk); 4048 inodedep = inodedep_lookup_ip(dp); 4049 if (DOINGSUJ(dvp)) { 4050 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4051 inoreflst); 4052 KASSERT(jaddref->ja_parent == ip->i_number, 4053 ("softdep_revert_mkdir: dotdot addref parent mismatch")); 4054 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4055 } 4056 inodedep = inodedep_lookup_ip(ip); 4057 if (DOINGSUJ(dvp)) { 4058 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4059 inoreflst); 4060 KASSERT(jaddref->ja_parent == dp->i_number, 4061 ("softdep_revert_mkdir: addref parent mismatch")); 4062 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4063 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4064 inoreflst); 4065 KASSERT(jaddref->ja_parent == ip->i_number, 4066 ("softdep_revert_mkdir: dot addref parent mismatch")); 4067 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4068 } 4069 FREE_LOCK(&lk); 4070 } 4071 4072 /* 4073 * Called to correct nlinkdelta after a failed rmdir. 4074 */ 4075 void 4076 softdep_revert_rmdir(dp, ip) 4077 struct inode *dp; 4078 struct inode *ip; 4079 { 4080 4081 ACQUIRE_LOCK(&lk); 4082 (void) inodedep_lookup_ip(ip); 4083 (void) inodedep_lookup_ip(dp); 4084 FREE_LOCK(&lk); 4085 } 4086 4087 /* 4088 * Protecting the freemaps (or bitmaps). 4089 * 4090 * To eliminate the need to execute fsck before mounting a filesystem 4091 * after a power failure, one must (conservatively) guarantee that the 4092 * on-disk copy of the bitmaps never indicate that a live inode or block is 4093 * free. So, when a block or inode is allocated, the bitmap should be 4094 * updated (on disk) before any new pointers. When a block or inode is 4095 * freed, the bitmap should not be updated until all pointers have been 4096 * reset. The latter dependency is handled by the delayed de-allocation 4097 * approach described below for block and inode de-allocation. The former 4098 * dependency is handled by calling the following procedure when a block or 4099 * inode is allocated. When an inode is allocated an "inodedep" is created 4100 * with its DEPCOMPLETE flag cleared until its bitmap is written to disk. 4101 * Each "inodedep" is also inserted into the hash indexing structure so 4102 * that any additional link additions can be made dependent on the inode 4103 * allocation. 4104 * 4105 * The ufs filesystem maintains a number of free block counts (e.g., per 4106 * cylinder group, per cylinder and per <cylinder, rotational position> pair) 4107 * in addition to the bitmaps. These counts are used to improve efficiency 4108 * during allocation and therefore must be consistent with the bitmaps. 4109 * There is no convenient way to guarantee post-crash consistency of these 4110 * counts with simple update ordering, for two main reasons: (1) The counts 4111 * and bitmaps for a single cylinder group block are not in the same disk 4112 * sector. If a disk write is interrupted (e.g., by power failure), one may 4113 * be written and the other not. (2) Some of the counts are located in the 4114 * superblock rather than the cylinder group block. So, we focus our soft 4115 * updates implementation on protecting the bitmaps. When mounting a 4116 * filesystem, we recompute the auxiliary counts from the bitmaps. 4117 */ 4118 4119 /* 4120 * Called just after updating the cylinder group block to allocate an inode. 4121 */ 4122 void 4123 softdep_setup_inomapdep(bp, ip, newinum) 4124 struct buf *bp; /* buffer for cylgroup block with inode map */ 4125 struct inode *ip; /* inode related to allocation */ 4126 ino_t newinum; /* new inode number being allocated */ 4127 { 4128 struct inodedep *inodedep; 4129 struct bmsafemap *bmsafemap; 4130 struct jaddref *jaddref; 4131 struct mount *mp; 4132 struct fs *fs; 4133 4134 mp = UFSTOVFS(ip->i_ump); 4135 fs = ip->i_ump->um_fs; 4136 jaddref = NULL; 4137 4138 /* 4139 * Allocate the journal reference add structure so that the bitmap 4140 * can be dependent on it. 4141 */ 4142 if (mp->mnt_kern_flag & MNTK_SUJ) { 4143 jaddref = newjaddref(ip, newinum, 0, 0, 0); 4144 jaddref->ja_state |= NEWBLOCK; 4145 } 4146 4147 /* 4148 * Create a dependency for the newly allocated inode. 4149 * Panic if it already exists as something is seriously wrong. 4150 * Otherwise add it to the dependency list for the buffer holding 4151 * the cylinder group map from which it was allocated. 4152 */ 4153 ACQUIRE_LOCK(&lk); 4154 if ((inodedep_lookup(mp, newinum, DEPALLOC|NODELAY, &inodedep))) 4155 panic("softdep_setup_inomapdep: dependency %p for new" 4156 "inode already exists", inodedep); 4157 bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum)); 4158 if (jaddref) { 4159 LIST_INSERT_HEAD(&bmsafemap->sm_jaddrefhd, jaddref, ja_bmdeps); 4160 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 4161 if_deps); 4162 } else { 4163 inodedep->id_state |= ONDEPLIST; 4164 LIST_INSERT_HEAD(&bmsafemap->sm_inodedephd, inodedep, id_deps); 4165 } 4166 inodedep->id_bmsafemap = bmsafemap; 4167 inodedep->id_state &= ~DEPCOMPLETE; 4168 FREE_LOCK(&lk); 4169 } 4170 4171 /* 4172 * Called just after updating the cylinder group block to 4173 * allocate block or fragment. 4174 */ 4175 void 4176 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 4177 struct buf *bp; /* buffer for cylgroup block with block map */ 4178 struct mount *mp; /* filesystem doing allocation */ 4179 ufs2_daddr_t newblkno; /* number of newly allocated block */ 4180 int frags; /* Number of fragments. */ 4181 int oldfrags; /* Previous number of fragments for extend. */ 4182 { 4183 struct newblk *newblk; 4184 struct bmsafemap *bmsafemap; 4185 struct jnewblk *jnewblk; 4186 struct fs *fs; 4187 4188 fs = VFSTOUFS(mp)->um_fs; 4189 jnewblk = NULL; 4190 /* 4191 * Create a dependency for the newly allocated block. 4192 * Add it to the dependency list for the buffer holding 4193 * the cylinder group map from which it was allocated. 4194 */ 4195 if (mp->mnt_kern_flag & MNTK_SUJ) { 4196 jnewblk = malloc(sizeof(*jnewblk), M_JNEWBLK, M_SOFTDEP_FLAGS); 4197 workitem_alloc(&jnewblk->jn_list, D_JNEWBLK, mp); 4198 jnewblk->jn_jsegdep = newjsegdep(&jnewblk->jn_list); 4199 jnewblk->jn_state = ATTACHED; 4200 jnewblk->jn_blkno = newblkno; 4201 jnewblk->jn_frags = frags; 4202 jnewblk->jn_oldfrags = oldfrags; 4203 #ifdef SUJ_DEBUG 4204 { 4205 struct cg *cgp; 4206 uint8_t *blksfree; 4207 long bno; 4208 int i; 4209 4210 cgp = (struct cg *)bp->b_data; 4211 blksfree = cg_blksfree(cgp); 4212 bno = dtogd(fs, jnewblk->jn_blkno); 4213 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 4214 i++) { 4215 if (isset(blksfree, bno + i)) 4216 panic("softdep_setup_blkmapdep: " 4217 "free fragment %d from %d-%d " 4218 "state 0x%X dep %p", i, 4219 jnewblk->jn_oldfrags, 4220 jnewblk->jn_frags, 4221 jnewblk->jn_state, 4222 jnewblk->jn_newblk); 4223 } 4224 } 4225 #endif 4226 } 4227 ACQUIRE_LOCK(&lk); 4228 if (newblk_lookup(mp, newblkno, DEPALLOC, &newblk) != 0) 4229 panic("softdep_setup_blkmapdep: found block"); 4230 newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(mp, bp, 4231 dtog(fs, newblkno)); 4232 if (jnewblk) { 4233 jnewblk->jn_newblk = newblk; 4234 LIST_INSERT_HEAD(&bmsafemap->sm_jnewblkhd, jnewblk, jn_deps); 4235 } else { 4236 newblk->nb_state |= ONDEPLIST; 4237 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps); 4238 } 4239 newblk->nb_bmsafemap = bmsafemap; 4240 newblk->nb_jnewblk = jnewblk; 4241 FREE_LOCK(&lk); 4242 } 4243 4244 #define BMSAFEMAP_HASH(fs, cg) \ 4245 (&bmsafemap_hashtbl[((((register_t)(fs)) >> 13) + (cg)) & bmsafemap_hash]) 4246 4247 static int 4248 bmsafemap_find(bmsafemaphd, mp, cg, bmsafemapp) 4249 struct bmsafemap_hashhead *bmsafemaphd; 4250 struct mount *mp; 4251 int cg; 4252 struct bmsafemap **bmsafemapp; 4253 { 4254 struct bmsafemap *bmsafemap; 4255 4256 LIST_FOREACH(bmsafemap, bmsafemaphd, sm_hash) 4257 if (bmsafemap->sm_list.wk_mp == mp && bmsafemap->sm_cg == cg) 4258 break; 4259 if (bmsafemap) { 4260 *bmsafemapp = bmsafemap; 4261 return (1); 4262 } 4263 *bmsafemapp = NULL; 4264 4265 return (0); 4266 } 4267 4268 /* 4269 * Find the bmsafemap associated with a cylinder group buffer. 4270 * If none exists, create one. The buffer must be locked when 4271 * this routine is called and this routine must be called with 4272 * splbio interrupts blocked. 4273 */ 4274 static struct bmsafemap * 4275 bmsafemap_lookup(mp, bp, cg) 4276 struct mount *mp; 4277 struct buf *bp; 4278 int cg; 4279 { 4280 struct bmsafemap_hashhead *bmsafemaphd; 4281 struct bmsafemap *bmsafemap, *collision; 4282 struct worklist *wk; 4283 struct fs *fs; 4284 4285 mtx_assert(&lk, MA_OWNED); 4286 if (bp) 4287 LIST_FOREACH(wk, &bp->b_dep, wk_list) 4288 if (wk->wk_type == D_BMSAFEMAP) 4289 return (WK_BMSAFEMAP(wk)); 4290 fs = VFSTOUFS(mp)->um_fs; 4291 bmsafemaphd = BMSAFEMAP_HASH(fs, cg); 4292 if (bmsafemap_find(bmsafemaphd, mp, cg, &bmsafemap) == 1) 4293 return (bmsafemap); 4294 FREE_LOCK(&lk); 4295 bmsafemap = malloc(sizeof(struct bmsafemap), 4296 M_BMSAFEMAP, M_SOFTDEP_FLAGS); 4297 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); 4298 bmsafemap->sm_buf = bp; 4299 LIST_INIT(&bmsafemap->sm_inodedephd); 4300 LIST_INIT(&bmsafemap->sm_inodedepwr); 4301 LIST_INIT(&bmsafemap->sm_newblkhd); 4302 LIST_INIT(&bmsafemap->sm_newblkwr); 4303 LIST_INIT(&bmsafemap->sm_jaddrefhd); 4304 LIST_INIT(&bmsafemap->sm_jnewblkhd); 4305 ACQUIRE_LOCK(&lk); 4306 if (bmsafemap_find(bmsafemaphd, mp, cg, &collision) == 1) { 4307 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 4308 return (collision); 4309 } 4310 bmsafemap->sm_cg = cg; 4311 LIST_INSERT_HEAD(bmsafemaphd, bmsafemap, sm_hash); 4312 WORKLIST_INSERT(&bp->b_dep, &bmsafemap->sm_list); 4313 return (bmsafemap); 4314 } 4315 4316 /* 4317 * Direct block allocation dependencies. 4318 * 4319 * When a new block is allocated, the corresponding disk locations must be 4320 * initialized (with zeros or new data) before the on-disk inode points to 4321 * them. Also, the freemap from which the block was allocated must be 4322 * updated (on disk) before the inode's pointer. These two dependencies are 4323 * independent of each other and are needed for all file blocks and indirect 4324 * blocks that are pointed to directly by the inode. Just before the 4325 * "in-core" version of the inode is updated with a newly allocated block 4326 * number, a procedure (below) is called to setup allocation dependency 4327 * structures. These structures are removed when the corresponding 4328 * dependencies are satisfied or when the block allocation becomes obsolete 4329 * (i.e., the file is deleted, the block is de-allocated, or the block is a 4330 * fragment that gets upgraded). All of these cases are handled in 4331 * procedures described later. 4332 * 4333 * When a file extension causes a fragment to be upgraded, either to a larger 4334 * fragment or to a full block, the on-disk location may change (if the 4335 * previous fragment could not simply be extended). In this case, the old 4336 * fragment must be de-allocated, but not until after the inode's pointer has 4337 * been updated. In most cases, this is handled by later procedures, which 4338 * will construct a "freefrag" structure to be added to the workitem queue 4339 * when the inode update is complete (or obsolete). The main exception to 4340 * this is when an allocation occurs while a pending allocation dependency 4341 * (for the same block pointer) remains. This case is handled in the main 4342 * allocation dependency setup procedure by immediately freeing the 4343 * unreferenced fragments. 4344 */ 4345 void 4346 softdep_setup_allocdirect(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 4347 struct inode *ip; /* inode to which block is being added */ 4348 ufs_lbn_t off; /* block pointer within inode */ 4349 ufs2_daddr_t newblkno; /* disk block number being added */ 4350 ufs2_daddr_t oldblkno; /* previous block number, 0 unless frag */ 4351 long newsize; /* size of new block */ 4352 long oldsize; /* size of new block */ 4353 struct buf *bp; /* bp for allocated block */ 4354 { 4355 struct allocdirect *adp, *oldadp; 4356 struct allocdirectlst *adphead; 4357 struct freefrag *freefrag; 4358 struct inodedep *inodedep; 4359 struct pagedep *pagedep; 4360 struct jnewblk *jnewblk; 4361 struct newblk *newblk; 4362 struct mount *mp; 4363 ufs_lbn_t lbn; 4364 4365 lbn = bp->b_lblkno; 4366 mp = UFSTOVFS(ip->i_ump); 4367 if (oldblkno && oldblkno != newblkno) 4368 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn); 4369 else 4370 freefrag = NULL; 4371 4372 ACQUIRE_LOCK(&lk); 4373 if (off >= NDADDR) { 4374 if (lbn > 0) 4375 panic("softdep_setup_allocdirect: bad lbn %jd, off %jd", 4376 lbn, off); 4377 /* allocating an indirect block */ 4378 if (oldblkno != 0) 4379 panic("softdep_setup_allocdirect: non-zero indir"); 4380 } else { 4381 if (off != lbn) 4382 panic("softdep_setup_allocdirect: lbn %jd != off %jd", 4383 lbn, off); 4384 /* 4385 * Allocating a direct block. 4386 * 4387 * If we are allocating a directory block, then we must 4388 * allocate an associated pagedep to track additions and 4389 * deletions. 4390 */ 4391 if ((ip->i_mode & IFMT) == IFDIR && 4392 pagedep_lookup(mp, ip->i_number, off, DEPALLOC, 4393 &pagedep) == 0) 4394 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 4395 } 4396 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 4397 panic("softdep_setup_allocdirect: lost block"); 4398 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 4399 ("softdep_setup_allocdirect: newblk already initialized")); 4400 /* 4401 * Convert the newblk to an allocdirect. 4402 */ 4403 newblk->nb_list.wk_type = D_ALLOCDIRECT; 4404 adp = (struct allocdirect *)newblk; 4405 newblk->nb_freefrag = freefrag; 4406 adp->ad_offset = off; 4407 adp->ad_oldblkno = oldblkno; 4408 adp->ad_newsize = newsize; 4409 adp->ad_oldsize = oldsize; 4410 4411 /* 4412 * Finish initializing the journal. 4413 */ 4414 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 4415 jnewblk->jn_ino = ip->i_number; 4416 jnewblk->jn_lbn = lbn; 4417 add_to_journal(&jnewblk->jn_list); 4418 } 4419 if (freefrag && freefrag->ff_jfreefrag != NULL) 4420 add_to_journal(&freefrag->ff_jfreefrag->fr_list); 4421 inodedep_lookup(mp, ip->i_number, DEPALLOC | NODELAY, &inodedep); 4422 adp->ad_inodedep = inodedep; 4423 4424 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 4425 /* 4426 * The list of allocdirects must be kept in sorted and ascending 4427 * order so that the rollback routines can quickly determine the 4428 * first uncommitted block (the size of the file stored on disk 4429 * ends at the end of the lowest committed fragment, or if there 4430 * are no fragments, at the end of the highest committed block). 4431 * Since files generally grow, the typical case is that the new 4432 * block is to be added at the end of the list. We speed this 4433 * special case by checking against the last allocdirect in the 4434 * list before laboriously traversing the list looking for the 4435 * insertion point. 4436 */ 4437 adphead = &inodedep->id_newinoupdt; 4438 oldadp = TAILQ_LAST(adphead, allocdirectlst); 4439 if (oldadp == NULL || oldadp->ad_offset <= off) { 4440 /* insert at end of list */ 4441 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 4442 if (oldadp != NULL && oldadp->ad_offset == off) 4443 allocdirect_merge(adphead, adp, oldadp); 4444 FREE_LOCK(&lk); 4445 return; 4446 } 4447 TAILQ_FOREACH(oldadp, adphead, ad_next) { 4448 if (oldadp->ad_offset >= off) 4449 break; 4450 } 4451 if (oldadp == NULL) 4452 panic("softdep_setup_allocdirect: lost entry"); 4453 /* insert in middle of list */ 4454 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 4455 if (oldadp->ad_offset == off) 4456 allocdirect_merge(adphead, adp, oldadp); 4457 4458 FREE_LOCK(&lk); 4459 } 4460 4461 /* 4462 * Replace an old allocdirect dependency with a newer one. 4463 * This routine must be called with splbio interrupts blocked. 4464 */ 4465 static void 4466 allocdirect_merge(adphead, newadp, oldadp) 4467 struct allocdirectlst *adphead; /* head of list holding allocdirects */ 4468 struct allocdirect *newadp; /* allocdirect being added */ 4469 struct allocdirect *oldadp; /* existing allocdirect being checked */ 4470 { 4471 struct worklist *wk; 4472 struct freefrag *freefrag; 4473 struct newdirblk *newdirblk; 4474 4475 freefrag = NULL; 4476 mtx_assert(&lk, MA_OWNED); 4477 if (newadp->ad_oldblkno != oldadp->ad_newblkno || 4478 newadp->ad_oldsize != oldadp->ad_newsize || 4479 newadp->ad_offset >= NDADDR) 4480 panic("%s %jd != new %jd || old size %ld != new %ld", 4481 "allocdirect_merge: old blkno", 4482 (intmax_t)newadp->ad_oldblkno, 4483 (intmax_t)oldadp->ad_newblkno, 4484 newadp->ad_oldsize, oldadp->ad_newsize); 4485 newadp->ad_oldblkno = oldadp->ad_oldblkno; 4486 newadp->ad_oldsize = oldadp->ad_oldsize; 4487 /* 4488 * If the old dependency had a fragment to free or had never 4489 * previously had a block allocated, then the new dependency 4490 * can immediately post its freefrag and adopt the old freefrag. 4491 * This action is done by swapping the freefrag dependencies. 4492 * The new dependency gains the old one's freefrag, and the 4493 * old one gets the new one and then immediately puts it on 4494 * the worklist when it is freed by free_newblk. It is 4495 * not possible to do this swap when the old dependency had a 4496 * non-zero size but no previous fragment to free. This condition 4497 * arises when the new block is an extension of the old block. 4498 * Here, the first part of the fragment allocated to the new 4499 * dependency is part of the block currently claimed on disk by 4500 * the old dependency, so cannot legitimately be freed until the 4501 * conditions for the new dependency are fulfilled. 4502 */ 4503 freefrag = newadp->ad_freefrag; 4504 if (oldadp->ad_freefrag != NULL || oldadp->ad_oldblkno == 0) { 4505 newadp->ad_freefrag = oldadp->ad_freefrag; 4506 oldadp->ad_freefrag = freefrag; 4507 } 4508 /* 4509 * If we are tracking a new directory-block allocation, 4510 * move it from the old allocdirect to the new allocdirect. 4511 */ 4512 if ((wk = LIST_FIRST(&oldadp->ad_newdirblk)) != NULL) { 4513 newdirblk = WK_NEWDIRBLK(wk); 4514 WORKLIST_REMOVE(&newdirblk->db_list); 4515 if (!LIST_EMPTY(&oldadp->ad_newdirblk)) 4516 panic("allocdirect_merge: extra newdirblk"); 4517 WORKLIST_INSERT(&newadp->ad_newdirblk, &newdirblk->db_list); 4518 } 4519 TAILQ_REMOVE(adphead, oldadp, ad_next); 4520 /* 4521 * We need to move any journal dependencies over to the freefrag 4522 * that releases this block if it exists. Otherwise we are 4523 * extending an existing block and we'll wait until that is 4524 * complete to release the journal space and extend the 4525 * new journal to cover this old space as well. 4526 */ 4527 if (freefrag == NULL) { 4528 struct jnewblk *jnewblk; 4529 struct jnewblk *njnewblk; 4530 4531 if (oldadp->ad_newblkno != newadp->ad_newblkno) 4532 panic("allocdirect_merge: %jd != %jd", 4533 oldadp->ad_newblkno, newadp->ad_newblkno); 4534 jnewblk = oldadp->ad_block.nb_jnewblk; 4535 cancel_newblk(&oldadp->ad_block, &newadp->ad_block.nb_jwork); 4536 /* 4537 * We have an unwritten jnewblk, we need to merge the 4538 * frag bits with our own. The newer adp's journal can not 4539 * be written prior to the old one so no need to check for 4540 * it here. 4541 */ 4542 if (jnewblk) { 4543 njnewblk = newadp->ad_block.nb_jnewblk; 4544 if (njnewblk == NULL) 4545 panic("allocdirect_merge: No jnewblk"); 4546 if (jnewblk->jn_state & UNDONE) { 4547 njnewblk->jn_state |= UNDONE | NEWBLOCK; 4548 njnewblk->jn_state &= ~ATTACHED; 4549 jnewblk->jn_state &= ~UNDONE; 4550 } 4551 njnewblk->jn_oldfrags = jnewblk->jn_oldfrags; 4552 WORKLIST_REMOVE(&jnewblk->jn_list); 4553 jnewblk->jn_state |= ATTACHED | COMPLETE; 4554 free_jnewblk(jnewblk); 4555 } 4556 } else { 4557 /* 4558 * We can skip journaling for this freefrag and just complete 4559 * any pending journal work for the allocdirect that is being 4560 * removed after the freefrag completes. 4561 */ 4562 if (freefrag->ff_jfreefrag) 4563 cancel_jfreefrag(freefrag->ff_jfreefrag); 4564 cancel_newblk(&oldadp->ad_block, &freefrag->ff_jwork); 4565 } 4566 free_newblk(&oldadp->ad_block); 4567 } 4568 4569 /* 4570 * Allocate a jfreefrag structure to journal a single block free. 4571 */ 4572 static struct jfreefrag * 4573 newjfreefrag(freefrag, ip, blkno, size, lbn) 4574 struct freefrag *freefrag; 4575 struct inode *ip; 4576 ufs2_daddr_t blkno; 4577 long size; 4578 ufs_lbn_t lbn; 4579 { 4580 struct jfreefrag *jfreefrag; 4581 struct fs *fs; 4582 4583 fs = ip->i_fs; 4584 jfreefrag = malloc(sizeof(struct jfreefrag), M_JFREEFRAG, 4585 M_SOFTDEP_FLAGS); 4586 workitem_alloc(&jfreefrag->fr_list, D_JFREEFRAG, UFSTOVFS(ip->i_ump)); 4587 jfreefrag->fr_jsegdep = newjsegdep(&jfreefrag->fr_list); 4588 jfreefrag->fr_state = ATTACHED | DEPCOMPLETE; 4589 jfreefrag->fr_ino = ip->i_number; 4590 jfreefrag->fr_lbn = lbn; 4591 jfreefrag->fr_blkno = blkno; 4592 jfreefrag->fr_frags = numfrags(fs, size); 4593 jfreefrag->fr_freefrag = freefrag; 4594 4595 return (jfreefrag); 4596 } 4597 4598 /* 4599 * Allocate a new freefrag structure. 4600 */ 4601 static struct freefrag * 4602 newfreefrag(ip, blkno, size, lbn) 4603 struct inode *ip; 4604 ufs2_daddr_t blkno; 4605 long size; 4606 ufs_lbn_t lbn; 4607 { 4608 struct freefrag *freefrag; 4609 struct fs *fs; 4610 4611 fs = ip->i_fs; 4612 if (fragnum(fs, blkno) + numfrags(fs, size) > fs->fs_frag) 4613 panic("newfreefrag: frag size"); 4614 freefrag = malloc(sizeof(struct freefrag), 4615 M_FREEFRAG, M_SOFTDEP_FLAGS); 4616 workitem_alloc(&freefrag->ff_list, D_FREEFRAG, UFSTOVFS(ip->i_ump)); 4617 freefrag->ff_state = ATTACHED; 4618 LIST_INIT(&freefrag->ff_jwork); 4619 freefrag->ff_inum = ip->i_number; 4620 freefrag->ff_blkno = blkno; 4621 freefrag->ff_fragsize = size; 4622 4623 if (fs->fs_flags & FS_SUJ) { 4624 freefrag->ff_jfreefrag = 4625 newjfreefrag(freefrag, ip, blkno, size, lbn); 4626 } else { 4627 freefrag->ff_state |= DEPCOMPLETE; 4628 freefrag->ff_jfreefrag = NULL; 4629 } 4630 4631 return (freefrag); 4632 } 4633 4634 /* 4635 * This workitem de-allocates fragments that were replaced during 4636 * file block allocation. 4637 */ 4638 static void 4639 handle_workitem_freefrag(freefrag) 4640 struct freefrag *freefrag; 4641 { 4642 struct ufsmount *ump = VFSTOUFS(freefrag->ff_list.wk_mp); 4643 struct workhead wkhd; 4644 4645 /* 4646 * It would be illegal to add new completion items to the 4647 * freefrag after it was schedule to be done so it must be 4648 * safe to modify the list head here. 4649 */ 4650 LIST_INIT(&wkhd); 4651 LIST_SWAP(&freefrag->ff_jwork, &wkhd, worklist, wk_list); 4652 ffs_blkfree(ump, ump->um_fs, ump->um_devvp, freefrag->ff_blkno, 4653 freefrag->ff_fragsize, freefrag->ff_inum, &wkhd); 4654 ACQUIRE_LOCK(&lk); 4655 WORKITEM_FREE(freefrag, D_FREEFRAG); 4656 FREE_LOCK(&lk); 4657 } 4658 4659 /* 4660 * Set up a dependency structure for an external attributes data block. 4661 * This routine follows much of the structure of softdep_setup_allocdirect. 4662 * See the description of softdep_setup_allocdirect above for details. 4663 */ 4664 void 4665 softdep_setup_allocext(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 4666 struct inode *ip; 4667 ufs_lbn_t off; 4668 ufs2_daddr_t newblkno; 4669 ufs2_daddr_t oldblkno; 4670 long newsize; 4671 long oldsize; 4672 struct buf *bp; 4673 { 4674 struct allocdirect *adp, *oldadp; 4675 struct allocdirectlst *adphead; 4676 struct freefrag *freefrag; 4677 struct inodedep *inodedep; 4678 struct jnewblk *jnewblk; 4679 struct newblk *newblk; 4680 struct mount *mp; 4681 ufs_lbn_t lbn; 4682 4683 if (off >= NXADDR) 4684 panic("softdep_setup_allocext: lbn %lld > NXADDR", 4685 (long long)off); 4686 4687 lbn = bp->b_lblkno; 4688 mp = UFSTOVFS(ip->i_ump); 4689 if (oldblkno && oldblkno != newblkno) 4690 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn); 4691 else 4692 freefrag = NULL; 4693 4694 ACQUIRE_LOCK(&lk); 4695 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 4696 panic("softdep_setup_allocext: lost block"); 4697 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 4698 ("softdep_setup_allocext: newblk already initialized")); 4699 /* 4700 * Convert the newblk to an allocdirect. 4701 */ 4702 newblk->nb_list.wk_type = D_ALLOCDIRECT; 4703 adp = (struct allocdirect *)newblk; 4704 newblk->nb_freefrag = freefrag; 4705 adp->ad_offset = off; 4706 adp->ad_oldblkno = oldblkno; 4707 adp->ad_newsize = newsize; 4708 adp->ad_oldsize = oldsize; 4709 adp->ad_state |= EXTDATA; 4710 4711 /* 4712 * Finish initializing the journal. 4713 */ 4714 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 4715 jnewblk->jn_ino = ip->i_number; 4716 jnewblk->jn_lbn = lbn; 4717 add_to_journal(&jnewblk->jn_list); 4718 } 4719 if (freefrag && freefrag->ff_jfreefrag != NULL) 4720 add_to_journal(&freefrag->ff_jfreefrag->fr_list); 4721 inodedep_lookup(mp, ip->i_number, DEPALLOC | NODELAY, &inodedep); 4722 adp->ad_inodedep = inodedep; 4723 4724 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 4725 /* 4726 * The list of allocdirects must be kept in sorted and ascending 4727 * order so that the rollback routines can quickly determine the 4728 * first uncommitted block (the size of the file stored on disk 4729 * ends at the end of the lowest committed fragment, or if there 4730 * are no fragments, at the end of the highest committed block). 4731 * Since files generally grow, the typical case is that the new 4732 * block is to be added at the end of the list. We speed this 4733 * special case by checking against the last allocdirect in the 4734 * list before laboriously traversing the list looking for the 4735 * insertion point. 4736 */ 4737 adphead = &inodedep->id_newextupdt; 4738 oldadp = TAILQ_LAST(adphead, allocdirectlst); 4739 if (oldadp == NULL || oldadp->ad_offset <= off) { 4740 /* insert at end of list */ 4741 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 4742 if (oldadp != NULL && oldadp->ad_offset == off) 4743 allocdirect_merge(adphead, adp, oldadp); 4744 FREE_LOCK(&lk); 4745 return; 4746 } 4747 TAILQ_FOREACH(oldadp, adphead, ad_next) { 4748 if (oldadp->ad_offset >= off) 4749 break; 4750 } 4751 if (oldadp == NULL) 4752 panic("softdep_setup_allocext: lost entry"); 4753 /* insert in middle of list */ 4754 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 4755 if (oldadp->ad_offset == off) 4756 allocdirect_merge(adphead, adp, oldadp); 4757 FREE_LOCK(&lk); 4758 } 4759 4760 /* 4761 * Indirect block allocation dependencies. 4762 * 4763 * The same dependencies that exist for a direct block also exist when 4764 * a new block is allocated and pointed to by an entry in a block of 4765 * indirect pointers. The undo/redo states described above are also 4766 * used here. Because an indirect block contains many pointers that 4767 * may have dependencies, a second copy of the entire in-memory indirect 4768 * block is kept. The buffer cache copy is always completely up-to-date. 4769 * The second copy, which is used only as a source for disk writes, 4770 * contains only the safe pointers (i.e., those that have no remaining 4771 * update dependencies). The second copy is freed when all pointers 4772 * are safe. The cache is not allowed to replace indirect blocks with 4773 * pending update dependencies. If a buffer containing an indirect 4774 * block with dependencies is written, these routines will mark it 4775 * dirty again. It can only be successfully written once all the 4776 * dependencies are removed. The ffs_fsync routine in conjunction with 4777 * softdep_sync_metadata work together to get all the dependencies 4778 * removed so that a file can be successfully written to disk. Three 4779 * procedures are used when setting up indirect block pointer 4780 * dependencies. The division is necessary because of the organization 4781 * of the "balloc" routine and because of the distinction between file 4782 * pages and file metadata blocks. 4783 */ 4784 4785 /* 4786 * Allocate a new allocindir structure. 4787 */ 4788 static struct allocindir * 4789 newallocindir(ip, ptrno, newblkno, oldblkno, lbn) 4790 struct inode *ip; /* inode for file being extended */ 4791 int ptrno; /* offset of pointer in indirect block */ 4792 ufs2_daddr_t newblkno; /* disk block number being added */ 4793 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 4794 ufs_lbn_t lbn; 4795 { 4796 struct newblk *newblk; 4797 struct allocindir *aip; 4798 struct freefrag *freefrag; 4799 struct jnewblk *jnewblk; 4800 4801 if (oldblkno) 4802 freefrag = newfreefrag(ip, oldblkno, ip->i_fs->fs_bsize, lbn); 4803 else 4804 freefrag = NULL; 4805 ACQUIRE_LOCK(&lk); 4806 if (newblk_lookup(UFSTOVFS(ip->i_ump), newblkno, 0, &newblk) == 0) 4807 panic("new_allocindir: lost block"); 4808 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 4809 ("newallocindir: newblk already initialized")); 4810 newblk->nb_list.wk_type = D_ALLOCINDIR; 4811 newblk->nb_freefrag = freefrag; 4812 aip = (struct allocindir *)newblk; 4813 aip->ai_offset = ptrno; 4814 aip->ai_oldblkno = oldblkno; 4815 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 4816 jnewblk->jn_ino = ip->i_number; 4817 jnewblk->jn_lbn = lbn; 4818 add_to_journal(&jnewblk->jn_list); 4819 } 4820 if (freefrag && freefrag->ff_jfreefrag != NULL) 4821 add_to_journal(&freefrag->ff_jfreefrag->fr_list); 4822 return (aip); 4823 } 4824 4825 /* 4826 * Called just before setting an indirect block pointer 4827 * to a newly allocated file page. 4828 */ 4829 void 4830 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 4831 struct inode *ip; /* inode for file being extended */ 4832 ufs_lbn_t lbn; /* allocated block number within file */ 4833 struct buf *bp; /* buffer with indirect blk referencing page */ 4834 int ptrno; /* offset of pointer in indirect block */ 4835 ufs2_daddr_t newblkno; /* disk block number being added */ 4836 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 4837 struct buf *nbp; /* buffer holding allocated page */ 4838 { 4839 struct inodedep *inodedep; 4840 struct allocindir *aip; 4841 struct pagedep *pagedep; 4842 struct mount *mp; 4843 4844 if (lbn != nbp->b_lblkno) 4845 panic("softdep_setup_allocindir_page: lbn %jd != lblkno %jd", 4846 lbn, bp->b_lblkno); 4847 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_page"); 4848 mp = UFSTOVFS(ip->i_ump); 4849 aip = newallocindir(ip, ptrno, newblkno, oldblkno, lbn); 4850 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 4851 /* 4852 * If we are allocating a directory page, then we must 4853 * allocate an associated pagedep to track additions and 4854 * deletions. 4855 */ 4856 if ((ip->i_mode & IFMT) == IFDIR && 4857 pagedep_lookup(mp, ip->i_number, lbn, DEPALLOC, &pagedep) == 0) 4858 WORKLIST_INSERT(&nbp->b_dep, &pagedep->pd_list); 4859 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 4860 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn); 4861 FREE_LOCK(&lk); 4862 } 4863 4864 /* 4865 * Called just before setting an indirect block pointer to a 4866 * newly allocated indirect block. 4867 */ 4868 void 4869 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 4870 struct buf *nbp; /* newly allocated indirect block */ 4871 struct inode *ip; /* inode for file being extended */ 4872 struct buf *bp; /* indirect block referencing allocated block */ 4873 int ptrno; /* offset of pointer in indirect block */ 4874 ufs2_daddr_t newblkno; /* disk block number being added */ 4875 { 4876 struct inodedep *inodedep; 4877 struct allocindir *aip; 4878 ufs_lbn_t lbn; 4879 4880 lbn = nbp->b_lblkno; 4881 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_meta"); 4882 aip = newallocindir(ip, ptrno, newblkno, 0, lbn); 4883 inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, DEPALLOC, &inodedep); 4884 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 4885 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn); 4886 FREE_LOCK(&lk); 4887 } 4888 4889 static void 4890 indirdep_complete(indirdep) 4891 struct indirdep *indirdep; 4892 { 4893 struct allocindir *aip; 4894 4895 LIST_REMOVE(indirdep, ir_next); 4896 indirdep->ir_state &= ~ONDEPLIST; 4897 4898 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) { 4899 LIST_REMOVE(aip, ai_next); 4900 free_newblk(&aip->ai_block); 4901 } 4902 /* 4903 * If this indirdep is not attached to a buf it was simply waiting 4904 * on completion to clear completehd. free_indirdep() asserts 4905 * that nothing is dangling. 4906 */ 4907 if ((indirdep->ir_state & ONWORKLIST) == 0) 4908 free_indirdep(indirdep); 4909 } 4910 4911 /* 4912 * Called to finish the allocation of the "aip" allocated 4913 * by one of the two routines above. 4914 */ 4915 static void 4916 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn) 4917 struct buf *bp; /* in-memory copy of the indirect block */ 4918 struct inode *ip; /* inode for file being extended */ 4919 struct inodedep *inodedep; /* Inodedep for ip */ 4920 struct allocindir *aip; /* allocindir allocated by the above routines */ 4921 ufs_lbn_t lbn; /* Logical block number for this block. */ 4922 { 4923 struct worklist *wk; 4924 struct fs *fs; 4925 struct newblk *newblk; 4926 struct indirdep *indirdep, *newindirdep; 4927 struct allocindir *oldaip; 4928 struct freefrag *freefrag; 4929 struct mount *mp; 4930 ufs2_daddr_t blkno; 4931 4932 mp = UFSTOVFS(ip->i_ump); 4933 fs = ip->i_fs; 4934 mtx_assert(&lk, MA_OWNED); 4935 if (bp->b_lblkno >= 0) 4936 panic("setup_allocindir_phase2: not indir blk"); 4937 for (freefrag = NULL, indirdep = NULL, newindirdep = NULL; ; ) { 4938 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 4939 if (wk->wk_type != D_INDIRDEP) 4940 continue; 4941 indirdep = WK_INDIRDEP(wk); 4942 break; 4943 } 4944 if (indirdep == NULL && newindirdep) { 4945 indirdep = newindirdep; 4946 newindirdep = NULL; 4947 WORKLIST_INSERT(&bp->b_dep, &indirdep->ir_list); 4948 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, 4949 &newblk)) { 4950 indirdep->ir_state |= ONDEPLIST; 4951 LIST_INSERT_HEAD(&newblk->nb_indirdeps, 4952 indirdep, ir_next); 4953 } else 4954 indirdep->ir_state |= DEPCOMPLETE; 4955 } 4956 if (indirdep) { 4957 aip->ai_indirdep = indirdep; 4958 /* 4959 * Check to see if there is an existing dependency 4960 * for this block. If there is, merge the old 4961 * dependency into the new one. This happens 4962 * as a result of reallocblk only. 4963 */ 4964 if (aip->ai_oldblkno == 0) 4965 oldaip = NULL; 4966 else 4967 4968 LIST_FOREACH(oldaip, &indirdep->ir_deplisthd, 4969 ai_next) 4970 if (oldaip->ai_offset == aip->ai_offset) 4971 break; 4972 if (oldaip != NULL) 4973 freefrag = allocindir_merge(aip, oldaip); 4974 LIST_INSERT_HEAD(&indirdep->ir_deplisthd, aip, ai_next); 4975 KASSERT(aip->ai_offset >= 0 && 4976 aip->ai_offset < NINDIR(ip->i_ump->um_fs), 4977 ("setup_allocindir_phase2: Bad offset %d", 4978 aip->ai_offset)); 4979 KASSERT(indirdep->ir_savebp != NULL, 4980 ("setup_allocindir_phase2 NULL ir_savebp")); 4981 if (ip->i_ump->um_fstype == UFS1) 4982 ((ufs1_daddr_t *)indirdep->ir_savebp->b_data) 4983 [aip->ai_offset] = aip->ai_oldblkno; 4984 else 4985 ((ufs2_daddr_t *)indirdep->ir_savebp->b_data) 4986 [aip->ai_offset] = aip->ai_oldblkno; 4987 FREE_LOCK(&lk); 4988 if (freefrag != NULL) 4989 handle_workitem_freefrag(freefrag); 4990 } else 4991 FREE_LOCK(&lk); 4992 if (newindirdep) { 4993 newindirdep->ir_savebp->b_flags |= B_INVAL | B_NOCACHE; 4994 brelse(newindirdep->ir_savebp); 4995 ACQUIRE_LOCK(&lk); 4996 WORKITEM_FREE((caddr_t)newindirdep, D_INDIRDEP); 4997 if (indirdep) 4998 break; 4999 FREE_LOCK(&lk); 5000 } 5001 if (indirdep) { 5002 ACQUIRE_LOCK(&lk); 5003 break; 5004 } 5005 newindirdep = malloc(sizeof(struct indirdep), 5006 M_INDIRDEP, M_SOFTDEP_FLAGS); 5007 workitem_alloc(&newindirdep->ir_list, D_INDIRDEP, mp); 5008 newindirdep->ir_state = ATTACHED; 5009 if (ip->i_ump->um_fstype == UFS1) 5010 newindirdep->ir_state |= UFS1FMT; 5011 newindirdep->ir_saveddata = NULL; 5012 LIST_INIT(&newindirdep->ir_deplisthd); 5013 LIST_INIT(&newindirdep->ir_donehd); 5014 LIST_INIT(&newindirdep->ir_writehd); 5015 LIST_INIT(&newindirdep->ir_completehd); 5016 LIST_INIT(&newindirdep->ir_jwork); 5017 if (bp->b_blkno == bp->b_lblkno) { 5018 ufs_bmaparray(bp->b_vp, bp->b_lblkno, &blkno, bp, 5019 NULL, NULL); 5020 bp->b_blkno = blkno; 5021 } 5022 newindirdep->ir_savebp = 5023 getblk(ip->i_devvp, bp->b_blkno, bp->b_bcount, 0, 0, 0); 5024 BUF_KERNPROC(newindirdep->ir_savebp); 5025 bcopy(bp->b_data, newindirdep->ir_savebp->b_data, bp->b_bcount); 5026 ACQUIRE_LOCK(&lk); 5027 } 5028 } 5029 5030 /* 5031 * Merge two allocindirs which refer to the same block. Move newblock 5032 * dependencies and setup the freefrags appropriately. 5033 */ 5034 static struct freefrag * 5035 allocindir_merge(aip, oldaip) 5036 struct allocindir *aip; 5037 struct allocindir *oldaip; 5038 { 5039 struct newdirblk *newdirblk; 5040 struct freefrag *freefrag; 5041 struct worklist *wk; 5042 5043 if (oldaip->ai_newblkno != aip->ai_oldblkno) 5044 panic("allocindir_merge: blkno"); 5045 aip->ai_oldblkno = oldaip->ai_oldblkno; 5046 freefrag = aip->ai_freefrag; 5047 aip->ai_freefrag = oldaip->ai_freefrag; 5048 oldaip->ai_freefrag = NULL; 5049 KASSERT(freefrag != NULL, ("setup_allocindir_phase2: No freefrag")); 5050 /* 5051 * If we are tracking a new directory-block allocation, 5052 * move it from the old allocindir to the new allocindir. 5053 */ 5054 if ((wk = LIST_FIRST(&oldaip->ai_newdirblk)) != NULL) { 5055 newdirblk = WK_NEWDIRBLK(wk); 5056 WORKLIST_REMOVE(&newdirblk->db_list); 5057 if (!LIST_EMPTY(&oldaip->ai_newdirblk)) 5058 panic("allocindir_merge: extra newdirblk"); 5059 WORKLIST_INSERT(&aip->ai_newdirblk, &newdirblk->db_list); 5060 } 5061 /* 5062 * We can skip journaling for this freefrag and just complete 5063 * any pending journal work for the allocindir that is being 5064 * removed after the freefrag completes. 5065 */ 5066 if (freefrag->ff_jfreefrag) 5067 cancel_jfreefrag(freefrag->ff_jfreefrag); 5068 LIST_REMOVE(oldaip, ai_next); 5069 cancel_newblk(&oldaip->ai_block, &freefrag->ff_jwork); 5070 free_newblk(&oldaip->ai_block); 5071 5072 return (freefrag); 5073 } 5074 5075 /* 5076 * Block de-allocation dependencies. 5077 * 5078 * When blocks are de-allocated, the on-disk pointers must be nullified before 5079 * the blocks are made available for use by other files. (The true 5080 * requirement is that old pointers must be nullified before new on-disk 5081 * pointers are set. We chose this slightly more stringent requirement to 5082 * reduce complexity.) Our implementation handles this dependency by updating 5083 * the inode (or indirect block) appropriately but delaying the actual block 5084 * de-allocation (i.e., freemap and free space count manipulation) until 5085 * after the updated versions reach stable storage. After the disk is 5086 * updated, the blocks can be safely de-allocated whenever it is convenient. 5087 * This implementation handles only the common case of reducing a file's 5088 * length to zero. Other cases are handled by the conventional synchronous 5089 * write approach. 5090 * 5091 * The ffs implementation with which we worked double-checks 5092 * the state of the block pointers and file size as it reduces 5093 * a file's length. Some of this code is replicated here in our 5094 * soft updates implementation. The freeblks->fb_chkcnt field is 5095 * used to transfer a part of this information to the procedure 5096 * that eventually de-allocates the blocks. 5097 * 5098 * This routine should be called from the routine that shortens 5099 * a file's length, before the inode's size or block pointers 5100 * are modified. It will save the block pointer information for 5101 * later release and zero the inode so that the calling routine 5102 * can release it. 5103 */ 5104 void 5105 softdep_setup_freeblocks(ip, length, flags) 5106 struct inode *ip; /* The inode whose length is to be reduced */ 5107 off_t length; /* The new length for the file */ 5108 int flags; /* IO_EXT and/or IO_NORMAL */ 5109 { 5110 struct ufs1_dinode *dp1; 5111 struct ufs2_dinode *dp2; 5112 struct freeblks *freeblks; 5113 struct inodedep *inodedep; 5114 struct allocdirect *adp; 5115 struct jfreeblk *jfreeblk; 5116 struct bufobj *bo; 5117 struct vnode *vp; 5118 struct buf *bp; 5119 struct fs *fs; 5120 ufs2_daddr_t extblocks, datablocks; 5121 struct mount *mp; 5122 int i, delay, error; 5123 ufs2_daddr_t blkno; 5124 ufs_lbn_t tmpval; 5125 ufs_lbn_t lbn; 5126 long oldextsize; 5127 long oldsize; 5128 int frags; 5129 int needj; 5130 5131 fs = ip->i_fs; 5132 mp = UFSTOVFS(ip->i_ump); 5133 if (length != 0) 5134 panic("softdep_setup_freeblocks: non-zero length"); 5135 freeblks = malloc(sizeof(struct freeblks), 5136 M_FREEBLKS, M_SOFTDEP_FLAGS|M_ZERO); 5137 workitem_alloc(&freeblks->fb_list, D_FREEBLKS, mp); 5138 LIST_INIT(&freeblks->fb_jfreeblkhd); 5139 LIST_INIT(&freeblks->fb_jwork); 5140 freeblks->fb_state = ATTACHED; 5141 freeblks->fb_uid = ip->i_uid; 5142 freeblks->fb_previousinum = ip->i_number; 5143 freeblks->fb_devvp = ip->i_devvp; 5144 freeblks->fb_chkcnt = 0; 5145 ACQUIRE_LOCK(&lk); 5146 /* 5147 * If we're truncating a removed file that will never be written 5148 * we don't need to journal the block frees. The canceled journals 5149 * for the allocations will suffice. 5150 */ 5151 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5152 if ((inodedep->id_state & (UNLINKED | DEPCOMPLETE)) == UNLINKED || 5153 (fs->fs_flags & FS_SUJ) == 0) 5154 needj = 0; 5155 else 5156 needj = 1; 5157 num_freeblkdep++; 5158 FREE_LOCK(&lk); 5159 extblocks = 0; 5160 if (fs->fs_magic == FS_UFS2_MAGIC) 5161 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize)); 5162 datablocks = DIP(ip, i_blocks) - extblocks; 5163 if ((flags & IO_NORMAL) != 0) { 5164 oldsize = ip->i_size; 5165 ip->i_size = 0; 5166 DIP_SET(ip, i_size, 0); 5167 freeblks->fb_chkcnt = datablocks; 5168 for (i = 0; i < NDADDR; i++) { 5169 blkno = DIP(ip, i_db[i]); 5170 DIP_SET(ip, i_db[i], 0); 5171 if (blkno == 0) 5172 continue; 5173 frags = sblksize(fs, oldsize, i); 5174 frags = numfrags(fs, frags); 5175 newfreework(freeblks, NULL, i, blkno, frags, needj); 5176 } 5177 for (i = 0, tmpval = NINDIR(fs), lbn = NDADDR; i < NIADDR; 5178 i++, tmpval *= NINDIR(fs)) { 5179 blkno = DIP(ip, i_ib[i]); 5180 DIP_SET(ip, i_ib[i], 0); 5181 if (blkno) 5182 newfreework(freeblks, NULL, -lbn - i, blkno, 5183 fs->fs_frag, needj); 5184 lbn += tmpval; 5185 } 5186 UFS_LOCK(ip->i_ump); 5187 fs->fs_pendingblocks += datablocks; 5188 UFS_UNLOCK(ip->i_ump); 5189 } 5190 if ((flags & IO_EXT) != 0) { 5191 oldextsize = ip->i_din2->di_extsize; 5192 ip->i_din2->di_extsize = 0; 5193 freeblks->fb_chkcnt += extblocks; 5194 for (i = 0; i < NXADDR; i++) { 5195 blkno = ip->i_din2->di_extb[i]; 5196 ip->i_din2->di_extb[i] = 0; 5197 if (blkno == 0) 5198 continue; 5199 frags = sblksize(fs, oldextsize, i); 5200 frags = numfrags(fs, frags); 5201 newfreework(freeblks, NULL, -1 - i, blkno, frags, 5202 needj); 5203 } 5204 } 5205 if (LIST_EMPTY(&freeblks->fb_jfreeblkhd)) 5206 needj = 0; 5207 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - freeblks->fb_chkcnt); 5208 /* 5209 * Push the zero'ed inode to to its disk buffer so that we are free 5210 * to delete its dependencies below. Once the dependencies are gone 5211 * the buffer can be safely released. 5212 */ 5213 if ((error = bread(ip->i_devvp, 5214 fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 5215 (int)fs->fs_bsize, NOCRED, &bp)) != 0) { 5216 brelse(bp); 5217 softdep_error("softdep_setup_freeblocks", error); 5218 } 5219 if (ip->i_ump->um_fstype == UFS1) { 5220 dp1 = ((struct ufs1_dinode *)bp->b_data + 5221 ino_to_fsbo(fs, ip->i_number)); 5222 ip->i_din1->di_freelink = dp1->di_freelink; 5223 *dp1 = *ip->i_din1; 5224 } else { 5225 dp2 = ((struct ufs2_dinode *)bp->b_data + 5226 ino_to_fsbo(fs, ip->i_number)); 5227 ip->i_din2->di_freelink = dp2->di_freelink; 5228 *dp2 = *ip->i_din2; 5229 } 5230 /* 5231 * Find and eliminate any inode dependencies. 5232 */ 5233 ACQUIRE_LOCK(&lk); 5234 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5235 if ((inodedep->id_state & IOSTARTED) != 0) 5236 panic("softdep_setup_freeblocks: inode busy"); 5237 /* 5238 * Add the freeblks structure to the list of operations that 5239 * must await the zero'ed inode being written to disk. If we 5240 * still have a bitmap dependency (delay == 0), then the inode 5241 * has never been written to disk, so we can process the 5242 * freeblks below once we have deleted the dependencies. 5243 */ 5244 delay = (inodedep->id_state & DEPCOMPLETE); 5245 if (delay) 5246 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list); 5247 else if (needj) 5248 freeblks->fb_state |= DEPCOMPLETE | COMPLETE; 5249 /* 5250 * Because the file length has been truncated to zero, any 5251 * pending block allocation dependency structures associated 5252 * with this inode are obsolete and can simply be de-allocated. 5253 * We must first merge the two dependency lists to get rid of 5254 * any duplicate freefrag structures, then purge the merged list. 5255 * If we still have a bitmap dependency, then the inode has never 5256 * been written to disk, so we can free any fragments without delay. 5257 */ 5258 if (flags & IO_NORMAL) { 5259 merge_inode_lists(&inodedep->id_newinoupdt, 5260 &inodedep->id_inoupdt); 5261 while ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != 0) 5262 cancel_allocdirect(&inodedep->id_inoupdt, adp, 5263 freeblks, delay); 5264 } 5265 if (flags & IO_EXT) { 5266 merge_inode_lists(&inodedep->id_newextupdt, 5267 &inodedep->id_extupdt); 5268 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != 0) 5269 cancel_allocdirect(&inodedep->id_extupdt, adp, 5270 freeblks, delay); 5271 } 5272 LIST_FOREACH(jfreeblk, &freeblks->fb_jfreeblkhd, jf_deps) 5273 add_to_journal(&jfreeblk->jf_list); 5274 5275 FREE_LOCK(&lk); 5276 bdwrite(bp); 5277 /* 5278 * We must wait for any I/O in progress to finish so that 5279 * all potential buffers on the dirty list will be visible. 5280 * Once they are all there, walk the list and get rid of 5281 * any dependencies. 5282 */ 5283 vp = ITOV(ip); 5284 bo = &vp->v_bufobj; 5285 BO_LOCK(bo); 5286 drain_output(vp); 5287 restart: 5288 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) { 5289 if (((flags & IO_EXT) == 0 && (bp->b_xflags & BX_ALTDATA)) || 5290 ((flags & IO_NORMAL) == 0 && 5291 (bp->b_xflags & BX_ALTDATA) == 0)) 5292 continue; 5293 if ((bp = getdirtybuf(bp, BO_MTX(bo), MNT_WAIT)) == NULL) 5294 goto restart; 5295 BO_UNLOCK(bo); 5296 ACQUIRE_LOCK(&lk); 5297 (void) inodedep_lookup(mp, ip->i_number, 0, &inodedep); 5298 if (deallocate_dependencies(bp, inodedep, freeblks)) 5299 bp->b_flags |= B_INVAL | B_NOCACHE; 5300 FREE_LOCK(&lk); 5301 brelse(bp); 5302 BO_LOCK(bo); 5303 goto restart; 5304 } 5305 BO_UNLOCK(bo); 5306 ACQUIRE_LOCK(&lk); 5307 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 5308 (void) free_inodedep(inodedep); 5309 5310 if (delay) { 5311 freeblks->fb_state |= DEPCOMPLETE; 5312 /* 5313 * If the inode with zeroed block pointers is now on disk 5314 * we can start freeing blocks. Add freeblks to the worklist 5315 * instead of calling handle_workitem_freeblocks directly as 5316 * it is more likely that additional IO is needed to complete 5317 * the request here than in the !delay case. 5318 */ 5319 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) 5320 add_to_worklist(&freeblks->fb_list, 1); 5321 } 5322 5323 FREE_LOCK(&lk); 5324 /* 5325 * If the inode has never been written to disk (delay == 0) and 5326 * we're not waiting on any journal writes, then we can process the 5327 * freeblks now that we have deleted the dependencies. 5328 */ 5329 if (!delay && !needj) 5330 handle_workitem_freeblocks(freeblks, 0); 5331 } 5332 5333 /* 5334 * Reclaim any dependency structures from a buffer that is about to 5335 * be reallocated to a new vnode. The buffer must be locked, thus, 5336 * no I/O completion operations can occur while we are manipulating 5337 * its associated dependencies. The mutex is held so that other I/O's 5338 * associated with related dependencies do not occur. Returns 1 if 5339 * all dependencies were cleared, 0 otherwise. 5340 */ 5341 static int 5342 deallocate_dependencies(bp, inodedep, freeblks) 5343 struct buf *bp; 5344 struct inodedep *inodedep; 5345 struct freeblks *freeblks; 5346 { 5347 struct worklist *wk; 5348 struct indirdep *indirdep; 5349 struct newdirblk *newdirblk; 5350 struct allocindir *aip; 5351 struct pagedep *pagedep; 5352 struct jremref *jremref; 5353 struct jmvref *jmvref; 5354 struct dirrem *dirrem; 5355 int i; 5356 5357 mtx_assert(&lk, MA_OWNED); 5358 while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) { 5359 switch (wk->wk_type) { 5360 5361 case D_INDIRDEP: 5362 indirdep = WK_INDIRDEP(wk); 5363 if (bp->b_lblkno >= 0 || 5364 bp->b_blkno != indirdep->ir_savebp->b_lblkno) 5365 panic("deallocate_dependencies: not indir"); 5366 cancel_indirdep(indirdep, bp, inodedep, freeblks); 5367 continue; 5368 5369 case D_PAGEDEP: 5370 pagedep = WK_PAGEDEP(wk); 5371 /* 5372 * There should be no directory add dependencies present 5373 * as the directory could not be truncated until all 5374 * children were removed. 5375 */ 5376 KASSERT(LIST_FIRST(&pagedep->pd_pendinghd) == NULL, 5377 ("deallocate_dependencies: pendinghd != NULL")); 5378 for (i = 0; i < DAHASHSZ; i++) 5379 KASSERT(LIST_FIRST(&pagedep->pd_diraddhd[i]) == NULL, 5380 ("deallocate_dependencies: diraddhd != NULL")); 5381 /* 5382 * Copy any directory remove dependencies to the list 5383 * to be processed after the zero'ed inode is written. 5384 * If the inode has already been written, then they 5385 * can be dumped directly onto the work list. 5386 */ 5387 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) { 5388 /* 5389 * If there are any dirrems we wait for 5390 * the journal write to complete and 5391 * then restart the buf scan as the lock 5392 * has been dropped. 5393 */ 5394 while ((jremref = 5395 LIST_FIRST(&dirrem->dm_jremrefhd)) 5396 != NULL) { 5397 stat_jwait_filepage++; 5398 jwait(&jremref->jr_list); 5399 return (0); 5400 } 5401 LIST_REMOVE(dirrem, dm_next); 5402 dirrem->dm_dirinum = pagedep->pd_ino; 5403 if (inodedep == NULL || 5404 (inodedep->id_state & ALLCOMPLETE) == 5405 ALLCOMPLETE) { 5406 dirrem->dm_state |= COMPLETE; 5407 add_to_worklist(&dirrem->dm_list, 0); 5408 } else 5409 WORKLIST_INSERT(&inodedep->id_bufwait, 5410 &dirrem->dm_list); 5411 } 5412 if ((pagedep->pd_state & NEWBLOCK) != 0) { 5413 newdirblk = pagedep->pd_newdirblk; 5414 WORKLIST_REMOVE(&newdirblk->db_list); 5415 free_newdirblk(newdirblk); 5416 } 5417 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) 5418 != NULL) { 5419 stat_jwait_filepage++; 5420 jwait(&jmvref->jm_list); 5421 return (0); 5422 } 5423 WORKLIST_REMOVE(&pagedep->pd_list); 5424 LIST_REMOVE(pagedep, pd_hash); 5425 WORKITEM_FREE(pagedep, D_PAGEDEP); 5426 continue; 5427 5428 case D_ALLOCINDIR: 5429 aip = WK_ALLOCINDIR(wk); 5430 cancel_allocindir(aip, inodedep, freeblks); 5431 continue; 5432 5433 case D_ALLOCDIRECT: 5434 case D_INODEDEP: 5435 panic("deallocate_dependencies: Unexpected type %s", 5436 TYPENAME(wk->wk_type)); 5437 /* NOTREACHED */ 5438 5439 default: 5440 panic("deallocate_dependencies: Unknown type %s", 5441 TYPENAME(wk->wk_type)); 5442 /* NOTREACHED */ 5443 } 5444 } 5445 5446 return (1); 5447 } 5448 5449 /* 5450 * An allocdirect is being canceled due to a truncate. We must make sure 5451 * the journal entry is released in concert with the blkfree that releases 5452 * the storage. Completed journal entries must not be released until the 5453 * space is no longer pointed to by the inode or in the bitmap. 5454 */ 5455 static void 5456 cancel_allocdirect(adphead, adp, freeblks, delay) 5457 struct allocdirectlst *adphead; 5458 struct allocdirect *adp; 5459 struct freeblks *freeblks; 5460 int delay; 5461 { 5462 struct freework *freework; 5463 struct newblk *newblk; 5464 struct worklist *wk; 5465 ufs_lbn_t lbn; 5466 5467 TAILQ_REMOVE(adphead, adp, ad_next); 5468 newblk = (struct newblk *)adp; 5469 /* 5470 * If the journal hasn't been written the jnewblk must be passed 5471 * to the call to ffs_freeblk that reclaims the space. We accomplish 5472 * this by linking the journal dependency into the freework to be 5473 * freed when freework_freeblock() is called. If the journal has 5474 * been written we can simply reclaim the journal space when the 5475 * freeblks work is complete. 5476 */ 5477 if (newblk->nb_jnewblk == NULL) { 5478 cancel_newblk(newblk, &freeblks->fb_jwork); 5479 goto found; 5480 } 5481 lbn = newblk->nb_jnewblk->jn_lbn; 5482 /* 5483 * Find the correct freework structure so it releases the canceled 5484 * journal when the bitmap is cleared. This preserves rollback 5485 * until the allocation is reverted. 5486 */ 5487 LIST_FOREACH(wk, &freeblks->fb_freeworkhd, wk_list) { 5488 freework = WK_FREEWORK(wk); 5489 if (freework->fw_lbn != lbn) 5490 continue; 5491 cancel_newblk(newblk, &freework->fw_jwork); 5492 goto found; 5493 } 5494 panic("cancel_allocdirect: Freework not found for lbn %jd\n", lbn); 5495 found: 5496 if (delay) 5497 WORKLIST_INSERT(&adp->ad_inodedep->id_bufwait, 5498 &newblk->nb_list); 5499 else 5500 free_newblk(newblk); 5501 return; 5502 } 5503 5504 5505 static void 5506 cancel_newblk(newblk, wkhd) 5507 struct newblk *newblk; 5508 struct workhead *wkhd; 5509 { 5510 struct indirdep *indirdep; 5511 struct allocindir *aip; 5512 5513 while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL) { 5514 indirdep->ir_state &= ~ONDEPLIST; 5515 LIST_REMOVE(indirdep, ir_next); 5516 /* 5517 * If an indirdep is not on the buf worklist we need to 5518 * free it here as deallocate_dependencies() will never 5519 * find it. These pointers were never visible on disk and 5520 * can be discarded immediately. 5521 */ 5522 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) { 5523 LIST_REMOVE(aip, ai_next); 5524 cancel_newblk(&aip->ai_block, wkhd); 5525 free_newblk(&aip->ai_block); 5526 } 5527 /* 5528 * If this indirdep is not attached to a buf it was simply 5529 * waiting on completion to clear completehd. free_indirdep() 5530 * asserts that nothing is dangling. 5531 */ 5532 if ((indirdep->ir_state & ONWORKLIST) == 0) 5533 free_indirdep(indirdep); 5534 } 5535 if (newblk->nb_state & ONDEPLIST) { 5536 newblk->nb_state &= ~ONDEPLIST; 5537 LIST_REMOVE(newblk, nb_deps); 5538 } 5539 if (newblk->nb_state & ONWORKLIST) 5540 WORKLIST_REMOVE(&newblk->nb_list); 5541 /* 5542 * If the journal entry hasn't been written we hold onto the dep 5543 * until it is safe to free along with the other journal work. 5544 */ 5545 if (newblk->nb_jnewblk != NULL) { 5546 cancel_jnewblk(newblk->nb_jnewblk, wkhd); 5547 newblk->nb_jnewblk = NULL; 5548 } 5549 if (!LIST_EMPTY(&newblk->nb_jwork)) 5550 jwork_move(wkhd, &newblk->nb_jwork); 5551 } 5552 5553 /* 5554 * Free a newblk. Generate a new freefrag work request if appropriate. 5555 * This must be called after the inode pointer and any direct block pointers 5556 * are valid or fully removed via truncate or frag extension. 5557 */ 5558 static void 5559 free_newblk(newblk) 5560 struct newblk *newblk; 5561 { 5562 struct indirdep *indirdep; 5563 struct newdirblk *newdirblk; 5564 struct freefrag *freefrag; 5565 struct worklist *wk; 5566 5567 mtx_assert(&lk, MA_OWNED); 5568 if (newblk->nb_state & ONDEPLIST) 5569 LIST_REMOVE(newblk, nb_deps); 5570 if (newblk->nb_state & ONWORKLIST) 5571 WORKLIST_REMOVE(&newblk->nb_list); 5572 LIST_REMOVE(newblk, nb_hash); 5573 if ((freefrag = newblk->nb_freefrag) != NULL) { 5574 freefrag->ff_state |= COMPLETE; 5575 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 5576 add_to_worklist(&freefrag->ff_list, 0); 5577 } 5578 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL) { 5579 newdirblk = WK_NEWDIRBLK(wk); 5580 WORKLIST_REMOVE(&newdirblk->db_list); 5581 if (!LIST_EMPTY(&newblk->nb_newdirblk)) 5582 panic("free_newblk: extra newdirblk"); 5583 free_newdirblk(newdirblk); 5584 } 5585 while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL) { 5586 indirdep->ir_state |= DEPCOMPLETE; 5587 indirdep_complete(indirdep); 5588 } 5589 KASSERT(newblk->nb_jnewblk == NULL, 5590 ("free_newblk; jnewblk %p still attached", newblk->nb_jnewblk)); 5591 handle_jwork(&newblk->nb_jwork); 5592 newblk->nb_list.wk_type = D_NEWBLK; 5593 WORKITEM_FREE(newblk, D_NEWBLK); 5594 } 5595 5596 /* 5597 * Free a newdirblk. Clear the NEWBLOCK flag on its associated pagedep. 5598 * This routine must be called with splbio interrupts blocked. 5599 */ 5600 static void 5601 free_newdirblk(newdirblk) 5602 struct newdirblk *newdirblk; 5603 { 5604 struct pagedep *pagedep; 5605 struct diradd *dap; 5606 struct worklist *wk; 5607 int i; 5608 5609 mtx_assert(&lk, MA_OWNED); 5610 /* 5611 * If the pagedep is still linked onto the directory buffer 5612 * dependency chain, then some of the entries on the 5613 * pd_pendinghd list may not be committed to disk yet. In 5614 * this case, we will simply clear the NEWBLOCK flag and 5615 * let the pd_pendinghd list be processed when the pagedep 5616 * is next written. If the pagedep is no longer on the buffer 5617 * dependency chain, then all the entries on the pd_pending 5618 * list are committed to disk and we can free them here. 5619 */ 5620 pagedep = newdirblk->db_pagedep; 5621 pagedep->pd_state &= ~NEWBLOCK; 5622 if ((pagedep->pd_state & ONWORKLIST) == 0) 5623 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 5624 free_diradd(dap, NULL); 5625 /* 5626 * If no dependencies remain, the pagedep will be freed. 5627 */ 5628 for (i = 0; i < DAHASHSZ; i++) 5629 if (!LIST_EMPTY(&pagedep->pd_diraddhd[i])) 5630 break; 5631 if (i == DAHASHSZ && (pagedep->pd_state & ONWORKLIST) == 0 && 5632 LIST_EMPTY(&pagedep->pd_jmvrefhd)) { 5633 KASSERT(LIST_FIRST(&pagedep->pd_dirremhd) == NULL, 5634 ("free_newdirblk: Freeing non-free pagedep %p", pagedep)); 5635 LIST_REMOVE(pagedep, pd_hash); 5636 WORKITEM_FREE(pagedep, D_PAGEDEP); 5637 } 5638 /* Should only ever be one item in the list. */ 5639 while ((wk = LIST_FIRST(&newdirblk->db_mkdir)) != NULL) { 5640 WORKLIST_REMOVE(wk); 5641 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 5642 } 5643 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 5644 } 5645 5646 /* 5647 * Prepare an inode to be freed. The actual free operation is not 5648 * done until the zero'ed inode has been written to disk. 5649 */ 5650 void 5651 softdep_freefile(pvp, ino, mode) 5652 struct vnode *pvp; 5653 ino_t ino; 5654 int mode; 5655 { 5656 struct inode *ip = VTOI(pvp); 5657 struct inodedep *inodedep; 5658 struct freefile *freefile; 5659 5660 /* 5661 * This sets up the inode de-allocation dependency. 5662 */ 5663 freefile = malloc(sizeof(struct freefile), 5664 M_FREEFILE, M_SOFTDEP_FLAGS); 5665 workitem_alloc(&freefile->fx_list, D_FREEFILE, pvp->v_mount); 5666 freefile->fx_mode = mode; 5667 freefile->fx_oldinum = ino; 5668 freefile->fx_devvp = ip->i_devvp; 5669 LIST_INIT(&freefile->fx_jwork); 5670 UFS_LOCK(ip->i_ump); 5671 ip->i_fs->fs_pendinginodes += 1; 5672 UFS_UNLOCK(ip->i_ump); 5673 5674 /* 5675 * If the inodedep does not exist, then the zero'ed inode has 5676 * been written to disk. If the allocated inode has never been 5677 * written to disk, then the on-disk inode is zero'ed. In either 5678 * case we can free the file immediately. If the journal was 5679 * canceled before being written the inode will never make it to 5680 * disk and we must send the canceled journal entrys to 5681 * ffs_freefile() to be cleared in conjunction with the bitmap. 5682 * Any blocks waiting on the inode to write can be safely freed 5683 * here as it will never been written. 5684 */ 5685 ACQUIRE_LOCK(&lk); 5686 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 5687 /* 5688 * Remove this inode from the unlinked list and set 5689 * GOINGAWAY as appropriate to indicate that this inode 5690 * will never be written. 5691 */ 5692 if (inodedep && inodedep->id_state & UNLINKED) { 5693 /* 5694 * Save the journal work to be freed with the bitmap 5695 * before we clear UNLINKED. Otherwise it can be lost 5696 * if the inode block is written. 5697 */ 5698 handle_bufwait(inodedep, &freefile->fx_jwork); 5699 clear_unlinked_inodedep(inodedep); 5700 /* Re-acquire inodedep as we've dropped lk. */ 5701 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 5702 if (inodedep && (inodedep->id_state & DEPCOMPLETE) == 0) 5703 inodedep->id_state |= GOINGAWAY; 5704 } 5705 if (inodedep == NULL || check_inode_unwritten(inodedep)) { 5706 FREE_LOCK(&lk); 5707 handle_workitem_freefile(freefile); 5708 return; 5709 } 5710 WORKLIST_INSERT(&inodedep->id_inowait, &freefile->fx_list); 5711 FREE_LOCK(&lk); 5712 if (ip->i_number == ino) 5713 ip->i_flag |= IN_MODIFIED; 5714 } 5715 5716 /* 5717 * Check to see if an inode has never been written to disk. If 5718 * so free the inodedep and return success, otherwise return failure. 5719 * This routine must be called with splbio interrupts blocked. 5720 * 5721 * If we still have a bitmap dependency, then the inode has never 5722 * been written to disk. Drop the dependency as it is no longer 5723 * necessary since the inode is being deallocated. We set the 5724 * ALLCOMPLETE flags since the bitmap now properly shows that the 5725 * inode is not allocated. Even if the inode is actively being 5726 * written, it has been rolled back to its zero'ed state, so we 5727 * are ensured that a zero inode is what is on the disk. For short 5728 * lived files, this change will usually result in removing all the 5729 * dependencies from the inode so that it can be freed immediately. 5730 */ 5731 static int 5732 check_inode_unwritten(inodedep) 5733 struct inodedep *inodedep; 5734 { 5735 5736 mtx_assert(&lk, MA_OWNED); 5737 5738 if ((inodedep->id_state & (DEPCOMPLETE | UNLINKED)) != 0 || 5739 !LIST_EMPTY(&inodedep->id_pendinghd) || 5740 !LIST_EMPTY(&inodedep->id_bufwait) || 5741 !LIST_EMPTY(&inodedep->id_inowait) || 5742 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 5743 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 5744 !TAILQ_EMPTY(&inodedep->id_extupdt) || 5745 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 5746 inodedep->id_mkdiradd != NULL || 5747 inodedep->id_nlinkdelta != 0) 5748 return (0); 5749 /* 5750 * Another process might be in initiate_write_inodeblock_ufs[12] 5751 * trying to allocate memory without holding "Softdep Lock". 5752 */ 5753 if ((inodedep->id_state & IOSTARTED) != 0 && 5754 inodedep->id_savedino1 == NULL) 5755 return (0); 5756 5757 if (inodedep->id_state & ONDEPLIST) 5758 LIST_REMOVE(inodedep, id_deps); 5759 inodedep->id_state &= ~ONDEPLIST; 5760 inodedep->id_state |= ALLCOMPLETE; 5761 inodedep->id_bmsafemap = NULL; 5762 if (inodedep->id_state & ONWORKLIST) 5763 WORKLIST_REMOVE(&inodedep->id_list); 5764 if (inodedep->id_savedino1 != NULL) { 5765 free(inodedep->id_savedino1, M_SAVEDINO); 5766 inodedep->id_savedino1 = NULL; 5767 } 5768 if (free_inodedep(inodedep) == 0) 5769 panic("check_inode_unwritten: busy inode"); 5770 return (1); 5771 } 5772 5773 /* 5774 * Try to free an inodedep structure. Return 1 if it could be freed. 5775 */ 5776 static int 5777 free_inodedep(inodedep) 5778 struct inodedep *inodedep; 5779 { 5780 5781 mtx_assert(&lk, MA_OWNED); 5782 if ((inodedep->id_state & (ONWORKLIST | UNLINKED)) != 0 || 5783 (inodedep->id_state & ALLCOMPLETE) != ALLCOMPLETE || 5784 !LIST_EMPTY(&inodedep->id_dirremhd) || 5785 !LIST_EMPTY(&inodedep->id_pendinghd) || 5786 !LIST_EMPTY(&inodedep->id_bufwait) || 5787 !LIST_EMPTY(&inodedep->id_inowait) || 5788 !TAILQ_EMPTY(&inodedep->id_inoreflst) || 5789 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 5790 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 5791 !TAILQ_EMPTY(&inodedep->id_extupdt) || 5792 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 5793 inodedep->id_mkdiradd != NULL || 5794 inodedep->id_nlinkdelta != 0 || 5795 inodedep->id_savedino1 != NULL) 5796 return (0); 5797 if (inodedep->id_state & ONDEPLIST) 5798 LIST_REMOVE(inodedep, id_deps); 5799 LIST_REMOVE(inodedep, id_hash); 5800 WORKITEM_FREE(inodedep, D_INODEDEP); 5801 num_inodedep -= 1; 5802 return (1); 5803 } 5804 5805 /* 5806 * Free the block referenced by a freework structure. The parent freeblks 5807 * structure is released and completed when the final cg bitmap reaches 5808 * the disk. This routine may be freeing a jnewblk which never made it to 5809 * disk in which case we do not have to wait as the operation is undone 5810 * in memory immediately. 5811 */ 5812 static void 5813 freework_freeblock(freework) 5814 struct freework *freework; 5815 { 5816 struct freeblks *freeblks; 5817 struct ufsmount *ump; 5818 struct workhead wkhd; 5819 struct fs *fs; 5820 int complete; 5821 int pending; 5822 int bsize; 5823 int needj; 5824 5825 freeblks = freework->fw_freeblks; 5826 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 5827 fs = ump->um_fs; 5828 needj = freeblks->fb_list.wk_mp->mnt_kern_flag & MNTK_SUJ; 5829 complete = 0; 5830 LIST_INIT(&wkhd); 5831 /* 5832 * If we are canceling an existing jnewblk pass it to the free 5833 * routine, otherwise pass the freeblk which will ultimately 5834 * release the freeblks. If we're not journaling, we can just 5835 * free the freeblks immediately. 5836 */ 5837 if (!LIST_EMPTY(&freework->fw_jwork)) { 5838 LIST_SWAP(&wkhd, &freework->fw_jwork, worklist, wk_list); 5839 complete = 1; 5840 } else if (needj) 5841 WORKLIST_INSERT_UNLOCKED(&wkhd, &freework->fw_list); 5842 bsize = lfragtosize(fs, freework->fw_frags); 5843 pending = btodb(bsize); 5844 ACQUIRE_LOCK(&lk); 5845 freeblks->fb_chkcnt -= pending; 5846 FREE_LOCK(&lk); 5847 /* 5848 * extattr blocks don't show up in pending blocks. XXX why? 5849 */ 5850 if (freework->fw_lbn >= 0 || freework->fw_lbn <= -NDADDR) { 5851 UFS_LOCK(ump); 5852 fs->fs_pendingblocks -= pending; 5853 UFS_UNLOCK(ump); 5854 } 5855 ffs_blkfree(ump, fs, freeblks->fb_devvp, freework->fw_blkno, 5856 bsize, freeblks->fb_previousinum, &wkhd); 5857 if (complete == 0 && needj) 5858 return; 5859 /* 5860 * The jnewblk will be discarded and the bits in the map never 5861 * made it to disk. We can immediately free the freeblk. 5862 */ 5863 ACQUIRE_LOCK(&lk); 5864 handle_written_freework(freework); 5865 FREE_LOCK(&lk); 5866 } 5867 5868 /* 5869 * Start, continue, or finish the process of freeing an indirect block tree. 5870 * The free operation may be paused at any point with fw_off containing the 5871 * offset to restart from. This enables us to implement some flow control 5872 * for large truncates which may fan out and generate a huge number of 5873 * dependencies. 5874 */ 5875 static void 5876 handle_workitem_indirblk(freework) 5877 struct freework *freework; 5878 { 5879 struct freeblks *freeblks; 5880 struct ufsmount *ump; 5881 struct fs *fs; 5882 5883 5884 freeblks = freework->fw_freeblks; 5885 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 5886 fs = ump->um_fs; 5887 if (freework->fw_off == NINDIR(fs)) 5888 freework_freeblock(freework); 5889 else 5890 indir_trunc(freework, fsbtodb(fs, freework->fw_blkno), 5891 freework->fw_lbn); 5892 } 5893 5894 /* 5895 * Called when a freework structure attached to a cg buf is written. The 5896 * ref on either the parent or the freeblks structure is released and 5897 * either may be added to the worklist if it is the final ref. 5898 */ 5899 static void 5900 handle_written_freework(freework) 5901 struct freework *freework; 5902 { 5903 struct freeblks *freeblks; 5904 struct freework *parent; 5905 5906 freeblks = freework->fw_freeblks; 5907 parent = freework->fw_parent; 5908 if (parent) { 5909 if (--parent->fw_ref != 0) 5910 parent = NULL; 5911 freeblks = NULL; 5912 } else if (--freeblks->fb_ref != 0) 5913 freeblks = NULL; 5914 WORKITEM_FREE(freework, D_FREEWORK); 5915 /* 5916 * Don't delay these block frees or it takes an intolerable amount 5917 * of time to process truncates and free their journal entries. 5918 */ 5919 if (freeblks) 5920 add_to_worklist(&freeblks->fb_list, 1); 5921 if (parent) 5922 add_to_worklist(&parent->fw_list, 1); 5923 } 5924 5925 /* 5926 * This workitem routine performs the block de-allocation. 5927 * The workitem is added to the pending list after the updated 5928 * inode block has been written to disk. As mentioned above, 5929 * checks regarding the number of blocks de-allocated (compared 5930 * to the number of blocks allocated for the file) are also 5931 * performed in this function. 5932 */ 5933 static void 5934 handle_workitem_freeblocks(freeblks, flags) 5935 struct freeblks *freeblks; 5936 int flags; 5937 { 5938 struct freework *freework; 5939 struct worklist *wk; 5940 5941 KASSERT(LIST_EMPTY(&freeblks->fb_jfreeblkhd), 5942 ("handle_workitem_freeblocks: Journal entries not written.")); 5943 if (LIST_EMPTY(&freeblks->fb_freeworkhd)) { 5944 handle_complete_freeblocks(freeblks); 5945 return; 5946 } 5947 freeblks->fb_ref++; 5948 while ((wk = LIST_FIRST(&freeblks->fb_freeworkhd)) != NULL) { 5949 KASSERT(wk->wk_type == D_FREEWORK, 5950 ("handle_workitem_freeblocks: Unknown type %s", 5951 TYPENAME(wk->wk_type))); 5952 WORKLIST_REMOVE_UNLOCKED(wk); 5953 freework = WK_FREEWORK(wk); 5954 if (freework->fw_lbn <= -NDADDR) 5955 handle_workitem_indirblk(freework); 5956 else 5957 freework_freeblock(freework); 5958 } 5959 ACQUIRE_LOCK(&lk); 5960 if (--freeblks->fb_ref != 0) 5961 freeblks = NULL; 5962 FREE_LOCK(&lk); 5963 if (freeblks) 5964 handle_complete_freeblocks(freeblks); 5965 } 5966 5967 /* 5968 * Once all of the freework workitems are complete we can retire the 5969 * freeblocks dependency and any journal work awaiting completion. This 5970 * can not be called until all other dependencies are stable on disk. 5971 */ 5972 static void 5973 handle_complete_freeblocks(freeblks) 5974 struct freeblks *freeblks; 5975 { 5976 struct inode *ip; 5977 struct vnode *vp; 5978 struct fs *fs; 5979 struct ufsmount *ump; 5980 int flags; 5981 5982 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 5983 fs = ump->um_fs; 5984 flags = LK_NOWAIT; 5985 5986 /* 5987 * If we still have not finished background cleanup, then check 5988 * to see if the block count needs to be adjusted. 5989 */ 5990 if (freeblks->fb_chkcnt != 0 && (fs->fs_flags & FS_UNCLEAN) != 0 && 5991 ffs_vgetf(freeblks->fb_list.wk_mp, freeblks->fb_previousinum, 5992 (flags & LK_NOWAIT) | LK_EXCLUSIVE, &vp, FFSV_FORCEINSMQ) == 0) { 5993 ip = VTOI(vp); 5994 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + freeblks->fb_chkcnt); 5995 ip->i_flag |= IN_CHANGE; 5996 vput(vp); 5997 } 5998 5999 #ifdef INVARIANTS 6000 if (freeblks->fb_chkcnt != 0 && 6001 ((fs->fs_flags & FS_UNCLEAN) == 0 || (flags & LK_NOWAIT) != 0)) 6002 printf("handle_workitem_freeblocks: block count\n"); 6003 #endif /* INVARIANTS */ 6004 6005 ACQUIRE_LOCK(&lk); 6006 /* 6007 * All of the freeblock deps must be complete prior to this call 6008 * so it's now safe to complete earlier outstanding journal entries. 6009 */ 6010 handle_jwork(&freeblks->fb_jwork); 6011 WORKITEM_FREE(freeblks, D_FREEBLKS); 6012 num_freeblkdep--; 6013 FREE_LOCK(&lk); 6014 } 6015 6016 /* 6017 * Release blocks associated with the inode ip and stored in the indirect 6018 * block dbn. If level is greater than SINGLE, the block is an indirect block 6019 * and recursive calls to indirtrunc must be used to cleanse other indirect 6020 * blocks. 6021 */ 6022 static void 6023 indir_trunc(freework, dbn, lbn) 6024 struct freework *freework; 6025 ufs2_daddr_t dbn; 6026 ufs_lbn_t lbn; 6027 { 6028 struct freework *nfreework; 6029 struct workhead wkhd; 6030 struct jnewblk *jnewblk; 6031 struct freeblks *freeblks; 6032 struct buf *bp; 6033 struct fs *fs; 6034 struct worklist *wkn; 6035 struct worklist *wk; 6036 struct indirdep *indirdep; 6037 struct ufsmount *ump; 6038 ufs1_daddr_t *bap1 = 0; 6039 ufs2_daddr_t nb, nnb, *bap2 = 0; 6040 ufs_lbn_t lbnadd; 6041 int i, nblocks, ufs1fmt; 6042 int fs_pendingblocks; 6043 int freedeps; 6044 int needj; 6045 int level; 6046 int cnt; 6047 6048 LIST_INIT(&wkhd); 6049 level = lbn_level(lbn); 6050 if (level == -1) 6051 panic("indir_trunc: Invalid lbn %jd\n", lbn); 6052 freeblks = freework->fw_freeblks; 6053 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 6054 fs = ump->um_fs; 6055 fs_pendingblocks = 0; 6056 freedeps = 0; 6057 needj = UFSTOVFS(ump)->mnt_kern_flag & MNTK_SUJ; 6058 lbnadd = 1; 6059 for (i = level; i > 0; i--) 6060 lbnadd *= NINDIR(fs); 6061 /* 6062 * Get buffer of block pointers to be freed. This routine is not 6063 * called until the zero'ed inode has been written, so it is safe 6064 * to free blocks as they are encountered. Because the inode has 6065 * been zero'ed, calls to bmap on these blocks will fail. So, we 6066 * have to use the on-disk address and the block device for the 6067 * filesystem to look them up. If the file was deleted before its 6068 * indirect blocks were all written to disk, the routine that set 6069 * us up (deallocate_dependencies) will have arranged to leave 6070 * a complete copy of the indirect block in memory for our use. 6071 * Otherwise we have to read the blocks in from the disk. 6072 */ 6073 #ifdef notyet 6074 bp = getblk(freeblks->fb_devvp, dbn, (int)fs->fs_bsize, 0, 0, 6075 GB_NOCREAT); 6076 #else 6077 bp = incore(&freeblks->fb_devvp->v_bufobj, dbn); 6078 #endif 6079 ACQUIRE_LOCK(&lk); 6080 if (bp != NULL && (wk = LIST_FIRST(&bp->b_dep)) != NULL) { 6081 if (wk->wk_type != D_INDIRDEP || 6082 (wk->wk_state & GOINGAWAY) == 0) 6083 panic("indir_trunc: lost indirdep %p", wk); 6084 indirdep = WK_INDIRDEP(wk); 6085 LIST_SWAP(&wkhd, &indirdep->ir_jwork, worklist, wk_list); 6086 free_indirdep(indirdep); 6087 if (!LIST_EMPTY(&bp->b_dep)) 6088 panic("indir_trunc: dangling dep %p", 6089 LIST_FIRST(&bp->b_dep)); 6090 ump->um_numindirdeps -= 1; 6091 FREE_LOCK(&lk); 6092 } else { 6093 #ifdef notyet 6094 if (bp) 6095 brelse(bp); 6096 #endif 6097 FREE_LOCK(&lk); 6098 if (bread(freeblks->fb_devvp, dbn, (int)fs->fs_bsize, 6099 NOCRED, &bp) != 0) { 6100 brelse(bp); 6101 return; 6102 } 6103 } 6104 /* 6105 * Recursively free indirect blocks. 6106 */ 6107 if (ump->um_fstype == UFS1) { 6108 ufs1fmt = 1; 6109 bap1 = (ufs1_daddr_t *)bp->b_data; 6110 } else { 6111 ufs1fmt = 0; 6112 bap2 = (ufs2_daddr_t *)bp->b_data; 6113 } 6114 /* 6115 * Reclaim indirect blocks which never made it to disk. 6116 */ 6117 cnt = 0; 6118 LIST_FOREACH_SAFE(wk, &wkhd, wk_list, wkn) { 6119 struct workhead freewk; 6120 if (wk->wk_type != D_JNEWBLK) 6121 continue; 6122 WORKLIST_REMOVE_UNLOCKED(wk); 6123 LIST_INIT(&freewk); 6124 WORKLIST_INSERT_UNLOCKED(&freewk, wk); 6125 jnewblk = WK_JNEWBLK(wk); 6126 if (jnewblk->jn_lbn > 0) 6127 i = (jnewblk->jn_lbn - -lbn) / lbnadd; 6128 else 6129 i = (jnewblk->jn_lbn - (lbn + 1)) / lbnadd; 6130 KASSERT(i >= 0 && i < NINDIR(fs), 6131 ("indir_trunc: Index out of range %d parent %jd lbn %jd", 6132 i, lbn, jnewblk->jn_lbn)); 6133 /* Clear the pointer so it isn't found below. */ 6134 if (ufs1fmt) { 6135 nb = bap1[i]; 6136 bap1[i] = 0; 6137 } else { 6138 nb = bap2[i]; 6139 bap2[i] = 0; 6140 } 6141 KASSERT(nb == jnewblk->jn_blkno, 6142 ("indir_trunc: Block mismatch %jd != %jd", 6143 nb, jnewblk->jn_blkno)); 6144 ffs_blkfree(ump, fs, freeblks->fb_devvp, jnewblk->jn_blkno, 6145 fs->fs_bsize, freeblks->fb_previousinum, &freewk); 6146 cnt++; 6147 } 6148 ACQUIRE_LOCK(&lk); 6149 if (needj) 6150 freework->fw_ref += NINDIR(fs) + 1; 6151 /* Any remaining journal work can be completed with freeblks. */ 6152 jwork_move(&freeblks->fb_jwork, &wkhd); 6153 FREE_LOCK(&lk); 6154 nblocks = btodb(fs->fs_bsize); 6155 if (ufs1fmt) 6156 nb = bap1[0]; 6157 else 6158 nb = bap2[0]; 6159 nfreework = freework; 6160 /* 6161 * Reclaim on disk blocks. 6162 */ 6163 for (i = freework->fw_off; i < NINDIR(fs); i++, nb = nnb) { 6164 if (i != NINDIR(fs) - 1) { 6165 if (ufs1fmt) 6166 nnb = bap1[i+1]; 6167 else 6168 nnb = bap2[i+1]; 6169 } else 6170 nnb = 0; 6171 if (nb == 0) 6172 continue; 6173 cnt++; 6174 if (level != 0) { 6175 ufs_lbn_t nlbn; 6176 6177 nlbn = (lbn + 1) - (i * lbnadd); 6178 if (needj != 0) { 6179 nfreework = newfreework(freeblks, freework, 6180 nlbn, nb, fs->fs_frag, 0); 6181 freedeps++; 6182 } 6183 indir_trunc(nfreework, fsbtodb(fs, nb), nlbn); 6184 } else { 6185 struct freedep *freedep; 6186 6187 /* 6188 * Attempt to aggregate freedep dependencies for 6189 * all blocks being released to the same CG. 6190 */ 6191 LIST_INIT(&wkhd); 6192 if (needj != 0 && 6193 (nnb == 0 || (dtog(fs, nb) != dtog(fs, nnb)))) { 6194 freedep = newfreedep(freework); 6195 WORKLIST_INSERT_UNLOCKED(&wkhd, 6196 &freedep->fd_list); 6197 freedeps++; 6198 } 6199 ffs_blkfree(ump, fs, freeblks->fb_devvp, nb, 6200 fs->fs_bsize, freeblks->fb_previousinum, &wkhd); 6201 } 6202 } 6203 if (level == 0) 6204 fs_pendingblocks = (nblocks * cnt); 6205 /* 6206 * If we're not journaling we can free the indirect now. Otherwise 6207 * setup the ref counts and offset so this indirect can be completed 6208 * when its children are free. 6209 */ 6210 if (needj == 0) { 6211 fs_pendingblocks += nblocks; 6212 dbn = dbtofsb(fs, dbn); 6213 ffs_blkfree(ump, fs, freeblks->fb_devvp, dbn, fs->fs_bsize, 6214 freeblks->fb_previousinum, NULL); 6215 ACQUIRE_LOCK(&lk); 6216 freeblks->fb_chkcnt -= fs_pendingblocks; 6217 if (freework->fw_blkno == dbn) 6218 handle_written_freework(freework); 6219 FREE_LOCK(&lk); 6220 freework = NULL; 6221 } else { 6222 ACQUIRE_LOCK(&lk); 6223 freework->fw_off = i; 6224 freework->fw_ref += freedeps; 6225 freework->fw_ref -= NINDIR(fs) + 1; 6226 if (freework->fw_ref != 0) 6227 freework = NULL; 6228 freeblks->fb_chkcnt -= fs_pendingblocks; 6229 FREE_LOCK(&lk); 6230 } 6231 if (fs_pendingblocks) { 6232 UFS_LOCK(ump); 6233 fs->fs_pendingblocks -= fs_pendingblocks; 6234 UFS_UNLOCK(ump); 6235 } 6236 bp->b_flags |= B_INVAL | B_NOCACHE; 6237 brelse(bp); 6238 if (freework) 6239 handle_workitem_indirblk(freework); 6240 return; 6241 } 6242 6243 /* 6244 * Cancel an allocindir when it is removed via truncation. 6245 */ 6246 static void 6247 cancel_allocindir(aip, inodedep, freeblks) 6248 struct allocindir *aip; 6249 struct inodedep *inodedep; 6250 struct freeblks *freeblks; 6251 { 6252 struct newblk *newblk; 6253 6254 /* 6255 * If the journal hasn't been written the jnewblk must be passed 6256 * to the call to ffs_freeblk that reclaims the space. We accomplish 6257 * this by linking the journal dependency into the indirdep to be 6258 * freed when indir_trunc() is called. If the journal has already 6259 * been written we can simply reclaim the journal space when the 6260 * freeblks work is complete. 6261 */ 6262 LIST_REMOVE(aip, ai_next); 6263 newblk = (struct newblk *)aip; 6264 if (newblk->nb_jnewblk == NULL) 6265 cancel_newblk(newblk, &freeblks->fb_jwork); 6266 else 6267 cancel_newblk(newblk, &aip->ai_indirdep->ir_jwork); 6268 if (inodedep && inodedep->id_state & DEPCOMPLETE) 6269 WORKLIST_INSERT(&inodedep->id_bufwait, &newblk->nb_list); 6270 else 6271 free_newblk(newblk); 6272 } 6273 6274 /* 6275 * Create the mkdir dependencies for . and .. in a new directory. Link them 6276 * in to a newdirblk so any subsequent additions are tracked properly. The 6277 * caller is responsible for adding the mkdir1 dependency to the journal 6278 * and updating id_mkdiradd. This function returns with lk held. 6279 */ 6280 static struct mkdir * 6281 setup_newdir(dap, newinum, dinum, newdirbp, mkdirp) 6282 struct diradd *dap; 6283 ino_t newinum; 6284 ino_t dinum; 6285 struct buf *newdirbp; 6286 struct mkdir **mkdirp; 6287 { 6288 struct newblk *newblk; 6289 struct pagedep *pagedep; 6290 struct inodedep *inodedep; 6291 struct newdirblk *newdirblk = 0; 6292 struct mkdir *mkdir1, *mkdir2; 6293 struct worklist *wk; 6294 struct jaddref *jaddref; 6295 struct mount *mp; 6296 6297 mp = dap->da_list.wk_mp; 6298 newdirblk = malloc(sizeof(struct newdirblk), M_NEWDIRBLK, 6299 M_SOFTDEP_FLAGS); 6300 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 6301 LIST_INIT(&newdirblk->db_mkdir); 6302 mkdir1 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 6303 workitem_alloc(&mkdir1->md_list, D_MKDIR, mp); 6304 mkdir1->md_state = ATTACHED | MKDIR_BODY; 6305 mkdir1->md_diradd = dap; 6306 mkdir1->md_jaddref = NULL; 6307 mkdir2 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 6308 workitem_alloc(&mkdir2->md_list, D_MKDIR, mp); 6309 mkdir2->md_state = ATTACHED | MKDIR_PARENT; 6310 mkdir2->md_diradd = dap; 6311 mkdir2->md_jaddref = NULL; 6312 if ((mp->mnt_kern_flag & MNTK_SUJ) == 0) { 6313 mkdir1->md_state |= DEPCOMPLETE; 6314 mkdir2->md_state |= DEPCOMPLETE; 6315 } 6316 /* 6317 * Dependency on "." and ".." being written to disk. 6318 */ 6319 mkdir1->md_buf = newdirbp; 6320 ACQUIRE_LOCK(&lk); 6321 LIST_INSERT_HEAD(&mkdirlisthd, mkdir1, md_mkdirs); 6322 /* 6323 * We must link the pagedep, allocdirect, and newdirblk for 6324 * the initial file page so the pointer to the new directory 6325 * is not written until the directory contents are live and 6326 * any subsequent additions are not marked live until the 6327 * block is reachable via the inode. 6328 */ 6329 if (pagedep_lookup(mp, newinum, 0, 0, &pagedep) == 0) 6330 panic("setup_newdir: lost pagedep"); 6331 LIST_FOREACH(wk, &newdirbp->b_dep, wk_list) 6332 if (wk->wk_type == D_ALLOCDIRECT) 6333 break; 6334 if (wk == NULL) 6335 panic("setup_newdir: lost allocdirect"); 6336 newblk = WK_NEWBLK(wk); 6337 pagedep->pd_state |= NEWBLOCK; 6338 pagedep->pd_newdirblk = newdirblk; 6339 newdirblk->db_pagedep = pagedep; 6340 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 6341 WORKLIST_INSERT(&newdirblk->db_mkdir, &mkdir1->md_list); 6342 /* 6343 * Look up the inodedep for the parent directory so that we 6344 * can link mkdir2 into the pending dotdot jaddref or 6345 * the inode write if there is none. If the inode is 6346 * ALLCOMPLETE and no jaddref is present all dependencies have 6347 * been satisfied and mkdir2 can be freed. 6348 */ 6349 inodedep_lookup(mp, dinum, 0, &inodedep); 6350 if (mp->mnt_kern_flag & MNTK_SUJ) { 6351 if (inodedep == NULL) 6352 panic("setup_newdir: Lost parent."); 6353 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 6354 inoreflst); 6355 KASSERT(jaddref != NULL && jaddref->ja_parent == newinum && 6356 (jaddref->ja_state & MKDIR_PARENT), 6357 ("setup_newdir: bad dotdot jaddref %p", jaddref)); 6358 LIST_INSERT_HEAD(&mkdirlisthd, mkdir2, md_mkdirs); 6359 mkdir2->md_jaddref = jaddref; 6360 jaddref->ja_mkdir = mkdir2; 6361 } else if (inodedep == NULL || 6362 (inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 6363 dap->da_state &= ~MKDIR_PARENT; 6364 WORKITEM_FREE(mkdir2, D_MKDIR); 6365 } else { 6366 LIST_INSERT_HEAD(&mkdirlisthd, mkdir2, md_mkdirs); 6367 WORKLIST_INSERT(&inodedep->id_bufwait,&mkdir2->md_list); 6368 } 6369 *mkdirp = mkdir2; 6370 6371 return (mkdir1); 6372 } 6373 6374 /* 6375 * Directory entry addition dependencies. 6376 * 6377 * When adding a new directory entry, the inode (with its incremented link 6378 * count) must be written to disk before the directory entry's pointer to it. 6379 * Also, if the inode is newly allocated, the corresponding freemap must be 6380 * updated (on disk) before the directory entry's pointer. These requirements 6381 * are met via undo/redo on the directory entry's pointer, which consists 6382 * simply of the inode number. 6383 * 6384 * As directory entries are added and deleted, the free space within a 6385 * directory block can become fragmented. The ufs filesystem will compact 6386 * a fragmented directory block to make space for a new entry. When this 6387 * occurs, the offsets of previously added entries change. Any "diradd" 6388 * dependency structures corresponding to these entries must be updated with 6389 * the new offsets. 6390 */ 6391 6392 /* 6393 * This routine is called after the in-memory inode's link 6394 * count has been incremented, but before the directory entry's 6395 * pointer to the inode has been set. 6396 */ 6397 int 6398 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 6399 struct buf *bp; /* buffer containing directory block */ 6400 struct inode *dp; /* inode for directory */ 6401 off_t diroffset; /* offset of new entry in directory */ 6402 ino_t newinum; /* inode referenced by new directory entry */ 6403 struct buf *newdirbp; /* non-NULL => contents of new mkdir */ 6404 int isnewblk; /* entry is in a newly allocated block */ 6405 { 6406 int offset; /* offset of new entry within directory block */ 6407 ufs_lbn_t lbn; /* block in directory containing new entry */ 6408 struct fs *fs; 6409 struct diradd *dap; 6410 struct newblk *newblk; 6411 struct pagedep *pagedep; 6412 struct inodedep *inodedep; 6413 struct newdirblk *newdirblk = 0; 6414 struct mkdir *mkdir1, *mkdir2; 6415 struct jaddref *jaddref; 6416 struct mount *mp; 6417 int isindir; 6418 6419 /* 6420 * Whiteouts have no dependencies. 6421 */ 6422 if (newinum == WINO) { 6423 if (newdirbp != NULL) 6424 bdwrite(newdirbp); 6425 return (0); 6426 } 6427 jaddref = NULL; 6428 mkdir1 = mkdir2 = NULL; 6429 mp = UFSTOVFS(dp->i_ump); 6430 fs = dp->i_fs; 6431 lbn = lblkno(fs, diroffset); 6432 offset = blkoff(fs, diroffset); 6433 dap = malloc(sizeof(struct diradd), M_DIRADD, 6434 M_SOFTDEP_FLAGS|M_ZERO); 6435 workitem_alloc(&dap->da_list, D_DIRADD, mp); 6436 dap->da_offset = offset; 6437 dap->da_newinum = newinum; 6438 dap->da_state = ATTACHED; 6439 LIST_INIT(&dap->da_jwork); 6440 isindir = bp->b_lblkno >= NDADDR; 6441 if (isnewblk && 6442 (isindir ? blkoff(fs, diroffset) : fragoff(fs, diroffset)) == 0) { 6443 newdirblk = malloc(sizeof(struct newdirblk), 6444 M_NEWDIRBLK, M_SOFTDEP_FLAGS); 6445 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 6446 LIST_INIT(&newdirblk->db_mkdir); 6447 } 6448 /* 6449 * If we're creating a new directory setup the dependencies and set 6450 * the dap state to wait for them. Otherwise it's COMPLETE and 6451 * we can move on. 6452 */ 6453 if (newdirbp == NULL) { 6454 dap->da_state |= DEPCOMPLETE; 6455 ACQUIRE_LOCK(&lk); 6456 } else { 6457 dap->da_state |= MKDIR_BODY | MKDIR_PARENT; 6458 mkdir1 = setup_newdir(dap, newinum, dp->i_number, newdirbp, 6459 &mkdir2); 6460 } 6461 /* 6462 * Link into parent directory pagedep to await its being written. 6463 */ 6464 if (pagedep_lookup(mp, dp->i_number, lbn, DEPALLOC, &pagedep) == 0) 6465 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 6466 #ifdef DEBUG 6467 if (diradd_lookup(pagedep, offset) != NULL) 6468 panic("softdep_setup_directory_add: %p already at off %d\n", 6469 diradd_lookup(pagedep, offset), offset); 6470 #endif 6471 dap->da_pagedep = pagedep; 6472 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], dap, 6473 da_pdlist); 6474 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 6475 /* 6476 * If we're journaling, link the diradd into the jaddref so it 6477 * may be completed after the journal entry is written. Otherwise, 6478 * link the diradd into its inodedep. If the inode is not yet 6479 * written place it on the bufwait list, otherwise do the post-inode 6480 * write processing to put it on the id_pendinghd list. 6481 */ 6482 if (mp->mnt_kern_flag & MNTK_SUJ) { 6483 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 6484 inoreflst); 6485 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 6486 ("softdep_setup_directory_add: bad jaddref %p", jaddref)); 6487 jaddref->ja_diroff = diroffset; 6488 jaddref->ja_diradd = dap; 6489 add_to_journal(&jaddref->ja_list); 6490 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) 6491 diradd_inode_written(dap, inodedep); 6492 else 6493 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 6494 /* 6495 * Add the journal entries for . and .. links now that the primary 6496 * link is written. 6497 */ 6498 if (mkdir1 != NULL && mp->mnt_kern_flag & MNTK_SUJ) { 6499 jaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref, 6500 inoreflst, if_deps); 6501 KASSERT(jaddref != NULL && 6502 jaddref->ja_ino == jaddref->ja_parent && 6503 (jaddref->ja_state & MKDIR_BODY), 6504 ("softdep_setup_directory_add: bad dot jaddref %p", 6505 jaddref)); 6506 mkdir1->md_jaddref = jaddref; 6507 jaddref->ja_mkdir = mkdir1; 6508 /* 6509 * It is important that the dotdot journal entry 6510 * is added prior to the dot entry since dot writes 6511 * both the dot and dotdot links. These both must 6512 * be added after the primary link for the journal 6513 * to remain consistent. 6514 */ 6515 add_to_journal(&mkdir2->md_jaddref->ja_list); 6516 add_to_journal(&jaddref->ja_list); 6517 } 6518 /* 6519 * If we are adding a new directory remember this diradd so that if 6520 * we rename it we can keep the dot and dotdot dependencies. If 6521 * we are adding a new name for an inode that has a mkdiradd we 6522 * must be in rename and we have to move the dot and dotdot 6523 * dependencies to this new name. The old name is being orphaned 6524 * soon. 6525 */ 6526 if (mkdir1 != NULL) { 6527 if (inodedep->id_mkdiradd != NULL) 6528 panic("softdep_setup_directory_add: Existing mkdir"); 6529 inodedep->id_mkdiradd = dap; 6530 } else if (inodedep->id_mkdiradd) 6531 merge_diradd(inodedep, dap); 6532 if (newdirblk) { 6533 /* 6534 * There is nothing to do if we are already tracking 6535 * this block. 6536 */ 6537 if ((pagedep->pd_state & NEWBLOCK) != 0) { 6538 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 6539 FREE_LOCK(&lk); 6540 return (0); 6541 } 6542 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk) 6543 == 0) 6544 panic("softdep_setup_directory_add: lost entry"); 6545 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 6546 pagedep->pd_state |= NEWBLOCK; 6547 pagedep->pd_newdirblk = newdirblk; 6548 newdirblk->db_pagedep = pagedep; 6549 FREE_LOCK(&lk); 6550 /* 6551 * If we extended into an indirect signal direnter to sync. 6552 */ 6553 if (isindir) 6554 return (1); 6555 return (0); 6556 } 6557 FREE_LOCK(&lk); 6558 return (0); 6559 } 6560 6561 /* 6562 * This procedure is called to change the offset of a directory 6563 * entry when compacting a directory block which must be owned 6564 * exclusively by the caller. Note that the actual entry movement 6565 * must be done in this procedure to ensure that no I/O completions 6566 * occur while the move is in progress. 6567 */ 6568 void 6569 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 6570 struct buf *bp; /* Buffer holding directory block. */ 6571 struct inode *dp; /* inode for directory */ 6572 caddr_t base; /* address of dp->i_offset */ 6573 caddr_t oldloc; /* address of old directory location */ 6574 caddr_t newloc; /* address of new directory location */ 6575 int entrysize; /* size of directory entry */ 6576 { 6577 int offset, oldoffset, newoffset; 6578 struct pagedep *pagedep; 6579 struct jmvref *jmvref; 6580 struct diradd *dap; 6581 struct direct *de; 6582 struct mount *mp; 6583 ufs_lbn_t lbn; 6584 int flags; 6585 6586 mp = UFSTOVFS(dp->i_ump); 6587 de = (struct direct *)oldloc; 6588 jmvref = NULL; 6589 flags = 0; 6590 /* 6591 * Moves are always journaled as it would be too complex to 6592 * determine if any affected adds or removes are present in the 6593 * journal. 6594 */ 6595 if (mp->mnt_kern_flag & MNTK_SUJ) { 6596 flags = DEPALLOC; 6597 jmvref = newjmvref(dp, de->d_ino, 6598 dp->i_offset + (oldloc - base), 6599 dp->i_offset + (newloc - base)); 6600 } 6601 lbn = lblkno(dp->i_fs, dp->i_offset); 6602 offset = blkoff(dp->i_fs, dp->i_offset); 6603 oldoffset = offset + (oldloc - base); 6604 newoffset = offset + (newloc - base); 6605 ACQUIRE_LOCK(&lk); 6606 if (pagedep_lookup(mp, dp->i_number, lbn, flags, &pagedep) == 0) { 6607 if (pagedep) 6608 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 6609 goto done; 6610 } 6611 dap = diradd_lookup(pagedep, oldoffset); 6612 if (dap) { 6613 dap->da_offset = newoffset; 6614 newoffset = DIRADDHASH(newoffset); 6615 oldoffset = DIRADDHASH(oldoffset); 6616 if ((dap->da_state & ALLCOMPLETE) != ALLCOMPLETE && 6617 newoffset != oldoffset) { 6618 LIST_REMOVE(dap, da_pdlist); 6619 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[newoffset], 6620 dap, da_pdlist); 6621 } 6622 } 6623 done: 6624 if (jmvref) { 6625 jmvref->jm_pagedep = pagedep; 6626 LIST_INSERT_HEAD(&pagedep->pd_jmvrefhd, jmvref, jm_deps); 6627 add_to_journal(&jmvref->jm_list); 6628 } 6629 bcopy(oldloc, newloc, entrysize); 6630 FREE_LOCK(&lk); 6631 } 6632 6633 /* 6634 * Move the mkdir dependencies and journal work from one diradd to another 6635 * when renaming a directory. The new name must depend on the mkdir deps 6636 * completing as the old name did. Directories can only have one valid link 6637 * at a time so one must be canonical. 6638 */ 6639 static void 6640 merge_diradd(inodedep, newdap) 6641 struct inodedep *inodedep; 6642 struct diradd *newdap; 6643 { 6644 struct diradd *olddap; 6645 struct mkdir *mkdir, *nextmd; 6646 short state; 6647 6648 olddap = inodedep->id_mkdiradd; 6649 inodedep->id_mkdiradd = newdap; 6650 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 6651 newdap->da_state &= ~DEPCOMPLETE; 6652 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; mkdir = nextmd) { 6653 nextmd = LIST_NEXT(mkdir, md_mkdirs); 6654 if (mkdir->md_diradd != olddap) 6655 continue; 6656 mkdir->md_diradd = newdap; 6657 state = mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY); 6658 newdap->da_state |= state; 6659 olddap->da_state &= ~state; 6660 if ((olddap->da_state & 6661 (MKDIR_PARENT | MKDIR_BODY)) == 0) 6662 break; 6663 } 6664 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 6665 panic("merge_diradd: unfound ref"); 6666 } 6667 /* 6668 * Any mkdir related journal items are not safe to be freed until 6669 * the new name is stable. 6670 */ 6671 jwork_move(&newdap->da_jwork, &olddap->da_jwork); 6672 olddap->da_state |= DEPCOMPLETE; 6673 complete_diradd(olddap); 6674 } 6675 6676 /* 6677 * Move the diradd to the pending list when all diradd dependencies are 6678 * complete. 6679 */ 6680 static void 6681 complete_diradd(dap) 6682 struct diradd *dap; 6683 { 6684 struct pagedep *pagedep; 6685 6686 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 6687 if (dap->da_state & DIRCHG) 6688 pagedep = dap->da_previous->dm_pagedep; 6689 else 6690 pagedep = dap->da_pagedep; 6691 LIST_REMOVE(dap, da_pdlist); 6692 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 6693 } 6694 } 6695 6696 /* 6697 * Cancel a diradd when a dirrem overlaps with it. We must cancel the journal 6698 * add entries and conditonally journal the remove. 6699 */ 6700 static void 6701 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref) 6702 struct diradd *dap; 6703 struct dirrem *dirrem; 6704 struct jremref *jremref; 6705 struct jremref *dotremref; 6706 struct jremref *dotdotremref; 6707 { 6708 struct inodedep *inodedep; 6709 struct jaddref *jaddref; 6710 struct inoref *inoref; 6711 struct mkdir *mkdir; 6712 6713 /* 6714 * If no remove references were allocated we're on a non-journaled 6715 * filesystem and can skip the cancel step. 6716 */ 6717 if (jremref == NULL) { 6718 free_diradd(dap, NULL); 6719 return; 6720 } 6721 /* 6722 * Cancel the primary name an free it if it does not require 6723 * journaling. 6724 */ 6725 if (inodedep_lookup(dap->da_list.wk_mp, dap->da_newinum, 6726 0, &inodedep) != 0) { 6727 /* Abort the addref that reference this diradd. */ 6728 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 6729 if (inoref->if_list.wk_type != D_JADDREF) 6730 continue; 6731 jaddref = (struct jaddref *)inoref; 6732 if (jaddref->ja_diradd != dap) 6733 continue; 6734 if (cancel_jaddref(jaddref, inodedep, 6735 &dirrem->dm_jwork) == 0) { 6736 free_jremref(jremref); 6737 jremref = NULL; 6738 } 6739 break; 6740 } 6741 } 6742 /* 6743 * Cancel subordinate names and free them if they do not require 6744 * journaling. 6745 */ 6746 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 6747 LIST_FOREACH(mkdir, &mkdirlisthd, md_mkdirs) { 6748 if (mkdir->md_diradd != dap) 6749 continue; 6750 if ((jaddref = mkdir->md_jaddref) == NULL) 6751 continue; 6752 mkdir->md_jaddref = NULL; 6753 if (mkdir->md_state & MKDIR_PARENT) { 6754 if (cancel_jaddref(jaddref, NULL, 6755 &dirrem->dm_jwork) == 0) { 6756 free_jremref(dotdotremref); 6757 dotdotremref = NULL; 6758 } 6759 } else { 6760 if (cancel_jaddref(jaddref, inodedep, 6761 &dirrem->dm_jwork) == 0) { 6762 free_jremref(dotremref); 6763 dotremref = NULL; 6764 } 6765 } 6766 } 6767 } 6768 6769 if (jremref) 6770 journal_jremref(dirrem, jremref, inodedep); 6771 if (dotremref) 6772 journal_jremref(dirrem, dotremref, inodedep); 6773 if (dotdotremref) 6774 journal_jremref(dirrem, dotdotremref, NULL); 6775 jwork_move(&dirrem->dm_jwork, &dap->da_jwork); 6776 free_diradd(dap, &dirrem->dm_jwork); 6777 } 6778 6779 /* 6780 * Free a diradd dependency structure. This routine must be called 6781 * with splbio interrupts blocked. 6782 */ 6783 static void 6784 free_diradd(dap, wkhd) 6785 struct diradd *dap; 6786 struct workhead *wkhd; 6787 { 6788 struct dirrem *dirrem; 6789 struct pagedep *pagedep; 6790 struct inodedep *inodedep; 6791 struct mkdir *mkdir, *nextmd; 6792 6793 mtx_assert(&lk, MA_OWNED); 6794 LIST_REMOVE(dap, da_pdlist); 6795 if (dap->da_state & ONWORKLIST) 6796 WORKLIST_REMOVE(&dap->da_list); 6797 if ((dap->da_state & DIRCHG) == 0) { 6798 pagedep = dap->da_pagedep; 6799 } else { 6800 dirrem = dap->da_previous; 6801 pagedep = dirrem->dm_pagedep; 6802 dirrem->dm_dirinum = pagedep->pd_ino; 6803 dirrem->dm_state |= COMPLETE; 6804 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 6805 add_to_worklist(&dirrem->dm_list, 0); 6806 } 6807 if (inodedep_lookup(pagedep->pd_list.wk_mp, dap->da_newinum, 6808 0, &inodedep) != 0) 6809 if (inodedep->id_mkdiradd == dap) 6810 inodedep->id_mkdiradd = NULL; 6811 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 6812 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; mkdir = nextmd) { 6813 nextmd = LIST_NEXT(mkdir, md_mkdirs); 6814 if (mkdir->md_diradd != dap) 6815 continue; 6816 dap->da_state &= 6817 ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 6818 LIST_REMOVE(mkdir, md_mkdirs); 6819 if (mkdir->md_state & ONWORKLIST) 6820 WORKLIST_REMOVE(&mkdir->md_list); 6821 if (mkdir->md_jaddref != NULL) 6822 panic("free_diradd: Unexpected jaddref"); 6823 WORKITEM_FREE(mkdir, D_MKDIR); 6824 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) 6825 break; 6826 } 6827 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 6828 panic("free_diradd: unfound ref"); 6829 } 6830 if (inodedep) 6831 free_inodedep(inodedep); 6832 /* 6833 * Free any journal segments waiting for the directory write. 6834 */ 6835 handle_jwork(&dap->da_jwork); 6836 WORKITEM_FREE(dap, D_DIRADD); 6837 } 6838 6839 /* 6840 * Directory entry removal dependencies. 6841 * 6842 * When removing a directory entry, the entry's inode pointer must be 6843 * zero'ed on disk before the corresponding inode's link count is decremented 6844 * (possibly freeing the inode for re-use). This dependency is handled by 6845 * updating the directory entry but delaying the inode count reduction until 6846 * after the directory block has been written to disk. After this point, the 6847 * inode count can be decremented whenever it is convenient. 6848 */ 6849 6850 /* 6851 * This routine should be called immediately after removing 6852 * a directory entry. The inode's link count should not be 6853 * decremented by the calling procedure -- the soft updates 6854 * code will do this task when it is safe. 6855 */ 6856 void 6857 softdep_setup_remove(bp, dp, ip, isrmdir) 6858 struct buf *bp; /* buffer containing directory block */ 6859 struct inode *dp; /* inode for the directory being modified */ 6860 struct inode *ip; /* inode for directory entry being removed */ 6861 int isrmdir; /* indicates if doing RMDIR */ 6862 { 6863 struct dirrem *dirrem, *prevdirrem; 6864 struct inodedep *inodedep; 6865 int direct; 6866 6867 /* 6868 * Allocate a new dirrem if appropriate and ACQUIRE_LOCK. We want 6869 * newdirrem() to setup the full directory remove which requires 6870 * isrmdir > 1. 6871 */ 6872 dirrem = newdirrem(bp, dp, ip, isrmdir?2:0, &prevdirrem); 6873 /* 6874 * Add the dirrem to the inodedep's pending remove list for quick 6875 * discovery later. 6876 */ 6877 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 6878 &inodedep) == 0) 6879 panic("softdep_setup_remove: Lost inodedep."); 6880 dirrem->dm_state |= ONDEPLIST; 6881 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 6882 6883 /* 6884 * If the COMPLETE flag is clear, then there were no active 6885 * entries and we want to roll back to a zeroed entry until 6886 * the new inode is committed to disk. If the COMPLETE flag is 6887 * set then we have deleted an entry that never made it to 6888 * disk. If the entry we deleted resulted from a name change, 6889 * then the old name still resides on disk. We cannot delete 6890 * its inode (returned to us in prevdirrem) until the zeroed 6891 * directory entry gets to disk. The new inode has never been 6892 * referenced on the disk, so can be deleted immediately. 6893 */ 6894 if ((dirrem->dm_state & COMPLETE) == 0) { 6895 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, dirrem, 6896 dm_next); 6897 FREE_LOCK(&lk); 6898 } else { 6899 if (prevdirrem != NULL) 6900 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, 6901 prevdirrem, dm_next); 6902 dirrem->dm_dirinum = dirrem->dm_pagedep->pd_ino; 6903 direct = LIST_EMPTY(&dirrem->dm_jremrefhd); 6904 FREE_LOCK(&lk); 6905 if (direct) 6906 handle_workitem_remove(dirrem, NULL); 6907 } 6908 } 6909 6910 /* 6911 * Check for an entry matching 'offset' on both the pd_dirraddhd list and the 6912 * pd_pendinghd list of a pagedep. 6913 */ 6914 static struct diradd * 6915 diradd_lookup(pagedep, offset) 6916 struct pagedep *pagedep; 6917 int offset; 6918 { 6919 struct diradd *dap; 6920 6921 LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(offset)], da_pdlist) 6922 if (dap->da_offset == offset) 6923 return (dap); 6924 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist) 6925 if (dap->da_offset == offset) 6926 return (dap); 6927 return (NULL); 6928 } 6929 6930 /* 6931 * Search for a .. diradd dependency in a directory that is being removed. 6932 * If the directory was renamed to a new parent we have a diradd rather 6933 * than a mkdir for the .. entry. We need to cancel it now before 6934 * it is found in truncate(). 6935 */ 6936 static struct jremref * 6937 cancel_diradd_dotdot(ip, dirrem, jremref) 6938 struct inode *ip; 6939 struct dirrem *dirrem; 6940 struct jremref *jremref; 6941 { 6942 struct pagedep *pagedep; 6943 struct diradd *dap; 6944 struct worklist *wk; 6945 6946 if (pagedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 0, 6947 &pagedep) == 0) 6948 return (jremref); 6949 dap = diradd_lookup(pagedep, DOTDOT_OFFSET); 6950 if (dap == NULL) 6951 return (jremref); 6952 cancel_diradd(dap, dirrem, jremref, NULL, NULL); 6953 /* 6954 * Mark any journal work as belonging to the parent so it is freed 6955 * with the .. reference. 6956 */ 6957 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 6958 wk->wk_state |= MKDIR_PARENT; 6959 return (NULL); 6960 } 6961 6962 /* 6963 * Cancel the MKDIR_PARENT mkdir component of a diradd when we're going to 6964 * replace it with a dirrem/diradd pair as a result of re-parenting a 6965 * directory. This ensures that we don't simultaneously have a mkdir and 6966 * a diradd for the same .. entry. 6967 */ 6968 static struct jremref * 6969 cancel_mkdir_dotdot(ip, dirrem, jremref) 6970 struct inode *ip; 6971 struct dirrem *dirrem; 6972 struct jremref *jremref; 6973 { 6974 struct inodedep *inodedep; 6975 struct jaddref *jaddref; 6976 struct mkdir *mkdir; 6977 struct diradd *dap; 6978 6979 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 6980 &inodedep) == 0) 6981 panic("cancel_mkdir_dotdot: Lost inodedep"); 6982 dap = inodedep->id_mkdiradd; 6983 if (dap == NULL || (dap->da_state & MKDIR_PARENT) == 0) 6984 return (jremref); 6985 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; 6986 mkdir = LIST_NEXT(mkdir, md_mkdirs)) 6987 if (mkdir->md_diradd == dap && mkdir->md_state & MKDIR_PARENT) 6988 break; 6989 if (mkdir == NULL) 6990 panic("cancel_mkdir_dotdot: Unable to find mkdir\n"); 6991 if ((jaddref = mkdir->md_jaddref) != NULL) { 6992 mkdir->md_jaddref = NULL; 6993 jaddref->ja_state &= ~MKDIR_PARENT; 6994 if (inodedep_lookup(UFSTOVFS(ip->i_ump), jaddref->ja_ino, 0, 6995 &inodedep) == 0) 6996 panic("cancel_mkdir_dotdot: Lost parent inodedep"); 6997 if (cancel_jaddref(jaddref, inodedep, &dirrem->dm_jwork)) { 6998 journal_jremref(dirrem, jremref, inodedep); 6999 jremref = NULL; 7000 } 7001 } 7002 if (mkdir->md_state & ONWORKLIST) 7003 WORKLIST_REMOVE(&mkdir->md_list); 7004 mkdir->md_state |= ALLCOMPLETE; 7005 complete_mkdir(mkdir); 7006 return (jremref); 7007 } 7008 7009 static void 7010 journal_jremref(dirrem, jremref, inodedep) 7011 struct dirrem *dirrem; 7012 struct jremref *jremref; 7013 struct inodedep *inodedep; 7014 { 7015 7016 if (inodedep == NULL) 7017 if (inodedep_lookup(jremref->jr_list.wk_mp, 7018 jremref->jr_ref.if_ino, 0, &inodedep) == 0) 7019 panic("journal_jremref: Lost inodedep"); 7020 LIST_INSERT_HEAD(&dirrem->dm_jremrefhd, jremref, jr_deps); 7021 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 7022 add_to_journal(&jremref->jr_list); 7023 } 7024 7025 static void 7026 dirrem_journal(dirrem, jremref, dotremref, dotdotremref) 7027 struct dirrem *dirrem; 7028 struct jremref *jremref; 7029 struct jremref *dotremref; 7030 struct jremref *dotdotremref; 7031 { 7032 struct inodedep *inodedep; 7033 7034 7035 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 0, 7036 &inodedep) == 0) 7037 panic("dirrem_journal: Lost inodedep"); 7038 journal_jremref(dirrem, jremref, inodedep); 7039 if (dotremref) 7040 journal_jremref(dirrem, dotremref, inodedep); 7041 if (dotdotremref) 7042 journal_jremref(dirrem, dotdotremref, NULL); 7043 } 7044 7045 /* 7046 * Allocate a new dirrem if appropriate and return it along with 7047 * its associated pagedep. Called without a lock, returns with lock. 7048 */ 7049 static long num_dirrem; /* number of dirrem allocated */ 7050 static struct dirrem * 7051 newdirrem(bp, dp, ip, isrmdir, prevdirremp) 7052 struct buf *bp; /* buffer containing directory block */ 7053 struct inode *dp; /* inode for the directory being modified */ 7054 struct inode *ip; /* inode for directory entry being removed */ 7055 int isrmdir; /* indicates if doing RMDIR */ 7056 struct dirrem **prevdirremp; /* previously referenced inode, if any */ 7057 { 7058 int offset; 7059 ufs_lbn_t lbn; 7060 struct diradd *dap; 7061 struct dirrem *dirrem; 7062 struct pagedep *pagedep; 7063 struct jremref *jremref; 7064 struct jremref *dotremref; 7065 struct jremref *dotdotremref; 7066 struct vnode *dvp; 7067 7068 /* 7069 * Whiteouts have no deletion dependencies. 7070 */ 7071 if (ip == NULL) 7072 panic("newdirrem: whiteout"); 7073 dvp = ITOV(dp); 7074 /* 7075 * If we are over our limit, try to improve the situation. 7076 * Limiting the number of dirrem structures will also limit 7077 * the number of freefile and freeblks structures. 7078 */ 7079 ACQUIRE_LOCK(&lk); 7080 if (!(ip->i_flags & SF_SNAPSHOT) && num_dirrem > max_softdeps / 2) 7081 (void) request_cleanup(ITOV(dp)->v_mount, FLUSH_REMOVE); 7082 num_dirrem += 1; 7083 FREE_LOCK(&lk); 7084 dirrem = malloc(sizeof(struct dirrem), 7085 M_DIRREM, M_SOFTDEP_FLAGS|M_ZERO); 7086 workitem_alloc(&dirrem->dm_list, D_DIRREM, dvp->v_mount); 7087 LIST_INIT(&dirrem->dm_jremrefhd); 7088 LIST_INIT(&dirrem->dm_jwork); 7089 dirrem->dm_state = isrmdir ? RMDIR : 0; 7090 dirrem->dm_oldinum = ip->i_number; 7091 *prevdirremp = NULL; 7092 /* 7093 * Allocate remove reference structures to track journal write 7094 * dependencies. We will always have one for the link and 7095 * when doing directories we will always have one more for dot. 7096 * When renaming a directory we skip the dotdot link change so 7097 * this is not needed. 7098 */ 7099 jremref = dotremref = dotdotremref = NULL; 7100 if (DOINGSUJ(dvp)) { 7101 if (isrmdir) { 7102 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 7103 ip->i_effnlink + 2); 7104 dotremref = newjremref(dirrem, ip, ip, DOT_OFFSET, 7105 ip->i_effnlink + 1); 7106 } else 7107 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 7108 ip->i_effnlink + 1); 7109 if (isrmdir > 1) { 7110 dotdotremref = newjremref(dirrem, ip, dp, DOTDOT_OFFSET, 7111 dp->i_effnlink + 1); 7112 dotdotremref->jr_state |= MKDIR_PARENT; 7113 } 7114 } 7115 ACQUIRE_LOCK(&lk); 7116 lbn = lblkno(dp->i_fs, dp->i_offset); 7117 offset = blkoff(dp->i_fs, dp->i_offset); 7118 if (pagedep_lookup(UFSTOVFS(dp->i_ump), dp->i_number, lbn, DEPALLOC, 7119 &pagedep) == 0) 7120 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 7121 dirrem->dm_pagedep = pagedep; 7122 /* 7123 * If we're renaming a .. link to a new directory, cancel any 7124 * existing MKDIR_PARENT mkdir. If it has already been canceled 7125 * the jremref is preserved for any potential diradd in this 7126 * location. This can not coincide with a rmdir. 7127 */ 7128 if (dp->i_offset == DOTDOT_OFFSET) { 7129 if (isrmdir) 7130 panic("newdirrem: .. directory change during remove?"); 7131 jremref = cancel_mkdir_dotdot(dp, dirrem, jremref); 7132 } 7133 /* 7134 * If we're removing a directory search for the .. dependency now and 7135 * cancel it. Any pending journal work will be added to the dirrem 7136 * to be completed when the workitem remove completes. 7137 */ 7138 if (isrmdir > 1) 7139 dotdotremref = cancel_diradd_dotdot(ip, dirrem, dotdotremref); 7140 /* 7141 * Check for a diradd dependency for the same directory entry. 7142 * If present, then both dependencies become obsolete and can 7143 * be de-allocated. 7144 */ 7145 dap = diradd_lookup(pagedep, offset); 7146 if (dap == NULL) { 7147 /* 7148 * Link the jremref structures into the dirrem so they are 7149 * written prior to the pagedep. 7150 */ 7151 if (jremref) 7152 dirrem_journal(dirrem, jremref, dotremref, 7153 dotdotremref); 7154 return (dirrem); 7155 } 7156 /* 7157 * Must be ATTACHED at this point. 7158 */ 7159 if ((dap->da_state & ATTACHED) == 0) 7160 panic("newdirrem: not ATTACHED"); 7161 if (dap->da_newinum != ip->i_number) 7162 panic("newdirrem: inum %d should be %d", 7163 ip->i_number, dap->da_newinum); 7164 /* 7165 * If we are deleting a changed name that never made it to disk, 7166 * then return the dirrem describing the previous inode (which 7167 * represents the inode currently referenced from this entry on disk). 7168 */ 7169 if ((dap->da_state & DIRCHG) != 0) { 7170 *prevdirremp = dap->da_previous; 7171 dap->da_state &= ~DIRCHG; 7172 dap->da_pagedep = pagedep; 7173 } 7174 /* 7175 * We are deleting an entry that never made it to disk. 7176 * Mark it COMPLETE so we can delete its inode immediately. 7177 */ 7178 dirrem->dm_state |= COMPLETE; 7179 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref); 7180 #ifdef SUJ_DEBUG 7181 if (isrmdir == 0) { 7182 struct worklist *wk; 7183 7184 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 7185 if (wk->wk_state & (MKDIR_BODY | MKDIR_PARENT)) 7186 panic("bad wk %p (0x%X)\n", wk, wk->wk_state); 7187 } 7188 #endif 7189 7190 return (dirrem); 7191 } 7192 7193 /* 7194 * Directory entry change dependencies. 7195 * 7196 * Changing an existing directory entry requires that an add operation 7197 * be completed first followed by a deletion. The semantics for the addition 7198 * are identical to the description of adding a new entry above except 7199 * that the rollback is to the old inode number rather than zero. Once 7200 * the addition dependency is completed, the removal is done as described 7201 * in the removal routine above. 7202 */ 7203 7204 /* 7205 * This routine should be called immediately after changing 7206 * a directory entry. The inode's link count should not be 7207 * decremented by the calling procedure -- the soft updates 7208 * code will perform this task when it is safe. 7209 */ 7210 void 7211 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 7212 struct buf *bp; /* buffer containing directory block */ 7213 struct inode *dp; /* inode for the directory being modified */ 7214 struct inode *ip; /* inode for directory entry being removed */ 7215 ino_t newinum; /* new inode number for changed entry */ 7216 int isrmdir; /* indicates if doing RMDIR */ 7217 { 7218 int offset; 7219 struct diradd *dap = NULL; 7220 struct dirrem *dirrem, *prevdirrem; 7221 struct pagedep *pagedep; 7222 struct inodedep *inodedep; 7223 struct jaddref *jaddref; 7224 struct mount *mp; 7225 7226 offset = blkoff(dp->i_fs, dp->i_offset); 7227 mp = UFSTOVFS(dp->i_ump); 7228 7229 /* 7230 * Whiteouts do not need diradd dependencies. 7231 */ 7232 if (newinum != WINO) { 7233 dap = malloc(sizeof(struct diradd), 7234 M_DIRADD, M_SOFTDEP_FLAGS|M_ZERO); 7235 workitem_alloc(&dap->da_list, D_DIRADD, mp); 7236 dap->da_state = DIRCHG | ATTACHED | DEPCOMPLETE; 7237 dap->da_offset = offset; 7238 dap->da_newinum = newinum; 7239 LIST_INIT(&dap->da_jwork); 7240 } 7241 7242 /* 7243 * Allocate a new dirrem and ACQUIRE_LOCK. 7244 */ 7245 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem); 7246 pagedep = dirrem->dm_pagedep; 7247 /* 7248 * The possible values for isrmdir: 7249 * 0 - non-directory file rename 7250 * 1 - directory rename within same directory 7251 * inum - directory rename to new directory of given inode number 7252 * When renaming to a new directory, we are both deleting and 7253 * creating a new directory entry, so the link count on the new 7254 * directory should not change. Thus we do not need the followup 7255 * dirrem which is usually done in handle_workitem_remove. We set 7256 * the DIRCHG flag to tell handle_workitem_remove to skip the 7257 * followup dirrem. 7258 */ 7259 if (isrmdir > 1) 7260 dirrem->dm_state |= DIRCHG; 7261 7262 /* 7263 * Whiteouts have no additional dependencies, 7264 * so just put the dirrem on the correct list. 7265 */ 7266 if (newinum == WINO) { 7267 if ((dirrem->dm_state & COMPLETE) == 0) { 7268 LIST_INSERT_HEAD(&pagedep->pd_dirremhd, dirrem, 7269 dm_next); 7270 } else { 7271 dirrem->dm_dirinum = pagedep->pd_ino; 7272 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 7273 add_to_worklist(&dirrem->dm_list, 0); 7274 } 7275 FREE_LOCK(&lk); 7276 return; 7277 } 7278 /* 7279 * Add the dirrem to the inodedep's pending remove list for quick 7280 * discovery later. A valid nlinkdelta ensures that this lookup 7281 * will not fail. 7282 */ 7283 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 7284 panic("softdep_setup_directory_change: Lost inodedep."); 7285 dirrem->dm_state |= ONDEPLIST; 7286 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 7287 7288 /* 7289 * If the COMPLETE flag is clear, then there were no active 7290 * entries and we want to roll back to the previous inode until 7291 * the new inode is committed to disk. If the COMPLETE flag is 7292 * set, then we have deleted an entry that never made it to disk. 7293 * If the entry we deleted resulted from a name change, then the old 7294 * inode reference still resides on disk. Any rollback that we do 7295 * needs to be to that old inode (returned to us in prevdirrem). If 7296 * the entry we deleted resulted from a create, then there is 7297 * no entry on the disk, so we want to roll back to zero rather 7298 * than the uncommitted inode. In either of the COMPLETE cases we 7299 * want to immediately free the unwritten and unreferenced inode. 7300 */ 7301 if ((dirrem->dm_state & COMPLETE) == 0) { 7302 dap->da_previous = dirrem; 7303 } else { 7304 if (prevdirrem != NULL) { 7305 dap->da_previous = prevdirrem; 7306 } else { 7307 dap->da_state &= ~DIRCHG; 7308 dap->da_pagedep = pagedep; 7309 } 7310 dirrem->dm_dirinum = pagedep->pd_ino; 7311 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 7312 add_to_worklist(&dirrem->dm_list, 0); 7313 } 7314 /* 7315 * Lookup the jaddref for this journal entry. We must finish 7316 * initializing it and make the diradd write dependent on it. 7317 * If we're not journaling Put it on the id_bufwait list if the inode 7318 * is not yet written. If it is written, do the post-inode write 7319 * processing to put it on the id_pendinghd list. 7320 */ 7321 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 7322 if (mp->mnt_kern_flag & MNTK_SUJ) { 7323 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 7324 inoreflst); 7325 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 7326 ("softdep_setup_directory_change: bad jaddref %p", 7327 jaddref)); 7328 jaddref->ja_diroff = dp->i_offset; 7329 jaddref->ja_diradd = dap; 7330 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 7331 dap, da_pdlist); 7332 add_to_journal(&jaddref->ja_list); 7333 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 7334 dap->da_state |= COMPLETE; 7335 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 7336 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 7337 } else { 7338 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 7339 dap, da_pdlist); 7340 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 7341 } 7342 /* 7343 * If we're making a new name for a directory that has not been 7344 * committed when need to move the dot and dotdot references to 7345 * this new name. 7346 */ 7347 if (inodedep->id_mkdiradd && dp->i_offset != DOTDOT_OFFSET) 7348 merge_diradd(inodedep, dap); 7349 FREE_LOCK(&lk); 7350 } 7351 7352 /* 7353 * Called whenever the link count on an inode is changed. 7354 * It creates an inode dependency so that the new reference(s) 7355 * to the inode cannot be committed to disk until the updated 7356 * inode has been written. 7357 */ 7358 void 7359 softdep_change_linkcnt(ip) 7360 struct inode *ip; /* the inode with the increased link count */ 7361 { 7362 struct inodedep *inodedep; 7363 7364 ACQUIRE_LOCK(&lk); 7365 inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, DEPALLOC, &inodedep); 7366 if (ip->i_nlink < ip->i_effnlink) 7367 panic("softdep_change_linkcnt: bad delta"); 7368 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 7369 FREE_LOCK(&lk); 7370 } 7371 7372 /* 7373 * Attach a sbdep dependency to the superblock buf so that we can keep 7374 * track of the head of the linked list of referenced but unlinked inodes. 7375 */ 7376 void 7377 softdep_setup_sbupdate(ump, fs, bp) 7378 struct ufsmount *ump; 7379 struct fs *fs; 7380 struct buf *bp; 7381 { 7382 struct sbdep *sbdep; 7383 struct worklist *wk; 7384 7385 if ((fs->fs_flags & FS_SUJ) == 0) 7386 return; 7387 LIST_FOREACH(wk, &bp->b_dep, wk_list) 7388 if (wk->wk_type == D_SBDEP) 7389 break; 7390 if (wk != NULL) 7391 return; 7392 sbdep = malloc(sizeof(struct sbdep), M_SBDEP, M_SOFTDEP_FLAGS); 7393 workitem_alloc(&sbdep->sb_list, D_SBDEP, UFSTOVFS(ump)); 7394 sbdep->sb_fs = fs; 7395 sbdep->sb_ump = ump; 7396 ACQUIRE_LOCK(&lk); 7397 WORKLIST_INSERT(&bp->b_dep, &sbdep->sb_list); 7398 FREE_LOCK(&lk); 7399 } 7400 7401 /* 7402 * Return the first unlinked inodedep which is ready to be the head of the 7403 * list. The inodedep and all those after it must have valid next pointers. 7404 */ 7405 static struct inodedep * 7406 first_unlinked_inodedep(ump) 7407 struct ufsmount *ump; 7408 { 7409 struct inodedep *inodedep; 7410 struct inodedep *idp; 7411 7412 for (inodedep = TAILQ_LAST(&ump->softdep_unlinked, inodedeplst); 7413 inodedep; inodedep = idp) { 7414 if ((inodedep->id_state & UNLINKNEXT) == 0) 7415 return (NULL); 7416 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 7417 if (idp == NULL || (idp->id_state & UNLINKNEXT) == 0) 7418 break; 7419 if ((inodedep->id_state & UNLINKPREV) == 0) 7420 panic("first_unlinked_inodedep: prev != next"); 7421 } 7422 if (inodedep == NULL) 7423 return (NULL); 7424 7425 return (inodedep); 7426 } 7427 7428 /* 7429 * Set the sujfree unlinked head pointer prior to writing a superblock. 7430 */ 7431 static void 7432 initiate_write_sbdep(sbdep) 7433 struct sbdep *sbdep; 7434 { 7435 struct inodedep *inodedep; 7436 struct fs *bpfs; 7437 struct fs *fs; 7438 7439 bpfs = sbdep->sb_fs; 7440 fs = sbdep->sb_ump->um_fs; 7441 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 7442 if (inodedep) { 7443 fs->fs_sujfree = inodedep->id_ino; 7444 inodedep->id_state |= UNLINKPREV; 7445 } else 7446 fs->fs_sujfree = 0; 7447 bpfs->fs_sujfree = fs->fs_sujfree; 7448 } 7449 7450 /* 7451 * After a superblock is written determine whether it must be written again 7452 * due to a changing unlinked list head. 7453 */ 7454 static int 7455 handle_written_sbdep(sbdep, bp) 7456 struct sbdep *sbdep; 7457 struct buf *bp; 7458 { 7459 struct inodedep *inodedep; 7460 struct mount *mp; 7461 struct fs *fs; 7462 7463 fs = sbdep->sb_fs; 7464 mp = UFSTOVFS(sbdep->sb_ump); 7465 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 7466 if ((inodedep && fs->fs_sujfree != inodedep->id_ino) || 7467 (inodedep == NULL && fs->fs_sujfree != 0)) { 7468 bdirty(bp); 7469 return (1); 7470 } 7471 WORKITEM_FREE(sbdep, D_SBDEP); 7472 if (fs->fs_sujfree == 0) 7473 return (0); 7474 if (inodedep_lookup(mp, fs->fs_sujfree, 0, &inodedep) == 0) 7475 panic("handle_written_sbdep: lost inodedep"); 7476 /* 7477 * Now that we have a record of this indode in stable store allow it 7478 * to be written to free up pending work. Inodes may see a lot of 7479 * write activity after they are unlinked which we must not hold up. 7480 */ 7481 for (; inodedep != NULL; inodedep = TAILQ_NEXT(inodedep, id_unlinked)) { 7482 if ((inodedep->id_state & UNLINKLINKS) != UNLINKLINKS) 7483 panic("handle_written_sbdep: Bad inodedep %p (0x%X)", 7484 inodedep, inodedep->id_state); 7485 if (inodedep->id_state & UNLINKONLIST) 7486 break; 7487 inodedep->id_state |= DEPCOMPLETE | UNLINKONLIST; 7488 } 7489 7490 return (0); 7491 } 7492 7493 /* 7494 * Mark an inodedep has unlinked and insert it into the in-memory unlinked 7495 * list. 7496 */ 7497 static void 7498 unlinked_inodedep(mp, inodedep) 7499 struct mount *mp; 7500 struct inodedep *inodedep; 7501 { 7502 struct ufsmount *ump; 7503 7504 if ((mp->mnt_kern_flag & MNTK_SUJ) == 0) 7505 return; 7506 ump = VFSTOUFS(mp); 7507 ump->um_fs->fs_fmod = 1; 7508 inodedep->id_state |= UNLINKED; 7509 TAILQ_INSERT_HEAD(&ump->softdep_unlinked, inodedep, id_unlinked); 7510 } 7511 7512 /* 7513 * Remove an inodedep from the unlinked inodedep list. This may require 7514 * disk writes if the inode has made it that far. 7515 */ 7516 static void 7517 clear_unlinked_inodedep(inodedep) 7518 struct inodedep *inodedep; 7519 { 7520 struct ufsmount *ump; 7521 struct inodedep *idp; 7522 struct inodedep *idn; 7523 struct fs *fs; 7524 struct buf *bp; 7525 ino_t ino; 7526 ino_t nino; 7527 ino_t pino; 7528 int error; 7529 7530 ump = VFSTOUFS(inodedep->id_list.wk_mp); 7531 fs = ump->um_fs; 7532 ino = inodedep->id_ino; 7533 error = 0; 7534 for (;;) { 7535 /* 7536 * If nothing has yet been written simply remove us from 7537 * the in memory list and return. This is the most common 7538 * case where handle_workitem_remove() loses the final 7539 * reference. 7540 */ 7541 if ((inodedep->id_state & UNLINKLINKS) == 0) 7542 break; 7543 /* 7544 * If we have a NEXT pointer and no PREV pointer we can simply 7545 * clear NEXT's PREV and remove ourselves from the list. Be 7546 * careful not to clear PREV if the superblock points at 7547 * next as well. 7548 */ 7549 idn = TAILQ_NEXT(inodedep, id_unlinked); 7550 if ((inodedep->id_state & UNLINKLINKS) == UNLINKNEXT) { 7551 if (idn && fs->fs_sujfree != idn->id_ino) 7552 idn->id_state &= ~UNLINKPREV; 7553 break; 7554 } 7555 /* 7556 * Here we have an inodedep which is actually linked into 7557 * the list. We must remove it by forcing a write to the 7558 * link before us, whether it be the superblock or an inode. 7559 * Unfortunately the list may change while we're waiting 7560 * on the buf lock for either resource so we must loop until 7561 * we lock. the right one. If both the superblock and an 7562 * inode point to this inode we must clear the inode first 7563 * followed by the superblock. 7564 */ 7565 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 7566 pino = 0; 7567 if (idp && (idp->id_state & UNLINKNEXT)) 7568 pino = idp->id_ino; 7569 FREE_LOCK(&lk); 7570 if (pino == 0) 7571 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 7572 (int)fs->fs_sbsize, 0, 0, 0); 7573 else 7574 error = bread(ump->um_devvp, 7575 fsbtodb(fs, ino_to_fsba(fs, pino)), 7576 (int)fs->fs_bsize, NOCRED, &bp); 7577 ACQUIRE_LOCK(&lk); 7578 if (error) 7579 break; 7580 /* If the list has changed restart the loop. */ 7581 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 7582 nino = 0; 7583 if (idp && (idp->id_state & UNLINKNEXT)) 7584 nino = idp->id_ino; 7585 if (nino != pino || 7586 (inodedep->id_state & UNLINKPREV) != UNLINKPREV) { 7587 FREE_LOCK(&lk); 7588 brelse(bp); 7589 ACQUIRE_LOCK(&lk); 7590 continue; 7591 } 7592 /* 7593 * Remove us from the in memory list. After this we cannot 7594 * access the inodedep. 7595 */ 7596 idn = TAILQ_NEXT(inodedep, id_unlinked); 7597 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS); 7598 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 7599 /* 7600 * Determine the next inode number. 7601 */ 7602 nino = 0; 7603 if (idn) { 7604 /* 7605 * If next isn't on the list we can just clear prev's 7606 * state and schedule it to be fixed later. No need 7607 * to synchronously write if we're not in the real 7608 * list. 7609 */ 7610 if ((idn->id_state & UNLINKPREV) == 0 && pino != 0) { 7611 idp->id_state &= ~UNLINKNEXT; 7612 if ((idp->id_state & ONWORKLIST) == 0) 7613 WORKLIST_INSERT(&bp->b_dep, 7614 &idp->id_list); 7615 FREE_LOCK(&lk); 7616 bawrite(bp); 7617 ACQUIRE_LOCK(&lk); 7618 return; 7619 } 7620 nino = idn->id_ino; 7621 } 7622 FREE_LOCK(&lk); 7623 /* 7624 * The predecessor's next pointer is manually updated here 7625 * so that the NEXT flag is never cleared for an element 7626 * that is in the list. 7627 */ 7628 if (pino == 0) { 7629 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 7630 ffs_oldfscompat_write((struct fs *)bp->b_data, ump); 7631 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data, 7632 bp); 7633 } else if (fs->fs_magic == FS_UFS1_MAGIC) 7634 ((struct ufs1_dinode *)bp->b_data + 7635 ino_to_fsbo(fs, pino))->di_freelink = nino; 7636 else 7637 ((struct ufs2_dinode *)bp->b_data + 7638 ino_to_fsbo(fs, pino))->di_freelink = nino; 7639 /* 7640 * If the bwrite fails we have no recourse to recover. The 7641 * filesystem is corrupted already. 7642 */ 7643 bwrite(bp); 7644 ACQUIRE_LOCK(&lk); 7645 /* 7646 * If the superblock pointer still needs to be cleared force 7647 * a write here. 7648 */ 7649 if (fs->fs_sujfree == ino) { 7650 FREE_LOCK(&lk); 7651 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 7652 (int)fs->fs_sbsize, 0, 0, 0); 7653 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 7654 ffs_oldfscompat_write((struct fs *)bp->b_data, ump); 7655 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data, 7656 bp); 7657 bwrite(bp); 7658 ACQUIRE_LOCK(&lk); 7659 } 7660 if (fs->fs_sujfree != ino) 7661 return; 7662 panic("clear_unlinked_inodedep: Failed to clear free head"); 7663 } 7664 if (inodedep->id_ino == fs->fs_sujfree) 7665 panic("clear_unlinked_inodedep: Freeing head of free list"); 7666 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS); 7667 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 7668 return; 7669 } 7670 7671 /* 7672 * This workitem decrements the inode's link count. 7673 * If the link count reaches zero, the file is removed. 7674 */ 7675 static void 7676 handle_workitem_remove(dirrem, xp) 7677 struct dirrem *dirrem; 7678 struct vnode *xp; 7679 { 7680 struct inodedep *inodedep; 7681 struct workhead dotdotwk; 7682 struct worklist *wk; 7683 struct ufsmount *ump; 7684 struct mount *mp; 7685 struct vnode *vp; 7686 struct inode *ip; 7687 ino_t oldinum; 7688 int error; 7689 7690 if (dirrem->dm_state & ONWORKLIST) 7691 panic("handle_workitem_remove: dirrem %p still on worklist", 7692 dirrem); 7693 oldinum = dirrem->dm_oldinum; 7694 mp = dirrem->dm_list.wk_mp; 7695 ump = VFSTOUFS(mp); 7696 if ((vp = xp) == NULL && 7697 (error = ffs_vgetf(mp, oldinum, LK_EXCLUSIVE, &vp, 7698 FFSV_FORCEINSMQ)) != 0) { 7699 softdep_error("handle_workitem_remove: vget", error); 7700 return; 7701 } 7702 ip = VTOI(vp); 7703 ACQUIRE_LOCK(&lk); 7704 if ((inodedep_lookup(mp, oldinum, 0, &inodedep)) == 0) 7705 panic("handle_workitem_remove: lost inodedep"); 7706 if (dirrem->dm_state & ONDEPLIST) 7707 LIST_REMOVE(dirrem, dm_inonext); 7708 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 7709 ("handle_workitem_remove: Journal entries not written.")); 7710 7711 /* 7712 * Move all dependencies waiting on the remove to complete 7713 * from the dirrem to the inode inowait list to be completed 7714 * after the inode has been updated and written to disk. Any 7715 * marked MKDIR_PARENT are saved to be completed when the .. ref 7716 * is removed. 7717 */ 7718 LIST_INIT(&dotdotwk); 7719 while ((wk = LIST_FIRST(&dirrem->dm_jwork)) != NULL) { 7720 WORKLIST_REMOVE(wk); 7721 if (wk->wk_state & MKDIR_PARENT) { 7722 wk->wk_state &= ~MKDIR_PARENT; 7723 WORKLIST_INSERT(&dotdotwk, wk); 7724 continue; 7725 } 7726 WORKLIST_INSERT(&inodedep->id_inowait, wk); 7727 } 7728 LIST_SWAP(&dirrem->dm_jwork, &dotdotwk, worklist, wk_list); 7729 /* 7730 * Normal file deletion. 7731 */ 7732 if ((dirrem->dm_state & RMDIR) == 0) { 7733 ip->i_nlink--; 7734 DIP_SET(ip, i_nlink, ip->i_nlink); 7735 ip->i_flag |= IN_CHANGE; 7736 if (ip->i_nlink < ip->i_effnlink) 7737 panic("handle_workitem_remove: bad file delta"); 7738 if (ip->i_nlink == 0) 7739 unlinked_inodedep(mp, inodedep); 7740 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 7741 num_dirrem -= 1; 7742 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 7743 ("handle_workitem_remove: worklist not empty. %s", 7744 TYPENAME(LIST_FIRST(&dirrem->dm_jwork)->wk_type))); 7745 WORKITEM_FREE(dirrem, D_DIRREM); 7746 FREE_LOCK(&lk); 7747 goto out; 7748 } 7749 /* 7750 * Directory deletion. Decrement reference count for both the 7751 * just deleted parent directory entry and the reference for ".". 7752 * Arrange to have the reference count on the parent decremented 7753 * to account for the loss of "..". 7754 */ 7755 ip->i_nlink -= 2; 7756 DIP_SET(ip, i_nlink, ip->i_nlink); 7757 ip->i_flag |= IN_CHANGE; 7758 if (ip->i_nlink < ip->i_effnlink) 7759 panic("handle_workitem_remove: bad dir delta"); 7760 if (ip->i_nlink == 0) 7761 unlinked_inodedep(mp, inodedep); 7762 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 7763 /* 7764 * Rename a directory to a new parent. Since, we are both deleting 7765 * and creating a new directory entry, the link count on the new 7766 * directory should not change. Thus we skip the followup dirrem. 7767 */ 7768 if (dirrem->dm_state & DIRCHG) { 7769 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 7770 ("handle_workitem_remove: DIRCHG and worklist not empty.")); 7771 num_dirrem -= 1; 7772 WORKITEM_FREE(dirrem, D_DIRREM); 7773 FREE_LOCK(&lk); 7774 goto out; 7775 } 7776 dirrem->dm_state = ONDEPLIST; 7777 dirrem->dm_oldinum = dirrem->dm_dirinum; 7778 /* 7779 * Place the dirrem on the parent's diremhd list. 7780 */ 7781 if (inodedep_lookup(mp, dirrem->dm_oldinum, 0, &inodedep) == 0) 7782 panic("handle_workitem_remove: lost dir inodedep"); 7783 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 7784 /* 7785 * If the allocated inode has never been written to disk, then 7786 * the on-disk inode is zero'ed and we can remove the file 7787 * immediately. When journaling if the inode has been marked 7788 * unlinked and not DEPCOMPLETE we know it can never be written. 7789 */ 7790 inodedep_lookup(mp, oldinum, 0, &inodedep); 7791 if (inodedep == NULL || 7792 (inodedep->id_state & (DEPCOMPLETE | UNLINKED)) == UNLINKED || 7793 check_inode_unwritten(inodedep)) { 7794 if (xp != NULL) 7795 add_to_worklist(&dirrem->dm_list, 0); 7796 FREE_LOCK(&lk); 7797 if (xp == NULL) { 7798 vput(vp); 7799 handle_workitem_remove(dirrem, NULL); 7800 } 7801 return; 7802 } 7803 WORKLIST_INSERT(&inodedep->id_inowait, &dirrem->dm_list); 7804 FREE_LOCK(&lk); 7805 ip->i_flag |= IN_CHANGE; 7806 out: 7807 ffs_update(vp, 0); 7808 if (xp == NULL) 7809 vput(vp); 7810 } 7811 7812 /* 7813 * Inode de-allocation dependencies. 7814 * 7815 * When an inode's link count is reduced to zero, it can be de-allocated. We 7816 * found it convenient to postpone de-allocation until after the inode is 7817 * written to disk with its new link count (zero). At this point, all of the 7818 * on-disk inode's block pointers are nullified and, with careful dependency 7819 * list ordering, all dependencies related to the inode will be satisfied and 7820 * the corresponding dependency structures de-allocated. So, if/when the 7821 * inode is reused, there will be no mixing of old dependencies with new 7822 * ones. This artificial dependency is set up by the block de-allocation 7823 * procedure above (softdep_setup_freeblocks) and completed by the 7824 * following procedure. 7825 */ 7826 static void 7827 handle_workitem_freefile(freefile) 7828 struct freefile *freefile; 7829 { 7830 struct workhead wkhd; 7831 struct fs *fs; 7832 struct inodedep *idp; 7833 struct ufsmount *ump; 7834 int error; 7835 7836 ump = VFSTOUFS(freefile->fx_list.wk_mp); 7837 fs = ump->um_fs; 7838 #ifdef DEBUG 7839 ACQUIRE_LOCK(&lk); 7840 error = inodedep_lookup(UFSTOVFS(ump), freefile->fx_oldinum, 0, &idp); 7841 FREE_LOCK(&lk); 7842 if (error) 7843 panic("handle_workitem_freefile: inodedep %p survived", idp); 7844 #endif 7845 UFS_LOCK(ump); 7846 fs->fs_pendinginodes -= 1; 7847 UFS_UNLOCK(ump); 7848 LIST_INIT(&wkhd); 7849 LIST_SWAP(&freefile->fx_jwork, &wkhd, worklist, wk_list); 7850 if ((error = ffs_freefile(ump, fs, freefile->fx_devvp, 7851 freefile->fx_oldinum, freefile->fx_mode, &wkhd)) != 0) 7852 softdep_error("handle_workitem_freefile", error); 7853 ACQUIRE_LOCK(&lk); 7854 WORKITEM_FREE(freefile, D_FREEFILE); 7855 FREE_LOCK(&lk); 7856 } 7857 7858 7859 /* 7860 * Helper function which unlinks marker element from work list and returns 7861 * the next element on the list. 7862 */ 7863 static __inline struct worklist * 7864 markernext(struct worklist *marker) 7865 { 7866 struct worklist *next; 7867 7868 next = LIST_NEXT(marker, wk_list); 7869 LIST_REMOVE(marker, wk_list); 7870 return next; 7871 } 7872 7873 /* 7874 * Disk writes. 7875 * 7876 * The dependency structures constructed above are most actively used when file 7877 * system blocks are written to disk. No constraints are placed on when a 7878 * block can be written, but unsatisfied update dependencies are made safe by 7879 * modifying (or replacing) the source memory for the duration of the disk 7880 * write. When the disk write completes, the memory block is again brought 7881 * up-to-date. 7882 * 7883 * In-core inode structure reclamation. 7884 * 7885 * Because there are a finite number of "in-core" inode structures, they are 7886 * reused regularly. By transferring all inode-related dependencies to the 7887 * in-memory inode block and indexing them separately (via "inodedep"s), we 7888 * can allow "in-core" inode structures to be reused at any time and avoid 7889 * any increase in contention. 7890 * 7891 * Called just before entering the device driver to initiate a new disk I/O. 7892 * The buffer must be locked, thus, no I/O completion operations can occur 7893 * while we are manipulating its associated dependencies. 7894 */ 7895 static void 7896 softdep_disk_io_initiation(bp) 7897 struct buf *bp; /* structure describing disk write to occur */ 7898 { 7899 struct worklist *wk; 7900 struct worklist marker; 7901 struct inodedep *inodedep; 7902 struct freeblks *freeblks; 7903 struct jfreeblk *jfreeblk; 7904 struct newblk *newblk; 7905 7906 /* 7907 * We only care about write operations. There should never 7908 * be dependencies for reads. 7909 */ 7910 if (bp->b_iocmd != BIO_WRITE) 7911 panic("softdep_disk_io_initiation: not write"); 7912 7913 if (bp->b_vflags & BV_BKGRDINPROG) 7914 panic("softdep_disk_io_initiation: Writing buffer with " 7915 "background write in progress: %p", bp); 7916 7917 marker.wk_type = D_LAST + 1; /* Not a normal workitem */ 7918 PHOLD(curproc); /* Don't swap out kernel stack */ 7919 7920 ACQUIRE_LOCK(&lk); 7921 /* 7922 * Do any necessary pre-I/O processing. 7923 */ 7924 for (wk = LIST_FIRST(&bp->b_dep); wk != NULL; 7925 wk = markernext(&marker)) { 7926 LIST_INSERT_AFTER(wk, &marker, wk_list); 7927 switch (wk->wk_type) { 7928 7929 case D_PAGEDEP: 7930 initiate_write_filepage(WK_PAGEDEP(wk), bp); 7931 continue; 7932 7933 case D_INODEDEP: 7934 inodedep = WK_INODEDEP(wk); 7935 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) 7936 initiate_write_inodeblock_ufs1(inodedep, bp); 7937 else 7938 initiate_write_inodeblock_ufs2(inodedep, bp); 7939 continue; 7940 7941 case D_INDIRDEP: 7942 initiate_write_indirdep(WK_INDIRDEP(wk), bp); 7943 continue; 7944 7945 case D_BMSAFEMAP: 7946 initiate_write_bmsafemap(WK_BMSAFEMAP(wk), bp); 7947 continue; 7948 7949 case D_JSEG: 7950 WK_JSEG(wk)->js_buf = NULL; 7951 continue; 7952 7953 case D_FREEBLKS: 7954 freeblks = WK_FREEBLKS(wk); 7955 jfreeblk = LIST_FIRST(&freeblks->fb_jfreeblkhd); 7956 /* 7957 * We have to wait for the jfreeblks to be journaled 7958 * before we can write an inodeblock with updated 7959 * pointers. Be careful to arrange the marker so 7960 * we revisit the jfreeblk if it's not removed by 7961 * the first jwait(). 7962 */ 7963 if (jfreeblk != NULL) { 7964 LIST_REMOVE(&marker, wk_list); 7965 LIST_INSERT_BEFORE(wk, &marker, wk_list); 7966 jwait(&jfreeblk->jf_list); 7967 } 7968 continue; 7969 case D_ALLOCDIRECT: 7970 case D_ALLOCINDIR: 7971 /* 7972 * We have to wait for the jnewblk to be journaled 7973 * before we can write to a block otherwise the 7974 * contents may be confused with an earlier file 7975 * at recovery time. Handle the marker as described 7976 * above. 7977 */ 7978 newblk = WK_NEWBLK(wk); 7979 if (newblk->nb_jnewblk != NULL) { 7980 LIST_REMOVE(&marker, wk_list); 7981 LIST_INSERT_BEFORE(wk, &marker, wk_list); 7982 jwait(&newblk->nb_jnewblk->jn_list); 7983 } 7984 continue; 7985 7986 case D_SBDEP: 7987 initiate_write_sbdep(WK_SBDEP(wk)); 7988 continue; 7989 7990 case D_MKDIR: 7991 case D_FREEWORK: 7992 case D_FREEDEP: 7993 case D_JSEGDEP: 7994 continue; 7995 7996 default: 7997 panic("handle_disk_io_initiation: Unexpected type %s", 7998 TYPENAME(wk->wk_type)); 7999 /* NOTREACHED */ 8000 } 8001 } 8002 FREE_LOCK(&lk); 8003 PRELE(curproc); /* Allow swapout of kernel stack */ 8004 } 8005 8006 /* 8007 * Called from within the procedure above to deal with unsatisfied 8008 * allocation dependencies in a directory. The buffer must be locked, 8009 * thus, no I/O completion operations can occur while we are 8010 * manipulating its associated dependencies. 8011 */ 8012 static void 8013 initiate_write_filepage(pagedep, bp) 8014 struct pagedep *pagedep; 8015 struct buf *bp; 8016 { 8017 struct jremref *jremref; 8018 struct jmvref *jmvref; 8019 struct dirrem *dirrem; 8020 struct diradd *dap; 8021 struct direct *ep; 8022 int i; 8023 8024 if (pagedep->pd_state & IOSTARTED) { 8025 /* 8026 * This can only happen if there is a driver that does not 8027 * understand chaining. Here biodone will reissue the call 8028 * to strategy for the incomplete buffers. 8029 */ 8030 printf("initiate_write_filepage: already started\n"); 8031 return; 8032 } 8033 pagedep->pd_state |= IOSTARTED; 8034 /* 8035 * Wait for all journal remove dependencies to hit the disk. 8036 * We can not allow any potentially conflicting directory adds 8037 * to be visible before removes and rollback is too difficult. 8038 * lk may be dropped and re-acquired, however we hold the buf 8039 * locked so the dependency can not go away. 8040 */ 8041 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) 8042 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) { 8043 stat_jwait_filepage++; 8044 jwait(&jremref->jr_list); 8045 } 8046 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) { 8047 stat_jwait_filepage++; 8048 jwait(&jmvref->jm_list); 8049 } 8050 for (i = 0; i < DAHASHSZ; i++) { 8051 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 8052 ep = (struct direct *) 8053 ((char *)bp->b_data + dap->da_offset); 8054 if (ep->d_ino != dap->da_newinum) 8055 panic("%s: dir inum %d != new %d", 8056 "initiate_write_filepage", 8057 ep->d_ino, dap->da_newinum); 8058 if (dap->da_state & DIRCHG) 8059 ep->d_ino = dap->da_previous->dm_oldinum; 8060 else 8061 ep->d_ino = 0; 8062 dap->da_state &= ~ATTACHED; 8063 dap->da_state |= UNDONE; 8064 } 8065 } 8066 } 8067 8068 /* 8069 * Version of initiate_write_inodeblock that handles UFS1 dinodes. 8070 * Note that any bug fixes made to this routine must be done in the 8071 * version found below. 8072 * 8073 * Called from within the procedure above to deal with unsatisfied 8074 * allocation dependencies in an inodeblock. The buffer must be 8075 * locked, thus, no I/O completion operations can occur while we 8076 * are manipulating its associated dependencies. 8077 */ 8078 static void 8079 initiate_write_inodeblock_ufs1(inodedep, bp) 8080 struct inodedep *inodedep; 8081 struct buf *bp; /* The inode block */ 8082 { 8083 struct allocdirect *adp, *lastadp; 8084 struct ufs1_dinode *dp; 8085 struct ufs1_dinode *sip; 8086 struct inoref *inoref; 8087 struct fs *fs; 8088 ufs_lbn_t i; 8089 #ifdef INVARIANTS 8090 ufs_lbn_t prevlbn = 0; 8091 #endif 8092 int deplist; 8093 8094 if (inodedep->id_state & IOSTARTED) 8095 panic("initiate_write_inodeblock_ufs1: already started"); 8096 inodedep->id_state |= IOSTARTED; 8097 fs = inodedep->id_fs; 8098 dp = (struct ufs1_dinode *)bp->b_data + 8099 ino_to_fsbo(fs, inodedep->id_ino); 8100 8101 /* 8102 * If we're on the unlinked list but have not yet written our 8103 * next pointer initialize it here. 8104 */ 8105 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 8106 struct inodedep *inon; 8107 8108 inon = TAILQ_NEXT(inodedep, id_unlinked); 8109 dp->di_freelink = inon ? inon->id_ino : 0; 8110 } 8111 /* 8112 * If the bitmap is not yet written, then the allocated 8113 * inode cannot be written to disk. 8114 */ 8115 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 8116 if (inodedep->id_savedino1 != NULL) 8117 panic("initiate_write_inodeblock_ufs1: I/O underway"); 8118 FREE_LOCK(&lk); 8119 sip = malloc(sizeof(struct ufs1_dinode), 8120 M_SAVEDINO, M_SOFTDEP_FLAGS); 8121 ACQUIRE_LOCK(&lk); 8122 inodedep->id_savedino1 = sip; 8123 *inodedep->id_savedino1 = *dp; 8124 bzero((caddr_t)dp, sizeof(struct ufs1_dinode)); 8125 dp->di_gen = inodedep->id_savedino1->di_gen; 8126 dp->di_freelink = inodedep->id_savedino1->di_freelink; 8127 return; 8128 } 8129 /* 8130 * If no dependencies, then there is nothing to roll back. 8131 */ 8132 inodedep->id_savedsize = dp->di_size; 8133 inodedep->id_savedextsize = 0; 8134 inodedep->id_savednlink = dp->di_nlink; 8135 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 8136 TAILQ_EMPTY(&inodedep->id_inoreflst)) 8137 return; 8138 /* 8139 * Revert the link count to that of the first unwritten journal entry. 8140 */ 8141 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 8142 if (inoref) 8143 dp->di_nlink = inoref->if_nlink; 8144 /* 8145 * Set the dependencies to busy. 8146 */ 8147 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8148 adp = TAILQ_NEXT(adp, ad_next)) { 8149 #ifdef INVARIANTS 8150 if (deplist != 0 && prevlbn >= adp->ad_offset) 8151 panic("softdep_write_inodeblock: lbn order"); 8152 prevlbn = adp->ad_offset; 8153 if (adp->ad_offset < NDADDR && 8154 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 8155 panic("%s: direct pointer #%jd mismatch %d != %jd", 8156 "softdep_write_inodeblock", 8157 (intmax_t)adp->ad_offset, 8158 dp->di_db[adp->ad_offset], 8159 (intmax_t)adp->ad_newblkno); 8160 if (adp->ad_offset >= NDADDR && 8161 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno) 8162 panic("%s: indirect pointer #%jd mismatch %d != %jd", 8163 "softdep_write_inodeblock", 8164 (intmax_t)adp->ad_offset - NDADDR, 8165 dp->di_ib[adp->ad_offset - NDADDR], 8166 (intmax_t)adp->ad_newblkno); 8167 deplist |= 1 << adp->ad_offset; 8168 if ((adp->ad_state & ATTACHED) == 0) 8169 panic("softdep_write_inodeblock: Unknown state 0x%x", 8170 adp->ad_state); 8171 #endif /* INVARIANTS */ 8172 adp->ad_state &= ~ATTACHED; 8173 adp->ad_state |= UNDONE; 8174 } 8175 /* 8176 * The on-disk inode cannot claim to be any larger than the last 8177 * fragment that has been written. Otherwise, the on-disk inode 8178 * might have fragments that were not the last block in the file 8179 * which would corrupt the filesystem. 8180 */ 8181 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8182 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 8183 if (adp->ad_offset >= NDADDR) 8184 break; 8185 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 8186 /* keep going until hitting a rollback to a frag */ 8187 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 8188 continue; 8189 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 8190 for (i = adp->ad_offset + 1; i < NDADDR; i++) { 8191 #ifdef INVARIANTS 8192 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 8193 panic("softdep_write_inodeblock: lost dep1"); 8194 #endif /* INVARIANTS */ 8195 dp->di_db[i] = 0; 8196 } 8197 for (i = 0; i < NIADDR; i++) { 8198 #ifdef INVARIANTS 8199 if (dp->di_ib[i] != 0 && 8200 (deplist & ((1 << NDADDR) << i)) == 0) 8201 panic("softdep_write_inodeblock: lost dep2"); 8202 #endif /* INVARIANTS */ 8203 dp->di_ib[i] = 0; 8204 } 8205 return; 8206 } 8207 /* 8208 * If we have zero'ed out the last allocated block of the file, 8209 * roll back the size to the last currently allocated block. 8210 * We know that this last allocated block is a full-sized as 8211 * we already checked for fragments in the loop above. 8212 */ 8213 if (lastadp != NULL && 8214 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 8215 for (i = lastadp->ad_offset; i >= 0; i--) 8216 if (dp->di_db[i] != 0) 8217 break; 8218 dp->di_size = (i + 1) * fs->fs_bsize; 8219 } 8220 /* 8221 * The only dependencies are for indirect blocks. 8222 * 8223 * The file size for indirect block additions is not guaranteed. 8224 * Such a guarantee would be non-trivial to achieve. The conventional 8225 * synchronous write implementation also does not make this guarantee. 8226 * Fsck should catch and fix discrepancies. Arguably, the file size 8227 * can be over-estimated without destroying integrity when the file 8228 * moves into the indirect blocks (i.e., is large). If we want to 8229 * postpone fsck, we are stuck with this argument. 8230 */ 8231 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 8232 dp->di_ib[adp->ad_offset - NDADDR] = 0; 8233 } 8234 8235 /* 8236 * Version of initiate_write_inodeblock that handles UFS2 dinodes. 8237 * Note that any bug fixes made to this routine must be done in the 8238 * version found above. 8239 * 8240 * Called from within the procedure above to deal with unsatisfied 8241 * allocation dependencies in an inodeblock. The buffer must be 8242 * locked, thus, no I/O completion operations can occur while we 8243 * are manipulating its associated dependencies. 8244 */ 8245 static void 8246 initiate_write_inodeblock_ufs2(inodedep, bp) 8247 struct inodedep *inodedep; 8248 struct buf *bp; /* The inode block */ 8249 { 8250 struct allocdirect *adp, *lastadp; 8251 struct ufs2_dinode *dp; 8252 struct ufs2_dinode *sip; 8253 struct inoref *inoref; 8254 struct fs *fs; 8255 ufs_lbn_t i; 8256 #ifdef INVARIANTS 8257 ufs_lbn_t prevlbn = 0; 8258 #endif 8259 int deplist; 8260 8261 if (inodedep->id_state & IOSTARTED) 8262 panic("initiate_write_inodeblock_ufs2: already started"); 8263 inodedep->id_state |= IOSTARTED; 8264 fs = inodedep->id_fs; 8265 dp = (struct ufs2_dinode *)bp->b_data + 8266 ino_to_fsbo(fs, inodedep->id_ino); 8267 8268 /* 8269 * If we're on the unlinked list but have not yet written our 8270 * next pointer initialize it here. 8271 */ 8272 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 8273 struct inodedep *inon; 8274 8275 inon = TAILQ_NEXT(inodedep, id_unlinked); 8276 dp->di_freelink = inon ? inon->id_ino : 0; 8277 } 8278 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == 8279 (UNLINKED | UNLINKNEXT)) { 8280 struct inodedep *inon; 8281 ino_t freelink; 8282 8283 inon = TAILQ_NEXT(inodedep, id_unlinked); 8284 freelink = inon ? inon->id_ino : 0; 8285 if (freelink != dp->di_freelink) 8286 panic("ino %p(0x%X) %d, %d != %d", 8287 inodedep, inodedep->id_state, inodedep->id_ino, 8288 freelink, dp->di_freelink); 8289 } 8290 /* 8291 * If the bitmap is not yet written, then the allocated 8292 * inode cannot be written to disk. 8293 */ 8294 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 8295 if (inodedep->id_savedino2 != NULL) 8296 panic("initiate_write_inodeblock_ufs2: I/O underway"); 8297 FREE_LOCK(&lk); 8298 sip = malloc(sizeof(struct ufs2_dinode), 8299 M_SAVEDINO, M_SOFTDEP_FLAGS); 8300 ACQUIRE_LOCK(&lk); 8301 inodedep->id_savedino2 = sip; 8302 *inodedep->id_savedino2 = *dp; 8303 bzero((caddr_t)dp, sizeof(struct ufs2_dinode)); 8304 dp->di_gen = inodedep->id_savedino2->di_gen; 8305 dp->di_freelink = inodedep->id_savedino2->di_freelink; 8306 return; 8307 } 8308 /* 8309 * If no dependencies, then there is nothing to roll back. 8310 */ 8311 inodedep->id_savedsize = dp->di_size; 8312 inodedep->id_savedextsize = dp->di_extsize; 8313 inodedep->id_savednlink = dp->di_nlink; 8314 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 8315 TAILQ_EMPTY(&inodedep->id_extupdt) && 8316 TAILQ_EMPTY(&inodedep->id_inoreflst)) 8317 return; 8318 /* 8319 * Revert the link count to that of the first unwritten journal entry. 8320 */ 8321 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 8322 if (inoref) 8323 dp->di_nlink = inoref->if_nlink; 8324 8325 /* 8326 * Set the ext data dependencies to busy. 8327 */ 8328 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 8329 adp = TAILQ_NEXT(adp, ad_next)) { 8330 #ifdef INVARIANTS 8331 if (deplist != 0 && prevlbn >= adp->ad_offset) 8332 panic("softdep_write_inodeblock: lbn order"); 8333 prevlbn = adp->ad_offset; 8334 if (dp->di_extb[adp->ad_offset] != adp->ad_newblkno) 8335 panic("%s: direct pointer #%jd mismatch %jd != %jd", 8336 "softdep_write_inodeblock", 8337 (intmax_t)adp->ad_offset, 8338 (intmax_t)dp->di_extb[adp->ad_offset], 8339 (intmax_t)adp->ad_newblkno); 8340 deplist |= 1 << adp->ad_offset; 8341 if ((adp->ad_state & ATTACHED) == 0) 8342 panic("softdep_write_inodeblock: Unknown state 0x%x", 8343 adp->ad_state); 8344 #endif /* INVARIANTS */ 8345 adp->ad_state &= ~ATTACHED; 8346 adp->ad_state |= UNDONE; 8347 } 8348 /* 8349 * The on-disk inode cannot claim to be any larger than the last 8350 * fragment that has been written. Otherwise, the on-disk inode 8351 * might have fragments that were not the last block in the ext 8352 * data which would corrupt the filesystem. 8353 */ 8354 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 8355 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 8356 dp->di_extb[adp->ad_offset] = adp->ad_oldblkno; 8357 /* keep going until hitting a rollback to a frag */ 8358 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 8359 continue; 8360 dp->di_extsize = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 8361 for (i = adp->ad_offset + 1; i < NXADDR; i++) { 8362 #ifdef INVARIANTS 8363 if (dp->di_extb[i] != 0 && (deplist & (1 << i)) == 0) 8364 panic("softdep_write_inodeblock: lost dep1"); 8365 #endif /* INVARIANTS */ 8366 dp->di_extb[i] = 0; 8367 } 8368 lastadp = NULL; 8369 break; 8370 } 8371 /* 8372 * If we have zero'ed out the last allocated block of the ext 8373 * data, roll back the size to the last currently allocated block. 8374 * We know that this last allocated block is a full-sized as 8375 * we already checked for fragments in the loop above. 8376 */ 8377 if (lastadp != NULL && 8378 dp->di_extsize <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 8379 for (i = lastadp->ad_offset; i >= 0; i--) 8380 if (dp->di_extb[i] != 0) 8381 break; 8382 dp->di_extsize = (i + 1) * fs->fs_bsize; 8383 } 8384 /* 8385 * Set the file data dependencies to busy. 8386 */ 8387 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8388 adp = TAILQ_NEXT(adp, ad_next)) { 8389 #ifdef INVARIANTS 8390 if (deplist != 0 && prevlbn >= adp->ad_offset) 8391 panic("softdep_write_inodeblock: lbn order"); 8392 prevlbn = adp->ad_offset; 8393 if (adp->ad_offset < NDADDR && 8394 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 8395 panic("%s: direct pointer #%jd mismatch %jd != %jd", 8396 "softdep_write_inodeblock", 8397 (intmax_t)adp->ad_offset, 8398 (intmax_t)dp->di_db[adp->ad_offset], 8399 (intmax_t)adp->ad_newblkno); 8400 if (adp->ad_offset >= NDADDR && 8401 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno) 8402 panic("%s indirect pointer #%jd mismatch %jd != %jd", 8403 "softdep_write_inodeblock:", 8404 (intmax_t)adp->ad_offset - NDADDR, 8405 (intmax_t)dp->di_ib[adp->ad_offset - NDADDR], 8406 (intmax_t)adp->ad_newblkno); 8407 deplist |= 1 << adp->ad_offset; 8408 if ((adp->ad_state & ATTACHED) == 0) 8409 panic("softdep_write_inodeblock: Unknown state 0x%x", 8410 adp->ad_state); 8411 #endif /* INVARIANTS */ 8412 adp->ad_state &= ~ATTACHED; 8413 adp->ad_state |= UNDONE; 8414 } 8415 /* 8416 * The on-disk inode cannot claim to be any larger than the last 8417 * fragment that has been written. Otherwise, the on-disk inode 8418 * might have fragments that were not the last block in the file 8419 * which would corrupt the filesystem. 8420 */ 8421 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8422 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 8423 if (adp->ad_offset >= NDADDR) 8424 break; 8425 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 8426 /* keep going until hitting a rollback to a frag */ 8427 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 8428 continue; 8429 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 8430 for (i = adp->ad_offset + 1; i < NDADDR; i++) { 8431 #ifdef INVARIANTS 8432 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 8433 panic("softdep_write_inodeblock: lost dep2"); 8434 #endif /* INVARIANTS */ 8435 dp->di_db[i] = 0; 8436 } 8437 for (i = 0; i < NIADDR; i++) { 8438 #ifdef INVARIANTS 8439 if (dp->di_ib[i] != 0 && 8440 (deplist & ((1 << NDADDR) << i)) == 0) 8441 panic("softdep_write_inodeblock: lost dep3"); 8442 #endif /* INVARIANTS */ 8443 dp->di_ib[i] = 0; 8444 } 8445 return; 8446 } 8447 /* 8448 * If we have zero'ed out the last allocated block of the file, 8449 * roll back the size to the last currently allocated block. 8450 * We know that this last allocated block is a full-sized as 8451 * we already checked for fragments in the loop above. 8452 */ 8453 if (lastadp != NULL && 8454 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 8455 for (i = lastadp->ad_offset; i >= 0; i--) 8456 if (dp->di_db[i] != 0) 8457 break; 8458 dp->di_size = (i + 1) * fs->fs_bsize; 8459 } 8460 /* 8461 * The only dependencies are for indirect blocks. 8462 * 8463 * The file size for indirect block additions is not guaranteed. 8464 * Such a guarantee would be non-trivial to achieve. The conventional 8465 * synchronous write implementation also does not make this guarantee. 8466 * Fsck should catch and fix discrepancies. Arguably, the file size 8467 * can be over-estimated without destroying integrity when the file 8468 * moves into the indirect blocks (i.e., is large). If we want to 8469 * postpone fsck, we are stuck with this argument. 8470 */ 8471 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 8472 dp->di_ib[adp->ad_offset - NDADDR] = 0; 8473 } 8474 8475 /* 8476 * Cancel an indirdep as a result of truncation. Release all of the 8477 * children allocindirs and place their journal work on the appropriate 8478 * list. 8479 */ 8480 static void 8481 cancel_indirdep(indirdep, bp, inodedep, freeblks) 8482 struct indirdep *indirdep; 8483 struct buf *bp; 8484 struct inodedep *inodedep; 8485 struct freeblks *freeblks; 8486 { 8487 struct allocindir *aip; 8488 8489 /* 8490 * None of the indirect pointers will ever be visible, 8491 * so they can simply be tossed. GOINGAWAY ensures 8492 * that allocated pointers will be saved in the buffer 8493 * cache until they are freed. Note that they will 8494 * only be able to be found by their physical address 8495 * since the inode mapping the logical address will 8496 * be gone. The save buffer used for the safe copy 8497 * was allocated in setup_allocindir_phase2 using 8498 * the physical address so it could be used for this 8499 * purpose. Hence we swap the safe copy with the real 8500 * copy, allowing the safe copy to be freed and holding 8501 * on to the real copy for later use in indir_trunc. 8502 */ 8503 if (indirdep->ir_state & GOINGAWAY) 8504 panic("cancel_indirdep: already gone"); 8505 if (indirdep->ir_state & ONDEPLIST) { 8506 indirdep->ir_state &= ~ONDEPLIST; 8507 LIST_REMOVE(indirdep, ir_next); 8508 } 8509 indirdep->ir_state |= GOINGAWAY; 8510 VFSTOUFS(indirdep->ir_list.wk_mp)->um_numindirdeps += 1; 8511 while ((aip = LIST_FIRST(&indirdep->ir_deplisthd)) != 0) 8512 cancel_allocindir(aip, inodedep, freeblks); 8513 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != 0) 8514 cancel_allocindir(aip, inodedep, freeblks); 8515 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != 0) 8516 cancel_allocindir(aip, inodedep, freeblks); 8517 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != 0) 8518 cancel_allocindir(aip, inodedep, freeblks); 8519 bcopy(bp->b_data, indirdep->ir_savebp->b_data, bp->b_bcount); 8520 WORKLIST_REMOVE(&indirdep->ir_list); 8521 WORKLIST_INSERT(&indirdep->ir_savebp->b_dep, &indirdep->ir_list); 8522 indirdep->ir_savebp = NULL; 8523 } 8524 8525 /* 8526 * Free an indirdep once it no longer has new pointers to track. 8527 */ 8528 static void 8529 free_indirdep(indirdep) 8530 struct indirdep *indirdep; 8531 { 8532 8533 KASSERT(LIST_EMPTY(&indirdep->ir_jwork), 8534 ("free_indirdep: Journal work not empty.")); 8535 KASSERT(LIST_EMPTY(&indirdep->ir_completehd), 8536 ("free_indirdep: Complete head not empty.")); 8537 KASSERT(LIST_EMPTY(&indirdep->ir_writehd), 8538 ("free_indirdep: write head not empty.")); 8539 KASSERT(LIST_EMPTY(&indirdep->ir_donehd), 8540 ("free_indirdep: done head not empty.")); 8541 KASSERT(LIST_EMPTY(&indirdep->ir_deplisthd), 8542 ("free_indirdep: deplist head not empty.")); 8543 KASSERT(indirdep->ir_savebp == NULL, 8544 ("free_indirdep: %p ir_savebp != NULL", indirdep)); 8545 KASSERT((indirdep->ir_state & ONDEPLIST) == 0, 8546 ("free_indirdep: %p still on deplist.", indirdep)); 8547 if (indirdep->ir_state & ONWORKLIST) 8548 WORKLIST_REMOVE(&indirdep->ir_list); 8549 WORKITEM_FREE(indirdep, D_INDIRDEP); 8550 } 8551 8552 /* 8553 * Called before a write to an indirdep. This routine is responsible for 8554 * rolling back pointers to a safe state which includes only those 8555 * allocindirs which have been completed. 8556 */ 8557 static void 8558 initiate_write_indirdep(indirdep, bp) 8559 struct indirdep *indirdep; 8560 struct buf *bp; 8561 { 8562 8563 if (indirdep->ir_state & GOINGAWAY) 8564 panic("disk_io_initiation: indirdep gone"); 8565 8566 /* 8567 * If there are no remaining dependencies, this will be writing 8568 * the real pointers. 8569 */ 8570 if (LIST_EMPTY(&indirdep->ir_deplisthd)) 8571 return; 8572 /* 8573 * Replace up-to-date version with safe version. 8574 */ 8575 FREE_LOCK(&lk); 8576 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP, 8577 M_SOFTDEP_FLAGS); 8578 ACQUIRE_LOCK(&lk); 8579 indirdep->ir_state &= ~ATTACHED; 8580 indirdep->ir_state |= UNDONE; 8581 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount); 8582 bcopy(indirdep->ir_savebp->b_data, bp->b_data, 8583 bp->b_bcount); 8584 } 8585 8586 /* 8587 * Called when an inode has been cleared in a cg bitmap. This finally 8588 * eliminates any canceled jaddrefs 8589 */ 8590 void 8591 softdep_setup_inofree(mp, bp, ino, wkhd) 8592 struct mount *mp; 8593 struct buf *bp; 8594 ino_t ino; 8595 struct workhead *wkhd; 8596 { 8597 struct worklist *wk, *wkn; 8598 struct inodedep *inodedep; 8599 uint8_t *inosused; 8600 struct cg *cgp; 8601 struct fs *fs; 8602 8603 ACQUIRE_LOCK(&lk); 8604 fs = VFSTOUFS(mp)->um_fs; 8605 cgp = (struct cg *)bp->b_data; 8606 inosused = cg_inosused(cgp); 8607 if (isset(inosused, ino % fs->fs_ipg)) 8608 panic("softdep_setup_inofree: inode %d not freed.", ino); 8609 if (inodedep_lookup(mp, ino, 0, &inodedep)) 8610 panic("softdep_setup_inofree: ino %d has existing inodedep %p", 8611 ino, inodedep); 8612 if (wkhd) { 8613 LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) { 8614 if (wk->wk_type != D_JADDREF) 8615 continue; 8616 WORKLIST_REMOVE(wk); 8617 /* 8618 * We can free immediately even if the jaddref 8619 * isn't attached in a background write as now 8620 * the bitmaps are reconciled. 8621 */ 8622 wk->wk_state |= COMPLETE | ATTACHED; 8623 free_jaddref(WK_JADDREF(wk)); 8624 } 8625 jwork_move(&bp->b_dep, wkhd); 8626 } 8627 FREE_LOCK(&lk); 8628 } 8629 8630 8631 /* 8632 * Called via ffs_blkfree() after a set of frags has been cleared from a cg 8633 * map. Any dependencies waiting for the write to clear are added to the 8634 * buf's list and any jnewblks that are being canceled are discarded 8635 * immediately. 8636 */ 8637 void 8638 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 8639 struct mount *mp; 8640 struct buf *bp; 8641 ufs2_daddr_t blkno; 8642 int frags; 8643 struct workhead *wkhd; 8644 { 8645 struct jnewblk *jnewblk; 8646 struct worklist *wk, *wkn; 8647 #ifdef SUJ_DEBUG 8648 struct bmsafemap *bmsafemap; 8649 struct fs *fs; 8650 uint8_t *blksfree; 8651 struct cg *cgp; 8652 ufs2_daddr_t jstart; 8653 ufs2_daddr_t jend; 8654 ufs2_daddr_t end; 8655 long bno; 8656 int i; 8657 #endif 8658 8659 ACQUIRE_LOCK(&lk); 8660 /* 8661 * Detach any jnewblks which have been canceled. They must linger 8662 * until the bitmap is cleared again by ffs_blkfree() to prevent 8663 * an unjournaled allocation from hitting the disk. 8664 */ 8665 if (wkhd) { 8666 LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) { 8667 if (wk->wk_type != D_JNEWBLK) 8668 continue; 8669 jnewblk = WK_JNEWBLK(wk); 8670 KASSERT(jnewblk->jn_state & GOINGAWAY, 8671 ("softdep_setup_blkfree: jnewblk not canceled.")); 8672 WORKLIST_REMOVE(wk); 8673 #ifdef SUJ_DEBUG 8674 /* 8675 * Assert that this block is free in the bitmap 8676 * before we discard the jnewblk. 8677 */ 8678 fs = VFSTOUFS(mp)->um_fs; 8679 cgp = (struct cg *)bp->b_data; 8680 blksfree = cg_blksfree(cgp); 8681 bno = dtogd(fs, jnewblk->jn_blkno); 8682 for (i = jnewblk->jn_oldfrags; 8683 i < jnewblk->jn_frags; i++) { 8684 if (isset(blksfree, bno + i)) 8685 continue; 8686 panic("softdep_setup_blkfree: not free"); 8687 } 8688 #endif 8689 /* 8690 * Even if it's not attached we can free immediately 8691 * as the new bitmap is correct. 8692 */ 8693 wk->wk_state |= COMPLETE | ATTACHED; 8694 free_jnewblk(jnewblk); 8695 } 8696 /* 8697 * The buf must be locked by the caller otherwise these could 8698 * be added while it's being written and the write would 8699 * complete them before they made it to disk. 8700 */ 8701 jwork_move(&bp->b_dep, wkhd); 8702 } 8703 8704 #ifdef SUJ_DEBUG 8705 /* 8706 * Assert that we are not freeing a block which has an outstanding 8707 * allocation dependency. 8708 */ 8709 fs = VFSTOUFS(mp)->um_fs; 8710 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno)); 8711 end = blkno + frags; 8712 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 8713 /* 8714 * Don't match against blocks that will be freed when the 8715 * background write is done. 8716 */ 8717 if ((jnewblk->jn_state & (ATTACHED | COMPLETE | DEPCOMPLETE)) == 8718 (COMPLETE | DEPCOMPLETE)) 8719 continue; 8720 jstart = jnewblk->jn_blkno + jnewblk->jn_oldfrags; 8721 jend = jnewblk->jn_blkno + jnewblk->jn_frags; 8722 if ((blkno >= jstart && blkno < jend) || 8723 (end > jstart && end <= jend)) { 8724 printf("state 0x%X %jd - %d %d dep %p\n", 8725 jnewblk->jn_state, jnewblk->jn_blkno, 8726 jnewblk->jn_oldfrags, jnewblk->jn_frags, 8727 jnewblk->jn_newblk); 8728 panic("softdep_setup_blkfree: " 8729 "%jd-%jd(%d) overlaps with %jd-%jd", 8730 blkno, end, frags, jstart, jend); 8731 } 8732 } 8733 #endif 8734 FREE_LOCK(&lk); 8735 } 8736 8737 static void 8738 initiate_write_bmsafemap(bmsafemap, bp) 8739 struct bmsafemap *bmsafemap; 8740 struct buf *bp; /* The cg block. */ 8741 { 8742 struct jaddref *jaddref; 8743 struct jnewblk *jnewblk; 8744 uint8_t *inosused; 8745 uint8_t *blksfree; 8746 struct cg *cgp; 8747 struct fs *fs; 8748 int cleared; 8749 ino_t ino; 8750 long bno; 8751 int i; 8752 8753 if (bmsafemap->sm_state & IOSTARTED) 8754 panic("initiate_write_bmsafemap: Already started\n"); 8755 bmsafemap->sm_state |= IOSTARTED; 8756 /* 8757 * Clear any inode allocations which are pending journal writes. 8758 */ 8759 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd) != NULL) { 8760 cgp = (struct cg *)bp->b_data; 8761 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 8762 inosused = cg_inosused(cgp); 8763 LIST_FOREACH(jaddref, &bmsafemap->sm_jaddrefhd, ja_bmdeps) { 8764 ino = jaddref->ja_ino % fs->fs_ipg; 8765 /* 8766 * If this is a background copy the inode may not 8767 * be marked used yet. 8768 */ 8769 if (isset(inosused, ino)) { 8770 if ((jaddref->ja_mode & IFMT) == IFDIR) 8771 cgp->cg_cs.cs_ndir--; 8772 cgp->cg_cs.cs_nifree++; 8773 clrbit(inosused, ino); 8774 jaddref->ja_state &= ~ATTACHED; 8775 jaddref->ja_state |= UNDONE; 8776 stat_jaddref++; 8777 } else if ((bp->b_xflags & BX_BKGRDMARKER) == 0) 8778 panic("initiate_write_bmsafemap: inode %d " 8779 "marked free", jaddref->ja_ino); 8780 } 8781 } 8782 /* 8783 * Clear any block allocations which are pending journal writes. 8784 */ 8785 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 8786 cgp = (struct cg *)bp->b_data; 8787 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 8788 blksfree = cg_blksfree(cgp); 8789 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 8790 bno = dtogd(fs, jnewblk->jn_blkno); 8791 cleared = 0; 8792 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 8793 i++) { 8794 if (isclr(blksfree, bno + i)) { 8795 cleared = 1; 8796 setbit(blksfree, bno + i); 8797 } 8798 } 8799 /* 8800 * We may not clear the block if it's a background 8801 * copy. In that case there is no reason to detach 8802 * it. 8803 */ 8804 if (cleared) { 8805 stat_jnewblk++; 8806 jnewblk->jn_state &= ~ATTACHED; 8807 jnewblk->jn_state |= UNDONE; 8808 } else if ((bp->b_xflags & BX_BKGRDMARKER) == 0) 8809 panic("initiate_write_bmsafemap: block %jd " 8810 "marked free", jnewblk->jn_blkno); 8811 } 8812 } 8813 /* 8814 * Move allocation lists to the written lists so they can be 8815 * cleared once the block write is complete. 8816 */ 8817 LIST_SWAP(&bmsafemap->sm_inodedephd, &bmsafemap->sm_inodedepwr, 8818 inodedep, id_deps); 8819 LIST_SWAP(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr, 8820 newblk, nb_deps); 8821 } 8822 8823 /* 8824 * This routine is called during the completion interrupt 8825 * service routine for a disk write (from the procedure called 8826 * by the device driver to inform the filesystem caches of 8827 * a request completion). It should be called early in this 8828 * procedure, before the block is made available to other 8829 * processes or other routines are called. 8830 * 8831 */ 8832 static void 8833 softdep_disk_write_complete(bp) 8834 struct buf *bp; /* describes the completed disk write */ 8835 { 8836 struct worklist *wk; 8837 struct worklist *owk; 8838 struct workhead reattach; 8839 struct buf *sbp; 8840 8841 /* 8842 * If an error occurred while doing the write, then the data 8843 * has not hit the disk and the dependencies cannot be unrolled. 8844 */ 8845 if ((bp->b_ioflags & BIO_ERROR) != 0 && (bp->b_flags & B_INVAL) == 0) 8846 return; 8847 LIST_INIT(&reattach); 8848 /* 8849 * This lock must not be released anywhere in this code segment. 8850 */ 8851 sbp = NULL; 8852 owk = NULL; 8853 ACQUIRE_LOCK(&lk); 8854 while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) { 8855 WORKLIST_REMOVE(wk); 8856 if (wk == owk) 8857 panic("duplicate worklist: %p\n", wk); 8858 owk = wk; 8859 switch (wk->wk_type) { 8860 8861 case D_PAGEDEP: 8862 if (handle_written_filepage(WK_PAGEDEP(wk), bp)) 8863 WORKLIST_INSERT(&reattach, wk); 8864 continue; 8865 8866 case D_INODEDEP: 8867 if (handle_written_inodeblock(WK_INODEDEP(wk), bp)) 8868 WORKLIST_INSERT(&reattach, wk); 8869 continue; 8870 8871 case D_BMSAFEMAP: 8872 if (handle_written_bmsafemap(WK_BMSAFEMAP(wk), bp)) 8873 WORKLIST_INSERT(&reattach, wk); 8874 continue; 8875 8876 case D_MKDIR: 8877 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 8878 continue; 8879 8880 case D_ALLOCDIRECT: 8881 wk->wk_state |= COMPLETE; 8882 handle_allocdirect_partdone(WK_ALLOCDIRECT(wk), NULL); 8883 continue; 8884 8885 case D_ALLOCINDIR: 8886 wk->wk_state |= COMPLETE; 8887 handle_allocindir_partdone(WK_ALLOCINDIR(wk)); 8888 continue; 8889 8890 case D_INDIRDEP: 8891 if (handle_written_indirdep(WK_INDIRDEP(wk), bp, &sbp)) 8892 WORKLIST_INSERT(&reattach, wk); 8893 continue; 8894 8895 case D_FREEBLKS: 8896 wk->wk_state |= COMPLETE; 8897 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE) 8898 add_to_worklist(wk, 1); 8899 continue; 8900 8901 case D_FREEWORK: 8902 handle_written_freework(WK_FREEWORK(wk)); 8903 break; 8904 8905 case D_FREEDEP: 8906 free_freedep(WK_FREEDEP(wk)); 8907 continue; 8908 8909 case D_JSEGDEP: 8910 free_jsegdep(WK_JSEGDEP(wk)); 8911 continue; 8912 8913 case D_JSEG: 8914 handle_written_jseg(WK_JSEG(wk), bp); 8915 continue; 8916 8917 case D_SBDEP: 8918 if (handle_written_sbdep(WK_SBDEP(wk), bp)) 8919 WORKLIST_INSERT(&reattach, wk); 8920 continue; 8921 8922 default: 8923 panic("handle_disk_write_complete: Unknown type %s", 8924 TYPENAME(wk->wk_type)); 8925 /* NOTREACHED */ 8926 } 8927 } 8928 /* 8929 * Reattach any requests that must be redone. 8930 */ 8931 while ((wk = LIST_FIRST(&reattach)) != NULL) { 8932 WORKLIST_REMOVE(wk); 8933 WORKLIST_INSERT(&bp->b_dep, wk); 8934 } 8935 FREE_LOCK(&lk); 8936 if (sbp) 8937 brelse(sbp); 8938 } 8939 8940 /* 8941 * Called from within softdep_disk_write_complete above. Note that 8942 * this routine is always called from interrupt level with further 8943 * splbio interrupts blocked. 8944 */ 8945 static void 8946 handle_allocdirect_partdone(adp, wkhd) 8947 struct allocdirect *adp; /* the completed allocdirect */ 8948 struct workhead *wkhd; /* Work to do when inode is writtne. */ 8949 { 8950 struct allocdirectlst *listhead; 8951 struct allocdirect *listadp; 8952 struct inodedep *inodedep; 8953 long bsize; 8954 8955 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 8956 return; 8957 /* 8958 * The on-disk inode cannot claim to be any larger than the last 8959 * fragment that has been written. Otherwise, the on-disk inode 8960 * might have fragments that were not the last block in the file 8961 * which would corrupt the filesystem. Thus, we cannot free any 8962 * allocdirects after one whose ad_oldblkno claims a fragment as 8963 * these blocks must be rolled back to zero before writing the inode. 8964 * We check the currently active set of allocdirects in id_inoupdt 8965 * or id_extupdt as appropriate. 8966 */ 8967 inodedep = adp->ad_inodedep; 8968 bsize = inodedep->id_fs->fs_bsize; 8969 if (adp->ad_state & EXTDATA) 8970 listhead = &inodedep->id_extupdt; 8971 else 8972 listhead = &inodedep->id_inoupdt; 8973 TAILQ_FOREACH(listadp, listhead, ad_next) { 8974 /* found our block */ 8975 if (listadp == adp) 8976 break; 8977 /* continue if ad_oldlbn is not a fragment */ 8978 if (listadp->ad_oldsize == 0 || 8979 listadp->ad_oldsize == bsize) 8980 continue; 8981 /* hit a fragment */ 8982 return; 8983 } 8984 /* 8985 * If we have reached the end of the current list without 8986 * finding the just finished dependency, then it must be 8987 * on the future dependency list. Future dependencies cannot 8988 * be freed until they are moved to the current list. 8989 */ 8990 if (listadp == NULL) { 8991 #ifdef DEBUG 8992 if (adp->ad_state & EXTDATA) 8993 listhead = &inodedep->id_newextupdt; 8994 else 8995 listhead = &inodedep->id_newinoupdt; 8996 TAILQ_FOREACH(listadp, listhead, ad_next) 8997 /* found our block */ 8998 if (listadp == adp) 8999 break; 9000 if (listadp == NULL) 9001 panic("handle_allocdirect_partdone: lost dep"); 9002 #endif /* DEBUG */ 9003 return; 9004 } 9005 /* 9006 * If we have found the just finished dependency, then queue 9007 * it along with anything that follows it that is complete. 9008 * Since the pointer has not yet been written in the inode 9009 * as the dependency prevents it, place the allocdirect on the 9010 * bufwait list where it will be freed once the pointer is 9011 * valid. 9012 */ 9013 if (wkhd == NULL) 9014 wkhd = &inodedep->id_bufwait; 9015 for (; adp; adp = listadp) { 9016 listadp = TAILQ_NEXT(adp, ad_next); 9017 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 9018 return; 9019 TAILQ_REMOVE(listhead, adp, ad_next); 9020 WORKLIST_INSERT(wkhd, &adp->ad_block.nb_list); 9021 } 9022 } 9023 9024 /* 9025 * Called from within softdep_disk_write_complete above. This routine 9026 * completes successfully written allocindirs. 9027 */ 9028 static void 9029 handle_allocindir_partdone(aip) 9030 struct allocindir *aip; /* the completed allocindir */ 9031 { 9032 struct indirdep *indirdep; 9033 9034 if ((aip->ai_state & ALLCOMPLETE) != ALLCOMPLETE) 9035 return; 9036 indirdep = aip->ai_indirdep; 9037 LIST_REMOVE(aip, ai_next); 9038 if (indirdep->ir_state & UNDONE) { 9039 LIST_INSERT_HEAD(&indirdep->ir_donehd, aip, ai_next); 9040 return; 9041 } 9042 if (indirdep->ir_state & UFS1FMT) 9043 ((ufs1_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 9044 aip->ai_newblkno; 9045 else 9046 ((ufs2_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 9047 aip->ai_newblkno; 9048 /* 9049 * Await the pointer write before freeing the allocindir. 9050 */ 9051 LIST_INSERT_HEAD(&indirdep->ir_writehd, aip, ai_next); 9052 } 9053 9054 /* 9055 * Release segments held on a jwork list. 9056 */ 9057 static void 9058 handle_jwork(wkhd) 9059 struct workhead *wkhd; 9060 { 9061 struct worklist *wk; 9062 9063 while ((wk = LIST_FIRST(wkhd)) != NULL) { 9064 WORKLIST_REMOVE(wk); 9065 switch (wk->wk_type) { 9066 case D_JSEGDEP: 9067 free_jsegdep(WK_JSEGDEP(wk)); 9068 continue; 9069 default: 9070 panic("handle_jwork: Unknown type %s\n", 9071 TYPENAME(wk->wk_type)); 9072 } 9073 } 9074 } 9075 9076 /* 9077 * Handle the bufwait list on an inode when it is safe to release items 9078 * held there. This normally happens after an inode block is written but 9079 * may be delayed and handle later if there are pending journal items that 9080 * are not yet safe to be released. 9081 */ 9082 static struct freefile * 9083 handle_bufwait(inodedep, refhd) 9084 struct inodedep *inodedep; 9085 struct workhead *refhd; 9086 { 9087 struct jaddref *jaddref; 9088 struct freefile *freefile; 9089 struct worklist *wk; 9090 9091 freefile = NULL; 9092 while ((wk = LIST_FIRST(&inodedep->id_bufwait)) != NULL) { 9093 WORKLIST_REMOVE(wk); 9094 switch (wk->wk_type) { 9095 case D_FREEFILE: 9096 /* 9097 * We defer adding freefile to the worklist 9098 * until all other additions have been made to 9099 * ensure that it will be done after all the 9100 * old blocks have been freed. 9101 */ 9102 if (freefile != NULL) 9103 panic("handle_bufwait: freefile"); 9104 freefile = WK_FREEFILE(wk); 9105 continue; 9106 9107 case D_MKDIR: 9108 handle_written_mkdir(WK_MKDIR(wk), MKDIR_PARENT); 9109 continue; 9110 9111 case D_DIRADD: 9112 diradd_inode_written(WK_DIRADD(wk), inodedep); 9113 continue; 9114 9115 case D_FREEFRAG: 9116 wk->wk_state |= COMPLETE; 9117 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE) 9118 add_to_worklist(wk, 0); 9119 continue; 9120 9121 case D_DIRREM: 9122 wk->wk_state |= COMPLETE; 9123 add_to_worklist(wk, 0); 9124 continue; 9125 9126 case D_ALLOCDIRECT: 9127 case D_ALLOCINDIR: 9128 free_newblk(WK_NEWBLK(wk)); 9129 continue; 9130 9131 case D_JNEWBLK: 9132 wk->wk_state |= COMPLETE; 9133 free_jnewblk(WK_JNEWBLK(wk)); 9134 continue; 9135 9136 /* 9137 * Save freed journal segments and add references on 9138 * the supplied list which will delay their release 9139 * until the cg bitmap is cleared on disk. 9140 */ 9141 case D_JSEGDEP: 9142 if (refhd == NULL) 9143 free_jsegdep(WK_JSEGDEP(wk)); 9144 else 9145 WORKLIST_INSERT(refhd, wk); 9146 continue; 9147 9148 case D_JADDREF: 9149 jaddref = WK_JADDREF(wk); 9150 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 9151 if_deps); 9152 /* 9153 * Transfer any jaddrefs to the list to be freed with 9154 * the bitmap if we're handling a removed file. 9155 */ 9156 if (refhd == NULL) { 9157 wk->wk_state |= COMPLETE; 9158 free_jaddref(jaddref); 9159 } else 9160 WORKLIST_INSERT(refhd, wk); 9161 continue; 9162 9163 default: 9164 panic("handle_bufwait: Unknown type %p(%s)", 9165 wk, TYPENAME(wk->wk_type)); 9166 /* NOTREACHED */ 9167 } 9168 } 9169 return (freefile); 9170 } 9171 /* 9172 * Called from within softdep_disk_write_complete above to restore 9173 * in-memory inode block contents to their most up-to-date state. Note 9174 * that this routine is always called from interrupt level with further 9175 * splbio interrupts blocked. 9176 */ 9177 static int 9178 handle_written_inodeblock(inodedep, bp) 9179 struct inodedep *inodedep; 9180 struct buf *bp; /* buffer containing the inode block */ 9181 { 9182 struct freefile *freefile; 9183 struct allocdirect *adp, *nextadp; 9184 struct ufs1_dinode *dp1 = NULL; 9185 struct ufs2_dinode *dp2 = NULL; 9186 struct workhead wkhd; 9187 int hadchanges, fstype; 9188 ino_t freelink; 9189 9190 LIST_INIT(&wkhd); 9191 hadchanges = 0; 9192 freefile = NULL; 9193 if ((inodedep->id_state & IOSTARTED) == 0) 9194 panic("handle_written_inodeblock: not started"); 9195 inodedep->id_state &= ~IOSTARTED; 9196 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) { 9197 fstype = UFS1; 9198 dp1 = (struct ufs1_dinode *)bp->b_data + 9199 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 9200 freelink = dp1->di_freelink; 9201 } else { 9202 fstype = UFS2; 9203 dp2 = (struct ufs2_dinode *)bp->b_data + 9204 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 9205 freelink = dp2->di_freelink; 9206 } 9207 /* 9208 * If we wrote a valid freelink pointer during the last write 9209 * record it here. 9210 */ 9211 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 9212 struct inodedep *inon; 9213 9214 inon = TAILQ_NEXT(inodedep, id_unlinked); 9215 if ((inon == NULL && freelink == 0) || 9216 (inon && inon->id_ino == freelink)) { 9217 if (inon) 9218 inon->id_state |= UNLINKPREV; 9219 inodedep->id_state |= UNLINKNEXT; 9220 } else 9221 hadchanges = 1; 9222 } 9223 /* Leave this inodeblock dirty until it's in the list. */ 9224 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) == UNLINKED) 9225 hadchanges = 1; 9226 /* 9227 * If we had to rollback the inode allocation because of 9228 * bitmaps being incomplete, then simply restore it. 9229 * Keep the block dirty so that it will not be reclaimed until 9230 * all associated dependencies have been cleared and the 9231 * corresponding updates written to disk. 9232 */ 9233 if (inodedep->id_savedino1 != NULL) { 9234 hadchanges = 1; 9235 if (fstype == UFS1) 9236 *dp1 = *inodedep->id_savedino1; 9237 else 9238 *dp2 = *inodedep->id_savedino2; 9239 free(inodedep->id_savedino1, M_SAVEDINO); 9240 inodedep->id_savedino1 = NULL; 9241 if ((bp->b_flags & B_DELWRI) == 0) 9242 stat_inode_bitmap++; 9243 bdirty(bp); 9244 /* 9245 * If the inode is clear here and GOINGAWAY it will never 9246 * be written. Process the bufwait and clear any pending 9247 * work which may include the freefile. 9248 */ 9249 if (inodedep->id_state & GOINGAWAY) 9250 goto bufwait; 9251 return (1); 9252 } 9253 inodedep->id_state |= COMPLETE; 9254 /* 9255 * Roll forward anything that had to be rolled back before 9256 * the inode could be updated. 9257 */ 9258 for (adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; adp = nextadp) { 9259 nextadp = TAILQ_NEXT(adp, ad_next); 9260 if (adp->ad_state & ATTACHED) 9261 panic("handle_written_inodeblock: new entry"); 9262 if (fstype == UFS1) { 9263 if (adp->ad_offset < NDADDR) { 9264 if (dp1->di_db[adp->ad_offset]!=adp->ad_oldblkno) 9265 panic("%s %s #%jd mismatch %d != %jd", 9266 "handle_written_inodeblock:", 9267 "direct pointer", 9268 (intmax_t)adp->ad_offset, 9269 dp1->di_db[adp->ad_offset], 9270 (intmax_t)adp->ad_oldblkno); 9271 dp1->di_db[adp->ad_offset] = adp->ad_newblkno; 9272 } else { 9273 if (dp1->di_ib[adp->ad_offset - NDADDR] != 0) 9274 panic("%s: %s #%jd allocated as %d", 9275 "handle_written_inodeblock", 9276 "indirect pointer", 9277 (intmax_t)adp->ad_offset - NDADDR, 9278 dp1->di_ib[adp->ad_offset - NDADDR]); 9279 dp1->di_ib[adp->ad_offset - NDADDR] = 9280 adp->ad_newblkno; 9281 } 9282 } else { 9283 if (adp->ad_offset < NDADDR) { 9284 if (dp2->di_db[adp->ad_offset]!=adp->ad_oldblkno) 9285 panic("%s: %s #%jd %s %jd != %jd", 9286 "handle_written_inodeblock", 9287 "direct pointer", 9288 (intmax_t)adp->ad_offset, "mismatch", 9289 (intmax_t)dp2->di_db[adp->ad_offset], 9290 (intmax_t)adp->ad_oldblkno); 9291 dp2->di_db[adp->ad_offset] = adp->ad_newblkno; 9292 } else { 9293 if (dp2->di_ib[adp->ad_offset - NDADDR] != 0) 9294 panic("%s: %s #%jd allocated as %jd", 9295 "handle_written_inodeblock", 9296 "indirect pointer", 9297 (intmax_t)adp->ad_offset - NDADDR, 9298 (intmax_t) 9299 dp2->di_ib[adp->ad_offset - NDADDR]); 9300 dp2->di_ib[adp->ad_offset - NDADDR] = 9301 adp->ad_newblkno; 9302 } 9303 } 9304 adp->ad_state &= ~UNDONE; 9305 adp->ad_state |= ATTACHED; 9306 hadchanges = 1; 9307 } 9308 for (adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; adp = nextadp) { 9309 nextadp = TAILQ_NEXT(adp, ad_next); 9310 if (adp->ad_state & ATTACHED) 9311 panic("handle_written_inodeblock: new entry"); 9312 if (dp2->di_extb[adp->ad_offset] != adp->ad_oldblkno) 9313 panic("%s: direct pointers #%jd %s %jd != %jd", 9314 "handle_written_inodeblock", 9315 (intmax_t)adp->ad_offset, "mismatch", 9316 (intmax_t)dp2->di_extb[adp->ad_offset], 9317 (intmax_t)adp->ad_oldblkno); 9318 dp2->di_extb[adp->ad_offset] = adp->ad_newblkno; 9319 adp->ad_state &= ~UNDONE; 9320 adp->ad_state |= ATTACHED; 9321 hadchanges = 1; 9322 } 9323 if (hadchanges && (bp->b_flags & B_DELWRI) == 0) 9324 stat_direct_blk_ptrs++; 9325 /* 9326 * Reset the file size to its most up-to-date value. 9327 */ 9328 if (inodedep->id_savedsize == -1 || inodedep->id_savedextsize == -1) 9329 panic("handle_written_inodeblock: bad size"); 9330 if (inodedep->id_savednlink > LINK_MAX) 9331 panic("handle_written_inodeblock: Invalid link count " 9332 "%d for inodedep %p", inodedep->id_savednlink, inodedep); 9333 if (fstype == UFS1) { 9334 if (dp1->di_nlink != inodedep->id_savednlink) { 9335 dp1->di_nlink = inodedep->id_savednlink; 9336 hadchanges = 1; 9337 } 9338 if (dp1->di_size != inodedep->id_savedsize) { 9339 dp1->di_size = inodedep->id_savedsize; 9340 hadchanges = 1; 9341 } 9342 } else { 9343 if (dp2->di_nlink != inodedep->id_savednlink) { 9344 dp2->di_nlink = inodedep->id_savednlink; 9345 hadchanges = 1; 9346 } 9347 if (dp2->di_size != inodedep->id_savedsize) { 9348 dp2->di_size = inodedep->id_savedsize; 9349 hadchanges = 1; 9350 } 9351 if (dp2->di_extsize != inodedep->id_savedextsize) { 9352 dp2->di_extsize = inodedep->id_savedextsize; 9353 hadchanges = 1; 9354 } 9355 } 9356 inodedep->id_savedsize = -1; 9357 inodedep->id_savedextsize = -1; 9358 inodedep->id_savednlink = -1; 9359 /* 9360 * If there were any rollbacks in the inode block, then it must be 9361 * marked dirty so that its will eventually get written back in 9362 * its correct form. 9363 */ 9364 if (hadchanges) 9365 bdirty(bp); 9366 bufwait: 9367 /* 9368 * Process any allocdirects that completed during the update. 9369 */ 9370 if ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL) 9371 handle_allocdirect_partdone(adp, &wkhd); 9372 if ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 9373 handle_allocdirect_partdone(adp, &wkhd); 9374 /* 9375 * Process deallocations that were held pending until the 9376 * inode had been written to disk. Freeing of the inode 9377 * is delayed until after all blocks have been freed to 9378 * avoid creation of new <vfsid, inum, lbn> triples 9379 * before the old ones have been deleted. Completely 9380 * unlinked inodes are not processed until the unlinked 9381 * inode list is written or the last reference is removed. 9382 */ 9383 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) != UNLINKED) { 9384 freefile = handle_bufwait(inodedep, NULL); 9385 if (freefile && !LIST_EMPTY(&wkhd)) { 9386 WORKLIST_INSERT(&wkhd, &freefile->fx_list); 9387 freefile = NULL; 9388 } 9389 } 9390 /* 9391 * Move rolled forward dependency completions to the bufwait list 9392 * now that those that were already written have been processed. 9393 */ 9394 if (!LIST_EMPTY(&wkhd) && hadchanges == 0) 9395 panic("handle_written_inodeblock: bufwait but no changes"); 9396 jwork_move(&inodedep->id_bufwait, &wkhd); 9397 9398 if (freefile != NULL) { 9399 /* 9400 * If the inode is goingaway it was never written. Fake up 9401 * the state here so free_inodedep() can succeed. 9402 */ 9403 if (inodedep->id_state & GOINGAWAY) 9404 inodedep->id_state |= COMPLETE | DEPCOMPLETE; 9405 if (free_inodedep(inodedep) == 0) 9406 panic("handle_written_inodeblock: live inodedep %p", 9407 inodedep); 9408 add_to_worklist(&freefile->fx_list, 0); 9409 return (0); 9410 } 9411 9412 /* 9413 * If no outstanding dependencies, free it. 9414 */ 9415 if (free_inodedep(inodedep) || 9416 (TAILQ_FIRST(&inodedep->id_inoreflst) == 0 && 9417 TAILQ_FIRST(&inodedep->id_inoupdt) == 0 && 9418 TAILQ_FIRST(&inodedep->id_extupdt) == 0 && 9419 LIST_FIRST(&inodedep->id_bufwait) == 0)) 9420 return (0); 9421 return (hadchanges); 9422 } 9423 9424 static int 9425 handle_written_indirdep(indirdep, bp, bpp) 9426 struct indirdep *indirdep; 9427 struct buf *bp; 9428 struct buf **bpp; 9429 { 9430 struct allocindir *aip; 9431 int chgs; 9432 9433 if (indirdep->ir_state & GOINGAWAY) 9434 panic("disk_write_complete: indirdep gone"); 9435 chgs = 0; 9436 /* 9437 * If there were rollbacks revert them here. 9438 */ 9439 if (indirdep->ir_saveddata) { 9440 bcopy(indirdep->ir_saveddata, bp->b_data, bp->b_bcount); 9441 free(indirdep->ir_saveddata, M_INDIRDEP); 9442 indirdep->ir_saveddata = 0; 9443 chgs = 1; 9444 } 9445 indirdep->ir_state &= ~UNDONE; 9446 indirdep->ir_state |= ATTACHED; 9447 /* 9448 * Move allocindirs with written pointers to the completehd if 9449 * the the indirdep's pointer is not yet written. Otherwise 9450 * free them here. 9451 */ 9452 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != 0) { 9453 LIST_REMOVE(aip, ai_next); 9454 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 9455 LIST_INSERT_HEAD(&indirdep->ir_completehd, aip, 9456 ai_next); 9457 continue; 9458 } 9459 free_newblk(&aip->ai_block); 9460 } 9461 /* 9462 * Move allocindirs that have finished dependency processing from 9463 * the done list to the write list after updating the pointers. 9464 */ 9465 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != 0) { 9466 handle_allocindir_partdone(aip); 9467 if (aip == LIST_FIRST(&indirdep->ir_donehd)) 9468 panic("disk_write_complete: not gone"); 9469 chgs = 1; 9470 } 9471 /* 9472 * If this indirdep has been detached from its newblk during 9473 * I/O we need to keep this dep attached to the buffer so 9474 * deallocate_dependencies can find it and properly resolve 9475 * any outstanding dependencies. 9476 */ 9477 if ((indirdep->ir_state & (ONDEPLIST | DEPCOMPLETE)) == 0) 9478 chgs = 1; 9479 if ((bp->b_flags & B_DELWRI) == 0) 9480 stat_indir_blk_ptrs++; 9481 /* 9482 * If there were no changes we can discard the savedbp and detach 9483 * ourselves from the buf. We are only carrying completed pointers 9484 * in this case. 9485 */ 9486 if (chgs == 0) { 9487 struct buf *sbp; 9488 9489 sbp = indirdep->ir_savebp; 9490 sbp->b_flags |= B_INVAL | B_NOCACHE; 9491 indirdep->ir_savebp = NULL; 9492 if (*bpp != NULL) 9493 panic("handle_written_indirdep: bp already exists."); 9494 *bpp = sbp; 9495 } else 9496 bdirty(bp); 9497 /* 9498 * If there are no fresh dependencies and none waiting on writes 9499 * we can free the indirdep. 9500 */ 9501 if ((indirdep->ir_state & DEPCOMPLETE) && chgs == 0) { 9502 if (indirdep->ir_state & ONDEPLIST) 9503 LIST_REMOVE(indirdep, ir_next); 9504 free_indirdep(indirdep); 9505 return (0); 9506 } 9507 9508 return (chgs); 9509 } 9510 9511 /* 9512 * Process a diradd entry after its dependent inode has been written. 9513 * This routine must be called with splbio interrupts blocked. 9514 */ 9515 static void 9516 diradd_inode_written(dap, inodedep) 9517 struct diradd *dap; 9518 struct inodedep *inodedep; 9519 { 9520 9521 dap->da_state |= COMPLETE; 9522 complete_diradd(dap); 9523 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 9524 } 9525 9526 /* 9527 * Returns true if the bmsafemap will have rollbacks when written. Must 9528 * only be called with lk and the buf lock on the cg held. 9529 */ 9530 static int 9531 bmsafemap_rollbacks(bmsafemap) 9532 struct bmsafemap *bmsafemap; 9533 { 9534 9535 return (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd) | 9536 !LIST_EMPTY(&bmsafemap->sm_jnewblkhd)); 9537 } 9538 9539 /* 9540 * Complete a write to a bmsafemap structure. Roll forward any bitmap 9541 * changes if it's not a background write. Set all written dependencies 9542 * to DEPCOMPLETE and free the structure if possible. 9543 */ 9544 static int 9545 handle_written_bmsafemap(bmsafemap, bp) 9546 struct bmsafemap *bmsafemap; 9547 struct buf *bp; 9548 { 9549 struct newblk *newblk; 9550 struct inodedep *inodedep; 9551 struct jaddref *jaddref, *jatmp; 9552 struct jnewblk *jnewblk, *jntmp; 9553 uint8_t *inosused; 9554 uint8_t *blksfree; 9555 struct cg *cgp; 9556 struct fs *fs; 9557 ino_t ino; 9558 long bno; 9559 int chgs; 9560 int i; 9561 9562 if ((bmsafemap->sm_state & IOSTARTED) == 0) 9563 panic("initiate_write_bmsafemap: Not started\n"); 9564 chgs = 0; 9565 bmsafemap->sm_state &= ~IOSTARTED; 9566 /* 9567 * Restore unwritten inode allocation pending jaddref writes. 9568 */ 9569 if (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd)) { 9570 cgp = (struct cg *)bp->b_data; 9571 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 9572 inosused = cg_inosused(cgp); 9573 LIST_FOREACH_SAFE(jaddref, &bmsafemap->sm_jaddrefhd, 9574 ja_bmdeps, jatmp) { 9575 if ((jaddref->ja_state & UNDONE) == 0) 9576 continue; 9577 ino = jaddref->ja_ino % fs->fs_ipg; 9578 if (isset(inosused, ino)) 9579 panic("handle_written_bmsafemap: " 9580 "re-allocated inode"); 9581 if ((bp->b_xflags & BX_BKGRDMARKER) == 0) { 9582 if ((jaddref->ja_mode & IFMT) == IFDIR) 9583 cgp->cg_cs.cs_ndir++; 9584 cgp->cg_cs.cs_nifree--; 9585 setbit(inosused, ino); 9586 chgs = 1; 9587 } 9588 jaddref->ja_state &= ~UNDONE; 9589 jaddref->ja_state |= ATTACHED; 9590 free_jaddref(jaddref); 9591 } 9592 } 9593 /* 9594 * Restore any block allocations which are pending journal writes. 9595 */ 9596 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 9597 cgp = (struct cg *)bp->b_data; 9598 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 9599 blksfree = cg_blksfree(cgp); 9600 LIST_FOREACH_SAFE(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps, 9601 jntmp) { 9602 if ((jnewblk->jn_state & UNDONE) == 0) 9603 continue; 9604 bno = dtogd(fs, jnewblk->jn_blkno); 9605 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 9606 i++) { 9607 if (bp->b_xflags & BX_BKGRDMARKER) 9608 break; 9609 if ((jnewblk->jn_state & NEWBLOCK) == 0 && 9610 isclr(blksfree, bno + i)) 9611 panic("handle_written_bmsafemap: " 9612 "re-allocated fragment"); 9613 clrbit(blksfree, bno + i); 9614 chgs = 1; 9615 } 9616 jnewblk->jn_state &= ~(UNDONE | NEWBLOCK); 9617 jnewblk->jn_state |= ATTACHED; 9618 free_jnewblk(jnewblk); 9619 } 9620 } 9621 while ((newblk = LIST_FIRST(&bmsafemap->sm_newblkwr))) { 9622 newblk->nb_state |= DEPCOMPLETE; 9623 newblk->nb_state &= ~ONDEPLIST; 9624 newblk->nb_bmsafemap = NULL; 9625 LIST_REMOVE(newblk, nb_deps); 9626 if (newblk->nb_list.wk_type == D_ALLOCDIRECT) 9627 handle_allocdirect_partdone( 9628 WK_ALLOCDIRECT(&newblk->nb_list), NULL); 9629 else if (newblk->nb_list.wk_type == D_ALLOCINDIR) 9630 handle_allocindir_partdone( 9631 WK_ALLOCINDIR(&newblk->nb_list)); 9632 else if (newblk->nb_list.wk_type != D_NEWBLK) 9633 panic("handle_written_bmsafemap: Unexpected type: %s", 9634 TYPENAME(newblk->nb_list.wk_type)); 9635 } 9636 while ((inodedep = LIST_FIRST(&bmsafemap->sm_inodedepwr)) != NULL) { 9637 inodedep->id_state |= DEPCOMPLETE; 9638 inodedep->id_state &= ~ONDEPLIST; 9639 LIST_REMOVE(inodedep, id_deps); 9640 inodedep->id_bmsafemap = NULL; 9641 } 9642 if (LIST_EMPTY(&bmsafemap->sm_jaddrefhd) && 9643 LIST_EMPTY(&bmsafemap->sm_jnewblkhd) && 9644 LIST_EMPTY(&bmsafemap->sm_newblkhd) && 9645 LIST_EMPTY(&bmsafemap->sm_inodedephd)) { 9646 if (chgs) 9647 bdirty(bp); 9648 LIST_REMOVE(bmsafemap, sm_hash); 9649 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 9650 return (0); 9651 } 9652 bdirty(bp); 9653 return (1); 9654 } 9655 9656 /* 9657 * Try to free a mkdir dependency. 9658 */ 9659 static void 9660 complete_mkdir(mkdir) 9661 struct mkdir *mkdir; 9662 { 9663 struct diradd *dap; 9664 9665 if ((mkdir->md_state & ALLCOMPLETE) != ALLCOMPLETE) 9666 return; 9667 LIST_REMOVE(mkdir, md_mkdirs); 9668 dap = mkdir->md_diradd; 9669 dap->da_state &= ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 9670 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) { 9671 dap->da_state |= DEPCOMPLETE; 9672 complete_diradd(dap); 9673 } 9674 WORKITEM_FREE(mkdir, D_MKDIR); 9675 } 9676 9677 /* 9678 * Handle the completion of a mkdir dependency. 9679 */ 9680 static void 9681 handle_written_mkdir(mkdir, type) 9682 struct mkdir *mkdir; 9683 int type; 9684 { 9685 9686 if ((mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)) != type) 9687 panic("handle_written_mkdir: bad type"); 9688 mkdir->md_state |= COMPLETE; 9689 complete_mkdir(mkdir); 9690 } 9691 9692 static void 9693 free_pagedep(pagedep) 9694 struct pagedep *pagedep; 9695 { 9696 int i; 9697 9698 if (pagedep->pd_state & (NEWBLOCK | ONWORKLIST)) 9699 return; 9700 for (i = 0; i < DAHASHSZ; i++) 9701 if (!LIST_EMPTY(&pagedep->pd_diraddhd[i])) 9702 return; 9703 if (!LIST_EMPTY(&pagedep->pd_jmvrefhd)) 9704 return; 9705 if (!LIST_EMPTY(&pagedep->pd_dirremhd)) 9706 return; 9707 if (!LIST_EMPTY(&pagedep->pd_pendinghd)) 9708 return; 9709 LIST_REMOVE(pagedep, pd_hash); 9710 WORKITEM_FREE(pagedep, D_PAGEDEP); 9711 } 9712 9713 /* 9714 * Called from within softdep_disk_write_complete above. 9715 * A write operation was just completed. Removed inodes can 9716 * now be freed and associated block pointers may be committed. 9717 * Note that this routine is always called from interrupt level 9718 * with further splbio interrupts blocked. 9719 */ 9720 static int 9721 handle_written_filepage(pagedep, bp) 9722 struct pagedep *pagedep; 9723 struct buf *bp; /* buffer containing the written page */ 9724 { 9725 struct dirrem *dirrem; 9726 struct diradd *dap, *nextdap; 9727 struct direct *ep; 9728 int i, chgs; 9729 9730 if ((pagedep->pd_state & IOSTARTED) == 0) 9731 panic("handle_written_filepage: not started"); 9732 pagedep->pd_state &= ~IOSTARTED; 9733 /* 9734 * Process any directory removals that have been committed. 9735 */ 9736 while ((dirrem = LIST_FIRST(&pagedep->pd_dirremhd)) != NULL) { 9737 LIST_REMOVE(dirrem, dm_next); 9738 dirrem->dm_state |= COMPLETE; 9739 dirrem->dm_dirinum = pagedep->pd_ino; 9740 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 9741 ("handle_written_filepage: Journal entries not written.")); 9742 add_to_worklist(&dirrem->dm_list, 0); 9743 } 9744 /* 9745 * Free any directory additions that have been committed. 9746 * If it is a newly allocated block, we have to wait until 9747 * the on-disk directory inode claims the new block. 9748 */ 9749 if ((pagedep->pd_state & NEWBLOCK) == 0) 9750 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 9751 free_diradd(dap, NULL); 9752 /* 9753 * Uncommitted directory entries must be restored. 9754 */ 9755 for (chgs = 0, i = 0; i < DAHASHSZ; i++) { 9756 for (dap = LIST_FIRST(&pagedep->pd_diraddhd[i]); dap; 9757 dap = nextdap) { 9758 nextdap = LIST_NEXT(dap, da_pdlist); 9759 if (dap->da_state & ATTACHED) 9760 panic("handle_written_filepage: attached"); 9761 ep = (struct direct *) 9762 ((char *)bp->b_data + dap->da_offset); 9763 ep->d_ino = dap->da_newinum; 9764 dap->da_state &= ~UNDONE; 9765 dap->da_state |= ATTACHED; 9766 chgs = 1; 9767 /* 9768 * If the inode referenced by the directory has 9769 * been written out, then the dependency can be 9770 * moved to the pending list. 9771 */ 9772 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 9773 LIST_REMOVE(dap, da_pdlist); 9774 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, 9775 da_pdlist); 9776 } 9777 } 9778 } 9779 /* 9780 * If there were any rollbacks in the directory, then it must be 9781 * marked dirty so that its will eventually get written back in 9782 * its correct form. 9783 */ 9784 if (chgs) { 9785 if ((bp->b_flags & B_DELWRI) == 0) 9786 stat_dir_entry++; 9787 bdirty(bp); 9788 return (1); 9789 } 9790 /* 9791 * If we are not waiting for a new directory block to be 9792 * claimed by its inode, then the pagedep will be freed. 9793 * Otherwise it will remain to track any new entries on 9794 * the page in case they are fsync'ed. 9795 */ 9796 if ((pagedep->pd_state & NEWBLOCK) == 0 && 9797 LIST_EMPTY(&pagedep->pd_jmvrefhd)) { 9798 LIST_REMOVE(pagedep, pd_hash); 9799 WORKITEM_FREE(pagedep, D_PAGEDEP); 9800 } 9801 return (0); 9802 } 9803 9804 /* 9805 * Writing back in-core inode structures. 9806 * 9807 * The filesystem only accesses an inode's contents when it occupies an 9808 * "in-core" inode structure. These "in-core" structures are separate from 9809 * the page frames used to cache inode blocks. Only the latter are 9810 * transferred to/from the disk. So, when the updated contents of the 9811 * "in-core" inode structure are copied to the corresponding in-memory inode 9812 * block, the dependencies are also transferred. The following procedure is 9813 * called when copying a dirty "in-core" inode to a cached inode block. 9814 */ 9815 9816 /* 9817 * Called when an inode is loaded from disk. If the effective link count 9818 * differed from the actual link count when it was last flushed, then we 9819 * need to ensure that the correct effective link count is put back. 9820 */ 9821 void 9822 softdep_load_inodeblock(ip) 9823 struct inode *ip; /* the "in_core" copy of the inode */ 9824 { 9825 struct inodedep *inodedep; 9826 9827 /* 9828 * Check for alternate nlink count. 9829 */ 9830 ip->i_effnlink = ip->i_nlink; 9831 ACQUIRE_LOCK(&lk); 9832 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 9833 &inodedep) == 0) { 9834 FREE_LOCK(&lk); 9835 return; 9836 } 9837 ip->i_effnlink -= inodedep->id_nlinkdelta; 9838 FREE_LOCK(&lk); 9839 } 9840 9841 /* 9842 * This routine is called just before the "in-core" inode 9843 * information is to be copied to the in-memory inode block. 9844 * Recall that an inode block contains several inodes. If 9845 * the force flag is set, then the dependencies will be 9846 * cleared so that the update can always be made. Note that 9847 * the buffer is locked when this routine is called, so we 9848 * will never be in the middle of writing the inode block 9849 * to disk. 9850 */ 9851 void 9852 softdep_update_inodeblock(ip, bp, waitfor) 9853 struct inode *ip; /* the "in_core" copy of the inode */ 9854 struct buf *bp; /* the buffer containing the inode block */ 9855 int waitfor; /* nonzero => update must be allowed */ 9856 { 9857 struct inodedep *inodedep; 9858 struct inoref *inoref; 9859 struct worklist *wk; 9860 struct mount *mp; 9861 struct buf *ibp; 9862 struct fs *fs; 9863 int error; 9864 9865 mp = UFSTOVFS(ip->i_ump); 9866 fs = ip->i_fs; 9867 /* 9868 * Preserve the freelink that is on disk. clear_unlinked_inodedep() 9869 * does not have access to the in-core ip so must write directly into 9870 * the inode block buffer when setting freelink. 9871 */ 9872 if (fs->fs_magic == FS_UFS1_MAGIC) 9873 DIP_SET(ip, i_freelink, ((struct ufs1_dinode *)bp->b_data + 9874 ino_to_fsbo(fs, ip->i_number))->di_freelink); 9875 else 9876 DIP_SET(ip, i_freelink, ((struct ufs2_dinode *)bp->b_data + 9877 ino_to_fsbo(fs, ip->i_number))->di_freelink); 9878 /* 9879 * If the effective link count is not equal to the actual link 9880 * count, then we must track the difference in an inodedep while 9881 * the inode is (potentially) tossed out of the cache. Otherwise, 9882 * if there is no existing inodedep, then there are no dependencies 9883 * to track. 9884 */ 9885 ACQUIRE_LOCK(&lk); 9886 again: 9887 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 9888 FREE_LOCK(&lk); 9889 if (ip->i_effnlink != ip->i_nlink) 9890 panic("softdep_update_inodeblock: bad link count"); 9891 return; 9892 } 9893 if (inodedep->id_nlinkdelta != ip->i_nlink - ip->i_effnlink) 9894 panic("softdep_update_inodeblock: bad delta"); 9895 /* 9896 * If we're flushing all dependencies we must also move any waiting 9897 * for journal writes onto the bufwait list prior to I/O. 9898 */ 9899 if (waitfor) { 9900 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 9901 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 9902 == DEPCOMPLETE) { 9903 stat_jwait_inode++; 9904 jwait(&inoref->if_list); 9905 goto again; 9906 } 9907 } 9908 } 9909 /* 9910 * Changes have been initiated. Anything depending on these 9911 * changes cannot occur until this inode has been written. 9912 */ 9913 inodedep->id_state &= ~COMPLETE; 9914 if ((inodedep->id_state & ONWORKLIST) == 0) 9915 WORKLIST_INSERT(&bp->b_dep, &inodedep->id_list); 9916 /* 9917 * Any new dependencies associated with the incore inode must 9918 * now be moved to the list associated with the buffer holding 9919 * the in-memory copy of the inode. Once merged process any 9920 * allocdirects that are completed by the merger. 9921 */ 9922 merge_inode_lists(&inodedep->id_newinoupdt, &inodedep->id_inoupdt); 9923 if (!TAILQ_EMPTY(&inodedep->id_inoupdt)) 9924 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_inoupdt), 9925 NULL); 9926 merge_inode_lists(&inodedep->id_newextupdt, &inodedep->id_extupdt); 9927 if (!TAILQ_EMPTY(&inodedep->id_extupdt)) 9928 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_extupdt), 9929 NULL); 9930 /* 9931 * Now that the inode has been pushed into the buffer, the 9932 * operations dependent on the inode being written to disk 9933 * can be moved to the id_bufwait so that they will be 9934 * processed when the buffer I/O completes. 9935 */ 9936 while ((wk = LIST_FIRST(&inodedep->id_inowait)) != NULL) { 9937 WORKLIST_REMOVE(wk); 9938 WORKLIST_INSERT(&inodedep->id_bufwait, wk); 9939 } 9940 /* 9941 * Newly allocated inodes cannot be written until the bitmap 9942 * that allocates them have been written (indicated by 9943 * DEPCOMPLETE being set in id_state). If we are doing a 9944 * forced sync (e.g., an fsync on a file), we force the bitmap 9945 * to be written so that the update can be done. 9946 */ 9947 if (waitfor == 0) { 9948 FREE_LOCK(&lk); 9949 return; 9950 } 9951 retry: 9952 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) != 0) { 9953 FREE_LOCK(&lk); 9954 return; 9955 } 9956 ibp = inodedep->id_bmsafemap->sm_buf; 9957 ibp = getdirtybuf(ibp, &lk, MNT_WAIT); 9958 if (ibp == NULL) { 9959 /* 9960 * If ibp came back as NULL, the dependency could have been 9961 * freed while we slept. Look it up again, and check to see 9962 * that it has completed. 9963 */ 9964 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 9965 goto retry; 9966 FREE_LOCK(&lk); 9967 return; 9968 } 9969 FREE_LOCK(&lk); 9970 if ((error = bwrite(ibp)) != 0) 9971 softdep_error("softdep_update_inodeblock: bwrite", error); 9972 } 9973 9974 /* 9975 * Merge the a new inode dependency list (such as id_newinoupdt) into an 9976 * old inode dependency list (such as id_inoupdt). This routine must be 9977 * called with splbio interrupts blocked. 9978 */ 9979 static void 9980 merge_inode_lists(newlisthead, oldlisthead) 9981 struct allocdirectlst *newlisthead; 9982 struct allocdirectlst *oldlisthead; 9983 { 9984 struct allocdirect *listadp, *newadp; 9985 9986 newadp = TAILQ_FIRST(newlisthead); 9987 for (listadp = TAILQ_FIRST(oldlisthead); listadp && newadp;) { 9988 if (listadp->ad_offset < newadp->ad_offset) { 9989 listadp = TAILQ_NEXT(listadp, ad_next); 9990 continue; 9991 } 9992 TAILQ_REMOVE(newlisthead, newadp, ad_next); 9993 TAILQ_INSERT_BEFORE(listadp, newadp, ad_next); 9994 if (listadp->ad_offset == newadp->ad_offset) { 9995 allocdirect_merge(oldlisthead, newadp, 9996 listadp); 9997 listadp = newadp; 9998 } 9999 newadp = TAILQ_FIRST(newlisthead); 10000 } 10001 while ((newadp = TAILQ_FIRST(newlisthead)) != NULL) { 10002 TAILQ_REMOVE(newlisthead, newadp, ad_next); 10003 TAILQ_INSERT_TAIL(oldlisthead, newadp, ad_next); 10004 } 10005 } 10006 10007 /* 10008 * If we are doing an fsync, then we must ensure that any directory 10009 * entries for the inode have been written after the inode gets to disk. 10010 */ 10011 int 10012 softdep_fsync(vp) 10013 struct vnode *vp; /* the "in_core" copy of the inode */ 10014 { 10015 struct inodedep *inodedep; 10016 struct pagedep *pagedep; 10017 struct inoref *inoref; 10018 struct worklist *wk; 10019 struct diradd *dap; 10020 struct mount *mp; 10021 struct vnode *pvp; 10022 struct inode *ip; 10023 struct buf *bp; 10024 struct fs *fs; 10025 struct thread *td = curthread; 10026 int error, flushparent, pagedep_new_block; 10027 ino_t parentino; 10028 ufs_lbn_t lbn; 10029 10030 ip = VTOI(vp); 10031 fs = ip->i_fs; 10032 mp = vp->v_mount; 10033 ACQUIRE_LOCK(&lk); 10034 restart: 10035 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 10036 FREE_LOCK(&lk); 10037 return (0); 10038 } 10039 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 10040 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 10041 == DEPCOMPLETE) { 10042 stat_jwait_inode++; 10043 jwait(&inoref->if_list); 10044 goto restart; 10045 } 10046 } 10047 if (!LIST_EMPTY(&inodedep->id_inowait) || 10048 !TAILQ_EMPTY(&inodedep->id_extupdt) || 10049 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 10050 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 10051 !TAILQ_EMPTY(&inodedep->id_newinoupdt)) 10052 panic("softdep_fsync: pending ops %p", inodedep); 10053 for (error = 0, flushparent = 0; ; ) { 10054 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) == NULL) 10055 break; 10056 if (wk->wk_type != D_DIRADD) 10057 panic("softdep_fsync: Unexpected type %s", 10058 TYPENAME(wk->wk_type)); 10059 dap = WK_DIRADD(wk); 10060 /* 10061 * Flush our parent if this directory entry has a MKDIR_PARENT 10062 * dependency or is contained in a newly allocated block. 10063 */ 10064 if (dap->da_state & DIRCHG) 10065 pagedep = dap->da_previous->dm_pagedep; 10066 else 10067 pagedep = dap->da_pagedep; 10068 parentino = pagedep->pd_ino; 10069 lbn = pagedep->pd_lbn; 10070 if ((dap->da_state & (MKDIR_BODY | COMPLETE)) != COMPLETE) 10071 panic("softdep_fsync: dirty"); 10072 if ((dap->da_state & MKDIR_PARENT) || 10073 (pagedep->pd_state & NEWBLOCK)) 10074 flushparent = 1; 10075 else 10076 flushparent = 0; 10077 /* 10078 * If we are being fsync'ed as part of vgone'ing this vnode, 10079 * then we will not be able to release and recover the 10080 * vnode below, so we just have to give up on writing its 10081 * directory entry out. It will eventually be written, just 10082 * not now, but then the user was not asking to have it 10083 * written, so we are not breaking any promises. 10084 */ 10085 if (vp->v_iflag & VI_DOOMED) 10086 break; 10087 /* 10088 * We prevent deadlock by always fetching inodes from the 10089 * root, moving down the directory tree. Thus, when fetching 10090 * our parent directory, we first try to get the lock. If 10091 * that fails, we must unlock ourselves before requesting 10092 * the lock on our parent. See the comment in ufs_lookup 10093 * for details on possible races. 10094 */ 10095 FREE_LOCK(&lk); 10096 if (ffs_vgetf(mp, parentino, LK_NOWAIT | LK_EXCLUSIVE, &pvp, 10097 FFSV_FORCEINSMQ)) { 10098 error = vfs_busy(mp, MBF_NOWAIT); 10099 if (error != 0) { 10100 vfs_ref(mp); 10101 VOP_UNLOCK(vp, 0); 10102 error = vfs_busy(mp, 0); 10103 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 10104 vfs_rel(mp); 10105 if (error != 0) 10106 return (ENOENT); 10107 if (vp->v_iflag & VI_DOOMED) { 10108 vfs_unbusy(mp); 10109 return (ENOENT); 10110 } 10111 } 10112 VOP_UNLOCK(vp, 0); 10113 error = ffs_vgetf(mp, parentino, LK_EXCLUSIVE, 10114 &pvp, FFSV_FORCEINSMQ); 10115 vfs_unbusy(mp); 10116 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 10117 if (vp->v_iflag & VI_DOOMED) { 10118 if (error == 0) 10119 vput(pvp); 10120 error = ENOENT; 10121 } 10122 if (error != 0) 10123 return (error); 10124 } 10125 /* 10126 * All MKDIR_PARENT dependencies and all the NEWBLOCK pagedeps 10127 * that are contained in direct blocks will be resolved by 10128 * doing a ffs_update. Pagedeps contained in indirect blocks 10129 * may require a complete sync'ing of the directory. So, we 10130 * try the cheap and fast ffs_update first, and if that fails, 10131 * then we do the slower ffs_syncvnode of the directory. 10132 */ 10133 if (flushparent) { 10134 int locked; 10135 10136 if ((error = ffs_update(pvp, 1)) != 0) { 10137 vput(pvp); 10138 return (error); 10139 } 10140 ACQUIRE_LOCK(&lk); 10141 locked = 1; 10142 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) { 10143 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) != NULL) { 10144 if (wk->wk_type != D_DIRADD) 10145 panic("softdep_fsync: Unexpected type %s", 10146 TYPENAME(wk->wk_type)); 10147 dap = WK_DIRADD(wk); 10148 if (dap->da_state & DIRCHG) 10149 pagedep = dap->da_previous->dm_pagedep; 10150 else 10151 pagedep = dap->da_pagedep; 10152 pagedep_new_block = pagedep->pd_state & NEWBLOCK; 10153 FREE_LOCK(&lk); 10154 locked = 0; 10155 if (pagedep_new_block && 10156 (error = ffs_syncvnode(pvp, MNT_WAIT))) { 10157 vput(pvp); 10158 return (error); 10159 } 10160 } 10161 } 10162 if (locked) 10163 FREE_LOCK(&lk); 10164 } 10165 /* 10166 * Flush directory page containing the inode's name. 10167 */ 10168 error = bread(pvp, lbn, blksize(fs, VTOI(pvp), lbn), td->td_ucred, 10169 &bp); 10170 if (error == 0) 10171 error = bwrite(bp); 10172 else 10173 brelse(bp); 10174 vput(pvp); 10175 if (error != 0) 10176 return (error); 10177 ACQUIRE_LOCK(&lk); 10178 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 10179 break; 10180 } 10181 FREE_LOCK(&lk); 10182 return (0); 10183 } 10184 10185 /* 10186 * Flush all the dirty bitmaps associated with the block device 10187 * before flushing the rest of the dirty blocks so as to reduce 10188 * the number of dependencies that will have to be rolled back. 10189 */ 10190 void 10191 softdep_fsync_mountdev(vp) 10192 struct vnode *vp; 10193 { 10194 struct buf *bp, *nbp; 10195 struct worklist *wk; 10196 struct bufobj *bo; 10197 10198 if (!vn_isdisk(vp, NULL)) 10199 panic("softdep_fsync_mountdev: vnode not a disk"); 10200 bo = &vp->v_bufobj; 10201 restart: 10202 BO_LOCK(bo); 10203 ACQUIRE_LOCK(&lk); 10204 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 10205 /* 10206 * If it is already scheduled, skip to the next buffer. 10207 */ 10208 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) 10209 continue; 10210 10211 if ((bp->b_flags & B_DELWRI) == 0) 10212 panic("softdep_fsync_mountdev: not dirty"); 10213 /* 10214 * We are only interested in bitmaps with outstanding 10215 * dependencies. 10216 */ 10217 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL || 10218 wk->wk_type != D_BMSAFEMAP || 10219 (bp->b_vflags & BV_BKGRDINPROG)) { 10220 BUF_UNLOCK(bp); 10221 continue; 10222 } 10223 FREE_LOCK(&lk); 10224 BO_UNLOCK(bo); 10225 bremfree(bp); 10226 (void) bawrite(bp); 10227 goto restart; 10228 } 10229 FREE_LOCK(&lk); 10230 drain_output(vp); 10231 BO_UNLOCK(bo); 10232 } 10233 10234 /* 10235 * This routine is called when we are trying to synchronously flush a 10236 * file. This routine must eliminate any filesystem metadata dependencies 10237 * so that the syncing routine can succeed by pushing the dirty blocks 10238 * associated with the file. If any I/O errors occur, they are returned. 10239 */ 10240 int 10241 softdep_sync_metadata(struct vnode *vp) 10242 { 10243 struct pagedep *pagedep; 10244 struct allocindir *aip; 10245 struct newblk *newblk; 10246 struct buf *bp, *nbp; 10247 struct worklist *wk; 10248 struct bufobj *bo; 10249 int i, error, waitfor; 10250 10251 if (!DOINGSOFTDEP(vp)) 10252 return (0); 10253 /* 10254 * Ensure that any direct block dependencies have been cleared. 10255 */ 10256 ACQUIRE_LOCK(&lk); 10257 if ((error = flush_inodedep_deps(vp->v_mount, VTOI(vp)->i_number))) { 10258 FREE_LOCK(&lk); 10259 return (error); 10260 } 10261 FREE_LOCK(&lk); 10262 /* 10263 * For most files, the only metadata dependencies are the 10264 * cylinder group maps that allocate their inode or blocks. 10265 * The block allocation dependencies can be found by traversing 10266 * the dependency lists for any buffers that remain on their 10267 * dirty buffer list. The inode allocation dependency will 10268 * be resolved when the inode is updated with MNT_WAIT. 10269 * This work is done in two passes. The first pass grabs most 10270 * of the buffers and begins asynchronously writing them. The 10271 * only way to wait for these asynchronous writes is to sleep 10272 * on the filesystem vnode which may stay busy for a long time 10273 * if the filesystem is active. So, instead, we make a second 10274 * pass over the dependencies blocking on each write. In the 10275 * usual case we will be blocking against a write that we 10276 * initiated, so when it is done the dependency will have been 10277 * resolved. Thus the second pass is expected to end quickly. 10278 */ 10279 waitfor = MNT_NOWAIT; 10280 bo = &vp->v_bufobj; 10281 10282 top: 10283 /* 10284 * We must wait for any I/O in progress to finish so that 10285 * all potential buffers on the dirty list will be visible. 10286 */ 10287 BO_LOCK(bo); 10288 drain_output(vp); 10289 while ((bp = TAILQ_FIRST(&bo->bo_dirty.bv_hd)) != NULL) { 10290 bp = getdirtybuf(bp, BO_MTX(bo), MNT_WAIT); 10291 if (bp) 10292 break; 10293 } 10294 BO_UNLOCK(bo); 10295 if (bp == NULL) 10296 return (0); 10297 loop: 10298 /* While syncing snapshots, we must allow recursive lookups */ 10299 BUF_AREC(bp); 10300 ACQUIRE_LOCK(&lk); 10301 /* 10302 * As we hold the buffer locked, none of its dependencies 10303 * will disappear. 10304 */ 10305 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 10306 switch (wk->wk_type) { 10307 10308 case D_ALLOCDIRECT: 10309 case D_ALLOCINDIR: 10310 newblk = WK_NEWBLK(wk); 10311 if (newblk->nb_jnewblk != NULL) { 10312 stat_jwait_newblk++; 10313 jwait(&newblk->nb_jnewblk->jn_list); 10314 goto restart; 10315 } 10316 if (newblk->nb_state & DEPCOMPLETE) 10317 continue; 10318 nbp = newblk->nb_bmsafemap->sm_buf; 10319 nbp = getdirtybuf(nbp, &lk, waitfor); 10320 if (nbp == NULL) 10321 continue; 10322 FREE_LOCK(&lk); 10323 if (waitfor == MNT_NOWAIT) { 10324 bawrite(nbp); 10325 } else if ((error = bwrite(nbp)) != 0) { 10326 break; 10327 } 10328 ACQUIRE_LOCK(&lk); 10329 continue; 10330 10331 case D_INDIRDEP: 10332 restart: 10333 10334 LIST_FOREACH(aip, 10335 &WK_INDIRDEP(wk)->ir_deplisthd, ai_next) { 10336 newblk = (struct newblk *)aip; 10337 if (newblk->nb_jnewblk != NULL) { 10338 stat_jwait_newblk++; 10339 jwait(&newblk->nb_jnewblk->jn_list); 10340 goto restart; 10341 } 10342 if (newblk->nb_state & DEPCOMPLETE) 10343 continue; 10344 nbp = newblk->nb_bmsafemap->sm_buf; 10345 nbp = getdirtybuf(nbp, &lk, MNT_WAIT); 10346 if (nbp == NULL) 10347 goto restart; 10348 FREE_LOCK(&lk); 10349 if ((error = bwrite(nbp)) != 0) { 10350 goto loop_end; 10351 } 10352 ACQUIRE_LOCK(&lk); 10353 goto restart; 10354 } 10355 continue; 10356 10357 case D_PAGEDEP: 10358 /* 10359 * We are trying to sync a directory that may 10360 * have dependencies on both its own metadata 10361 * and/or dependencies on the inodes of any 10362 * recently allocated files. We walk its diradd 10363 * lists pushing out the associated inode. 10364 */ 10365 pagedep = WK_PAGEDEP(wk); 10366 for (i = 0; i < DAHASHSZ; i++) { 10367 if (LIST_FIRST(&pagedep->pd_diraddhd[i]) == 0) 10368 continue; 10369 if ((error = 10370 flush_pagedep_deps(vp, wk->wk_mp, 10371 &pagedep->pd_diraddhd[i]))) { 10372 FREE_LOCK(&lk); 10373 goto loop_end; 10374 } 10375 } 10376 continue; 10377 10378 default: 10379 panic("softdep_sync_metadata: Unknown type %s", 10380 TYPENAME(wk->wk_type)); 10381 /* NOTREACHED */ 10382 } 10383 loop_end: 10384 /* We reach here only in error and unlocked */ 10385 if (error == 0) 10386 panic("softdep_sync_metadata: zero error"); 10387 BUF_NOREC(bp); 10388 bawrite(bp); 10389 return (error); 10390 } 10391 FREE_LOCK(&lk); 10392 BO_LOCK(bo); 10393 while ((nbp = TAILQ_NEXT(bp, b_bobufs)) != NULL) { 10394 nbp = getdirtybuf(nbp, BO_MTX(bo), MNT_WAIT); 10395 if (nbp) 10396 break; 10397 } 10398 BO_UNLOCK(bo); 10399 BUF_NOREC(bp); 10400 bawrite(bp); 10401 if (nbp != NULL) { 10402 bp = nbp; 10403 goto loop; 10404 } 10405 /* 10406 * The brief unlock is to allow any pent up dependency 10407 * processing to be done. Then proceed with the second pass. 10408 */ 10409 if (waitfor == MNT_NOWAIT) { 10410 waitfor = MNT_WAIT; 10411 goto top; 10412 } 10413 10414 /* 10415 * If we have managed to get rid of all the dirty buffers, 10416 * then we are done. For certain directories and block 10417 * devices, we may need to do further work. 10418 * 10419 * We must wait for any I/O in progress to finish so that 10420 * all potential buffers on the dirty list will be visible. 10421 */ 10422 BO_LOCK(bo); 10423 drain_output(vp); 10424 BO_UNLOCK(bo); 10425 return ffs_update(vp, 1); 10426 /* return (0); */ 10427 } 10428 10429 /* 10430 * Flush the dependencies associated with an inodedep. 10431 * Called with splbio blocked. 10432 */ 10433 static int 10434 flush_inodedep_deps(mp, ino) 10435 struct mount *mp; 10436 ino_t ino; 10437 { 10438 struct inodedep *inodedep; 10439 struct inoref *inoref; 10440 int error, waitfor; 10441 10442 /* 10443 * This work is done in two passes. The first pass grabs most 10444 * of the buffers and begins asynchronously writing them. The 10445 * only way to wait for these asynchronous writes is to sleep 10446 * on the filesystem vnode which may stay busy for a long time 10447 * if the filesystem is active. So, instead, we make a second 10448 * pass over the dependencies blocking on each write. In the 10449 * usual case we will be blocking against a write that we 10450 * initiated, so when it is done the dependency will have been 10451 * resolved. Thus the second pass is expected to end quickly. 10452 * We give a brief window at the top of the loop to allow 10453 * any pending I/O to complete. 10454 */ 10455 for (error = 0, waitfor = MNT_NOWAIT; ; ) { 10456 if (error) 10457 return (error); 10458 FREE_LOCK(&lk); 10459 ACQUIRE_LOCK(&lk); 10460 restart: 10461 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 10462 return (0); 10463 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 10464 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 10465 == DEPCOMPLETE) { 10466 stat_jwait_inode++; 10467 jwait(&inoref->if_list); 10468 goto restart; 10469 } 10470 } 10471 if (flush_deplist(&inodedep->id_inoupdt, waitfor, &error) || 10472 flush_deplist(&inodedep->id_newinoupdt, waitfor, &error) || 10473 flush_deplist(&inodedep->id_extupdt, waitfor, &error) || 10474 flush_deplist(&inodedep->id_newextupdt, waitfor, &error)) 10475 continue; 10476 /* 10477 * If pass2, we are done, otherwise do pass 2. 10478 */ 10479 if (waitfor == MNT_WAIT) 10480 break; 10481 waitfor = MNT_WAIT; 10482 } 10483 /* 10484 * Try freeing inodedep in case all dependencies have been removed. 10485 */ 10486 if (inodedep_lookup(mp, ino, 0, &inodedep) != 0) 10487 (void) free_inodedep(inodedep); 10488 return (0); 10489 } 10490 10491 /* 10492 * Flush an inode dependency list. 10493 * Called with splbio blocked. 10494 */ 10495 static int 10496 flush_deplist(listhead, waitfor, errorp) 10497 struct allocdirectlst *listhead; 10498 int waitfor; 10499 int *errorp; 10500 { 10501 struct allocdirect *adp; 10502 struct newblk *newblk; 10503 struct buf *bp; 10504 10505 mtx_assert(&lk, MA_OWNED); 10506 TAILQ_FOREACH(adp, listhead, ad_next) { 10507 newblk = (struct newblk *)adp; 10508 if (newblk->nb_jnewblk != NULL) { 10509 stat_jwait_newblk++; 10510 jwait(&newblk->nb_jnewblk->jn_list); 10511 return (1); 10512 } 10513 if (newblk->nb_state & DEPCOMPLETE) 10514 continue; 10515 bp = newblk->nb_bmsafemap->sm_buf; 10516 bp = getdirtybuf(bp, &lk, waitfor); 10517 if (bp == NULL) { 10518 if (waitfor == MNT_NOWAIT) 10519 continue; 10520 return (1); 10521 } 10522 FREE_LOCK(&lk); 10523 if (waitfor == MNT_NOWAIT) { 10524 bawrite(bp); 10525 } else if ((*errorp = bwrite(bp)) != 0) { 10526 ACQUIRE_LOCK(&lk); 10527 return (1); 10528 } 10529 ACQUIRE_LOCK(&lk); 10530 return (1); 10531 } 10532 return (0); 10533 } 10534 10535 /* 10536 * Flush dependencies associated with an allocdirect block. 10537 */ 10538 static int 10539 flush_newblk_dep(vp, mp, lbn) 10540 struct vnode *vp; 10541 struct mount *mp; 10542 ufs_lbn_t lbn; 10543 { 10544 struct newblk *newblk; 10545 struct bufobj *bo; 10546 struct inode *ip; 10547 struct buf *bp; 10548 ufs2_daddr_t blkno; 10549 int error; 10550 10551 error = 0; 10552 bo = &vp->v_bufobj; 10553 ip = VTOI(vp); 10554 blkno = DIP(ip, i_db[lbn]); 10555 if (blkno == 0) 10556 panic("flush_newblk_dep: Missing block"); 10557 ACQUIRE_LOCK(&lk); 10558 /* 10559 * Loop until all dependencies related to this block are satisfied. 10560 * We must be careful to restart after each sleep in case a write 10561 * completes some part of this process for us. 10562 */ 10563 for (;;) { 10564 if (newblk_lookup(mp, blkno, 0, &newblk) == 0) { 10565 FREE_LOCK(&lk); 10566 break; 10567 } 10568 if (newblk->nb_list.wk_type != D_ALLOCDIRECT) 10569 panic("flush_newblk_deps: Bad newblk %p", newblk); 10570 /* 10571 * Flush the journal. 10572 */ 10573 if (newblk->nb_jnewblk != NULL) { 10574 stat_jwait_newblk++; 10575 jwait(&newblk->nb_jnewblk->jn_list); 10576 continue; 10577 } 10578 /* 10579 * Write the bitmap dependency. 10580 */ 10581 if ((newblk->nb_state & DEPCOMPLETE) == 0) { 10582 bp = newblk->nb_bmsafemap->sm_buf; 10583 bp = getdirtybuf(bp, &lk, MNT_WAIT); 10584 if (bp == NULL) 10585 continue; 10586 FREE_LOCK(&lk); 10587 error = bwrite(bp); 10588 if (error) 10589 break; 10590 ACQUIRE_LOCK(&lk); 10591 continue; 10592 } 10593 /* 10594 * Write the buffer. 10595 */ 10596 FREE_LOCK(&lk); 10597 BO_LOCK(bo); 10598 bp = gbincore(bo, lbn); 10599 if (bp != NULL) { 10600 error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 10601 LK_INTERLOCK, BO_MTX(bo)); 10602 if (error == ENOLCK) { 10603 ACQUIRE_LOCK(&lk); 10604 continue; /* Slept, retry */ 10605 } 10606 if (error != 0) 10607 break; /* Failed */ 10608 if (bp->b_flags & B_DELWRI) { 10609 bremfree(bp); 10610 error = bwrite(bp); 10611 if (error) 10612 break; 10613 } else 10614 BUF_UNLOCK(bp); 10615 } else 10616 BO_UNLOCK(bo); 10617 /* 10618 * We have to wait for the direct pointers to 10619 * point at the newdirblk before the dependency 10620 * will go away. 10621 */ 10622 error = ffs_update(vp, MNT_WAIT); 10623 if (error) 10624 break; 10625 ACQUIRE_LOCK(&lk); 10626 } 10627 return (error); 10628 } 10629 10630 /* 10631 * Eliminate a pagedep dependency by flushing out all its diradd dependencies. 10632 * Called with splbio blocked. 10633 */ 10634 static int 10635 flush_pagedep_deps(pvp, mp, diraddhdp) 10636 struct vnode *pvp; 10637 struct mount *mp; 10638 struct diraddhd *diraddhdp; 10639 { 10640 struct inodedep *inodedep; 10641 struct inoref *inoref; 10642 struct ufsmount *ump; 10643 struct diradd *dap; 10644 struct vnode *vp; 10645 int error = 0; 10646 struct buf *bp; 10647 ino_t inum; 10648 10649 ump = VFSTOUFS(mp); 10650 restart: 10651 while ((dap = LIST_FIRST(diraddhdp)) != NULL) { 10652 /* 10653 * Flush ourselves if this directory entry 10654 * has a MKDIR_PARENT dependency. 10655 */ 10656 if (dap->da_state & MKDIR_PARENT) { 10657 FREE_LOCK(&lk); 10658 if ((error = ffs_update(pvp, MNT_WAIT)) != 0) 10659 break; 10660 ACQUIRE_LOCK(&lk); 10661 /* 10662 * If that cleared dependencies, go on to next. 10663 */ 10664 if (dap != LIST_FIRST(diraddhdp)) 10665 continue; 10666 if (dap->da_state & MKDIR_PARENT) 10667 panic("flush_pagedep_deps: MKDIR_PARENT"); 10668 } 10669 /* 10670 * A newly allocated directory must have its "." and 10671 * ".." entries written out before its name can be 10672 * committed in its parent. 10673 */ 10674 inum = dap->da_newinum; 10675 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 10676 panic("flush_pagedep_deps: lost inode1"); 10677 /* 10678 * Wait for any pending journal adds to complete so we don't 10679 * cause rollbacks while syncing. 10680 */ 10681 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 10682 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 10683 == DEPCOMPLETE) { 10684 stat_jwait_inode++; 10685 jwait(&inoref->if_list); 10686 goto restart; 10687 } 10688 } 10689 if (dap->da_state & MKDIR_BODY) { 10690 FREE_LOCK(&lk); 10691 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 10692 FFSV_FORCEINSMQ))) 10693 break; 10694 error = flush_newblk_dep(vp, mp, 0); 10695 /* 10696 * If we still have the dependency we might need to 10697 * update the vnode to sync the new link count to 10698 * disk. 10699 */ 10700 if (error == 0 && dap == LIST_FIRST(diraddhdp)) 10701 error = ffs_update(vp, MNT_WAIT); 10702 vput(vp); 10703 if (error != 0) 10704 break; 10705 ACQUIRE_LOCK(&lk); 10706 /* 10707 * If that cleared dependencies, go on to next. 10708 */ 10709 if (dap != LIST_FIRST(diraddhdp)) 10710 continue; 10711 if (dap->da_state & MKDIR_BODY) { 10712 inodedep_lookup(UFSTOVFS(ump), inum, 0, 10713 &inodedep); 10714 panic("flush_pagedep_deps: MKDIR_BODY " 10715 "inodedep %p dap %p vp %p", 10716 inodedep, dap, vp); 10717 } 10718 } 10719 /* 10720 * Flush the inode on which the directory entry depends. 10721 * Having accounted for MKDIR_PARENT and MKDIR_BODY above, 10722 * the only remaining dependency is that the updated inode 10723 * count must get pushed to disk. The inode has already 10724 * been pushed into its inode buffer (via VOP_UPDATE) at 10725 * the time of the reference count change. So we need only 10726 * locate that buffer, ensure that there will be no rollback 10727 * caused by a bitmap dependency, then write the inode buffer. 10728 */ 10729 retry: 10730 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 10731 panic("flush_pagedep_deps: lost inode"); 10732 /* 10733 * If the inode still has bitmap dependencies, 10734 * push them to disk. 10735 */ 10736 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) == 0) { 10737 bp = inodedep->id_bmsafemap->sm_buf; 10738 bp = getdirtybuf(bp, &lk, MNT_WAIT); 10739 if (bp == NULL) 10740 goto retry; 10741 FREE_LOCK(&lk); 10742 if ((error = bwrite(bp)) != 0) 10743 break; 10744 ACQUIRE_LOCK(&lk); 10745 if (dap != LIST_FIRST(diraddhdp)) 10746 continue; 10747 } 10748 /* 10749 * If the inode is still sitting in a buffer waiting 10750 * to be written or waiting for the link count to be 10751 * adjusted update it here to flush it to disk. 10752 */ 10753 if (dap == LIST_FIRST(diraddhdp)) { 10754 FREE_LOCK(&lk); 10755 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 10756 FFSV_FORCEINSMQ))) 10757 break; 10758 error = ffs_update(vp, MNT_WAIT); 10759 vput(vp); 10760 if (error) 10761 break; 10762 ACQUIRE_LOCK(&lk); 10763 } 10764 /* 10765 * If we have failed to get rid of all the dependencies 10766 * then something is seriously wrong. 10767 */ 10768 if (dap == LIST_FIRST(diraddhdp)) { 10769 inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep); 10770 panic("flush_pagedep_deps: failed to flush " 10771 "inodedep %p ino %d dap %p", inodedep, inum, dap); 10772 } 10773 } 10774 if (error) 10775 ACQUIRE_LOCK(&lk); 10776 return (error); 10777 } 10778 10779 /* 10780 * A large burst of file addition or deletion activity can drive the 10781 * memory load excessively high. First attempt to slow things down 10782 * using the techniques below. If that fails, this routine requests 10783 * the offending operations to fall back to running synchronously 10784 * until the memory load returns to a reasonable level. 10785 */ 10786 int 10787 softdep_slowdown(vp) 10788 struct vnode *vp; 10789 { 10790 struct ufsmount *ump; 10791 int jlow; 10792 int max_softdeps_hard; 10793 10794 ACQUIRE_LOCK(&lk); 10795 jlow = 0; 10796 /* 10797 * Check for journal space if needed. 10798 */ 10799 if (DOINGSUJ(vp)) { 10800 ump = VFSTOUFS(vp->v_mount); 10801 if (journal_space(ump, 0) == 0) 10802 jlow = 1; 10803 } 10804 max_softdeps_hard = max_softdeps * 11 / 10; 10805 if (num_dirrem < max_softdeps_hard / 2 && 10806 num_inodedep < max_softdeps_hard && 10807 VFSTOUFS(vp->v_mount)->um_numindirdeps < maxindirdeps && 10808 num_freeblkdep < max_softdeps_hard && jlow == 0) { 10809 FREE_LOCK(&lk); 10810 return (0); 10811 } 10812 if (VFSTOUFS(vp->v_mount)->um_numindirdeps >= maxindirdeps || jlow) 10813 softdep_speedup(); 10814 stat_sync_limit_hit += 1; 10815 FREE_LOCK(&lk); 10816 return (1); 10817 } 10818 10819 /* 10820 * Called by the allocation routines when they are about to fail 10821 * in the hope that we can free up some disk space. 10822 * 10823 * First check to see if the work list has anything on it. If it has, 10824 * clean up entries until we successfully free some space. Because this 10825 * process holds inodes locked, we cannot handle any remove requests 10826 * that might block on a locked inode as that could lead to deadlock. 10827 * If the worklist yields no free space, encourage the syncer daemon 10828 * to help us. In no event will we try for longer than tickdelay seconds. 10829 */ 10830 int 10831 softdep_request_cleanup(fs, vp) 10832 struct fs *fs; 10833 struct vnode *vp; 10834 { 10835 struct ufsmount *ump; 10836 long starttime; 10837 ufs2_daddr_t needed; 10838 int error; 10839 10840 ump = VTOI(vp)->i_ump; 10841 mtx_assert(UFS_MTX(ump), MA_OWNED); 10842 needed = fs->fs_cstotal.cs_nbfree + fs->fs_contigsumsize; 10843 starttime = time_second + tickdelay; 10844 /* 10845 * If we are being called because of a process doing a 10846 * copy-on-write, then it is not safe to update the vnode 10847 * as we may recurse into the copy-on-write routine. 10848 */ 10849 if (!(curthread->td_pflags & TDP_COWINPROGRESS)) { 10850 UFS_UNLOCK(ump); 10851 error = ffs_update(vp, 1); 10852 UFS_LOCK(ump); 10853 if (error != 0) 10854 return (0); 10855 } 10856 while (fs->fs_pendingblocks > 0 && fs->fs_cstotal.cs_nbfree <= needed) { 10857 if (time_second > starttime) 10858 return (0); 10859 UFS_UNLOCK(ump); 10860 ACQUIRE_LOCK(&lk); 10861 process_removes(vp); 10862 if (ump->softdep_on_worklist > 0 && 10863 process_worklist_item(UFSTOVFS(ump), LK_NOWAIT) != -1) { 10864 stat_worklist_push += 1; 10865 FREE_LOCK(&lk); 10866 UFS_LOCK(ump); 10867 continue; 10868 } 10869 request_cleanup(UFSTOVFS(ump), FLUSH_REMOVE_WAIT); 10870 FREE_LOCK(&lk); 10871 UFS_LOCK(ump); 10872 } 10873 return (1); 10874 } 10875 10876 /* 10877 * If memory utilization has gotten too high, deliberately slow things 10878 * down and speed up the I/O processing. 10879 */ 10880 extern struct thread *syncertd; 10881 static int 10882 request_cleanup(mp, resource) 10883 struct mount *mp; 10884 int resource; 10885 { 10886 struct thread *td = curthread; 10887 struct ufsmount *ump; 10888 10889 mtx_assert(&lk, MA_OWNED); 10890 /* 10891 * We never hold up the filesystem syncer or buf daemon. 10892 */ 10893 if (td->td_pflags & (TDP_SOFTDEP|TDP_NORUNNINGBUF)) 10894 return (0); 10895 ump = VFSTOUFS(mp); 10896 /* 10897 * First check to see if the work list has gotten backlogged. 10898 * If it has, co-opt this process to help clean up two entries. 10899 * Because this process may hold inodes locked, we cannot 10900 * handle any remove requests that might block on a locked 10901 * inode as that could lead to deadlock. We set TDP_SOFTDEP 10902 * to avoid recursively processing the worklist. 10903 */ 10904 if (ump->softdep_on_worklist > max_softdeps / 10) { 10905 td->td_pflags |= TDP_SOFTDEP; 10906 process_worklist_item(mp, LK_NOWAIT); 10907 process_worklist_item(mp, LK_NOWAIT); 10908 td->td_pflags &= ~TDP_SOFTDEP; 10909 stat_worklist_push += 2; 10910 return(1); 10911 } 10912 /* 10913 * Next, we attempt to speed up the syncer process. If that 10914 * is successful, then we allow the process to continue. 10915 */ 10916 if (softdep_speedup() && resource != FLUSH_REMOVE_WAIT) 10917 return(0); 10918 /* 10919 * If we are resource constrained on inode dependencies, try 10920 * flushing some dirty inodes. Otherwise, we are constrained 10921 * by file deletions, so try accelerating flushes of directories 10922 * with removal dependencies. We would like to do the cleanup 10923 * here, but we probably hold an inode locked at this point and 10924 * that might deadlock against one that we try to clean. So, 10925 * the best that we can do is request the syncer daemon to do 10926 * the cleanup for us. 10927 */ 10928 switch (resource) { 10929 10930 case FLUSH_INODES: 10931 stat_ino_limit_push += 1; 10932 req_clear_inodedeps += 1; 10933 stat_countp = &stat_ino_limit_hit; 10934 break; 10935 10936 case FLUSH_REMOVE: 10937 case FLUSH_REMOVE_WAIT: 10938 stat_blk_limit_push += 1; 10939 req_clear_remove += 1; 10940 stat_countp = &stat_blk_limit_hit; 10941 break; 10942 10943 default: 10944 panic("request_cleanup: unknown type"); 10945 } 10946 /* 10947 * Hopefully the syncer daemon will catch up and awaken us. 10948 * We wait at most tickdelay before proceeding in any case. 10949 */ 10950 proc_waiting += 1; 10951 if (callout_pending(&softdep_callout) == FALSE) 10952 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2, 10953 pause_timer, 0); 10954 10955 msleep((caddr_t)&proc_waiting, &lk, PPAUSE, "softupdate", 0); 10956 proc_waiting -= 1; 10957 return (1); 10958 } 10959 10960 /* 10961 * Awaken processes pausing in request_cleanup and clear proc_waiting 10962 * to indicate that there is no longer a timer running. 10963 */ 10964 static void 10965 pause_timer(arg) 10966 void *arg; 10967 { 10968 10969 /* 10970 * The callout_ API has acquired mtx and will hold it around this 10971 * function call. 10972 */ 10973 *stat_countp += 1; 10974 wakeup_one(&proc_waiting); 10975 if (proc_waiting > 0) 10976 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2, 10977 pause_timer, 0); 10978 } 10979 10980 /* 10981 * Flush out a directory with at least one removal dependency in an effort to 10982 * reduce the number of dirrem, freefile, and freeblks dependency structures. 10983 */ 10984 static void 10985 clear_remove(td) 10986 struct thread *td; 10987 { 10988 struct pagedep_hashhead *pagedephd; 10989 struct pagedep *pagedep; 10990 static int next = 0; 10991 struct mount *mp; 10992 struct vnode *vp; 10993 struct bufobj *bo; 10994 int error, cnt; 10995 ino_t ino; 10996 10997 mtx_assert(&lk, MA_OWNED); 10998 10999 for (cnt = 0; cnt < pagedep_hash; cnt++) { 11000 pagedephd = &pagedep_hashtbl[next++]; 11001 if (next >= pagedep_hash) 11002 next = 0; 11003 LIST_FOREACH(pagedep, pagedephd, pd_hash) { 11004 if (LIST_EMPTY(&pagedep->pd_dirremhd)) 11005 continue; 11006 mp = pagedep->pd_list.wk_mp; 11007 ino = pagedep->pd_ino; 11008 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 11009 continue; 11010 FREE_LOCK(&lk); 11011 11012 /* 11013 * Let unmount clear deps 11014 */ 11015 error = vfs_busy(mp, MBF_NOWAIT); 11016 if (error != 0) 11017 goto finish_write; 11018 error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 11019 FFSV_FORCEINSMQ); 11020 vfs_unbusy(mp); 11021 if (error != 0) { 11022 softdep_error("clear_remove: vget", error); 11023 goto finish_write; 11024 } 11025 if ((error = ffs_syncvnode(vp, MNT_NOWAIT))) 11026 softdep_error("clear_remove: fsync", error); 11027 bo = &vp->v_bufobj; 11028 BO_LOCK(bo); 11029 drain_output(vp); 11030 BO_UNLOCK(bo); 11031 vput(vp); 11032 finish_write: 11033 vn_finished_write(mp); 11034 ACQUIRE_LOCK(&lk); 11035 return; 11036 } 11037 } 11038 } 11039 11040 /* 11041 * Clear out a block of dirty inodes in an effort to reduce 11042 * the number of inodedep dependency structures. 11043 */ 11044 static void 11045 clear_inodedeps(td) 11046 struct thread *td; 11047 { 11048 struct inodedep_hashhead *inodedephd; 11049 struct inodedep *inodedep; 11050 static int next = 0; 11051 struct mount *mp; 11052 struct vnode *vp; 11053 struct fs *fs; 11054 int error, cnt; 11055 ino_t firstino, lastino, ino; 11056 11057 mtx_assert(&lk, MA_OWNED); 11058 /* 11059 * Pick a random inode dependency to be cleared. 11060 * We will then gather up all the inodes in its block 11061 * that have dependencies and flush them out. 11062 */ 11063 for (cnt = 0; cnt < inodedep_hash; cnt++) { 11064 inodedephd = &inodedep_hashtbl[next++]; 11065 if (next >= inodedep_hash) 11066 next = 0; 11067 if ((inodedep = LIST_FIRST(inodedephd)) != NULL) 11068 break; 11069 } 11070 if (inodedep == NULL) 11071 return; 11072 fs = inodedep->id_fs; 11073 mp = inodedep->id_list.wk_mp; 11074 /* 11075 * Find the last inode in the block with dependencies. 11076 */ 11077 firstino = inodedep->id_ino & ~(INOPB(fs) - 1); 11078 for (lastino = firstino + INOPB(fs) - 1; lastino > firstino; lastino--) 11079 if (inodedep_lookup(mp, lastino, 0, &inodedep) != 0) 11080 break; 11081 /* 11082 * Asynchronously push all but the last inode with dependencies. 11083 * Synchronously push the last inode with dependencies to ensure 11084 * that the inode block gets written to free up the inodedeps. 11085 */ 11086 for (ino = firstino; ino <= lastino; ino++) { 11087 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 11088 continue; 11089 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 11090 continue; 11091 FREE_LOCK(&lk); 11092 error = vfs_busy(mp, MBF_NOWAIT); /* Let unmount clear deps */ 11093 if (error != 0) { 11094 vn_finished_write(mp); 11095 ACQUIRE_LOCK(&lk); 11096 return; 11097 } 11098 if ((error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 11099 FFSV_FORCEINSMQ)) != 0) { 11100 softdep_error("clear_inodedeps: vget", error); 11101 vfs_unbusy(mp); 11102 vn_finished_write(mp); 11103 ACQUIRE_LOCK(&lk); 11104 return; 11105 } 11106 vfs_unbusy(mp); 11107 if (ino == lastino) { 11108 if ((error = ffs_syncvnode(vp, MNT_WAIT))) 11109 softdep_error("clear_inodedeps: fsync1", error); 11110 } else { 11111 if ((error = ffs_syncvnode(vp, MNT_NOWAIT))) 11112 softdep_error("clear_inodedeps: fsync2", error); 11113 BO_LOCK(&vp->v_bufobj); 11114 drain_output(vp); 11115 BO_UNLOCK(&vp->v_bufobj); 11116 } 11117 vput(vp); 11118 vn_finished_write(mp); 11119 ACQUIRE_LOCK(&lk); 11120 } 11121 } 11122 11123 /* 11124 * Function to determine if the buffer has outstanding dependencies 11125 * that will cause a roll-back if the buffer is written. If wantcount 11126 * is set, return number of dependencies, otherwise just yes or no. 11127 */ 11128 static int 11129 softdep_count_dependencies(bp, wantcount) 11130 struct buf *bp; 11131 int wantcount; 11132 { 11133 struct worklist *wk; 11134 struct bmsafemap *bmsafemap; 11135 struct inodedep *inodedep; 11136 struct indirdep *indirdep; 11137 struct freeblks *freeblks; 11138 struct allocindir *aip; 11139 struct pagedep *pagedep; 11140 struct dirrem *dirrem; 11141 struct newblk *newblk; 11142 struct mkdir *mkdir; 11143 struct diradd *dap; 11144 int i, retval; 11145 11146 retval = 0; 11147 ACQUIRE_LOCK(&lk); 11148 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 11149 switch (wk->wk_type) { 11150 11151 case D_INODEDEP: 11152 inodedep = WK_INODEDEP(wk); 11153 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 11154 /* bitmap allocation dependency */ 11155 retval += 1; 11156 if (!wantcount) 11157 goto out; 11158 } 11159 if (TAILQ_FIRST(&inodedep->id_inoupdt)) { 11160 /* direct block pointer dependency */ 11161 retval += 1; 11162 if (!wantcount) 11163 goto out; 11164 } 11165 if (TAILQ_FIRST(&inodedep->id_extupdt)) { 11166 /* direct block pointer dependency */ 11167 retval += 1; 11168 if (!wantcount) 11169 goto out; 11170 } 11171 if (TAILQ_FIRST(&inodedep->id_inoreflst)) { 11172 /* Add reference dependency. */ 11173 retval += 1; 11174 if (!wantcount) 11175 goto out; 11176 } 11177 continue; 11178 11179 case D_INDIRDEP: 11180 indirdep = WK_INDIRDEP(wk); 11181 11182 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) { 11183 /* indirect block pointer dependency */ 11184 retval += 1; 11185 if (!wantcount) 11186 goto out; 11187 } 11188 continue; 11189 11190 case D_PAGEDEP: 11191 pagedep = WK_PAGEDEP(wk); 11192 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) { 11193 if (LIST_FIRST(&dirrem->dm_jremrefhd)) { 11194 /* Journal remove ref dependency. */ 11195 retval += 1; 11196 if (!wantcount) 11197 goto out; 11198 } 11199 } 11200 for (i = 0; i < DAHASHSZ; i++) { 11201 11202 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 11203 /* directory entry dependency */ 11204 retval += 1; 11205 if (!wantcount) 11206 goto out; 11207 } 11208 } 11209 continue; 11210 11211 case D_BMSAFEMAP: 11212 bmsafemap = WK_BMSAFEMAP(wk); 11213 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd)) { 11214 /* Add reference dependency. */ 11215 retval += 1; 11216 if (!wantcount) 11217 goto out; 11218 } 11219 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd)) { 11220 /* Allocate block dependency. */ 11221 retval += 1; 11222 if (!wantcount) 11223 goto out; 11224 } 11225 continue; 11226 11227 case D_FREEBLKS: 11228 freeblks = WK_FREEBLKS(wk); 11229 if (LIST_FIRST(&freeblks->fb_jfreeblkhd)) { 11230 /* Freeblk journal dependency. */ 11231 retval += 1; 11232 if (!wantcount) 11233 goto out; 11234 } 11235 continue; 11236 11237 case D_ALLOCDIRECT: 11238 case D_ALLOCINDIR: 11239 newblk = WK_NEWBLK(wk); 11240 if (newblk->nb_jnewblk) { 11241 /* Journal allocate dependency. */ 11242 retval += 1; 11243 if (!wantcount) 11244 goto out; 11245 } 11246 continue; 11247 11248 case D_MKDIR: 11249 mkdir = WK_MKDIR(wk); 11250 if (mkdir->md_jaddref) { 11251 /* Journal reference dependency. */ 11252 retval += 1; 11253 if (!wantcount) 11254 goto out; 11255 } 11256 continue; 11257 11258 case D_FREEWORK: 11259 case D_FREEDEP: 11260 case D_JSEGDEP: 11261 case D_JSEG: 11262 case D_SBDEP: 11263 /* never a dependency on these blocks */ 11264 continue; 11265 11266 default: 11267 panic("softdep_count_dependencies: Unexpected type %s", 11268 TYPENAME(wk->wk_type)); 11269 /* NOTREACHED */ 11270 } 11271 } 11272 out: 11273 FREE_LOCK(&lk); 11274 return retval; 11275 } 11276 11277 /* 11278 * Acquire exclusive access to a buffer. 11279 * Must be called with a locked mtx parameter. 11280 * Return acquired buffer or NULL on failure. 11281 */ 11282 static struct buf * 11283 getdirtybuf(bp, mtx, waitfor) 11284 struct buf *bp; 11285 struct mtx *mtx; 11286 int waitfor; 11287 { 11288 int error; 11289 11290 mtx_assert(mtx, MA_OWNED); 11291 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) { 11292 if (waitfor != MNT_WAIT) 11293 return (NULL); 11294 error = BUF_LOCK(bp, 11295 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, mtx); 11296 /* 11297 * Even if we sucessfully acquire bp here, we have dropped 11298 * mtx, which may violates our guarantee. 11299 */ 11300 if (error == 0) 11301 BUF_UNLOCK(bp); 11302 else if (error != ENOLCK) 11303 panic("getdirtybuf: inconsistent lock: %d", error); 11304 mtx_lock(mtx); 11305 return (NULL); 11306 } 11307 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 11308 if (mtx == &lk && waitfor == MNT_WAIT) { 11309 mtx_unlock(mtx); 11310 BO_LOCK(bp->b_bufobj); 11311 BUF_UNLOCK(bp); 11312 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 11313 bp->b_vflags |= BV_BKGRDWAIT; 11314 msleep(&bp->b_xflags, BO_MTX(bp->b_bufobj), 11315 PRIBIO | PDROP, "getbuf", 0); 11316 } else 11317 BO_UNLOCK(bp->b_bufobj); 11318 mtx_lock(mtx); 11319 return (NULL); 11320 } 11321 BUF_UNLOCK(bp); 11322 if (waitfor != MNT_WAIT) 11323 return (NULL); 11324 /* 11325 * The mtx argument must be bp->b_vp's mutex in 11326 * this case. 11327 */ 11328 #ifdef DEBUG_VFS_LOCKS 11329 if (bp->b_vp->v_type != VCHR) 11330 ASSERT_BO_LOCKED(bp->b_bufobj); 11331 #endif 11332 bp->b_vflags |= BV_BKGRDWAIT; 11333 msleep(&bp->b_xflags, mtx, PRIBIO, "getbuf", 0); 11334 return (NULL); 11335 } 11336 if ((bp->b_flags & B_DELWRI) == 0) { 11337 BUF_UNLOCK(bp); 11338 return (NULL); 11339 } 11340 bremfree(bp); 11341 return (bp); 11342 } 11343 11344 11345 /* 11346 * Check if it is safe to suspend the file system now. On entry, 11347 * the vnode interlock for devvp should be held. Return 0 with 11348 * the mount interlock held if the file system can be suspended now, 11349 * otherwise return EAGAIN with the mount interlock held. 11350 */ 11351 int 11352 softdep_check_suspend(struct mount *mp, 11353 struct vnode *devvp, 11354 int softdep_deps, 11355 int softdep_accdeps, 11356 int secondary_writes, 11357 int secondary_accwrites) 11358 { 11359 struct bufobj *bo; 11360 struct ufsmount *ump; 11361 int error; 11362 11363 ump = VFSTOUFS(mp); 11364 bo = &devvp->v_bufobj; 11365 ASSERT_BO_LOCKED(bo); 11366 11367 for (;;) { 11368 if (!TRY_ACQUIRE_LOCK(&lk)) { 11369 BO_UNLOCK(bo); 11370 ACQUIRE_LOCK(&lk); 11371 FREE_LOCK(&lk); 11372 BO_LOCK(bo); 11373 continue; 11374 } 11375 MNT_ILOCK(mp); 11376 if (mp->mnt_secondary_writes != 0) { 11377 FREE_LOCK(&lk); 11378 BO_UNLOCK(bo); 11379 msleep(&mp->mnt_secondary_writes, 11380 MNT_MTX(mp), 11381 (PUSER - 1) | PDROP, "secwr", 0); 11382 BO_LOCK(bo); 11383 continue; 11384 } 11385 break; 11386 } 11387 11388 /* 11389 * Reasons for needing more work before suspend: 11390 * - Dirty buffers on devvp. 11391 * - Softdep activity occurred after start of vnode sync loop 11392 * - Secondary writes occurred after start of vnode sync loop 11393 */ 11394 error = 0; 11395 if (bo->bo_numoutput > 0 || 11396 bo->bo_dirty.bv_cnt > 0 || 11397 softdep_deps != 0 || 11398 ump->softdep_deps != 0 || 11399 softdep_accdeps != ump->softdep_accdeps || 11400 secondary_writes != 0 || 11401 mp->mnt_secondary_writes != 0 || 11402 secondary_accwrites != mp->mnt_secondary_accwrites) 11403 error = EAGAIN; 11404 FREE_LOCK(&lk); 11405 BO_UNLOCK(bo); 11406 return (error); 11407 } 11408 11409 11410 /* 11411 * Get the number of dependency structures for the file system, both 11412 * the current number and the total number allocated. These will 11413 * later be used to detect that softdep processing has occurred. 11414 */ 11415 void 11416 softdep_get_depcounts(struct mount *mp, 11417 int *softdep_depsp, 11418 int *softdep_accdepsp) 11419 { 11420 struct ufsmount *ump; 11421 11422 ump = VFSTOUFS(mp); 11423 ACQUIRE_LOCK(&lk); 11424 *softdep_depsp = ump->softdep_deps; 11425 *softdep_accdepsp = ump->softdep_accdeps; 11426 FREE_LOCK(&lk); 11427 } 11428 11429 /* 11430 * Wait for pending output on a vnode to complete. 11431 * Must be called with vnode lock and interlock locked. 11432 * 11433 * XXX: Should just be a call to bufobj_wwait(). 11434 */ 11435 static void 11436 drain_output(vp) 11437 struct vnode *vp; 11438 { 11439 struct bufobj *bo; 11440 11441 bo = &vp->v_bufobj; 11442 ASSERT_VOP_LOCKED(vp, "drain_output"); 11443 ASSERT_BO_LOCKED(bo); 11444 11445 while (bo->bo_numoutput) { 11446 bo->bo_flag |= BO_WWAIT; 11447 msleep((caddr_t)&bo->bo_numoutput, 11448 BO_MTX(bo), PRIBIO + 1, "drainvp", 0); 11449 } 11450 } 11451 11452 /* 11453 * Called whenever a buffer that is being invalidated or reallocated 11454 * contains dependencies. This should only happen if an I/O error has 11455 * occurred. The routine is called with the buffer locked. 11456 */ 11457 static void 11458 softdep_deallocate_dependencies(bp) 11459 struct buf *bp; 11460 { 11461 11462 if ((bp->b_ioflags & BIO_ERROR) == 0) 11463 panic("softdep_deallocate_dependencies: dangling deps"); 11464 softdep_error(bp->b_vp->v_mount->mnt_stat.f_mntonname, bp->b_error); 11465 panic("softdep_deallocate_dependencies: unrecovered I/O error"); 11466 } 11467 11468 /* 11469 * Function to handle asynchronous write errors in the filesystem. 11470 */ 11471 static void 11472 softdep_error(func, error) 11473 char *func; 11474 int error; 11475 { 11476 11477 /* XXX should do something better! */ 11478 printf("%s: got error %d while accessing filesystem\n", func, error); 11479 } 11480 11481 #ifdef DDB 11482 11483 static void 11484 inodedep_print(struct inodedep *inodedep, int verbose) 11485 { 11486 db_printf("%p fs %p st %x ino %jd inoblk %jd delta %d nlink %d" 11487 " saveino %p\n", 11488 inodedep, inodedep->id_fs, inodedep->id_state, 11489 (intmax_t)inodedep->id_ino, 11490 (intmax_t)fsbtodb(inodedep->id_fs, 11491 ino_to_fsba(inodedep->id_fs, inodedep->id_ino)), 11492 inodedep->id_nlinkdelta, inodedep->id_savednlink, 11493 inodedep->id_savedino1); 11494 11495 if (verbose == 0) 11496 return; 11497 11498 db_printf("\tpendinghd %p, bufwait %p, inowait %p, inoreflst %p, " 11499 "mkdiradd %p\n", 11500 LIST_FIRST(&inodedep->id_pendinghd), 11501 LIST_FIRST(&inodedep->id_bufwait), 11502 LIST_FIRST(&inodedep->id_inowait), 11503 TAILQ_FIRST(&inodedep->id_inoreflst), 11504 inodedep->id_mkdiradd); 11505 db_printf("\tinoupdt %p, newinoupdt %p, extupdt %p, newextupdt %p\n", 11506 TAILQ_FIRST(&inodedep->id_inoupdt), 11507 TAILQ_FIRST(&inodedep->id_newinoupdt), 11508 TAILQ_FIRST(&inodedep->id_extupdt), 11509 TAILQ_FIRST(&inodedep->id_newextupdt)); 11510 } 11511 11512 DB_SHOW_COMMAND(inodedep, db_show_inodedep) 11513 { 11514 11515 if (have_addr == 0) { 11516 db_printf("Address required\n"); 11517 return; 11518 } 11519 inodedep_print((struct inodedep*)addr, 1); 11520 } 11521 11522 DB_SHOW_COMMAND(inodedeps, db_show_inodedeps) 11523 { 11524 struct inodedep_hashhead *inodedephd; 11525 struct inodedep *inodedep; 11526 struct fs *fs; 11527 int cnt; 11528 11529 fs = have_addr ? (struct fs *)addr : NULL; 11530 for (cnt = 0; cnt < inodedep_hash; cnt++) { 11531 inodedephd = &inodedep_hashtbl[cnt]; 11532 LIST_FOREACH(inodedep, inodedephd, id_hash) { 11533 if (fs != NULL && fs != inodedep->id_fs) 11534 continue; 11535 inodedep_print(inodedep, 0); 11536 } 11537 } 11538 } 11539 11540 DB_SHOW_COMMAND(worklist, db_show_worklist) 11541 { 11542 struct worklist *wk; 11543 11544 if (have_addr == 0) { 11545 db_printf("Address required\n"); 11546 return; 11547 } 11548 wk = (struct worklist *)addr; 11549 printf("worklist: %p type %s state 0x%X\n", 11550 wk, TYPENAME(wk->wk_type), wk->wk_state); 11551 } 11552 11553 DB_SHOW_COMMAND(workhead, db_show_workhead) 11554 { 11555 struct workhead *wkhd; 11556 struct worklist *wk; 11557 int i; 11558 11559 if (have_addr == 0) { 11560 db_printf("Address required\n"); 11561 return; 11562 } 11563 wkhd = (struct workhead *)addr; 11564 wk = LIST_FIRST(wkhd); 11565 for (i = 0; i < 100 && wk != NULL; i++, wk = LIST_NEXT(wk, wk_list)) 11566 db_printf("worklist: %p type %s state 0x%X", 11567 wk, TYPENAME(wk->wk_type), wk->wk_state); 11568 if (i == 100) 11569 db_printf("workhead overflow"); 11570 printf("\n"); 11571 } 11572 11573 11574 DB_SHOW_COMMAND(mkdirs, db_show_mkdirs) 11575 { 11576 struct jaddref *jaddref; 11577 struct diradd *diradd; 11578 struct mkdir *mkdir; 11579 11580 LIST_FOREACH(mkdir, &mkdirlisthd, md_mkdirs) { 11581 diradd = mkdir->md_diradd; 11582 db_printf("mkdir: %p state 0x%X dap %p state 0x%X", 11583 mkdir, mkdir->md_state, diradd, diradd->da_state); 11584 if ((jaddref = mkdir->md_jaddref) != NULL) 11585 db_printf(" jaddref %p jaddref state 0x%X", 11586 jaddref, jaddref->ja_state); 11587 db_printf("\n"); 11588 } 11589 } 11590 11591 #endif /* DDB */ 11592 11593 #endif /* SOFTUPDATES */ 11594