1 /*- 2 * Copyright 1998, 2000 Marshall Kirk McKusick. 3 * Copyright 2009, 2010 Jeffrey W. Roberson <jeff@FreeBSD.org> 4 * All rights reserved. 5 * 6 * The soft updates code is derived from the appendix of a University 7 * of Michigan technical report (Gregory R. Ganger and Yale N. Patt, 8 * "Soft Updates: A Solution to the Metadata Update Problem in File 9 * Systems", CSE-TR-254-95, August 1995). 10 * 11 * Further information about soft updates can be obtained from: 12 * 13 * Marshall Kirk McKusick http://www.mckusick.com/softdep/ 14 * 1614 Oxford Street mckusick@mckusick.com 15 * Berkeley, CA 94709-1608 +1-510-843-9542 16 * USA 17 * 18 * Redistribution and use in source and binary forms, with or without 19 * modification, are permitted provided that the following conditions 20 * are met: 21 * 22 * 1. Redistributions of source code must retain the above copyright 23 * notice, this list of conditions and the following disclaimer. 24 * 2. Redistributions in binary form must reproduce the above copyright 25 * notice, this list of conditions and the following disclaimer in the 26 * documentation and/or other materials provided with the distribution. 27 * 28 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS ``AS IS'' AND ANY EXPRESS OR 29 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 30 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 31 * IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, 32 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 33 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 34 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 35 * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR 36 * TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE 37 * USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 38 * 39 * from: @(#)ffs_softdep.c 9.59 (McKusick) 6/21/00 40 */ 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include "opt_ffs.h" 46 #include "opt_ddb.h" 47 48 /* 49 * For now we want the safety net that the DEBUG flag provides. 50 */ 51 #ifndef DEBUG 52 #define DEBUG 53 #endif 54 55 #include <sys/param.h> 56 #include <sys/kernel.h> 57 #include <sys/systm.h> 58 #include <sys/bio.h> 59 #include <sys/buf.h> 60 #include <sys/kdb.h> 61 #include <sys/kthread.h> 62 #include <sys/lock.h> 63 #include <sys/malloc.h> 64 #include <sys/mount.h> 65 #include <sys/mutex.h> 66 #include <sys/namei.h> 67 #include <sys/proc.h> 68 #include <sys/stat.h> 69 #include <sys/sysctl.h> 70 #include <sys/syslog.h> 71 #include <sys/vnode.h> 72 #include <sys/conf.h> 73 #include <ufs/ufs/dir.h> 74 #include <ufs/ufs/extattr.h> 75 #include <ufs/ufs/quota.h> 76 #include <ufs/ufs/inode.h> 77 #include <ufs/ufs/ufsmount.h> 78 #include <ufs/ffs/fs.h> 79 #include <ufs/ffs/softdep.h> 80 #include <ufs/ffs/ffs_extern.h> 81 #include <ufs/ufs/ufs_extern.h> 82 83 #include <vm/vm.h> 84 85 #include <ddb/ddb.h> 86 87 #ifndef SOFTUPDATES 88 89 int 90 softdep_flushfiles(oldmnt, flags, td) 91 struct mount *oldmnt; 92 int flags; 93 struct thread *td; 94 { 95 96 panic("softdep_flushfiles called"); 97 } 98 99 int 100 softdep_mount(devvp, mp, fs, cred) 101 struct vnode *devvp; 102 struct mount *mp; 103 struct fs *fs; 104 struct ucred *cred; 105 { 106 107 return (0); 108 } 109 110 void 111 softdep_initialize() 112 { 113 114 return; 115 } 116 117 void 118 softdep_uninitialize() 119 { 120 121 return; 122 } 123 124 void 125 softdep_unmount(mp) 126 struct mount *mp; 127 { 128 129 } 130 131 void 132 softdep_setup_sbupdate(ump, fs, bp) 133 struct ufsmount *ump; 134 struct fs *fs; 135 struct buf *bp; 136 { 137 } 138 139 void 140 softdep_setup_inomapdep(bp, ip, newinum) 141 struct buf *bp; 142 struct inode *ip; 143 ino_t newinum; 144 { 145 146 panic("softdep_setup_inomapdep called"); 147 } 148 149 void 150 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 151 struct buf *bp; 152 struct mount *mp; 153 ufs2_daddr_t newblkno; 154 int frags; 155 int oldfrags; 156 { 157 158 panic("softdep_setup_blkmapdep called"); 159 } 160 161 void 162 softdep_setup_allocdirect(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 163 struct inode *ip; 164 ufs_lbn_t lbn; 165 ufs2_daddr_t newblkno; 166 ufs2_daddr_t oldblkno; 167 long newsize; 168 long oldsize; 169 struct buf *bp; 170 { 171 172 panic("softdep_setup_allocdirect called"); 173 } 174 175 void 176 softdep_setup_allocext(ip, lbn, newblkno, oldblkno, newsize, oldsize, bp) 177 struct inode *ip; 178 ufs_lbn_t lbn; 179 ufs2_daddr_t newblkno; 180 ufs2_daddr_t oldblkno; 181 long newsize; 182 long oldsize; 183 struct buf *bp; 184 { 185 186 panic("softdep_setup_allocext called"); 187 } 188 189 void 190 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 191 struct inode *ip; 192 ufs_lbn_t lbn; 193 struct buf *bp; 194 int ptrno; 195 ufs2_daddr_t newblkno; 196 ufs2_daddr_t oldblkno; 197 struct buf *nbp; 198 { 199 200 panic("softdep_setup_allocindir_page called"); 201 } 202 203 void 204 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 205 struct buf *nbp; 206 struct inode *ip; 207 struct buf *bp; 208 int ptrno; 209 ufs2_daddr_t newblkno; 210 { 211 212 panic("softdep_setup_allocindir_meta called"); 213 } 214 215 void 216 softdep_setup_freeblocks(ip, length, flags) 217 struct inode *ip; 218 off_t length; 219 int flags; 220 { 221 222 panic("softdep_setup_freeblocks called"); 223 } 224 225 void 226 softdep_freefile(pvp, ino, mode) 227 struct vnode *pvp; 228 ino_t ino; 229 int mode; 230 { 231 232 panic("softdep_freefile called"); 233 } 234 235 int 236 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 237 struct buf *bp; 238 struct inode *dp; 239 off_t diroffset; 240 ino_t newinum; 241 struct buf *newdirbp; 242 int isnewblk; 243 { 244 245 panic("softdep_setup_directory_add called"); 246 } 247 248 void 249 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 250 struct buf *bp; 251 struct inode *dp; 252 caddr_t base; 253 caddr_t oldloc; 254 caddr_t newloc; 255 int entrysize; 256 { 257 258 panic("softdep_change_directoryentry_offset called"); 259 } 260 261 void 262 softdep_setup_remove(bp, dp, ip, isrmdir) 263 struct buf *bp; 264 struct inode *dp; 265 struct inode *ip; 266 int isrmdir; 267 { 268 269 panic("softdep_setup_remove called"); 270 } 271 272 void 273 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 274 struct buf *bp; 275 struct inode *dp; 276 struct inode *ip; 277 ino_t newinum; 278 int isrmdir; 279 { 280 281 panic("softdep_setup_directory_change called"); 282 } 283 284 void * 285 softdep_setup_trunc(vp, length, flags) 286 struct vnode *vp; 287 off_t length; 288 int flags; 289 { 290 291 panic("%s called", __FUNCTION__); 292 293 return (NULL); 294 } 295 296 int 297 softdep_complete_trunc(vp, cookie) 298 struct vnode *vp; 299 void *cookie; 300 { 301 302 panic("%s called", __FUNCTION__); 303 304 return (0); 305 } 306 307 void 308 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 309 struct mount *mp; 310 struct buf *bp; 311 ufs2_daddr_t blkno; 312 int frags; 313 struct workhead *wkhd; 314 { 315 316 panic("%s called", __FUNCTION__); 317 } 318 319 void 320 softdep_setup_inofree(mp, bp, ino, wkhd) 321 struct mount *mp; 322 struct buf *bp; 323 ino_t ino; 324 struct workhead *wkhd; 325 { 326 327 panic("%s called", __FUNCTION__); 328 } 329 330 void 331 softdep_setup_unlink(dp, ip) 332 struct inode *dp; 333 struct inode *ip; 334 { 335 336 panic("%s called", __FUNCTION__); 337 } 338 339 void 340 softdep_setup_link(dp, ip) 341 struct inode *dp; 342 struct inode *ip; 343 { 344 345 panic("%s called", __FUNCTION__); 346 } 347 348 void 349 softdep_revert_link(dp, ip) 350 struct inode *dp; 351 struct inode *ip; 352 { 353 354 panic("%s called", __FUNCTION__); 355 } 356 357 void 358 softdep_setup_rmdir(dp, ip) 359 struct inode *dp; 360 struct inode *ip; 361 { 362 363 panic("%s called", __FUNCTION__); 364 } 365 366 void 367 softdep_revert_rmdir(dp, ip) 368 struct inode *dp; 369 struct inode *ip; 370 { 371 372 panic("%s called", __FUNCTION__); 373 } 374 375 void 376 softdep_setup_create(dp, ip) 377 struct inode *dp; 378 struct inode *ip; 379 { 380 381 panic("%s called", __FUNCTION__); 382 } 383 384 void 385 softdep_revert_create(dp, ip) 386 struct inode *dp; 387 struct inode *ip; 388 { 389 390 panic("%s called", __FUNCTION__); 391 } 392 393 void 394 softdep_setup_mkdir(dp, ip) 395 struct inode *dp; 396 struct inode *ip; 397 { 398 399 panic("%s called", __FUNCTION__); 400 } 401 402 void 403 softdep_revert_mkdir(dp, ip) 404 struct inode *dp; 405 struct inode *ip; 406 { 407 408 panic("%s called", __FUNCTION__); 409 } 410 411 void 412 softdep_setup_dotdot_link(dp, ip) 413 struct inode *dp; 414 struct inode *ip; 415 { 416 417 panic("%s called", __FUNCTION__); 418 } 419 420 int 421 softdep_prealloc(vp, waitok) 422 struct vnode *vp; 423 int waitok; 424 { 425 426 panic("%s called", __FUNCTION__); 427 428 return (0); 429 } 430 431 int 432 softdep_journal_lookup(mp, vpp) 433 struct mount *mp; 434 struct vnode **vpp; 435 { 436 437 return (ENOENT); 438 } 439 440 void 441 softdep_change_linkcnt(ip) 442 struct inode *ip; 443 { 444 445 panic("softdep_change_linkcnt called"); 446 } 447 448 void 449 softdep_load_inodeblock(ip) 450 struct inode *ip; 451 { 452 453 panic("softdep_load_inodeblock called"); 454 } 455 456 void 457 softdep_update_inodeblock(ip, bp, waitfor) 458 struct inode *ip; 459 struct buf *bp; 460 int waitfor; 461 { 462 463 panic("softdep_update_inodeblock called"); 464 } 465 466 int 467 softdep_fsync(vp) 468 struct vnode *vp; /* the "in_core" copy of the inode */ 469 { 470 471 return (0); 472 } 473 474 void 475 softdep_fsync_mountdev(vp) 476 struct vnode *vp; 477 { 478 479 return; 480 } 481 482 int 483 softdep_flushworklist(oldmnt, countp, td) 484 struct mount *oldmnt; 485 int *countp; 486 struct thread *td; 487 { 488 489 *countp = 0; 490 return (0); 491 } 492 493 int 494 softdep_sync_metadata(struct vnode *vp) 495 { 496 497 return (0); 498 } 499 500 int 501 softdep_slowdown(vp) 502 struct vnode *vp; 503 { 504 505 panic("softdep_slowdown called"); 506 } 507 508 void 509 softdep_releasefile(ip) 510 struct inode *ip; /* inode with the zero effective link count */ 511 { 512 513 panic("softdep_releasefile called"); 514 } 515 516 int 517 softdep_request_cleanup(fs, vp) 518 struct fs *fs; 519 struct vnode *vp; 520 { 521 522 return (0); 523 } 524 525 int 526 softdep_check_suspend(struct mount *mp, 527 struct vnode *devvp, 528 int softdep_deps, 529 int softdep_accdeps, 530 int secondary_writes, 531 int secondary_accwrites) 532 { 533 struct bufobj *bo; 534 int error; 535 536 (void) softdep_deps, 537 (void) softdep_accdeps; 538 539 bo = &devvp->v_bufobj; 540 ASSERT_BO_LOCKED(bo); 541 542 MNT_ILOCK(mp); 543 while (mp->mnt_secondary_writes != 0) { 544 BO_UNLOCK(bo); 545 msleep(&mp->mnt_secondary_writes, MNT_MTX(mp), 546 (PUSER - 1) | PDROP, "secwr", 0); 547 BO_LOCK(bo); 548 MNT_ILOCK(mp); 549 } 550 551 /* 552 * Reasons for needing more work before suspend: 553 * - Dirty buffers on devvp. 554 * - Secondary writes occurred after start of vnode sync loop 555 */ 556 error = 0; 557 if (bo->bo_numoutput > 0 || 558 bo->bo_dirty.bv_cnt > 0 || 559 secondary_writes != 0 || 560 mp->mnt_secondary_writes != 0 || 561 secondary_accwrites != mp->mnt_secondary_accwrites) 562 error = EAGAIN; 563 BO_UNLOCK(bo); 564 return (error); 565 } 566 567 void 568 softdep_get_depcounts(struct mount *mp, 569 int *softdepactivep, 570 int *softdepactiveaccp) 571 { 572 (void) mp; 573 *softdepactivep = 0; 574 *softdepactiveaccp = 0; 575 } 576 577 #else 578 /* 579 * These definitions need to be adapted to the system to which 580 * this file is being ported. 581 */ 582 583 #define M_SOFTDEP_FLAGS (M_WAITOK) 584 585 #define D_PAGEDEP 0 586 #define D_INODEDEP 1 587 #define D_BMSAFEMAP 2 588 #define D_NEWBLK 3 589 #define D_ALLOCDIRECT 4 590 #define D_INDIRDEP 5 591 #define D_ALLOCINDIR 6 592 #define D_FREEFRAG 7 593 #define D_FREEBLKS 8 594 #define D_FREEFILE 9 595 #define D_DIRADD 10 596 #define D_MKDIR 11 597 #define D_DIRREM 12 598 #define D_NEWDIRBLK 13 599 #define D_FREEWORK 14 600 #define D_FREEDEP 15 601 #define D_JADDREF 16 602 #define D_JREMREF 17 603 #define D_JMVREF 18 604 #define D_JNEWBLK 19 605 #define D_JFREEBLK 20 606 #define D_JFREEFRAG 21 607 #define D_JSEG 22 608 #define D_JSEGDEP 23 609 #define D_SBDEP 24 610 #define D_JTRUNC 25 611 #define D_LAST D_JTRUNC 612 613 unsigned long dep_current[D_LAST + 1]; 614 unsigned long dep_total[D_LAST + 1]; 615 616 617 SYSCTL_NODE(_debug, OID_AUTO, softdep, CTLFLAG_RW, 0, "soft updates stats"); 618 SYSCTL_NODE(_debug_softdep, OID_AUTO, total, CTLFLAG_RW, 0, 619 "total dependencies allocated"); 620 SYSCTL_NODE(_debug_softdep, OID_AUTO, current, CTLFLAG_RW, 0, 621 "current dependencies allocated"); 622 623 #define SOFTDEP_TYPE(type, str, long) \ 624 static MALLOC_DEFINE(M_ ## type, #str, long); \ 625 SYSCTL_LONG(_debug_softdep_total, OID_AUTO, str, CTLFLAG_RD, \ 626 &dep_total[D_ ## type], 0, ""); \ 627 SYSCTL_LONG(_debug_softdep_current, OID_AUTO, str, CTLFLAG_RD, \ 628 &dep_current[D_ ## type], 0, ""); 629 630 SOFTDEP_TYPE(PAGEDEP, pagedep, "File page dependencies"); 631 SOFTDEP_TYPE(INODEDEP, inodedep, "Inode dependencies"); 632 SOFTDEP_TYPE(BMSAFEMAP, bmsafemap, 633 "Block or frag allocated from cyl group map"); 634 SOFTDEP_TYPE(NEWBLK, newblk, "New block or frag allocation dependency"); 635 SOFTDEP_TYPE(ALLOCDIRECT, allocdirect, "Block or frag dependency for an inode"); 636 SOFTDEP_TYPE(INDIRDEP, indirdep, "Indirect block dependencies"); 637 SOFTDEP_TYPE(ALLOCINDIR, allocindir, "Block dependency for an indirect block"); 638 SOFTDEP_TYPE(FREEFRAG, freefrag, "Previously used frag for an inode"); 639 SOFTDEP_TYPE(FREEBLKS, freeblks, "Blocks freed from an inode"); 640 SOFTDEP_TYPE(FREEFILE, freefile, "Inode deallocated"); 641 SOFTDEP_TYPE(DIRADD, diradd, "New directory entry"); 642 SOFTDEP_TYPE(MKDIR, mkdir, "New directory"); 643 SOFTDEP_TYPE(DIRREM, dirrem, "Directory entry deleted"); 644 SOFTDEP_TYPE(NEWDIRBLK, newdirblk, "Unclaimed new directory block"); 645 SOFTDEP_TYPE(FREEWORK, freework, "free an inode block"); 646 SOFTDEP_TYPE(FREEDEP, freedep, "track a block free"); 647 SOFTDEP_TYPE(JADDREF, jaddref, "Journal inode ref add"); 648 SOFTDEP_TYPE(JREMREF, jremref, "Journal inode ref remove"); 649 SOFTDEP_TYPE(JMVREF, jmvref, "Journal inode ref move"); 650 SOFTDEP_TYPE(JNEWBLK, jnewblk, "Journal new block"); 651 SOFTDEP_TYPE(JFREEBLK, jfreeblk, "Journal free block"); 652 SOFTDEP_TYPE(JFREEFRAG, jfreefrag, "Journal free frag"); 653 SOFTDEP_TYPE(JSEG, jseg, "Journal segment"); 654 SOFTDEP_TYPE(JSEGDEP, jsegdep, "Journal segment complete"); 655 SOFTDEP_TYPE(SBDEP, sbdep, "Superblock write dependency"); 656 SOFTDEP_TYPE(JTRUNC, jtrunc, "Journal inode truncation"); 657 658 static MALLOC_DEFINE(M_SAVEDINO, "savedino", "Saved inodes"); 659 static MALLOC_DEFINE(M_JBLOCKS, "jblocks", "Journal block locations"); 660 661 /* 662 * translate from workitem type to memory type 663 * MUST match the defines above, such that memtype[D_XXX] == M_XXX 664 */ 665 static struct malloc_type *memtype[] = { 666 M_PAGEDEP, 667 M_INODEDEP, 668 M_BMSAFEMAP, 669 M_NEWBLK, 670 M_ALLOCDIRECT, 671 M_INDIRDEP, 672 M_ALLOCINDIR, 673 M_FREEFRAG, 674 M_FREEBLKS, 675 M_FREEFILE, 676 M_DIRADD, 677 M_MKDIR, 678 M_DIRREM, 679 M_NEWDIRBLK, 680 M_FREEWORK, 681 M_FREEDEP, 682 M_JADDREF, 683 M_JREMREF, 684 M_JMVREF, 685 M_JNEWBLK, 686 M_JFREEBLK, 687 M_JFREEFRAG, 688 M_JSEG, 689 M_JSEGDEP, 690 M_SBDEP, 691 M_JTRUNC 692 }; 693 694 #define DtoM(type) (memtype[type]) 695 696 /* 697 * Names of malloc types. 698 */ 699 #define TYPENAME(type) \ 700 ((unsigned)(type) <= D_LAST ? memtype[type]->ks_shortdesc : "???") 701 /* 702 * End system adaptation definitions. 703 */ 704 705 #define DOTDOT_OFFSET offsetof(struct dirtemplate, dotdot_ino) 706 #define DOT_OFFSET offsetof(struct dirtemplate, dot_ino) 707 708 /* 709 * Forward declarations. 710 */ 711 struct inodedep_hashhead; 712 struct newblk_hashhead; 713 struct pagedep_hashhead; 714 struct bmsafemap_hashhead; 715 716 /* 717 * Internal function prototypes. 718 */ 719 static void softdep_error(char *, int); 720 static void drain_output(struct vnode *); 721 static struct buf *getdirtybuf(struct buf *, struct mtx *, int); 722 static void clear_remove(struct thread *); 723 static void clear_inodedeps(struct thread *); 724 static void unlinked_inodedep(struct mount *, struct inodedep *); 725 static void clear_unlinked_inodedep(struct inodedep *); 726 static struct inodedep *first_unlinked_inodedep(struct ufsmount *); 727 static int flush_pagedep_deps(struct vnode *, struct mount *, 728 struct diraddhd *); 729 static void free_pagedep(struct pagedep *); 730 static int flush_newblk_dep(struct vnode *, struct mount *, ufs_lbn_t); 731 static int flush_inodedep_deps(struct mount *, ino_t); 732 static int flush_deplist(struct allocdirectlst *, int, int *); 733 static int handle_written_filepage(struct pagedep *, struct buf *); 734 static int handle_written_sbdep(struct sbdep *, struct buf *); 735 static void initiate_write_sbdep(struct sbdep *); 736 static void diradd_inode_written(struct diradd *, struct inodedep *); 737 static int handle_written_indirdep(struct indirdep *, struct buf *, 738 struct buf**); 739 static int handle_written_inodeblock(struct inodedep *, struct buf *); 740 static int handle_written_bmsafemap(struct bmsafemap *, struct buf *); 741 static void handle_written_jaddref(struct jaddref *); 742 static void handle_written_jremref(struct jremref *); 743 static void handle_written_jseg(struct jseg *, struct buf *); 744 static void handle_written_jnewblk(struct jnewblk *); 745 static void handle_written_jfreeblk(struct jfreeblk *); 746 static void handle_written_jfreefrag(struct jfreefrag *); 747 static void complete_jseg(struct jseg *); 748 static void jseg_write(struct fs *, struct jblocks *, struct jseg *, 749 uint8_t *); 750 static void jaddref_write(struct jaddref *, struct jseg *, uint8_t *); 751 static void jremref_write(struct jremref *, struct jseg *, uint8_t *); 752 static void jmvref_write(struct jmvref *, struct jseg *, uint8_t *); 753 static void jtrunc_write(struct jtrunc *, struct jseg *, uint8_t *); 754 static void jnewblk_write(struct jnewblk *, struct jseg *, uint8_t *); 755 static void jfreeblk_write(struct jfreeblk *, struct jseg *, uint8_t *); 756 static void jfreefrag_write(struct jfreefrag *, struct jseg *, uint8_t *); 757 static inline void inoref_write(struct inoref *, struct jseg *, 758 struct jrefrec *); 759 static void handle_allocdirect_partdone(struct allocdirect *, 760 struct workhead *); 761 static void cancel_newblk(struct newblk *, struct workhead *); 762 static void indirdep_complete(struct indirdep *); 763 static void handle_allocindir_partdone(struct allocindir *); 764 static void initiate_write_filepage(struct pagedep *, struct buf *); 765 static void initiate_write_indirdep(struct indirdep*, struct buf *); 766 static void handle_written_mkdir(struct mkdir *, int); 767 static void initiate_write_bmsafemap(struct bmsafemap *, struct buf *); 768 static void initiate_write_inodeblock_ufs1(struct inodedep *, struct buf *); 769 static void initiate_write_inodeblock_ufs2(struct inodedep *, struct buf *); 770 static void handle_workitem_freefile(struct freefile *); 771 static void handle_workitem_remove(struct dirrem *, struct vnode *); 772 static struct dirrem *newdirrem(struct buf *, struct inode *, 773 struct inode *, int, struct dirrem **); 774 static void cancel_indirdep(struct indirdep *, struct buf *, struct inodedep *, 775 struct freeblks *); 776 static void free_indirdep(struct indirdep *); 777 static void free_diradd(struct diradd *, struct workhead *); 778 static void merge_diradd(struct inodedep *, struct diradd *); 779 static void complete_diradd(struct diradd *); 780 static struct diradd *diradd_lookup(struct pagedep *, int); 781 static struct jremref *cancel_diradd_dotdot(struct inode *, struct dirrem *, 782 struct jremref *); 783 static struct jremref *cancel_mkdir_dotdot(struct inode *, struct dirrem *, 784 struct jremref *); 785 static void cancel_diradd(struct diradd *, struct dirrem *, struct jremref *, 786 struct jremref *, struct jremref *); 787 static void dirrem_journal(struct dirrem *, struct jremref *, struct jremref *, 788 struct jremref *); 789 static void cancel_allocindir(struct allocindir *, struct inodedep *, 790 struct freeblks *); 791 static void complete_mkdir(struct mkdir *); 792 static void free_newdirblk(struct newdirblk *); 793 static void free_jremref(struct jremref *); 794 static void free_jaddref(struct jaddref *); 795 static void free_jsegdep(struct jsegdep *); 796 static void free_jseg(struct jseg *); 797 static void free_jnewblk(struct jnewblk *); 798 static void free_jfreeblk(struct jfreeblk *); 799 static void free_jfreefrag(struct jfreefrag *); 800 static void free_freedep(struct freedep *); 801 static void journal_jremref(struct dirrem *, struct jremref *, 802 struct inodedep *); 803 static void cancel_jnewblk(struct jnewblk *, struct workhead *); 804 static int cancel_jaddref(struct jaddref *, struct inodedep *, 805 struct workhead *); 806 static void cancel_jfreefrag(struct jfreefrag *); 807 static void indir_trunc(struct freework *, ufs2_daddr_t, ufs_lbn_t); 808 static int deallocate_dependencies(struct buf *, struct inodedep *, 809 struct freeblks *); 810 static void free_newblk(struct newblk *); 811 static void cancel_allocdirect(struct allocdirectlst *, 812 struct allocdirect *, struct freeblks *, int); 813 static int check_inode_unwritten(struct inodedep *); 814 static int free_inodedep(struct inodedep *); 815 static void freework_freeblock(struct freework *); 816 static void handle_workitem_freeblocks(struct freeblks *, int); 817 static void handle_complete_freeblocks(struct freeblks *); 818 static void handle_workitem_indirblk(struct freework *); 819 static void handle_written_freework(struct freework *); 820 static void merge_inode_lists(struct allocdirectlst *,struct allocdirectlst *); 821 static void setup_allocindir_phase2(struct buf *, struct inode *, 822 struct inodedep *, struct allocindir *, ufs_lbn_t); 823 static struct allocindir *newallocindir(struct inode *, int, ufs2_daddr_t, 824 ufs2_daddr_t, ufs_lbn_t); 825 static void handle_workitem_freefrag(struct freefrag *); 826 static struct freefrag *newfreefrag(struct inode *, ufs2_daddr_t, long, 827 ufs_lbn_t); 828 static void allocdirect_merge(struct allocdirectlst *, 829 struct allocdirect *, struct allocdirect *); 830 static struct freefrag *allocindir_merge(struct allocindir *, 831 struct allocindir *); 832 static int bmsafemap_find(struct bmsafemap_hashhead *, struct mount *, int, 833 struct bmsafemap **); 834 static struct bmsafemap *bmsafemap_lookup(struct mount *, struct buf *, 835 int cg); 836 static int newblk_find(struct newblk_hashhead *, struct mount *, ufs2_daddr_t, 837 int, struct newblk **); 838 static int newblk_lookup(struct mount *, ufs2_daddr_t, int, struct newblk **); 839 static int inodedep_find(struct inodedep_hashhead *, struct fs *, ino_t, 840 struct inodedep **); 841 static int inodedep_lookup(struct mount *, ino_t, int, struct inodedep **); 842 static int pagedep_lookup(struct mount *, ino_t, ufs_lbn_t, int, 843 struct pagedep **); 844 static int pagedep_find(struct pagedep_hashhead *, ino_t, ufs_lbn_t, 845 struct mount *mp, int, struct pagedep **); 846 static void pause_timer(void *); 847 static int request_cleanup(struct mount *, int); 848 static int process_worklist_item(struct mount *, int); 849 static void process_removes(struct vnode *); 850 static void jwork_move(struct workhead *, struct workhead *); 851 static void add_to_worklist(struct worklist *, int); 852 static void remove_from_worklist(struct worklist *); 853 static void softdep_flush(void); 854 static int softdep_speedup(void); 855 static void worklist_speedup(void); 856 static int journal_mount(struct mount *, struct fs *, struct ucred *); 857 static void journal_unmount(struct mount *); 858 static int journal_space(struct ufsmount *, int); 859 static void journal_suspend(struct ufsmount *); 860 static int journal_unsuspend(struct ufsmount *ump); 861 static void softdep_prelink(struct vnode *, struct vnode *); 862 static void add_to_journal(struct worklist *); 863 static void remove_from_journal(struct worklist *); 864 static void softdep_process_journal(struct mount *, int); 865 static struct jremref *newjremref(struct dirrem *, struct inode *, 866 struct inode *ip, off_t, nlink_t); 867 static struct jaddref *newjaddref(struct inode *, ino_t, off_t, int16_t, 868 uint16_t); 869 static inline void newinoref(struct inoref *, ino_t, ino_t, off_t, nlink_t, 870 uint16_t); 871 static inline struct jsegdep *inoref_jseg(struct inoref *); 872 static struct jmvref *newjmvref(struct inode *, ino_t, off_t, off_t); 873 static struct jfreeblk *newjfreeblk(struct freeblks *, ufs_lbn_t, 874 ufs2_daddr_t, int); 875 static struct jfreefrag *newjfreefrag(struct freefrag *, struct inode *, 876 ufs2_daddr_t, long, ufs_lbn_t); 877 static struct freework *newfreework(struct freeblks *, struct freework *, 878 ufs_lbn_t, ufs2_daddr_t, int, int); 879 static void jwait(struct worklist *wk); 880 static struct inodedep *inodedep_lookup_ip(struct inode *); 881 static int bmsafemap_rollbacks(struct bmsafemap *); 882 static struct freefile *handle_bufwait(struct inodedep *, struct workhead *); 883 static void handle_jwork(struct workhead *); 884 static struct mkdir *setup_newdir(struct diradd *, ino_t, ino_t, struct buf *, 885 struct mkdir **); 886 static struct jblocks *jblocks_create(void); 887 static ufs2_daddr_t jblocks_alloc(struct jblocks *, int, int *); 888 static void jblocks_free(struct jblocks *, struct mount *, int); 889 static void jblocks_destroy(struct jblocks *); 890 static void jblocks_add(struct jblocks *, ufs2_daddr_t, int); 891 892 /* 893 * Exported softdep operations. 894 */ 895 static void softdep_disk_io_initiation(struct buf *); 896 static void softdep_disk_write_complete(struct buf *); 897 static void softdep_deallocate_dependencies(struct buf *); 898 static int softdep_count_dependencies(struct buf *bp, int); 899 900 static struct mtx lk; 901 MTX_SYSINIT(softdep_lock, &lk, "Softdep Lock", MTX_DEF); 902 903 #define TRY_ACQUIRE_LOCK(lk) mtx_trylock(lk) 904 #define ACQUIRE_LOCK(lk) mtx_lock(lk) 905 #define FREE_LOCK(lk) mtx_unlock(lk) 906 907 #define BUF_AREC(bp) lockallowrecurse(&(bp)->b_lock) 908 #define BUF_NOREC(bp) lockdisablerecurse(&(bp)->b_lock) 909 910 /* 911 * Worklist queue management. 912 * These routines require that the lock be held. 913 */ 914 #ifndef /* NOT */ DEBUG 915 #define WORKLIST_INSERT(head, item) do { \ 916 (item)->wk_state |= ONWORKLIST; \ 917 LIST_INSERT_HEAD(head, item, wk_list); \ 918 } while (0) 919 #define WORKLIST_REMOVE(item) do { \ 920 (item)->wk_state &= ~ONWORKLIST; \ 921 LIST_REMOVE(item, wk_list); \ 922 } while (0) 923 #define WORKLIST_INSERT_UNLOCKED WORKLIST_INSERT 924 #define WORKLIST_REMOVE_UNLOCKED WORKLIST_REMOVE 925 926 #else /* DEBUG */ 927 static void worklist_insert(struct workhead *, struct worklist *, int); 928 static void worklist_remove(struct worklist *, int); 929 930 #define WORKLIST_INSERT(head, item) worklist_insert(head, item, 1) 931 #define WORKLIST_INSERT_UNLOCKED(head, item) worklist_insert(head, item, 0) 932 #define WORKLIST_REMOVE(item) worklist_remove(item, 1) 933 #define WORKLIST_REMOVE_UNLOCKED(item) worklist_remove(item, 0) 934 935 static void 936 worklist_insert(head, item, locked) 937 struct workhead *head; 938 struct worklist *item; 939 int locked; 940 { 941 942 if (locked) 943 mtx_assert(&lk, MA_OWNED); 944 if (item->wk_state & ONWORKLIST) 945 panic("worklist_insert: %p %s(0x%X) already on list", 946 item, TYPENAME(item->wk_type), item->wk_state); 947 item->wk_state |= ONWORKLIST; 948 LIST_INSERT_HEAD(head, item, wk_list); 949 } 950 951 static void 952 worklist_remove(item, locked) 953 struct worklist *item; 954 int locked; 955 { 956 957 if (locked) 958 mtx_assert(&lk, MA_OWNED); 959 if ((item->wk_state & ONWORKLIST) == 0) 960 panic("worklist_remove: %p %s(0x%X) not on list", 961 item, TYPENAME(item->wk_type), item->wk_state); 962 item->wk_state &= ~ONWORKLIST; 963 LIST_REMOVE(item, wk_list); 964 } 965 #endif /* DEBUG */ 966 967 /* 968 * Merge two jsegdeps keeping only the oldest one as newer references 969 * can't be discarded until after older references. 970 */ 971 static inline struct jsegdep * 972 jsegdep_merge(struct jsegdep *one, struct jsegdep *two) 973 { 974 struct jsegdep *swp; 975 976 if (two == NULL) 977 return (one); 978 979 if (one->jd_seg->js_seq > two->jd_seg->js_seq) { 980 swp = one; 981 one = two; 982 two = swp; 983 } 984 WORKLIST_REMOVE(&two->jd_list); 985 free_jsegdep(two); 986 987 return (one); 988 } 989 990 /* 991 * If two freedeps are compatible free one to reduce list size. 992 */ 993 static inline struct freedep * 994 freedep_merge(struct freedep *one, struct freedep *two) 995 { 996 if (two == NULL) 997 return (one); 998 999 if (one->fd_freework == two->fd_freework) { 1000 WORKLIST_REMOVE(&two->fd_list); 1001 free_freedep(two); 1002 } 1003 return (one); 1004 } 1005 1006 /* 1007 * Move journal work from one list to another. Duplicate freedeps and 1008 * jsegdeps are coalesced to keep the lists as small as possible. 1009 */ 1010 static void 1011 jwork_move(dst, src) 1012 struct workhead *dst; 1013 struct workhead *src; 1014 { 1015 struct freedep *freedep; 1016 struct jsegdep *jsegdep; 1017 struct worklist *wkn; 1018 struct worklist *wk; 1019 1020 KASSERT(dst != src, 1021 ("jwork_move: dst == src")); 1022 freedep = NULL; 1023 jsegdep = NULL; 1024 LIST_FOREACH_SAFE(wk, dst, wk_list, wkn) { 1025 if (wk->wk_type == D_JSEGDEP) 1026 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1027 if (wk->wk_type == D_FREEDEP) 1028 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1029 } 1030 1031 mtx_assert(&lk, MA_OWNED); 1032 while ((wk = LIST_FIRST(src)) != NULL) { 1033 WORKLIST_REMOVE(wk); 1034 WORKLIST_INSERT(dst, wk); 1035 if (wk->wk_type == D_JSEGDEP) { 1036 jsegdep = jsegdep_merge(WK_JSEGDEP(wk), jsegdep); 1037 continue; 1038 } 1039 if (wk->wk_type == D_FREEDEP) 1040 freedep = freedep_merge(WK_FREEDEP(wk), freedep); 1041 } 1042 } 1043 1044 /* 1045 * Routines for tracking and managing workitems. 1046 */ 1047 static void workitem_free(struct worklist *, int); 1048 static void workitem_alloc(struct worklist *, int, struct mount *); 1049 1050 #define WORKITEM_FREE(item, type) workitem_free((struct worklist *)(item), (type)) 1051 1052 static void 1053 workitem_free(item, type) 1054 struct worklist *item; 1055 int type; 1056 { 1057 struct ufsmount *ump; 1058 mtx_assert(&lk, MA_OWNED); 1059 1060 #ifdef DEBUG 1061 if (item->wk_state & ONWORKLIST) 1062 panic("workitem_free: %s(0x%X) still on list", 1063 TYPENAME(item->wk_type), item->wk_state); 1064 if (item->wk_type != type) 1065 panic("workitem_free: type mismatch %s != %s", 1066 TYPENAME(item->wk_type), TYPENAME(type)); 1067 #endif 1068 ump = VFSTOUFS(item->wk_mp); 1069 if (--ump->softdep_deps == 0 && ump->softdep_req) 1070 wakeup(&ump->softdep_deps); 1071 dep_current[type]--; 1072 free(item, DtoM(type)); 1073 } 1074 1075 static void 1076 workitem_alloc(item, type, mp) 1077 struct worklist *item; 1078 int type; 1079 struct mount *mp; 1080 { 1081 item->wk_type = type; 1082 item->wk_mp = mp; 1083 item->wk_state = 0; 1084 ACQUIRE_LOCK(&lk); 1085 dep_current[type]++; 1086 dep_total[type]++; 1087 VFSTOUFS(mp)->softdep_deps++; 1088 VFSTOUFS(mp)->softdep_accdeps++; 1089 FREE_LOCK(&lk); 1090 } 1091 1092 /* 1093 * Workitem queue management 1094 */ 1095 static int max_softdeps; /* maximum number of structs before slowdown */ 1096 static int maxindirdeps = 50; /* max number of indirdeps before slowdown */ 1097 static int tickdelay = 2; /* number of ticks to pause during slowdown */ 1098 static int proc_waiting; /* tracks whether we have a timeout posted */ 1099 static int *stat_countp; /* statistic to count in proc_waiting timeout */ 1100 static struct callout softdep_callout; 1101 static int req_pending; 1102 static int req_clear_inodedeps; /* syncer process flush some inodedeps */ 1103 #define FLUSH_INODES 1 1104 static int req_clear_remove; /* syncer process flush some freeblks */ 1105 #define FLUSH_REMOVE 2 1106 #define FLUSH_REMOVE_WAIT 3 1107 static long num_freeblkdep; /* number of freeblks workitems allocated */ 1108 1109 /* 1110 * runtime statistics 1111 */ 1112 static int stat_worklist_push; /* number of worklist cleanups */ 1113 static int stat_blk_limit_push; /* number of times block limit neared */ 1114 static int stat_ino_limit_push; /* number of times inode limit neared */ 1115 static int stat_blk_limit_hit; /* number of times block slowdown imposed */ 1116 static int stat_ino_limit_hit; /* number of times inode slowdown imposed */ 1117 static int stat_sync_limit_hit; /* number of synchronous slowdowns imposed */ 1118 static int stat_indir_blk_ptrs; /* bufs redirtied as indir ptrs not written */ 1119 static int stat_inode_bitmap; /* bufs redirtied as inode bitmap not written */ 1120 static int stat_direct_blk_ptrs;/* bufs redirtied as direct ptrs not written */ 1121 static int stat_dir_entry; /* bufs redirtied as dir entry cannot write */ 1122 static int stat_jaddref; /* bufs redirtied as ino bitmap can not write */ 1123 static int stat_jnewblk; /* bufs redirtied as blk bitmap can not write */ 1124 static int stat_journal_min; /* Times hit journal min threshold */ 1125 static int stat_journal_low; /* Times hit journal low threshold */ 1126 static int stat_journal_wait; /* Times blocked in jwait(). */ 1127 static int stat_jwait_filepage; /* Times blocked in jwait() for filepage. */ 1128 static int stat_jwait_freeblks; /* Times blocked in jwait() for freeblks. */ 1129 static int stat_jwait_inode; /* Times blocked in jwait() for inodes. */ 1130 static int stat_jwait_newblk; /* Times blocked in jwait() for newblks. */ 1131 1132 SYSCTL_INT(_debug_softdep, OID_AUTO, max_softdeps, CTLFLAG_RW, 1133 &max_softdeps, 0, ""); 1134 SYSCTL_INT(_debug_softdep, OID_AUTO, tickdelay, CTLFLAG_RW, 1135 &tickdelay, 0, ""); 1136 SYSCTL_INT(_debug_softdep, OID_AUTO, maxindirdeps, CTLFLAG_RW, 1137 &maxindirdeps, 0, ""); 1138 SYSCTL_INT(_debug_softdep, OID_AUTO, worklist_push, CTLFLAG_RW, 1139 &stat_worklist_push, 0,""); 1140 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_push, CTLFLAG_RW, 1141 &stat_blk_limit_push, 0,""); 1142 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_push, CTLFLAG_RW, 1143 &stat_ino_limit_push, 0,""); 1144 SYSCTL_INT(_debug_softdep, OID_AUTO, blk_limit_hit, CTLFLAG_RW, 1145 &stat_blk_limit_hit, 0, ""); 1146 SYSCTL_INT(_debug_softdep, OID_AUTO, ino_limit_hit, CTLFLAG_RW, 1147 &stat_ino_limit_hit, 0, ""); 1148 SYSCTL_INT(_debug_softdep, OID_AUTO, sync_limit_hit, CTLFLAG_RW, 1149 &stat_sync_limit_hit, 0, ""); 1150 SYSCTL_INT(_debug_softdep, OID_AUTO, indir_blk_ptrs, CTLFLAG_RW, 1151 &stat_indir_blk_ptrs, 0, ""); 1152 SYSCTL_INT(_debug_softdep, OID_AUTO, inode_bitmap, CTLFLAG_RW, 1153 &stat_inode_bitmap, 0, ""); 1154 SYSCTL_INT(_debug_softdep, OID_AUTO, direct_blk_ptrs, CTLFLAG_RW, 1155 &stat_direct_blk_ptrs, 0, ""); 1156 SYSCTL_INT(_debug_softdep, OID_AUTO, dir_entry, CTLFLAG_RW, 1157 &stat_dir_entry, 0, ""); 1158 SYSCTL_INT(_debug_softdep, OID_AUTO, jaddref_rollback, CTLFLAG_RW, 1159 &stat_jaddref, 0, ""); 1160 SYSCTL_INT(_debug_softdep, OID_AUTO, jnewblk_rollback, CTLFLAG_RW, 1161 &stat_jnewblk, 0, ""); 1162 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_low, CTLFLAG_RW, 1163 &stat_journal_low, 0, ""); 1164 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_min, CTLFLAG_RW, 1165 &stat_journal_min, 0, ""); 1166 SYSCTL_INT(_debug_softdep, OID_AUTO, journal_wait, CTLFLAG_RW, 1167 &stat_journal_wait, 0, ""); 1168 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_filepage, CTLFLAG_RW, 1169 &stat_jwait_filepage, 0, ""); 1170 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_freeblks, CTLFLAG_RW, 1171 &stat_jwait_freeblks, 0, ""); 1172 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_inode, CTLFLAG_RW, 1173 &stat_jwait_inode, 0, ""); 1174 SYSCTL_INT(_debug_softdep, OID_AUTO, jwait_newblk, CTLFLAG_RW, 1175 &stat_jwait_newblk, 0, ""); 1176 1177 SYSCTL_DECL(_vfs_ffs); 1178 1179 LIST_HEAD(bmsafemap_hashhead, bmsafemap) *bmsafemap_hashtbl; 1180 static u_long bmsafemap_hash; /* size of hash table - 1 */ 1181 1182 static int compute_summary_at_mount = 0; /* Whether to recompute the summary at mount time */ 1183 SYSCTL_INT(_vfs_ffs, OID_AUTO, compute_summary_at_mount, CTLFLAG_RW, 1184 &compute_summary_at_mount, 0, "Recompute summary at mount"); 1185 1186 static struct proc *softdepproc; 1187 static struct kproc_desc softdep_kp = { 1188 "softdepflush", 1189 softdep_flush, 1190 &softdepproc 1191 }; 1192 SYSINIT(sdproc, SI_SUB_KTHREAD_UPDATE, SI_ORDER_ANY, kproc_start, 1193 &softdep_kp); 1194 1195 static void 1196 softdep_flush(void) 1197 { 1198 struct mount *nmp; 1199 struct mount *mp; 1200 struct ufsmount *ump; 1201 struct thread *td; 1202 int remaining; 1203 int progress; 1204 int vfslocked; 1205 1206 td = curthread; 1207 td->td_pflags |= TDP_NORUNNINGBUF; 1208 1209 for (;;) { 1210 kproc_suspend_check(softdepproc); 1211 vfslocked = VFS_LOCK_GIANT((struct mount *)NULL); 1212 ACQUIRE_LOCK(&lk); 1213 /* 1214 * If requested, try removing inode or removal dependencies. 1215 */ 1216 if (req_clear_inodedeps) { 1217 clear_inodedeps(td); 1218 req_clear_inodedeps -= 1; 1219 wakeup_one(&proc_waiting); 1220 } 1221 if (req_clear_remove) { 1222 clear_remove(td); 1223 req_clear_remove -= 1; 1224 wakeup_one(&proc_waiting); 1225 } 1226 FREE_LOCK(&lk); 1227 VFS_UNLOCK_GIANT(vfslocked); 1228 remaining = progress = 0; 1229 mtx_lock(&mountlist_mtx); 1230 for (mp = TAILQ_FIRST(&mountlist); mp != NULL; mp = nmp) { 1231 nmp = TAILQ_NEXT(mp, mnt_list); 1232 if ((mp->mnt_flag & MNT_SOFTDEP) == 0) 1233 continue; 1234 if (vfs_busy(mp, MBF_NOWAIT | MBF_MNTLSTLOCK)) 1235 continue; 1236 vfslocked = VFS_LOCK_GIANT(mp); 1237 progress += softdep_process_worklist(mp, 0); 1238 ump = VFSTOUFS(mp); 1239 remaining += ump->softdep_on_worklist - 1240 ump->softdep_on_worklist_inprogress; 1241 VFS_UNLOCK_GIANT(vfslocked); 1242 mtx_lock(&mountlist_mtx); 1243 nmp = TAILQ_NEXT(mp, mnt_list); 1244 vfs_unbusy(mp); 1245 } 1246 mtx_unlock(&mountlist_mtx); 1247 if (remaining && progress) 1248 continue; 1249 ACQUIRE_LOCK(&lk); 1250 if (!req_pending) 1251 msleep(&req_pending, &lk, PVM, "sdflush", hz); 1252 req_pending = 0; 1253 FREE_LOCK(&lk); 1254 } 1255 } 1256 1257 static void 1258 worklist_speedup(void) 1259 { 1260 mtx_assert(&lk, MA_OWNED); 1261 if (req_pending == 0) { 1262 req_pending = 1; 1263 wakeup(&req_pending); 1264 } 1265 } 1266 1267 static int 1268 softdep_speedup(void) 1269 { 1270 1271 worklist_speedup(); 1272 bd_speedup(); 1273 return speedup_syncer(); 1274 } 1275 1276 /* 1277 * Add an item to the end of the work queue. 1278 * This routine requires that the lock be held. 1279 * This is the only routine that adds items to the list. 1280 * The following routine is the only one that removes items 1281 * and does so in order from first to last. 1282 */ 1283 static void 1284 add_to_worklist(wk, nodelay) 1285 struct worklist *wk; 1286 int nodelay; 1287 { 1288 struct ufsmount *ump; 1289 1290 mtx_assert(&lk, MA_OWNED); 1291 ump = VFSTOUFS(wk->wk_mp); 1292 if (wk->wk_state & ONWORKLIST) 1293 panic("add_to_worklist: %s(0x%X) already on list", 1294 TYPENAME(wk->wk_type), wk->wk_state); 1295 wk->wk_state |= ONWORKLIST; 1296 if (LIST_EMPTY(&ump->softdep_workitem_pending)) 1297 LIST_INSERT_HEAD(&ump->softdep_workitem_pending, wk, wk_list); 1298 else 1299 LIST_INSERT_AFTER(ump->softdep_worklist_tail, wk, wk_list); 1300 ump->softdep_worklist_tail = wk; 1301 ump->softdep_on_worklist += 1; 1302 if (nodelay) 1303 worklist_speedup(); 1304 } 1305 1306 /* 1307 * Remove the item to be processed. If we are removing the last 1308 * item on the list, we need to recalculate the tail pointer. 1309 */ 1310 static void 1311 remove_from_worklist(wk) 1312 struct worklist *wk; 1313 { 1314 struct ufsmount *ump; 1315 struct worklist *wkend; 1316 1317 ump = VFSTOUFS(wk->wk_mp); 1318 WORKLIST_REMOVE(wk); 1319 if (wk == ump->softdep_worklist_tail) { 1320 LIST_FOREACH(wkend, &ump->softdep_workitem_pending, wk_list) 1321 if (LIST_NEXT(wkend, wk_list) == NULL) 1322 break; 1323 ump->softdep_worklist_tail = wkend; 1324 } 1325 ump->softdep_on_worklist -= 1; 1326 } 1327 1328 /* 1329 * Process that runs once per second to handle items in the background queue. 1330 * 1331 * Note that we ensure that everything is done in the order in which they 1332 * appear in the queue. The code below depends on this property to ensure 1333 * that blocks of a file are freed before the inode itself is freed. This 1334 * ordering ensures that no new <vfsid, inum, lbn> triples will be generated 1335 * until all the old ones have been purged from the dependency lists. 1336 */ 1337 int 1338 softdep_process_worklist(mp, full) 1339 struct mount *mp; 1340 int full; 1341 { 1342 struct thread *td = curthread; 1343 int cnt, matchcnt, loopcount; 1344 struct ufsmount *ump; 1345 long starttime; 1346 1347 KASSERT(mp != NULL, ("softdep_process_worklist: NULL mp")); 1348 /* 1349 * Record the process identifier of our caller so that we can give 1350 * this process preferential treatment in request_cleanup below. 1351 */ 1352 matchcnt = 0; 1353 ump = VFSTOUFS(mp); 1354 ACQUIRE_LOCK(&lk); 1355 loopcount = 1; 1356 starttime = time_second; 1357 softdep_process_journal(mp, full?MNT_WAIT:0); 1358 while (ump->softdep_on_worklist > 0) { 1359 if ((cnt = process_worklist_item(mp, LK_NOWAIT)) == -1) 1360 break; 1361 else 1362 matchcnt += cnt; 1363 /* 1364 * If requested, try removing inode or removal dependencies. 1365 */ 1366 if (req_clear_inodedeps) { 1367 clear_inodedeps(td); 1368 req_clear_inodedeps -= 1; 1369 wakeup_one(&proc_waiting); 1370 } 1371 if (req_clear_remove) { 1372 clear_remove(td); 1373 req_clear_remove -= 1; 1374 wakeup_one(&proc_waiting); 1375 } 1376 /* 1377 * We do not generally want to stop for buffer space, but if 1378 * we are really being a buffer hog, we will stop and wait. 1379 */ 1380 if (loopcount++ % 128 == 0) { 1381 FREE_LOCK(&lk); 1382 uio_yield(); 1383 bwillwrite(); 1384 ACQUIRE_LOCK(&lk); 1385 } 1386 /* 1387 * Never allow processing to run for more than one 1388 * second. Otherwise the other mountpoints may get 1389 * excessively backlogged. 1390 */ 1391 if (!full && starttime != time_second) 1392 break; 1393 } 1394 if (full == 0) 1395 journal_unsuspend(ump); 1396 FREE_LOCK(&lk); 1397 return (matchcnt); 1398 } 1399 1400 /* 1401 * Process all removes associated with a vnode if we are running out of 1402 * journal space. Any other process which attempts to flush these will 1403 * be unable as we have the vnodes locked. 1404 */ 1405 static void 1406 process_removes(vp) 1407 struct vnode *vp; 1408 { 1409 struct inodedep *inodedep; 1410 struct dirrem *dirrem; 1411 struct mount *mp; 1412 ino_t inum; 1413 1414 mtx_assert(&lk, MA_OWNED); 1415 1416 mp = vp->v_mount; 1417 inum = VTOI(vp)->i_number; 1418 for (;;) { 1419 if (inodedep_lookup(mp, inum, 0, &inodedep) == 0) 1420 return; 1421 LIST_FOREACH(dirrem, &inodedep->id_dirremhd, dm_inonext) 1422 if ((dirrem->dm_state & (COMPLETE | ONWORKLIST)) == 1423 (COMPLETE | ONWORKLIST)) 1424 break; 1425 if (dirrem == NULL) 1426 return; 1427 /* 1428 * If another thread is trying to lock this vnode it will 1429 * fail but we must wait for it to do so before we can 1430 * proceed. 1431 */ 1432 if (dirrem->dm_state & INPROGRESS) { 1433 dirrem->dm_state |= IOWAITING; 1434 msleep(&dirrem->dm_list, &lk, PVM, "pwrwait", 0); 1435 continue; 1436 } 1437 remove_from_worklist(&dirrem->dm_list); 1438 FREE_LOCK(&lk); 1439 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1440 panic("process_removes: suspended filesystem"); 1441 handle_workitem_remove(dirrem, vp); 1442 vn_finished_secondary_write(mp); 1443 ACQUIRE_LOCK(&lk); 1444 } 1445 } 1446 1447 /* 1448 * Process one item on the worklist. 1449 */ 1450 static int 1451 process_worklist_item(mp, flags) 1452 struct mount *mp; 1453 int flags; 1454 { 1455 struct worklist *wk; 1456 struct ufsmount *ump; 1457 struct vnode *vp; 1458 int matchcnt = 0; 1459 1460 mtx_assert(&lk, MA_OWNED); 1461 KASSERT(mp != NULL, ("process_worklist_item: NULL mp")); 1462 /* 1463 * If we are being called because of a process doing a 1464 * copy-on-write, then it is not safe to write as we may 1465 * recurse into the copy-on-write routine. 1466 */ 1467 if (curthread->td_pflags & TDP_COWINPROGRESS) 1468 return (-1); 1469 /* 1470 * Normally we just process each item on the worklist in order. 1471 * However, if we are in a situation where we cannot lock any 1472 * inodes, we have to skip over any dirrem requests whose 1473 * vnodes are resident and locked. 1474 */ 1475 vp = NULL; 1476 ump = VFSTOUFS(mp); 1477 LIST_FOREACH(wk, &ump->softdep_workitem_pending, wk_list) { 1478 if (wk->wk_state & INPROGRESS) 1479 continue; 1480 if ((flags & LK_NOWAIT) == 0 || wk->wk_type != D_DIRREM) 1481 break; 1482 wk->wk_state |= INPROGRESS; 1483 ump->softdep_on_worklist_inprogress++; 1484 FREE_LOCK(&lk); 1485 ffs_vgetf(mp, WK_DIRREM(wk)->dm_oldinum, 1486 LK_NOWAIT | LK_EXCLUSIVE, &vp, FFSV_FORCEINSMQ); 1487 ACQUIRE_LOCK(&lk); 1488 if (wk->wk_state & IOWAITING) { 1489 wk->wk_state &= ~IOWAITING; 1490 wakeup(wk); 1491 } 1492 wk->wk_state &= ~INPROGRESS; 1493 ump->softdep_on_worklist_inprogress--; 1494 if (vp != NULL) 1495 break; 1496 } 1497 if (wk == 0) 1498 return (-1); 1499 remove_from_worklist(wk); 1500 FREE_LOCK(&lk); 1501 if (vn_start_secondary_write(NULL, &mp, V_NOWAIT)) 1502 panic("process_worklist_item: suspended filesystem"); 1503 matchcnt++; 1504 switch (wk->wk_type) { 1505 1506 case D_DIRREM: 1507 /* removal of a directory entry */ 1508 handle_workitem_remove(WK_DIRREM(wk), vp); 1509 if (vp) 1510 vput(vp); 1511 break; 1512 1513 case D_FREEBLKS: 1514 /* releasing blocks and/or fragments from a file */ 1515 handle_workitem_freeblocks(WK_FREEBLKS(wk), flags & LK_NOWAIT); 1516 break; 1517 1518 case D_FREEFRAG: 1519 /* releasing a fragment when replaced as a file grows */ 1520 handle_workitem_freefrag(WK_FREEFRAG(wk)); 1521 break; 1522 1523 case D_FREEFILE: 1524 /* releasing an inode when its link count drops to 0 */ 1525 handle_workitem_freefile(WK_FREEFILE(wk)); 1526 break; 1527 1528 case D_FREEWORK: 1529 /* Final block in an indirect was freed. */ 1530 handle_workitem_indirblk(WK_FREEWORK(wk)); 1531 break; 1532 1533 default: 1534 panic("%s_process_worklist: Unknown type %s", 1535 "softdep", TYPENAME(wk->wk_type)); 1536 /* NOTREACHED */ 1537 } 1538 vn_finished_secondary_write(mp); 1539 ACQUIRE_LOCK(&lk); 1540 return (matchcnt); 1541 } 1542 1543 /* 1544 * Move dependencies from one buffer to another. 1545 */ 1546 int 1547 softdep_move_dependencies(oldbp, newbp) 1548 struct buf *oldbp; 1549 struct buf *newbp; 1550 { 1551 struct worklist *wk, *wktail; 1552 int dirty; 1553 1554 dirty = 0; 1555 wktail = NULL; 1556 ACQUIRE_LOCK(&lk); 1557 while ((wk = LIST_FIRST(&oldbp->b_dep)) != NULL) { 1558 LIST_REMOVE(wk, wk_list); 1559 if (wk->wk_type == D_BMSAFEMAP && 1560 bmsafemap_rollbacks(WK_BMSAFEMAP(wk))) 1561 dirty = 1; 1562 if (wktail == 0) 1563 LIST_INSERT_HEAD(&newbp->b_dep, wk, wk_list); 1564 else 1565 LIST_INSERT_AFTER(wktail, wk, wk_list); 1566 wktail = wk; 1567 } 1568 FREE_LOCK(&lk); 1569 1570 return (dirty); 1571 } 1572 1573 /* 1574 * Purge the work list of all items associated with a particular mount point. 1575 */ 1576 int 1577 softdep_flushworklist(oldmnt, countp, td) 1578 struct mount *oldmnt; 1579 int *countp; 1580 struct thread *td; 1581 { 1582 struct vnode *devvp; 1583 int count, error = 0; 1584 struct ufsmount *ump; 1585 1586 /* 1587 * Alternately flush the block device associated with the mount 1588 * point and process any dependencies that the flushing 1589 * creates. We continue until no more worklist dependencies 1590 * are found. 1591 */ 1592 *countp = 0; 1593 ump = VFSTOUFS(oldmnt); 1594 devvp = ump->um_devvp; 1595 while ((count = softdep_process_worklist(oldmnt, 1)) > 0) { 1596 *countp += count; 1597 vn_lock(devvp, LK_EXCLUSIVE | LK_RETRY); 1598 error = VOP_FSYNC(devvp, MNT_WAIT, td); 1599 VOP_UNLOCK(devvp, 0); 1600 if (error) 1601 break; 1602 } 1603 return (error); 1604 } 1605 1606 int 1607 softdep_waitidle(struct mount *mp) 1608 { 1609 struct ufsmount *ump; 1610 int error; 1611 int i; 1612 1613 ump = VFSTOUFS(mp); 1614 ACQUIRE_LOCK(&lk); 1615 for (i = 0; i < 10 && ump->softdep_deps; i++) { 1616 ump->softdep_req = 1; 1617 if (ump->softdep_on_worklist) 1618 panic("softdep_waitidle: work added after flush."); 1619 msleep(&ump->softdep_deps, &lk, PVM, "softdeps", 1); 1620 } 1621 ump->softdep_req = 0; 1622 FREE_LOCK(&lk); 1623 error = 0; 1624 if (i == 10) { 1625 error = EBUSY; 1626 printf("softdep_waitidle: Failed to flush worklist for %p\n", 1627 mp); 1628 } 1629 1630 return (error); 1631 } 1632 1633 /* 1634 * Flush all vnodes and worklist items associated with a specified mount point. 1635 */ 1636 int 1637 softdep_flushfiles(oldmnt, flags, td) 1638 struct mount *oldmnt; 1639 int flags; 1640 struct thread *td; 1641 { 1642 int error, depcount, loopcnt, retry_flush_count, retry; 1643 1644 loopcnt = 10; 1645 retry_flush_count = 3; 1646 retry_flush: 1647 error = 0; 1648 1649 /* 1650 * Alternately flush the vnodes associated with the mount 1651 * point and process any dependencies that the flushing 1652 * creates. In theory, this loop can happen at most twice, 1653 * but we give it a few extra just to be sure. 1654 */ 1655 for (; loopcnt > 0; loopcnt--) { 1656 /* 1657 * Do another flush in case any vnodes were brought in 1658 * as part of the cleanup operations. 1659 */ 1660 if ((error = ffs_flushfiles(oldmnt, flags, td)) != 0) 1661 break; 1662 if ((error = softdep_flushworklist(oldmnt, &depcount, td)) != 0 || 1663 depcount == 0) 1664 break; 1665 } 1666 /* 1667 * If we are unmounting then it is an error to fail. If we 1668 * are simply trying to downgrade to read-only, then filesystem 1669 * activity can keep us busy forever, so we just fail with EBUSY. 1670 */ 1671 if (loopcnt == 0) { 1672 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) 1673 panic("softdep_flushfiles: looping"); 1674 error = EBUSY; 1675 } 1676 if (!error) 1677 error = softdep_waitidle(oldmnt); 1678 if (!error) { 1679 if (oldmnt->mnt_kern_flag & MNTK_UNMOUNT) { 1680 retry = 0; 1681 MNT_ILOCK(oldmnt); 1682 KASSERT((oldmnt->mnt_kern_flag & MNTK_NOINSMNTQ) != 0, 1683 ("softdep_flushfiles: !MNTK_NOINSMNTQ")); 1684 if (oldmnt->mnt_nvnodelistsize > 0) { 1685 if (--retry_flush_count > 0) { 1686 retry = 1; 1687 loopcnt = 3; 1688 } else 1689 error = EBUSY; 1690 } 1691 MNT_IUNLOCK(oldmnt); 1692 if (retry) 1693 goto retry_flush; 1694 } 1695 } 1696 return (error); 1697 } 1698 1699 /* 1700 * Structure hashing. 1701 * 1702 * There are three types of structures that can be looked up: 1703 * 1) pagedep structures identified by mount point, inode number, 1704 * and logical block. 1705 * 2) inodedep structures identified by mount point and inode number. 1706 * 3) newblk structures identified by mount point and 1707 * physical block number. 1708 * 1709 * The "pagedep" and "inodedep" dependency structures are hashed 1710 * separately from the file blocks and inodes to which they correspond. 1711 * This separation helps when the in-memory copy of an inode or 1712 * file block must be replaced. It also obviates the need to access 1713 * an inode or file page when simply updating (or de-allocating) 1714 * dependency structures. Lookup of newblk structures is needed to 1715 * find newly allocated blocks when trying to associate them with 1716 * their allocdirect or allocindir structure. 1717 * 1718 * The lookup routines optionally create and hash a new instance when 1719 * an existing entry is not found. 1720 */ 1721 #define DEPALLOC 0x0001 /* allocate structure if lookup fails */ 1722 #define NODELAY 0x0002 /* cannot do background work */ 1723 1724 /* 1725 * Structures and routines associated with pagedep caching. 1726 */ 1727 LIST_HEAD(pagedep_hashhead, pagedep) *pagedep_hashtbl; 1728 u_long pagedep_hash; /* size of hash table - 1 */ 1729 #define PAGEDEP_HASH(mp, inum, lbn) \ 1730 (&pagedep_hashtbl[((((register_t)(mp)) >> 13) + (inum) + (lbn)) & \ 1731 pagedep_hash]) 1732 1733 static int 1734 pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp) 1735 struct pagedep_hashhead *pagedephd; 1736 ino_t ino; 1737 ufs_lbn_t lbn; 1738 struct mount *mp; 1739 int flags; 1740 struct pagedep **pagedeppp; 1741 { 1742 struct pagedep *pagedep; 1743 1744 LIST_FOREACH(pagedep, pagedephd, pd_hash) 1745 if (ino == pagedep->pd_ino && 1746 lbn == pagedep->pd_lbn && 1747 mp == pagedep->pd_list.wk_mp) 1748 break; 1749 if (pagedep) { 1750 *pagedeppp = pagedep; 1751 if ((flags & DEPALLOC) != 0 && 1752 (pagedep->pd_state & ONWORKLIST) == 0) 1753 return (0); 1754 return (1); 1755 } 1756 *pagedeppp = NULL; 1757 return (0); 1758 } 1759 /* 1760 * Look up a pagedep. Return 1 if found, 0 if not found or found 1761 * when asked to allocate but not associated with any buffer. 1762 * If not found, allocate if DEPALLOC flag is passed. 1763 * Found or allocated entry is returned in pagedeppp. 1764 * This routine must be called with splbio interrupts blocked. 1765 */ 1766 static int 1767 pagedep_lookup(mp, ino, lbn, flags, pagedeppp) 1768 struct mount *mp; 1769 ino_t ino; 1770 ufs_lbn_t lbn; 1771 int flags; 1772 struct pagedep **pagedeppp; 1773 { 1774 struct pagedep *pagedep; 1775 struct pagedep_hashhead *pagedephd; 1776 int ret; 1777 int i; 1778 1779 mtx_assert(&lk, MA_OWNED); 1780 pagedephd = PAGEDEP_HASH(mp, ino, lbn); 1781 1782 ret = pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp); 1783 if (*pagedeppp || (flags & DEPALLOC) == 0) 1784 return (ret); 1785 FREE_LOCK(&lk); 1786 pagedep = malloc(sizeof(struct pagedep), 1787 M_PAGEDEP, M_SOFTDEP_FLAGS|M_ZERO); 1788 workitem_alloc(&pagedep->pd_list, D_PAGEDEP, mp); 1789 ACQUIRE_LOCK(&lk); 1790 ret = pagedep_find(pagedephd, ino, lbn, mp, flags, pagedeppp); 1791 if (*pagedeppp) { 1792 WORKITEM_FREE(pagedep, D_PAGEDEP); 1793 return (ret); 1794 } 1795 pagedep->pd_ino = ino; 1796 pagedep->pd_lbn = lbn; 1797 LIST_INIT(&pagedep->pd_dirremhd); 1798 LIST_INIT(&pagedep->pd_pendinghd); 1799 for (i = 0; i < DAHASHSZ; i++) 1800 LIST_INIT(&pagedep->pd_diraddhd[i]); 1801 LIST_INSERT_HEAD(pagedephd, pagedep, pd_hash); 1802 *pagedeppp = pagedep; 1803 return (0); 1804 } 1805 1806 /* 1807 * Structures and routines associated with inodedep caching. 1808 */ 1809 LIST_HEAD(inodedep_hashhead, inodedep) *inodedep_hashtbl; 1810 static u_long inodedep_hash; /* size of hash table - 1 */ 1811 static long num_inodedep; /* number of inodedep allocated */ 1812 #define INODEDEP_HASH(fs, inum) \ 1813 (&inodedep_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & inodedep_hash]) 1814 1815 static int 1816 inodedep_find(inodedephd, fs, inum, inodedeppp) 1817 struct inodedep_hashhead *inodedephd; 1818 struct fs *fs; 1819 ino_t inum; 1820 struct inodedep **inodedeppp; 1821 { 1822 struct inodedep *inodedep; 1823 1824 LIST_FOREACH(inodedep, inodedephd, id_hash) 1825 if (inum == inodedep->id_ino && fs == inodedep->id_fs) 1826 break; 1827 if (inodedep) { 1828 *inodedeppp = inodedep; 1829 return (1); 1830 } 1831 *inodedeppp = NULL; 1832 1833 return (0); 1834 } 1835 /* 1836 * Look up an inodedep. Return 1 if found, 0 if not found. 1837 * If not found, allocate if DEPALLOC flag is passed. 1838 * Found or allocated entry is returned in inodedeppp. 1839 * This routine must be called with splbio interrupts blocked. 1840 */ 1841 static int 1842 inodedep_lookup(mp, inum, flags, inodedeppp) 1843 struct mount *mp; 1844 ino_t inum; 1845 int flags; 1846 struct inodedep **inodedeppp; 1847 { 1848 struct inodedep *inodedep; 1849 struct inodedep_hashhead *inodedephd; 1850 struct fs *fs; 1851 1852 mtx_assert(&lk, MA_OWNED); 1853 fs = VFSTOUFS(mp)->um_fs; 1854 inodedephd = INODEDEP_HASH(fs, inum); 1855 1856 if (inodedep_find(inodedephd, fs, inum, inodedeppp)) 1857 return (1); 1858 if ((flags & DEPALLOC) == 0) 1859 return (0); 1860 /* 1861 * If we are over our limit, try to improve the situation. 1862 */ 1863 if (num_inodedep > max_softdeps && (flags & NODELAY) == 0) 1864 request_cleanup(mp, FLUSH_INODES); 1865 FREE_LOCK(&lk); 1866 inodedep = malloc(sizeof(struct inodedep), 1867 M_INODEDEP, M_SOFTDEP_FLAGS); 1868 workitem_alloc(&inodedep->id_list, D_INODEDEP, mp); 1869 ACQUIRE_LOCK(&lk); 1870 if (inodedep_find(inodedephd, fs, inum, inodedeppp)) { 1871 WORKITEM_FREE(inodedep, D_INODEDEP); 1872 return (1); 1873 } 1874 num_inodedep += 1; 1875 inodedep->id_fs = fs; 1876 inodedep->id_ino = inum; 1877 inodedep->id_state = ALLCOMPLETE; 1878 inodedep->id_nlinkdelta = 0; 1879 inodedep->id_savedino1 = NULL; 1880 inodedep->id_savedsize = -1; 1881 inodedep->id_savedextsize = -1; 1882 inodedep->id_savednlink = -1; 1883 inodedep->id_bmsafemap = NULL; 1884 inodedep->id_mkdiradd = NULL; 1885 LIST_INIT(&inodedep->id_dirremhd); 1886 LIST_INIT(&inodedep->id_pendinghd); 1887 LIST_INIT(&inodedep->id_inowait); 1888 LIST_INIT(&inodedep->id_bufwait); 1889 TAILQ_INIT(&inodedep->id_inoreflst); 1890 TAILQ_INIT(&inodedep->id_inoupdt); 1891 TAILQ_INIT(&inodedep->id_newinoupdt); 1892 TAILQ_INIT(&inodedep->id_extupdt); 1893 TAILQ_INIT(&inodedep->id_newextupdt); 1894 LIST_INSERT_HEAD(inodedephd, inodedep, id_hash); 1895 *inodedeppp = inodedep; 1896 return (0); 1897 } 1898 1899 /* 1900 * Structures and routines associated with newblk caching. 1901 */ 1902 LIST_HEAD(newblk_hashhead, newblk) *newblk_hashtbl; 1903 u_long newblk_hash; /* size of hash table - 1 */ 1904 #define NEWBLK_HASH(fs, inum) \ 1905 (&newblk_hashtbl[((((register_t)(fs)) >> 13) + (inum)) & newblk_hash]) 1906 1907 static int 1908 newblk_find(newblkhd, mp, newblkno, flags, newblkpp) 1909 struct newblk_hashhead *newblkhd; 1910 struct mount *mp; 1911 ufs2_daddr_t newblkno; 1912 int flags; 1913 struct newblk **newblkpp; 1914 { 1915 struct newblk *newblk; 1916 1917 LIST_FOREACH(newblk, newblkhd, nb_hash) { 1918 if (newblkno != newblk->nb_newblkno) 1919 continue; 1920 if (mp != newblk->nb_list.wk_mp) 1921 continue; 1922 /* 1923 * If we're creating a new dependency don't match those that 1924 * have already been converted to allocdirects. This is for 1925 * a frag extend. 1926 */ 1927 if ((flags & DEPALLOC) && newblk->nb_list.wk_type != D_NEWBLK) 1928 continue; 1929 break; 1930 } 1931 if (newblk) { 1932 *newblkpp = newblk; 1933 return (1); 1934 } 1935 *newblkpp = NULL; 1936 return (0); 1937 } 1938 1939 /* 1940 * Look up a newblk. Return 1 if found, 0 if not found. 1941 * If not found, allocate if DEPALLOC flag is passed. 1942 * Found or allocated entry is returned in newblkpp. 1943 */ 1944 static int 1945 newblk_lookup(mp, newblkno, flags, newblkpp) 1946 struct mount *mp; 1947 ufs2_daddr_t newblkno; 1948 int flags; 1949 struct newblk **newblkpp; 1950 { 1951 struct newblk *newblk; 1952 struct newblk_hashhead *newblkhd; 1953 1954 newblkhd = NEWBLK_HASH(VFSTOUFS(mp)->um_fs, newblkno); 1955 if (newblk_find(newblkhd, mp, newblkno, flags, newblkpp)) 1956 return (1); 1957 if ((flags & DEPALLOC) == 0) 1958 return (0); 1959 FREE_LOCK(&lk); 1960 newblk = malloc(sizeof(union allblk), M_NEWBLK, 1961 M_SOFTDEP_FLAGS | M_ZERO); 1962 workitem_alloc(&newblk->nb_list, D_NEWBLK, mp); 1963 ACQUIRE_LOCK(&lk); 1964 if (newblk_find(newblkhd, mp, newblkno, flags, newblkpp)) { 1965 WORKITEM_FREE(newblk, D_NEWBLK); 1966 return (1); 1967 } 1968 newblk->nb_freefrag = NULL; 1969 LIST_INIT(&newblk->nb_indirdeps); 1970 LIST_INIT(&newblk->nb_newdirblk); 1971 LIST_INIT(&newblk->nb_jwork); 1972 newblk->nb_state = ATTACHED; 1973 newblk->nb_newblkno = newblkno; 1974 LIST_INSERT_HEAD(newblkhd, newblk, nb_hash); 1975 *newblkpp = newblk; 1976 return (0); 1977 } 1978 1979 /* 1980 * Executed during filesystem system initialization before 1981 * mounting any filesystems. 1982 */ 1983 void 1984 softdep_initialize() 1985 { 1986 1987 LIST_INIT(&mkdirlisthd); 1988 max_softdeps = desiredvnodes * 4; 1989 pagedep_hashtbl = hashinit(desiredvnodes / 5, M_PAGEDEP, &pagedep_hash); 1990 inodedep_hashtbl = hashinit(desiredvnodes, M_INODEDEP, &inodedep_hash); 1991 newblk_hashtbl = hashinit(desiredvnodes / 5, M_NEWBLK, &newblk_hash); 1992 bmsafemap_hashtbl = hashinit(1024, M_BMSAFEMAP, &bmsafemap_hash); 1993 1994 /* initialise bioops hack */ 1995 bioops.io_start = softdep_disk_io_initiation; 1996 bioops.io_complete = softdep_disk_write_complete; 1997 bioops.io_deallocate = softdep_deallocate_dependencies; 1998 bioops.io_countdeps = softdep_count_dependencies; 1999 2000 /* Initialize the callout with an mtx. */ 2001 callout_init_mtx(&softdep_callout, &lk, 0); 2002 } 2003 2004 /* 2005 * Executed after all filesystems have been unmounted during 2006 * filesystem module unload. 2007 */ 2008 void 2009 softdep_uninitialize() 2010 { 2011 2012 callout_drain(&softdep_callout); 2013 hashdestroy(pagedep_hashtbl, M_PAGEDEP, pagedep_hash); 2014 hashdestroy(inodedep_hashtbl, M_INODEDEP, inodedep_hash); 2015 hashdestroy(newblk_hashtbl, M_NEWBLK, newblk_hash); 2016 hashdestroy(bmsafemap_hashtbl, M_BMSAFEMAP, bmsafemap_hash); 2017 } 2018 2019 /* 2020 * Called at mount time to notify the dependency code that a 2021 * filesystem wishes to use it. 2022 */ 2023 int 2024 softdep_mount(devvp, mp, fs, cred) 2025 struct vnode *devvp; 2026 struct mount *mp; 2027 struct fs *fs; 2028 struct ucred *cred; 2029 { 2030 struct csum_total cstotal; 2031 struct ufsmount *ump; 2032 struct cg *cgp; 2033 struct buf *bp; 2034 int error, cyl; 2035 2036 MNT_ILOCK(mp); 2037 mp->mnt_flag = (mp->mnt_flag & ~MNT_ASYNC) | MNT_SOFTDEP; 2038 if ((mp->mnt_kern_flag & MNTK_SOFTDEP) == 0) { 2039 mp->mnt_kern_flag = (mp->mnt_kern_flag & ~MNTK_ASYNC) | 2040 MNTK_SOFTDEP; 2041 mp->mnt_noasync++; 2042 } 2043 MNT_IUNLOCK(mp); 2044 ump = VFSTOUFS(mp); 2045 LIST_INIT(&ump->softdep_workitem_pending); 2046 LIST_INIT(&ump->softdep_journal_pending); 2047 TAILQ_INIT(&ump->softdep_unlinked); 2048 ump->softdep_worklist_tail = NULL; 2049 ump->softdep_on_worklist = 0; 2050 ump->softdep_deps = 0; 2051 if ((fs->fs_flags & FS_SUJ) && 2052 (error = journal_mount(mp, fs, cred)) != 0) { 2053 printf("Failed to start journal: %d\n", error); 2054 return (error); 2055 } 2056 /* 2057 * When doing soft updates, the counters in the 2058 * superblock may have gotten out of sync. Recomputation 2059 * can take a long time and can be deferred for background 2060 * fsck. However, the old behavior of scanning the cylinder 2061 * groups and recalculating them at mount time is available 2062 * by setting vfs.ffs.compute_summary_at_mount to one. 2063 */ 2064 if (compute_summary_at_mount == 0 || fs->fs_clean != 0) 2065 return (0); 2066 bzero(&cstotal, sizeof cstotal); 2067 for (cyl = 0; cyl < fs->fs_ncg; cyl++) { 2068 if ((error = bread(devvp, fsbtodb(fs, cgtod(fs, cyl)), 2069 fs->fs_cgsize, cred, &bp)) != 0) { 2070 brelse(bp); 2071 return (error); 2072 } 2073 cgp = (struct cg *)bp->b_data; 2074 cstotal.cs_nffree += cgp->cg_cs.cs_nffree; 2075 cstotal.cs_nbfree += cgp->cg_cs.cs_nbfree; 2076 cstotal.cs_nifree += cgp->cg_cs.cs_nifree; 2077 cstotal.cs_ndir += cgp->cg_cs.cs_ndir; 2078 fs->fs_cs(fs, cyl) = cgp->cg_cs; 2079 brelse(bp); 2080 } 2081 #ifdef DEBUG 2082 if (bcmp(&cstotal, &fs->fs_cstotal, sizeof cstotal)) 2083 printf("%s: superblock summary recomputed\n", fs->fs_fsmnt); 2084 #endif 2085 bcopy(&cstotal, &fs->fs_cstotal, sizeof cstotal); 2086 return (0); 2087 } 2088 2089 void 2090 softdep_unmount(mp) 2091 struct mount *mp; 2092 { 2093 2094 if (mp->mnt_kern_flag & MNTK_SUJ) 2095 journal_unmount(mp); 2096 } 2097 2098 struct jblocks { 2099 struct jseglst jb_segs; /* TAILQ of current segments. */ 2100 struct jseg *jb_writeseg; /* Next write to complete. */ 2101 struct jextent *jb_extent; /* Extent array. */ 2102 uint64_t jb_nextseq; /* Next sequence number. */ 2103 uint64_t jb_oldestseq; /* Oldest active sequence number. */ 2104 int jb_avail; /* Available extents. */ 2105 int jb_used; /* Last used extent. */ 2106 int jb_head; /* Allocator head. */ 2107 int jb_off; /* Allocator extent offset. */ 2108 int jb_blocks; /* Total disk blocks covered. */ 2109 int jb_free; /* Total disk blocks free. */ 2110 int jb_min; /* Minimum free space. */ 2111 int jb_low; /* Low on space. */ 2112 int jb_age; /* Insertion time of oldest rec. */ 2113 int jb_suspended; /* Did journal suspend writes? */ 2114 }; 2115 2116 struct jextent { 2117 ufs2_daddr_t je_daddr; /* Disk block address. */ 2118 int je_blocks; /* Disk block count. */ 2119 }; 2120 2121 static struct jblocks * 2122 jblocks_create(void) 2123 { 2124 struct jblocks *jblocks; 2125 2126 jblocks = malloc(sizeof(*jblocks), M_JBLOCKS, M_WAITOK | M_ZERO); 2127 TAILQ_INIT(&jblocks->jb_segs); 2128 jblocks->jb_avail = 10; 2129 jblocks->jb_extent = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2130 M_JBLOCKS, M_WAITOK | M_ZERO); 2131 2132 return (jblocks); 2133 } 2134 2135 static ufs2_daddr_t 2136 jblocks_alloc(jblocks, bytes, actual) 2137 struct jblocks *jblocks; 2138 int bytes; 2139 int *actual; 2140 { 2141 ufs2_daddr_t daddr; 2142 struct jextent *jext; 2143 int freecnt; 2144 int blocks; 2145 2146 blocks = bytes / DEV_BSIZE; 2147 jext = &jblocks->jb_extent[jblocks->jb_head]; 2148 freecnt = jext->je_blocks - jblocks->jb_off; 2149 if (freecnt == 0) { 2150 jblocks->jb_off = 0; 2151 if (++jblocks->jb_head > jblocks->jb_used) 2152 jblocks->jb_head = 0; 2153 jext = &jblocks->jb_extent[jblocks->jb_head]; 2154 freecnt = jext->je_blocks; 2155 } 2156 if (freecnt > blocks) 2157 freecnt = blocks; 2158 *actual = freecnt * DEV_BSIZE; 2159 daddr = jext->je_daddr + jblocks->jb_off; 2160 jblocks->jb_off += freecnt; 2161 jblocks->jb_free -= freecnt; 2162 2163 return (daddr); 2164 } 2165 2166 static void 2167 jblocks_free(jblocks, mp, bytes) 2168 struct jblocks *jblocks; 2169 struct mount *mp; 2170 int bytes; 2171 { 2172 2173 jblocks->jb_free += bytes / DEV_BSIZE; 2174 if (jblocks->jb_suspended) 2175 worklist_speedup(); 2176 wakeup(jblocks); 2177 } 2178 2179 static void 2180 jblocks_destroy(jblocks) 2181 struct jblocks *jblocks; 2182 { 2183 2184 if (jblocks->jb_extent) 2185 free(jblocks->jb_extent, M_JBLOCKS); 2186 free(jblocks, M_JBLOCKS); 2187 } 2188 2189 static void 2190 jblocks_add(jblocks, daddr, blocks) 2191 struct jblocks *jblocks; 2192 ufs2_daddr_t daddr; 2193 int blocks; 2194 { 2195 struct jextent *jext; 2196 2197 jblocks->jb_blocks += blocks; 2198 jblocks->jb_free += blocks; 2199 jext = &jblocks->jb_extent[jblocks->jb_used]; 2200 /* Adding the first block. */ 2201 if (jext->je_daddr == 0) { 2202 jext->je_daddr = daddr; 2203 jext->je_blocks = blocks; 2204 return; 2205 } 2206 /* Extending the last extent. */ 2207 if (jext->je_daddr + jext->je_blocks == daddr) { 2208 jext->je_blocks += blocks; 2209 return; 2210 } 2211 /* Adding a new extent. */ 2212 if (++jblocks->jb_used == jblocks->jb_avail) { 2213 jblocks->jb_avail *= 2; 2214 jext = malloc(sizeof(struct jextent) * jblocks->jb_avail, 2215 M_JBLOCKS, M_WAITOK | M_ZERO); 2216 memcpy(jext, jblocks->jb_extent, 2217 sizeof(struct jextent) * jblocks->jb_used); 2218 free(jblocks->jb_extent, M_JBLOCKS); 2219 jblocks->jb_extent = jext; 2220 } 2221 jext = &jblocks->jb_extent[jblocks->jb_used]; 2222 jext->je_daddr = daddr; 2223 jext->je_blocks = blocks; 2224 return; 2225 } 2226 2227 int 2228 softdep_journal_lookup(mp, vpp) 2229 struct mount *mp; 2230 struct vnode **vpp; 2231 { 2232 struct componentname cnp; 2233 struct vnode *dvp; 2234 ino_t sujournal; 2235 int error; 2236 2237 error = VFS_VGET(mp, ROOTINO, LK_EXCLUSIVE, &dvp); 2238 if (error) 2239 return (error); 2240 bzero(&cnp, sizeof(cnp)); 2241 cnp.cn_nameiop = LOOKUP; 2242 cnp.cn_flags = ISLASTCN; 2243 cnp.cn_thread = curthread; 2244 cnp.cn_cred = curthread->td_ucred; 2245 cnp.cn_pnbuf = SUJ_FILE; 2246 cnp.cn_nameptr = SUJ_FILE; 2247 cnp.cn_namelen = strlen(SUJ_FILE); 2248 error = ufs_lookup_ino(dvp, NULL, &cnp, &sujournal); 2249 vput(dvp); 2250 if (error != 0) 2251 return (error); 2252 error = VFS_VGET(mp, sujournal, LK_EXCLUSIVE, vpp); 2253 return (error); 2254 } 2255 2256 /* 2257 * Open and verify the journal file. 2258 */ 2259 static int 2260 journal_mount(mp, fs, cred) 2261 struct mount *mp; 2262 struct fs *fs; 2263 struct ucred *cred; 2264 { 2265 struct jblocks *jblocks; 2266 struct vnode *vp; 2267 struct inode *ip; 2268 ufs2_daddr_t blkno; 2269 int bcount; 2270 int error; 2271 int i; 2272 2273 error = softdep_journal_lookup(mp, &vp); 2274 if (error != 0) { 2275 printf("Failed to find journal. Use tunefs to create one\n"); 2276 return (error); 2277 } 2278 ip = VTOI(vp); 2279 if (ip->i_size < SUJ_MIN) { 2280 error = ENOSPC; 2281 goto out; 2282 } 2283 bcount = lblkno(fs, ip->i_size); /* Only use whole blocks. */ 2284 jblocks = jblocks_create(); 2285 for (i = 0; i < bcount; i++) { 2286 error = ufs_bmaparray(vp, i, &blkno, NULL, NULL, NULL); 2287 if (error) 2288 break; 2289 jblocks_add(jblocks, blkno, fsbtodb(fs, fs->fs_frag)); 2290 } 2291 if (error) { 2292 jblocks_destroy(jblocks); 2293 goto out; 2294 } 2295 jblocks->jb_low = jblocks->jb_free / 3; /* Reserve 33%. */ 2296 jblocks->jb_min = jblocks->jb_free / 10; /* Suspend at 10%. */ 2297 VFSTOUFS(mp)->softdep_jblocks = jblocks; 2298 out: 2299 if (error == 0) { 2300 MNT_ILOCK(mp); 2301 mp->mnt_kern_flag |= MNTK_SUJ; 2302 MNT_IUNLOCK(mp); 2303 /* 2304 * Only validate the journal contents if the 2305 * filesystem is clean, otherwise we write the logs 2306 * but they'll never be used. If the filesystem was 2307 * still dirty when we mounted it the journal is 2308 * invalid and a new journal can only be valid if it 2309 * starts from a clean mount. 2310 */ 2311 if (fs->fs_clean) { 2312 DIP_SET(ip, i_modrev, fs->fs_mtime); 2313 ip->i_flags |= IN_MODIFIED; 2314 ffs_update(vp, 1); 2315 } 2316 } 2317 vput(vp); 2318 return (error); 2319 } 2320 2321 static void 2322 journal_unmount(mp) 2323 struct mount *mp; 2324 { 2325 struct ufsmount *ump; 2326 2327 ump = VFSTOUFS(mp); 2328 if (ump->softdep_jblocks) 2329 jblocks_destroy(ump->softdep_jblocks); 2330 ump->softdep_jblocks = NULL; 2331 } 2332 2333 /* 2334 * Called when a journal record is ready to be written. Space is allocated 2335 * and the journal entry is created when the journal is flushed to stable 2336 * store. 2337 */ 2338 static void 2339 add_to_journal(wk) 2340 struct worklist *wk; 2341 { 2342 struct ufsmount *ump; 2343 2344 mtx_assert(&lk, MA_OWNED); 2345 ump = VFSTOUFS(wk->wk_mp); 2346 if (wk->wk_state & ONWORKLIST) 2347 panic("add_to_journal: %s(0x%X) already on list", 2348 TYPENAME(wk->wk_type), wk->wk_state); 2349 wk->wk_state |= ONWORKLIST | DEPCOMPLETE; 2350 if (LIST_EMPTY(&ump->softdep_journal_pending)) { 2351 ump->softdep_jblocks->jb_age = ticks; 2352 LIST_INSERT_HEAD(&ump->softdep_journal_pending, wk, wk_list); 2353 } else 2354 LIST_INSERT_AFTER(ump->softdep_journal_tail, wk, wk_list); 2355 ump->softdep_journal_tail = wk; 2356 ump->softdep_on_journal += 1; 2357 } 2358 2359 /* 2360 * Remove an arbitrary item for the journal worklist maintain the tail 2361 * pointer. This happens when a new operation obviates the need to 2362 * journal an old operation. 2363 */ 2364 static void 2365 remove_from_journal(wk) 2366 struct worklist *wk; 2367 { 2368 struct ufsmount *ump; 2369 2370 mtx_assert(&lk, MA_OWNED); 2371 ump = VFSTOUFS(wk->wk_mp); 2372 #ifdef SUJ_DEBUG 2373 { 2374 struct worklist *wkn; 2375 2376 LIST_FOREACH(wkn, &ump->softdep_journal_pending, wk_list) 2377 if (wkn == wk) 2378 break; 2379 if (wkn == NULL) 2380 panic("remove_from_journal: %p is not in journal", wk); 2381 } 2382 #endif 2383 /* 2384 * We emulate a TAILQ to save space in most structures which do not 2385 * require TAILQ semantics. Here we must update the tail position 2386 * when removing the tail which is not the final entry. This works 2387 * only if the worklist linkage are at the beginning of the structure. 2388 */ 2389 if (ump->softdep_journal_tail == wk) 2390 ump->softdep_journal_tail = 2391 (struct worklist *)wk->wk_list.le_prev; 2392 2393 WORKLIST_REMOVE(wk); 2394 ump->softdep_on_journal -= 1; 2395 } 2396 2397 /* 2398 * Check for journal space as well as dependency limits so the prelink 2399 * code can throttle both journaled and non-journaled filesystems. 2400 * Threshold is 0 for low and 1 for min. 2401 */ 2402 static int 2403 journal_space(ump, thresh) 2404 struct ufsmount *ump; 2405 int thresh; 2406 { 2407 struct jblocks *jblocks; 2408 int avail; 2409 2410 jblocks = ump->softdep_jblocks; 2411 if (jblocks == NULL) 2412 return (1); 2413 /* 2414 * We use a tighter restriction here to prevent request_cleanup() 2415 * running in threads from running into locks we currently hold. 2416 */ 2417 if (num_inodedep > (max_softdeps / 10) * 9) 2418 return (0); 2419 if (thresh) 2420 thresh = jblocks->jb_min; 2421 else 2422 thresh = jblocks->jb_low; 2423 avail = (ump->softdep_on_journal * JREC_SIZE) / DEV_BSIZE; 2424 avail = jblocks->jb_free - avail; 2425 2426 return (avail > thresh); 2427 } 2428 2429 static void 2430 journal_suspend(ump) 2431 struct ufsmount *ump; 2432 { 2433 struct jblocks *jblocks; 2434 struct mount *mp; 2435 2436 mp = UFSTOVFS(ump); 2437 jblocks = ump->softdep_jblocks; 2438 MNT_ILOCK(mp); 2439 if ((mp->mnt_kern_flag & MNTK_SUSPEND) == 0) { 2440 stat_journal_min++; 2441 mp->mnt_kern_flag |= MNTK_SUSPEND; 2442 mp->mnt_susp_owner = FIRST_THREAD_IN_PROC(softdepproc); 2443 } 2444 jblocks->jb_suspended = 1; 2445 MNT_IUNLOCK(mp); 2446 } 2447 2448 static int 2449 journal_unsuspend(struct ufsmount *ump) 2450 { 2451 struct jblocks *jblocks; 2452 struct mount *mp; 2453 2454 mp = UFSTOVFS(ump); 2455 jblocks = ump->softdep_jblocks; 2456 2457 if (jblocks != NULL && jblocks->jb_suspended && 2458 journal_space(ump, jblocks->jb_min)) { 2459 jblocks->jb_suspended = 0; 2460 FREE_LOCK(&lk); 2461 mp->mnt_susp_owner = curthread; 2462 vfs_write_resume(mp); 2463 ACQUIRE_LOCK(&lk); 2464 return (1); 2465 } 2466 return (0); 2467 } 2468 2469 /* 2470 * Called before any allocation function to be certain that there is 2471 * sufficient space in the journal prior to creating any new records. 2472 * Since in the case of block allocation we may have multiple locked 2473 * buffers at the time of the actual allocation we can not block 2474 * when the journal records are created. Doing so would create a deadlock 2475 * if any of these buffers needed to be flushed to reclaim space. Instead 2476 * we require a sufficiently large amount of available space such that 2477 * each thread in the system could have passed this allocation check and 2478 * still have sufficient free space. With 20% of a minimum journal size 2479 * of 1MB we have 6553 records available. 2480 */ 2481 int 2482 softdep_prealloc(vp, waitok) 2483 struct vnode *vp; 2484 int waitok; 2485 { 2486 struct ufsmount *ump; 2487 2488 if (DOINGSUJ(vp) == 0) 2489 return (0); 2490 ump = VFSTOUFS(vp->v_mount); 2491 ACQUIRE_LOCK(&lk); 2492 if (journal_space(ump, 0)) { 2493 FREE_LOCK(&lk); 2494 return (0); 2495 } 2496 stat_journal_low++; 2497 FREE_LOCK(&lk); 2498 if (waitok == MNT_NOWAIT) 2499 return (ENOSPC); 2500 /* 2501 * Attempt to sync this vnode once to flush any journal 2502 * work attached to it. 2503 */ 2504 if ((curthread->td_pflags & TDP_COWINPROGRESS) == 0) 2505 ffs_syncvnode(vp, waitok); 2506 ACQUIRE_LOCK(&lk); 2507 process_removes(vp); 2508 if (journal_space(ump, 0) == 0) { 2509 softdep_speedup(); 2510 if (journal_space(ump, 1) == 0) 2511 journal_suspend(ump); 2512 } 2513 FREE_LOCK(&lk); 2514 2515 return (0); 2516 } 2517 2518 /* 2519 * Before adjusting a link count on a vnode verify that we have sufficient 2520 * journal space. If not, process operations that depend on the currently 2521 * locked pair of vnodes to try to flush space as the syncer, buf daemon, 2522 * and softdep flush threads can not acquire these locks to reclaim space. 2523 */ 2524 static void 2525 softdep_prelink(dvp, vp) 2526 struct vnode *dvp; 2527 struct vnode *vp; 2528 { 2529 struct ufsmount *ump; 2530 2531 ump = VFSTOUFS(dvp->v_mount); 2532 mtx_assert(&lk, MA_OWNED); 2533 if (journal_space(ump, 0)) 2534 return; 2535 stat_journal_low++; 2536 FREE_LOCK(&lk); 2537 if (vp) 2538 ffs_syncvnode(vp, MNT_NOWAIT); 2539 ffs_syncvnode(dvp, MNT_WAIT); 2540 ACQUIRE_LOCK(&lk); 2541 /* Process vp before dvp as it may create .. removes. */ 2542 if (vp) 2543 process_removes(vp); 2544 process_removes(dvp); 2545 softdep_speedup(); 2546 process_worklist_item(UFSTOVFS(ump), LK_NOWAIT); 2547 process_worklist_item(UFSTOVFS(ump), LK_NOWAIT); 2548 if (journal_space(ump, 0) == 0) { 2549 softdep_speedup(); 2550 if (journal_space(ump, 1) == 0) 2551 journal_suspend(ump); 2552 } 2553 } 2554 2555 static void 2556 jseg_write(fs, jblocks, jseg, data) 2557 struct fs *fs; 2558 struct jblocks *jblocks; 2559 struct jseg *jseg; 2560 uint8_t *data; 2561 { 2562 struct jsegrec *rec; 2563 2564 rec = (struct jsegrec *)data; 2565 rec->jsr_seq = jseg->js_seq; 2566 rec->jsr_oldest = jblocks->jb_oldestseq; 2567 rec->jsr_cnt = jseg->js_cnt; 2568 rec->jsr_blocks = jseg->js_size / DEV_BSIZE; 2569 rec->jsr_crc = 0; 2570 rec->jsr_time = fs->fs_mtime; 2571 } 2572 2573 static inline void 2574 inoref_write(inoref, jseg, rec) 2575 struct inoref *inoref; 2576 struct jseg *jseg; 2577 struct jrefrec *rec; 2578 { 2579 2580 inoref->if_jsegdep->jd_seg = jseg; 2581 rec->jr_ino = inoref->if_ino; 2582 rec->jr_parent = inoref->if_parent; 2583 rec->jr_nlink = inoref->if_nlink; 2584 rec->jr_mode = inoref->if_mode; 2585 rec->jr_diroff = inoref->if_diroff; 2586 } 2587 2588 static void 2589 jaddref_write(jaddref, jseg, data) 2590 struct jaddref *jaddref; 2591 struct jseg *jseg; 2592 uint8_t *data; 2593 { 2594 struct jrefrec *rec; 2595 2596 rec = (struct jrefrec *)data; 2597 rec->jr_op = JOP_ADDREF; 2598 inoref_write(&jaddref->ja_ref, jseg, rec); 2599 } 2600 2601 static void 2602 jremref_write(jremref, jseg, data) 2603 struct jremref *jremref; 2604 struct jseg *jseg; 2605 uint8_t *data; 2606 { 2607 struct jrefrec *rec; 2608 2609 rec = (struct jrefrec *)data; 2610 rec->jr_op = JOP_REMREF; 2611 inoref_write(&jremref->jr_ref, jseg, rec); 2612 } 2613 2614 static void 2615 jmvref_write(jmvref, jseg, data) 2616 struct jmvref *jmvref; 2617 struct jseg *jseg; 2618 uint8_t *data; 2619 { 2620 struct jmvrec *rec; 2621 2622 rec = (struct jmvrec *)data; 2623 rec->jm_op = JOP_MVREF; 2624 rec->jm_ino = jmvref->jm_ino; 2625 rec->jm_parent = jmvref->jm_parent; 2626 rec->jm_oldoff = jmvref->jm_oldoff; 2627 rec->jm_newoff = jmvref->jm_newoff; 2628 } 2629 2630 static void 2631 jnewblk_write(jnewblk, jseg, data) 2632 struct jnewblk *jnewblk; 2633 struct jseg *jseg; 2634 uint8_t *data; 2635 { 2636 struct jblkrec *rec; 2637 2638 jnewblk->jn_jsegdep->jd_seg = jseg; 2639 rec = (struct jblkrec *)data; 2640 rec->jb_op = JOP_NEWBLK; 2641 rec->jb_ino = jnewblk->jn_ino; 2642 rec->jb_blkno = jnewblk->jn_blkno; 2643 rec->jb_lbn = jnewblk->jn_lbn; 2644 rec->jb_frags = jnewblk->jn_frags; 2645 rec->jb_oldfrags = jnewblk->jn_oldfrags; 2646 } 2647 2648 static void 2649 jfreeblk_write(jfreeblk, jseg, data) 2650 struct jfreeblk *jfreeblk; 2651 struct jseg *jseg; 2652 uint8_t *data; 2653 { 2654 struct jblkrec *rec; 2655 2656 jfreeblk->jf_jsegdep->jd_seg = jseg; 2657 rec = (struct jblkrec *)data; 2658 rec->jb_op = JOP_FREEBLK; 2659 rec->jb_ino = jfreeblk->jf_ino; 2660 rec->jb_blkno = jfreeblk->jf_blkno; 2661 rec->jb_lbn = jfreeblk->jf_lbn; 2662 rec->jb_frags = jfreeblk->jf_frags; 2663 rec->jb_oldfrags = 0; 2664 } 2665 2666 static void 2667 jfreefrag_write(jfreefrag, jseg, data) 2668 struct jfreefrag *jfreefrag; 2669 struct jseg *jseg; 2670 uint8_t *data; 2671 { 2672 struct jblkrec *rec; 2673 2674 jfreefrag->fr_jsegdep->jd_seg = jseg; 2675 rec = (struct jblkrec *)data; 2676 rec->jb_op = JOP_FREEBLK; 2677 rec->jb_ino = jfreefrag->fr_ino; 2678 rec->jb_blkno = jfreefrag->fr_blkno; 2679 rec->jb_lbn = jfreefrag->fr_lbn; 2680 rec->jb_frags = jfreefrag->fr_frags; 2681 rec->jb_oldfrags = 0; 2682 } 2683 2684 static void 2685 jtrunc_write(jtrunc, jseg, data) 2686 struct jtrunc *jtrunc; 2687 struct jseg *jseg; 2688 uint8_t *data; 2689 { 2690 struct jtrncrec *rec; 2691 2692 rec = (struct jtrncrec *)data; 2693 rec->jt_op = JOP_TRUNC; 2694 rec->jt_ino = jtrunc->jt_ino; 2695 rec->jt_size = jtrunc->jt_size; 2696 rec->jt_extsize = jtrunc->jt_extsize; 2697 } 2698 2699 /* 2700 * Flush some journal records to disk. 2701 */ 2702 static void 2703 softdep_process_journal(mp, flags) 2704 struct mount *mp; 2705 int flags; 2706 { 2707 struct jblocks *jblocks; 2708 struct ufsmount *ump; 2709 struct worklist *wk; 2710 struct jseg *jseg; 2711 struct buf *bp; 2712 uint8_t *data; 2713 struct fs *fs; 2714 int segwritten; 2715 int jrecmin; /* Minimum records per block. */ 2716 int jrecmax; /* Maximum records per block. */ 2717 int size; 2718 int cnt; 2719 int off; 2720 2721 if ((mp->mnt_kern_flag & MNTK_SUJ) == 0) 2722 return; 2723 ump = VFSTOUFS(mp); 2724 fs = ump->um_fs; 2725 jblocks = ump->softdep_jblocks; 2726 /* 2727 * We write anywhere between a disk block and fs block. The upper 2728 * bound is picked to prevent buffer cache fragmentation and limit 2729 * processing time per I/O. 2730 */ 2731 jrecmin = (DEV_BSIZE / JREC_SIZE) - 1; /* -1 for seg header */ 2732 jrecmax = (fs->fs_bsize / DEV_BSIZE) * jrecmin; 2733 segwritten = 0; 2734 while ((cnt = ump->softdep_on_journal) != 0) { 2735 /* 2736 * Create a new segment to hold as many as 'cnt' journal 2737 * entries and add them to the segment. Notice cnt is 2738 * off by one to account for the space required by the 2739 * jsegrec. If we don't have a full block to log skip it 2740 * unless we haven't written anything. 2741 */ 2742 cnt++; 2743 if (cnt < jrecmax && segwritten) 2744 break; 2745 /* 2746 * Verify some free journal space. softdep_prealloc() should 2747 * guarantee that we don't run out so this is indicative of 2748 * a problem with the flow control. Try to recover 2749 * gracefully in any event. 2750 */ 2751 while (jblocks->jb_free == 0) { 2752 if (flags != MNT_WAIT) 2753 break; 2754 printf("softdep: Out of journal space!\n"); 2755 softdep_speedup(); 2756 msleep(jblocks, &lk, PRIBIO, "jblocks", hz); 2757 } 2758 FREE_LOCK(&lk); 2759 jseg = malloc(sizeof(*jseg), M_JSEG, M_SOFTDEP_FLAGS); 2760 workitem_alloc(&jseg->js_list, D_JSEG, mp); 2761 LIST_INIT(&jseg->js_entries); 2762 jseg->js_state = ATTACHED; 2763 jseg->js_jblocks = jblocks; 2764 bp = geteblk(fs->fs_bsize, 0); 2765 ACQUIRE_LOCK(&lk); 2766 /* 2767 * If there was a race while we were allocating the block 2768 * and jseg the entry we care about was likely written. 2769 * We bail out in both the WAIT and NOWAIT case and assume 2770 * the caller will loop if the entry it cares about is 2771 * not written. 2772 */ 2773 if (ump->softdep_on_journal == 0 || jblocks->jb_free == 0) { 2774 bp->b_flags |= B_INVAL | B_NOCACHE; 2775 WORKITEM_FREE(jseg, D_JSEG); 2776 FREE_LOCK(&lk); 2777 brelse(bp); 2778 ACQUIRE_LOCK(&lk); 2779 break; 2780 } 2781 /* 2782 * Calculate the disk block size required for the available 2783 * records rounded to the min size. 2784 */ 2785 cnt = ump->softdep_on_journal; 2786 if (cnt < jrecmax) 2787 size = howmany(cnt, jrecmin) * DEV_BSIZE; 2788 else 2789 size = fs->fs_bsize; 2790 /* 2791 * Allocate a disk block for this journal data and account 2792 * for truncation of the requested size if enough contiguous 2793 * space was not available. 2794 */ 2795 bp->b_blkno = jblocks_alloc(jblocks, size, &size); 2796 bp->b_lblkno = bp->b_blkno; 2797 bp->b_offset = bp->b_blkno * DEV_BSIZE; 2798 bp->b_bcount = size; 2799 bp->b_bufobj = &ump->um_devvp->v_bufobj; 2800 bp->b_flags &= ~B_INVAL; 2801 bp->b_flags |= B_VALIDSUSPWRT | B_NOCOPY; 2802 /* 2803 * Initialize our jseg with cnt records. Assign the next 2804 * sequence number to it and link it in-order. 2805 */ 2806 cnt = MIN(ump->softdep_on_journal, 2807 (size / DEV_BSIZE) * jrecmin); 2808 jseg->js_buf = bp; 2809 jseg->js_cnt = cnt; 2810 jseg->js_refs = cnt + 1; /* Self ref. */ 2811 jseg->js_size = size; 2812 jseg->js_seq = jblocks->jb_nextseq++; 2813 if (TAILQ_EMPTY(&jblocks->jb_segs)) 2814 jblocks->jb_oldestseq = jseg->js_seq; 2815 TAILQ_INSERT_TAIL(&jblocks->jb_segs, jseg, js_next); 2816 if (jblocks->jb_writeseg == NULL) 2817 jblocks->jb_writeseg = jseg; 2818 /* 2819 * Start filling in records from the pending list. 2820 */ 2821 data = bp->b_data; 2822 off = 0; 2823 while ((wk = LIST_FIRST(&ump->softdep_journal_pending)) 2824 != NULL) { 2825 /* Place a segment header on every device block. */ 2826 if ((off % DEV_BSIZE) == 0) { 2827 jseg_write(fs, jblocks, jseg, data); 2828 off += JREC_SIZE; 2829 data = bp->b_data + off; 2830 } 2831 remove_from_journal(wk); 2832 wk->wk_state |= IOSTARTED; 2833 WORKLIST_INSERT(&jseg->js_entries, wk); 2834 switch (wk->wk_type) { 2835 case D_JADDREF: 2836 jaddref_write(WK_JADDREF(wk), jseg, data); 2837 break; 2838 case D_JREMREF: 2839 jremref_write(WK_JREMREF(wk), jseg, data); 2840 break; 2841 case D_JMVREF: 2842 jmvref_write(WK_JMVREF(wk), jseg, data); 2843 break; 2844 case D_JNEWBLK: 2845 jnewblk_write(WK_JNEWBLK(wk), jseg, data); 2846 break; 2847 case D_JFREEBLK: 2848 jfreeblk_write(WK_JFREEBLK(wk), jseg, data); 2849 break; 2850 case D_JFREEFRAG: 2851 jfreefrag_write(WK_JFREEFRAG(wk), jseg, data); 2852 break; 2853 case D_JTRUNC: 2854 jtrunc_write(WK_JTRUNC(wk), jseg, data); 2855 break; 2856 default: 2857 panic("process_journal: Unknown type %s", 2858 TYPENAME(wk->wk_type)); 2859 /* NOTREACHED */ 2860 } 2861 if (--cnt == 0) 2862 break; 2863 off += JREC_SIZE; 2864 data = bp->b_data + off; 2865 } 2866 /* 2867 * Write this one buffer and continue. 2868 */ 2869 WORKLIST_INSERT(&bp->b_dep, &jseg->js_list); 2870 FREE_LOCK(&lk); 2871 BO_LOCK(bp->b_bufobj); 2872 bgetvp(ump->um_devvp, bp); 2873 BO_UNLOCK(bp->b_bufobj); 2874 if (flags == MNT_NOWAIT) 2875 bawrite(bp); 2876 else 2877 bwrite(bp); 2878 ACQUIRE_LOCK(&lk); 2879 } 2880 /* 2881 * If we've suspended the filesystem because we ran out of journal 2882 * space either try to sync it here to make some progress or 2883 * unsuspend it if we already have. 2884 */ 2885 if (flags == 0 && jblocks->jb_suspended) { 2886 if (journal_unsuspend(ump)) 2887 return; 2888 FREE_LOCK(&lk); 2889 VFS_SYNC(mp, MNT_NOWAIT); 2890 ffs_sbupdate(ump, MNT_WAIT, 0); 2891 ACQUIRE_LOCK(&lk); 2892 } 2893 } 2894 2895 /* 2896 * Complete a jseg, allowing all dependencies awaiting journal writes 2897 * to proceed. Each journal dependency also attaches a jsegdep to dependent 2898 * structures so that the journal segment can be freed to reclaim space. 2899 */ 2900 static void 2901 complete_jseg(jseg) 2902 struct jseg *jseg; 2903 { 2904 struct worklist *wk; 2905 struct jmvref *jmvref; 2906 int waiting; 2907 #ifdef INVARIANTS 2908 int i = 0; 2909 #endif 2910 2911 while ((wk = LIST_FIRST(&jseg->js_entries)) != NULL) { 2912 WORKLIST_REMOVE(wk); 2913 waiting = wk->wk_state & IOWAITING; 2914 wk->wk_state &= ~(IOSTARTED | IOWAITING); 2915 wk->wk_state |= COMPLETE; 2916 KASSERT(i++ < jseg->js_cnt, 2917 ("handle_written_jseg: overflow %d >= %d", 2918 i - 1, jseg->js_cnt)); 2919 switch (wk->wk_type) { 2920 case D_JADDREF: 2921 handle_written_jaddref(WK_JADDREF(wk)); 2922 break; 2923 case D_JREMREF: 2924 handle_written_jremref(WK_JREMREF(wk)); 2925 break; 2926 case D_JMVREF: 2927 /* No jsegdep here. */ 2928 free_jseg(jseg); 2929 jmvref = WK_JMVREF(wk); 2930 LIST_REMOVE(jmvref, jm_deps); 2931 free_pagedep(jmvref->jm_pagedep); 2932 WORKITEM_FREE(jmvref, D_JMVREF); 2933 break; 2934 case D_JNEWBLK: 2935 handle_written_jnewblk(WK_JNEWBLK(wk)); 2936 break; 2937 case D_JFREEBLK: 2938 handle_written_jfreeblk(WK_JFREEBLK(wk)); 2939 break; 2940 case D_JFREEFRAG: 2941 handle_written_jfreefrag(WK_JFREEFRAG(wk)); 2942 break; 2943 case D_JTRUNC: 2944 WK_JTRUNC(wk)->jt_jsegdep->jd_seg = jseg; 2945 WORKITEM_FREE(wk, D_JTRUNC); 2946 break; 2947 default: 2948 panic("handle_written_jseg: Unknown type %s", 2949 TYPENAME(wk->wk_type)); 2950 /* NOTREACHED */ 2951 } 2952 if (waiting) 2953 wakeup(wk); 2954 } 2955 /* Release the self reference so the structure may be freed. */ 2956 free_jseg(jseg); 2957 } 2958 2959 /* 2960 * Mark a jseg as DEPCOMPLETE and throw away the buffer. Handle jseg 2961 * completions in order only. 2962 */ 2963 static void 2964 handle_written_jseg(jseg, bp) 2965 struct jseg *jseg; 2966 struct buf *bp; 2967 { 2968 struct jblocks *jblocks; 2969 struct jseg *jsegn; 2970 2971 if (jseg->js_refs == 0) 2972 panic("handle_written_jseg: No self-reference on %p", jseg); 2973 jseg->js_state |= DEPCOMPLETE; 2974 /* 2975 * We'll never need this buffer again, set flags so it will be 2976 * discarded. 2977 */ 2978 bp->b_flags |= B_INVAL | B_NOCACHE; 2979 jblocks = jseg->js_jblocks; 2980 /* 2981 * Don't allow out of order completions. If this isn't the first 2982 * block wait for it to write before we're done. 2983 */ 2984 if (jseg != jblocks->jb_writeseg) 2985 return; 2986 /* Iterate through available jsegs processing their entries. */ 2987 do { 2988 jsegn = TAILQ_NEXT(jseg, js_next); 2989 complete_jseg(jseg); 2990 jseg = jsegn; 2991 } while (jseg && jseg->js_state & DEPCOMPLETE); 2992 jblocks->jb_writeseg = jseg; 2993 } 2994 2995 static inline struct jsegdep * 2996 inoref_jseg(inoref) 2997 struct inoref *inoref; 2998 { 2999 struct jsegdep *jsegdep; 3000 3001 jsegdep = inoref->if_jsegdep; 3002 inoref->if_jsegdep = NULL; 3003 3004 return (jsegdep); 3005 } 3006 3007 /* 3008 * Called once a jremref has made it to stable store. The jremref is marked 3009 * complete and we attempt to free it. Any pagedeps writes sleeping waiting 3010 * for the jremref to complete will be awoken by free_jremref. 3011 */ 3012 static void 3013 handle_written_jremref(jremref) 3014 struct jremref *jremref; 3015 { 3016 struct inodedep *inodedep; 3017 struct jsegdep *jsegdep; 3018 struct dirrem *dirrem; 3019 3020 /* Grab the jsegdep. */ 3021 jsegdep = inoref_jseg(&jremref->jr_ref); 3022 /* 3023 * Remove us from the inoref list. 3024 */ 3025 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 3026 0, &inodedep) == 0) 3027 panic("handle_written_jremref: Lost inodedep"); 3028 TAILQ_REMOVE(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 3029 /* 3030 * Complete the dirrem. 3031 */ 3032 dirrem = jremref->jr_dirrem; 3033 jremref->jr_dirrem = NULL; 3034 LIST_REMOVE(jremref, jr_deps); 3035 jsegdep->jd_state |= jremref->jr_state & MKDIR_PARENT; 3036 WORKLIST_INSERT(&dirrem->dm_jwork, &jsegdep->jd_list); 3037 if (LIST_EMPTY(&dirrem->dm_jremrefhd) && 3038 (dirrem->dm_state & COMPLETE) != 0) 3039 add_to_worklist(&dirrem->dm_list, 0); 3040 free_jremref(jremref); 3041 } 3042 3043 /* 3044 * Called once a jaddref has made it to stable store. The dependency is 3045 * marked complete and any dependent structures are added to the inode 3046 * bufwait list to be completed as soon as it is written. If a bitmap write 3047 * depends on this entry we move the inode into the inodedephd of the 3048 * bmsafemap dependency and attempt to remove the jaddref from the bmsafemap. 3049 */ 3050 static void 3051 handle_written_jaddref(jaddref) 3052 struct jaddref *jaddref; 3053 { 3054 struct jsegdep *jsegdep; 3055 struct inodedep *inodedep; 3056 struct diradd *diradd; 3057 struct mkdir *mkdir; 3058 3059 /* Grab the jsegdep. */ 3060 jsegdep = inoref_jseg(&jaddref->ja_ref); 3061 mkdir = NULL; 3062 diradd = NULL; 3063 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 3064 0, &inodedep) == 0) 3065 panic("handle_written_jaddref: Lost inodedep."); 3066 if (jaddref->ja_diradd == NULL) 3067 panic("handle_written_jaddref: No dependency"); 3068 if (jaddref->ja_diradd->da_list.wk_type == D_DIRADD) { 3069 diradd = jaddref->ja_diradd; 3070 WORKLIST_INSERT(&inodedep->id_bufwait, &diradd->da_list); 3071 } else if (jaddref->ja_state & MKDIR_PARENT) { 3072 mkdir = jaddref->ja_mkdir; 3073 WORKLIST_INSERT(&inodedep->id_bufwait, &mkdir->md_list); 3074 } else if (jaddref->ja_state & MKDIR_BODY) 3075 mkdir = jaddref->ja_mkdir; 3076 else 3077 panic("handle_written_jaddref: Unknown dependency %p", 3078 jaddref->ja_diradd); 3079 jaddref->ja_diradd = NULL; /* also clears ja_mkdir */ 3080 /* 3081 * Remove us from the inode list. 3082 */ 3083 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, if_deps); 3084 /* 3085 * The mkdir may be waiting on the jaddref to clear before freeing. 3086 */ 3087 if (mkdir) { 3088 KASSERT(mkdir->md_list.wk_type == D_MKDIR, 3089 ("handle_written_jaddref: Incorrect type for mkdir %s", 3090 TYPENAME(mkdir->md_list.wk_type))); 3091 mkdir->md_jaddref = NULL; 3092 diradd = mkdir->md_diradd; 3093 mkdir->md_state |= DEPCOMPLETE; 3094 complete_mkdir(mkdir); 3095 } 3096 WORKLIST_INSERT(&diradd->da_jwork, &jsegdep->jd_list); 3097 if (jaddref->ja_state & NEWBLOCK) { 3098 inodedep->id_state |= ONDEPLIST; 3099 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_inodedephd, 3100 inodedep, id_deps); 3101 } 3102 free_jaddref(jaddref); 3103 } 3104 3105 /* 3106 * Called once a jnewblk journal is written. The allocdirect or allocindir 3107 * is placed in the bmsafemap to await notification of a written bitmap. 3108 */ 3109 static void 3110 handle_written_jnewblk(jnewblk) 3111 struct jnewblk *jnewblk; 3112 { 3113 struct bmsafemap *bmsafemap; 3114 struct jsegdep *jsegdep; 3115 struct newblk *newblk; 3116 3117 /* Grab the jsegdep. */ 3118 jsegdep = jnewblk->jn_jsegdep; 3119 jnewblk->jn_jsegdep = NULL; 3120 /* 3121 * Add the written block to the bmsafemap so it can be notified when 3122 * the bitmap is on disk. 3123 */ 3124 newblk = jnewblk->jn_newblk; 3125 jnewblk->jn_newblk = NULL; 3126 if (newblk == NULL) 3127 panic("handle_written_jnewblk: No dependency for the segdep."); 3128 3129 newblk->nb_jnewblk = NULL; 3130 bmsafemap = newblk->nb_bmsafemap; 3131 WORKLIST_INSERT(&newblk->nb_jwork, &jsegdep->jd_list); 3132 newblk->nb_state |= ONDEPLIST; 3133 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps); 3134 free_jnewblk(jnewblk); 3135 } 3136 3137 /* 3138 * Cancel a jfreefrag that won't be needed, probably due to colliding with 3139 * an in-flight allocation that has not yet been committed. Divorce us 3140 * from the freefrag and mark it DEPCOMPLETE so that it may be added 3141 * to the worklist. 3142 */ 3143 static void 3144 cancel_jfreefrag(jfreefrag) 3145 struct jfreefrag *jfreefrag; 3146 { 3147 struct freefrag *freefrag; 3148 3149 if (jfreefrag->fr_jsegdep) { 3150 free_jsegdep(jfreefrag->fr_jsegdep); 3151 jfreefrag->fr_jsegdep = NULL; 3152 } 3153 freefrag = jfreefrag->fr_freefrag; 3154 jfreefrag->fr_freefrag = NULL; 3155 freefrag->ff_jfreefrag = NULL; 3156 free_jfreefrag(jfreefrag); 3157 freefrag->ff_state |= DEPCOMPLETE; 3158 } 3159 3160 /* 3161 * Free a jfreefrag when the parent freefrag is rendered obsolete. 3162 */ 3163 static void 3164 free_jfreefrag(jfreefrag) 3165 struct jfreefrag *jfreefrag; 3166 { 3167 3168 if (jfreefrag->fr_state & IOSTARTED) 3169 WORKLIST_REMOVE(&jfreefrag->fr_list); 3170 else if (jfreefrag->fr_state & ONWORKLIST) 3171 remove_from_journal(&jfreefrag->fr_list); 3172 if (jfreefrag->fr_freefrag != NULL) 3173 panic("free_jfreefrag: Still attached to a freefrag."); 3174 WORKITEM_FREE(jfreefrag, D_JFREEFRAG); 3175 } 3176 3177 /* 3178 * Called when the journal write for a jfreefrag completes. The parent 3179 * freefrag is added to the worklist if this completes its dependencies. 3180 */ 3181 static void 3182 handle_written_jfreefrag(jfreefrag) 3183 struct jfreefrag *jfreefrag; 3184 { 3185 struct jsegdep *jsegdep; 3186 struct freefrag *freefrag; 3187 3188 /* Grab the jsegdep. */ 3189 jsegdep = jfreefrag->fr_jsegdep; 3190 jfreefrag->fr_jsegdep = NULL; 3191 freefrag = jfreefrag->fr_freefrag; 3192 if (freefrag == NULL) 3193 panic("handle_written_jfreefrag: No freefrag."); 3194 freefrag->ff_state |= DEPCOMPLETE; 3195 freefrag->ff_jfreefrag = NULL; 3196 WORKLIST_INSERT(&freefrag->ff_jwork, &jsegdep->jd_list); 3197 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 3198 add_to_worklist(&freefrag->ff_list, 0); 3199 jfreefrag->fr_freefrag = NULL; 3200 free_jfreefrag(jfreefrag); 3201 } 3202 3203 /* 3204 * Called when the journal write for a jfreeblk completes. The jfreeblk 3205 * is removed from the freeblks list of pending journal writes and the 3206 * jsegdep is moved to the freeblks jwork to be completed when all blocks 3207 * have been reclaimed. 3208 */ 3209 static void 3210 handle_written_jfreeblk(jfreeblk) 3211 struct jfreeblk *jfreeblk; 3212 { 3213 struct freeblks *freeblks; 3214 struct jsegdep *jsegdep; 3215 3216 /* Grab the jsegdep. */ 3217 jsegdep = jfreeblk->jf_jsegdep; 3218 jfreeblk->jf_jsegdep = NULL; 3219 freeblks = jfreeblk->jf_freeblks; 3220 LIST_REMOVE(jfreeblk, jf_deps); 3221 WORKLIST_INSERT(&freeblks->fb_jwork, &jsegdep->jd_list); 3222 /* 3223 * If the freeblks is all journaled, we can add it to the worklist. 3224 */ 3225 if (LIST_EMPTY(&freeblks->fb_jfreeblkhd) && 3226 (freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) { 3227 /* Remove from the b_dep that is waiting on this write. */ 3228 if (freeblks->fb_state & ONWORKLIST) 3229 WORKLIST_REMOVE(&freeblks->fb_list); 3230 add_to_worklist(&freeblks->fb_list, 1); 3231 } 3232 3233 free_jfreeblk(jfreeblk); 3234 } 3235 3236 static struct jsegdep * 3237 newjsegdep(struct worklist *wk) 3238 { 3239 struct jsegdep *jsegdep; 3240 3241 jsegdep = malloc(sizeof(*jsegdep), M_JSEGDEP, M_SOFTDEP_FLAGS); 3242 workitem_alloc(&jsegdep->jd_list, D_JSEGDEP, wk->wk_mp); 3243 jsegdep->jd_seg = NULL; 3244 3245 return (jsegdep); 3246 } 3247 3248 static struct jmvref * 3249 newjmvref(dp, ino, oldoff, newoff) 3250 struct inode *dp; 3251 ino_t ino; 3252 off_t oldoff; 3253 off_t newoff; 3254 { 3255 struct jmvref *jmvref; 3256 3257 jmvref = malloc(sizeof(*jmvref), M_JMVREF, M_SOFTDEP_FLAGS); 3258 workitem_alloc(&jmvref->jm_list, D_JMVREF, UFSTOVFS(dp->i_ump)); 3259 jmvref->jm_list.wk_state = ATTACHED | DEPCOMPLETE; 3260 jmvref->jm_parent = dp->i_number; 3261 jmvref->jm_ino = ino; 3262 jmvref->jm_oldoff = oldoff; 3263 jmvref->jm_newoff = newoff; 3264 3265 return (jmvref); 3266 } 3267 3268 /* 3269 * Allocate a new jremref that tracks the removal of ip from dp with the 3270 * directory entry offset of diroff. Mark the entry as ATTACHED and 3271 * DEPCOMPLETE as we have all the information required for the journal write 3272 * and the directory has already been removed from the buffer. The caller 3273 * is responsible for linking the jremref into the pagedep and adding it 3274 * to the journal to write. The MKDIR_PARENT flag is set if we're doing 3275 * a DOTDOT addition so handle_workitem_remove() can properly assign 3276 * the jsegdep when we're done. 3277 */ 3278 static struct jremref * 3279 newjremref(struct dirrem *dirrem, struct inode *dp, struct inode *ip, 3280 off_t diroff, nlink_t nlink) 3281 { 3282 struct jremref *jremref; 3283 3284 jremref = malloc(sizeof(*jremref), M_JREMREF, M_SOFTDEP_FLAGS); 3285 workitem_alloc(&jremref->jr_list, D_JREMREF, UFSTOVFS(dp->i_ump)); 3286 jremref->jr_state = ATTACHED; 3287 newinoref(&jremref->jr_ref, ip->i_number, dp->i_number, diroff, 3288 nlink, ip->i_mode); 3289 jremref->jr_dirrem = dirrem; 3290 3291 return (jremref); 3292 } 3293 3294 static inline void 3295 newinoref(struct inoref *inoref, ino_t ino, ino_t parent, off_t diroff, 3296 nlink_t nlink, uint16_t mode) 3297 { 3298 3299 inoref->if_jsegdep = newjsegdep(&inoref->if_list); 3300 inoref->if_diroff = diroff; 3301 inoref->if_ino = ino; 3302 inoref->if_parent = parent; 3303 inoref->if_nlink = nlink; 3304 inoref->if_mode = mode; 3305 } 3306 3307 /* 3308 * Allocate a new jaddref to track the addition of ino to dp at diroff. The 3309 * directory offset may not be known until later. The caller is responsible 3310 * adding the entry to the journal when this information is available. nlink 3311 * should be the link count prior to the addition and mode is only required 3312 * to have the correct FMT. 3313 */ 3314 static struct jaddref * 3315 newjaddref(struct inode *dp, ino_t ino, off_t diroff, int16_t nlink, 3316 uint16_t mode) 3317 { 3318 struct jaddref *jaddref; 3319 3320 jaddref = malloc(sizeof(*jaddref), M_JADDREF, M_SOFTDEP_FLAGS); 3321 workitem_alloc(&jaddref->ja_list, D_JADDREF, UFSTOVFS(dp->i_ump)); 3322 jaddref->ja_state = ATTACHED; 3323 jaddref->ja_mkdir = NULL; 3324 newinoref(&jaddref->ja_ref, ino, dp->i_number, diroff, nlink, mode); 3325 3326 return (jaddref); 3327 } 3328 3329 /* 3330 * Create a new free dependency for a freework. The caller is responsible 3331 * for adjusting the reference count when it has the lock held. The freedep 3332 * will track an outstanding bitmap write that will ultimately clear the 3333 * freework to continue. 3334 */ 3335 static struct freedep * 3336 newfreedep(struct freework *freework) 3337 { 3338 struct freedep *freedep; 3339 3340 freedep = malloc(sizeof(*freedep), M_FREEDEP, M_SOFTDEP_FLAGS); 3341 workitem_alloc(&freedep->fd_list, D_FREEDEP, freework->fw_list.wk_mp); 3342 freedep->fd_freework = freework; 3343 3344 return (freedep); 3345 } 3346 3347 /* 3348 * Free a freedep structure once the buffer it is linked to is written. If 3349 * this is the last reference to the freework schedule it for completion. 3350 */ 3351 static void 3352 free_freedep(freedep) 3353 struct freedep *freedep; 3354 { 3355 3356 if (--freedep->fd_freework->fw_ref == 0) 3357 add_to_worklist(&freedep->fd_freework->fw_list, 1); 3358 WORKITEM_FREE(freedep, D_FREEDEP); 3359 } 3360 3361 /* 3362 * Allocate a new freework structure that may be a level in an indirect 3363 * when parent is not NULL or a top level block when it is. The top level 3364 * freework structures are allocated without lk held and before the freeblks 3365 * is visible outside of softdep_setup_freeblocks(). 3366 */ 3367 static struct freework * 3368 newfreework(freeblks, parent, lbn, nb, frags, journal) 3369 struct freeblks *freeblks; 3370 struct freework *parent; 3371 ufs_lbn_t lbn; 3372 ufs2_daddr_t nb; 3373 int frags; 3374 int journal; 3375 { 3376 struct freework *freework; 3377 3378 freework = malloc(sizeof(*freework), M_FREEWORK, M_SOFTDEP_FLAGS); 3379 workitem_alloc(&freework->fw_list, D_FREEWORK, freeblks->fb_list.wk_mp); 3380 freework->fw_freeblks = freeblks; 3381 freework->fw_parent = parent; 3382 freework->fw_lbn = lbn; 3383 freework->fw_blkno = nb; 3384 freework->fw_frags = frags; 3385 freework->fw_ref = 0; 3386 freework->fw_off = 0; 3387 LIST_INIT(&freework->fw_jwork); 3388 3389 if (parent == NULL) { 3390 WORKLIST_INSERT_UNLOCKED(&freeblks->fb_freeworkhd, 3391 &freework->fw_list); 3392 freeblks->fb_ref++; 3393 } 3394 if (journal) 3395 newjfreeblk(freeblks, lbn, nb, frags); 3396 3397 return (freework); 3398 } 3399 3400 /* 3401 * Allocate a new jfreeblk to journal top level block pointer when truncating 3402 * a file. The caller must add this to the worklist when lk is held. 3403 */ 3404 static struct jfreeblk * 3405 newjfreeblk(freeblks, lbn, blkno, frags) 3406 struct freeblks *freeblks; 3407 ufs_lbn_t lbn; 3408 ufs2_daddr_t blkno; 3409 int frags; 3410 { 3411 struct jfreeblk *jfreeblk; 3412 3413 jfreeblk = malloc(sizeof(*jfreeblk), M_JFREEBLK, M_SOFTDEP_FLAGS); 3414 workitem_alloc(&jfreeblk->jf_list, D_JFREEBLK, freeblks->fb_list.wk_mp); 3415 jfreeblk->jf_jsegdep = newjsegdep(&jfreeblk->jf_list); 3416 jfreeblk->jf_state = ATTACHED | DEPCOMPLETE; 3417 jfreeblk->jf_ino = freeblks->fb_previousinum; 3418 jfreeblk->jf_lbn = lbn; 3419 jfreeblk->jf_blkno = blkno; 3420 jfreeblk->jf_frags = frags; 3421 jfreeblk->jf_freeblks = freeblks; 3422 LIST_INSERT_HEAD(&freeblks->fb_jfreeblkhd, jfreeblk, jf_deps); 3423 3424 return (jfreeblk); 3425 } 3426 3427 static void move_newblock_dep(struct jaddref *, struct inodedep *); 3428 /* 3429 * If we're canceling a new bitmap we have to search for another ref 3430 * to move into the bmsafemap dep. This might be better expressed 3431 * with another structure. 3432 */ 3433 static void 3434 move_newblock_dep(jaddref, inodedep) 3435 struct jaddref *jaddref; 3436 struct inodedep *inodedep; 3437 { 3438 struct inoref *inoref; 3439 struct jaddref *jaddrefn; 3440 3441 jaddrefn = NULL; 3442 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 3443 inoref = TAILQ_NEXT(inoref, if_deps)) { 3444 if ((jaddref->ja_state & NEWBLOCK) && 3445 inoref->if_list.wk_type == D_JADDREF) { 3446 jaddrefn = (struct jaddref *)inoref; 3447 break; 3448 } 3449 } 3450 if (jaddrefn == NULL) 3451 return; 3452 jaddrefn->ja_state &= ~(ATTACHED | UNDONE); 3453 jaddrefn->ja_state |= jaddref->ja_state & 3454 (ATTACHED | UNDONE | NEWBLOCK); 3455 jaddref->ja_state &= ~(ATTACHED | UNDONE | NEWBLOCK); 3456 jaddref->ja_state |= ATTACHED; 3457 LIST_REMOVE(jaddref, ja_bmdeps); 3458 LIST_INSERT_HEAD(&inodedep->id_bmsafemap->sm_jaddrefhd, jaddrefn, 3459 ja_bmdeps); 3460 } 3461 3462 /* 3463 * Cancel a jaddref either before it has been written or while it is being 3464 * written. This happens when a link is removed before the add reaches 3465 * the disk. The jaddref dependency is kept linked into the bmsafemap 3466 * and inode to prevent the link count or bitmap from reaching the disk 3467 * until handle_workitem_remove() re-adjusts the counts and bitmaps as 3468 * required. 3469 * 3470 * Returns 1 if the canceled addref requires journaling of the remove and 3471 * 0 otherwise. 3472 */ 3473 static int 3474 cancel_jaddref(jaddref, inodedep, wkhd) 3475 struct jaddref *jaddref; 3476 struct inodedep *inodedep; 3477 struct workhead *wkhd; 3478 { 3479 struct inoref *inoref; 3480 struct jsegdep *jsegdep; 3481 int needsj; 3482 3483 KASSERT((jaddref->ja_state & COMPLETE) == 0, 3484 ("cancel_jaddref: Canceling complete jaddref")); 3485 if (jaddref->ja_state & (IOSTARTED | COMPLETE)) 3486 needsj = 1; 3487 else 3488 needsj = 0; 3489 if (inodedep == NULL) 3490 if (inodedep_lookup(jaddref->ja_list.wk_mp, jaddref->ja_ino, 3491 0, &inodedep) == 0) 3492 panic("cancel_jaddref: Lost inodedep"); 3493 /* 3494 * We must adjust the nlink of any reference operation that follows 3495 * us so that it is consistent with the in-memory reference. This 3496 * ensures that inode nlink rollbacks always have the correct link. 3497 */ 3498 if (needsj == 0) 3499 for (inoref = TAILQ_NEXT(&jaddref->ja_ref, if_deps); inoref; 3500 inoref = TAILQ_NEXT(inoref, if_deps)) 3501 inoref->if_nlink--; 3502 jsegdep = inoref_jseg(&jaddref->ja_ref); 3503 if (jaddref->ja_state & NEWBLOCK) 3504 move_newblock_dep(jaddref, inodedep); 3505 if (jaddref->ja_state & IOWAITING) { 3506 jaddref->ja_state &= ~IOWAITING; 3507 wakeup(&jaddref->ja_list); 3508 } 3509 jaddref->ja_mkdir = NULL; 3510 if (jaddref->ja_state & IOSTARTED) { 3511 jaddref->ja_state &= ~IOSTARTED; 3512 WORKLIST_REMOVE(&jaddref->ja_list); 3513 WORKLIST_INSERT(wkhd, &jsegdep->jd_list); 3514 } else { 3515 free_jsegdep(jsegdep); 3516 if (jaddref->ja_state & DEPCOMPLETE) 3517 remove_from_journal(&jaddref->ja_list); 3518 } 3519 /* 3520 * Leave NEWBLOCK jaddrefs on the inodedep so handle_workitem_remove 3521 * can arrange for them to be freed with the bitmap. Otherwise we 3522 * no longer need this addref attached to the inoreflst and it 3523 * will incorrectly adjust nlink if we leave it. 3524 */ 3525 if ((jaddref->ja_state & NEWBLOCK) == 0) { 3526 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 3527 if_deps); 3528 jaddref->ja_state |= COMPLETE; 3529 free_jaddref(jaddref); 3530 return (needsj); 3531 } 3532 jaddref->ja_state |= GOINGAWAY; 3533 /* 3534 * Leave the head of the list for jsegdeps for fast merging. 3535 */ 3536 if (LIST_FIRST(wkhd) != NULL) { 3537 jaddref->ja_state |= ONWORKLIST; 3538 LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jaddref->ja_list, wk_list); 3539 } else 3540 WORKLIST_INSERT(wkhd, &jaddref->ja_list); 3541 3542 return (needsj); 3543 } 3544 3545 /* 3546 * Attempt to free a jaddref structure when some work completes. This 3547 * should only succeed once the entry is written and all dependencies have 3548 * been notified. 3549 */ 3550 static void 3551 free_jaddref(jaddref) 3552 struct jaddref *jaddref; 3553 { 3554 3555 if ((jaddref->ja_state & ALLCOMPLETE) != ALLCOMPLETE) 3556 return; 3557 if (jaddref->ja_ref.if_jsegdep) 3558 panic("free_jaddref: segdep attached to jaddref %p(0x%X)\n", 3559 jaddref, jaddref->ja_state); 3560 if (jaddref->ja_state & NEWBLOCK) 3561 LIST_REMOVE(jaddref, ja_bmdeps); 3562 if (jaddref->ja_state & (IOSTARTED | ONWORKLIST)) 3563 panic("free_jaddref: Bad state %p(0x%X)", 3564 jaddref, jaddref->ja_state); 3565 if (jaddref->ja_mkdir != NULL) 3566 panic("free_jaddref: Work pending, 0x%X\n", jaddref->ja_state); 3567 WORKITEM_FREE(jaddref, D_JADDREF); 3568 } 3569 3570 /* 3571 * Free a jremref structure once it has been written or discarded. 3572 */ 3573 static void 3574 free_jremref(jremref) 3575 struct jremref *jremref; 3576 { 3577 3578 if (jremref->jr_ref.if_jsegdep) 3579 free_jsegdep(jremref->jr_ref.if_jsegdep); 3580 if (jremref->jr_state & IOSTARTED) 3581 panic("free_jremref: IO still pending"); 3582 WORKITEM_FREE(jremref, D_JREMREF); 3583 } 3584 3585 /* 3586 * Free a jnewblk structure. 3587 */ 3588 static void 3589 free_jnewblk(jnewblk) 3590 struct jnewblk *jnewblk; 3591 { 3592 3593 if ((jnewblk->jn_state & ALLCOMPLETE) != ALLCOMPLETE) 3594 return; 3595 LIST_REMOVE(jnewblk, jn_deps); 3596 if (jnewblk->jn_newblk != NULL) 3597 panic("free_jnewblk: Dependency still attached."); 3598 WORKITEM_FREE(jnewblk, D_JNEWBLK); 3599 } 3600 3601 /* 3602 * Cancel a jnewblk which has been superseded by a freeblk. The jnewblk 3603 * is kept linked into the bmsafemap until the free completes, thus 3604 * preventing the modified state from ever reaching disk. The free 3605 * routine must pass this structure via ffs_blkfree() to 3606 * softdep_setup_freeblks() so there is no race in releasing the space. 3607 */ 3608 static void 3609 cancel_jnewblk(jnewblk, wkhd) 3610 struct jnewblk *jnewblk; 3611 struct workhead *wkhd; 3612 { 3613 struct jsegdep *jsegdep; 3614 3615 jsegdep = jnewblk->jn_jsegdep; 3616 jnewblk->jn_jsegdep = NULL; 3617 free_jsegdep(jsegdep); 3618 jnewblk->jn_newblk = NULL; 3619 jnewblk->jn_state |= GOINGAWAY; 3620 if (jnewblk->jn_state & IOSTARTED) { 3621 jnewblk->jn_state &= ~IOSTARTED; 3622 WORKLIST_REMOVE(&jnewblk->jn_list); 3623 } else 3624 remove_from_journal(&jnewblk->jn_list); 3625 /* 3626 * Leave the head of the list for jsegdeps for fast merging. 3627 */ 3628 if (LIST_FIRST(wkhd) != NULL) { 3629 jnewblk->jn_state |= ONWORKLIST; 3630 LIST_INSERT_AFTER(LIST_FIRST(wkhd), &jnewblk->jn_list, wk_list); 3631 } else 3632 WORKLIST_INSERT(wkhd, &jnewblk->jn_list); 3633 if (jnewblk->jn_state & IOWAITING) { 3634 jnewblk->jn_state &= ~IOWAITING; 3635 wakeup(&jnewblk->jn_list); 3636 } 3637 } 3638 3639 static void 3640 free_jfreeblk(jfreeblk) 3641 struct jfreeblk *jfreeblk; 3642 { 3643 3644 WORKITEM_FREE(jfreeblk, D_JFREEBLK); 3645 } 3646 3647 /* 3648 * Release one reference to a jseg and free it if the count reaches 0. This 3649 * should eventually reclaim journal space as well. 3650 */ 3651 static void 3652 free_jseg(jseg) 3653 struct jseg *jseg; 3654 { 3655 struct jblocks *jblocks; 3656 3657 KASSERT(jseg->js_refs > 0, 3658 ("free_jseg: Invalid refcnt %d", jseg->js_refs)); 3659 if (--jseg->js_refs != 0) 3660 return; 3661 /* 3662 * Free only those jsegs which have none allocated before them to 3663 * preserve the journal space ordering. 3664 */ 3665 jblocks = jseg->js_jblocks; 3666 while ((jseg = TAILQ_FIRST(&jblocks->jb_segs)) != NULL) { 3667 jblocks->jb_oldestseq = jseg->js_seq; 3668 if (jseg->js_refs != 0) 3669 break; 3670 TAILQ_REMOVE(&jblocks->jb_segs, jseg, js_next); 3671 jblocks_free(jblocks, jseg->js_list.wk_mp, jseg->js_size); 3672 KASSERT(LIST_EMPTY(&jseg->js_entries), 3673 ("free_jseg: Freed jseg has valid entries.")); 3674 WORKITEM_FREE(jseg, D_JSEG); 3675 } 3676 } 3677 3678 /* 3679 * Release a jsegdep and decrement the jseg count. 3680 */ 3681 static void 3682 free_jsegdep(jsegdep) 3683 struct jsegdep *jsegdep; 3684 { 3685 3686 if (jsegdep->jd_seg) 3687 free_jseg(jsegdep->jd_seg); 3688 WORKITEM_FREE(jsegdep, D_JSEGDEP); 3689 } 3690 3691 /* 3692 * Wait for a journal item to make it to disk. Initiate journal processing 3693 * if required. 3694 */ 3695 static void 3696 jwait(wk) 3697 struct worklist *wk; 3698 { 3699 3700 stat_journal_wait++; 3701 /* 3702 * If IO has not started we process the journal. We can't mark the 3703 * worklist item as IOWAITING because we drop the lock while 3704 * processing the journal and the worklist entry may be freed after 3705 * this point. The caller may call back in and re-issue the request. 3706 */ 3707 if ((wk->wk_state & IOSTARTED) == 0) { 3708 softdep_process_journal(wk->wk_mp, MNT_WAIT); 3709 return; 3710 } 3711 wk->wk_state |= IOWAITING; 3712 msleep(wk, &lk, PRIBIO, "jwait", 0); 3713 } 3714 3715 /* 3716 * Lookup an inodedep based on an inode pointer and set the nlinkdelta as 3717 * appropriate. This is a convenience function to reduce duplicate code 3718 * for the setup and revert functions below. 3719 */ 3720 static struct inodedep * 3721 inodedep_lookup_ip(ip) 3722 struct inode *ip; 3723 { 3724 struct inodedep *inodedep; 3725 3726 KASSERT(ip->i_nlink >= ip->i_effnlink, 3727 ("inodedep_lookup_ip: bad delta")); 3728 (void) inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 3729 DEPALLOC, &inodedep); 3730 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 3731 3732 return (inodedep); 3733 } 3734 3735 /* 3736 * Create a journal entry that describes a truncate that we're about to 3737 * perform. The inode allocations and frees between here and the completion 3738 * of the operation are done asynchronously and without journaling. At 3739 * the end of the operation the vnode is sync'd and the journal space 3740 * is released. Recovery will discover the partially completed truncate 3741 * and complete it. 3742 */ 3743 void * 3744 softdep_setup_trunc(vp, length, flags) 3745 struct vnode *vp; 3746 off_t length; 3747 int flags; 3748 { 3749 struct jsegdep *jsegdep; 3750 struct jtrunc *jtrunc; 3751 struct ufsmount *ump; 3752 struct inode *ip; 3753 3754 softdep_prealloc(vp, MNT_WAIT); 3755 ip = VTOI(vp); 3756 ump = VFSTOUFS(vp->v_mount); 3757 jtrunc = malloc(sizeof(*jtrunc), M_JTRUNC, M_SOFTDEP_FLAGS); 3758 workitem_alloc(&jtrunc->jt_list, D_JTRUNC, vp->v_mount); 3759 jsegdep = jtrunc->jt_jsegdep = newjsegdep(&jtrunc->jt_list); 3760 jtrunc->jt_ino = ip->i_number; 3761 jtrunc->jt_extsize = 0; 3762 jtrunc->jt_size = length; 3763 if ((flags & IO_EXT) == 0 && ump->um_fstype == UFS2) 3764 jtrunc->jt_extsize = ip->i_din2->di_extsize; 3765 if ((flags & IO_NORMAL) == 0) 3766 jtrunc->jt_size = DIP(ip, i_size); 3767 ACQUIRE_LOCK(&lk); 3768 add_to_journal(&jtrunc->jt_list); 3769 while (jsegdep->jd_seg == NULL) { 3770 stat_jwait_freeblks++; 3771 jwait(&jtrunc->jt_list); 3772 } 3773 FREE_LOCK(&lk); 3774 3775 return (jsegdep); 3776 } 3777 3778 /* 3779 * After synchronous truncation is complete we free sync the vnode and 3780 * release the jsegdep so the journal space can be freed. 3781 */ 3782 int 3783 softdep_complete_trunc(vp, cookie) 3784 struct vnode *vp; 3785 void *cookie; 3786 { 3787 int error; 3788 3789 error = ffs_syncvnode(vp, MNT_WAIT); 3790 ACQUIRE_LOCK(&lk); 3791 free_jsegdep((struct jsegdep *)cookie); 3792 FREE_LOCK(&lk); 3793 3794 return (error); 3795 } 3796 3797 /* 3798 * Called prior to creating a new inode and linking it to a directory. The 3799 * jaddref structure must already be allocated by softdep_setup_inomapdep 3800 * and it is discovered here so we can initialize the mode and update 3801 * nlinkdelta. 3802 */ 3803 void 3804 softdep_setup_create(dp, ip) 3805 struct inode *dp; 3806 struct inode *ip; 3807 { 3808 struct inodedep *inodedep; 3809 struct jaddref *jaddref; 3810 struct vnode *dvp; 3811 3812 KASSERT(ip->i_nlink == 1, 3813 ("softdep_setup_create: Invalid link count.")); 3814 dvp = ITOV(dp); 3815 ACQUIRE_LOCK(&lk); 3816 inodedep = inodedep_lookup_ip(ip); 3817 if (DOINGSUJ(dvp)) { 3818 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 3819 inoreflst); 3820 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 3821 ("softdep_setup_create: No addref structure present.")); 3822 jaddref->ja_mode = ip->i_mode; 3823 } 3824 softdep_prelink(dvp, NULL); 3825 FREE_LOCK(&lk); 3826 } 3827 3828 /* 3829 * Create a jaddref structure to track the addition of a DOTDOT link when 3830 * we are reparenting an inode as part of a rename. This jaddref will be 3831 * found by softdep_setup_directory_change. Adjusts nlinkdelta for 3832 * non-journaling softdep. 3833 */ 3834 void 3835 softdep_setup_dotdot_link(dp, ip) 3836 struct inode *dp; 3837 struct inode *ip; 3838 { 3839 struct inodedep *inodedep; 3840 struct jaddref *jaddref; 3841 struct vnode *dvp; 3842 struct vnode *vp; 3843 3844 dvp = ITOV(dp); 3845 vp = ITOV(ip); 3846 jaddref = NULL; 3847 /* 3848 * We don't set MKDIR_PARENT as this is not tied to a mkdir and 3849 * is used as a normal link would be. 3850 */ 3851 if (DOINGSUJ(dvp)) 3852 jaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 3853 dp->i_effnlink - 1, dp->i_mode); 3854 ACQUIRE_LOCK(&lk); 3855 inodedep = inodedep_lookup_ip(dp); 3856 if (jaddref) 3857 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 3858 if_deps); 3859 softdep_prelink(dvp, ITOV(ip)); 3860 FREE_LOCK(&lk); 3861 } 3862 3863 /* 3864 * Create a jaddref structure to track a new link to an inode. The directory 3865 * offset is not known until softdep_setup_directory_add or 3866 * softdep_setup_directory_change. Adjusts nlinkdelta for non-journaling 3867 * softdep. 3868 */ 3869 void 3870 softdep_setup_link(dp, ip) 3871 struct inode *dp; 3872 struct inode *ip; 3873 { 3874 struct inodedep *inodedep; 3875 struct jaddref *jaddref; 3876 struct vnode *dvp; 3877 3878 dvp = ITOV(dp); 3879 jaddref = NULL; 3880 if (DOINGSUJ(dvp)) 3881 jaddref = newjaddref(dp, ip->i_number, 0, ip->i_effnlink - 1, 3882 ip->i_mode); 3883 ACQUIRE_LOCK(&lk); 3884 inodedep = inodedep_lookup_ip(ip); 3885 if (jaddref) 3886 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 3887 if_deps); 3888 softdep_prelink(dvp, ITOV(ip)); 3889 FREE_LOCK(&lk); 3890 } 3891 3892 /* 3893 * Called to create the jaddref structures to track . and .. references as 3894 * well as lookup and further initialize the incomplete jaddref created 3895 * by softdep_setup_inomapdep when the inode was allocated. Adjusts 3896 * nlinkdelta for non-journaling softdep. 3897 */ 3898 void 3899 softdep_setup_mkdir(dp, ip) 3900 struct inode *dp; 3901 struct inode *ip; 3902 { 3903 struct inodedep *inodedep; 3904 struct jaddref *dotdotaddref; 3905 struct jaddref *dotaddref; 3906 struct jaddref *jaddref; 3907 struct vnode *dvp; 3908 3909 dvp = ITOV(dp); 3910 dotaddref = dotdotaddref = NULL; 3911 if (DOINGSUJ(dvp)) { 3912 dotaddref = newjaddref(ip, ip->i_number, DOT_OFFSET, 1, 3913 ip->i_mode); 3914 dotaddref->ja_state |= MKDIR_BODY; 3915 dotdotaddref = newjaddref(ip, dp->i_number, DOTDOT_OFFSET, 3916 dp->i_effnlink - 1, dp->i_mode); 3917 dotdotaddref->ja_state |= MKDIR_PARENT; 3918 } 3919 ACQUIRE_LOCK(&lk); 3920 inodedep = inodedep_lookup_ip(ip); 3921 if (DOINGSUJ(dvp)) { 3922 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 3923 inoreflst); 3924 KASSERT(jaddref != NULL, 3925 ("softdep_setup_mkdir: No addref structure present.")); 3926 KASSERT(jaddref->ja_parent == dp->i_number, 3927 ("softdep_setup_mkdir: bad parent %d", 3928 jaddref->ja_parent)); 3929 jaddref->ja_mode = ip->i_mode; 3930 TAILQ_INSERT_BEFORE(&jaddref->ja_ref, &dotaddref->ja_ref, 3931 if_deps); 3932 } 3933 inodedep = inodedep_lookup_ip(dp); 3934 if (DOINGSUJ(dvp)) 3935 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, 3936 &dotdotaddref->ja_ref, if_deps); 3937 softdep_prelink(ITOV(dp), NULL); 3938 FREE_LOCK(&lk); 3939 } 3940 3941 /* 3942 * Called to track nlinkdelta of the inode and parent directories prior to 3943 * unlinking a directory. 3944 */ 3945 void 3946 softdep_setup_rmdir(dp, ip) 3947 struct inode *dp; 3948 struct inode *ip; 3949 { 3950 struct vnode *dvp; 3951 3952 dvp = ITOV(dp); 3953 ACQUIRE_LOCK(&lk); 3954 (void) inodedep_lookup_ip(ip); 3955 (void) inodedep_lookup_ip(dp); 3956 softdep_prelink(dvp, ITOV(ip)); 3957 FREE_LOCK(&lk); 3958 } 3959 3960 /* 3961 * Called to track nlinkdelta of the inode and parent directories prior to 3962 * unlink. 3963 */ 3964 void 3965 softdep_setup_unlink(dp, ip) 3966 struct inode *dp; 3967 struct inode *ip; 3968 { 3969 struct vnode *dvp; 3970 3971 dvp = ITOV(dp); 3972 ACQUIRE_LOCK(&lk); 3973 (void) inodedep_lookup_ip(ip); 3974 (void) inodedep_lookup_ip(dp); 3975 softdep_prelink(dvp, ITOV(ip)); 3976 FREE_LOCK(&lk); 3977 } 3978 3979 /* 3980 * Called to release the journal structures created by a failed non-directory 3981 * creation. Adjusts nlinkdelta for non-journaling softdep. 3982 */ 3983 void 3984 softdep_revert_create(dp, ip) 3985 struct inode *dp; 3986 struct inode *ip; 3987 { 3988 struct inodedep *inodedep; 3989 struct jaddref *jaddref; 3990 struct vnode *dvp; 3991 3992 dvp = ITOV(dp); 3993 ACQUIRE_LOCK(&lk); 3994 inodedep = inodedep_lookup_ip(ip); 3995 if (DOINGSUJ(dvp)) { 3996 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 3997 inoreflst); 3998 KASSERT(jaddref->ja_parent == dp->i_number, 3999 ("softdep_revert_create: addref parent mismatch")); 4000 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4001 } 4002 FREE_LOCK(&lk); 4003 } 4004 4005 /* 4006 * Called to release the journal structures created by a failed dotdot link 4007 * creation. Adjusts nlinkdelta for non-journaling softdep. 4008 */ 4009 void 4010 softdep_revert_dotdot_link(dp, ip) 4011 struct inode *dp; 4012 struct inode *ip; 4013 { 4014 struct inodedep *inodedep; 4015 struct jaddref *jaddref; 4016 struct vnode *dvp; 4017 4018 dvp = ITOV(dp); 4019 ACQUIRE_LOCK(&lk); 4020 inodedep = inodedep_lookup_ip(dp); 4021 if (DOINGSUJ(dvp)) { 4022 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4023 inoreflst); 4024 KASSERT(jaddref->ja_parent == ip->i_number, 4025 ("softdep_revert_dotdot_link: addref parent mismatch")); 4026 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4027 } 4028 FREE_LOCK(&lk); 4029 } 4030 4031 /* 4032 * Called to release the journal structures created by a failed link 4033 * addition. Adjusts nlinkdelta for non-journaling softdep. 4034 */ 4035 void 4036 softdep_revert_link(dp, ip) 4037 struct inode *dp; 4038 struct inode *ip; 4039 { 4040 struct inodedep *inodedep; 4041 struct jaddref *jaddref; 4042 struct vnode *dvp; 4043 4044 dvp = ITOV(dp); 4045 ACQUIRE_LOCK(&lk); 4046 inodedep = inodedep_lookup_ip(ip); 4047 if (DOINGSUJ(dvp)) { 4048 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4049 inoreflst); 4050 KASSERT(jaddref->ja_parent == dp->i_number, 4051 ("softdep_revert_link: addref parent mismatch")); 4052 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4053 } 4054 FREE_LOCK(&lk); 4055 } 4056 4057 /* 4058 * Called to release the journal structures created by a failed mkdir 4059 * attempt. Adjusts nlinkdelta for non-journaling softdep. 4060 */ 4061 void 4062 softdep_revert_mkdir(dp, ip) 4063 struct inode *dp; 4064 struct inode *ip; 4065 { 4066 struct inodedep *inodedep; 4067 struct jaddref *jaddref; 4068 struct vnode *dvp; 4069 4070 dvp = ITOV(dp); 4071 4072 ACQUIRE_LOCK(&lk); 4073 inodedep = inodedep_lookup_ip(dp); 4074 if (DOINGSUJ(dvp)) { 4075 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4076 inoreflst); 4077 KASSERT(jaddref->ja_parent == ip->i_number, 4078 ("softdep_revert_mkdir: dotdot addref parent mismatch")); 4079 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4080 } 4081 inodedep = inodedep_lookup_ip(ip); 4082 if (DOINGSUJ(dvp)) { 4083 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4084 inoreflst); 4085 KASSERT(jaddref->ja_parent == dp->i_number, 4086 ("softdep_revert_mkdir: addref parent mismatch")); 4087 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4088 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 4089 inoreflst); 4090 KASSERT(jaddref->ja_parent == ip->i_number, 4091 ("softdep_revert_mkdir: dot addref parent mismatch")); 4092 cancel_jaddref(jaddref, inodedep, &inodedep->id_inowait); 4093 } 4094 FREE_LOCK(&lk); 4095 } 4096 4097 /* 4098 * Called to correct nlinkdelta after a failed rmdir. 4099 */ 4100 void 4101 softdep_revert_rmdir(dp, ip) 4102 struct inode *dp; 4103 struct inode *ip; 4104 { 4105 4106 ACQUIRE_LOCK(&lk); 4107 (void) inodedep_lookup_ip(ip); 4108 (void) inodedep_lookup_ip(dp); 4109 FREE_LOCK(&lk); 4110 } 4111 4112 /* 4113 * Protecting the freemaps (or bitmaps). 4114 * 4115 * To eliminate the need to execute fsck before mounting a filesystem 4116 * after a power failure, one must (conservatively) guarantee that the 4117 * on-disk copy of the bitmaps never indicate that a live inode or block is 4118 * free. So, when a block or inode is allocated, the bitmap should be 4119 * updated (on disk) before any new pointers. When a block or inode is 4120 * freed, the bitmap should not be updated until all pointers have been 4121 * reset. The latter dependency is handled by the delayed de-allocation 4122 * approach described below for block and inode de-allocation. The former 4123 * dependency is handled by calling the following procedure when a block or 4124 * inode is allocated. When an inode is allocated an "inodedep" is created 4125 * with its DEPCOMPLETE flag cleared until its bitmap is written to disk. 4126 * Each "inodedep" is also inserted into the hash indexing structure so 4127 * that any additional link additions can be made dependent on the inode 4128 * allocation. 4129 * 4130 * The ufs filesystem maintains a number of free block counts (e.g., per 4131 * cylinder group, per cylinder and per <cylinder, rotational position> pair) 4132 * in addition to the bitmaps. These counts are used to improve efficiency 4133 * during allocation and therefore must be consistent with the bitmaps. 4134 * There is no convenient way to guarantee post-crash consistency of these 4135 * counts with simple update ordering, for two main reasons: (1) The counts 4136 * and bitmaps for a single cylinder group block are not in the same disk 4137 * sector. If a disk write is interrupted (e.g., by power failure), one may 4138 * be written and the other not. (2) Some of the counts are located in the 4139 * superblock rather than the cylinder group block. So, we focus our soft 4140 * updates implementation on protecting the bitmaps. When mounting a 4141 * filesystem, we recompute the auxiliary counts from the bitmaps. 4142 */ 4143 4144 /* 4145 * Called just after updating the cylinder group block to allocate an inode. 4146 */ 4147 void 4148 softdep_setup_inomapdep(bp, ip, newinum) 4149 struct buf *bp; /* buffer for cylgroup block with inode map */ 4150 struct inode *ip; /* inode related to allocation */ 4151 ino_t newinum; /* new inode number being allocated */ 4152 { 4153 struct inodedep *inodedep; 4154 struct bmsafemap *bmsafemap; 4155 struct jaddref *jaddref; 4156 struct mount *mp; 4157 struct fs *fs; 4158 4159 mp = UFSTOVFS(ip->i_ump); 4160 fs = ip->i_ump->um_fs; 4161 jaddref = NULL; 4162 4163 /* 4164 * Allocate the journal reference add structure so that the bitmap 4165 * can be dependent on it. 4166 */ 4167 if (mp->mnt_kern_flag & MNTK_SUJ) { 4168 jaddref = newjaddref(ip, newinum, 0, 0, 0); 4169 jaddref->ja_state |= NEWBLOCK; 4170 } 4171 4172 /* 4173 * Create a dependency for the newly allocated inode. 4174 * Panic if it already exists as something is seriously wrong. 4175 * Otherwise add it to the dependency list for the buffer holding 4176 * the cylinder group map from which it was allocated. 4177 */ 4178 ACQUIRE_LOCK(&lk); 4179 if ((inodedep_lookup(mp, newinum, DEPALLOC|NODELAY, &inodedep))) 4180 panic("softdep_setup_inomapdep: dependency %p for new" 4181 "inode already exists", inodedep); 4182 bmsafemap = bmsafemap_lookup(mp, bp, ino_to_cg(fs, newinum)); 4183 if (jaddref) { 4184 LIST_INSERT_HEAD(&bmsafemap->sm_jaddrefhd, jaddref, ja_bmdeps); 4185 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jaddref->ja_ref, 4186 if_deps); 4187 } else { 4188 inodedep->id_state |= ONDEPLIST; 4189 LIST_INSERT_HEAD(&bmsafemap->sm_inodedephd, inodedep, id_deps); 4190 } 4191 inodedep->id_bmsafemap = bmsafemap; 4192 inodedep->id_state &= ~DEPCOMPLETE; 4193 FREE_LOCK(&lk); 4194 } 4195 4196 /* 4197 * Called just after updating the cylinder group block to 4198 * allocate block or fragment. 4199 */ 4200 void 4201 softdep_setup_blkmapdep(bp, mp, newblkno, frags, oldfrags) 4202 struct buf *bp; /* buffer for cylgroup block with block map */ 4203 struct mount *mp; /* filesystem doing allocation */ 4204 ufs2_daddr_t newblkno; /* number of newly allocated block */ 4205 int frags; /* Number of fragments. */ 4206 int oldfrags; /* Previous number of fragments for extend. */ 4207 { 4208 struct newblk *newblk; 4209 struct bmsafemap *bmsafemap; 4210 struct jnewblk *jnewblk; 4211 struct fs *fs; 4212 4213 fs = VFSTOUFS(mp)->um_fs; 4214 jnewblk = NULL; 4215 /* 4216 * Create a dependency for the newly allocated block. 4217 * Add it to the dependency list for the buffer holding 4218 * the cylinder group map from which it was allocated. 4219 */ 4220 if (mp->mnt_kern_flag & MNTK_SUJ) { 4221 jnewblk = malloc(sizeof(*jnewblk), M_JNEWBLK, M_SOFTDEP_FLAGS); 4222 workitem_alloc(&jnewblk->jn_list, D_JNEWBLK, mp); 4223 jnewblk->jn_jsegdep = newjsegdep(&jnewblk->jn_list); 4224 jnewblk->jn_state = ATTACHED; 4225 jnewblk->jn_blkno = newblkno; 4226 jnewblk->jn_frags = frags; 4227 jnewblk->jn_oldfrags = oldfrags; 4228 #ifdef SUJ_DEBUG 4229 { 4230 struct cg *cgp; 4231 uint8_t *blksfree; 4232 long bno; 4233 int i; 4234 4235 cgp = (struct cg *)bp->b_data; 4236 blksfree = cg_blksfree(cgp); 4237 bno = dtogd(fs, jnewblk->jn_blkno); 4238 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 4239 i++) { 4240 if (isset(blksfree, bno + i)) 4241 panic("softdep_setup_blkmapdep: " 4242 "free fragment %d from %d-%d " 4243 "state 0x%X dep %p", i, 4244 jnewblk->jn_oldfrags, 4245 jnewblk->jn_frags, 4246 jnewblk->jn_state, 4247 jnewblk->jn_newblk); 4248 } 4249 } 4250 #endif 4251 } 4252 ACQUIRE_LOCK(&lk); 4253 if (newblk_lookup(mp, newblkno, DEPALLOC, &newblk) != 0) 4254 panic("softdep_setup_blkmapdep: found block"); 4255 newblk->nb_bmsafemap = bmsafemap = bmsafemap_lookup(mp, bp, 4256 dtog(fs, newblkno)); 4257 if (jnewblk) { 4258 jnewblk->jn_newblk = newblk; 4259 LIST_INSERT_HEAD(&bmsafemap->sm_jnewblkhd, jnewblk, jn_deps); 4260 } else { 4261 newblk->nb_state |= ONDEPLIST; 4262 LIST_INSERT_HEAD(&bmsafemap->sm_newblkhd, newblk, nb_deps); 4263 } 4264 newblk->nb_bmsafemap = bmsafemap; 4265 newblk->nb_jnewblk = jnewblk; 4266 FREE_LOCK(&lk); 4267 } 4268 4269 #define BMSAFEMAP_HASH(fs, cg) \ 4270 (&bmsafemap_hashtbl[((((register_t)(fs)) >> 13) + (cg)) & bmsafemap_hash]) 4271 4272 static int 4273 bmsafemap_find(bmsafemaphd, mp, cg, bmsafemapp) 4274 struct bmsafemap_hashhead *bmsafemaphd; 4275 struct mount *mp; 4276 int cg; 4277 struct bmsafemap **bmsafemapp; 4278 { 4279 struct bmsafemap *bmsafemap; 4280 4281 LIST_FOREACH(bmsafemap, bmsafemaphd, sm_hash) 4282 if (bmsafemap->sm_list.wk_mp == mp && bmsafemap->sm_cg == cg) 4283 break; 4284 if (bmsafemap) { 4285 *bmsafemapp = bmsafemap; 4286 return (1); 4287 } 4288 *bmsafemapp = NULL; 4289 4290 return (0); 4291 } 4292 4293 /* 4294 * Find the bmsafemap associated with a cylinder group buffer. 4295 * If none exists, create one. The buffer must be locked when 4296 * this routine is called and this routine must be called with 4297 * splbio interrupts blocked. 4298 */ 4299 static struct bmsafemap * 4300 bmsafemap_lookup(mp, bp, cg) 4301 struct mount *mp; 4302 struct buf *bp; 4303 int cg; 4304 { 4305 struct bmsafemap_hashhead *bmsafemaphd; 4306 struct bmsafemap *bmsafemap, *collision; 4307 struct worklist *wk; 4308 struct fs *fs; 4309 4310 mtx_assert(&lk, MA_OWNED); 4311 if (bp) 4312 LIST_FOREACH(wk, &bp->b_dep, wk_list) 4313 if (wk->wk_type == D_BMSAFEMAP) 4314 return (WK_BMSAFEMAP(wk)); 4315 fs = VFSTOUFS(mp)->um_fs; 4316 bmsafemaphd = BMSAFEMAP_HASH(fs, cg); 4317 if (bmsafemap_find(bmsafemaphd, mp, cg, &bmsafemap) == 1) 4318 return (bmsafemap); 4319 FREE_LOCK(&lk); 4320 bmsafemap = malloc(sizeof(struct bmsafemap), 4321 M_BMSAFEMAP, M_SOFTDEP_FLAGS); 4322 workitem_alloc(&bmsafemap->sm_list, D_BMSAFEMAP, mp); 4323 bmsafemap->sm_buf = bp; 4324 LIST_INIT(&bmsafemap->sm_inodedephd); 4325 LIST_INIT(&bmsafemap->sm_inodedepwr); 4326 LIST_INIT(&bmsafemap->sm_newblkhd); 4327 LIST_INIT(&bmsafemap->sm_newblkwr); 4328 LIST_INIT(&bmsafemap->sm_jaddrefhd); 4329 LIST_INIT(&bmsafemap->sm_jnewblkhd); 4330 ACQUIRE_LOCK(&lk); 4331 if (bmsafemap_find(bmsafemaphd, mp, cg, &collision) == 1) { 4332 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 4333 return (collision); 4334 } 4335 bmsafemap->sm_cg = cg; 4336 LIST_INSERT_HEAD(bmsafemaphd, bmsafemap, sm_hash); 4337 WORKLIST_INSERT(&bp->b_dep, &bmsafemap->sm_list); 4338 return (bmsafemap); 4339 } 4340 4341 /* 4342 * Direct block allocation dependencies. 4343 * 4344 * When a new block is allocated, the corresponding disk locations must be 4345 * initialized (with zeros or new data) before the on-disk inode points to 4346 * them. Also, the freemap from which the block was allocated must be 4347 * updated (on disk) before the inode's pointer. These two dependencies are 4348 * independent of each other and are needed for all file blocks and indirect 4349 * blocks that are pointed to directly by the inode. Just before the 4350 * "in-core" version of the inode is updated with a newly allocated block 4351 * number, a procedure (below) is called to setup allocation dependency 4352 * structures. These structures are removed when the corresponding 4353 * dependencies are satisfied or when the block allocation becomes obsolete 4354 * (i.e., the file is deleted, the block is de-allocated, or the block is a 4355 * fragment that gets upgraded). All of these cases are handled in 4356 * procedures described later. 4357 * 4358 * When a file extension causes a fragment to be upgraded, either to a larger 4359 * fragment or to a full block, the on-disk location may change (if the 4360 * previous fragment could not simply be extended). In this case, the old 4361 * fragment must be de-allocated, but not until after the inode's pointer has 4362 * been updated. In most cases, this is handled by later procedures, which 4363 * will construct a "freefrag" structure to be added to the workitem queue 4364 * when the inode update is complete (or obsolete). The main exception to 4365 * this is when an allocation occurs while a pending allocation dependency 4366 * (for the same block pointer) remains. This case is handled in the main 4367 * allocation dependency setup procedure by immediately freeing the 4368 * unreferenced fragments. 4369 */ 4370 void 4371 softdep_setup_allocdirect(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 4372 struct inode *ip; /* inode to which block is being added */ 4373 ufs_lbn_t off; /* block pointer within inode */ 4374 ufs2_daddr_t newblkno; /* disk block number being added */ 4375 ufs2_daddr_t oldblkno; /* previous block number, 0 unless frag */ 4376 long newsize; /* size of new block */ 4377 long oldsize; /* size of new block */ 4378 struct buf *bp; /* bp for allocated block */ 4379 { 4380 struct allocdirect *adp, *oldadp; 4381 struct allocdirectlst *adphead; 4382 struct freefrag *freefrag; 4383 struct inodedep *inodedep; 4384 struct pagedep *pagedep; 4385 struct jnewblk *jnewblk; 4386 struct newblk *newblk; 4387 struct mount *mp; 4388 ufs_lbn_t lbn; 4389 4390 lbn = bp->b_lblkno; 4391 mp = UFSTOVFS(ip->i_ump); 4392 if (oldblkno && oldblkno != newblkno) 4393 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn); 4394 else 4395 freefrag = NULL; 4396 4397 ACQUIRE_LOCK(&lk); 4398 if (off >= NDADDR) { 4399 if (lbn > 0) 4400 panic("softdep_setup_allocdirect: bad lbn %jd, off %jd", 4401 lbn, off); 4402 /* allocating an indirect block */ 4403 if (oldblkno != 0) 4404 panic("softdep_setup_allocdirect: non-zero indir"); 4405 } else { 4406 if (off != lbn) 4407 panic("softdep_setup_allocdirect: lbn %jd != off %jd", 4408 lbn, off); 4409 /* 4410 * Allocating a direct block. 4411 * 4412 * If we are allocating a directory block, then we must 4413 * allocate an associated pagedep to track additions and 4414 * deletions. 4415 */ 4416 if ((ip->i_mode & IFMT) == IFDIR && 4417 pagedep_lookup(mp, ip->i_number, off, DEPALLOC, 4418 &pagedep) == 0) 4419 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 4420 } 4421 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 4422 panic("softdep_setup_allocdirect: lost block"); 4423 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 4424 ("softdep_setup_allocdirect: newblk already initialized")); 4425 /* 4426 * Convert the newblk to an allocdirect. 4427 */ 4428 newblk->nb_list.wk_type = D_ALLOCDIRECT; 4429 adp = (struct allocdirect *)newblk; 4430 newblk->nb_freefrag = freefrag; 4431 adp->ad_offset = off; 4432 adp->ad_oldblkno = oldblkno; 4433 adp->ad_newsize = newsize; 4434 adp->ad_oldsize = oldsize; 4435 4436 /* 4437 * Finish initializing the journal. 4438 */ 4439 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 4440 jnewblk->jn_ino = ip->i_number; 4441 jnewblk->jn_lbn = lbn; 4442 add_to_journal(&jnewblk->jn_list); 4443 } 4444 if (freefrag && freefrag->ff_jfreefrag != NULL) 4445 add_to_journal(&freefrag->ff_jfreefrag->fr_list); 4446 inodedep_lookup(mp, ip->i_number, DEPALLOC | NODELAY, &inodedep); 4447 adp->ad_inodedep = inodedep; 4448 4449 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 4450 /* 4451 * The list of allocdirects must be kept in sorted and ascending 4452 * order so that the rollback routines can quickly determine the 4453 * first uncommitted block (the size of the file stored on disk 4454 * ends at the end of the lowest committed fragment, or if there 4455 * are no fragments, at the end of the highest committed block). 4456 * Since files generally grow, the typical case is that the new 4457 * block is to be added at the end of the list. We speed this 4458 * special case by checking against the last allocdirect in the 4459 * list before laboriously traversing the list looking for the 4460 * insertion point. 4461 */ 4462 adphead = &inodedep->id_newinoupdt; 4463 oldadp = TAILQ_LAST(adphead, allocdirectlst); 4464 if (oldadp == NULL || oldadp->ad_offset <= off) { 4465 /* insert at end of list */ 4466 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 4467 if (oldadp != NULL && oldadp->ad_offset == off) 4468 allocdirect_merge(adphead, adp, oldadp); 4469 FREE_LOCK(&lk); 4470 return; 4471 } 4472 TAILQ_FOREACH(oldadp, adphead, ad_next) { 4473 if (oldadp->ad_offset >= off) 4474 break; 4475 } 4476 if (oldadp == NULL) 4477 panic("softdep_setup_allocdirect: lost entry"); 4478 /* insert in middle of list */ 4479 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 4480 if (oldadp->ad_offset == off) 4481 allocdirect_merge(adphead, adp, oldadp); 4482 4483 FREE_LOCK(&lk); 4484 } 4485 4486 /* 4487 * Replace an old allocdirect dependency with a newer one. 4488 * This routine must be called with splbio interrupts blocked. 4489 */ 4490 static void 4491 allocdirect_merge(adphead, newadp, oldadp) 4492 struct allocdirectlst *adphead; /* head of list holding allocdirects */ 4493 struct allocdirect *newadp; /* allocdirect being added */ 4494 struct allocdirect *oldadp; /* existing allocdirect being checked */ 4495 { 4496 struct worklist *wk; 4497 struct freefrag *freefrag; 4498 struct newdirblk *newdirblk; 4499 4500 freefrag = NULL; 4501 mtx_assert(&lk, MA_OWNED); 4502 if (newadp->ad_oldblkno != oldadp->ad_newblkno || 4503 newadp->ad_oldsize != oldadp->ad_newsize || 4504 newadp->ad_offset >= NDADDR) 4505 panic("%s %jd != new %jd || old size %ld != new %ld", 4506 "allocdirect_merge: old blkno", 4507 (intmax_t)newadp->ad_oldblkno, 4508 (intmax_t)oldadp->ad_newblkno, 4509 newadp->ad_oldsize, oldadp->ad_newsize); 4510 newadp->ad_oldblkno = oldadp->ad_oldblkno; 4511 newadp->ad_oldsize = oldadp->ad_oldsize; 4512 /* 4513 * If the old dependency had a fragment to free or had never 4514 * previously had a block allocated, then the new dependency 4515 * can immediately post its freefrag and adopt the old freefrag. 4516 * This action is done by swapping the freefrag dependencies. 4517 * The new dependency gains the old one's freefrag, and the 4518 * old one gets the new one and then immediately puts it on 4519 * the worklist when it is freed by free_newblk. It is 4520 * not possible to do this swap when the old dependency had a 4521 * non-zero size but no previous fragment to free. This condition 4522 * arises when the new block is an extension of the old block. 4523 * Here, the first part of the fragment allocated to the new 4524 * dependency is part of the block currently claimed on disk by 4525 * the old dependency, so cannot legitimately be freed until the 4526 * conditions for the new dependency are fulfilled. 4527 */ 4528 freefrag = newadp->ad_freefrag; 4529 if (oldadp->ad_freefrag != NULL || oldadp->ad_oldblkno == 0) { 4530 newadp->ad_freefrag = oldadp->ad_freefrag; 4531 oldadp->ad_freefrag = freefrag; 4532 } 4533 /* 4534 * If we are tracking a new directory-block allocation, 4535 * move it from the old allocdirect to the new allocdirect. 4536 */ 4537 if ((wk = LIST_FIRST(&oldadp->ad_newdirblk)) != NULL) { 4538 newdirblk = WK_NEWDIRBLK(wk); 4539 WORKLIST_REMOVE(&newdirblk->db_list); 4540 if (!LIST_EMPTY(&oldadp->ad_newdirblk)) 4541 panic("allocdirect_merge: extra newdirblk"); 4542 WORKLIST_INSERT(&newadp->ad_newdirblk, &newdirblk->db_list); 4543 } 4544 TAILQ_REMOVE(adphead, oldadp, ad_next); 4545 /* 4546 * We need to move any journal dependencies over to the freefrag 4547 * that releases this block if it exists. Otherwise we are 4548 * extending an existing block and we'll wait until that is 4549 * complete to release the journal space and extend the 4550 * new journal to cover this old space as well. 4551 */ 4552 if (freefrag == NULL) { 4553 struct jnewblk *jnewblk; 4554 struct jnewblk *njnewblk; 4555 4556 if (oldadp->ad_newblkno != newadp->ad_newblkno) 4557 panic("allocdirect_merge: %jd != %jd", 4558 oldadp->ad_newblkno, newadp->ad_newblkno); 4559 jnewblk = oldadp->ad_block.nb_jnewblk; 4560 cancel_newblk(&oldadp->ad_block, &newadp->ad_block.nb_jwork); 4561 /* 4562 * We have an unwritten jnewblk, we need to merge the 4563 * frag bits with our own. The newer adp's journal can not 4564 * be written prior to the old one so no need to check for 4565 * it here. 4566 */ 4567 if (jnewblk) { 4568 njnewblk = newadp->ad_block.nb_jnewblk; 4569 if (njnewblk == NULL) 4570 panic("allocdirect_merge: No jnewblk"); 4571 if (jnewblk->jn_state & UNDONE) { 4572 njnewblk->jn_state |= UNDONE | NEWBLOCK; 4573 njnewblk->jn_state &= ~ATTACHED; 4574 jnewblk->jn_state &= ~UNDONE; 4575 } 4576 njnewblk->jn_oldfrags = jnewblk->jn_oldfrags; 4577 WORKLIST_REMOVE(&jnewblk->jn_list); 4578 jnewblk->jn_state |= ATTACHED | COMPLETE; 4579 free_jnewblk(jnewblk); 4580 } 4581 } else { 4582 /* 4583 * We can skip journaling for this freefrag and just complete 4584 * any pending journal work for the allocdirect that is being 4585 * removed after the freefrag completes. 4586 */ 4587 if (freefrag->ff_jfreefrag) 4588 cancel_jfreefrag(freefrag->ff_jfreefrag); 4589 cancel_newblk(&oldadp->ad_block, &freefrag->ff_jwork); 4590 } 4591 free_newblk(&oldadp->ad_block); 4592 } 4593 4594 /* 4595 * Allocate a jfreefrag structure to journal a single block free. 4596 */ 4597 static struct jfreefrag * 4598 newjfreefrag(freefrag, ip, blkno, size, lbn) 4599 struct freefrag *freefrag; 4600 struct inode *ip; 4601 ufs2_daddr_t blkno; 4602 long size; 4603 ufs_lbn_t lbn; 4604 { 4605 struct jfreefrag *jfreefrag; 4606 struct fs *fs; 4607 4608 fs = ip->i_fs; 4609 jfreefrag = malloc(sizeof(struct jfreefrag), M_JFREEFRAG, 4610 M_SOFTDEP_FLAGS); 4611 workitem_alloc(&jfreefrag->fr_list, D_JFREEFRAG, UFSTOVFS(ip->i_ump)); 4612 jfreefrag->fr_jsegdep = newjsegdep(&jfreefrag->fr_list); 4613 jfreefrag->fr_state = ATTACHED | DEPCOMPLETE; 4614 jfreefrag->fr_ino = ip->i_number; 4615 jfreefrag->fr_lbn = lbn; 4616 jfreefrag->fr_blkno = blkno; 4617 jfreefrag->fr_frags = numfrags(fs, size); 4618 jfreefrag->fr_freefrag = freefrag; 4619 4620 return (jfreefrag); 4621 } 4622 4623 /* 4624 * Allocate a new freefrag structure. 4625 */ 4626 static struct freefrag * 4627 newfreefrag(ip, blkno, size, lbn) 4628 struct inode *ip; 4629 ufs2_daddr_t blkno; 4630 long size; 4631 ufs_lbn_t lbn; 4632 { 4633 struct freefrag *freefrag; 4634 struct fs *fs; 4635 4636 fs = ip->i_fs; 4637 if (fragnum(fs, blkno) + numfrags(fs, size) > fs->fs_frag) 4638 panic("newfreefrag: frag size"); 4639 freefrag = malloc(sizeof(struct freefrag), 4640 M_FREEFRAG, M_SOFTDEP_FLAGS); 4641 workitem_alloc(&freefrag->ff_list, D_FREEFRAG, UFSTOVFS(ip->i_ump)); 4642 freefrag->ff_state = ATTACHED; 4643 LIST_INIT(&freefrag->ff_jwork); 4644 freefrag->ff_inum = ip->i_number; 4645 freefrag->ff_blkno = blkno; 4646 freefrag->ff_fragsize = size; 4647 4648 if (fs->fs_flags & FS_SUJ) { 4649 freefrag->ff_jfreefrag = 4650 newjfreefrag(freefrag, ip, blkno, size, lbn); 4651 } else { 4652 freefrag->ff_state |= DEPCOMPLETE; 4653 freefrag->ff_jfreefrag = NULL; 4654 } 4655 4656 return (freefrag); 4657 } 4658 4659 /* 4660 * This workitem de-allocates fragments that were replaced during 4661 * file block allocation. 4662 */ 4663 static void 4664 handle_workitem_freefrag(freefrag) 4665 struct freefrag *freefrag; 4666 { 4667 struct ufsmount *ump = VFSTOUFS(freefrag->ff_list.wk_mp); 4668 struct workhead wkhd; 4669 4670 /* 4671 * It would be illegal to add new completion items to the 4672 * freefrag after it was schedule to be done so it must be 4673 * safe to modify the list head here. 4674 */ 4675 LIST_INIT(&wkhd); 4676 LIST_SWAP(&freefrag->ff_jwork, &wkhd, worklist, wk_list); 4677 ffs_blkfree(ump, ump->um_fs, ump->um_devvp, freefrag->ff_blkno, 4678 freefrag->ff_fragsize, freefrag->ff_inum, &wkhd); 4679 ACQUIRE_LOCK(&lk); 4680 WORKITEM_FREE(freefrag, D_FREEFRAG); 4681 FREE_LOCK(&lk); 4682 } 4683 4684 /* 4685 * Set up a dependency structure for an external attributes data block. 4686 * This routine follows much of the structure of softdep_setup_allocdirect. 4687 * See the description of softdep_setup_allocdirect above for details. 4688 */ 4689 void 4690 softdep_setup_allocext(ip, off, newblkno, oldblkno, newsize, oldsize, bp) 4691 struct inode *ip; 4692 ufs_lbn_t off; 4693 ufs2_daddr_t newblkno; 4694 ufs2_daddr_t oldblkno; 4695 long newsize; 4696 long oldsize; 4697 struct buf *bp; 4698 { 4699 struct allocdirect *adp, *oldadp; 4700 struct allocdirectlst *adphead; 4701 struct freefrag *freefrag; 4702 struct inodedep *inodedep; 4703 struct jnewblk *jnewblk; 4704 struct newblk *newblk; 4705 struct mount *mp; 4706 ufs_lbn_t lbn; 4707 4708 if (off >= NXADDR) 4709 panic("softdep_setup_allocext: lbn %lld > NXADDR", 4710 (long long)off); 4711 4712 lbn = bp->b_lblkno; 4713 mp = UFSTOVFS(ip->i_ump); 4714 if (oldblkno && oldblkno != newblkno) 4715 freefrag = newfreefrag(ip, oldblkno, oldsize, lbn); 4716 else 4717 freefrag = NULL; 4718 4719 ACQUIRE_LOCK(&lk); 4720 if (newblk_lookup(mp, newblkno, 0, &newblk) == 0) 4721 panic("softdep_setup_allocext: lost block"); 4722 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 4723 ("softdep_setup_allocext: newblk already initialized")); 4724 /* 4725 * Convert the newblk to an allocdirect. 4726 */ 4727 newblk->nb_list.wk_type = D_ALLOCDIRECT; 4728 adp = (struct allocdirect *)newblk; 4729 newblk->nb_freefrag = freefrag; 4730 adp->ad_offset = off; 4731 adp->ad_oldblkno = oldblkno; 4732 adp->ad_newsize = newsize; 4733 adp->ad_oldsize = oldsize; 4734 adp->ad_state |= EXTDATA; 4735 4736 /* 4737 * Finish initializing the journal. 4738 */ 4739 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 4740 jnewblk->jn_ino = ip->i_number; 4741 jnewblk->jn_lbn = lbn; 4742 add_to_journal(&jnewblk->jn_list); 4743 } 4744 if (freefrag && freefrag->ff_jfreefrag != NULL) 4745 add_to_journal(&freefrag->ff_jfreefrag->fr_list); 4746 inodedep_lookup(mp, ip->i_number, DEPALLOC | NODELAY, &inodedep); 4747 adp->ad_inodedep = inodedep; 4748 4749 WORKLIST_INSERT(&bp->b_dep, &newblk->nb_list); 4750 /* 4751 * The list of allocdirects must be kept in sorted and ascending 4752 * order so that the rollback routines can quickly determine the 4753 * first uncommitted block (the size of the file stored on disk 4754 * ends at the end of the lowest committed fragment, or if there 4755 * are no fragments, at the end of the highest committed block). 4756 * Since files generally grow, the typical case is that the new 4757 * block is to be added at the end of the list. We speed this 4758 * special case by checking against the last allocdirect in the 4759 * list before laboriously traversing the list looking for the 4760 * insertion point. 4761 */ 4762 adphead = &inodedep->id_newextupdt; 4763 oldadp = TAILQ_LAST(adphead, allocdirectlst); 4764 if (oldadp == NULL || oldadp->ad_offset <= off) { 4765 /* insert at end of list */ 4766 TAILQ_INSERT_TAIL(adphead, adp, ad_next); 4767 if (oldadp != NULL && oldadp->ad_offset == off) 4768 allocdirect_merge(adphead, adp, oldadp); 4769 FREE_LOCK(&lk); 4770 return; 4771 } 4772 TAILQ_FOREACH(oldadp, adphead, ad_next) { 4773 if (oldadp->ad_offset >= off) 4774 break; 4775 } 4776 if (oldadp == NULL) 4777 panic("softdep_setup_allocext: lost entry"); 4778 /* insert in middle of list */ 4779 TAILQ_INSERT_BEFORE(oldadp, adp, ad_next); 4780 if (oldadp->ad_offset == off) 4781 allocdirect_merge(adphead, adp, oldadp); 4782 FREE_LOCK(&lk); 4783 } 4784 4785 /* 4786 * Indirect block allocation dependencies. 4787 * 4788 * The same dependencies that exist for a direct block also exist when 4789 * a new block is allocated and pointed to by an entry in a block of 4790 * indirect pointers. The undo/redo states described above are also 4791 * used here. Because an indirect block contains many pointers that 4792 * may have dependencies, a second copy of the entire in-memory indirect 4793 * block is kept. The buffer cache copy is always completely up-to-date. 4794 * The second copy, which is used only as a source for disk writes, 4795 * contains only the safe pointers (i.e., those that have no remaining 4796 * update dependencies). The second copy is freed when all pointers 4797 * are safe. The cache is not allowed to replace indirect blocks with 4798 * pending update dependencies. If a buffer containing an indirect 4799 * block with dependencies is written, these routines will mark it 4800 * dirty again. It can only be successfully written once all the 4801 * dependencies are removed. The ffs_fsync routine in conjunction with 4802 * softdep_sync_metadata work together to get all the dependencies 4803 * removed so that a file can be successfully written to disk. Three 4804 * procedures are used when setting up indirect block pointer 4805 * dependencies. The division is necessary because of the organization 4806 * of the "balloc" routine and because of the distinction between file 4807 * pages and file metadata blocks. 4808 */ 4809 4810 /* 4811 * Allocate a new allocindir structure. 4812 */ 4813 static struct allocindir * 4814 newallocindir(ip, ptrno, newblkno, oldblkno, lbn) 4815 struct inode *ip; /* inode for file being extended */ 4816 int ptrno; /* offset of pointer in indirect block */ 4817 ufs2_daddr_t newblkno; /* disk block number being added */ 4818 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 4819 ufs_lbn_t lbn; 4820 { 4821 struct newblk *newblk; 4822 struct allocindir *aip; 4823 struct freefrag *freefrag; 4824 struct jnewblk *jnewblk; 4825 4826 if (oldblkno) 4827 freefrag = newfreefrag(ip, oldblkno, ip->i_fs->fs_bsize, lbn); 4828 else 4829 freefrag = NULL; 4830 ACQUIRE_LOCK(&lk); 4831 if (newblk_lookup(UFSTOVFS(ip->i_ump), newblkno, 0, &newblk) == 0) 4832 panic("new_allocindir: lost block"); 4833 KASSERT(newblk->nb_list.wk_type == D_NEWBLK, 4834 ("newallocindir: newblk already initialized")); 4835 newblk->nb_list.wk_type = D_ALLOCINDIR; 4836 newblk->nb_freefrag = freefrag; 4837 aip = (struct allocindir *)newblk; 4838 aip->ai_offset = ptrno; 4839 aip->ai_oldblkno = oldblkno; 4840 if ((jnewblk = newblk->nb_jnewblk) != NULL) { 4841 jnewblk->jn_ino = ip->i_number; 4842 jnewblk->jn_lbn = lbn; 4843 add_to_journal(&jnewblk->jn_list); 4844 } 4845 if (freefrag && freefrag->ff_jfreefrag != NULL) 4846 add_to_journal(&freefrag->ff_jfreefrag->fr_list); 4847 return (aip); 4848 } 4849 4850 /* 4851 * Called just before setting an indirect block pointer 4852 * to a newly allocated file page. 4853 */ 4854 void 4855 softdep_setup_allocindir_page(ip, lbn, bp, ptrno, newblkno, oldblkno, nbp) 4856 struct inode *ip; /* inode for file being extended */ 4857 ufs_lbn_t lbn; /* allocated block number within file */ 4858 struct buf *bp; /* buffer with indirect blk referencing page */ 4859 int ptrno; /* offset of pointer in indirect block */ 4860 ufs2_daddr_t newblkno; /* disk block number being added */ 4861 ufs2_daddr_t oldblkno; /* previous block number, 0 if none */ 4862 struct buf *nbp; /* buffer holding allocated page */ 4863 { 4864 struct inodedep *inodedep; 4865 struct allocindir *aip; 4866 struct pagedep *pagedep; 4867 struct mount *mp; 4868 4869 if (lbn != nbp->b_lblkno) 4870 panic("softdep_setup_allocindir_page: lbn %jd != lblkno %jd", 4871 lbn, bp->b_lblkno); 4872 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_page"); 4873 mp = UFSTOVFS(ip->i_ump); 4874 aip = newallocindir(ip, ptrno, newblkno, oldblkno, lbn); 4875 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 4876 /* 4877 * If we are allocating a directory page, then we must 4878 * allocate an associated pagedep to track additions and 4879 * deletions. 4880 */ 4881 if ((ip->i_mode & IFMT) == IFDIR && 4882 pagedep_lookup(mp, ip->i_number, lbn, DEPALLOC, &pagedep) == 0) 4883 WORKLIST_INSERT(&nbp->b_dep, &pagedep->pd_list); 4884 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 4885 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn); 4886 FREE_LOCK(&lk); 4887 } 4888 4889 /* 4890 * Called just before setting an indirect block pointer to a 4891 * newly allocated indirect block. 4892 */ 4893 void 4894 softdep_setup_allocindir_meta(nbp, ip, bp, ptrno, newblkno) 4895 struct buf *nbp; /* newly allocated indirect block */ 4896 struct inode *ip; /* inode for file being extended */ 4897 struct buf *bp; /* indirect block referencing allocated block */ 4898 int ptrno; /* offset of pointer in indirect block */ 4899 ufs2_daddr_t newblkno; /* disk block number being added */ 4900 { 4901 struct inodedep *inodedep; 4902 struct allocindir *aip; 4903 ufs_lbn_t lbn; 4904 4905 lbn = nbp->b_lblkno; 4906 ASSERT_VOP_LOCKED(ITOV(ip), "softdep_setup_allocindir_meta"); 4907 aip = newallocindir(ip, ptrno, newblkno, 0, lbn); 4908 inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, DEPALLOC, &inodedep); 4909 WORKLIST_INSERT(&nbp->b_dep, &aip->ai_block.nb_list); 4910 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn); 4911 FREE_LOCK(&lk); 4912 } 4913 4914 static void 4915 indirdep_complete(indirdep) 4916 struct indirdep *indirdep; 4917 { 4918 struct allocindir *aip; 4919 4920 LIST_REMOVE(indirdep, ir_next); 4921 indirdep->ir_state &= ~ONDEPLIST; 4922 4923 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) { 4924 LIST_REMOVE(aip, ai_next); 4925 free_newblk(&aip->ai_block); 4926 } 4927 /* 4928 * If this indirdep is not attached to a buf it was simply waiting 4929 * on completion to clear completehd. free_indirdep() asserts 4930 * that nothing is dangling. 4931 */ 4932 if ((indirdep->ir_state & ONWORKLIST) == 0) 4933 free_indirdep(indirdep); 4934 } 4935 4936 /* 4937 * Called to finish the allocation of the "aip" allocated 4938 * by one of the two routines above. 4939 */ 4940 static void 4941 setup_allocindir_phase2(bp, ip, inodedep, aip, lbn) 4942 struct buf *bp; /* in-memory copy of the indirect block */ 4943 struct inode *ip; /* inode for file being extended */ 4944 struct inodedep *inodedep; /* Inodedep for ip */ 4945 struct allocindir *aip; /* allocindir allocated by the above routines */ 4946 ufs_lbn_t lbn; /* Logical block number for this block. */ 4947 { 4948 struct worklist *wk; 4949 struct fs *fs; 4950 struct newblk *newblk; 4951 struct indirdep *indirdep, *newindirdep; 4952 struct allocindir *oldaip; 4953 struct freefrag *freefrag; 4954 struct mount *mp; 4955 ufs2_daddr_t blkno; 4956 4957 mp = UFSTOVFS(ip->i_ump); 4958 fs = ip->i_fs; 4959 mtx_assert(&lk, MA_OWNED); 4960 if (bp->b_lblkno >= 0) 4961 panic("setup_allocindir_phase2: not indir blk"); 4962 for (freefrag = NULL, indirdep = NULL, newindirdep = NULL; ; ) { 4963 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 4964 if (wk->wk_type != D_INDIRDEP) 4965 continue; 4966 indirdep = WK_INDIRDEP(wk); 4967 break; 4968 } 4969 if (indirdep == NULL && newindirdep) { 4970 indirdep = newindirdep; 4971 newindirdep = NULL; 4972 WORKLIST_INSERT(&bp->b_dep, &indirdep->ir_list); 4973 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, 4974 &newblk)) { 4975 indirdep->ir_state |= ONDEPLIST; 4976 LIST_INSERT_HEAD(&newblk->nb_indirdeps, 4977 indirdep, ir_next); 4978 } else 4979 indirdep->ir_state |= DEPCOMPLETE; 4980 } 4981 if (indirdep) { 4982 aip->ai_indirdep = indirdep; 4983 /* 4984 * Check to see if there is an existing dependency 4985 * for this block. If there is, merge the old 4986 * dependency into the new one. This happens 4987 * as a result of reallocblk only. 4988 */ 4989 if (aip->ai_oldblkno == 0) 4990 oldaip = NULL; 4991 else 4992 4993 LIST_FOREACH(oldaip, &indirdep->ir_deplisthd, 4994 ai_next) 4995 if (oldaip->ai_offset == aip->ai_offset) 4996 break; 4997 if (oldaip != NULL) 4998 freefrag = allocindir_merge(aip, oldaip); 4999 LIST_INSERT_HEAD(&indirdep->ir_deplisthd, aip, ai_next); 5000 KASSERT(aip->ai_offset >= 0 && 5001 aip->ai_offset < NINDIR(ip->i_ump->um_fs), 5002 ("setup_allocindir_phase2: Bad offset %d", 5003 aip->ai_offset)); 5004 KASSERT(indirdep->ir_savebp != NULL, 5005 ("setup_allocindir_phase2 NULL ir_savebp")); 5006 if (ip->i_ump->um_fstype == UFS1) 5007 ((ufs1_daddr_t *)indirdep->ir_savebp->b_data) 5008 [aip->ai_offset] = aip->ai_oldblkno; 5009 else 5010 ((ufs2_daddr_t *)indirdep->ir_savebp->b_data) 5011 [aip->ai_offset] = aip->ai_oldblkno; 5012 FREE_LOCK(&lk); 5013 if (freefrag != NULL) 5014 handle_workitem_freefrag(freefrag); 5015 } else 5016 FREE_LOCK(&lk); 5017 if (newindirdep) { 5018 newindirdep->ir_savebp->b_flags |= B_INVAL | B_NOCACHE; 5019 brelse(newindirdep->ir_savebp); 5020 ACQUIRE_LOCK(&lk); 5021 WORKITEM_FREE((caddr_t)newindirdep, D_INDIRDEP); 5022 if (indirdep) 5023 break; 5024 FREE_LOCK(&lk); 5025 } 5026 if (indirdep) { 5027 ACQUIRE_LOCK(&lk); 5028 break; 5029 } 5030 newindirdep = malloc(sizeof(struct indirdep), 5031 M_INDIRDEP, M_SOFTDEP_FLAGS); 5032 workitem_alloc(&newindirdep->ir_list, D_INDIRDEP, mp); 5033 newindirdep->ir_state = ATTACHED; 5034 if (ip->i_ump->um_fstype == UFS1) 5035 newindirdep->ir_state |= UFS1FMT; 5036 newindirdep->ir_saveddata = NULL; 5037 LIST_INIT(&newindirdep->ir_deplisthd); 5038 LIST_INIT(&newindirdep->ir_donehd); 5039 LIST_INIT(&newindirdep->ir_writehd); 5040 LIST_INIT(&newindirdep->ir_completehd); 5041 LIST_INIT(&newindirdep->ir_jwork); 5042 if (bp->b_blkno == bp->b_lblkno) { 5043 ufs_bmaparray(bp->b_vp, bp->b_lblkno, &blkno, bp, 5044 NULL, NULL); 5045 bp->b_blkno = blkno; 5046 } 5047 newindirdep->ir_savebp = 5048 getblk(ip->i_devvp, bp->b_blkno, bp->b_bcount, 0, 0, 0); 5049 BUF_KERNPROC(newindirdep->ir_savebp); 5050 bcopy(bp->b_data, newindirdep->ir_savebp->b_data, bp->b_bcount); 5051 ACQUIRE_LOCK(&lk); 5052 } 5053 } 5054 5055 /* 5056 * Merge two allocindirs which refer to the same block. Move newblock 5057 * dependencies and setup the freefrags appropriately. 5058 */ 5059 static struct freefrag * 5060 allocindir_merge(aip, oldaip) 5061 struct allocindir *aip; 5062 struct allocindir *oldaip; 5063 { 5064 struct newdirblk *newdirblk; 5065 struct freefrag *freefrag; 5066 struct worklist *wk; 5067 5068 if (oldaip->ai_newblkno != aip->ai_oldblkno) 5069 panic("allocindir_merge: blkno"); 5070 aip->ai_oldblkno = oldaip->ai_oldblkno; 5071 freefrag = aip->ai_freefrag; 5072 aip->ai_freefrag = oldaip->ai_freefrag; 5073 oldaip->ai_freefrag = NULL; 5074 KASSERT(freefrag != NULL, ("setup_allocindir_phase2: No freefrag")); 5075 /* 5076 * If we are tracking a new directory-block allocation, 5077 * move it from the old allocindir to the new allocindir. 5078 */ 5079 if ((wk = LIST_FIRST(&oldaip->ai_newdirblk)) != NULL) { 5080 newdirblk = WK_NEWDIRBLK(wk); 5081 WORKLIST_REMOVE(&newdirblk->db_list); 5082 if (!LIST_EMPTY(&oldaip->ai_newdirblk)) 5083 panic("allocindir_merge: extra newdirblk"); 5084 WORKLIST_INSERT(&aip->ai_newdirblk, &newdirblk->db_list); 5085 } 5086 /* 5087 * We can skip journaling for this freefrag and just complete 5088 * any pending journal work for the allocindir that is being 5089 * removed after the freefrag completes. 5090 */ 5091 if (freefrag->ff_jfreefrag) 5092 cancel_jfreefrag(freefrag->ff_jfreefrag); 5093 LIST_REMOVE(oldaip, ai_next); 5094 cancel_newblk(&oldaip->ai_block, &freefrag->ff_jwork); 5095 free_newblk(&oldaip->ai_block); 5096 5097 return (freefrag); 5098 } 5099 5100 /* 5101 * Block de-allocation dependencies. 5102 * 5103 * When blocks are de-allocated, the on-disk pointers must be nullified before 5104 * the blocks are made available for use by other files. (The true 5105 * requirement is that old pointers must be nullified before new on-disk 5106 * pointers are set. We chose this slightly more stringent requirement to 5107 * reduce complexity.) Our implementation handles this dependency by updating 5108 * the inode (or indirect block) appropriately but delaying the actual block 5109 * de-allocation (i.e., freemap and free space count manipulation) until 5110 * after the updated versions reach stable storage. After the disk is 5111 * updated, the blocks can be safely de-allocated whenever it is convenient. 5112 * This implementation handles only the common case of reducing a file's 5113 * length to zero. Other cases are handled by the conventional synchronous 5114 * write approach. 5115 * 5116 * The ffs implementation with which we worked double-checks 5117 * the state of the block pointers and file size as it reduces 5118 * a file's length. Some of this code is replicated here in our 5119 * soft updates implementation. The freeblks->fb_chkcnt field is 5120 * used to transfer a part of this information to the procedure 5121 * that eventually de-allocates the blocks. 5122 * 5123 * This routine should be called from the routine that shortens 5124 * a file's length, before the inode's size or block pointers 5125 * are modified. It will save the block pointer information for 5126 * later release and zero the inode so that the calling routine 5127 * can release it. 5128 */ 5129 void 5130 softdep_setup_freeblocks(ip, length, flags) 5131 struct inode *ip; /* The inode whose length is to be reduced */ 5132 off_t length; /* The new length for the file */ 5133 int flags; /* IO_EXT and/or IO_NORMAL */ 5134 { 5135 struct ufs1_dinode *dp1; 5136 struct ufs2_dinode *dp2; 5137 struct freeblks *freeblks; 5138 struct inodedep *inodedep; 5139 struct allocdirect *adp; 5140 struct jfreeblk *jfreeblk; 5141 struct bufobj *bo; 5142 struct vnode *vp; 5143 struct buf *bp; 5144 struct fs *fs; 5145 ufs2_daddr_t extblocks, datablocks; 5146 struct mount *mp; 5147 int i, delay, error; 5148 ufs2_daddr_t blkno; 5149 ufs_lbn_t tmpval; 5150 ufs_lbn_t lbn; 5151 long oldextsize; 5152 long oldsize; 5153 int frags; 5154 int needj; 5155 5156 fs = ip->i_fs; 5157 mp = UFSTOVFS(ip->i_ump); 5158 if (length != 0) 5159 panic("softdep_setup_freeblocks: non-zero length"); 5160 freeblks = malloc(sizeof(struct freeblks), 5161 M_FREEBLKS, M_SOFTDEP_FLAGS|M_ZERO); 5162 workitem_alloc(&freeblks->fb_list, D_FREEBLKS, mp); 5163 LIST_INIT(&freeblks->fb_jfreeblkhd); 5164 LIST_INIT(&freeblks->fb_jwork); 5165 freeblks->fb_state = ATTACHED; 5166 freeblks->fb_uid = ip->i_uid; 5167 freeblks->fb_previousinum = ip->i_number; 5168 freeblks->fb_devvp = ip->i_devvp; 5169 freeblks->fb_chkcnt = 0; 5170 ACQUIRE_LOCK(&lk); 5171 /* 5172 * If we're truncating a removed file that will never be written 5173 * we don't need to journal the block frees. The canceled journals 5174 * for the allocations will suffice. 5175 */ 5176 inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5177 if ((inodedep->id_state & (UNLINKED | DEPCOMPLETE)) == UNLINKED || 5178 (fs->fs_flags & FS_SUJ) == 0) 5179 needj = 0; 5180 else 5181 needj = 1; 5182 num_freeblkdep++; 5183 FREE_LOCK(&lk); 5184 extblocks = 0; 5185 if (fs->fs_magic == FS_UFS2_MAGIC) 5186 extblocks = btodb(fragroundup(fs, ip->i_din2->di_extsize)); 5187 datablocks = DIP(ip, i_blocks) - extblocks; 5188 if ((flags & IO_NORMAL) != 0) { 5189 oldsize = ip->i_size; 5190 ip->i_size = 0; 5191 DIP_SET(ip, i_size, 0); 5192 freeblks->fb_chkcnt = datablocks; 5193 for (i = 0; i < NDADDR; i++) { 5194 blkno = DIP(ip, i_db[i]); 5195 DIP_SET(ip, i_db[i], 0); 5196 if (blkno == 0) 5197 continue; 5198 frags = sblksize(fs, oldsize, i); 5199 frags = numfrags(fs, frags); 5200 newfreework(freeblks, NULL, i, blkno, frags, needj); 5201 } 5202 for (i = 0, tmpval = NINDIR(fs), lbn = NDADDR; i < NIADDR; 5203 i++, tmpval *= NINDIR(fs)) { 5204 blkno = DIP(ip, i_ib[i]); 5205 DIP_SET(ip, i_ib[i], 0); 5206 if (blkno) 5207 newfreework(freeblks, NULL, -lbn - i, blkno, 5208 fs->fs_frag, needj); 5209 lbn += tmpval; 5210 } 5211 UFS_LOCK(ip->i_ump); 5212 fs->fs_pendingblocks += datablocks; 5213 UFS_UNLOCK(ip->i_ump); 5214 } 5215 if ((flags & IO_EXT) != 0) { 5216 oldextsize = ip->i_din2->di_extsize; 5217 ip->i_din2->di_extsize = 0; 5218 freeblks->fb_chkcnt += extblocks; 5219 for (i = 0; i < NXADDR; i++) { 5220 blkno = ip->i_din2->di_extb[i]; 5221 ip->i_din2->di_extb[i] = 0; 5222 if (blkno == 0) 5223 continue; 5224 frags = sblksize(fs, oldextsize, i); 5225 frags = numfrags(fs, frags); 5226 newfreework(freeblks, NULL, -1 - i, blkno, frags, 5227 needj); 5228 } 5229 } 5230 if (LIST_EMPTY(&freeblks->fb_jfreeblkhd)) 5231 needj = 0; 5232 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) - freeblks->fb_chkcnt); 5233 /* 5234 * Push the zero'ed inode to to its disk buffer so that we are free 5235 * to delete its dependencies below. Once the dependencies are gone 5236 * the buffer can be safely released. 5237 */ 5238 if ((error = bread(ip->i_devvp, 5239 fsbtodb(fs, ino_to_fsba(fs, ip->i_number)), 5240 (int)fs->fs_bsize, NOCRED, &bp)) != 0) { 5241 brelse(bp); 5242 softdep_error("softdep_setup_freeblocks", error); 5243 } 5244 if (ip->i_ump->um_fstype == UFS1) { 5245 dp1 = ((struct ufs1_dinode *)bp->b_data + 5246 ino_to_fsbo(fs, ip->i_number)); 5247 ip->i_din1->di_freelink = dp1->di_freelink; 5248 *dp1 = *ip->i_din1; 5249 } else { 5250 dp2 = ((struct ufs2_dinode *)bp->b_data + 5251 ino_to_fsbo(fs, ip->i_number)); 5252 ip->i_din2->di_freelink = dp2->di_freelink; 5253 *dp2 = *ip->i_din2; 5254 } 5255 /* 5256 * Find and eliminate any inode dependencies. 5257 */ 5258 ACQUIRE_LOCK(&lk); 5259 (void) inodedep_lookup(mp, ip->i_number, DEPALLOC, &inodedep); 5260 if ((inodedep->id_state & IOSTARTED) != 0) 5261 panic("softdep_setup_freeblocks: inode busy"); 5262 /* 5263 * Add the freeblks structure to the list of operations that 5264 * must await the zero'ed inode being written to disk. If we 5265 * still have a bitmap dependency (delay == 0), then the inode 5266 * has never been written to disk, so we can process the 5267 * freeblks below once we have deleted the dependencies. 5268 */ 5269 delay = (inodedep->id_state & DEPCOMPLETE); 5270 if (delay) 5271 WORKLIST_INSERT(&bp->b_dep, &freeblks->fb_list); 5272 else if (needj) 5273 freeblks->fb_state |= COMPLETE; 5274 /* 5275 * Because the file length has been truncated to zero, any 5276 * pending block allocation dependency structures associated 5277 * with this inode are obsolete and can simply be de-allocated. 5278 * We must first merge the two dependency lists to get rid of 5279 * any duplicate freefrag structures, then purge the merged list. 5280 * If we still have a bitmap dependency, then the inode has never 5281 * been written to disk, so we can free any fragments without delay. 5282 */ 5283 if (flags & IO_NORMAL) { 5284 merge_inode_lists(&inodedep->id_newinoupdt, 5285 &inodedep->id_inoupdt); 5286 while ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != 0) 5287 cancel_allocdirect(&inodedep->id_inoupdt, adp, 5288 freeblks, delay); 5289 } 5290 if (flags & IO_EXT) { 5291 merge_inode_lists(&inodedep->id_newextupdt, 5292 &inodedep->id_extupdt); 5293 while ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != 0) 5294 cancel_allocdirect(&inodedep->id_extupdt, adp, 5295 freeblks, delay); 5296 } 5297 LIST_FOREACH(jfreeblk, &freeblks->fb_jfreeblkhd, jf_deps) 5298 add_to_journal(&jfreeblk->jf_list); 5299 5300 FREE_LOCK(&lk); 5301 bdwrite(bp); 5302 /* 5303 * We must wait for any I/O in progress to finish so that 5304 * all potential buffers on the dirty list will be visible. 5305 * Once they are all there, walk the list and get rid of 5306 * any dependencies. 5307 */ 5308 vp = ITOV(ip); 5309 bo = &vp->v_bufobj; 5310 BO_LOCK(bo); 5311 drain_output(vp); 5312 restart: 5313 TAILQ_FOREACH(bp, &bo->bo_dirty.bv_hd, b_bobufs) { 5314 if (((flags & IO_EXT) == 0 && (bp->b_xflags & BX_ALTDATA)) || 5315 ((flags & IO_NORMAL) == 0 && 5316 (bp->b_xflags & BX_ALTDATA) == 0)) 5317 continue; 5318 if ((bp = getdirtybuf(bp, BO_MTX(bo), MNT_WAIT)) == NULL) 5319 goto restart; 5320 BO_UNLOCK(bo); 5321 ACQUIRE_LOCK(&lk); 5322 (void) inodedep_lookup(mp, ip->i_number, 0, &inodedep); 5323 if (deallocate_dependencies(bp, inodedep, freeblks)) 5324 bp->b_flags |= B_INVAL | B_NOCACHE; 5325 FREE_LOCK(&lk); 5326 brelse(bp); 5327 BO_LOCK(bo); 5328 goto restart; 5329 } 5330 BO_UNLOCK(bo); 5331 ACQUIRE_LOCK(&lk); 5332 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 5333 (void) free_inodedep(inodedep); 5334 5335 if (delay || needj) 5336 freeblks->fb_state |= DEPCOMPLETE; 5337 if (delay) { 5338 /* 5339 * If the inode with zeroed block pointers is now on disk 5340 * we can start freeing blocks. Add freeblks to the worklist 5341 * instead of calling handle_workitem_freeblocks directly as 5342 * it is more likely that additional IO is needed to complete 5343 * the request here than in the !delay case. 5344 */ 5345 if ((freeblks->fb_state & ALLCOMPLETE) == ALLCOMPLETE) 5346 add_to_worklist(&freeblks->fb_list, 1); 5347 } 5348 if (needj && LIST_EMPTY(&freeblks->fb_jfreeblkhd)) 5349 needj = 0; 5350 5351 FREE_LOCK(&lk); 5352 /* 5353 * If the inode has never been written to disk (delay == 0) and 5354 * we're not waiting on any journal writes, then we can process the 5355 * freeblks now that we have deleted the dependencies. 5356 */ 5357 if (!delay && !needj) 5358 handle_workitem_freeblocks(freeblks, 0); 5359 } 5360 5361 /* 5362 * Reclaim any dependency structures from a buffer that is about to 5363 * be reallocated to a new vnode. The buffer must be locked, thus, 5364 * no I/O completion operations can occur while we are manipulating 5365 * its associated dependencies. The mutex is held so that other I/O's 5366 * associated with related dependencies do not occur. Returns 1 if 5367 * all dependencies were cleared, 0 otherwise. 5368 */ 5369 static int 5370 deallocate_dependencies(bp, inodedep, freeblks) 5371 struct buf *bp; 5372 struct inodedep *inodedep; 5373 struct freeblks *freeblks; 5374 { 5375 struct worklist *wk; 5376 struct indirdep *indirdep; 5377 struct newdirblk *newdirblk; 5378 struct allocindir *aip; 5379 struct pagedep *pagedep; 5380 struct jremref *jremref; 5381 struct jmvref *jmvref; 5382 struct dirrem *dirrem; 5383 int i; 5384 5385 mtx_assert(&lk, MA_OWNED); 5386 while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) { 5387 switch (wk->wk_type) { 5388 5389 case D_INDIRDEP: 5390 indirdep = WK_INDIRDEP(wk); 5391 if (bp->b_lblkno >= 0 || 5392 bp->b_blkno != indirdep->ir_savebp->b_lblkno) 5393 panic("deallocate_dependencies: not indir"); 5394 cancel_indirdep(indirdep, bp, inodedep, freeblks); 5395 continue; 5396 5397 case D_PAGEDEP: 5398 pagedep = WK_PAGEDEP(wk); 5399 /* 5400 * There should be no directory add dependencies present 5401 * as the directory could not be truncated until all 5402 * children were removed. 5403 */ 5404 KASSERT(LIST_FIRST(&pagedep->pd_pendinghd) == NULL, 5405 ("deallocate_dependencies: pendinghd != NULL")); 5406 for (i = 0; i < DAHASHSZ; i++) 5407 KASSERT(LIST_FIRST(&pagedep->pd_diraddhd[i]) == NULL, 5408 ("deallocate_dependencies: diraddhd != NULL")); 5409 /* 5410 * Copy any directory remove dependencies to the list 5411 * to be processed after the zero'ed inode is written. 5412 * If the inode has already been written, then they 5413 * can be dumped directly onto the work list. 5414 */ 5415 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) { 5416 /* 5417 * If there are any dirrems we wait for 5418 * the journal write to complete and 5419 * then restart the buf scan as the lock 5420 * has been dropped. 5421 */ 5422 while ((jremref = 5423 LIST_FIRST(&dirrem->dm_jremrefhd)) 5424 != NULL) { 5425 stat_jwait_filepage++; 5426 jwait(&jremref->jr_list); 5427 return (0); 5428 } 5429 LIST_REMOVE(dirrem, dm_next); 5430 dirrem->dm_dirinum = pagedep->pd_ino; 5431 if (inodedep == NULL || 5432 (inodedep->id_state & ALLCOMPLETE) == 5433 ALLCOMPLETE) { 5434 dirrem->dm_state |= COMPLETE; 5435 add_to_worklist(&dirrem->dm_list, 0); 5436 } else 5437 WORKLIST_INSERT(&inodedep->id_bufwait, 5438 &dirrem->dm_list); 5439 } 5440 if ((pagedep->pd_state & NEWBLOCK) != 0) { 5441 newdirblk = pagedep->pd_newdirblk; 5442 WORKLIST_REMOVE(&newdirblk->db_list); 5443 free_newdirblk(newdirblk); 5444 } 5445 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) 5446 != NULL) { 5447 stat_jwait_filepage++; 5448 jwait(&jmvref->jm_list); 5449 return (0); 5450 } 5451 WORKLIST_REMOVE(&pagedep->pd_list); 5452 LIST_REMOVE(pagedep, pd_hash); 5453 WORKITEM_FREE(pagedep, D_PAGEDEP); 5454 continue; 5455 5456 case D_ALLOCINDIR: 5457 aip = WK_ALLOCINDIR(wk); 5458 cancel_allocindir(aip, inodedep, freeblks); 5459 continue; 5460 5461 case D_ALLOCDIRECT: 5462 case D_INODEDEP: 5463 panic("deallocate_dependencies: Unexpected type %s", 5464 TYPENAME(wk->wk_type)); 5465 /* NOTREACHED */ 5466 5467 default: 5468 panic("deallocate_dependencies: Unknown type %s", 5469 TYPENAME(wk->wk_type)); 5470 /* NOTREACHED */ 5471 } 5472 } 5473 5474 return (1); 5475 } 5476 5477 /* 5478 * An allocdirect is being canceled due to a truncate. We must make sure 5479 * the journal entry is released in concert with the blkfree that releases 5480 * the storage. Completed journal entries must not be released until the 5481 * space is no longer pointed to by the inode or in the bitmap. 5482 */ 5483 static void 5484 cancel_allocdirect(adphead, adp, freeblks, delay) 5485 struct allocdirectlst *adphead; 5486 struct allocdirect *adp; 5487 struct freeblks *freeblks; 5488 int delay; 5489 { 5490 struct freework *freework; 5491 struct newblk *newblk; 5492 struct worklist *wk; 5493 ufs_lbn_t lbn; 5494 5495 TAILQ_REMOVE(adphead, adp, ad_next); 5496 newblk = (struct newblk *)adp; 5497 /* 5498 * If the journal hasn't been written the jnewblk must be passed 5499 * to the call to ffs_blkfree that reclaims the space. We accomplish 5500 * this by linking the journal dependency into the freework to be 5501 * freed when freework_freeblock() is called. If the journal has 5502 * been written we can simply reclaim the journal space when the 5503 * freeblks work is complete. 5504 */ 5505 if (newblk->nb_jnewblk == NULL) { 5506 cancel_newblk(newblk, &freeblks->fb_jwork); 5507 goto found; 5508 } 5509 lbn = newblk->nb_jnewblk->jn_lbn; 5510 /* 5511 * Find the correct freework structure so it releases the canceled 5512 * journal when the bitmap is cleared. This preserves rollback 5513 * until the allocation is reverted. 5514 */ 5515 LIST_FOREACH(wk, &freeblks->fb_freeworkhd, wk_list) { 5516 freework = WK_FREEWORK(wk); 5517 if (freework->fw_lbn != lbn) 5518 continue; 5519 cancel_newblk(newblk, &freework->fw_jwork); 5520 goto found; 5521 } 5522 panic("cancel_allocdirect: Freework not found for lbn %jd\n", lbn); 5523 found: 5524 if (delay) 5525 WORKLIST_INSERT(&adp->ad_inodedep->id_bufwait, 5526 &newblk->nb_list); 5527 else 5528 free_newblk(newblk); 5529 return; 5530 } 5531 5532 5533 static void 5534 cancel_newblk(newblk, wkhd) 5535 struct newblk *newblk; 5536 struct workhead *wkhd; 5537 { 5538 struct indirdep *indirdep; 5539 struct allocindir *aip; 5540 5541 while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL) { 5542 indirdep->ir_state &= ~ONDEPLIST; 5543 LIST_REMOVE(indirdep, ir_next); 5544 /* 5545 * If an indirdep is not on the buf worklist we need to 5546 * free it here as deallocate_dependencies() will never 5547 * find it. These pointers were never visible on disk and 5548 * can be discarded immediately. 5549 */ 5550 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != NULL) { 5551 LIST_REMOVE(aip, ai_next); 5552 cancel_newblk(&aip->ai_block, wkhd); 5553 free_newblk(&aip->ai_block); 5554 } 5555 /* 5556 * If this indirdep is not attached to a buf it was simply 5557 * waiting on completion to clear completehd. free_indirdep() 5558 * asserts that nothing is dangling. 5559 */ 5560 if ((indirdep->ir_state & ONWORKLIST) == 0) 5561 free_indirdep(indirdep); 5562 } 5563 if (newblk->nb_state & ONDEPLIST) { 5564 newblk->nb_state &= ~ONDEPLIST; 5565 LIST_REMOVE(newblk, nb_deps); 5566 } 5567 if (newblk->nb_state & ONWORKLIST) 5568 WORKLIST_REMOVE(&newblk->nb_list); 5569 /* 5570 * If the journal entry hasn't been written we hold onto the dep 5571 * until it is safe to free along with the other journal work. 5572 */ 5573 if (newblk->nb_jnewblk != NULL) { 5574 cancel_jnewblk(newblk->nb_jnewblk, wkhd); 5575 newblk->nb_jnewblk = NULL; 5576 } 5577 if (!LIST_EMPTY(&newblk->nb_jwork)) 5578 jwork_move(wkhd, &newblk->nb_jwork); 5579 } 5580 5581 /* 5582 * Free a newblk. Generate a new freefrag work request if appropriate. 5583 * This must be called after the inode pointer and any direct block pointers 5584 * are valid or fully removed via truncate or frag extension. 5585 */ 5586 static void 5587 free_newblk(newblk) 5588 struct newblk *newblk; 5589 { 5590 struct indirdep *indirdep; 5591 struct newdirblk *newdirblk; 5592 struct freefrag *freefrag; 5593 struct worklist *wk; 5594 5595 mtx_assert(&lk, MA_OWNED); 5596 if (newblk->nb_state & ONDEPLIST) 5597 LIST_REMOVE(newblk, nb_deps); 5598 if (newblk->nb_state & ONWORKLIST) 5599 WORKLIST_REMOVE(&newblk->nb_list); 5600 LIST_REMOVE(newblk, nb_hash); 5601 if ((freefrag = newblk->nb_freefrag) != NULL) { 5602 freefrag->ff_state |= COMPLETE; 5603 if ((freefrag->ff_state & ALLCOMPLETE) == ALLCOMPLETE) 5604 add_to_worklist(&freefrag->ff_list, 0); 5605 } 5606 if ((wk = LIST_FIRST(&newblk->nb_newdirblk)) != NULL) { 5607 newdirblk = WK_NEWDIRBLK(wk); 5608 WORKLIST_REMOVE(&newdirblk->db_list); 5609 if (!LIST_EMPTY(&newblk->nb_newdirblk)) 5610 panic("free_newblk: extra newdirblk"); 5611 free_newdirblk(newdirblk); 5612 } 5613 while ((indirdep = LIST_FIRST(&newblk->nb_indirdeps)) != NULL) { 5614 indirdep->ir_state |= DEPCOMPLETE; 5615 indirdep_complete(indirdep); 5616 } 5617 KASSERT(newblk->nb_jnewblk == NULL, 5618 ("free_newblk; jnewblk %p still attached", newblk->nb_jnewblk)); 5619 handle_jwork(&newblk->nb_jwork); 5620 newblk->nb_list.wk_type = D_NEWBLK; 5621 WORKITEM_FREE(newblk, D_NEWBLK); 5622 } 5623 5624 /* 5625 * Free a newdirblk. Clear the NEWBLOCK flag on its associated pagedep. 5626 * This routine must be called with splbio interrupts blocked. 5627 */ 5628 static void 5629 free_newdirblk(newdirblk) 5630 struct newdirblk *newdirblk; 5631 { 5632 struct pagedep *pagedep; 5633 struct diradd *dap; 5634 struct worklist *wk; 5635 int i; 5636 5637 mtx_assert(&lk, MA_OWNED); 5638 /* 5639 * If the pagedep is still linked onto the directory buffer 5640 * dependency chain, then some of the entries on the 5641 * pd_pendinghd list may not be committed to disk yet. In 5642 * this case, we will simply clear the NEWBLOCK flag and 5643 * let the pd_pendinghd list be processed when the pagedep 5644 * is next written. If the pagedep is no longer on the buffer 5645 * dependency chain, then all the entries on the pd_pending 5646 * list are committed to disk and we can free them here. 5647 */ 5648 pagedep = newdirblk->db_pagedep; 5649 pagedep->pd_state &= ~NEWBLOCK; 5650 if ((pagedep->pd_state & ONWORKLIST) == 0) 5651 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 5652 free_diradd(dap, NULL); 5653 /* 5654 * If no dependencies remain, the pagedep will be freed. 5655 */ 5656 for (i = 0; i < DAHASHSZ; i++) 5657 if (!LIST_EMPTY(&pagedep->pd_diraddhd[i])) 5658 break; 5659 if (i == DAHASHSZ && (pagedep->pd_state & ONWORKLIST) == 0 && 5660 LIST_EMPTY(&pagedep->pd_jmvrefhd)) { 5661 KASSERT(LIST_FIRST(&pagedep->pd_dirremhd) == NULL, 5662 ("free_newdirblk: Freeing non-free pagedep %p", pagedep)); 5663 LIST_REMOVE(pagedep, pd_hash); 5664 WORKITEM_FREE(pagedep, D_PAGEDEP); 5665 } 5666 /* Should only ever be one item in the list. */ 5667 while ((wk = LIST_FIRST(&newdirblk->db_mkdir)) != NULL) { 5668 WORKLIST_REMOVE(wk); 5669 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 5670 } 5671 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 5672 } 5673 5674 /* 5675 * Prepare an inode to be freed. The actual free operation is not 5676 * done until the zero'ed inode has been written to disk. 5677 */ 5678 void 5679 softdep_freefile(pvp, ino, mode) 5680 struct vnode *pvp; 5681 ino_t ino; 5682 int mode; 5683 { 5684 struct inode *ip = VTOI(pvp); 5685 struct inodedep *inodedep; 5686 struct freefile *freefile; 5687 5688 /* 5689 * This sets up the inode de-allocation dependency. 5690 */ 5691 freefile = malloc(sizeof(struct freefile), 5692 M_FREEFILE, M_SOFTDEP_FLAGS); 5693 workitem_alloc(&freefile->fx_list, D_FREEFILE, pvp->v_mount); 5694 freefile->fx_mode = mode; 5695 freefile->fx_oldinum = ino; 5696 freefile->fx_devvp = ip->i_devvp; 5697 LIST_INIT(&freefile->fx_jwork); 5698 UFS_LOCK(ip->i_ump); 5699 ip->i_fs->fs_pendinginodes += 1; 5700 UFS_UNLOCK(ip->i_ump); 5701 5702 /* 5703 * If the inodedep does not exist, then the zero'ed inode has 5704 * been written to disk. If the allocated inode has never been 5705 * written to disk, then the on-disk inode is zero'ed. In either 5706 * case we can free the file immediately. If the journal was 5707 * canceled before being written the inode will never make it to 5708 * disk and we must send the canceled journal entrys to 5709 * ffs_freefile() to be cleared in conjunction with the bitmap. 5710 * Any blocks waiting on the inode to write can be safely freed 5711 * here as it will never been written. 5712 */ 5713 ACQUIRE_LOCK(&lk); 5714 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 5715 /* 5716 * Remove this inode from the unlinked list and set 5717 * GOINGAWAY as appropriate to indicate that this inode 5718 * will never be written. 5719 */ 5720 if (inodedep && inodedep->id_state & UNLINKED) { 5721 /* 5722 * Save the journal work to be freed with the bitmap 5723 * before we clear UNLINKED. Otherwise it can be lost 5724 * if the inode block is written. 5725 */ 5726 handle_bufwait(inodedep, &freefile->fx_jwork); 5727 clear_unlinked_inodedep(inodedep); 5728 /* Re-acquire inodedep as we've dropped lk. */ 5729 inodedep_lookup(pvp->v_mount, ino, 0, &inodedep); 5730 if (inodedep && (inodedep->id_state & DEPCOMPLETE) == 0) 5731 inodedep->id_state |= GOINGAWAY; 5732 } 5733 if (inodedep == NULL || check_inode_unwritten(inodedep)) { 5734 FREE_LOCK(&lk); 5735 handle_workitem_freefile(freefile); 5736 return; 5737 } 5738 WORKLIST_INSERT(&inodedep->id_inowait, &freefile->fx_list); 5739 FREE_LOCK(&lk); 5740 if (ip->i_number == ino) 5741 ip->i_flag |= IN_MODIFIED; 5742 } 5743 5744 /* 5745 * Check to see if an inode has never been written to disk. If 5746 * so free the inodedep and return success, otherwise return failure. 5747 * This routine must be called with splbio interrupts blocked. 5748 * 5749 * If we still have a bitmap dependency, then the inode has never 5750 * been written to disk. Drop the dependency as it is no longer 5751 * necessary since the inode is being deallocated. We set the 5752 * ALLCOMPLETE flags since the bitmap now properly shows that the 5753 * inode is not allocated. Even if the inode is actively being 5754 * written, it has been rolled back to its zero'ed state, so we 5755 * are ensured that a zero inode is what is on the disk. For short 5756 * lived files, this change will usually result in removing all the 5757 * dependencies from the inode so that it can be freed immediately. 5758 */ 5759 static int 5760 check_inode_unwritten(inodedep) 5761 struct inodedep *inodedep; 5762 { 5763 5764 mtx_assert(&lk, MA_OWNED); 5765 5766 if ((inodedep->id_state & (DEPCOMPLETE | UNLINKED)) != 0 || 5767 !LIST_EMPTY(&inodedep->id_pendinghd) || 5768 !LIST_EMPTY(&inodedep->id_bufwait) || 5769 !LIST_EMPTY(&inodedep->id_inowait) || 5770 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 5771 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 5772 !TAILQ_EMPTY(&inodedep->id_extupdt) || 5773 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 5774 inodedep->id_mkdiradd != NULL || 5775 inodedep->id_nlinkdelta != 0) 5776 return (0); 5777 /* 5778 * Another process might be in initiate_write_inodeblock_ufs[12] 5779 * trying to allocate memory without holding "Softdep Lock". 5780 */ 5781 if ((inodedep->id_state & IOSTARTED) != 0 && 5782 inodedep->id_savedino1 == NULL) 5783 return (0); 5784 5785 if (inodedep->id_state & ONDEPLIST) 5786 LIST_REMOVE(inodedep, id_deps); 5787 inodedep->id_state &= ~ONDEPLIST; 5788 inodedep->id_state |= ALLCOMPLETE; 5789 inodedep->id_bmsafemap = NULL; 5790 if (inodedep->id_state & ONWORKLIST) 5791 WORKLIST_REMOVE(&inodedep->id_list); 5792 if (inodedep->id_savedino1 != NULL) { 5793 free(inodedep->id_savedino1, M_SAVEDINO); 5794 inodedep->id_savedino1 = NULL; 5795 } 5796 if (free_inodedep(inodedep) == 0) 5797 panic("check_inode_unwritten: busy inode"); 5798 return (1); 5799 } 5800 5801 /* 5802 * Try to free an inodedep structure. Return 1 if it could be freed. 5803 */ 5804 static int 5805 free_inodedep(inodedep) 5806 struct inodedep *inodedep; 5807 { 5808 5809 mtx_assert(&lk, MA_OWNED); 5810 if ((inodedep->id_state & (ONWORKLIST | UNLINKED)) != 0 || 5811 (inodedep->id_state & ALLCOMPLETE) != ALLCOMPLETE || 5812 !LIST_EMPTY(&inodedep->id_dirremhd) || 5813 !LIST_EMPTY(&inodedep->id_pendinghd) || 5814 !LIST_EMPTY(&inodedep->id_bufwait) || 5815 !LIST_EMPTY(&inodedep->id_inowait) || 5816 !TAILQ_EMPTY(&inodedep->id_inoreflst) || 5817 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 5818 !TAILQ_EMPTY(&inodedep->id_newinoupdt) || 5819 !TAILQ_EMPTY(&inodedep->id_extupdt) || 5820 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 5821 inodedep->id_mkdiradd != NULL || 5822 inodedep->id_nlinkdelta != 0 || 5823 inodedep->id_savedino1 != NULL) 5824 return (0); 5825 if (inodedep->id_state & ONDEPLIST) 5826 LIST_REMOVE(inodedep, id_deps); 5827 LIST_REMOVE(inodedep, id_hash); 5828 WORKITEM_FREE(inodedep, D_INODEDEP); 5829 num_inodedep -= 1; 5830 return (1); 5831 } 5832 5833 /* 5834 * Free the block referenced by a freework structure. The parent freeblks 5835 * structure is released and completed when the final cg bitmap reaches 5836 * the disk. This routine may be freeing a jnewblk which never made it to 5837 * disk in which case we do not have to wait as the operation is undone 5838 * in memory immediately. 5839 */ 5840 static void 5841 freework_freeblock(freework) 5842 struct freework *freework; 5843 { 5844 struct freeblks *freeblks; 5845 struct ufsmount *ump; 5846 struct workhead wkhd; 5847 struct fs *fs; 5848 int complete; 5849 int pending; 5850 int bsize; 5851 int needj; 5852 5853 freeblks = freework->fw_freeblks; 5854 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 5855 fs = ump->um_fs; 5856 needj = freeblks->fb_list.wk_mp->mnt_kern_flag & MNTK_SUJ; 5857 complete = 0; 5858 LIST_INIT(&wkhd); 5859 /* 5860 * If we are canceling an existing jnewblk pass it to the free 5861 * routine, otherwise pass the freeblk which will ultimately 5862 * release the freeblks. If we're not journaling, we can just 5863 * free the freeblks immediately. 5864 */ 5865 if (!LIST_EMPTY(&freework->fw_jwork)) { 5866 LIST_SWAP(&wkhd, &freework->fw_jwork, worklist, wk_list); 5867 complete = 1; 5868 } else if (needj) 5869 WORKLIST_INSERT_UNLOCKED(&wkhd, &freework->fw_list); 5870 bsize = lfragtosize(fs, freework->fw_frags); 5871 pending = btodb(bsize); 5872 ACQUIRE_LOCK(&lk); 5873 freeblks->fb_chkcnt -= pending; 5874 FREE_LOCK(&lk); 5875 /* 5876 * extattr blocks don't show up in pending blocks. XXX why? 5877 */ 5878 if (freework->fw_lbn >= 0 || freework->fw_lbn <= -NDADDR) { 5879 UFS_LOCK(ump); 5880 fs->fs_pendingblocks -= pending; 5881 UFS_UNLOCK(ump); 5882 } 5883 ffs_blkfree(ump, fs, freeblks->fb_devvp, freework->fw_blkno, 5884 bsize, freeblks->fb_previousinum, &wkhd); 5885 if (complete == 0 && needj) 5886 return; 5887 /* 5888 * The jnewblk will be discarded and the bits in the map never 5889 * made it to disk. We can immediately free the freeblk. 5890 */ 5891 ACQUIRE_LOCK(&lk); 5892 handle_written_freework(freework); 5893 FREE_LOCK(&lk); 5894 } 5895 5896 /* 5897 * Start, continue, or finish the process of freeing an indirect block tree. 5898 * The free operation may be paused at any point with fw_off containing the 5899 * offset to restart from. This enables us to implement some flow control 5900 * for large truncates which may fan out and generate a huge number of 5901 * dependencies. 5902 */ 5903 static void 5904 handle_workitem_indirblk(freework) 5905 struct freework *freework; 5906 { 5907 struct freeblks *freeblks; 5908 struct ufsmount *ump; 5909 struct fs *fs; 5910 5911 5912 freeblks = freework->fw_freeblks; 5913 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 5914 fs = ump->um_fs; 5915 if (freework->fw_off == NINDIR(fs)) 5916 freework_freeblock(freework); 5917 else 5918 indir_trunc(freework, fsbtodb(fs, freework->fw_blkno), 5919 freework->fw_lbn); 5920 } 5921 5922 /* 5923 * Called when a freework structure attached to a cg buf is written. The 5924 * ref on either the parent or the freeblks structure is released and 5925 * either may be added to the worklist if it is the final ref. 5926 */ 5927 static void 5928 handle_written_freework(freework) 5929 struct freework *freework; 5930 { 5931 struct freeblks *freeblks; 5932 struct freework *parent; 5933 5934 freeblks = freework->fw_freeblks; 5935 parent = freework->fw_parent; 5936 if (parent) { 5937 if (--parent->fw_ref != 0) 5938 parent = NULL; 5939 freeblks = NULL; 5940 } else if (--freeblks->fb_ref != 0) 5941 freeblks = NULL; 5942 WORKITEM_FREE(freework, D_FREEWORK); 5943 /* 5944 * Don't delay these block frees or it takes an intolerable amount 5945 * of time to process truncates and free their journal entries. 5946 */ 5947 if (freeblks) 5948 add_to_worklist(&freeblks->fb_list, 1); 5949 if (parent) 5950 add_to_worklist(&parent->fw_list, 1); 5951 } 5952 5953 /* 5954 * This workitem routine performs the block de-allocation. 5955 * The workitem is added to the pending list after the updated 5956 * inode block has been written to disk. As mentioned above, 5957 * checks regarding the number of blocks de-allocated (compared 5958 * to the number of blocks allocated for the file) are also 5959 * performed in this function. 5960 */ 5961 static void 5962 handle_workitem_freeblocks(freeblks, flags) 5963 struct freeblks *freeblks; 5964 int flags; 5965 { 5966 struct freework *freework; 5967 struct worklist *wk; 5968 5969 KASSERT(LIST_EMPTY(&freeblks->fb_jfreeblkhd), 5970 ("handle_workitem_freeblocks: Journal entries not written.")); 5971 if (LIST_EMPTY(&freeblks->fb_freeworkhd)) { 5972 handle_complete_freeblocks(freeblks); 5973 return; 5974 } 5975 freeblks->fb_ref++; 5976 while ((wk = LIST_FIRST(&freeblks->fb_freeworkhd)) != NULL) { 5977 KASSERT(wk->wk_type == D_FREEWORK, 5978 ("handle_workitem_freeblocks: Unknown type %s", 5979 TYPENAME(wk->wk_type))); 5980 WORKLIST_REMOVE_UNLOCKED(wk); 5981 freework = WK_FREEWORK(wk); 5982 if (freework->fw_lbn <= -NDADDR) 5983 handle_workitem_indirblk(freework); 5984 else 5985 freework_freeblock(freework); 5986 } 5987 ACQUIRE_LOCK(&lk); 5988 if (--freeblks->fb_ref != 0) 5989 freeblks = NULL; 5990 FREE_LOCK(&lk); 5991 if (freeblks) 5992 handle_complete_freeblocks(freeblks); 5993 } 5994 5995 /* 5996 * Once all of the freework workitems are complete we can retire the 5997 * freeblocks dependency and any journal work awaiting completion. This 5998 * can not be called until all other dependencies are stable on disk. 5999 */ 6000 static void 6001 handle_complete_freeblocks(freeblks) 6002 struct freeblks *freeblks; 6003 { 6004 struct inode *ip; 6005 struct vnode *vp; 6006 struct fs *fs; 6007 struct ufsmount *ump; 6008 int flags; 6009 6010 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 6011 fs = ump->um_fs; 6012 flags = LK_NOWAIT; 6013 6014 /* 6015 * If we still have not finished background cleanup, then check 6016 * to see if the block count needs to be adjusted. 6017 */ 6018 if (freeblks->fb_chkcnt != 0 && (fs->fs_flags & FS_UNCLEAN) != 0 && 6019 ffs_vgetf(freeblks->fb_list.wk_mp, freeblks->fb_previousinum, 6020 (flags & LK_NOWAIT) | LK_EXCLUSIVE, &vp, FFSV_FORCEINSMQ) == 0) { 6021 ip = VTOI(vp); 6022 DIP_SET(ip, i_blocks, DIP(ip, i_blocks) + freeblks->fb_chkcnt); 6023 ip->i_flag |= IN_CHANGE; 6024 vput(vp); 6025 } 6026 6027 KASSERT(freeblks->fb_chkcnt == 0 || 6028 ((fs->fs_flags & FS_UNCLEAN) != 0 && (flags & LK_NOWAIT) == 0), 6029 ("handle_workitem_freeblocks: inode %ju block count %jd\n", 6030 (uintmax_t)freeblks->fb_previousinum, 6031 (intmax_t)freeblks->fb_chkcnt)); 6032 6033 ACQUIRE_LOCK(&lk); 6034 /* 6035 * All of the freeblock deps must be complete prior to this call 6036 * so it's now safe to complete earlier outstanding journal entries. 6037 */ 6038 handle_jwork(&freeblks->fb_jwork); 6039 WORKITEM_FREE(freeblks, D_FREEBLKS); 6040 num_freeblkdep--; 6041 FREE_LOCK(&lk); 6042 } 6043 6044 /* 6045 * Release blocks associated with the inode ip and stored in the indirect 6046 * block dbn. If level is greater than SINGLE, the block is an indirect block 6047 * and recursive calls to indirtrunc must be used to cleanse other indirect 6048 * blocks. 6049 */ 6050 static void 6051 indir_trunc(freework, dbn, lbn) 6052 struct freework *freework; 6053 ufs2_daddr_t dbn; 6054 ufs_lbn_t lbn; 6055 { 6056 struct freework *nfreework; 6057 struct workhead wkhd; 6058 struct jnewblk *jnewblk; 6059 struct freeblks *freeblks; 6060 struct buf *bp; 6061 struct fs *fs; 6062 struct worklist *wkn; 6063 struct worklist *wk; 6064 struct indirdep *indirdep; 6065 struct ufsmount *ump; 6066 ufs1_daddr_t *bap1 = 0; 6067 ufs2_daddr_t nb, nnb, *bap2 = 0; 6068 ufs_lbn_t lbnadd; 6069 int i, nblocks, ufs1fmt; 6070 int fs_pendingblocks; 6071 int freedeps; 6072 int needj; 6073 int level; 6074 int cnt; 6075 6076 LIST_INIT(&wkhd); 6077 level = lbn_level(lbn); 6078 if (level == -1) 6079 panic("indir_trunc: Invalid lbn %jd\n", lbn); 6080 freeblks = freework->fw_freeblks; 6081 ump = VFSTOUFS(freeblks->fb_list.wk_mp); 6082 fs = ump->um_fs; 6083 fs_pendingblocks = 0; 6084 freedeps = 0; 6085 needj = UFSTOVFS(ump)->mnt_kern_flag & MNTK_SUJ; 6086 lbnadd = lbn_offset(fs, level); 6087 /* 6088 * Get buffer of block pointers to be freed. This routine is not 6089 * called until the zero'ed inode has been written, so it is safe 6090 * to free blocks as they are encountered. Because the inode has 6091 * been zero'ed, calls to bmap on these blocks will fail. So, we 6092 * have to use the on-disk address and the block device for the 6093 * filesystem to look them up. If the file was deleted before its 6094 * indirect blocks were all written to disk, the routine that set 6095 * us up (deallocate_dependencies) will have arranged to leave 6096 * a complete copy of the indirect block in memory for our use. 6097 * Otherwise we have to read the blocks in from the disk. 6098 */ 6099 #ifdef notyet 6100 bp = getblk(freeblks->fb_devvp, dbn, (int)fs->fs_bsize, 0, 0, 6101 GB_NOCREAT); 6102 #else 6103 bp = incore(&freeblks->fb_devvp->v_bufobj, dbn); 6104 #endif 6105 ACQUIRE_LOCK(&lk); 6106 if (bp != NULL && (wk = LIST_FIRST(&bp->b_dep)) != NULL) { 6107 if (wk->wk_type != D_INDIRDEP || 6108 (wk->wk_state & GOINGAWAY) == 0) 6109 panic("indir_trunc: lost indirdep %p", wk); 6110 indirdep = WK_INDIRDEP(wk); 6111 LIST_SWAP(&wkhd, &indirdep->ir_jwork, worklist, wk_list); 6112 free_indirdep(indirdep); 6113 if (!LIST_EMPTY(&bp->b_dep)) 6114 panic("indir_trunc: dangling dep %p", 6115 LIST_FIRST(&bp->b_dep)); 6116 ump->um_numindirdeps -= 1; 6117 FREE_LOCK(&lk); 6118 } else { 6119 #ifdef notyet 6120 if (bp) 6121 brelse(bp); 6122 #endif 6123 FREE_LOCK(&lk); 6124 if (bread(freeblks->fb_devvp, dbn, (int)fs->fs_bsize, 6125 NOCRED, &bp) != 0) { 6126 brelse(bp); 6127 return; 6128 } 6129 } 6130 /* 6131 * Recursively free indirect blocks. 6132 */ 6133 if (ump->um_fstype == UFS1) { 6134 ufs1fmt = 1; 6135 bap1 = (ufs1_daddr_t *)bp->b_data; 6136 } else { 6137 ufs1fmt = 0; 6138 bap2 = (ufs2_daddr_t *)bp->b_data; 6139 } 6140 /* 6141 * Reclaim indirect blocks which never made it to disk. 6142 */ 6143 cnt = 0; 6144 LIST_FOREACH_SAFE(wk, &wkhd, wk_list, wkn) { 6145 struct workhead freewk; 6146 if (wk->wk_type != D_JNEWBLK) 6147 continue; 6148 WORKLIST_REMOVE_UNLOCKED(wk); 6149 LIST_INIT(&freewk); 6150 WORKLIST_INSERT_UNLOCKED(&freewk, wk); 6151 jnewblk = WK_JNEWBLK(wk); 6152 if (jnewblk->jn_lbn > 0) 6153 i = (jnewblk->jn_lbn - -lbn) / lbnadd; 6154 else 6155 i = (jnewblk->jn_lbn - (lbn + 1)) / lbnadd; 6156 KASSERT(i >= 0 && i < NINDIR(fs), 6157 ("indir_trunc: Index out of range %d parent %jd lbn %jd", 6158 i, lbn, jnewblk->jn_lbn)); 6159 /* Clear the pointer so it isn't found below. */ 6160 if (ufs1fmt) { 6161 nb = bap1[i]; 6162 bap1[i] = 0; 6163 } else { 6164 nb = bap2[i]; 6165 bap2[i] = 0; 6166 } 6167 KASSERT(nb == jnewblk->jn_blkno, 6168 ("indir_trunc: Block mismatch %jd != %jd", 6169 nb, jnewblk->jn_blkno)); 6170 ffs_blkfree(ump, fs, freeblks->fb_devvp, jnewblk->jn_blkno, 6171 fs->fs_bsize, freeblks->fb_previousinum, &freewk); 6172 cnt++; 6173 } 6174 ACQUIRE_LOCK(&lk); 6175 if (needj) 6176 freework->fw_ref += NINDIR(fs) + 1; 6177 /* Any remaining journal work can be completed with freeblks. */ 6178 jwork_move(&freeblks->fb_jwork, &wkhd); 6179 FREE_LOCK(&lk); 6180 nblocks = btodb(fs->fs_bsize); 6181 if (ufs1fmt) 6182 nb = bap1[0]; 6183 else 6184 nb = bap2[0]; 6185 nfreework = freework; 6186 /* 6187 * Reclaim on disk blocks. 6188 */ 6189 for (i = freework->fw_off; i < NINDIR(fs); i++, nb = nnb) { 6190 if (i != NINDIR(fs) - 1) { 6191 if (ufs1fmt) 6192 nnb = bap1[i+1]; 6193 else 6194 nnb = bap2[i+1]; 6195 } else 6196 nnb = 0; 6197 if (nb == 0) 6198 continue; 6199 cnt++; 6200 if (level != 0) { 6201 ufs_lbn_t nlbn; 6202 6203 nlbn = (lbn + 1) - (i * lbnadd); 6204 if (needj != 0) { 6205 nfreework = newfreework(freeblks, freework, 6206 nlbn, nb, fs->fs_frag, 0); 6207 freedeps++; 6208 } 6209 indir_trunc(nfreework, fsbtodb(fs, nb), nlbn); 6210 } else { 6211 struct freedep *freedep; 6212 6213 /* 6214 * Attempt to aggregate freedep dependencies for 6215 * all blocks being released to the same CG. 6216 */ 6217 LIST_INIT(&wkhd); 6218 if (needj != 0 && 6219 (nnb == 0 || (dtog(fs, nb) != dtog(fs, nnb)))) { 6220 freedep = newfreedep(freework); 6221 WORKLIST_INSERT_UNLOCKED(&wkhd, 6222 &freedep->fd_list); 6223 freedeps++; 6224 } 6225 ffs_blkfree(ump, fs, freeblks->fb_devvp, nb, 6226 fs->fs_bsize, freeblks->fb_previousinum, &wkhd); 6227 } 6228 } 6229 if (level == 0) 6230 fs_pendingblocks = (nblocks * cnt); 6231 /* 6232 * If we're not journaling we can free the indirect now. Otherwise 6233 * setup the ref counts and offset so this indirect can be completed 6234 * when its children are free. 6235 */ 6236 if (needj == 0) { 6237 fs_pendingblocks += nblocks; 6238 dbn = dbtofsb(fs, dbn); 6239 ffs_blkfree(ump, fs, freeblks->fb_devvp, dbn, fs->fs_bsize, 6240 freeblks->fb_previousinum, NULL); 6241 ACQUIRE_LOCK(&lk); 6242 freeblks->fb_chkcnt -= fs_pendingblocks; 6243 if (freework->fw_blkno == dbn) 6244 handle_written_freework(freework); 6245 FREE_LOCK(&lk); 6246 freework = NULL; 6247 } else { 6248 ACQUIRE_LOCK(&lk); 6249 freework->fw_off = i; 6250 freework->fw_ref += freedeps; 6251 freework->fw_ref -= NINDIR(fs) + 1; 6252 if (freework->fw_ref != 0) 6253 freework = NULL; 6254 freeblks->fb_chkcnt -= fs_pendingblocks; 6255 FREE_LOCK(&lk); 6256 } 6257 if (fs_pendingblocks) { 6258 UFS_LOCK(ump); 6259 fs->fs_pendingblocks -= fs_pendingblocks; 6260 UFS_UNLOCK(ump); 6261 } 6262 bp->b_flags |= B_INVAL | B_NOCACHE; 6263 brelse(bp); 6264 if (freework) 6265 handle_workitem_indirblk(freework); 6266 return; 6267 } 6268 6269 /* 6270 * Cancel an allocindir when it is removed via truncation. 6271 */ 6272 static void 6273 cancel_allocindir(aip, inodedep, freeblks) 6274 struct allocindir *aip; 6275 struct inodedep *inodedep; 6276 struct freeblks *freeblks; 6277 { 6278 struct newblk *newblk; 6279 6280 /* 6281 * If the journal hasn't been written the jnewblk must be passed 6282 * to the call to ffs_blkfree that reclaims the space. We accomplish 6283 * this by linking the journal dependency into the indirdep to be 6284 * freed when indir_trunc() is called. If the journal has already 6285 * been written we can simply reclaim the journal space when the 6286 * freeblks work is complete. 6287 */ 6288 LIST_REMOVE(aip, ai_next); 6289 newblk = (struct newblk *)aip; 6290 if (newblk->nb_jnewblk == NULL) 6291 cancel_newblk(newblk, &freeblks->fb_jwork); 6292 else 6293 cancel_newblk(newblk, &aip->ai_indirdep->ir_jwork); 6294 if (inodedep && inodedep->id_state & DEPCOMPLETE) 6295 WORKLIST_INSERT(&inodedep->id_bufwait, &newblk->nb_list); 6296 else 6297 free_newblk(newblk); 6298 } 6299 6300 /* 6301 * Create the mkdir dependencies for . and .. in a new directory. Link them 6302 * in to a newdirblk so any subsequent additions are tracked properly. The 6303 * caller is responsible for adding the mkdir1 dependency to the journal 6304 * and updating id_mkdiradd. This function returns with lk held. 6305 */ 6306 static struct mkdir * 6307 setup_newdir(dap, newinum, dinum, newdirbp, mkdirp) 6308 struct diradd *dap; 6309 ino_t newinum; 6310 ino_t dinum; 6311 struct buf *newdirbp; 6312 struct mkdir **mkdirp; 6313 { 6314 struct newblk *newblk; 6315 struct pagedep *pagedep; 6316 struct inodedep *inodedep; 6317 struct newdirblk *newdirblk = 0; 6318 struct mkdir *mkdir1, *mkdir2; 6319 struct worklist *wk; 6320 struct jaddref *jaddref; 6321 struct mount *mp; 6322 6323 mp = dap->da_list.wk_mp; 6324 newdirblk = malloc(sizeof(struct newdirblk), M_NEWDIRBLK, 6325 M_SOFTDEP_FLAGS); 6326 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 6327 LIST_INIT(&newdirblk->db_mkdir); 6328 mkdir1 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 6329 workitem_alloc(&mkdir1->md_list, D_MKDIR, mp); 6330 mkdir1->md_state = ATTACHED | MKDIR_BODY; 6331 mkdir1->md_diradd = dap; 6332 mkdir1->md_jaddref = NULL; 6333 mkdir2 = malloc(sizeof(struct mkdir), M_MKDIR, M_SOFTDEP_FLAGS); 6334 workitem_alloc(&mkdir2->md_list, D_MKDIR, mp); 6335 mkdir2->md_state = ATTACHED | MKDIR_PARENT; 6336 mkdir2->md_diradd = dap; 6337 mkdir2->md_jaddref = NULL; 6338 if ((mp->mnt_kern_flag & MNTK_SUJ) == 0) { 6339 mkdir1->md_state |= DEPCOMPLETE; 6340 mkdir2->md_state |= DEPCOMPLETE; 6341 } 6342 /* 6343 * Dependency on "." and ".." being written to disk. 6344 */ 6345 mkdir1->md_buf = newdirbp; 6346 ACQUIRE_LOCK(&lk); 6347 LIST_INSERT_HEAD(&mkdirlisthd, mkdir1, md_mkdirs); 6348 /* 6349 * We must link the pagedep, allocdirect, and newdirblk for 6350 * the initial file page so the pointer to the new directory 6351 * is not written until the directory contents are live and 6352 * any subsequent additions are not marked live until the 6353 * block is reachable via the inode. 6354 */ 6355 if (pagedep_lookup(mp, newinum, 0, 0, &pagedep) == 0) 6356 panic("setup_newdir: lost pagedep"); 6357 LIST_FOREACH(wk, &newdirbp->b_dep, wk_list) 6358 if (wk->wk_type == D_ALLOCDIRECT) 6359 break; 6360 if (wk == NULL) 6361 panic("setup_newdir: lost allocdirect"); 6362 newblk = WK_NEWBLK(wk); 6363 pagedep->pd_state |= NEWBLOCK; 6364 pagedep->pd_newdirblk = newdirblk; 6365 newdirblk->db_pagedep = pagedep; 6366 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 6367 WORKLIST_INSERT(&newdirblk->db_mkdir, &mkdir1->md_list); 6368 /* 6369 * Look up the inodedep for the parent directory so that we 6370 * can link mkdir2 into the pending dotdot jaddref or 6371 * the inode write if there is none. If the inode is 6372 * ALLCOMPLETE and no jaddref is present all dependencies have 6373 * been satisfied and mkdir2 can be freed. 6374 */ 6375 inodedep_lookup(mp, dinum, 0, &inodedep); 6376 if (mp->mnt_kern_flag & MNTK_SUJ) { 6377 if (inodedep == NULL) 6378 panic("setup_newdir: Lost parent."); 6379 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 6380 inoreflst); 6381 KASSERT(jaddref != NULL && jaddref->ja_parent == newinum && 6382 (jaddref->ja_state & MKDIR_PARENT), 6383 ("setup_newdir: bad dotdot jaddref %p", jaddref)); 6384 LIST_INSERT_HEAD(&mkdirlisthd, mkdir2, md_mkdirs); 6385 mkdir2->md_jaddref = jaddref; 6386 jaddref->ja_mkdir = mkdir2; 6387 } else if (inodedep == NULL || 6388 (inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 6389 dap->da_state &= ~MKDIR_PARENT; 6390 WORKITEM_FREE(mkdir2, D_MKDIR); 6391 } else { 6392 LIST_INSERT_HEAD(&mkdirlisthd, mkdir2, md_mkdirs); 6393 WORKLIST_INSERT(&inodedep->id_bufwait,&mkdir2->md_list); 6394 } 6395 *mkdirp = mkdir2; 6396 6397 return (mkdir1); 6398 } 6399 6400 /* 6401 * Directory entry addition dependencies. 6402 * 6403 * When adding a new directory entry, the inode (with its incremented link 6404 * count) must be written to disk before the directory entry's pointer to it. 6405 * Also, if the inode is newly allocated, the corresponding freemap must be 6406 * updated (on disk) before the directory entry's pointer. These requirements 6407 * are met via undo/redo on the directory entry's pointer, which consists 6408 * simply of the inode number. 6409 * 6410 * As directory entries are added and deleted, the free space within a 6411 * directory block can become fragmented. The ufs filesystem will compact 6412 * a fragmented directory block to make space for a new entry. When this 6413 * occurs, the offsets of previously added entries change. Any "diradd" 6414 * dependency structures corresponding to these entries must be updated with 6415 * the new offsets. 6416 */ 6417 6418 /* 6419 * This routine is called after the in-memory inode's link 6420 * count has been incremented, but before the directory entry's 6421 * pointer to the inode has been set. 6422 */ 6423 int 6424 softdep_setup_directory_add(bp, dp, diroffset, newinum, newdirbp, isnewblk) 6425 struct buf *bp; /* buffer containing directory block */ 6426 struct inode *dp; /* inode for directory */ 6427 off_t diroffset; /* offset of new entry in directory */ 6428 ino_t newinum; /* inode referenced by new directory entry */ 6429 struct buf *newdirbp; /* non-NULL => contents of new mkdir */ 6430 int isnewblk; /* entry is in a newly allocated block */ 6431 { 6432 int offset; /* offset of new entry within directory block */ 6433 ufs_lbn_t lbn; /* block in directory containing new entry */ 6434 struct fs *fs; 6435 struct diradd *dap; 6436 struct newblk *newblk; 6437 struct pagedep *pagedep; 6438 struct inodedep *inodedep; 6439 struct newdirblk *newdirblk = 0; 6440 struct mkdir *mkdir1, *mkdir2; 6441 struct jaddref *jaddref; 6442 struct mount *mp; 6443 int isindir; 6444 6445 /* 6446 * Whiteouts have no dependencies. 6447 */ 6448 if (newinum == WINO) { 6449 if (newdirbp != NULL) 6450 bdwrite(newdirbp); 6451 return (0); 6452 } 6453 jaddref = NULL; 6454 mkdir1 = mkdir2 = NULL; 6455 mp = UFSTOVFS(dp->i_ump); 6456 fs = dp->i_fs; 6457 lbn = lblkno(fs, diroffset); 6458 offset = blkoff(fs, diroffset); 6459 dap = malloc(sizeof(struct diradd), M_DIRADD, 6460 M_SOFTDEP_FLAGS|M_ZERO); 6461 workitem_alloc(&dap->da_list, D_DIRADD, mp); 6462 dap->da_offset = offset; 6463 dap->da_newinum = newinum; 6464 dap->da_state = ATTACHED; 6465 LIST_INIT(&dap->da_jwork); 6466 isindir = bp->b_lblkno >= NDADDR; 6467 if (isnewblk && 6468 (isindir ? blkoff(fs, diroffset) : fragoff(fs, diroffset)) == 0) { 6469 newdirblk = malloc(sizeof(struct newdirblk), 6470 M_NEWDIRBLK, M_SOFTDEP_FLAGS); 6471 workitem_alloc(&newdirblk->db_list, D_NEWDIRBLK, mp); 6472 LIST_INIT(&newdirblk->db_mkdir); 6473 } 6474 /* 6475 * If we're creating a new directory setup the dependencies and set 6476 * the dap state to wait for them. Otherwise it's COMPLETE and 6477 * we can move on. 6478 */ 6479 if (newdirbp == NULL) { 6480 dap->da_state |= DEPCOMPLETE; 6481 ACQUIRE_LOCK(&lk); 6482 } else { 6483 dap->da_state |= MKDIR_BODY | MKDIR_PARENT; 6484 mkdir1 = setup_newdir(dap, newinum, dp->i_number, newdirbp, 6485 &mkdir2); 6486 } 6487 /* 6488 * Link into parent directory pagedep to await its being written. 6489 */ 6490 if (pagedep_lookup(mp, dp->i_number, lbn, DEPALLOC, &pagedep) == 0) 6491 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 6492 #ifdef DEBUG 6493 if (diradd_lookup(pagedep, offset) != NULL) 6494 panic("softdep_setup_directory_add: %p already at off %d\n", 6495 diradd_lookup(pagedep, offset), offset); 6496 #endif 6497 dap->da_pagedep = pagedep; 6498 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], dap, 6499 da_pdlist); 6500 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 6501 /* 6502 * If we're journaling, link the diradd into the jaddref so it 6503 * may be completed after the journal entry is written. Otherwise, 6504 * link the diradd into its inodedep. If the inode is not yet 6505 * written place it on the bufwait list, otherwise do the post-inode 6506 * write processing to put it on the id_pendinghd list. 6507 */ 6508 if (mp->mnt_kern_flag & MNTK_SUJ) { 6509 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 6510 inoreflst); 6511 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 6512 ("softdep_setup_directory_add: bad jaddref %p", jaddref)); 6513 jaddref->ja_diroff = diroffset; 6514 jaddref->ja_diradd = dap; 6515 add_to_journal(&jaddref->ja_list); 6516 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) 6517 diradd_inode_written(dap, inodedep); 6518 else 6519 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 6520 /* 6521 * Add the journal entries for . and .. links now that the primary 6522 * link is written. 6523 */ 6524 if (mkdir1 != NULL && mp->mnt_kern_flag & MNTK_SUJ) { 6525 jaddref = (struct jaddref *)TAILQ_PREV(&jaddref->ja_ref, 6526 inoreflst, if_deps); 6527 KASSERT(jaddref != NULL && 6528 jaddref->ja_ino == jaddref->ja_parent && 6529 (jaddref->ja_state & MKDIR_BODY), 6530 ("softdep_setup_directory_add: bad dot jaddref %p", 6531 jaddref)); 6532 mkdir1->md_jaddref = jaddref; 6533 jaddref->ja_mkdir = mkdir1; 6534 /* 6535 * It is important that the dotdot journal entry 6536 * is added prior to the dot entry since dot writes 6537 * both the dot and dotdot links. These both must 6538 * be added after the primary link for the journal 6539 * to remain consistent. 6540 */ 6541 add_to_journal(&mkdir2->md_jaddref->ja_list); 6542 add_to_journal(&jaddref->ja_list); 6543 } 6544 /* 6545 * If we are adding a new directory remember this diradd so that if 6546 * we rename it we can keep the dot and dotdot dependencies. If 6547 * we are adding a new name for an inode that has a mkdiradd we 6548 * must be in rename and we have to move the dot and dotdot 6549 * dependencies to this new name. The old name is being orphaned 6550 * soon. 6551 */ 6552 if (mkdir1 != NULL) { 6553 if (inodedep->id_mkdiradd != NULL) 6554 panic("softdep_setup_directory_add: Existing mkdir"); 6555 inodedep->id_mkdiradd = dap; 6556 } else if (inodedep->id_mkdiradd) 6557 merge_diradd(inodedep, dap); 6558 if (newdirblk) { 6559 /* 6560 * There is nothing to do if we are already tracking 6561 * this block. 6562 */ 6563 if ((pagedep->pd_state & NEWBLOCK) != 0) { 6564 WORKITEM_FREE(newdirblk, D_NEWDIRBLK); 6565 FREE_LOCK(&lk); 6566 return (0); 6567 } 6568 if (newblk_lookup(mp, dbtofsb(fs, bp->b_blkno), 0, &newblk) 6569 == 0) 6570 panic("softdep_setup_directory_add: lost entry"); 6571 WORKLIST_INSERT(&newblk->nb_newdirblk, &newdirblk->db_list); 6572 pagedep->pd_state |= NEWBLOCK; 6573 pagedep->pd_newdirblk = newdirblk; 6574 newdirblk->db_pagedep = pagedep; 6575 FREE_LOCK(&lk); 6576 /* 6577 * If we extended into an indirect signal direnter to sync. 6578 */ 6579 if (isindir) 6580 return (1); 6581 return (0); 6582 } 6583 FREE_LOCK(&lk); 6584 return (0); 6585 } 6586 6587 /* 6588 * This procedure is called to change the offset of a directory 6589 * entry when compacting a directory block which must be owned 6590 * exclusively by the caller. Note that the actual entry movement 6591 * must be done in this procedure to ensure that no I/O completions 6592 * occur while the move is in progress. 6593 */ 6594 void 6595 softdep_change_directoryentry_offset(bp, dp, base, oldloc, newloc, entrysize) 6596 struct buf *bp; /* Buffer holding directory block. */ 6597 struct inode *dp; /* inode for directory */ 6598 caddr_t base; /* address of dp->i_offset */ 6599 caddr_t oldloc; /* address of old directory location */ 6600 caddr_t newloc; /* address of new directory location */ 6601 int entrysize; /* size of directory entry */ 6602 { 6603 int offset, oldoffset, newoffset; 6604 struct pagedep *pagedep; 6605 struct jmvref *jmvref; 6606 struct diradd *dap; 6607 struct direct *de; 6608 struct mount *mp; 6609 ufs_lbn_t lbn; 6610 int flags; 6611 6612 mp = UFSTOVFS(dp->i_ump); 6613 de = (struct direct *)oldloc; 6614 jmvref = NULL; 6615 flags = 0; 6616 /* 6617 * Moves are always journaled as it would be too complex to 6618 * determine if any affected adds or removes are present in the 6619 * journal. 6620 */ 6621 if (mp->mnt_kern_flag & MNTK_SUJ) { 6622 flags = DEPALLOC; 6623 jmvref = newjmvref(dp, de->d_ino, 6624 dp->i_offset + (oldloc - base), 6625 dp->i_offset + (newloc - base)); 6626 } 6627 lbn = lblkno(dp->i_fs, dp->i_offset); 6628 offset = blkoff(dp->i_fs, dp->i_offset); 6629 oldoffset = offset + (oldloc - base); 6630 newoffset = offset + (newloc - base); 6631 ACQUIRE_LOCK(&lk); 6632 if (pagedep_lookup(mp, dp->i_number, lbn, flags, &pagedep) == 0) { 6633 if (pagedep) 6634 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 6635 goto done; 6636 } 6637 dap = diradd_lookup(pagedep, oldoffset); 6638 if (dap) { 6639 dap->da_offset = newoffset; 6640 newoffset = DIRADDHASH(newoffset); 6641 oldoffset = DIRADDHASH(oldoffset); 6642 if ((dap->da_state & ALLCOMPLETE) != ALLCOMPLETE && 6643 newoffset != oldoffset) { 6644 LIST_REMOVE(dap, da_pdlist); 6645 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[newoffset], 6646 dap, da_pdlist); 6647 } 6648 } 6649 done: 6650 if (jmvref) { 6651 jmvref->jm_pagedep = pagedep; 6652 LIST_INSERT_HEAD(&pagedep->pd_jmvrefhd, jmvref, jm_deps); 6653 add_to_journal(&jmvref->jm_list); 6654 } 6655 bcopy(oldloc, newloc, entrysize); 6656 FREE_LOCK(&lk); 6657 } 6658 6659 /* 6660 * Move the mkdir dependencies and journal work from one diradd to another 6661 * when renaming a directory. The new name must depend on the mkdir deps 6662 * completing as the old name did. Directories can only have one valid link 6663 * at a time so one must be canonical. 6664 */ 6665 static void 6666 merge_diradd(inodedep, newdap) 6667 struct inodedep *inodedep; 6668 struct diradd *newdap; 6669 { 6670 struct diradd *olddap; 6671 struct mkdir *mkdir, *nextmd; 6672 short state; 6673 6674 olddap = inodedep->id_mkdiradd; 6675 inodedep->id_mkdiradd = newdap; 6676 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 6677 newdap->da_state &= ~DEPCOMPLETE; 6678 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; mkdir = nextmd) { 6679 nextmd = LIST_NEXT(mkdir, md_mkdirs); 6680 if (mkdir->md_diradd != olddap) 6681 continue; 6682 mkdir->md_diradd = newdap; 6683 state = mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY); 6684 newdap->da_state |= state; 6685 olddap->da_state &= ~state; 6686 if ((olddap->da_state & 6687 (MKDIR_PARENT | MKDIR_BODY)) == 0) 6688 break; 6689 } 6690 if ((olddap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 6691 panic("merge_diradd: unfound ref"); 6692 } 6693 /* 6694 * Any mkdir related journal items are not safe to be freed until 6695 * the new name is stable. 6696 */ 6697 jwork_move(&newdap->da_jwork, &olddap->da_jwork); 6698 olddap->da_state |= DEPCOMPLETE; 6699 complete_diradd(olddap); 6700 } 6701 6702 /* 6703 * Move the diradd to the pending list when all diradd dependencies are 6704 * complete. 6705 */ 6706 static void 6707 complete_diradd(dap) 6708 struct diradd *dap; 6709 { 6710 struct pagedep *pagedep; 6711 6712 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 6713 if (dap->da_state & DIRCHG) 6714 pagedep = dap->da_previous->dm_pagedep; 6715 else 6716 pagedep = dap->da_pagedep; 6717 LIST_REMOVE(dap, da_pdlist); 6718 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 6719 } 6720 } 6721 6722 /* 6723 * Cancel a diradd when a dirrem overlaps with it. We must cancel the journal 6724 * add entries and conditonally journal the remove. 6725 */ 6726 static void 6727 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref) 6728 struct diradd *dap; 6729 struct dirrem *dirrem; 6730 struct jremref *jremref; 6731 struct jremref *dotremref; 6732 struct jremref *dotdotremref; 6733 { 6734 struct inodedep *inodedep; 6735 struct jaddref *jaddref; 6736 struct inoref *inoref; 6737 struct mkdir *mkdir; 6738 6739 /* 6740 * If no remove references were allocated we're on a non-journaled 6741 * filesystem and can skip the cancel step. 6742 */ 6743 if (jremref == NULL) { 6744 free_diradd(dap, NULL); 6745 return; 6746 } 6747 /* 6748 * Cancel the primary name an free it if it does not require 6749 * journaling. 6750 */ 6751 if (inodedep_lookup(dap->da_list.wk_mp, dap->da_newinum, 6752 0, &inodedep) != 0) { 6753 /* Abort the addref that reference this diradd. */ 6754 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 6755 if (inoref->if_list.wk_type != D_JADDREF) 6756 continue; 6757 jaddref = (struct jaddref *)inoref; 6758 if (jaddref->ja_diradd != dap) 6759 continue; 6760 if (cancel_jaddref(jaddref, inodedep, 6761 &dirrem->dm_jwork) == 0) { 6762 free_jremref(jremref); 6763 jremref = NULL; 6764 } 6765 break; 6766 } 6767 } 6768 /* 6769 * Cancel subordinate names and free them if they do not require 6770 * journaling. 6771 */ 6772 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 6773 LIST_FOREACH(mkdir, &mkdirlisthd, md_mkdirs) { 6774 if (mkdir->md_diradd != dap) 6775 continue; 6776 if ((jaddref = mkdir->md_jaddref) == NULL) 6777 continue; 6778 mkdir->md_jaddref = NULL; 6779 if (mkdir->md_state & MKDIR_PARENT) { 6780 if (cancel_jaddref(jaddref, NULL, 6781 &dirrem->dm_jwork) == 0) { 6782 free_jremref(dotdotremref); 6783 dotdotremref = NULL; 6784 } 6785 } else { 6786 if (cancel_jaddref(jaddref, inodedep, 6787 &dirrem->dm_jwork) == 0) { 6788 free_jremref(dotremref); 6789 dotremref = NULL; 6790 } 6791 } 6792 } 6793 } 6794 6795 if (jremref) 6796 journal_jremref(dirrem, jremref, inodedep); 6797 if (dotremref) 6798 journal_jremref(dirrem, dotremref, inodedep); 6799 if (dotdotremref) 6800 journal_jremref(dirrem, dotdotremref, NULL); 6801 jwork_move(&dirrem->dm_jwork, &dap->da_jwork); 6802 free_diradd(dap, &dirrem->dm_jwork); 6803 } 6804 6805 /* 6806 * Free a diradd dependency structure. This routine must be called 6807 * with splbio interrupts blocked. 6808 */ 6809 static void 6810 free_diradd(dap, wkhd) 6811 struct diradd *dap; 6812 struct workhead *wkhd; 6813 { 6814 struct dirrem *dirrem; 6815 struct pagedep *pagedep; 6816 struct inodedep *inodedep; 6817 struct mkdir *mkdir, *nextmd; 6818 6819 mtx_assert(&lk, MA_OWNED); 6820 LIST_REMOVE(dap, da_pdlist); 6821 if (dap->da_state & ONWORKLIST) 6822 WORKLIST_REMOVE(&dap->da_list); 6823 if ((dap->da_state & DIRCHG) == 0) { 6824 pagedep = dap->da_pagedep; 6825 } else { 6826 dirrem = dap->da_previous; 6827 pagedep = dirrem->dm_pagedep; 6828 dirrem->dm_dirinum = pagedep->pd_ino; 6829 dirrem->dm_state |= COMPLETE; 6830 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 6831 add_to_worklist(&dirrem->dm_list, 0); 6832 } 6833 if (inodedep_lookup(pagedep->pd_list.wk_mp, dap->da_newinum, 6834 0, &inodedep) != 0) 6835 if (inodedep->id_mkdiradd == dap) 6836 inodedep->id_mkdiradd = NULL; 6837 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) { 6838 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; mkdir = nextmd) { 6839 nextmd = LIST_NEXT(mkdir, md_mkdirs); 6840 if (mkdir->md_diradd != dap) 6841 continue; 6842 dap->da_state &= 6843 ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 6844 LIST_REMOVE(mkdir, md_mkdirs); 6845 if (mkdir->md_state & ONWORKLIST) 6846 WORKLIST_REMOVE(&mkdir->md_list); 6847 if (mkdir->md_jaddref != NULL) 6848 panic("free_diradd: Unexpected jaddref"); 6849 WORKITEM_FREE(mkdir, D_MKDIR); 6850 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) 6851 break; 6852 } 6853 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) != 0) 6854 panic("free_diradd: unfound ref"); 6855 } 6856 if (inodedep) 6857 free_inodedep(inodedep); 6858 /* 6859 * Free any journal segments waiting for the directory write. 6860 */ 6861 handle_jwork(&dap->da_jwork); 6862 WORKITEM_FREE(dap, D_DIRADD); 6863 } 6864 6865 /* 6866 * Directory entry removal dependencies. 6867 * 6868 * When removing a directory entry, the entry's inode pointer must be 6869 * zero'ed on disk before the corresponding inode's link count is decremented 6870 * (possibly freeing the inode for re-use). This dependency is handled by 6871 * updating the directory entry but delaying the inode count reduction until 6872 * after the directory block has been written to disk. After this point, the 6873 * inode count can be decremented whenever it is convenient. 6874 */ 6875 6876 /* 6877 * This routine should be called immediately after removing 6878 * a directory entry. The inode's link count should not be 6879 * decremented by the calling procedure -- the soft updates 6880 * code will do this task when it is safe. 6881 */ 6882 void 6883 softdep_setup_remove(bp, dp, ip, isrmdir) 6884 struct buf *bp; /* buffer containing directory block */ 6885 struct inode *dp; /* inode for the directory being modified */ 6886 struct inode *ip; /* inode for directory entry being removed */ 6887 int isrmdir; /* indicates if doing RMDIR */ 6888 { 6889 struct dirrem *dirrem, *prevdirrem; 6890 struct inodedep *inodedep; 6891 int direct; 6892 6893 /* 6894 * Allocate a new dirrem if appropriate and ACQUIRE_LOCK. We want 6895 * newdirrem() to setup the full directory remove which requires 6896 * isrmdir > 1. 6897 */ 6898 dirrem = newdirrem(bp, dp, ip, isrmdir?2:0, &prevdirrem); 6899 /* 6900 * Add the dirrem to the inodedep's pending remove list for quick 6901 * discovery later. 6902 */ 6903 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 6904 &inodedep) == 0) 6905 panic("softdep_setup_remove: Lost inodedep."); 6906 dirrem->dm_state |= ONDEPLIST; 6907 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 6908 6909 /* 6910 * If the COMPLETE flag is clear, then there were no active 6911 * entries and we want to roll back to a zeroed entry until 6912 * the new inode is committed to disk. If the COMPLETE flag is 6913 * set then we have deleted an entry that never made it to 6914 * disk. If the entry we deleted resulted from a name change, 6915 * then the old name still resides on disk. We cannot delete 6916 * its inode (returned to us in prevdirrem) until the zeroed 6917 * directory entry gets to disk. The new inode has never been 6918 * referenced on the disk, so can be deleted immediately. 6919 */ 6920 if ((dirrem->dm_state & COMPLETE) == 0) { 6921 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, dirrem, 6922 dm_next); 6923 FREE_LOCK(&lk); 6924 } else { 6925 if (prevdirrem != NULL) 6926 LIST_INSERT_HEAD(&dirrem->dm_pagedep->pd_dirremhd, 6927 prevdirrem, dm_next); 6928 dirrem->dm_dirinum = dirrem->dm_pagedep->pd_ino; 6929 direct = LIST_EMPTY(&dirrem->dm_jremrefhd); 6930 FREE_LOCK(&lk); 6931 if (direct) 6932 handle_workitem_remove(dirrem, NULL); 6933 } 6934 } 6935 6936 /* 6937 * Check for an entry matching 'offset' on both the pd_dirraddhd list and the 6938 * pd_pendinghd list of a pagedep. 6939 */ 6940 static struct diradd * 6941 diradd_lookup(pagedep, offset) 6942 struct pagedep *pagedep; 6943 int offset; 6944 { 6945 struct diradd *dap; 6946 6947 LIST_FOREACH(dap, &pagedep->pd_diraddhd[DIRADDHASH(offset)], da_pdlist) 6948 if (dap->da_offset == offset) 6949 return (dap); 6950 LIST_FOREACH(dap, &pagedep->pd_pendinghd, da_pdlist) 6951 if (dap->da_offset == offset) 6952 return (dap); 6953 return (NULL); 6954 } 6955 6956 /* 6957 * Search for a .. diradd dependency in a directory that is being removed. 6958 * If the directory was renamed to a new parent we have a diradd rather 6959 * than a mkdir for the .. entry. We need to cancel it now before 6960 * it is found in truncate(). 6961 */ 6962 static struct jremref * 6963 cancel_diradd_dotdot(ip, dirrem, jremref) 6964 struct inode *ip; 6965 struct dirrem *dirrem; 6966 struct jremref *jremref; 6967 { 6968 struct pagedep *pagedep; 6969 struct diradd *dap; 6970 struct worklist *wk; 6971 6972 if (pagedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 0, 6973 &pagedep) == 0) 6974 return (jremref); 6975 dap = diradd_lookup(pagedep, DOTDOT_OFFSET); 6976 if (dap == NULL) 6977 return (jremref); 6978 cancel_diradd(dap, dirrem, jremref, NULL, NULL); 6979 /* 6980 * Mark any journal work as belonging to the parent so it is freed 6981 * with the .. reference. 6982 */ 6983 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 6984 wk->wk_state |= MKDIR_PARENT; 6985 return (NULL); 6986 } 6987 6988 /* 6989 * Cancel the MKDIR_PARENT mkdir component of a diradd when we're going to 6990 * replace it with a dirrem/diradd pair as a result of re-parenting a 6991 * directory. This ensures that we don't simultaneously have a mkdir and 6992 * a diradd for the same .. entry. 6993 */ 6994 static struct jremref * 6995 cancel_mkdir_dotdot(ip, dirrem, jremref) 6996 struct inode *ip; 6997 struct dirrem *dirrem; 6998 struct jremref *jremref; 6999 { 7000 struct inodedep *inodedep; 7001 struct jaddref *jaddref; 7002 struct mkdir *mkdir; 7003 struct diradd *dap; 7004 7005 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 7006 &inodedep) == 0) 7007 panic("cancel_mkdir_dotdot: Lost inodedep"); 7008 dap = inodedep->id_mkdiradd; 7009 if (dap == NULL || (dap->da_state & MKDIR_PARENT) == 0) 7010 return (jremref); 7011 for (mkdir = LIST_FIRST(&mkdirlisthd); mkdir; 7012 mkdir = LIST_NEXT(mkdir, md_mkdirs)) 7013 if (mkdir->md_diradd == dap && mkdir->md_state & MKDIR_PARENT) 7014 break; 7015 if (mkdir == NULL) 7016 panic("cancel_mkdir_dotdot: Unable to find mkdir\n"); 7017 if ((jaddref = mkdir->md_jaddref) != NULL) { 7018 mkdir->md_jaddref = NULL; 7019 jaddref->ja_state &= ~MKDIR_PARENT; 7020 if (inodedep_lookup(UFSTOVFS(ip->i_ump), jaddref->ja_ino, 0, 7021 &inodedep) == 0) 7022 panic("cancel_mkdir_dotdot: Lost parent inodedep"); 7023 if (cancel_jaddref(jaddref, inodedep, &dirrem->dm_jwork)) { 7024 journal_jremref(dirrem, jremref, inodedep); 7025 jremref = NULL; 7026 } 7027 } 7028 if (mkdir->md_state & ONWORKLIST) 7029 WORKLIST_REMOVE(&mkdir->md_list); 7030 mkdir->md_state |= ALLCOMPLETE; 7031 complete_mkdir(mkdir); 7032 return (jremref); 7033 } 7034 7035 static void 7036 journal_jremref(dirrem, jremref, inodedep) 7037 struct dirrem *dirrem; 7038 struct jremref *jremref; 7039 struct inodedep *inodedep; 7040 { 7041 7042 if (inodedep == NULL) 7043 if (inodedep_lookup(jremref->jr_list.wk_mp, 7044 jremref->jr_ref.if_ino, 0, &inodedep) == 0) 7045 panic("journal_jremref: Lost inodedep"); 7046 LIST_INSERT_HEAD(&dirrem->dm_jremrefhd, jremref, jr_deps); 7047 TAILQ_INSERT_TAIL(&inodedep->id_inoreflst, &jremref->jr_ref, if_deps); 7048 add_to_journal(&jremref->jr_list); 7049 } 7050 7051 static void 7052 dirrem_journal(dirrem, jremref, dotremref, dotdotremref) 7053 struct dirrem *dirrem; 7054 struct jremref *jremref; 7055 struct jremref *dotremref; 7056 struct jremref *dotdotremref; 7057 { 7058 struct inodedep *inodedep; 7059 7060 7061 if (inodedep_lookup(jremref->jr_list.wk_mp, jremref->jr_ref.if_ino, 0, 7062 &inodedep) == 0) 7063 panic("dirrem_journal: Lost inodedep"); 7064 journal_jremref(dirrem, jremref, inodedep); 7065 if (dotremref) 7066 journal_jremref(dirrem, dotremref, inodedep); 7067 if (dotdotremref) 7068 journal_jremref(dirrem, dotdotremref, NULL); 7069 } 7070 7071 /* 7072 * Allocate a new dirrem if appropriate and return it along with 7073 * its associated pagedep. Called without a lock, returns with lock. 7074 */ 7075 static long num_dirrem; /* number of dirrem allocated */ 7076 static struct dirrem * 7077 newdirrem(bp, dp, ip, isrmdir, prevdirremp) 7078 struct buf *bp; /* buffer containing directory block */ 7079 struct inode *dp; /* inode for the directory being modified */ 7080 struct inode *ip; /* inode for directory entry being removed */ 7081 int isrmdir; /* indicates if doing RMDIR */ 7082 struct dirrem **prevdirremp; /* previously referenced inode, if any */ 7083 { 7084 int offset; 7085 ufs_lbn_t lbn; 7086 struct diradd *dap; 7087 struct dirrem *dirrem; 7088 struct pagedep *pagedep; 7089 struct jremref *jremref; 7090 struct jremref *dotremref; 7091 struct jremref *dotdotremref; 7092 struct vnode *dvp; 7093 7094 /* 7095 * Whiteouts have no deletion dependencies. 7096 */ 7097 if (ip == NULL) 7098 panic("newdirrem: whiteout"); 7099 dvp = ITOV(dp); 7100 /* 7101 * If we are over our limit, try to improve the situation. 7102 * Limiting the number of dirrem structures will also limit 7103 * the number of freefile and freeblks structures. 7104 */ 7105 ACQUIRE_LOCK(&lk); 7106 if (!(ip->i_flags & SF_SNAPSHOT) && num_dirrem > max_softdeps / 2) 7107 (void) request_cleanup(ITOV(dp)->v_mount, FLUSH_REMOVE); 7108 num_dirrem += 1; 7109 FREE_LOCK(&lk); 7110 dirrem = malloc(sizeof(struct dirrem), 7111 M_DIRREM, M_SOFTDEP_FLAGS|M_ZERO); 7112 workitem_alloc(&dirrem->dm_list, D_DIRREM, dvp->v_mount); 7113 LIST_INIT(&dirrem->dm_jremrefhd); 7114 LIST_INIT(&dirrem->dm_jwork); 7115 dirrem->dm_state = isrmdir ? RMDIR : 0; 7116 dirrem->dm_oldinum = ip->i_number; 7117 *prevdirremp = NULL; 7118 /* 7119 * Allocate remove reference structures to track journal write 7120 * dependencies. We will always have one for the link and 7121 * when doing directories we will always have one more for dot. 7122 * When renaming a directory we skip the dotdot link change so 7123 * this is not needed. 7124 */ 7125 jremref = dotremref = dotdotremref = NULL; 7126 if (DOINGSUJ(dvp)) { 7127 if (isrmdir) { 7128 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 7129 ip->i_effnlink + 2); 7130 dotremref = newjremref(dirrem, ip, ip, DOT_OFFSET, 7131 ip->i_effnlink + 1); 7132 } else 7133 jremref = newjremref(dirrem, dp, ip, dp->i_offset, 7134 ip->i_effnlink + 1); 7135 if (isrmdir > 1) { 7136 dotdotremref = newjremref(dirrem, ip, dp, DOTDOT_OFFSET, 7137 dp->i_effnlink + 1); 7138 dotdotremref->jr_state |= MKDIR_PARENT; 7139 } 7140 } 7141 ACQUIRE_LOCK(&lk); 7142 lbn = lblkno(dp->i_fs, dp->i_offset); 7143 offset = blkoff(dp->i_fs, dp->i_offset); 7144 if (pagedep_lookup(UFSTOVFS(dp->i_ump), dp->i_number, lbn, DEPALLOC, 7145 &pagedep) == 0) 7146 WORKLIST_INSERT(&bp->b_dep, &pagedep->pd_list); 7147 dirrem->dm_pagedep = pagedep; 7148 /* 7149 * If we're renaming a .. link to a new directory, cancel any 7150 * existing MKDIR_PARENT mkdir. If it has already been canceled 7151 * the jremref is preserved for any potential diradd in this 7152 * location. This can not coincide with a rmdir. 7153 */ 7154 if (dp->i_offset == DOTDOT_OFFSET) { 7155 if (isrmdir) 7156 panic("newdirrem: .. directory change during remove?"); 7157 jremref = cancel_mkdir_dotdot(dp, dirrem, jremref); 7158 } 7159 /* 7160 * If we're removing a directory search for the .. dependency now and 7161 * cancel it. Any pending journal work will be added to the dirrem 7162 * to be completed when the workitem remove completes. 7163 */ 7164 if (isrmdir > 1) 7165 dotdotremref = cancel_diradd_dotdot(ip, dirrem, dotdotremref); 7166 /* 7167 * Check for a diradd dependency for the same directory entry. 7168 * If present, then both dependencies become obsolete and can 7169 * be de-allocated. 7170 */ 7171 dap = diradd_lookup(pagedep, offset); 7172 if (dap == NULL) { 7173 /* 7174 * Link the jremref structures into the dirrem so they are 7175 * written prior to the pagedep. 7176 */ 7177 if (jremref) 7178 dirrem_journal(dirrem, jremref, dotremref, 7179 dotdotremref); 7180 return (dirrem); 7181 } 7182 /* 7183 * Must be ATTACHED at this point. 7184 */ 7185 if ((dap->da_state & ATTACHED) == 0) 7186 panic("newdirrem: not ATTACHED"); 7187 if (dap->da_newinum != ip->i_number) 7188 panic("newdirrem: inum %d should be %d", 7189 ip->i_number, dap->da_newinum); 7190 /* 7191 * If we are deleting a changed name that never made it to disk, 7192 * then return the dirrem describing the previous inode (which 7193 * represents the inode currently referenced from this entry on disk). 7194 */ 7195 if ((dap->da_state & DIRCHG) != 0) { 7196 *prevdirremp = dap->da_previous; 7197 dap->da_state &= ~DIRCHG; 7198 dap->da_pagedep = pagedep; 7199 } 7200 /* 7201 * We are deleting an entry that never made it to disk. 7202 * Mark it COMPLETE so we can delete its inode immediately. 7203 */ 7204 dirrem->dm_state |= COMPLETE; 7205 cancel_diradd(dap, dirrem, jremref, dotremref, dotdotremref); 7206 #ifdef SUJ_DEBUG 7207 if (isrmdir == 0) { 7208 struct worklist *wk; 7209 7210 LIST_FOREACH(wk, &dirrem->dm_jwork, wk_list) 7211 if (wk->wk_state & (MKDIR_BODY | MKDIR_PARENT)) 7212 panic("bad wk %p (0x%X)\n", wk, wk->wk_state); 7213 } 7214 #endif 7215 7216 return (dirrem); 7217 } 7218 7219 /* 7220 * Directory entry change dependencies. 7221 * 7222 * Changing an existing directory entry requires that an add operation 7223 * be completed first followed by a deletion. The semantics for the addition 7224 * are identical to the description of adding a new entry above except 7225 * that the rollback is to the old inode number rather than zero. Once 7226 * the addition dependency is completed, the removal is done as described 7227 * in the removal routine above. 7228 */ 7229 7230 /* 7231 * This routine should be called immediately after changing 7232 * a directory entry. The inode's link count should not be 7233 * decremented by the calling procedure -- the soft updates 7234 * code will perform this task when it is safe. 7235 */ 7236 void 7237 softdep_setup_directory_change(bp, dp, ip, newinum, isrmdir) 7238 struct buf *bp; /* buffer containing directory block */ 7239 struct inode *dp; /* inode for the directory being modified */ 7240 struct inode *ip; /* inode for directory entry being removed */ 7241 ino_t newinum; /* new inode number for changed entry */ 7242 int isrmdir; /* indicates if doing RMDIR */ 7243 { 7244 int offset; 7245 struct diradd *dap = NULL; 7246 struct dirrem *dirrem, *prevdirrem; 7247 struct pagedep *pagedep; 7248 struct inodedep *inodedep; 7249 struct jaddref *jaddref; 7250 struct mount *mp; 7251 7252 offset = blkoff(dp->i_fs, dp->i_offset); 7253 mp = UFSTOVFS(dp->i_ump); 7254 7255 /* 7256 * Whiteouts do not need diradd dependencies. 7257 */ 7258 if (newinum != WINO) { 7259 dap = malloc(sizeof(struct diradd), 7260 M_DIRADD, M_SOFTDEP_FLAGS|M_ZERO); 7261 workitem_alloc(&dap->da_list, D_DIRADD, mp); 7262 dap->da_state = DIRCHG | ATTACHED | DEPCOMPLETE; 7263 dap->da_offset = offset; 7264 dap->da_newinum = newinum; 7265 LIST_INIT(&dap->da_jwork); 7266 } 7267 7268 /* 7269 * Allocate a new dirrem and ACQUIRE_LOCK. 7270 */ 7271 dirrem = newdirrem(bp, dp, ip, isrmdir, &prevdirrem); 7272 pagedep = dirrem->dm_pagedep; 7273 /* 7274 * The possible values for isrmdir: 7275 * 0 - non-directory file rename 7276 * 1 - directory rename within same directory 7277 * inum - directory rename to new directory of given inode number 7278 * When renaming to a new directory, we are both deleting and 7279 * creating a new directory entry, so the link count on the new 7280 * directory should not change. Thus we do not need the followup 7281 * dirrem which is usually done in handle_workitem_remove. We set 7282 * the DIRCHG flag to tell handle_workitem_remove to skip the 7283 * followup dirrem. 7284 */ 7285 if (isrmdir > 1) 7286 dirrem->dm_state |= DIRCHG; 7287 7288 /* 7289 * Whiteouts have no additional dependencies, 7290 * so just put the dirrem on the correct list. 7291 */ 7292 if (newinum == WINO) { 7293 if ((dirrem->dm_state & COMPLETE) == 0) { 7294 LIST_INSERT_HEAD(&pagedep->pd_dirremhd, dirrem, 7295 dm_next); 7296 } else { 7297 dirrem->dm_dirinum = pagedep->pd_ino; 7298 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 7299 add_to_worklist(&dirrem->dm_list, 0); 7300 } 7301 FREE_LOCK(&lk); 7302 return; 7303 } 7304 /* 7305 * Add the dirrem to the inodedep's pending remove list for quick 7306 * discovery later. A valid nlinkdelta ensures that this lookup 7307 * will not fail. 7308 */ 7309 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 7310 panic("softdep_setup_directory_change: Lost inodedep."); 7311 dirrem->dm_state |= ONDEPLIST; 7312 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 7313 7314 /* 7315 * If the COMPLETE flag is clear, then there were no active 7316 * entries and we want to roll back to the previous inode until 7317 * the new inode is committed to disk. If the COMPLETE flag is 7318 * set, then we have deleted an entry that never made it to disk. 7319 * If the entry we deleted resulted from a name change, then the old 7320 * inode reference still resides on disk. Any rollback that we do 7321 * needs to be to that old inode (returned to us in prevdirrem). If 7322 * the entry we deleted resulted from a create, then there is 7323 * no entry on the disk, so we want to roll back to zero rather 7324 * than the uncommitted inode. In either of the COMPLETE cases we 7325 * want to immediately free the unwritten and unreferenced inode. 7326 */ 7327 if ((dirrem->dm_state & COMPLETE) == 0) { 7328 dap->da_previous = dirrem; 7329 } else { 7330 if (prevdirrem != NULL) { 7331 dap->da_previous = prevdirrem; 7332 } else { 7333 dap->da_state &= ~DIRCHG; 7334 dap->da_pagedep = pagedep; 7335 } 7336 dirrem->dm_dirinum = pagedep->pd_ino; 7337 if (LIST_EMPTY(&dirrem->dm_jremrefhd)) 7338 add_to_worklist(&dirrem->dm_list, 0); 7339 } 7340 /* 7341 * Lookup the jaddref for this journal entry. We must finish 7342 * initializing it and make the diradd write dependent on it. 7343 * If we're not journaling Put it on the id_bufwait list if the inode 7344 * is not yet written. If it is written, do the post-inode write 7345 * processing to put it on the id_pendinghd list. 7346 */ 7347 inodedep_lookup(mp, newinum, DEPALLOC, &inodedep); 7348 if (mp->mnt_kern_flag & MNTK_SUJ) { 7349 jaddref = (struct jaddref *)TAILQ_LAST(&inodedep->id_inoreflst, 7350 inoreflst); 7351 KASSERT(jaddref != NULL && jaddref->ja_parent == dp->i_number, 7352 ("softdep_setup_directory_change: bad jaddref %p", 7353 jaddref)); 7354 jaddref->ja_diroff = dp->i_offset; 7355 jaddref->ja_diradd = dap; 7356 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 7357 dap, da_pdlist); 7358 add_to_journal(&jaddref->ja_list); 7359 } else if ((inodedep->id_state & ALLCOMPLETE) == ALLCOMPLETE) { 7360 dap->da_state |= COMPLETE; 7361 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, da_pdlist); 7362 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 7363 } else { 7364 LIST_INSERT_HEAD(&pagedep->pd_diraddhd[DIRADDHASH(offset)], 7365 dap, da_pdlist); 7366 WORKLIST_INSERT(&inodedep->id_bufwait, &dap->da_list); 7367 } 7368 /* 7369 * If we're making a new name for a directory that has not been 7370 * committed when need to move the dot and dotdot references to 7371 * this new name. 7372 */ 7373 if (inodedep->id_mkdiradd && dp->i_offset != DOTDOT_OFFSET) 7374 merge_diradd(inodedep, dap); 7375 FREE_LOCK(&lk); 7376 } 7377 7378 /* 7379 * Called whenever the link count on an inode is changed. 7380 * It creates an inode dependency so that the new reference(s) 7381 * to the inode cannot be committed to disk until the updated 7382 * inode has been written. 7383 */ 7384 void 7385 softdep_change_linkcnt(ip) 7386 struct inode *ip; /* the inode with the increased link count */ 7387 { 7388 struct inodedep *inodedep; 7389 7390 ACQUIRE_LOCK(&lk); 7391 inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, DEPALLOC, &inodedep); 7392 if (ip->i_nlink < ip->i_effnlink) 7393 panic("softdep_change_linkcnt: bad delta"); 7394 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 7395 FREE_LOCK(&lk); 7396 } 7397 7398 /* 7399 * Attach a sbdep dependency to the superblock buf so that we can keep 7400 * track of the head of the linked list of referenced but unlinked inodes. 7401 */ 7402 void 7403 softdep_setup_sbupdate(ump, fs, bp) 7404 struct ufsmount *ump; 7405 struct fs *fs; 7406 struct buf *bp; 7407 { 7408 struct sbdep *sbdep; 7409 struct worklist *wk; 7410 7411 if ((fs->fs_flags & FS_SUJ) == 0) 7412 return; 7413 LIST_FOREACH(wk, &bp->b_dep, wk_list) 7414 if (wk->wk_type == D_SBDEP) 7415 break; 7416 if (wk != NULL) 7417 return; 7418 sbdep = malloc(sizeof(struct sbdep), M_SBDEP, M_SOFTDEP_FLAGS); 7419 workitem_alloc(&sbdep->sb_list, D_SBDEP, UFSTOVFS(ump)); 7420 sbdep->sb_fs = fs; 7421 sbdep->sb_ump = ump; 7422 ACQUIRE_LOCK(&lk); 7423 WORKLIST_INSERT(&bp->b_dep, &sbdep->sb_list); 7424 FREE_LOCK(&lk); 7425 } 7426 7427 /* 7428 * Return the first unlinked inodedep which is ready to be the head of the 7429 * list. The inodedep and all those after it must have valid next pointers. 7430 */ 7431 static struct inodedep * 7432 first_unlinked_inodedep(ump) 7433 struct ufsmount *ump; 7434 { 7435 struct inodedep *inodedep; 7436 struct inodedep *idp; 7437 7438 for (inodedep = TAILQ_LAST(&ump->softdep_unlinked, inodedeplst); 7439 inodedep; inodedep = idp) { 7440 if ((inodedep->id_state & UNLINKNEXT) == 0) 7441 return (NULL); 7442 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 7443 if (idp == NULL || (idp->id_state & UNLINKNEXT) == 0) 7444 break; 7445 if ((inodedep->id_state & UNLINKPREV) == 0) 7446 panic("first_unlinked_inodedep: prev != next"); 7447 } 7448 if (inodedep == NULL) 7449 return (NULL); 7450 7451 return (inodedep); 7452 } 7453 7454 /* 7455 * Set the sujfree unlinked head pointer prior to writing a superblock. 7456 */ 7457 static void 7458 initiate_write_sbdep(sbdep) 7459 struct sbdep *sbdep; 7460 { 7461 struct inodedep *inodedep; 7462 struct fs *bpfs; 7463 struct fs *fs; 7464 7465 bpfs = sbdep->sb_fs; 7466 fs = sbdep->sb_ump->um_fs; 7467 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 7468 if (inodedep) { 7469 fs->fs_sujfree = inodedep->id_ino; 7470 inodedep->id_state |= UNLINKPREV; 7471 } else 7472 fs->fs_sujfree = 0; 7473 bpfs->fs_sujfree = fs->fs_sujfree; 7474 } 7475 7476 /* 7477 * After a superblock is written determine whether it must be written again 7478 * due to a changing unlinked list head. 7479 */ 7480 static int 7481 handle_written_sbdep(sbdep, bp) 7482 struct sbdep *sbdep; 7483 struct buf *bp; 7484 { 7485 struct inodedep *inodedep; 7486 struct mount *mp; 7487 struct fs *fs; 7488 7489 fs = sbdep->sb_fs; 7490 mp = UFSTOVFS(sbdep->sb_ump); 7491 inodedep = first_unlinked_inodedep(sbdep->sb_ump); 7492 if ((inodedep && fs->fs_sujfree != inodedep->id_ino) || 7493 (inodedep == NULL && fs->fs_sujfree != 0)) { 7494 bdirty(bp); 7495 return (1); 7496 } 7497 WORKITEM_FREE(sbdep, D_SBDEP); 7498 if (fs->fs_sujfree == 0) 7499 return (0); 7500 if (inodedep_lookup(mp, fs->fs_sujfree, 0, &inodedep) == 0) 7501 panic("handle_written_sbdep: lost inodedep"); 7502 /* 7503 * Now that we have a record of this inode in stable store allow it 7504 * to be written to free up pending work. Inodes may see a lot of 7505 * write activity after they are unlinked which we must not hold up. 7506 */ 7507 for (; inodedep != NULL; inodedep = TAILQ_NEXT(inodedep, id_unlinked)) { 7508 if ((inodedep->id_state & UNLINKLINKS) != UNLINKLINKS) 7509 panic("handle_written_sbdep: Bad inodedep %p (0x%X)", 7510 inodedep, inodedep->id_state); 7511 if (inodedep->id_state & UNLINKONLIST) 7512 break; 7513 inodedep->id_state |= DEPCOMPLETE | UNLINKONLIST; 7514 } 7515 7516 return (0); 7517 } 7518 7519 /* 7520 * Mark an inodedep as unlinked and insert it into the in-memory unlinked list. 7521 */ 7522 static void 7523 unlinked_inodedep(mp, inodedep) 7524 struct mount *mp; 7525 struct inodedep *inodedep; 7526 { 7527 struct ufsmount *ump; 7528 7529 if ((mp->mnt_kern_flag & MNTK_SUJ) == 0) 7530 return; 7531 ump = VFSTOUFS(mp); 7532 ump->um_fs->fs_fmod = 1; 7533 inodedep->id_state |= UNLINKED; 7534 TAILQ_INSERT_HEAD(&ump->softdep_unlinked, inodedep, id_unlinked); 7535 } 7536 7537 /* 7538 * Remove an inodedep from the unlinked inodedep list. This may require 7539 * disk writes if the inode has made it that far. 7540 */ 7541 static void 7542 clear_unlinked_inodedep(inodedep) 7543 struct inodedep *inodedep; 7544 { 7545 struct ufsmount *ump; 7546 struct inodedep *idp; 7547 struct inodedep *idn; 7548 struct fs *fs; 7549 struct buf *bp; 7550 ino_t ino; 7551 ino_t nino; 7552 ino_t pino; 7553 int error; 7554 7555 ump = VFSTOUFS(inodedep->id_list.wk_mp); 7556 fs = ump->um_fs; 7557 ino = inodedep->id_ino; 7558 error = 0; 7559 for (;;) { 7560 /* 7561 * If nothing has yet been written simply remove us from 7562 * the in memory list and return. This is the most common 7563 * case where handle_workitem_remove() loses the final 7564 * reference. 7565 */ 7566 if ((inodedep->id_state & UNLINKLINKS) == 0) 7567 break; 7568 /* 7569 * If we have a NEXT pointer and no PREV pointer we can simply 7570 * clear NEXT's PREV and remove ourselves from the list. Be 7571 * careful not to clear PREV if the superblock points at 7572 * next as well. 7573 */ 7574 idn = TAILQ_NEXT(inodedep, id_unlinked); 7575 if ((inodedep->id_state & UNLINKLINKS) == UNLINKNEXT) { 7576 if (idn && fs->fs_sujfree != idn->id_ino) 7577 idn->id_state &= ~UNLINKPREV; 7578 break; 7579 } 7580 /* 7581 * Here we have an inodedep which is actually linked into 7582 * the list. We must remove it by forcing a write to the 7583 * link before us, whether it be the superblock or an inode. 7584 * Unfortunately the list may change while we're waiting 7585 * on the buf lock for either resource so we must loop until 7586 * we lock the right one. If both the superblock and an 7587 * inode point to this inode we must clear the inode first 7588 * followed by the superblock. 7589 */ 7590 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 7591 pino = 0; 7592 if (idp && (idp->id_state & UNLINKNEXT)) 7593 pino = idp->id_ino; 7594 FREE_LOCK(&lk); 7595 if (pino == 0) 7596 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 7597 (int)fs->fs_sbsize, 0, 0, 0); 7598 else 7599 error = bread(ump->um_devvp, 7600 fsbtodb(fs, ino_to_fsba(fs, pino)), 7601 (int)fs->fs_bsize, NOCRED, &bp); 7602 ACQUIRE_LOCK(&lk); 7603 if (error) 7604 break; 7605 /* If the list has changed restart the loop. */ 7606 idp = TAILQ_PREV(inodedep, inodedeplst, id_unlinked); 7607 nino = 0; 7608 if (idp && (idp->id_state & UNLINKNEXT)) 7609 nino = idp->id_ino; 7610 if (nino != pino || 7611 (inodedep->id_state & UNLINKPREV) != UNLINKPREV) { 7612 FREE_LOCK(&lk); 7613 brelse(bp); 7614 ACQUIRE_LOCK(&lk); 7615 continue; 7616 } 7617 /* 7618 * Remove us from the in memory list. After this we cannot 7619 * access the inodedep. 7620 */ 7621 idn = TAILQ_NEXT(inodedep, id_unlinked); 7622 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS); 7623 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 7624 /* 7625 * Determine the next inode number. 7626 */ 7627 nino = 0; 7628 if (idn) { 7629 /* 7630 * If next isn't on the list we can just clear prev's 7631 * state and schedule it to be fixed later. No need 7632 * to synchronously write if we're not in the real 7633 * list. 7634 */ 7635 if ((idn->id_state & UNLINKPREV) == 0 && pino != 0) { 7636 idp->id_state &= ~UNLINKNEXT; 7637 if ((idp->id_state & ONWORKLIST) == 0) 7638 WORKLIST_INSERT(&bp->b_dep, 7639 &idp->id_list); 7640 FREE_LOCK(&lk); 7641 bawrite(bp); 7642 ACQUIRE_LOCK(&lk); 7643 return; 7644 } 7645 nino = idn->id_ino; 7646 } 7647 FREE_LOCK(&lk); 7648 /* 7649 * The predecessor's next pointer is manually updated here 7650 * so that the NEXT flag is never cleared for an element 7651 * that is in the list. 7652 */ 7653 if (pino == 0) { 7654 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 7655 ffs_oldfscompat_write((struct fs *)bp->b_data, ump); 7656 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data, 7657 bp); 7658 } else if (fs->fs_magic == FS_UFS1_MAGIC) 7659 ((struct ufs1_dinode *)bp->b_data + 7660 ino_to_fsbo(fs, pino))->di_freelink = nino; 7661 else 7662 ((struct ufs2_dinode *)bp->b_data + 7663 ino_to_fsbo(fs, pino))->di_freelink = nino; 7664 /* 7665 * If the bwrite fails we have no recourse to recover. The 7666 * filesystem is corrupted already. 7667 */ 7668 bwrite(bp); 7669 ACQUIRE_LOCK(&lk); 7670 /* 7671 * If the superblock pointer still needs to be cleared force 7672 * a write here. 7673 */ 7674 if (fs->fs_sujfree == ino) { 7675 FREE_LOCK(&lk); 7676 bp = getblk(ump->um_devvp, btodb(fs->fs_sblockloc), 7677 (int)fs->fs_sbsize, 0, 0, 0); 7678 bcopy((caddr_t)fs, bp->b_data, (u_int)fs->fs_sbsize); 7679 ffs_oldfscompat_write((struct fs *)bp->b_data, ump); 7680 softdep_setup_sbupdate(ump, (struct fs *)bp->b_data, 7681 bp); 7682 bwrite(bp); 7683 ACQUIRE_LOCK(&lk); 7684 } 7685 if (fs->fs_sujfree != ino) 7686 return; 7687 panic("clear_unlinked_inodedep: Failed to clear free head"); 7688 } 7689 if (inodedep->id_ino == fs->fs_sujfree) 7690 panic("clear_unlinked_inodedep: Freeing head of free list"); 7691 inodedep->id_state &= ~(UNLINKED | UNLINKLINKS); 7692 TAILQ_REMOVE(&ump->softdep_unlinked, inodedep, id_unlinked); 7693 return; 7694 } 7695 7696 /* 7697 * This workitem decrements the inode's link count. 7698 * If the link count reaches zero, the file is removed. 7699 */ 7700 static void 7701 handle_workitem_remove(dirrem, xp) 7702 struct dirrem *dirrem; 7703 struct vnode *xp; 7704 { 7705 struct inodedep *inodedep; 7706 struct workhead dotdotwk; 7707 struct worklist *wk; 7708 struct ufsmount *ump; 7709 struct mount *mp; 7710 struct vnode *vp; 7711 struct inode *ip; 7712 ino_t oldinum; 7713 int error; 7714 7715 if (dirrem->dm_state & ONWORKLIST) 7716 panic("handle_workitem_remove: dirrem %p still on worklist", 7717 dirrem); 7718 oldinum = dirrem->dm_oldinum; 7719 mp = dirrem->dm_list.wk_mp; 7720 ump = VFSTOUFS(mp); 7721 if ((vp = xp) == NULL && 7722 (error = ffs_vgetf(mp, oldinum, LK_EXCLUSIVE, &vp, 7723 FFSV_FORCEINSMQ)) != 0) { 7724 softdep_error("handle_workitem_remove: vget", error); 7725 return; 7726 } 7727 ip = VTOI(vp); 7728 ACQUIRE_LOCK(&lk); 7729 if ((inodedep_lookup(mp, oldinum, 0, &inodedep)) == 0) 7730 panic("handle_workitem_remove: lost inodedep"); 7731 if (dirrem->dm_state & ONDEPLIST) 7732 LIST_REMOVE(dirrem, dm_inonext); 7733 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 7734 ("handle_workitem_remove: Journal entries not written.")); 7735 7736 /* 7737 * Move all dependencies waiting on the remove to complete 7738 * from the dirrem to the inode inowait list to be completed 7739 * after the inode has been updated and written to disk. Any 7740 * marked MKDIR_PARENT are saved to be completed when the .. ref 7741 * is removed. 7742 */ 7743 LIST_INIT(&dotdotwk); 7744 while ((wk = LIST_FIRST(&dirrem->dm_jwork)) != NULL) { 7745 WORKLIST_REMOVE(wk); 7746 if (wk->wk_state & MKDIR_PARENT) { 7747 wk->wk_state &= ~MKDIR_PARENT; 7748 WORKLIST_INSERT(&dotdotwk, wk); 7749 continue; 7750 } 7751 WORKLIST_INSERT(&inodedep->id_inowait, wk); 7752 } 7753 LIST_SWAP(&dirrem->dm_jwork, &dotdotwk, worklist, wk_list); 7754 /* 7755 * Normal file deletion. 7756 */ 7757 if ((dirrem->dm_state & RMDIR) == 0) { 7758 ip->i_nlink--; 7759 DIP_SET(ip, i_nlink, ip->i_nlink); 7760 ip->i_flag |= IN_CHANGE; 7761 if (ip->i_nlink < ip->i_effnlink) 7762 panic("handle_workitem_remove: bad file delta"); 7763 if (ip->i_nlink == 0) 7764 unlinked_inodedep(mp, inodedep); 7765 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 7766 num_dirrem -= 1; 7767 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 7768 ("handle_workitem_remove: worklist not empty. %s", 7769 TYPENAME(LIST_FIRST(&dirrem->dm_jwork)->wk_type))); 7770 WORKITEM_FREE(dirrem, D_DIRREM); 7771 FREE_LOCK(&lk); 7772 goto out; 7773 } 7774 /* 7775 * Directory deletion. Decrement reference count for both the 7776 * just deleted parent directory entry and the reference for ".". 7777 * Arrange to have the reference count on the parent decremented 7778 * to account for the loss of "..". 7779 */ 7780 ip->i_nlink -= 2; 7781 DIP_SET(ip, i_nlink, ip->i_nlink); 7782 ip->i_flag |= IN_CHANGE; 7783 if (ip->i_nlink < ip->i_effnlink) 7784 panic("handle_workitem_remove: bad dir delta"); 7785 if (ip->i_nlink == 0) 7786 unlinked_inodedep(mp, inodedep); 7787 inodedep->id_nlinkdelta = ip->i_nlink - ip->i_effnlink; 7788 /* 7789 * Rename a directory to a new parent. Since, we are both deleting 7790 * and creating a new directory entry, the link count on the new 7791 * directory should not change. Thus we skip the followup dirrem. 7792 */ 7793 if (dirrem->dm_state & DIRCHG) { 7794 KASSERT(LIST_EMPTY(&dirrem->dm_jwork), 7795 ("handle_workitem_remove: DIRCHG and worklist not empty.")); 7796 num_dirrem -= 1; 7797 WORKITEM_FREE(dirrem, D_DIRREM); 7798 FREE_LOCK(&lk); 7799 goto out; 7800 } 7801 dirrem->dm_state = ONDEPLIST; 7802 dirrem->dm_oldinum = dirrem->dm_dirinum; 7803 /* 7804 * Place the dirrem on the parent's diremhd list. 7805 */ 7806 if (inodedep_lookup(mp, dirrem->dm_oldinum, 0, &inodedep) == 0) 7807 panic("handle_workitem_remove: lost dir inodedep"); 7808 LIST_INSERT_HEAD(&inodedep->id_dirremhd, dirrem, dm_inonext); 7809 /* 7810 * If the allocated inode has never been written to disk, then 7811 * the on-disk inode is zero'ed and we can remove the file 7812 * immediately. When journaling if the inode has been marked 7813 * unlinked and not DEPCOMPLETE we know it can never be written. 7814 */ 7815 inodedep_lookup(mp, oldinum, 0, &inodedep); 7816 if (inodedep == NULL || 7817 (inodedep->id_state & (DEPCOMPLETE | UNLINKED)) == UNLINKED || 7818 check_inode_unwritten(inodedep)) { 7819 if (xp != NULL) 7820 add_to_worklist(&dirrem->dm_list, 0); 7821 FREE_LOCK(&lk); 7822 if (xp == NULL) { 7823 vput(vp); 7824 handle_workitem_remove(dirrem, NULL); 7825 } 7826 return; 7827 } 7828 WORKLIST_INSERT(&inodedep->id_inowait, &dirrem->dm_list); 7829 FREE_LOCK(&lk); 7830 ip->i_flag |= IN_CHANGE; 7831 out: 7832 ffs_update(vp, 0); 7833 if (xp == NULL) 7834 vput(vp); 7835 } 7836 7837 /* 7838 * Inode de-allocation dependencies. 7839 * 7840 * When an inode's link count is reduced to zero, it can be de-allocated. We 7841 * found it convenient to postpone de-allocation until after the inode is 7842 * written to disk with its new link count (zero). At this point, all of the 7843 * on-disk inode's block pointers are nullified and, with careful dependency 7844 * list ordering, all dependencies related to the inode will be satisfied and 7845 * the corresponding dependency structures de-allocated. So, if/when the 7846 * inode is reused, there will be no mixing of old dependencies with new 7847 * ones. This artificial dependency is set up by the block de-allocation 7848 * procedure above (softdep_setup_freeblocks) and completed by the 7849 * following procedure. 7850 */ 7851 static void 7852 handle_workitem_freefile(freefile) 7853 struct freefile *freefile; 7854 { 7855 struct workhead wkhd; 7856 struct fs *fs; 7857 struct inodedep *idp; 7858 struct ufsmount *ump; 7859 int error; 7860 7861 ump = VFSTOUFS(freefile->fx_list.wk_mp); 7862 fs = ump->um_fs; 7863 #ifdef DEBUG 7864 ACQUIRE_LOCK(&lk); 7865 error = inodedep_lookup(UFSTOVFS(ump), freefile->fx_oldinum, 0, &idp); 7866 FREE_LOCK(&lk); 7867 if (error) 7868 panic("handle_workitem_freefile: inodedep %p survived", idp); 7869 #endif 7870 UFS_LOCK(ump); 7871 fs->fs_pendinginodes -= 1; 7872 UFS_UNLOCK(ump); 7873 LIST_INIT(&wkhd); 7874 LIST_SWAP(&freefile->fx_jwork, &wkhd, worklist, wk_list); 7875 if ((error = ffs_freefile(ump, fs, freefile->fx_devvp, 7876 freefile->fx_oldinum, freefile->fx_mode, &wkhd)) != 0) 7877 softdep_error("handle_workitem_freefile", error); 7878 ACQUIRE_LOCK(&lk); 7879 WORKITEM_FREE(freefile, D_FREEFILE); 7880 FREE_LOCK(&lk); 7881 } 7882 7883 7884 /* 7885 * Helper function which unlinks marker element from work list and returns 7886 * the next element on the list. 7887 */ 7888 static __inline struct worklist * 7889 markernext(struct worklist *marker) 7890 { 7891 struct worklist *next; 7892 7893 next = LIST_NEXT(marker, wk_list); 7894 LIST_REMOVE(marker, wk_list); 7895 return next; 7896 } 7897 7898 /* 7899 * Disk writes. 7900 * 7901 * The dependency structures constructed above are most actively used when file 7902 * system blocks are written to disk. No constraints are placed on when a 7903 * block can be written, but unsatisfied update dependencies are made safe by 7904 * modifying (or replacing) the source memory for the duration of the disk 7905 * write. When the disk write completes, the memory block is again brought 7906 * up-to-date. 7907 * 7908 * In-core inode structure reclamation. 7909 * 7910 * Because there are a finite number of "in-core" inode structures, they are 7911 * reused regularly. By transferring all inode-related dependencies to the 7912 * in-memory inode block and indexing them separately (via "inodedep"s), we 7913 * can allow "in-core" inode structures to be reused at any time and avoid 7914 * any increase in contention. 7915 * 7916 * Called just before entering the device driver to initiate a new disk I/O. 7917 * The buffer must be locked, thus, no I/O completion operations can occur 7918 * while we are manipulating its associated dependencies. 7919 */ 7920 static void 7921 softdep_disk_io_initiation(bp) 7922 struct buf *bp; /* structure describing disk write to occur */ 7923 { 7924 struct worklist *wk; 7925 struct worklist marker; 7926 struct inodedep *inodedep; 7927 struct freeblks *freeblks; 7928 struct jfreeblk *jfreeblk; 7929 struct newblk *newblk; 7930 7931 /* 7932 * We only care about write operations. There should never 7933 * be dependencies for reads. 7934 */ 7935 if (bp->b_iocmd != BIO_WRITE) 7936 panic("softdep_disk_io_initiation: not write"); 7937 7938 if (bp->b_vflags & BV_BKGRDINPROG) 7939 panic("softdep_disk_io_initiation: Writing buffer with " 7940 "background write in progress: %p", bp); 7941 7942 marker.wk_type = D_LAST + 1; /* Not a normal workitem */ 7943 PHOLD(curproc); /* Don't swap out kernel stack */ 7944 7945 ACQUIRE_LOCK(&lk); 7946 /* 7947 * Do any necessary pre-I/O processing. 7948 */ 7949 for (wk = LIST_FIRST(&bp->b_dep); wk != NULL; 7950 wk = markernext(&marker)) { 7951 LIST_INSERT_AFTER(wk, &marker, wk_list); 7952 switch (wk->wk_type) { 7953 7954 case D_PAGEDEP: 7955 initiate_write_filepage(WK_PAGEDEP(wk), bp); 7956 continue; 7957 7958 case D_INODEDEP: 7959 inodedep = WK_INODEDEP(wk); 7960 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) 7961 initiate_write_inodeblock_ufs1(inodedep, bp); 7962 else 7963 initiate_write_inodeblock_ufs2(inodedep, bp); 7964 continue; 7965 7966 case D_INDIRDEP: 7967 initiate_write_indirdep(WK_INDIRDEP(wk), bp); 7968 continue; 7969 7970 case D_BMSAFEMAP: 7971 initiate_write_bmsafemap(WK_BMSAFEMAP(wk), bp); 7972 continue; 7973 7974 case D_JSEG: 7975 WK_JSEG(wk)->js_buf = NULL; 7976 continue; 7977 7978 case D_FREEBLKS: 7979 freeblks = WK_FREEBLKS(wk); 7980 jfreeblk = LIST_FIRST(&freeblks->fb_jfreeblkhd); 7981 /* 7982 * We have to wait for the jfreeblks to be journaled 7983 * before we can write an inodeblock with updated 7984 * pointers. Be careful to arrange the marker so 7985 * we revisit the jfreeblk if it's not removed by 7986 * the first jwait(). 7987 */ 7988 if (jfreeblk != NULL) { 7989 LIST_REMOVE(&marker, wk_list); 7990 LIST_INSERT_BEFORE(wk, &marker, wk_list); 7991 jwait(&jfreeblk->jf_list); 7992 } 7993 continue; 7994 case D_ALLOCDIRECT: 7995 case D_ALLOCINDIR: 7996 /* 7997 * We have to wait for the jnewblk to be journaled 7998 * before we can write to a block otherwise the 7999 * contents may be confused with an earlier file 8000 * at recovery time. Handle the marker as described 8001 * above. 8002 */ 8003 newblk = WK_NEWBLK(wk); 8004 if (newblk->nb_jnewblk != NULL) { 8005 LIST_REMOVE(&marker, wk_list); 8006 LIST_INSERT_BEFORE(wk, &marker, wk_list); 8007 jwait(&newblk->nb_jnewblk->jn_list); 8008 } 8009 continue; 8010 8011 case D_SBDEP: 8012 initiate_write_sbdep(WK_SBDEP(wk)); 8013 continue; 8014 8015 case D_MKDIR: 8016 case D_FREEWORK: 8017 case D_FREEDEP: 8018 case D_JSEGDEP: 8019 continue; 8020 8021 default: 8022 panic("handle_disk_io_initiation: Unexpected type %s", 8023 TYPENAME(wk->wk_type)); 8024 /* NOTREACHED */ 8025 } 8026 } 8027 FREE_LOCK(&lk); 8028 PRELE(curproc); /* Allow swapout of kernel stack */ 8029 } 8030 8031 /* 8032 * Called from within the procedure above to deal with unsatisfied 8033 * allocation dependencies in a directory. The buffer must be locked, 8034 * thus, no I/O completion operations can occur while we are 8035 * manipulating its associated dependencies. 8036 */ 8037 static void 8038 initiate_write_filepage(pagedep, bp) 8039 struct pagedep *pagedep; 8040 struct buf *bp; 8041 { 8042 struct jremref *jremref; 8043 struct jmvref *jmvref; 8044 struct dirrem *dirrem; 8045 struct diradd *dap; 8046 struct direct *ep; 8047 int i; 8048 8049 if (pagedep->pd_state & IOSTARTED) { 8050 /* 8051 * This can only happen if there is a driver that does not 8052 * understand chaining. Here biodone will reissue the call 8053 * to strategy for the incomplete buffers. 8054 */ 8055 printf("initiate_write_filepage: already started\n"); 8056 return; 8057 } 8058 pagedep->pd_state |= IOSTARTED; 8059 /* 8060 * Wait for all journal remove dependencies to hit the disk. 8061 * We can not allow any potentially conflicting directory adds 8062 * to be visible before removes and rollback is too difficult. 8063 * lk may be dropped and re-acquired, however we hold the buf 8064 * locked so the dependency can not go away. 8065 */ 8066 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) 8067 while ((jremref = LIST_FIRST(&dirrem->dm_jremrefhd)) != NULL) { 8068 stat_jwait_filepage++; 8069 jwait(&jremref->jr_list); 8070 } 8071 while ((jmvref = LIST_FIRST(&pagedep->pd_jmvrefhd)) != NULL) { 8072 stat_jwait_filepage++; 8073 jwait(&jmvref->jm_list); 8074 } 8075 for (i = 0; i < DAHASHSZ; i++) { 8076 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 8077 ep = (struct direct *) 8078 ((char *)bp->b_data + dap->da_offset); 8079 if (ep->d_ino != dap->da_newinum) 8080 panic("%s: dir inum %d != new %d", 8081 "initiate_write_filepage", 8082 ep->d_ino, dap->da_newinum); 8083 if (dap->da_state & DIRCHG) 8084 ep->d_ino = dap->da_previous->dm_oldinum; 8085 else 8086 ep->d_ino = 0; 8087 dap->da_state &= ~ATTACHED; 8088 dap->da_state |= UNDONE; 8089 } 8090 } 8091 } 8092 8093 /* 8094 * Version of initiate_write_inodeblock that handles UFS1 dinodes. 8095 * Note that any bug fixes made to this routine must be done in the 8096 * version found below. 8097 * 8098 * Called from within the procedure above to deal with unsatisfied 8099 * allocation dependencies in an inodeblock. The buffer must be 8100 * locked, thus, no I/O completion operations can occur while we 8101 * are manipulating its associated dependencies. 8102 */ 8103 static void 8104 initiate_write_inodeblock_ufs1(inodedep, bp) 8105 struct inodedep *inodedep; 8106 struct buf *bp; /* The inode block */ 8107 { 8108 struct allocdirect *adp, *lastadp; 8109 struct ufs1_dinode *dp; 8110 struct ufs1_dinode *sip; 8111 struct inoref *inoref; 8112 struct fs *fs; 8113 ufs_lbn_t i; 8114 #ifdef INVARIANTS 8115 ufs_lbn_t prevlbn = 0; 8116 #endif 8117 int deplist; 8118 8119 if (inodedep->id_state & IOSTARTED) 8120 panic("initiate_write_inodeblock_ufs1: already started"); 8121 inodedep->id_state |= IOSTARTED; 8122 fs = inodedep->id_fs; 8123 dp = (struct ufs1_dinode *)bp->b_data + 8124 ino_to_fsbo(fs, inodedep->id_ino); 8125 8126 /* 8127 * If we're on the unlinked list but have not yet written our 8128 * next pointer initialize it here. 8129 */ 8130 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 8131 struct inodedep *inon; 8132 8133 inon = TAILQ_NEXT(inodedep, id_unlinked); 8134 dp->di_freelink = inon ? inon->id_ino : 0; 8135 } 8136 /* 8137 * If the bitmap is not yet written, then the allocated 8138 * inode cannot be written to disk. 8139 */ 8140 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 8141 if (inodedep->id_savedino1 != NULL) 8142 panic("initiate_write_inodeblock_ufs1: I/O underway"); 8143 FREE_LOCK(&lk); 8144 sip = malloc(sizeof(struct ufs1_dinode), 8145 M_SAVEDINO, M_SOFTDEP_FLAGS); 8146 ACQUIRE_LOCK(&lk); 8147 inodedep->id_savedino1 = sip; 8148 *inodedep->id_savedino1 = *dp; 8149 bzero((caddr_t)dp, sizeof(struct ufs1_dinode)); 8150 dp->di_gen = inodedep->id_savedino1->di_gen; 8151 dp->di_freelink = inodedep->id_savedino1->di_freelink; 8152 return; 8153 } 8154 /* 8155 * If no dependencies, then there is nothing to roll back. 8156 */ 8157 inodedep->id_savedsize = dp->di_size; 8158 inodedep->id_savedextsize = 0; 8159 inodedep->id_savednlink = dp->di_nlink; 8160 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 8161 TAILQ_EMPTY(&inodedep->id_inoreflst)) 8162 return; 8163 /* 8164 * Revert the link count to that of the first unwritten journal entry. 8165 */ 8166 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 8167 if (inoref) 8168 dp->di_nlink = inoref->if_nlink; 8169 /* 8170 * Set the dependencies to busy. 8171 */ 8172 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8173 adp = TAILQ_NEXT(adp, ad_next)) { 8174 #ifdef INVARIANTS 8175 if (deplist != 0 && prevlbn >= adp->ad_offset) 8176 panic("softdep_write_inodeblock: lbn order"); 8177 prevlbn = adp->ad_offset; 8178 if (adp->ad_offset < NDADDR && 8179 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 8180 panic("%s: direct pointer #%jd mismatch %d != %jd", 8181 "softdep_write_inodeblock", 8182 (intmax_t)adp->ad_offset, 8183 dp->di_db[adp->ad_offset], 8184 (intmax_t)adp->ad_newblkno); 8185 if (adp->ad_offset >= NDADDR && 8186 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno) 8187 panic("%s: indirect pointer #%jd mismatch %d != %jd", 8188 "softdep_write_inodeblock", 8189 (intmax_t)adp->ad_offset - NDADDR, 8190 dp->di_ib[adp->ad_offset - NDADDR], 8191 (intmax_t)adp->ad_newblkno); 8192 deplist |= 1 << adp->ad_offset; 8193 if ((adp->ad_state & ATTACHED) == 0) 8194 panic("softdep_write_inodeblock: Unknown state 0x%x", 8195 adp->ad_state); 8196 #endif /* INVARIANTS */ 8197 adp->ad_state &= ~ATTACHED; 8198 adp->ad_state |= UNDONE; 8199 } 8200 /* 8201 * The on-disk inode cannot claim to be any larger than the last 8202 * fragment that has been written. Otherwise, the on-disk inode 8203 * might have fragments that were not the last block in the file 8204 * which would corrupt the filesystem. 8205 */ 8206 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8207 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 8208 if (adp->ad_offset >= NDADDR) 8209 break; 8210 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 8211 /* keep going until hitting a rollback to a frag */ 8212 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 8213 continue; 8214 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 8215 for (i = adp->ad_offset + 1; i < NDADDR; i++) { 8216 #ifdef INVARIANTS 8217 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 8218 panic("softdep_write_inodeblock: lost dep1"); 8219 #endif /* INVARIANTS */ 8220 dp->di_db[i] = 0; 8221 } 8222 for (i = 0; i < NIADDR; i++) { 8223 #ifdef INVARIANTS 8224 if (dp->di_ib[i] != 0 && 8225 (deplist & ((1 << NDADDR) << i)) == 0) 8226 panic("softdep_write_inodeblock: lost dep2"); 8227 #endif /* INVARIANTS */ 8228 dp->di_ib[i] = 0; 8229 } 8230 return; 8231 } 8232 /* 8233 * If we have zero'ed out the last allocated block of the file, 8234 * roll back the size to the last currently allocated block. 8235 * We know that this last allocated block is a full-sized as 8236 * we already checked for fragments in the loop above. 8237 */ 8238 if (lastadp != NULL && 8239 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 8240 for (i = lastadp->ad_offset; i >= 0; i--) 8241 if (dp->di_db[i] != 0) 8242 break; 8243 dp->di_size = (i + 1) * fs->fs_bsize; 8244 } 8245 /* 8246 * The only dependencies are for indirect blocks. 8247 * 8248 * The file size for indirect block additions is not guaranteed. 8249 * Such a guarantee would be non-trivial to achieve. The conventional 8250 * synchronous write implementation also does not make this guarantee. 8251 * Fsck should catch and fix discrepancies. Arguably, the file size 8252 * can be over-estimated without destroying integrity when the file 8253 * moves into the indirect blocks (i.e., is large). If we want to 8254 * postpone fsck, we are stuck with this argument. 8255 */ 8256 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 8257 dp->di_ib[adp->ad_offset - NDADDR] = 0; 8258 } 8259 8260 /* 8261 * Version of initiate_write_inodeblock that handles UFS2 dinodes. 8262 * Note that any bug fixes made to this routine must be done in the 8263 * version found above. 8264 * 8265 * Called from within the procedure above to deal with unsatisfied 8266 * allocation dependencies in an inodeblock. The buffer must be 8267 * locked, thus, no I/O completion operations can occur while we 8268 * are manipulating its associated dependencies. 8269 */ 8270 static void 8271 initiate_write_inodeblock_ufs2(inodedep, bp) 8272 struct inodedep *inodedep; 8273 struct buf *bp; /* The inode block */ 8274 { 8275 struct allocdirect *adp, *lastadp; 8276 struct ufs2_dinode *dp; 8277 struct ufs2_dinode *sip; 8278 struct inoref *inoref; 8279 struct fs *fs; 8280 ufs_lbn_t i; 8281 #ifdef INVARIANTS 8282 ufs_lbn_t prevlbn = 0; 8283 #endif 8284 int deplist; 8285 8286 if (inodedep->id_state & IOSTARTED) 8287 panic("initiate_write_inodeblock_ufs2: already started"); 8288 inodedep->id_state |= IOSTARTED; 8289 fs = inodedep->id_fs; 8290 dp = (struct ufs2_dinode *)bp->b_data + 8291 ino_to_fsbo(fs, inodedep->id_ino); 8292 8293 /* 8294 * If we're on the unlinked list but have not yet written our 8295 * next pointer initialize it here. 8296 */ 8297 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 8298 struct inodedep *inon; 8299 8300 inon = TAILQ_NEXT(inodedep, id_unlinked); 8301 dp->di_freelink = inon ? inon->id_ino : 0; 8302 } 8303 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == 8304 (UNLINKED | UNLINKNEXT)) { 8305 struct inodedep *inon; 8306 ino_t freelink; 8307 8308 inon = TAILQ_NEXT(inodedep, id_unlinked); 8309 freelink = inon ? inon->id_ino : 0; 8310 if (freelink != dp->di_freelink) 8311 panic("ino %p(0x%X) %d, %d != %d", 8312 inodedep, inodedep->id_state, inodedep->id_ino, 8313 freelink, dp->di_freelink); 8314 } 8315 /* 8316 * If the bitmap is not yet written, then the allocated 8317 * inode cannot be written to disk. 8318 */ 8319 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 8320 if (inodedep->id_savedino2 != NULL) 8321 panic("initiate_write_inodeblock_ufs2: I/O underway"); 8322 FREE_LOCK(&lk); 8323 sip = malloc(sizeof(struct ufs2_dinode), 8324 M_SAVEDINO, M_SOFTDEP_FLAGS); 8325 ACQUIRE_LOCK(&lk); 8326 inodedep->id_savedino2 = sip; 8327 *inodedep->id_savedino2 = *dp; 8328 bzero((caddr_t)dp, sizeof(struct ufs2_dinode)); 8329 dp->di_gen = inodedep->id_savedino2->di_gen; 8330 dp->di_freelink = inodedep->id_savedino2->di_freelink; 8331 return; 8332 } 8333 /* 8334 * If no dependencies, then there is nothing to roll back. 8335 */ 8336 inodedep->id_savedsize = dp->di_size; 8337 inodedep->id_savedextsize = dp->di_extsize; 8338 inodedep->id_savednlink = dp->di_nlink; 8339 if (TAILQ_EMPTY(&inodedep->id_inoupdt) && 8340 TAILQ_EMPTY(&inodedep->id_extupdt) && 8341 TAILQ_EMPTY(&inodedep->id_inoreflst)) 8342 return; 8343 /* 8344 * Revert the link count to that of the first unwritten journal entry. 8345 */ 8346 inoref = TAILQ_FIRST(&inodedep->id_inoreflst); 8347 if (inoref) 8348 dp->di_nlink = inoref->if_nlink; 8349 8350 /* 8351 * Set the ext data dependencies to busy. 8352 */ 8353 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 8354 adp = TAILQ_NEXT(adp, ad_next)) { 8355 #ifdef INVARIANTS 8356 if (deplist != 0 && prevlbn >= adp->ad_offset) 8357 panic("softdep_write_inodeblock: lbn order"); 8358 prevlbn = adp->ad_offset; 8359 if (dp->di_extb[adp->ad_offset] != adp->ad_newblkno) 8360 panic("%s: direct pointer #%jd mismatch %jd != %jd", 8361 "softdep_write_inodeblock", 8362 (intmax_t)adp->ad_offset, 8363 (intmax_t)dp->di_extb[adp->ad_offset], 8364 (intmax_t)adp->ad_newblkno); 8365 deplist |= 1 << adp->ad_offset; 8366 if ((adp->ad_state & ATTACHED) == 0) 8367 panic("softdep_write_inodeblock: Unknown state 0x%x", 8368 adp->ad_state); 8369 #endif /* INVARIANTS */ 8370 adp->ad_state &= ~ATTACHED; 8371 adp->ad_state |= UNDONE; 8372 } 8373 /* 8374 * The on-disk inode cannot claim to be any larger than the last 8375 * fragment that has been written. Otherwise, the on-disk inode 8376 * might have fragments that were not the last block in the ext 8377 * data which would corrupt the filesystem. 8378 */ 8379 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; 8380 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 8381 dp->di_extb[adp->ad_offset] = adp->ad_oldblkno; 8382 /* keep going until hitting a rollback to a frag */ 8383 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 8384 continue; 8385 dp->di_extsize = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 8386 for (i = adp->ad_offset + 1; i < NXADDR; i++) { 8387 #ifdef INVARIANTS 8388 if (dp->di_extb[i] != 0 && (deplist & (1 << i)) == 0) 8389 panic("softdep_write_inodeblock: lost dep1"); 8390 #endif /* INVARIANTS */ 8391 dp->di_extb[i] = 0; 8392 } 8393 lastadp = NULL; 8394 break; 8395 } 8396 /* 8397 * If we have zero'ed out the last allocated block of the ext 8398 * data, roll back the size to the last currently allocated block. 8399 * We know that this last allocated block is a full-sized as 8400 * we already checked for fragments in the loop above. 8401 */ 8402 if (lastadp != NULL && 8403 dp->di_extsize <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 8404 for (i = lastadp->ad_offset; i >= 0; i--) 8405 if (dp->di_extb[i] != 0) 8406 break; 8407 dp->di_extsize = (i + 1) * fs->fs_bsize; 8408 } 8409 /* 8410 * Set the file data dependencies to busy. 8411 */ 8412 for (deplist = 0, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8413 adp = TAILQ_NEXT(adp, ad_next)) { 8414 #ifdef INVARIANTS 8415 if (deplist != 0 && prevlbn >= adp->ad_offset) 8416 panic("softdep_write_inodeblock: lbn order"); 8417 prevlbn = adp->ad_offset; 8418 if (adp->ad_offset < NDADDR && 8419 dp->di_db[adp->ad_offset] != adp->ad_newblkno) 8420 panic("%s: direct pointer #%jd mismatch %jd != %jd", 8421 "softdep_write_inodeblock", 8422 (intmax_t)adp->ad_offset, 8423 (intmax_t)dp->di_db[adp->ad_offset], 8424 (intmax_t)adp->ad_newblkno); 8425 if (adp->ad_offset >= NDADDR && 8426 dp->di_ib[adp->ad_offset - NDADDR] != adp->ad_newblkno) 8427 panic("%s indirect pointer #%jd mismatch %jd != %jd", 8428 "softdep_write_inodeblock:", 8429 (intmax_t)adp->ad_offset - NDADDR, 8430 (intmax_t)dp->di_ib[adp->ad_offset - NDADDR], 8431 (intmax_t)adp->ad_newblkno); 8432 deplist |= 1 << adp->ad_offset; 8433 if ((adp->ad_state & ATTACHED) == 0) 8434 panic("softdep_write_inodeblock: Unknown state 0x%x", 8435 adp->ad_state); 8436 #endif /* INVARIANTS */ 8437 adp->ad_state &= ~ATTACHED; 8438 adp->ad_state |= UNDONE; 8439 } 8440 /* 8441 * The on-disk inode cannot claim to be any larger than the last 8442 * fragment that has been written. Otherwise, the on-disk inode 8443 * might have fragments that were not the last block in the file 8444 * which would corrupt the filesystem. 8445 */ 8446 for (lastadp = NULL, adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; 8447 lastadp = adp, adp = TAILQ_NEXT(adp, ad_next)) { 8448 if (adp->ad_offset >= NDADDR) 8449 break; 8450 dp->di_db[adp->ad_offset] = adp->ad_oldblkno; 8451 /* keep going until hitting a rollback to a frag */ 8452 if (adp->ad_oldsize == 0 || adp->ad_oldsize == fs->fs_bsize) 8453 continue; 8454 dp->di_size = fs->fs_bsize * adp->ad_offset + adp->ad_oldsize; 8455 for (i = adp->ad_offset + 1; i < NDADDR; i++) { 8456 #ifdef INVARIANTS 8457 if (dp->di_db[i] != 0 && (deplist & (1 << i)) == 0) 8458 panic("softdep_write_inodeblock: lost dep2"); 8459 #endif /* INVARIANTS */ 8460 dp->di_db[i] = 0; 8461 } 8462 for (i = 0; i < NIADDR; i++) { 8463 #ifdef INVARIANTS 8464 if (dp->di_ib[i] != 0 && 8465 (deplist & ((1 << NDADDR) << i)) == 0) 8466 panic("softdep_write_inodeblock: lost dep3"); 8467 #endif /* INVARIANTS */ 8468 dp->di_ib[i] = 0; 8469 } 8470 return; 8471 } 8472 /* 8473 * If we have zero'ed out the last allocated block of the file, 8474 * roll back the size to the last currently allocated block. 8475 * We know that this last allocated block is a full-sized as 8476 * we already checked for fragments in the loop above. 8477 */ 8478 if (lastadp != NULL && 8479 dp->di_size <= (lastadp->ad_offset + 1) * fs->fs_bsize) { 8480 for (i = lastadp->ad_offset; i >= 0; i--) 8481 if (dp->di_db[i] != 0) 8482 break; 8483 dp->di_size = (i + 1) * fs->fs_bsize; 8484 } 8485 /* 8486 * The only dependencies are for indirect blocks. 8487 * 8488 * The file size for indirect block additions is not guaranteed. 8489 * Such a guarantee would be non-trivial to achieve. The conventional 8490 * synchronous write implementation also does not make this guarantee. 8491 * Fsck should catch and fix discrepancies. Arguably, the file size 8492 * can be over-estimated without destroying integrity when the file 8493 * moves into the indirect blocks (i.e., is large). If we want to 8494 * postpone fsck, we are stuck with this argument. 8495 */ 8496 for (; adp; adp = TAILQ_NEXT(adp, ad_next)) 8497 dp->di_ib[adp->ad_offset - NDADDR] = 0; 8498 } 8499 8500 /* 8501 * Cancel an indirdep as a result of truncation. Release all of the 8502 * children allocindirs and place their journal work on the appropriate 8503 * list. 8504 */ 8505 static void 8506 cancel_indirdep(indirdep, bp, inodedep, freeblks) 8507 struct indirdep *indirdep; 8508 struct buf *bp; 8509 struct inodedep *inodedep; 8510 struct freeblks *freeblks; 8511 { 8512 struct allocindir *aip; 8513 8514 /* 8515 * None of the indirect pointers will ever be visible, 8516 * so they can simply be tossed. GOINGAWAY ensures 8517 * that allocated pointers will be saved in the buffer 8518 * cache until they are freed. Note that they will 8519 * only be able to be found by their physical address 8520 * since the inode mapping the logical address will 8521 * be gone. The save buffer used for the safe copy 8522 * was allocated in setup_allocindir_phase2 using 8523 * the physical address so it could be used for this 8524 * purpose. Hence we swap the safe copy with the real 8525 * copy, allowing the safe copy to be freed and holding 8526 * on to the real copy for later use in indir_trunc. 8527 */ 8528 if (indirdep->ir_state & GOINGAWAY) 8529 panic("cancel_indirdep: already gone"); 8530 if (indirdep->ir_state & ONDEPLIST) { 8531 indirdep->ir_state &= ~ONDEPLIST; 8532 LIST_REMOVE(indirdep, ir_next); 8533 } 8534 indirdep->ir_state |= GOINGAWAY; 8535 VFSTOUFS(indirdep->ir_list.wk_mp)->um_numindirdeps += 1; 8536 while ((aip = LIST_FIRST(&indirdep->ir_deplisthd)) != 0) 8537 cancel_allocindir(aip, inodedep, freeblks); 8538 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != 0) 8539 cancel_allocindir(aip, inodedep, freeblks); 8540 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != 0) 8541 cancel_allocindir(aip, inodedep, freeblks); 8542 while ((aip = LIST_FIRST(&indirdep->ir_completehd)) != 0) 8543 cancel_allocindir(aip, inodedep, freeblks); 8544 bcopy(bp->b_data, indirdep->ir_savebp->b_data, bp->b_bcount); 8545 WORKLIST_REMOVE(&indirdep->ir_list); 8546 WORKLIST_INSERT(&indirdep->ir_savebp->b_dep, &indirdep->ir_list); 8547 indirdep->ir_savebp = NULL; 8548 } 8549 8550 /* 8551 * Free an indirdep once it no longer has new pointers to track. 8552 */ 8553 static void 8554 free_indirdep(indirdep) 8555 struct indirdep *indirdep; 8556 { 8557 8558 KASSERT(LIST_EMPTY(&indirdep->ir_jwork), 8559 ("free_indirdep: Journal work not empty.")); 8560 KASSERT(LIST_EMPTY(&indirdep->ir_completehd), 8561 ("free_indirdep: Complete head not empty.")); 8562 KASSERT(LIST_EMPTY(&indirdep->ir_writehd), 8563 ("free_indirdep: write head not empty.")); 8564 KASSERT(LIST_EMPTY(&indirdep->ir_donehd), 8565 ("free_indirdep: done head not empty.")); 8566 KASSERT(LIST_EMPTY(&indirdep->ir_deplisthd), 8567 ("free_indirdep: deplist head not empty.")); 8568 KASSERT(indirdep->ir_savebp == NULL, 8569 ("free_indirdep: %p ir_savebp != NULL", indirdep)); 8570 KASSERT((indirdep->ir_state & ONDEPLIST) == 0, 8571 ("free_indirdep: %p still on deplist.", indirdep)); 8572 if (indirdep->ir_state & ONWORKLIST) 8573 WORKLIST_REMOVE(&indirdep->ir_list); 8574 WORKITEM_FREE(indirdep, D_INDIRDEP); 8575 } 8576 8577 /* 8578 * Called before a write to an indirdep. This routine is responsible for 8579 * rolling back pointers to a safe state which includes only those 8580 * allocindirs which have been completed. 8581 */ 8582 static void 8583 initiate_write_indirdep(indirdep, bp) 8584 struct indirdep *indirdep; 8585 struct buf *bp; 8586 { 8587 8588 if (indirdep->ir_state & GOINGAWAY) 8589 panic("disk_io_initiation: indirdep gone"); 8590 8591 /* 8592 * If there are no remaining dependencies, this will be writing 8593 * the real pointers. 8594 */ 8595 if (LIST_EMPTY(&indirdep->ir_deplisthd)) 8596 return; 8597 /* 8598 * Replace up-to-date version with safe version. 8599 */ 8600 FREE_LOCK(&lk); 8601 indirdep->ir_saveddata = malloc(bp->b_bcount, M_INDIRDEP, 8602 M_SOFTDEP_FLAGS); 8603 ACQUIRE_LOCK(&lk); 8604 indirdep->ir_state &= ~ATTACHED; 8605 indirdep->ir_state |= UNDONE; 8606 bcopy(bp->b_data, indirdep->ir_saveddata, bp->b_bcount); 8607 bcopy(indirdep->ir_savebp->b_data, bp->b_data, 8608 bp->b_bcount); 8609 } 8610 8611 /* 8612 * Called when an inode has been cleared in a cg bitmap. This finally 8613 * eliminates any canceled jaddrefs 8614 */ 8615 void 8616 softdep_setup_inofree(mp, bp, ino, wkhd) 8617 struct mount *mp; 8618 struct buf *bp; 8619 ino_t ino; 8620 struct workhead *wkhd; 8621 { 8622 struct worklist *wk, *wkn; 8623 struct inodedep *inodedep; 8624 uint8_t *inosused; 8625 struct cg *cgp; 8626 struct fs *fs; 8627 8628 ACQUIRE_LOCK(&lk); 8629 fs = VFSTOUFS(mp)->um_fs; 8630 cgp = (struct cg *)bp->b_data; 8631 inosused = cg_inosused(cgp); 8632 if (isset(inosused, ino % fs->fs_ipg)) 8633 panic("softdep_setup_inofree: inode %d not freed.", ino); 8634 if (inodedep_lookup(mp, ino, 0, &inodedep)) 8635 panic("softdep_setup_inofree: ino %d has existing inodedep %p", 8636 ino, inodedep); 8637 if (wkhd) { 8638 LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) { 8639 if (wk->wk_type != D_JADDREF) 8640 continue; 8641 WORKLIST_REMOVE(wk); 8642 /* 8643 * We can free immediately even if the jaddref 8644 * isn't attached in a background write as now 8645 * the bitmaps are reconciled. 8646 */ 8647 wk->wk_state |= COMPLETE | ATTACHED; 8648 free_jaddref(WK_JADDREF(wk)); 8649 } 8650 jwork_move(&bp->b_dep, wkhd); 8651 } 8652 FREE_LOCK(&lk); 8653 } 8654 8655 8656 /* 8657 * Called via ffs_blkfree() after a set of frags has been cleared from a cg 8658 * map. Any dependencies waiting for the write to clear are added to the 8659 * buf's list and any jnewblks that are being canceled are discarded 8660 * immediately. 8661 */ 8662 void 8663 softdep_setup_blkfree(mp, bp, blkno, frags, wkhd) 8664 struct mount *mp; 8665 struct buf *bp; 8666 ufs2_daddr_t blkno; 8667 int frags; 8668 struct workhead *wkhd; 8669 { 8670 struct jnewblk *jnewblk; 8671 struct worklist *wk, *wkn; 8672 #ifdef SUJ_DEBUG 8673 struct bmsafemap *bmsafemap; 8674 struct fs *fs; 8675 uint8_t *blksfree; 8676 struct cg *cgp; 8677 ufs2_daddr_t jstart; 8678 ufs2_daddr_t jend; 8679 ufs2_daddr_t end; 8680 long bno; 8681 int i; 8682 #endif 8683 8684 ACQUIRE_LOCK(&lk); 8685 /* 8686 * Detach any jnewblks which have been canceled. They must linger 8687 * until the bitmap is cleared again by ffs_blkfree() to prevent 8688 * an unjournaled allocation from hitting the disk. 8689 */ 8690 if (wkhd) { 8691 LIST_FOREACH_SAFE(wk, wkhd, wk_list, wkn) { 8692 if (wk->wk_type != D_JNEWBLK) 8693 continue; 8694 jnewblk = WK_JNEWBLK(wk); 8695 KASSERT(jnewblk->jn_state & GOINGAWAY, 8696 ("softdep_setup_blkfree: jnewblk not canceled.")); 8697 WORKLIST_REMOVE(wk); 8698 #ifdef SUJ_DEBUG 8699 /* 8700 * Assert that this block is free in the bitmap 8701 * before we discard the jnewblk. 8702 */ 8703 fs = VFSTOUFS(mp)->um_fs; 8704 cgp = (struct cg *)bp->b_data; 8705 blksfree = cg_blksfree(cgp); 8706 bno = dtogd(fs, jnewblk->jn_blkno); 8707 for (i = jnewblk->jn_oldfrags; 8708 i < jnewblk->jn_frags; i++) { 8709 if (isset(blksfree, bno + i)) 8710 continue; 8711 panic("softdep_setup_blkfree: not free"); 8712 } 8713 #endif 8714 /* 8715 * Even if it's not attached we can free immediately 8716 * as the new bitmap is correct. 8717 */ 8718 wk->wk_state |= COMPLETE | ATTACHED; 8719 free_jnewblk(jnewblk); 8720 } 8721 /* 8722 * The buf must be locked by the caller otherwise these could 8723 * be added while it's being written and the write would 8724 * complete them before they made it to disk. 8725 */ 8726 jwork_move(&bp->b_dep, wkhd); 8727 } 8728 8729 #ifdef SUJ_DEBUG 8730 /* 8731 * Assert that we are not freeing a block which has an outstanding 8732 * allocation dependency. 8733 */ 8734 fs = VFSTOUFS(mp)->um_fs; 8735 bmsafemap = bmsafemap_lookup(mp, bp, dtog(fs, blkno)); 8736 end = blkno + frags; 8737 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 8738 /* 8739 * Don't match against blocks that will be freed when the 8740 * background write is done. 8741 */ 8742 if ((jnewblk->jn_state & (ATTACHED | COMPLETE | DEPCOMPLETE)) == 8743 (COMPLETE | DEPCOMPLETE)) 8744 continue; 8745 jstart = jnewblk->jn_blkno + jnewblk->jn_oldfrags; 8746 jend = jnewblk->jn_blkno + jnewblk->jn_frags; 8747 if ((blkno >= jstart && blkno < jend) || 8748 (end > jstart && end <= jend)) { 8749 printf("state 0x%X %jd - %d %d dep %p\n", 8750 jnewblk->jn_state, jnewblk->jn_blkno, 8751 jnewblk->jn_oldfrags, jnewblk->jn_frags, 8752 jnewblk->jn_newblk); 8753 panic("softdep_setup_blkfree: " 8754 "%jd-%jd(%d) overlaps with %jd-%jd", 8755 blkno, end, frags, jstart, jend); 8756 } 8757 } 8758 #endif 8759 FREE_LOCK(&lk); 8760 } 8761 8762 static void 8763 initiate_write_bmsafemap(bmsafemap, bp) 8764 struct bmsafemap *bmsafemap; 8765 struct buf *bp; /* The cg block. */ 8766 { 8767 struct jaddref *jaddref; 8768 struct jnewblk *jnewblk; 8769 uint8_t *inosused; 8770 uint8_t *blksfree; 8771 struct cg *cgp; 8772 struct fs *fs; 8773 int cleared; 8774 ino_t ino; 8775 long bno; 8776 int i; 8777 8778 if (bmsafemap->sm_state & IOSTARTED) 8779 panic("initiate_write_bmsafemap: Already started\n"); 8780 bmsafemap->sm_state |= IOSTARTED; 8781 /* 8782 * Clear any inode allocations which are pending journal writes. 8783 */ 8784 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd) != NULL) { 8785 cgp = (struct cg *)bp->b_data; 8786 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 8787 inosused = cg_inosused(cgp); 8788 LIST_FOREACH(jaddref, &bmsafemap->sm_jaddrefhd, ja_bmdeps) { 8789 ino = jaddref->ja_ino % fs->fs_ipg; 8790 /* 8791 * If this is a background copy the inode may not 8792 * be marked used yet. 8793 */ 8794 if (isset(inosused, ino)) { 8795 if ((jaddref->ja_mode & IFMT) == IFDIR) 8796 cgp->cg_cs.cs_ndir--; 8797 cgp->cg_cs.cs_nifree++; 8798 clrbit(inosused, ino); 8799 jaddref->ja_state &= ~ATTACHED; 8800 jaddref->ja_state |= UNDONE; 8801 stat_jaddref++; 8802 } else if ((bp->b_xflags & BX_BKGRDMARKER) == 0) 8803 panic("initiate_write_bmsafemap: inode %d " 8804 "marked free", jaddref->ja_ino); 8805 } 8806 } 8807 /* 8808 * Clear any block allocations which are pending journal writes. 8809 */ 8810 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 8811 cgp = (struct cg *)bp->b_data; 8812 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 8813 blksfree = cg_blksfree(cgp); 8814 LIST_FOREACH(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps) { 8815 bno = dtogd(fs, jnewblk->jn_blkno); 8816 cleared = 0; 8817 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 8818 i++) { 8819 if (isclr(blksfree, bno + i)) { 8820 cleared = 1; 8821 setbit(blksfree, bno + i); 8822 } 8823 } 8824 /* 8825 * We may not clear the block if it's a background 8826 * copy. In that case there is no reason to detach 8827 * it. 8828 */ 8829 if (cleared) { 8830 stat_jnewblk++; 8831 jnewblk->jn_state &= ~ATTACHED; 8832 jnewblk->jn_state |= UNDONE; 8833 } else if ((bp->b_xflags & BX_BKGRDMARKER) == 0) 8834 panic("initiate_write_bmsafemap: block %jd " 8835 "marked free", jnewblk->jn_blkno); 8836 } 8837 } 8838 /* 8839 * Move allocation lists to the written lists so they can be 8840 * cleared once the block write is complete. 8841 */ 8842 LIST_SWAP(&bmsafemap->sm_inodedephd, &bmsafemap->sm_inodedepwr, 8843 inodedep, id_deps); 8844 LIST_SWAP(&bmsafemap->sm_newblkhd, &bmsafemap->sm_newblkwr, 8845 newblk, nb_deps); 8846 } 8847 8848 /* 8849 * This routine is called during the completion interrupt 8850 * service routine for a disk write (from the procedure called 8851 * by the device driver to inform the filesystem caches of 8852 * a request completion). It should be called early in this 8853 * procedure, before the block is made available to other 8854 * processes or other routines are called. 8855 * 8856 */ 8857 static void 8858 softdep_disk_write_complete(bp) 8859 struct buf *bp; /* describes the completed disk write */ 8860 { 8861 struct worklist *wk; 8862 struct worklist *owk; 8863 struct workhead reattach; 8864 struct buf *sbp; 8865 8866 /* 8867 * If an error occurred while doing the write, then the data 8868 * has not hit the disk and the dependencies cannot be unrolled. 8869 */ 8870 if ((bp->b_ioflags & BIO_ERROR) != 0 && (bp->b_flags & B_INVAL) == 0) 8871 return; 8872 LIST_INIT(&reattach); 8873 /* 8874 * This lock must not be released anywhere in this code segment. 8875 */ 8876 sbp = NULL; 8877 owk = NULL; 8878 ACQUIRE_LOCK(&lk); 8879 while ((wk = LIST_FIRST(&bp->b_dep)) != NULL) { 8880 WORKLIST_REMOVE(wk); 8881 if (wk == owk) 8882 panic("duplicate worklist: %p\n", wk); 8883 owk = wk; 8884 switch (wk->wk_type) { 8885 8886 case D_PAGEDEP: 8887 if (handle_written_filepage(WK_PAGEDEP(wk), bp)) 8888 WORKLIST_INSERT(&reattach, wk); 8889 continue; 8890 8891 case D_INODEDEP: 8892 if (handle_written_inodeblock(WK_INODEDEP(wk), bp)) 8893 WORKLIST_INSERT(&reattach, wk); 8894 continue; 8895 8896 case D_BMSAFEMAP: 8897 if (handle_written_bmsafemap(WK_BMSAFEMAP(wk), bp)) 8898 WORKLIST_INSERT(&reattach, wk); 8899 continue; 8900 8901 case D_MKDIR: 8902 handle_written_mkdir(WK_MKDIR(wk), MKDIR_BODY); 8903 continue; 8904 8905 case D_ALLOCDIRECT: 8906 wk->wk_state |= COMPLETE; 8907 handle_allocdirect_partdone(WK_ALLOCDIRECT(wk), NULL); 8908 continue; 8909 8910 case D_ALLOCINDIR: 8911 wk->wk_state |= COMPLETE; 8912 handle_allocindir_partdone(WK_ALLOCINDIR(wk)); 8913 continue; 8914 8915 case D_INDIRDEP: 8916 if (handle_written_indirdep(WK_INDIRDEP(wk), bp, &sbp)) 8917 WORKLIST_INSERT(&reattach, wk); 8918 continue; 8919 8920 case D_FREEBLKS: 8921 wk->wk_state |= COMPLETE; 8922 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE) 8923 add_to_worklist(wk, 1); 8924 continue; 8925 8926 case D_FREEWORK: 8927 handle_written_freework(WK_FREEWORK(wk)); 8928 break; 8929 8930 case D_FREEDEP: 8931 free_freedep(WK_FREEDEP(wk)); 8932 continue; 8933 8934 case D_JSEGDEP: 8935 free_jsegdep(WK_JSEGDEP(wk)); 8936 continue; 8937 8938 case D_JSEG: 8939 handle_written_jseg(WK_JSEG(wk), bp); 8940 continue; 8941 8942 case D_SBDEP: 8943 if (handle_written_sbdep(WK_SBDEP(wk), bp)) 8944 WORKLIST_INSERT(&reattach, wk); 8945 continue; 8946 8947 default: 8948 panic("handle_disk_write_complete: Unknown type %s", 8949 TYPENAME(wk->wk_type)); 8950 /* NOTREACHED */ 8951 } 8952 } 8953 /* 8954 * Reattach any requests that must be redone. 8955 */ 8956 while ((wk = LIST_FIRST(&reattach)) != NULL) { 8957 WORKLIST_REMOVE(wk); 8958 WORKLIST_INSERT(&bp->b_dep, wk); 8959 } 8960 FREE_LOCK(&lk); 8961 if (sbp) 8962 brelse(sbp); 8963 } 8964 8965 /* 8966 * Called from within softdep_disk_write_complete above. Note that 8967 * this routine is always called from interrupt level with further 8968 * splbio interrupts blocked. 8969 */ 8970 static void 8971 handle_allocdirect_partdone(adp, wkhd) 8972 struct allocdirect *adp; /* the completed allocdirect */ 8973 struct workhead *wkhd; /* Work to do when inode is writtne. */ 8974 { 8975 struct allocdirectlst *listhead; 8976 struct allocdirect *listadp; 8977 struct inodedep *inodedep; 8978 long bsize; 8979 8980 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 8981 return; 8982 /* 8983 * The on-disk inode cannot claim to be any larger than the last 8984 * fragment that has been written. Otherwise, the on-disk inode 8985 * might have fragments that were not the last block in the file 8986 * which would corrupt the filesystem. Thus, we cannot free any 8987 * allocdirects after one whose ad_oldblkno claims a fragment as 8988 * these blocks must be rolled back to zero before writing the inode. 8989 * We check the currently active set of allocdirects in id_inoupdt 8990 * or id_extupdt as appropriate. 8991 */ 8992 inodedep = adp->ad_inodedep; 8993 bsize = inodedep->id_fs->fs_bsize; 8994 if (adp->ad_state & EXTDATA) 8995 listhead = &inodedep->id_extupdt; 8996 else 8997 listhead = &inodedep->id_inoupdt; 8998 TAILQ_FOREACH(listadp, listhead, ad_next) { 8999 /* found our block */ 9000 if (listadp == adp) 9001 break; 9002 /* continue if ad_oldlbn is not a fragment */ 9003 if (listadp->ad_oldsize == 0 || 9004 listadp->ad_oldsize == bsize) 9005 continue; 9006 /* hit a fragment */ 9007 return; 9008 } 9009 /* 9010 * If we have reached the end of the current list without 9011 * finding the just finished dependency, then it must be 9012 * on the future dependency list. Future dependencies cannot 9013 * be freed until they are moved to the current list. 9014 */ 9015 if (listadp == NULL) { 9016 #ifdef DEBUG 9017 if (adp->ad_state & EXTDATA) 9018 listhead = &inodedep->id_newextupdt; 9019 else 9020 listhead = &inodedep->id_newinoupdt; 9021 TAILQ_FOREACH(listadp, listhead, ad_next) 9022 /* found our block */ 9023 if (listadp == adp) 9024 break; 9025 if (listadp == NULL) 9026 panic("handle_allocdirect_partdone: lost dep"); 9027 #endif /* DEBUG */ 9028 return; 9029 } 9030 /* 9031 * If we have found the just finished dependency, then queue 9032 * it along with anything that follows it that is complete. 9033 * Since the pointer has not yet been written in the inode 9034 * as the dependency prevents it, place the allocdirect on the 9035 * bufwait list where it will be freed once the pointer is 9036 * valid. 9037 */ 9038 if (wkhd == NULL) 9039 wkhd = &inodedep->id_bufwait; 9040 for (; adp; adp = listadp) { 9041 listadp = TAILQ_NEXT(adp, ad_next); 9042 if ((adp->ad_state & ALLCOMPLETE) != ALLCOMPLETE) 9043 return; 9044 TAILQ_REMOVE(listhead, adp, ad_next); 9045 WORKLIST_INSERT(wkhd, &adp->ad_block.nb_list); 9046 } 9047 } 9048 9049 /* 9050 * Called from within softdep_disk_write_complete above. This routine 9051 * completes successfully written allocindirs. 9052 */ 9053 static void 9054 handle_allocindir_partdone(aip) 9055 struct allocindir *aip; /* the completed allocindir */ 9056 { 9057 struct indirdep *indirdep; 9058 9059 if ((aip->ai_state & ALLCOMPLETE) != ALLCOMPLETE) 9060 return; 9061 indirdep = aip->ai_indirdep; 9062 LIST_REMOVE(aip, ai_next); 9063 if (indirdep->ir_state & UNDONE) { 9064 LIST_INSERT_HEAD(&indirdep->ir_donehd, aip, ai_next); 9065 return; 9066 } 9067 if (indirdep->ir_state & UFS1FMT) 9068 ((ufs1_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 9069 aip->ai_newblkno; 9070 else 9071 ((ufs2_daddr_t *)indirdep->ir_savebp->b_data)[aip->ai_offset] = 9072 aip->ai_newblkno; 9073 /* 9074 * Await the pointer write before freeing the allocindir. 9075 */ 9076 LIST_INSERT_HEAD(&indirdep->ir_writehd, aip, ai_next); 9077 } 9078 9079 /* 9080 * Release segments held on a jwork list. 9081 */ 9082 static void 9083 handle_jwork(wkhd) 9084 struct workhead *wkhd; 9085 { 9086 struct worklist *wk; 9087 9088 while ((wk = LIST_FIRST(wkhd)) != NULL) { 9089 WORKLIST_REMOVE(wk); 9090 switch (wk->wk_type) { 9091 case D_JSEGDEP: 9092 free_jsegdep(WK_JSEGDEP(wk)); 9093 continue; 9094 default: 9095 panic("handle_jwork: Unknown type %s\n", 9096 TYPENAME(wk->wk_type)); 9097 } 9098 } 9099 } 9100 9101 /* 9102 * Handle the bufwait list on an inode when it is safe to release items 9103 * held there. This normally happens after an inode block is written but 9104 * may be delayed and handled later if there are pending journal items that 9105 * are not yet safe to be released. 9106 */ 9107 static struct freefile * 9108 handle_bufwait(inodedep, refhd) 9109 struct inodedep *inodedep; 9110 struct workhead *refhd; 9111 { 9112 struct jaddref *jaddref; 9113 struct freefile *freefile; 9114 struct worklist *wk; 9115 9116 freefile = NULL; 9117 while ((wk = LIST_FIRST(&inodedep->id_bufwait)) != NULL) { 9118 WORKLIST_REMOVE(wk); 9119 switch (wk->wk_type) { 9120 case D_FREEFILE: 9121 /* 9122 * We defer adding freefile to the worklist 9123 * until all other additions have been made to 9124 * ensure that it will be done after all the 9125 * old blocks have been freed. 9126 */ 9127 if (freefile != NULL) 9128 panic("handle_bufwait: freefile"); 9129 freefile = WK_FREEFILE(wk); 9130 continue; 9131 9132 case D_MKDIR: 9133 handle_written_mkdir(WK_MKDIR(wk), MKDIR_PARENT); 9134 continue; 9135 9136 case D_DIRADD: 9137 diradd_inode_written(WK_DIRADD(wk), inodedep); 9138 continue; 9139 9140 case D_FREEFRAG: 9141 wk->wk_state |= COMPLETE; 9142 if ((wk->wk_state & ALLCOMPLETE) == ALLCOMPLETE) 9143 add_to_worklist(wk, 0); 9144 continue; 9145 9146 case D_DIRREM: 9147 wk->wk_state |= COMPLETE; 9148 add_to_worklist(wk, 0); 9149 continue; 9150 9151 case D_ALLOCDIRECT: 9152 case D_ALLOCINDIR: 9153 free_newblk(WK_NEWBLK(wk)); 9154 continue; 9155 9156 case D_JNEWBLK: 9157 wk->wk_state |= COMPLETE; 9158 free_jnewblk(WK_JNEWBLK(wk)); 9159 continue; 9160 9161 /* 9162 * Save freed journal segments and add references on 9163 * the supplied list which will delay their release 9164 * until the cg bitmap is cleared on disk. 9165 */ 9166 case D_JSEGDEP: 9167 if (refhd == NULL) 9168 free_jsegdep(WK_JSEGDEP(wk)); 9169 else 9170 WORKLIST_INSERT(refhd, wk); 9171 continue; 9172 9173 case D_JADDREF: 9174 jaddref = WK_JADDREF(wk); 9175 TAILQ_REMOVE(&inodedep->id_inoreflst, &jaddref->ja_ref, 9176 if_deps); 9177 /* 9178 * Transfer any jaddrefs to the list to be freed with 9179 * the bitmap if we're handling a removed file. 9180 */ 9181 if (refhd == NULL) { 9182 wk->wk_state |= COMPLETE; 9183 free_jaddref(jaddref); 9184 } else 9185 WORKLIST_INSERT(refhd, wk); 9186 continue; 9187 9188 default: 9189 panic("handle_bufwait: Unknown type %p(%s)", 9190 wk, TYPENAME(wk->wk_type)); 9191 /* NOTREACHED */ 9192 } 9193 } 9194 return (freefile); 9195 } 9196 /* 9197 * Called from within softdep_disk_write_complete above to restore 9198 * in-memory inode block contents to their most up-to-date state. Note 9199 * that this routine is always called from interrupt level with further 9200 * splbio interrupts blocked. 9201 */ 9202 static int 9203 handle_written_inodeblock(inodedep, bp) 9204 struct inodedep *inodedep; 9205 struct buf *bp; /* buffer containing the inode block */ 9206 { 9207 struct freefile *freefile; 9208 struct allocdirect *adp, *nextadp; 9209 struct ufs1_dinode *dp1 = NULL; 9210 struct ufs2_dinode *dp2 = NULL; 9211 struct workhead wkhd; 9212 int hadchanges, fstype; 9213 ino_t freelink; 9214 9215 LIST_INIT(&wkhd); 9216 hadchanges = 0; 9217 freefile = NULL; 9218 if ((inodedep->id_state & IOSTARTED) == 0) 9219 panic("handle_written_inodeblock: not started"); 9220 inodedep->id_state &= ~IOSTARTED; 9221 if (inodedep->id_fs->fs_magic == FS_UFS1_MAGIC) { 9222 fstype = UFS1; 9223 dp1 = (struct ufs1_dinode *)bp->b_data + 9224 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 9225 freelink = dp1->di_freelink; 9226 } else { 9227 fstype = UFS2; 9228 dp2 = (struct ufs2_dinode *)bp->b_data + 9229 ino_to_fsbo(inodedep->id_fs, inodedep->id_ino); 9230 freelink = dp2->di_freelink; 9231 } 9232 /* 9233 * If we wrote a valid freelink pointer during the last write 9234 * record it here. 9235 */ 9236 if ((inodedep->id_state & (UNLINKED | UNLINKNEXT)) == UNLINKED) { 9237 struct inodedep *inon; 9238 9239 inon = TAILQ_NEXT(inodedep, id_unlinked); 9240 if ((inon == NULL && freelink == 0) || 9241 (inon && inon->id_ino == freelink)) { 9242 if (inon) 9243 inon->id_state |= UNLINKPREV; 9244 inodedep->id_state |= UNLINKNEXT; 9245 } else 9246 hadchanges = 1; 9247 } 9248 /* Leave this inodeblock dirty until it's in the list. */ 9249 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) == UNLINKED) 9250 hadchanges = 1; 9251 /* 9252 * If we had to rollback the inode allocation because of 9253 * bitmaps being incomplete, then simply restore it. 9254 * Keep the block dirty so that it will not be reclaimed until 9255 * all associated dependencies have been cleared and the 9256 * corresponding updates written to disk. 9257 */ 9258 if (inodedep->id_savedino1 != NULL) { 9259 hadchanges = 1; 9260 if (fstype == UFS1) 9261 *dp1 = *inodedep->id_savedino1; 9262 else 9263 *dp2 = *inodedep->id_savedino2; 9264 free(inodedep->id_savedino1, M_SAVEDINO); 9265 inodedep->id_savedino1 = NULL; 9266 if ((bp->b_flags & B_DELWRI) == 0) 9267 stat_inode_bitmap++; 9268 bdirty(bp); 9269 /* 9270 * If the inode is clear here and GOINGAWAY it will never 9271 * be written. Process the bufwait and clear any pending 9272 * work which may include the freefile. 9273 */ 9274 if (inodedep->id_state & GOINGAWAY) 9275 goto bufwait; 9276 return (1); 9277 } 9278 inodedep->id_state |= COMPLETE; 9279 /* 9280 * Roll forward anything that had to be rolled back before 9281 * the inode could be updated. 9282 */ 9283 for (adp = TAILQ_FIRST(&inodedep->id_inoupdt); adp; adp = nextadp) { 9284 nextadp = TAILQ_NEXT(adp, ad_next); 9285 if (adp->ad_state & ATTACHED) 9286 panic("handle_written_inodeblock: new entry"); 9287 if (fstype == UFS1) { 9288 if (adp->ad_offset < NDADDR) { 9289 if (dp1->di_db[adp->ad_offset]!=adp->ad_oldblkno) 9290 panic("%s %s #%jd mismatch %d != %jd", 9291 "handle_written_inodeblock:", 9292 "direct pointer", 9293 (intmax_t)adp->ad_offset, 9294 dp1->di_db[adp->ad_offset], 9295 (intmax_t)adp->ad_oldblkno); 9296 dp1->di_db[adp->ad_offset] = adp->ad_newblkno; 9297 } else { 9298 if (dp1->di_ib[adp->ad_offset - NDADDR] != 0) 9299 panic("%s: %s #%jd allocated as %d", 9300 "handle_written_inodeblock", 9301 "indirect pointer", 9302 (intmax_t)adp->ad_offset - NDADDR, 9303 dp1->di_ib[adp->ad_offset - NDADDR]); 9304 dp1->di_ib[adp->ad_offset - NDADDR] = 9305 adp->ad_newblkno; 9306 } 9307 } else { 9308 if (adp->ad_offset < NDADDR) { 9309 if (dp2->di_db[adp->ad_offset]!=adp->ad_oldblkno) 9310 panic("%s: %s #%jd %s %jd != %jd", 9311 "handle_written_inodeblock", 9312 "direct pointer", 9313 (intmax_t)adp->ad_offset, "mismatch", 9314 (intmax_t)dp2->di_db[adp->ad_offset], 9315 (intmax_t)adp->ad_oldblkno); 9316 dp2->di_db[adp->ad_offset] = adp->ad_newblkno; 9317 } else { 9318 if (dp2->di_ib[adp->ad_offset - NDADDR] != 0) 9319 panic("%s: %s #%jd allocated as %jd", 9320 "handle_written_inodeblock", 9321 "indirect pointer", 9322 (intmax_t)adp->ad_offset - NDADDR, 9323 (intmax_t) 9324 dp2->di_ib[adp->ad_offset - NDADDR]); 9325 dp2->di_ib[adp->ad_offset - NDADDR] = 9326 adp->ad_newblkno; 9327 } 9328 } 9329 adp->ad_state &= ~UNDONE; 9330 adp->ad_state |= ATTACHED; 9331 hadchanges = 1; 9332 } 9333 for (adp = TAILQ_FIRST(&inodedep->id_extupdt); adp; adp = nextadp) { 9334 nextadp = TAILQ_NEXT(adp, ad_next); 9335 if (adp->ad_state & ATTACHED) 9336 panic("handle_written_inodeblock: new entry"); 9337 if (dp2->di_extb[adp->ad_offset] != adp->ad_oldblkno) 9338 panic("%s: direct pointers #%jd %s %jd != %jd", 9339 "handle_written_inodeblock", 9340 (intmax_t)adp->ad_offset, "mismatch", 9341 (intmax_t)dp2->di_extb[adp->ad_offset], 9342 (intmax_t)adp->ad_oldblkno); 9343 dp2->di_extb[adp->ad_offset] = adp->ad_newblkno; 9344 adp->ad_state &= ~UNDONE; 9345 adp->ad_state |= ATTACHED; 9346 hadchanges = 1; 9347 } 9348 if (hadchanges && (bp->b_flags & B_DELWRI) == 0) 9349 stat_direct_blk_ptrs++; 9350 /* 9351 * Reset the file size to its most up-to-date value. 9352 */ 9353 if (inodedep->id_savedsize == -1 || inodedep->id_savedextsize == -1) 9354 panic("handle_written_inodeblock: bad size"); 9355 if (inodedep->id_savednlink > LINK_MAX) 9356 panic("handle_written_inodeblock: Invalid link count " 9357 "%d for inodedep %p", inodedep->id_savednlink, inodedep); 9358 if (fstype == UFS1) { 9359 if (dp1->di_nlink != inodedep->id_savednlink) { 9360 dp1->di_nlink = inodedep->id_savednlink; 9361 hadchanges = 1; 9362 } 9363 if (dp1->di_size != inodedep->id_savedsize) { 9364 dp1->di_size = inodedep->id_savedsize; 9365 hadchanges = 1; 9366 } 9367 } else { 9368 if (dp2->di_nlink != inodedep->id_savednlink) { 9369 dp2->di_nlink = inodedep->id_savednlink; 9370 hadchanges = 1; 9371 } 9372 if (dp2->di_size != inodedep->id_savedsize) { 9373 dp2->di_size = inodedep->id_savedsize; 9374 hadchanges = 1; 9375 } 9376 if (dp2->di_extsize != inodedep->id_savedextsize) { 9377 dp2->di_extsize = inodedep->id_savedextsize; 9378 hadchanges = 1; 9379 } 9380 } 9381 inodedep->id_savedsize = -1; 9382 inodedep->id_savedextsize = -1; 9383 inodedep->id_savednlink = -1; 9384 /* 9385 * If there were any rollbacks in the inode block, then it must be 9386 * marked dirty so that its will eventually get written back in 9387 * its correct form. 9388 */ 9389 if (hadchanges) 9390 bdirty(bp); 9391 bufwait: 9392 /* 9393 * Process any allocdirects that completed during the update. 9394 */ 9395 if ((adp = TAILQ_FIRST(&inodedep->id_inoupdt)) != NULL) 9396 handle_allocdirect_partdone(adp, &wkhd); 9397 if ((adp = TAILQ_FIRST(&inodedep->id_extupdt)) != NULL) 9398 handle_allocdirect_partdone(adp, &wkhd); 9399 /* 9400 * Process deallocations that were held pending until the 9401 * inode had been written to disk. Freeing of the inode 9402 * is delayed until after all blocks have been freed to 9403 * avoid creation of new <vfsid, inum, lbn> triples 9404 * before the old ones have been deleted. Completely 9405 * unlinked inodes are not processed until the unlinked 9406 * inode list is written or the last reference is removed. 9407 */ 9408 if ((inodedep->id_state & (UNLINKED | UNLINKONLIST)) != UNLINKED) { 9409 freefile = handle_bufwait(inodedep, NULL); 9410 if (freefile && !LIST_EMPTY(&wkhd)) { 9411 WORKLIST_INSERT(&wkhd, &freefile->fx_list); 9412 freefile = NULL; 9413 } 9414 } 9415 /* 9416 * Move rolled forward dependency completions to the bufwait list 9417 * now that those that were already written have been processed. 9418 */ 9419 if (!LIST_EMPTY(&wkhd) && hadchanges == 0) 9420 panic("handle_written_inodeblock: bufwait but no changes"); 9421 jwork_move(&inodedep->id_bufwait, &wkhd); 9422 9423 if (freefile != NULL) { 9424 /* 9425 * If the inode is goingaway it was never written. Fake up 9426 * the state here so free_inodedep() can succeed. 9427 */ 9428 if (inodedep->id_state & GOINGAWAY) 9429 inodedep->id_state |= COMPLETE | DEPCOMPLETE; 9430 if (free_inodedep(inodedep) == 0) 9431 panic("handle_written_inodeblock: live inodedep %p", 9432 inodedep); 9433 add_to_worklist(&freefile->fx_list, 0); 9434 return (0); 9435 } 9436 9437 /* 9438 * If no outstanding dependencies, free it. 9439 */ 9440 if (free_inodedep(inodedep) || 9441 (TAILQ_FIRST(&inodedep->id_inoreflst) == 0 && 9442 TAILQ_FIRST(&inodedep->id_inoupdt) == 0 && 9443 TAILQ_FIRST(&inodedep->id_extupdt) == 0 && 9444 LIST_FIRST(&inodedep->id_bufwait) == 0)) 9445 return (0); 9446 return (hadchanges); 9447 } 9448 9449 static int 9450 handle_written_indirdep(indirdep, bp, bpp) 9451 struct indirdep *indirdep; 9452 struct buf *bp; 9453 struct buf **bpp; 9454 { 9455 struct allocindir *aip; 9456 int chgs; 9457 9458 if (indirdep->ir_state & GOINGAWAY) 9459 panic("disk_write_complete: indirdep gone"); 9460 chgs = 0; 9461 /* 9462 * If there were rollbacks revert them here. 9463 */ 9464 if (indirdep->ir_saveddata) { 9465 bcopy(indirdep->ir_saveddata, bp->b_data, bp->b_bcount); 9466 free(indirdep->ir_saveddata, M_INDIRDEP); 9467 indirdep->ir_saveddata = 0; 9468 chgs = 1; 9469 } 9470 indirdep->ir_state &= ~UNDONE; 9471 indirdep->ir_state |= ATTACHED; 9472 /* 9473 * Move allocindirs with written pointers to the completehd if 9474 * the indirdep's pointer is not yet written. Otherwise 9475 * free them here. 9476 */ 9477 while ((aip = LIST_FIRST(&indirdep->ir_writehd)) != 0) { 9478 LIST_REMOVE(aip, ai_next); 9479 if ((indirdep->ir_state & DEPCOMPLETE) == 0) { 9480 LIST_INSERT_HEAD(&indirdep->ir_completehd, aip, 9481 ai_next); 9482 continue; 9483 } 9484 free_newblk(&aip->ai_block); 9485 } 9486 /* 9487 * Move allocindirs that have finished dependency processing from 9488 * the done list to the write list after updating the pointers. 9489 */ 9490 while ((aip = LIST_FIRST(&indirdep->ir_donehd)) != 0) { 9491 handle_allocindir_partdone(aip); 9492 if (aip == LIST_FIRST(&indirdep->ir_donehd)) 9493 panic("disk_write_complete: not gone"); 9494 chgs = 1; 9495 } 9496 /* 9497 * If this indirdep has been detached from its newblk during 9498 * I/O we need to keep this dep attached to the buffer so 9499 * deallocate_dependencies can find it and properly resolve 9500 * any outstanding dependencies. 9501 */ 9502 if ((indirdep->ir_state & (ONDEPLIST | DEPCOMPLETE)) == 0) 9503 chgs = 1; 9504 if ((bp->b_flags & B_DELWRI) == 0) 9505 stat_indir_blk_ptrs++; 9506 /* 9507 * If there were no changes we can discard the savedbp and detach 9508 * ourselves from the buf. We are only carrying completed pointers 9509 * in this case. 9510 */ 9511 if (chgs == 0) { 9512 struct buf *sbp; 9513 9514 sbp = indirdep->ir_savebp; 9515 sbp->b_flags |= B_INVAL | B_NOCACHE; 9516 indirdep->ir_savebp = NULL; 9517 if (*bpp != NULL) 9518 panic("handle_written_indirdep: bp already exists."); 9519 *bpp = sbp; 9520 } else 9521 bdirty(bp); 9522 /* 9523 * If there are no fresh dependencies and none waiting on writes 9524 * we can free the indirdep. 9525 */ 9526 if ((indirdep->ir_state & DEPCOMPLETE) && chgs == 0) { 9527 if (indirdep->ir_state & ONDEPLIST) 9528 LIST_REMOVE(indirdep, ir_next); 9529 free_indirdep(indirdep); 9530 return (0); 9531 } 9532 9533 return (chgs); 9534 } 9535 9536 /* 9537 * Process a diradd entry after its dependent inode has been written. 9538 * This routine must be called with splbio interrupts blocked. 9539 */ 9540 static void 9541 diradd_inode_written(dap, inodedep) 9542 struct diradd *dap; 9543 struct inodedep *inodedep; 9544 { 9545 9546 dap->da_state |= COMPLETE; 9547 complete_diradd(dap); 9548 WORKLIST_INSERT(&inodedep->id_pendinghd, &dap->da_list); 9549 } 9550 9551 /* 9552 * Returns true if the bmsafemap will have rollbacks when written. Must 9553 * only be called with lk and the buf lock on the cg held. 9554 */ 9555 static int 9556 bmsafemap_rollbacks(bmsafemap) 9557 struct bmsafemap *bmsafemap; 9558 { 9559 9560 return (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd) | 9561 !LIST_EMPTY(&bmsafemap->sm_jnewblkhd)); 9562 } 9563 9564 /* 9565 * Complete a write to a bmsafemap structure. Roll forward any bitmap 9566 * changes if it's not a background write. Set all written dependencies 9567 * to DEPCOMPLETE and free the structure if possible. 9568 */ 9569 static int 9570 handle_written_bmsafemap(bmsafemap, bp) 9571 struct bmsafemap *bmsafemap; 9572 struct buf *bp; 9573 { 9574 struct newblk *newblk; 9575 struct inodedep *inodedep; 9576 struct jaddref *jaddref, *jatmp; 9577 struct jnewblk *jnewblk, *jntmp; 9578 uint8_t *inosused; 9579 uint8_t *blksfree; 9580 struct cg *cgp; 9581 struct fs *fs; 9582 ino_t ino; 9583 long bno; 9584 int chgs; 9585 int i; 9586 9587 if ((bmsafemap->sm_state & IOSTARTED) == 0) 9588 panic("initiate_write_bmsafemap: Not started\n"); 9589 chgs = 0; 9590 bmsafemap->sm_state &= ~IOSTARTED; 9591 /* 9592 * Restore unwritten inode allocation pending jaddref writes. 9593 */ 9594 if (!LIST_EMPTY(&bmsafemap->sm_jaddrefhd)) { 9595 cgp = (struct cg *)bp->b_data; 9596 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 9597 inosused = cg_inosused(cgp); 9598 LIST_FOREACH_SAFE(jaddref, &bmsafemap->sm_jaddrefhd, 9599 ja_bmdeps, jatmp) { 9600 if ((jaddref->ja_state & UNDONE) == 0) 9601 continue; 9602 ino = jaddref->ja_ino % fs->fs_ipg; 9603 if (isset(inosused, ino)) 9604 panic("handle_written_bmsafemap: " 9605 "re-allocated inode"); 9606 if ((bp->b_xflags & BX_BKGRDMARKER) == 0) { 9607 if ((jaddref->ja_mode & IFMT) == IFDIR) 9608 cgp->cg_cs.cs_ndir++; 9609 cgp->cg_cs.cs_nifree--; 9610 setbit(inosused, ino); 9611 chgs = 1; 9612 } 9613 jaddref->ja_state &= ~UNDONE; 9614 jaddref->ja_state |= ATTACHED; 9615 free_jaddref(jaddref); 9616 } 9617 } 9618 /* 9619 * Restore any block allocations which are pending journal writes. 9620 */ 9621 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd) != NULL) { 9622 cgp = (struct cg *)bp->b_data; 9623 fs = VFSTOUFS(bmsafemap->sm_list.wk_mp)->um_fs; 9624 blksfree = cg_blksfree(cgp); 9625 LIST_FOREACH_SAFE(jnewblk, &bmsafemap->sm_jnewblkhd, jn_deps, 9626 jntmp) { 9627 if ((jnewblk->jn_state & UNDONE) == 0) 9628 continue; 9629 bno = dtogd(fs, jnewblk->jn_blkno); 9630 for (i = jnewblk->jn_oldfrags; i < jnewblk->jn_frags; 9631 i++) { 9632 if (bp->b_xflags & BX_BKGRDMARKER) 9633 break; 9634 if ((jnewblk->jn_state & NEWBLOCK) == 0 && 9635 isclr(blksfree, bno + i)) 9636 panic("handle_written_bmsafemap: " 9637 "re-allocated fragment"); 9638 clrbit(blksfree, bno + i); 9639 chgs = 1; 9640 } 9641 jnewblk->jn_state &= ~(UNDONE | NEWBLOCK); 9642 jnewblk->jn_state |= ATTACHED; 9643 free_jnewblk(jnewblk); 9644 } 9645 } 9646 while ((newblk = LIST_FIRST(&bmsafemap->sm_newblkwr))) { 9647 newblk->nb_state |= DEPCOMPLETE; 9648 newblk->nb_state &= ~ONDEPLIST; 9649 newblk->nb_bmsafemap = NULL; 9650 LIST_REMOVE(newblk, nb_deps); 9651 if (newblk->nb_list.wk_type == D_ALLOCDIRECT) 9652 handle_allocdirect_partdone( 9653 WK_ALLOCDIRECT(&newblk->nb_list), NULL); 9654 else if (newblk->nb_list.wk_type == D_ALLOCINDIR) 9655 handle_allocindir_partdone( 9656 WK_ALLOCINDIR(&newblk->nb_list)); 9657 else if (newblk->nb_list.wk_type != D_NEWBLK) 9658 panic("handle_written_bmsafemap: Unexpected type: %s", 9659 TYPENAME(newblk->nb_list.wk_type)); 9660 } 9661 while ((inodedep = LIST_FIRST(&bmsafemap->sm_inodedepwr)) != NULL) { 9662 inodedep->id_state |= DEPCOMPLETE; 9663 inodedep->id_state &= ~ONDEPLIST; 9664 LIST_REMOVE(inodedep, id_deps); 9665 inodedep->id_bmsafemap = NULL; 9666 } 9667 if (LIST_EMPTY(&bmsafemap->sm_jaddrefhd) && 9668 LIST_EMPTY(&bmsafemap->sm_jnewblkhd) && 9669 LIST_EMPTY(&bmsafemap->sm_newblkhd) && 9670 LIST_EMPTY(&bmsafemap->sm_inodedephd)) { 9671 if (chgs) 9672 bdirty(bp); 9673 LIST_REMOVE(bmsafemap, sm_hash); 9674 WORKITEM_FREE(bmsafemap, D_BMSAFEMAP); 9675 return (0); 9676 } 9677 bdirty(bp); 9678 return (1); 9679 } 9680 9681 /* 9682 * Try to free a mkdir dependency. 9683 */ 9684 static void 9685 complete_mkdir(mkdir) 9686 struct mkdir *mkdir; 9687 { 9688 struct diradd *dap; 9689 9690 if ((mkdir->md_state & ALLCOMPLETE) != ALLCOMPLETE) 9691 return; 9692 LIST_REMOVE(mkdir, md_mkdirs); 9693 dap = mkdir->md_diradd; 9694 dap->da_state &= ~(mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)); 9695 if ((dap->da_state & (MKDIR_PARENT | MKDIR_BODY)) == 0) { 9696 dap->da_state |= DEPCOMPLETE; 9697 complete_diradd(dap); 9698 } 9699 WORKITEM_FREE(mkdir, D_MKDIR); 9700 } 9701 9702 /* 9703 * Handle the completion of a mkdir dependency. 9704 */ 9705 static void 9706 handle_written_mkdir(mkdir, type) 9707 struct mkdir *mkdir; 9708 int type; 9709 { 9710 9711 if ((mkdir->md_state & (MKDIR_PARENT | MKDIR_BODY)) != type) 9712 panic("handle_written_mkdir: bad type"); 9713 mkdir->md_state |= COMPLETE; 9714 complete_mkdir(mkdir); 9715 } 9716 9717 static void 9718 free_pagedep(pagedep) 9719 struct pagedep *pagedep; 9720 { 9721 int i; 9722 9723 if (pagedep->pd_state & (NEWBLOCK | ONWORKLIST)) 9724 return; 9725 for (i = 0; i < DAHASHSZ; i++) 9726 if (!LIST_EMPTY(&pagedep->pd_diraddhd[i])) 9727 return; 9728 if (!LIST_EMPTY(&pagedep->pd_jmvrefhd)) 9729 return; 9730 if (!LIST_EMPTY(&pagedep->pd_dirremhd)) 9731 return; 9732 if (!LIST_EMPTY(&pagedep->pd_pendinghd)) 9733 return; 9734 LIST_REMOVE(pagedep, pd_hash); 9735 WORKITEM_FREE(pagedep, D_PAGEDEP); 9736 } 9737 9738 /* 9739 * Called from within softdep_disk_write_complete above. 9740 * A write operation was just completed. Removed inodes can 9741 * now be freed and associated block pointers may be committed. 9742 * Note that this routine is always called from interrupt level 9743 * with further splbio interrupts blocked. 9744 */ 9745 static int 9746 handle_written_filepage(pagedep, bp) 9747 struct pagedep *pagedep; 9748 struct buf *bp; /* buffer containing the written page */ 9749 { 9750 struct dirrem *dirrem; 9751 struct diradd *dap, *nextdap; 9752 struct direct *ep; 9753 int i, chgs; 9754 9755 if ((pagedep->pd_state & IOSTARTED) == 0) 9756 panic("handle_written_filepage: not started"); 9757 pagedep->pd_state &= ~IOSTARTED; 9758 /* 9759 * Process any directory removals that have been committed. 9760 */ 9761 while ((dirrem = LIST_FIRST(&pagedep->pd_dirremhd)) != NULL) { 9762 LIST_REMOVE(dirrem, dm_next); 9763 dirrem->dm_state |= COMPLETE; 9764 dirrem->dm_dirinum = pagedep->pd_ino; 9765 KASSERT(LIST_EMPTY(&dirrem->dm_jremrefhd), 9766 ("handle_written_filepage: Journal entries not written.")); 9767 add_to_worklist(&dirrem->dm_list, 0); 9768 } 9769 /* 9770 * Free any directory additions that have been committed. 9771 * If it is a newly allocated block, we have to wait until 9772 * the on-disk directory inode claims the new block. 9773 */ 9774 if ((pagedep->pd_state & NEWBLOCK) == 0) 9775 while ((dap = LIST_FIRST(&pagedep->pd_pendinghd)) != NULL) 9776 free_diradd(dap, NULL); 9777 /* 9778 * Uncommitted directory entries must be restored. 9779 */ 9780 for (chgs = 0, i = 0; i < DAHASHSZ; i++) { 9781 for (dap = LIST_FIRST(&pagedep->pd_diraddhd[i]); dap; 9782 dap = nextdap) { 9783 nextdap = LIST_NEXT(dap, da_pdlist); 9784 if (dap->da_state & ATTACHED) 9785 panic("handle_written_filepage: attached"); 9786 ep = (struct direct *) 9787 ((char *)bp->b_data + dap->da_offset); 9788 ep->d_ino = dap->da_newinum; 9789 dap->da_state &= ~UNDONE; 9790 dap->da_state |= ATTACHED; 9791 chgs = 1; 9792 /* 9793 * If the inode referenced by the directory has 9794 * been written out, then the dependency can be 9795 * moved to the pending list. 9796 */ 9797 if ((dap->da_state & ALLCOMPLETE) == ALLCOMPLETE) { 9798 LIST_REMOVE(dap, da_pdlist); 9799 LIST_INSERT_HEAD(&pagedep->pd_pendinghd, dap, 9800 da_pdlist); 9801 } 9802 } 9803 } 9804 /* 9805 * If there were any rollbacks in the directory, then it must be 9806 * marked dirty so that its will eventually get written back in 9807 * its correct form. 9808 */ 9809 if (chgs) { 9810 if ((bp->b_flags & B_DELWRI) == 0) 9811 stat_dir_entry++; 9812 bdirty(bp); 9813 return (1); 9814 } 9815 /* 9816 * If we are not waiting for a new directory block to be 9817 * claimed by its inode, then the pagedep will be freed. 9818 * Otherwise it will remain to track any new entries on 9819 * the page in case they are fsync'ed. 9820 */ 9821 if ((pagedep->pd_state & NEWBLOCK) == 0 && 9822 LIST_EMPTY(&pagedep->pd_jmvrefhd)) { 9823 LIST_REMOVE(pagedep, pd_hash); 9824 WORKITEM_FREE(pagedep, D_PAGEDEP); 9825 } 9826 return (0); 9827 } 9828 9829 /* 9830 * Writing back in-core inode structures. 9831 * 9832 * The filesystem only accesses an inode's contents when it occupies an 9833 * "in-core" inode structure. These "in-core" structures are separate from 9834 * the page frames used to cache inode blocks. Only the latter are 9835 * transferred to/from the disk. So, when the updated contents of the 9836 * "in-core" inode structure are copied to the corresponding in-memory inode 9837 * block, the dependencies are also transferred. The following procedure is 9838 * called when copying a dirty "in-core" inode to a cached inode block. 9839 */ 9840 9841 /* 9842 * Called when an inode is loaded from disk. If the effective link count 9843 * differed from the actual link count when it was last flushed, then we 9844 * need to ensure that the correct effective link count is put back. 9845 */ 9846 void 9847 softdep_load_inodeblock(ip) 9848 struct inode *ip; /* the "in_core" copy of the inode */ 9849 { 9850 struct inodedep *inodedep; 9851 9852 /* 9853 * Check for alternate nlink count. 9854 */ 9855 ip->i_effnlink = ip->i_nlink; 9856 ACQUIRE_LOCK(&lk); 9857 if (inodedep_lookup(UFSTOVFS(ip->i_ump), ip->i_number, 0, 9858 &inodedep) == 0) { 9859 FREE_LOCK(&lk); 9860 return; 9861 } 9862 ip->i_effnlink -= inodedep->id_nlinkdelta; 9863 FREE_LOCK(&lk); 9864 } 9865 9866 /* 9867 * This routine is called just before the "in-core" inode 9868 * information is to be copied to the in-memory inode block. 9869 * Recall that an inode block contains several inodes. If 9870 * the force flag is set, then the dependencies will be 9871 * cleared so that the update can always be made. Note that 9872 * the buffer is locked when this routine is called, so we 9873 * will never be in the middle of writing the inode block 9874 * to disk. 9875 */ 9876 void 9877 softdep_update_inodeblock(ip, bp, waitfor) 9878 struct inode *ip; /* the "in_core" copy of the inode */ 9879 struct buf *bp; /* the buffer containing the inode block */ 9880 int waitfor; /* nonzero => update must be allowed */ 9881 { 9882 struct inodedep *inodedep; 9883 struct inoref *inoref; 9884 struct worklist *wk; 9885 struct mount *mp; 9886 struct buf *ibp; 9887 struct fs *fs; 9888 int error; 9889 9890 mp = UFSTOVFS(ip->i_ump); 9891 fs = ip->i_fs; 9892 /* 9893 * Preserve the freelink that is on disk. clear_unlinked_inodedep() 9894 * does not have access to the in-core ip so must write directly into 9895 * the inode block buffer when setting freelink. 9896 */ 9897 if (fs->fs_magic == FS_UFS1_MAGIC) 9898 DIP_SET(ip, i_freelink, ((struct ufs1_dinode *)bp->b_data + 9899 ino_to_fsbo(fs, ip->i_number))->di_freelink); 9900 else 9901 DIP_SET(ip, i_freelink, ((struct ufs2_dinode *)bp->b_data + 9902 ino_to_fsbo(fs, ip->i_number))->di_freelink); 9903 /* 9904 * If the effective link count is not equal to the actual link 9905 * count, then we must track the difference in an inodedep while 9906 * the inode is (potentially) tossed out of the cache. Otherwise, 9907 * if there is no existing inodedep, then there are no dependencies 9908 * to track. 9909 */ 9910 ACQUIRE_LOCK(&lk); 9911 again: 9912 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 9913 FREE_LOCK(&lk); 9914 if (ip->i_effnlink != ip->i_nlink) 9915 panic("softdep_update_inodeblock: bad link count"); 9916 return; 9917 } 9918 if (inodedep->id_nlinkdelta != ip->i_nlink - ip->i_effnlink) 9919 panic("softdep_update_inodeblock: bad delta"); 9920 /* 9921 * If we're flushing all dependencies we must also move any waiting 9922 * for journal writes onto the bufwait list prior to I/O. 9923 */ 9924 if (waitfor) { 9925 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 9926 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 9927 == DEPCOMPLETE) { 9928 stat_jwait_inode++; 9929 jwait(&inoref->if_list); 9930 goto again; 9931 } 9932 } 9933 } 9934 /* 9935 * Changes have been initiated. Anything depending on these 9936 * changes cannot occur until this inode has been written. 9937 */ 9938 inodedep->id_state &= ~COMPLETE; 9939 if ((inodedep->id_state & ONWORKLIST) == 0) 9940 WORKLIST_INSERT(&bp->b_dep, &inodedep->id_list); 9941 /* 9942 * Any new dependencies associated with the incore inode must 9943 * now be moved to the list associated with the buffer holding 9944 * the in-memory copy of the inode. Once merged process any 9945 * allocdirects that are completed by the merger. 9946 */ 9947 merge_inode_lists(&inodedep->id_newinoupdt, &inodedep->id_inoupdt); 9948 if (!TAILQ_EMPTY(&inodedep->id_inoupdt)) 9949 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_inoupdt), 9950 NULL); 9951 merge_inode_lists(&inodedep->id_newextupdt, &inodedep->id_extupdt); 9952 if (!TAILQ_EMPTY(&inodedep->id_extupdt)) 9953 handle_allocdirect_partdone(TAILQ_FIRST(&inodedep->id_extupdt), 9954 NULL); 9955 /* 9956 * Now that the inode has been pushed into the buffer, the 9957 * operations dependent on the inode being written to disk 9958 * can be moved to the id_bufwait so that they will be 9959 * processed when the buffer I/O completes. 9960 */ 9961 while ((wk = LIST_FIRST(&inodedep->id_inowait)) != NULL) { 9962 WORKLIST_REMOVE(wk); 9963 WORKLIST_INSERT(&inodedep->id_bufwait, wk); 9964 } 9965 /* 9966 * Newly allocated inodes cannot be written until the bitmap 9967 * that allocates them have been written (indicated by 9968 * DEPCOMPLETE being set in id_state). If we are doing a 9969 * forced sync (e.g., an fsync on a file), we force the bitmap 9970 * to be written so that the update can be done. 9971 */ 9972 if (waitfor == 0) { 9973 FREE_LOCK(&lk); 9974 return; 9975 } 9976 retry: 9977 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) != 0) { 9978 FREE_LOCK(&lk); 9979 return; 9980 } 9981 ibp = inodedep->id_bmsafemap->sm_buf; 9982 ibp = getdirtybuf(ibp, &lk, MNT_WAIT); 9983 if (ibp == NULL) { 9984 /* 9985 * If ibp came back as NULL, the dependency could have been 9986 * freed while we slept. Look it up again, and check to see 9987 * that it has completed. 9988 */ 9989 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) 9990 goto retry; 9991 FREE_LOCK(&lk); 9992 return; 9993 } 9994 FREE_LOCK(&lk); 9995 if ((error = bwrite(ibp)) != 0) 9996 softdep_error("softdep_update_inodeblock: bwrite", error); 9997 } 9998 9999 /* 10000 * Merge the a new inode dependency list (such as id_newinoupdt) into an 10001 * old inode dependency list (such as id_inoupdt). This routine must be 10002 * called with splbio interrupts blocked. 10003 */ 10004 static void 10005 merge_inode_lists(newlisthead, oldlisthead) 10006 struct allocdirectlst *newlisthead; 10007 struct allocdirectlst *oldlisthead; 10008 { 10009 struct allocdirect *listadp, *newadp; 10010 10011 newadp = TAILQ_FIRST(newlisthead); 10012 for (listadp = TAILQ_FIRST(oldlisthead); listadp && newadp;) { 10013 if (listadp->ad_offset < newadp->ad_offset) { 10014 listadp = TAILQ_NEXT(listadp, ad_next); 10015 continue; 10016 } 10017 TAILQ_REMOVE(newlisthead, newadp, ad_next); 10018 TAILQ_INSERT_BEFORE(listadp, newadp, ad_next); 10019 if (listadp->ad_offset == newadp->ad_offset) { 10020 allocdirect_merge(oldlisthead, newadp, 10021 listadp); 10022 listadp = newadp; 10023 } 10024 newadp = TAILQ_FIRST(newlisthead); 10025 } 10026 while ((newadp = TAILQ_FIRST(newlisthead)) != NULL) { 10027 TAILQ_REMOVE(newlisthead, newadp, ad_next); 10028 TAILQ_INSERT_TAIL(oldlisthead, newadp, ad_next); 10029 } 10030 } 10031 10032 /* 10033 * If we are doing an fsync, then we must ensure that any directory 10034 * entries for the inode have been written after the inode gets to disk. 10035 */ 10036 int 10037 softdep_fsync(vp) 10038 struct vnode *vp; /* the "in_core" copy of the inode */ 10039 { 10040 struct inodedep *inodedep; 10041 struct pagedep *pagedep; 10042 struct inoref *inoref; 10043 struct worklist *wk; 10044 struct diradd *dap; 10045 struct mount *mp; 10046 struct vnode *pvp; 10047 struct inode *ip; 10048 struct buf *bp; 10049 struct fs *fs; 10050 struct thread *td = curthread; 10051 int error, flushparent, pagedep_new_block; 10052 ino_t parentino; 10053 ufs_lbn_t lbn; 10054 10055 ip = VTOI(vp); 10056 fs = ip->i_fs; 10057 mp = vp->v_mount; 10058 ACQUIRE_LOCK(&lk); 10059 restart: 10060 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) { 10061 FREE_LOCK(&lk); 10062 return (0); 10063 } 10064 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 10065 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 10066 == DEPCOMPLETE) { 10067 stat_jwait_inode++; 10068 jwait(&inoref->if_list); 10069 goto restart; 10070 } 10071 } 10072 if (!LIST_EMPTY(&inodedep->id_inowait) || 10073 !TAILQ_EMPTY(&inodedep->id_extupdt) || 10074 !TAILQ_EMPTY(&inodedep->id_newextupdt) || 10075 !TAILQ_EMPTY(&inodedep->id_inoupdt) || 10076 !TAILQ_EMPTY(&inodedep->id_newinoupdt)) 10077 panic("softdep_fsync: pending ops %p", inodedep); 10078 for (error = 0, flushparent = 0; ; ) { 10079 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) == NULL) 10080 break; 10081 if (wk->wk_type != D_DIRADD) 10082 panic("softdep_fsync: Unexpected type %s", 10083 TYPENAME(wk->wk_type)); 10084 dap = WK_DIRADD(wk); 10085 /* 10086 * Flush our parent if this directory entry has a MKDIR_PARENT 10087 * dependency or is contained in a newly allocated block. 10088 */ 10089 if (dap->da_state & DIRCHG) 10090 pagedep = dap->da_previous->dm_pagedep; 10091 else 10092 pagedep = dap->da_pagedep; 10093 parentino = pagedep->pd_ino; 10094 lbn = pagedep->pd_lbn; 10095 if ((dap->da_state & (MKDIR_BODY | COMPLETE)) != COMPLETE) 10096 panic("softdep_fsync: dirty"); 10097 if ((dap->da_state & MKDIR_PARENT) || 10098 (pagedep->pd_state & NEWBLOCK)) 10099 flushparent = 1; 10100 else 10101 flushparent = 0; 10102 /* 10103 * If we are being fsync'ed as part of vgone'ing this vnode, 10104 * then we will not be able to release and recover the 10105 * vnode below, so we just have to give up on writing its 10106 * directory entry out. It will eventually be written, just 10107 * not now, but then the user was not asking to have it 10108 * written, so we are not breaking any promises. 10109 */ 10110 if (vp->v_iflag & VI_DOOMED) 10111 break; 10112 /* 10113 * We prevent deadlock by always fetching inodes from the 10114 * root, moving down the directory tree. Thus, when fetching 10115 * our parent directory, we first try to get the lock. If 10116 * that fails, we must unlock ourselves before requesting 10117 * the lock on our parent. See the comment in ufs_lookup 10118 * for details on possible races. 10119 */ 10120 FREE_LOCK(&lk); 10121 if (ffs_vgetf(mp, parentino, LK_NOWAIT | LK_EXCLUSIVE, &pvp, 10122 FFSV_FORCEINSMQ)) { 10123 error = vfs_busy(mp, MBF_NOWAIT); 10124 if (error != 0) { 10125 vfs_ref(mp); 10126 VOP_UNLOCK(vp, 0); 10127 error = vfs_busy(mp, 0); 10128 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 10129 vfs_rel(mp); 10130 if (error != 0) 10131 return (ENOENT); 10132 if (vp->v_iflag & VI_DOOMED) { 10133 vfs_unbusy(mp); 10134 return (ENOENT); 10135 } 10136 } 10137 VOP_UNLOCK(vp, 0); 10138 error = ffs_vgetf(mp, parentino, LK_EXCLUSIVE, 10139 &pvp, FFSV_FORCEINSMQ); 10140 vfs_unbusy(mp); 10141 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); 10142 if (vp->v_iflag & VI_DOOMED) { 10143 if (error == 0) 10144 vput(pvp); 10145 error = ENOENT; 10146 } 10147 if (error != 0) 10148 return (error); 10149 } 10150 /* 10151 * All MKDIR_PARENT dependencies and all the NEWBLOCK pagedeps 10152 * that are contained in direct blocks will be resolved by 10153 * doing a ffs_update. Pagedeps contained in indirect blocks 10154 * may require a complete sync'ing of the directory. So, we 10155 * try the cheap and fast ffs_update first, and if that fails, 10156 * then we do the slower ffs_syncvnode of the directory. 10157 */ 10158 if (flushparent) { 10159 int locked; 10160 10161 if ((error = ffs_update(pvp, 1)) != 0) { 10162 vput(pvp); 10163 return (error); 10164 } 10165 ACQUIRE_LOCK(&lk); 10166 locked = 1; 10167 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) != 0) { 10168 if ((wk = LIST_FIRST(&inodedep->id_pendinghd)) != NULL) { 10169 if (wk->wk_type != D_DIRADD) 10170 panic("softdep_fsync: Unexpected type %s", 10171 TYPENAME(wk->wk_type)); 10172 dap = WK_DIRADD(wk); 10173 if (dap->da_state & DIRCHG) 10174 pagedep = dap->da_previous->dm_pagedep; 10175 else 10176 pagedep = dap->da_pagedep; 10177 pagedep_new_block = pagedep->pd_state & NEWBLOCK; 10178 FREE_LOCK(&lk); 10179 locked = 0; 10180 if (pagedep_new_block && 10181 (error = ffs_syncvnode(pvp, MNT_WAIT))) { 10182 vput(pvp); 10183 return (error); 10184 } 10185 } 10186 } 10187 if (locked) 10188 FREE_LOCK(&lk); 10189 } 10190 /* 10191 * Flush directory page containing the inode's name. 10192 */ 10193 error = bread(pvp, lbn, blksize(fs, VTOI(pvp), lbn), td->td_ucred, 10194 &bp); 10195 if (error == 0) 10196 error = bwrite(bp); 10197 else 10198 brelse(bp); 10199 vput(pvp); 10200 if (error != 0) 10201 return (error); 10202 ACQUIRE_LOCK(&lk); 10203 if (inodedep_lookup(mp, ip->i_number, 0, &inodedep) == 0) 10204 break; 10205 } 10206 FREE_LOCK(&lk); 10207 return (0); 10208 } 10209 10210 /* 10211 * Flush all the dirty bitmaps associated with the block device 10212 * before flushing the rest of the dirty blocks so as to reduce 10213 * the number of dependencies that will have to be rolled back. 10214 */ 10215 void 10216 softdep_fsync_mountdev(vp) 10217 struct vnode *vp; 10218 { 10219 struct buf *bp, *nbp; 10220 struct worklist *wk; 10221 struct bufobj *bo; 10222 10223 if (!vn_isdisk(vp, NULL)) 10224 panic("softdep_fsync_mountdev: vnode not a disk"); 10225 bo = &vp->v_bufobj; 10226 restart: 10227 BO_LOCK(bo); 10228 ACQUIRE_LOCK(&lk); 10229 TAILQ_FOREACH_SAFE(bp, &bo->bo_dirty.bv_hd, b_bobufs, nbp) { 10230 /* 10231 * If it is already scheduled, skip to the next buffer. 10232 */ 10233 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL)) 10234 continue; 10235 10236 if ((bp->b_flags & B_DELWRI) == 0) 10237 panic("softdep_fsync_mountdev: not dirty"); 10238 /* 10239 * We are only interested in bitmaps with outstanding 10240 * dependencies. 10241 */ 10242 if ((wk = LIST_FIRST(&bp->b_dep)) == NULL || 10243 wk->wk_type != D_BMSAFEMAP || 10244 (bp->b_vflags & BV_BKGRDINPROG)) { 10245 BUF_UNLOCK(bp); 10246 continue; 10247 } 10248 FREE_LOCK(&lk); 10249 BO_UNLOCK(bo); 10250 bremfree(bp); 10251 (void) bawrite(bp); 10252 goto restart; 10253 } 10254 FREE_LOCK(&lk); 10255 drain_output(vp); 10256 BO_UNLOCK(bo); 10257 } 10258 10259 /* 10260 * This routine is called when we are trying to synchronously flush a 10261 * file. This routine must eliminate any filesystem metadata dependencies 10262 * so that the syncing routine can succeed by pushing the dirty blocks 10263 * associated with the file. If any I/O errors occur, they are returned. 10264 */ 10265 int 10266 softdep_sync_metadata(struct vnode *vp) 10267 { 10268 struct pagedep *pagedep; 10269 struct allocindir *aip; 10270 struct newblk *newblk; 10271 struct buf *bp, *nbp; 10272 struct worklist *wk; 10273 struct bufobj *bo; 10274 int i, error, waitfor; 10275 10276 if (!DOINGSOFTDEP(vp)) 10277 return (0); 10278 /* 10279 * Ensure that any direct block dependencies have been cleared. 10280 */ 10281 ACQUIRE_LOCK(&lk); 10282 if ((error = flush_inodedep_deps(vp->v_mount, VTOI(vp)->i_number))) { 10283 FREE_LOCK(&lk); 10284 return (error); 10285 } 10286 FREE_LOCK(&lk); 10287 /* 10288 * For most files, the only metadata dependencies are the 10289 * cylinder group maps that allocate their inode or blocks. 10290 * The block allocation dependencies can be found by traversing 10291 * the dependency lists for any buffers that remain on their 10292 * dirty buffer list. The inode allocation dependency will 10293 * be resolved when the inode is updated with MNT_WAIT. 10294 * This work is done in two passes. The first pass grabs most 10295 * of the buffers and begins asynchronously writing them. The 10296 * only way to wait for these asynchronous writes is to sleep 10297 * on the filesystem vnode which may stay busy for a long time 10298 * if the filesystem is active. So, instead, we make a second 10299 * pass over the dependencies blocking on each write. In the 10300 * usual case we will be blocking against a write that we 10301 * initiated, so when it is done the dependency will have been 10302 * resolved. Thus the second pass is expected to end quickly. 10303 */ 10304 waitfor = MNT_NOWAIT; 10305 bo = &vp->v_bufobj; 10306 10307 top: 10308 /* 10309 * We must wait for any I/O in progress to finish so that 10310 * all potential buffers on the dirty list will be visible. 10311 */ 10312 BO_LOCK(bo); 10313 drain_output(vp); 10314 while ((bp = TAILQ_FIRST(&bo->bo_dirty.bv_hd)) != NULL) { 10315 bp = getdirtybuf(bp, BO_MTX(bo), MNT_WAIT); 10316 if (bp) 10317 break; 10318 } 10319 BO_UNLOCK(bo); 10320 if (bp == NULL) 10321 return (0); 10322 loop: 10323 /* While syncing snapshots, we must allow recursive lookups */ 10324 BUF_AREC(bp); 10325 ACQUIRE_LOCK(&lk); 10326 /* 10327 * As we hold the buffer locked, none of its dependencies 10328 * will disappear. 10329 */ 10330 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 10331 switch (wk->wk_type) { 10332 10333 case D_ALLOCDIRECT: 10334 case D_ALLOCINDIR: 10335 newblk = WK_NEWBLK(wk); 10336 if (newblk->nb_jnewblk != NULL) { 10337 stat_jwait_newblk++; 10338 jwait(&newblk->nb_jnewblk->jn_list); 10339 goto restart; 10340 } 10341 if (newblk->nb_state & DEPCOMPLETE) 10342 continue; 10343 nbp = newblk->nb_bmsafemap->sm_buf; 10344 nbp = getdirtybuf(nbp, &lk, waitfor); 10345 if (nbp == NULL) 10346 continue; 10347 FREE_LOCK(&lk); 10348 if (waitfor == MNT_NOWAIT) { 10349 bawrite(nbp); 10350 } else if ((error = bwrite(nbp)) != 0) { 10351 break; 10352 } 10353 ACQUIRE_LOCK(&lk); 10354 continue; 10355 10356 case D_INDIRDEP: 10357 restart: 10358 10359 LIST_FOREACH(aip, 10360 &WK_INDIRDEP(wk)->ir_deplisthd, ai_next) { 10361 newblk = (struct newblk *)aip; 10362 if (newblk->nb_jnewblk != NULL) { 10363 stat_jwait_newblk++; 10364 jwait(&newblk->nb_jnewblk->jn_list); 10365 goto restart; 10366 } 10367 if (newblk->nb_state & DEPCOMPLETE) 10368 continue; 10369 nbp = newblk->nb_bmsafemap->sm_buf; 10370 nbp = getdirtybuf(nbp, &lk, MNT_WAIT); 10371 if (nbp == NULL) 10372 goto restart; 10373 FREE_LOCK(&lk); 10374 if ((error = bwrite(nbp)) != 0) { 10375 goto loop_end; 10376 } 10377 ACQUIRE_LOCK(&lk); 10378 goto restart; 10379 } 10380 continue; 10381 10382 case D_PAGEDEP: 10383 /* 10384 * We are trying to sync a directory that may 10385 * have dependencies on both its own metadata 10386 * and/or dependencies on the inodes of any 10387 * recently allocated files. We walk its diradd 10388 * lists pushing out the associated inode. 10389 */ 10390 pagedep = WK_PAGEDEP(wk); 10391 for (i = 0; i < DAHASHSZ; i++) { 10392 if (LIST_FIRST(&pagedep->pd_diraddhd[i]) == 0) 10393 continue; 10394 if ((error = 10395 flush_pagedep_deps(vp, wk->wk_mp, 10396 &pagedep->pd_diraddhd[i]))) { 10397 FREE_LOCK(&lk); 10398 goto loop_end; 10399 } 10400 } 10401 continue; 10402 10403 default: 10404 panic("softdep_sync_metadata: Unknown type %s", 10405 TYPENAME(wk->wk_type)); 10406 /* NOTREACHED */ 10407 } 10408 loop_end: 10409 /* We reach here only in error and unlocked */ 10410 if (error == 0) 10411 panic("softdep_sync_metadata: zero error"); 10412 BUF_NOREC(bp); 10413 bawrite(bp); 10414 return (error); 10415 } 10416 FREE_LOCK(&lk); 10417 BO_LOCK(bo); 10418 while ((nbp = TAILQ_NEXT(bp, b_bobufs)) != NULL) { 10419 nbp = getdirtybuf(nbp, BO_MTX(bo), MNT_WAIT); 10420 if (nbp) 10421 break; 10422 } 10423 BO_UNLOCK(bo); 10424 BUF_NOREC(bp); 10425 bawrite(bp); 10426 if (nbp != NULL) { 10427 bp = nbp; 10428 goto loop; 10429 } 10430 /* 10431 * The brief unlock is to allow any pent up dependency 10432 * processing to be done. Then proceed with the second pass. 10433 */ 10434 if (waitfor == MNT_NOWAIT) { 10435 waitfor = MNT_WAIT; 10436 goto top; 10437 } 10438 10439 /* 10440 * If we have managed to get rid of all the dirty buffers, 10441 * then we are done. For certain directories and block 10442 * devices, we may need to do further work. 10443 * 10444 * We must wait for any I/O in progress to finish so that 10445 * all potential buffers on the dirty list will be visible. 10446 */ 10447 BO_LOCK(bo); 10448 drain_output(vp); 10449 BO_UNLOCK(bo); 10450 return ffs_update(vp, 1); 10451 /* return (0); */ 10452 } 10453 10454 /* 10455 * Flush the dependencies associated with an inodedep. 10456 * Called with splbio blocked. 10457 */ 10458 static int 10459 flush_inodedep_deps(mp, ino) 10460 struct mount *mp; 10461 ino_t ino; 10462 { 10463 struct inodedep *inodedep; 10464 struct inoref *inoref; 10465 int error, waitfor; 10466 10467 /* 10468 * This work is done in two passes. The first pass grabs most 10469 * of the buffers and begins asynchronously writing them. The 10470 * only way to wait for these asynchronous writes is to sleep 10471 * on the filesystem vnode which may stay busy for a long time 10472 * if the filesystem is active. So, instead, we make a second 10473 * pass over the dependencies blocking on each write. In the 10474 * usual case we will be blocking against a write that we 10475 * initiated, so when it is done the dependency will have been 10476 * resolved. Thus the second pass is expected to end quickly. 10477 * We give a brief window at the top of the loop to allow 10478 * any pending I/O to complete. 10479 */ 10480 for (error = 0, waitfor = MNT_NOWAIT; ; ) { 10481 if (error) 10482 return (error); 10483 FREE_LOCK(&lk); 10484 ACQUIRE_LOCK(&lk); 10485 restart: 10486 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 10487 return (0); 10488 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 10489 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 10490 == DEPCOMPLETE) { 10491 stat_jwait_inode++; 10492 jwait(&inoref->if_list); 10493 goto restart; 10494 } 10495 } 10496 if (flush_deplist(&inodedep->id_inoupdt, waitfor, &error) || 10497 flush_deplist(&inodedep->id_newinoupdt, waitfor, &error) || 10498 flush_deplist(&inodedep->id_extupdt, waitfor, &error) || 10499 flush_deplist(&inodedep->id_newextupdt, waitfor, &error)) 10500 continue; 10501 /* 10502 * If pass2, we are done, otherwise do pass 2. 10503 */ 10504 if (waitfor == MNT_WAIT) 10505 break; 10506 waitfor = MNT_WAIT; 10507 } 10508 /* 10509 * Try freeing inodedep in case all dependencies have been removed. 10510 */ 10511 if (inodedep_lookup(mp, ino, 0, &inodedep) != 0) 10512 (void) free_inodedep(inodedep); 10513 return (0); 10514 } 10515 10516 /* 10517 * Flush an inode dependency list. 10518 * Called with splbio blocked. 10519 */ 10520 static int 10521 flush_deplist(listhead, waitfor, errorp) 10522 struct allocdirectlst *listhead; 10523 int waitfor; 10524 int *errorp; 10525 { 10526 struct allocdirect *adp; 10527 struct newblk *newblk; 10528 struct buf *bp; 10529 10530 mtx_assert(&lk, MA_OWNED); 10531 TAILQ_FOREACH(adp, listhead, ad_next) { 10532 newblk = (struct newblk *)adp; 10533 if (newblk->nb_jnewblk != NULL) { 10534 stat_jwait_newblk++; 10535 jwait(&newblk->nb_jnewblk->jn_list); 10536 return (1); 10537 } 10538 if (newblk->nb_state & DEPCOMPLETE) 10539 continue; 10540 bp = newblk->nb_bmsafemap->sm_buf; 10541 bp = getdirtybuf(bp, &lk, waitfor); 10542 if (bp == NULL) { 10543 if (waitfor == MNT_NOWAIT) 10544 continue; 10545 return (1); 10546 } 10547 FREE_LOCK(&lk); 10548 if (waitfor == MNT_NOWAIT) { 10549 bawrite(bp); 10550 } else if ((*errorp = bwrite(bp)) != 0) { 10551 ACQUIRE_LOCK(&lk); 10552 return (1); 10553 } 10554 ACQUIRE_LOCK(&lk); 10555 return (1); 10556 } 10557 return (0); 10558 } 10559 10560 /* 10561 * Flush dependencies associated with an allocdirect block. 10562 */ 10563 static int 10564 flush_newblk_dep(vp, mp, lbn) 10565 struct vnode *vp; 10566 struct mount *mp; 10567 ufs_lbn_t lbn; 10568 { 10569 struct newblk *newblk; 10570 struct bufobj *bo; 10571 struct inode *ip; 10572 struct buf *bp; 10573 ufs2_daddr_t blkno; 10574 int error; 10575 10576 error = 0; 10577 bo = &vp->v_bufobj; 10578 ip = VTOI(vp); 10579 blkno = DIP(ip, i_db[lbn]); 10580 if (blkno == 0) 10581 panic("flush_newblk_dep: Missing block"); 10582 ACQUIRE_LOCK(&lk); 10583 /* 10584 * Loop until all dependencies related to this block are satisfied. 10585 * We must be careful to restart after each sleep in case a write 10586 * completes some part of this process for us. 10587 */ 10588 for (;;) { 10589 if (newblk_lookup(mp, blkno, 0, &newblk) == 0) { 10590 FREE_LOCK(&lk); 10591 break; 10592 } 10593 if (newblk->nb_list.wk_type != D_ALLOCDIRECT) 10594 panic("flush_newblk_deps: Bad newblk %p", newblk); 10595 /* 10596 * Flush the journal. 10597 */ 10598 if (newblk->nb_jnewblk != NULL) { 10599 stat_jwait_newblk++; 10600 jwait(&newblk->nb_jnewblk->jn_list); 10601 continue; 10602 } 10603 /* 10604 * Write the bitmap dependency. 10605 */ 10606 if ((newblk->nb_state & DEPCOMPLETE) == 0) { 10607 bp = newblk->nb_bmsafemap->sm_buf; 10608 bp = getdirtybuf(bp, &lk, MNT_WAIT); 10609 if (bp == NULL) 10610 continue; 10611 FREE_LOCK(&lk); 10612 error = bwrite(bp); 10613 if (error) 10614 break; 10615 ACQUIRE_LOCK(&lk); 10616 continue; 10617 } 10618 /* 10619 * Write the buffer. 10620 */ 10621 FREE_LOCK(&lk); 10622 BO_LOCK(bo); 10623 bp = gbincore(bo, lbn); 10624 if (bp != NULL) { 10625 error = BUF_LOCK(bp, LK_EXCLUSIVE | LK_SLEEPFAIL | 10626 LK_INTERLOCK, BO_MTX(bo)); 10627 if (error == ENOLCK) { 10628 ACQUIRE_LOCK(&lk); 10629 continue; /* Slept, retry */ 10630 } 10631 if (error != 0) 10632 break; /* Failed */ 10633 if (bp->b_flags & B_DELWRI) { 10634 bremfree(bp); 10635 error = bwrite(bp); 10636 if (error) 10637 break; 10638 } else 10639 BUF_UNLOCK(bp); 10640 } else 10641 BO_UNLOCK(bo); 10642 /* 10643 * We have to wait for the direct pointers to 10644 * point at the newdirblk before the dependency 10645 * will go away. 10646 */ 10647 error = ffs_update(vp, MNT_WAIT); 10648 if (error) 10649 break; 10650 ACQUIRE_LOCK(&lk); 10651 } 10652 return (error); 10653 } 10654 10655 /* 10656 * Eliminate a pagedep dependency by flushing out all its diradd dependencies. 10657 * Called with splbio blocked. 10658 */ 10659 static int 10660 flush_pagedep_deps(pvp, mp, diraddhdp) 10661 struct vnode *pvp; 10662 struct mount *mp; 10663 struct diraddhd *diraddhdp; 10664 { 10665 struct inodedep *inodedep; 10666 struct inoref *inoref; 10667 struct ufsmount *ump; 10668 struct diradd *dap; 10669 struct vnode *vp; 10670 int error = 0; 10671 struct buf *bp; 10672 ino_t inum; 10673 10674 ump = VFSTOUFS(mp); 10675 restart: 10676 while ((dap = LIST_FIRST(diraddhdp)) != NULL) { 10677 /* 10678 * Flush ourselves if this directory entry 10679 * has a MKDIR_PARENT dependency. 10680 */ 10681 if (dap->da_state & MKDIR_PARENT) { 10682 FREE_LOCK(&lk); 10683 if ((error = ffs_update(pvp, MNT_WAIT)) != 0) 10684 break; 10685 ACQUIRE_LOCK(&lk); 10686 /* 10687 * If that cleared dependencies, go on to next. 10688 */ 10689 if (dap != LIST_FIRST(diraddhdp)) 10690 continue; 10691 if (dap->da_state & MKDIR_PARENT) 10692 panic("flush_pagedep_deps: MKDIR_PARENT"); 10693 } 10694 /* 10695 * A newly allocated directory must have its "." and 10696 * ".." entries written out before its name can be 10697 * committed in its parent. 10698 */ 10699 inum = dap->da_newinum; 10700 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 10701 panic("flush_pagedep_deps: lost inode1"); 10702 /* 10703 * Wait for any pending journal adds to complete so we don't 10704 * cause rollbacks while syncing. 10705 */ 10706 TAILQ_FOREACH(inoref, &inodedep->id_inoreflst, if_deps) { 10707 if ((inoref->if_state & (DEPCOMPLETE | GOINGAWAY)) 10708 == DEPCOMPLETE) { 10709 stat_jwait_inode++; 10710 jwait(&inoref->if_list); 10711 goto restart; 10712 } 10713 } 10714 if (dap->da_state & MKDIR_BODY) { 10715 FREE_LOCK(&lk); 10716 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 10717 FFSV_FORCEINSMQ))) 10718 break; 10719 error = flush_newblk_dep(vp, mp, 0); 10720 /* 10721 * If we still have the dependency we might need to 10722 * update the vnode to sync the new link count to 10723 * disk. 10724 */ 10725 if (error == 0 && dap == LIST_FIRST(diraddhdp)) 10726 error = ffs_update(vp, MNT_WAIT); 10727 vput(vp); 10728 if (error != 0) 10729 break; 10730 ACQUIRE_LOCK(&lk); 10731 /* 10732 * If that cleared dependencies, go on to next. 10733 */ 10734 if (dap != LIST_FIRST(diraddhdp)) 10735 continue; 10736 if (dap->da_state & MKDIR_BODY) { 10737 inodedep_lookup(UFSTOVFS(ump), inum, 0, 10738 &inodedep); 10739 panic("flush_pagedep_deps: MKDIR_BODY " 10740 "inodedep %p dap %p vp %p", 10741 inodedep, dap, vp); 10742 } 10743 } 10744 /* 10745 * Flush the inode on which the directory entry depends. 10746 * Having accounted for MKDIR_PARENT and MKDIR_BODY above, 10747 * the only remaining dependency is that the updated inode 10748 * count must get pushed to disk. The inode has already 10749 * been pushed into its inode buffer (via VOP_UPDATE) at 10750 * the time of the reference count change. So we need only 10751 * locate that buffer, ensure that there will be no rollback 10752 * caused by a bitmap dependency, then write the inode buffer. 10753 */ 10754 retry: 10755 if (inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep) == 0) 10756 panic("flush_pagedep_deps: lost inode"); 10757 /* 10758 * If the inode still has bitmap dependencies, 10759 * push them to disk. 10760 */ 10761 if ((inodedep->id_state & (DEPCOMPLETE | GOINGAWAY)) == 0) { 10762 bp = inodedep->id_bmsafemap->sm_buf; 10763 bp = getdirtybuf(bp, &lk, MNT_WAIT); 10764 if (bp == NULL) 10765 goto retry; 10766 FREE_LOCK(&lk); 10767 if ((error = bwrite(bp)) != 0) 10768 break; 10769 ACQUIRE_LOCK(&lk); 10770 if (dap != LIST_FIRST(diraddhdp)) 10771 continue; 10772 } 10773 /* 10774 * If the inode is still sitting in a buffer waiting 10775 * to be written or waiting for the link count to be 10776 * adjusted update it here to flush it to disk. 10777 */ 10778 if (dap == LIST_FIRST(diraddhdp)) { 10779 FREE_LOCK(&lk); 10780 if ((error = ffs_vgetf(mp, inum, LK_EXCLUSIVE, &vp, 10781 FFSV_FORCEINSMQ))) 10782 break; 10783 error = ffs_update(vp, MNT_WAIT); 10784 vput(vp); 10785 if (error) 10786 break; 10787 ACQUIRE_LOCK(&lk); 10788 } 10789 /* 10790 * If we have failed to get rid of all the dependencies 10791 * then something is seriously wrong. 10792 */ 10793 if (dap == LIST_FIRST(diraddhdp)) { 10794 inodedep_lookup(UFSTOVFS(ump), inum, 0, &inodedep); 10795 panic("flush_pagedep_deps: failed to flush " 10796 "inodedep %p ino %d dap %p", inodedep, inum, dap); 10797 } 10798 } 10799 if (error) 10800 ACQUIRE_LOCK(&lk); 10801 return (error); 10802 } 10803 10804 /* 10805 * A large burst of file addition or deletion activity can drive the 10806 * memory load excessively high. First attempt to slow things down 10807 * using the techniques below. If that fails, this routine requests 10808 * the offending operations to fall back to running synchronously 10809 * until the memory load returns to a reasonable level. 10810 */ 10811 int 10812 softdep_slowdown(vp) 10813 struct vnode *vp; 10814 { 10815 struct ufsmount *ump; 10816 int jlow; 10817 int max_softdeps_hard; 10818 10819 ACQUIRE_LOCK(&lk); 10820 jlow = 0; 10821 /* 10822 * Check for journal space if needed. 10823 */ 10824 if (DOINGSUJ(vp)) { 10825 ump = VFSTOUFS(vp->v_mount); 10826 if (journal_space(ump, 0) == 0) 10827 jlow = 1; 10828 } 10829 max_softdeps_hard = max_softdeps * 11 / 10; 10830 if (num_dirrem < max_softdeps_hard / 2 && 10831 num_inodedep < max_softdeps_hard && 10832 VFSTOUFS(vp->v_mount)->um_numindirdeps < maxindirdeps && 10833 num_freeblkdep < max_softdeps_hard && jlow == 0) { 10834 FREE_LOCK(&lk); 10835 return (0); 10836 } 10837 if (VFSTOUFS(vp->v_mount)->um_numindirdeps >= maxindirdeps || jlow) 10838 softdep_speedup(); 10839 stat_sync_limit_hit += 1; 10840 FREE_LOCK(&lk); 10841 return (1); 10842 } 10843 10844 /* 10845 * Called by the allocation routines when they are about to fail 10846 * in the hope that we can free up some disk space. 10847 * 10848 * First check to see if the work list has anything on it. If it has, 10849 * clean up entries until we successfully free some space. Because this 10850 * process holds inodes locked, we cannot handle any remove requests 10851 * that might block on a locked inode as that could lead to deadlock. 10852 * If the worklist yields no free space, encourage the syncer daemon 10853 * to help us. In no event will we try for longer than tickdelay seconds. 10854 */ 10855 int 10856 softdep_request_cleanup(fs, vp) 10857 struct fs *fs; 10858 struct vnode *vp; 10859 { 10860 struct ufsmount *ump; 10861 long starttime; 10862 ufs2_daddr_t needed; 10863 int error; 10864 10865 ump = VTOI(vp)->i_ump; 10866 mtx_assert(UFS_MTX(ump), MA_OWNED); 10867 needed = fs->fs_cstotal.cs_nbfree + fs->fs_contigsumsize; 10868 starttime = time_second + tickdelay; 10869 /* 10870 * If we are being called because of a process doing a 10871 * copy-on-write, then it is not safe to update the vnode 10872 * as we may recurse into the copy-on-write routine. 10873 */ 10874 if (!(curthread->td_pflags & TDP_COWINPROGRESS)) { 10875 UFS_UNLOCK(ump); 10876 error = ffs_update(vp, 1); 10877 UFS_LOCK(ump); 10878 if (error != 0) 10879 return (0); 10880 } 10881 while (fs->fs_pendingblocks > 0 && fs->fs_cstotal.cs_nbfree <= needed) { 10882 if (time_second > starttime) 10883 return (0); 10884 UFS_UNLOCK(ump); 10885 ACQUIRE_LOCK(&lk); 10886 process_removes(vp); 10887 if (ump->softdep_on_worklist > 0 && 10888 process_worklist_item(UFSTOVFS(ump), LK_NOWAIT) != -1) { 10889 stat_worklist_push += 1; 10890 FREE_LOCK(&lk); 10891 UFS_LOCK(ump); 10892 continue; 10893 } 10894 request_cleanup(UFSTOVFS(ump), FLUSH_REMOVE_WAIT); 10895 FREE_LOCK(&lk); 10896 UFS_LOCK(ump); 10897 } 10898 return (1); 10899 } 10900 10901 /* 10902 * If memory utilization has gotten too high, deliberately slow things 10903 * down and speed up the I/O processing. 10904 */ 10905 extern struct thread *syncertd; 10906 static int 10907 request_cleanup(mp, resource) 10908 struct mount *mp; 10909 int resource; 10910 { 10911 struct thread *td = curthread; 10912 struct ufsmount *ump; 10913 10914 mtx_assert(&lk, MA_OWNED); 10915 /* 10916 * We never hold up the filesystem syncer or buf daemon. 10917 */ 10918 if (td->td_pflags & (TDP_SOFTDEP|TDP_NORUNNINGBUF)) 10919 return (0); 10920 ump = VFSTOUFS(mp); 10921 /* 10922 * First check to see if the work list has gotten backlogged. 10923 * If it has, co-opt this process to help clean up two entries. 10924 * Because this process may hold inodes locked, we cannot 10925 * handle any remove requests that might block on a locked 10926 * inode as that could lead to deadlock. We set TDP_SOFTDEP 10927 * to avoid recursively processing the worklist. 10928 */ 10929 if (ump->softdep_on_worklist > max_softdeps / 10) { 10930 td->td_pflags |= TDP_SOFTDEP; 10931 process_worklist_item(mp, LK_NOWAIT); 10932 process_worklist_item(mp, LK_NOWAIT); 10933 td->td_pflags &= ~TDP_SOFTDEP; 10934 stat_worklist_push += 2; 10935 return(1); 10936 } 10937 /* 10938 * Next, we attempt to speed up the syncer process. If that 10939 * is successful, then we allow the process to continue. 10940 */ 10941 if (softdep_speedup() && resource != FLUSH_REMOVE_WAIT) 10942 return(0); 10943 /* 10944 * If we are resource constrained on inode dependencies, try 10945 * flushing some dirty inodes. Otherwise, we are constrained 10946 * by file deletions, so try accelerating flushes of directories 10947 * with removal dependencies. We would like to do the cleanup 10948 * here, but we probably hold an inode locked at this point and 10949 * that might deadlock against one that we try to clean. So, 10950 * the best that we can do is request the syncer daemon to do 10951 * the cleanup for us. 10952 */ 10953 switch (resource) { 10954 10955 case FLUSH_INODES: 10956 stat_ino_limit_push += 1; 10957 req_clear_inodedeps += 1; 10958 stat_countp = &stat_ino_limit_hit; 10959 break; 10960 10961 case FLUSH_REMOVE: 10962 case FLUSH_REMOVE_WAIT: 10963 stat_blk_limit_push += 1; 10964 req_clear_remove += 1; 10965 stat_countp = &stat_blk_limit_hit; 10966 break; 10967 10968 default: 10969 panic("request_cleanup: unknown type"); 10970 } 10971 /* 10972 * Hopefully the syncer daemon will catch up and awaken us. 10973 * We wait at most tickdelay before proceeding in any case. 10974 */ 10975 proc_waiting += 1; 10976 if (callout_pending(&softdep_callout) == FALSE) 10977 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2, 10978 pause_timer, 0); 10979 10980 msleep((caddr_t)&proc_waiting, &lk, PPAUSE, "softupdate", 0); 10981 proc_waiting -= 1; 10982 return (1); 10983 } 10984 10985 /* 10986 * Awaken processes pausing in request_cleanup and clear proc_waiting 10987 * to indicate that there is no longer a timer running. 10988 */ 10989 static void 10990 pause_timer(arg) 10991 void *arg; 10992 { 10993 10994 /* 10995 * The callout_ API has acquired mtx and will hold it around this 10996 * function call. 10997 */ 10998 *stat_countp += 1; 10999 wakeup_one(&proc_waiting); 11000 if (proc_waiting > 0) 11001 callout_reset(&softdep_callout, tickdelay > 2 ? tickdelay : 2, 11002 pause_timer, 0); 11003 } 11004 11005 /* 11006 * Flush out a directory with at least one removal dependency in an effort to 11007 * reduce the number of dirrem, freefile, and freeblks dependency structures. 11008 */ 11009 static void 11010 clear_remove(td) 11011 struct thread *td; 11012 { 11013 struct pagedep_hashhead *pagedephd; 11014 struct pagedep *pagedep; 11015 static int next = 0; 11016 struct mount *mp; 11017 struct vnode *vp; 11018 struct bufobj *bo; 11019 int error, cnt; 11020 ino_t ino; 11021 11022 mtx_assert(&lk, MA_OWNED); 11023 11024 for (cnt = 0; cnt < pagedep_hash; cnt++) { 11025 pagedephd = &pagedep_hashtbl[next++]; 11026 if (next >= pagedep_hash) 11027 next = 0; 11028 LIST_FOREACH(pagedep, pagedephd, pd_hash) { 11029 if (LIST_EMPTY(&pagedep->pd_dirremhd)) 11030 continue; 11031 mp = pagedep->pd_list.wk_mp; 11032 ino = pagedep->pd_ino; 11033 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 11034 continue; 11035 FREE_LOCK(&lk); 11036 11037 /* 11038 * Let unmount clear deps 11039 */ 11040 error = vfs_busy(mp, MBF_NOWAIT); 11041 if (error != 0) 11042 goto finish_write; 11043 error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 11044 FFSV_FORCEINSMQ); 11045 vfs_unbusy(mp); 11046 if (error != 0) { 11047 softdep_error("clear_remove: vget", error); 11048 goto finish_write; 11049 } 11050 if ((error = ffs_syncvnode(vp, MNT_NOWAIT))) 11051 softdep_error("clear_remove: fsync", error); 11052 bo = &vp->v_bufobj; 11053 BO_LOCK(bo); 11054 drain_output(vp); 11055 BO_UNLOCK(bo); 11056 vput(vp); 11057 finish_write: 11058 vn_finished_write(mp); 11059 ACQUIRE_LOCK(&lk); 11060 return; 11061 } 11062 } 11063 } 11064 11065 /* 11066 * Clear out a block of dirty inodes in an effort to reduce 11067 * the number of inodedep dependency structures. 11068 */ 11069 static void 11070 clear_inodedeps(td) 11071 struct thread *td; 11072 { 11073 struct inodedep_hashhead *inodedephd; 11074 struct inodedep *inodedep; 11075 static int next = 0; 11076 struct mount *mp; 11077 struct vnode *vp; 11078 struct fs *fs; 11079 int error, cnt; 11080 ino_t firstino, lastino, ino; 11081 11082 mtx_assert(&lk, MA_OWNED); 11083 /* 11084 * Pick a random inode dependency to be cleared. 11085 * We will then gather up all the inodes in its block 11086 * that have dependencies and flush them out. 11087 */ 11088 for (cnt = 0; cnt < inodedep_hash; cnt++) { 11089 inodedephd = &inodedep_hashtbl[next++]; 11090 if (next >= inodedep_hash) 11091 next = 0; 11092 if ((inodedep = LIST_FIRST(inodedephd)) != NULL) 11093 break; 11094 } 11095 if (inodedep == NULL) 11096 return; 11097 fs = inodedep->id_fs; 11098 mp = inodedep->id_list.wk_mp; 11099 /* 11100 * Find the last inode in the block with dependencies. 11101 */ 11102 firstino = inodedep->id_ino & ~(INOPB(fs) - 1); 11103 for (lastino = firstino + INOPB(fs) - 1; lastino > firstino; lastino--) 11104 if (inodedep_lookup(mp, lastino, 0, &inodedep) != 0) 11105 break; 11106 /* 11107 * Asynchronously push all but the last inode with dependencies. 11108 * Synchronously push the last inode with dependencies to ensure 11109 * that the inode block gets written to free up the inodedeps. 11110 */ 11111 for (ino = firstino; ino <= lastino; ino++) { 11112 if (inodedep_lookup(mp, ino, 0, &inodedep) == 0) 11113 continue; 11114 if (vn_start_write(NULL, &mp, V_NOWAIT) != 0) 11115 continue; 11116 FREE_LOCK(&lk); 11117 error = vfs_busy(mp, MBF_NOWAIT); /* Let unmount clear deps */ 11118 if (error != 0) { 11119 vn_finished_write(mp); 11120 ACQUIRE_LOCK(&lk); 11121 return; 11122 } 11123 if ((error = ffs_vgetf(mp, ino, LK_EXCLUSIVE, &vp, 11124 FFSV_FORCEINSMQ)) != 0) { 11125 softdep_error("clear_inodedeps: vget", error); 11126 vfs_unbusy(mp); 11127 vn_finished_write(mp); 11128 ACQUIRE_LOCK(&lk); 11129 return; 11130 } 11131 vfs_unbusy(mp); 11132 if (ino == lastino) { 11133 if ((error = ffs_syncvnode(vp, MNT_WAIT))) 11134 softdep_error("clear_inodedeps: fsync1", error); 11135 } else { 11136 if ((error = ffs_syncvnode(vp, MNT_NOWAIT))) 11137 softdep_error("clear_inodedeps: fsync2", error); 11138 BO_LOCK(&vp->v_bufobj); 11139 drain_output(vp); 11140 BO_UNLOCK(&vp->v_bufobj); 11141 } 11142 vput(vp); 11143 vn_finished_write(mp); 11144 ACQUIRE_LOCK(&lk); 11145 } 11146 } 11147 11148 /* 11149 * Function to determine if the buffer has outstanding dependencies 11150 * that will cause a roll-back if the buffer is written. If wantcount 11151 * is set, return number of dependencies, otherwise just yes or no. 11152 */ 11153 static int 11154 softdep_count_dependencies(bp, wantcount) 11155 struct buf *bp; 11156 int wantcount; 11157 { 11158 struct worklist *wk; 11159 struct bmsafemap *bmsafemap; 11160 struct inodedep *inodedep; 11161 struct indirdep *indirdep; 11162 struct freeblks *freeblks; 11163 struct allocindir *aip; 11164 struct pagedep *pagedep; 11165 struct dirrem *dirrem; 11166 struct newblk *newblk; 11167 struct mkdir *mkdir; 11168 struct diradd *dap; 11169 int i, retval; 11170 11171 retval = 0; 11172 ACQUIRE_LOCK(&lk); 11173 LIST_FOREACH(wk, &bp->b_dep, wk_list) { 11174 switch (wk->wk_type) { 11175 11176 case D_INODEDEP: 11177 inodedep = WK_INODEDEP(wk); 11178 if ((inodedep->id_state & DEPCOMPLETE) == 0) { 11179 /* bitmap allocation dependency */ 11180 retval += 1; 11181 if (!wantcount) 11182 goto out; 11183 } 11184 if (TAILQ_FIRST(&inodedep->id_inoupdt)) { 11185 /* direct block pointer dependency */ 11186 retval += 1; 11187 if (!wantcount) 11188 goto out; 11189 } 11190 if (TAILQ_FIRST(&inodedep->id_extupdt)) { 11191 /* direct block pointer dependency */ 11192 retval += 1; 11193 if (!wantcount) 11194 goto out; 11195 } 11196 if (TAILQ_FIRST(&inodedep->id_inoreflst)) { 11197 /* Add reference dependency. */ 11198 retval += 1; 11199 if (!wantcount) 11200 goto out; 11201 } 11202 continue; 11203 11204 case D_INDIRDEP: 11205 indirdep = WK_INDIRDEP(wk); 11206 11207 LIST_FOREACH(aip, &indirdep->ir_deplisthd, ai_next) { 11208 /* indirect block pointer dependency */ 11209 retval += 1; 11210 if (!wantcount) 11211 goto out; 11212 } 11213 continue; 11214 11215 case D_PAGEDEP: 11216 pagedep = WK_PAGEDEP(wk); 11217 LIST_FOREACH(dirrem, &pagedep->pd_dirremhd, dm_next) { 11218 if (LIST_FIRST(&dirrem->dm_jremrefhd)) { 11219 /* Journal remove ref dependency. */ 11220 retval += 1; 11221 if (!wantcount) 11222 goto out; 11223 } 11224 } 11225 for (i = 0; i < DAHASHSZ; i++) { 11226 11227 LIST_FOREACH(dap, &pagedep->pd_diraddhd[i], da_pdlist) { 11228 /* directory entry dependency */ 11229 retval += 1; 11230 if (!wantcount) 11231 goto out; 11232 } 11233 } 11234 continue; 11235 11236 case D_BMSAFEMAP: 11237 bmsafemap = WK_BMSAFEMAP(wk); 11238 if (LIST_FIRST(&bmsafemap->sm_jaddrefhd)) { 11239 /* Add reference dependency. */ 11240 retval += 1; 11241 if (!wantcount) 11242 goto out; 11243 } 11244 if (LIST_FIRST(&bmsafemap->sm_jnewblkhd)) { 11245 /* Allocate block dependency. */ 11246 retval += 1; 11247 if (!wantcount) 11248 goto out; 11249 } 11250 continue; 11251 11252 case D_FREEBLKS: 11253 freeblks = WK_FREEBLKS(wk); 11254 if (LIST_FIRST(&freeblks->fb_jfreeblkhd)) { 11255 /* Freeblk journal dependency. */ 11256 retval += 1; 11257 if (!wantcount) 11258 goto out; 11259 } 11260 continue; 11261 11262 case D_ALLOCDIRECT: 11263 case D_ALLOCINDIR: 11264 newblk = WK_NEWBLK(wk); 11265 if (newblk->nb_jnewblk) { 11266 /* Journal allocate dependency. */ 11267 retval += 1; 11268 if (!wantcount) 11269 goto out; 11270 } 11271 continue; 11272 11273 case D_MKDIR: 11274 mkdir = WK_MKDIR(wk); 11275 if (mkdir->md_jaddref) { 11276 /* Journal reference dependency. */ 11277 retval += 1; 11278 if (!wantcount) 11279 goto out; 11280 } 11281 continue; 11282 11283 case D_FREEWORK: 11284 case D_FREEDEP: 11285 case D_JSEGDEP: 11286 case D_JSEG: 11287 case D_SBDEP: 11288 /* never a dependency on these blocks */ 11289 continue; 11290 11291 default: 11292 panic("softdep_count_dependencies: Unexpected type %s", 11293 TYPENAME(wk->wk_type)); 11294 /* NOTREACHED */ 11295 } 11296 } 11297 out: 11298 FREE_LOCK(&lk); 11299 return retval; 11300 } 11301 11302 /* 11303 * Acquire exclusive access to a buffer. 11304 * Must be called with a locked mtx parameter. 11305 * Return acquired buffer or NULL on failure. 11306 */ 11307 static struct buf * 11308 getdirtybuf(bp, mtx, waitfor) 11309 struct buf *bp; 11310 struct mtx *mtx; 11311 int waitfor; 11312 { 11313 int error; 11314 11315 mtx_assert(mtx, MA_OWNED); 11316 if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0) { 11317 if (waitfor != MNT_WAIT) 11318 return (NULL); 11319 error = BUF_LOCK(bp, 11320 LK_EXCLUSIVE | LK_SLEEPFAIL | LK_INTERLOCK, mtx); 11321 /* 11322 * Even if we sucessfully acquire bp here, we have dropped 11323 * mtx, which may violates our guarantee. 11324 */ 11325 if (error == 0) 11326 BUF_UNLOCK(bp); 11327 else if (error != ENOLCK) 11328 panic("getdirtybuf: inconsistent lock: %d", error); 11329 mtx_lock(mtx); 11330 return (NULL); 11331 } 11332 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 11333 if (mtx == &lk && waitfor == MNT_WAIT) { 11334 mtx_unlock(mtx); 11335 BO_LOCK(bp->b_bufobj); 11336 BUF_UNLOCK(bp); 11337 if ((bp->b_vflags & BV_BKGRDINPROG) != 0) { 11338 bp->b_vflags |= BV_BKGRDWAIT; 11339 msleep(&bp->b_xflags, BO_MTX(bp->b_bufobj), 11340 PRIBIO | PDROP, "getbuf", 0); 11341 } else 11342 BO_UNLOCK(bp->b_bufobj); 11343 mtx_lock(mtx); 11344 return (NULL); 11345 } 11346 BUF_UNLOCK(bp); 11347 if (waitfor != MNT_WAIT) 11348 return (NULL); 11349 /* 11350 * The mtx argument must be bp->b_vp's mutex in 11351 * this case. 11352 */ 11353 #ifdef DEBUG_VFS_LOCKS 11354 if (bp->b_vp->v_type != VCHR) 11355 ASSERT_BO_LOCKED(bp->b_bufobj); 11356 #endif 11357 bp->b_vflags |= BV_BKGRDWAIT; 11358 msleep(&bp->b_xflags, mtx, PRIBIO, "getbuf", 0); 11359 return (NULL); 11360 } 11361 if ((bp->b_flags & B_DELWRI) == 0) { 11362 BUF_UNLOCK(bp); 11363 return (NULL); 11364 } 11365 bremfree(bp); 11366 return (bp); 11367 } 11368 11369 11370 /* 11371 * Check if it is safe to suspend the file system now. On entry, 11372 * the vnode interlock for devvp should be held. Return 0 with 11373 * the mount interlock held if the file system can be suspended now, 11374 * otherwise return EAGAIN with the mount interlock held. 11375 */ 11376 int 11377 softdep_check_suspend(struct mount *mp, 11378 struct vnode *devvp, 11379 int softdep_deps, 11380 int softdep_accdeps, 11381 int secondary_writes, 11382 int secondary_accwrites) 11383 { 11384 struct bufobj *bo; 11385 struct ufsmount *ump; 11386 int error; 11387 11388 ump = VFSTOUFS(mp); 11389 bo = &devvp->v_bufobj; 11390 ASSERT_BO_LOCKED(bo); 11391 11392 for (;;) { 11393 if (!TRY_ACQUIRE_LOCK(&lk)) { 11394 BO_UNLOCK(bo); 11395 ACQUIRE_LOCK(&lk); 11396 FREE_LOCK(&lk); 11397 BO_LOCK(bo); 11398 continue; 11399 } 11400 MNT_ILOCK(mp); 11401 if (mp->mnt_secondary_writes != 0) { 11402 FREE_LOCK(&lk); 11403 BO_UNLOCK(bo); 11404 msleep(&mp->mnt_secondary_writes, 11405 MNT_MTX(mp), 11406 (PUSER - 1) | PDROP, "secwr", 0); 11407 BO_LOCK(bo); 11408 continue; 11409 } 11410 break; 11411 } 11412 11413 /* 11414 * Reasons for needing more work before suspend: 11415 * - Dirty buffers on devvp. 11416 * - Softdep activity occurred after start of vnode sync loop 11417 * - Secondary writes occurred after start of vnode sync loop 11418 */ 11419 error = 0; 11420 if (bo->bo_numoutput > 0 || 11421 bo->bo_dirty.bv_cnt > 0 || 11422 softdep_deps != 0 || 11423 ump->softdep_deps != 0 || 11424 softdep_accdeps != ump->softdep_accdeps || 11425 secondary_writes != 0 || 11426 mp->mnt_secondary_writes != 0 || 11427 secondary_accwrites != mp->mnt_secondary_accwrites) 11428 error = EAGAIN; 11429 FREE_LOCK(&lk); 11430 BO_UNLOCK(bo); 11431 return (error); 11432 } 11433 11434 11435 /* 11436 * Get the number of dependency structures for the file system, both 11437 * the current number and the total number allocated. These will 11438 * later be used to detect that softdep processing has occurred. 11439 */ 11440 void 11441 softdep_get_depcounts(struct mount *mp, 11442 int *softdep_depsp, 11443 int *softdep_accdepsp) 11444 { 11445 struct ufsmount *ump; 11446 11447 ump = VFSTOUFS(mp); 11448 ACQUIRE_LOCK(&lk); 11449 *softdep_depsp = ump->softdep_deps; 11450 *softdep_accdepsp = ump->softdep_accdeps; 11451 FREE_LOCK(&lk); 11452 } 11453 11454 /* 11455 * Wait for pending output on a vnode to complete. 11456 * Must be called with vnode lock and interlock locked. 11457 * 11458 * XXX: Should just be a call to bufobj_wwait(). 11459 */ 11460 static void 11461 drain_output(vp) 11462 struct vnode *vp; 11463 { 11464 struct bufobj *bo; 11465 11466 bo = &vp->v_bufobj; 11467 ASSERT_VOP_LOCKED(vp, "drain_output"); 11468 ASSERT_BO_LOCKED(bo); 11469 11470 while (bo->bo_numoutput) { 11471 bo->bo_flag |= BO_WWAIT; 11472 msleep((caddr_t)&bo->bo_numoutput, 11473 BO_MTX(bo), PRIBIO + 1, "drainvp", 0); 11474 } 11475 } 11476 11477 /* 11478 * Called whenever a buffer that is being invalidated or reallocated 11479 * contains dependencies. This should only happen if an I/O error has 11480 * occurred. The routine is called with the buffer locked. 11481 */ 11482 static void 11483 softdep_deallocate_dependencies(bp) 11484 struct buf *bp; 11485 { 11486 11487 if ((bp->b_ioflags & BIO_ERROR) == 0) 11488 panic("softdep_deallocate_dependencies: dangling deps"); 11489 softdep_error(bp->b_vp->v_mount->mnt_stat.f_mntonname, bp->b_error); 11490 panic("softdep_deallocate_dependencies: unrecovered I/O error"); 11491 } 11492 11493 /* 11494 * Function to handle asynchronous write errors in the filesystem. 11495 */ 11496 static void 11497 softdep_error(func, error) 11498 char *func; 11499 int error; 11500 { 11501 11502 /* XXX should do something better! */ 11503 printf("%s: got error %d while accessing filesystem\n", func, error); 11504 } 11505 11506 #ifdef DDB 11507 11508 static void 11509 inodedep_print(struct inodedep *inodedep, int verbose) 11510 { 11511 db_printf("%p fs %p st %x ino %jd inoblk %jd delta %d nlink %d" 11512 " saveino %p\n", 11513 inodedep, inodedep->id_fs, inodedep->id_state, 11514 (intmax_t)inodedep->id_ino, 11515 (intmax_t)fsbtodb(inodedep->id_fs, 11516 ino_to_fsba(inodedep->id_fs, inodedep->id_ino)), 11517 inodedep->id_nlinkdelta, inodedep->id_savednlink, 11518 inodedep->id_savedino1); 11519 11520 if (verbose == 0) 11521 return; 11522 11523 db_printf("\tpendinghd %p, bufwait %p, inowait %p, inoreflst %p, " 11524 "mkdiradd %p\n", 11525 LIST_FIRST(&inodedep->id_pendinghd), 11526 LIST_FIRST(&inodedep->id_bufwait), 11527 LIST_FIRST(&inodedep->id_inowait), 11528 TAILQ_FIRST(&inodedep->id_inoreflst), 11529 inodedep->id_mkdiradd); 11530 db_printf("\tinoupdt %p, newinoupdt %p, extupdt %p, newextupdt %p\n", 11531 TAILQ_FIRST(&inodedep->id_inoupdt), 11532 TAILQ_FIRST(&inodedep->id_newinoupdt), 11533 TAILQ_FIRST(&inodedep->id_extupdt), 11534 TAILQ_FIRST(&inodedep->id_newextupdt)); 11535 } 11536 11537 DB_SHOW_COMMAND(inodedep, db_show_inodedep) 11538 { 11539 11540 if (have_addr == 0) { 11541 db_printf("Address required\n"); 11542 return; 11543 } 11544 inodedep_print((struct inodedep*)addr, 1); 11545 } 11546 11547 DB_SHOW_COMMAND(inodedeps, db_show_inodedeps) 11548 { 11549 struct inodedep_hashhead *inodedephd; 11550 struct inodedep *inodedep; 11551 struct fs *fs; 11552 int cnt; 11553 11554 fs = have_addr ? (struct fs *)addr : NULL; 11555 for (cnt = 0; cnt < inodedep_hash; cnt++) { 11556 inodedephd = &inodedep_hashtbl[cnt]; 11557 LIST_FOREACH(inodedep, inodedephd, id_hash) { 11558 if (fs != NULL && fs != inodedep->id_fs) 11559 continue; 11560 inodedep_print(inodedep, 0); 11561 } 11562 } 11563 } 11564 11565 DB_SHOW_COMMAND(worklist, db_show_worklist) 11566 { 11567 struct worklist *wk; 11568 11569 if (have_addr == 0) { 11570 db_printf("Address required\n"); 11571 return; 11572 } 11573 wk = (struct worklist *)addr; 11574 printf("worklist: %p type %s state 0x%X\n", 11575 wk, TYPENAME(wk->wk_type), wk->wk_state); 11576 } 11577 11578 DB_SHOW_COMMAND(workhead, db_show_workhead) 11579 { 11580 struct workhead *wkhd; 11581 struct worklist *wk; 11582 int i; 11583 11584 if (have_addr == 0) { 11585 db_printf("Address required\n"); 11586 return; 11587 } 11588 wkhd = (struct workhead *)addr; 11589 wk = LIST_FIRST(wkhd); 11590 for (i = 0; i < 100 && wk != NULL; i++, wk = LIST_NEXT(wk, wk_list)) 11591 db_printf("worklist: %p type %s state 0x%X", 11592 wk, TYPENAME(wk->wk_type), wk->wk_state); 11593 if (i == 100) 11594 db_printf("workhead overflow"); 11595 printf("\n"); 11596 } 11597 11598 11599 DB_SHOW_COMMAND(mkdirs, db_show_mkdirs) 11600 { 11601 struct jaddref *jaddref; 11602 struct diradd *diradd; 11603 struct mkdir *mkdir; 11604 11605 LIST_FOREACH(mkdir, &mkdirlisthd, md_mkdirs) { 11606 diradd = mkdir->md_diradd; 11607 db_printf("mkdir: %p state 0x%X dap %p state 0x%X", 11608 mkdir, mkdir->md_state, diradd, diradd->da_state); 11609 if ((jaddref = mkdir->md_jaddref) != NULL) 11610 db_printf(" jaddref %p jaddref state 0x%X", 11611 jaddref, jaddref->ja_state); 11612 db_printf("\n"); 11613 } 11614 } 11615 11616 #endif /* DDB */ 11617 11618 #endif /* SOFTUPDATES */ 11619