1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 #include <sys/zfs_context.h> 29 #include <sys/spa.h> 30 #include <sys/dmu.h> 31 #include <sys/zap.h> 32 #include <sys/arc.h> 33 #include <sys/stat.h> 34 #include <sys/resource.h> 35 #include <sys/zil.h> 36 #include <sys/zil_impl.h> 37 #include <sys/dsl_dataset.h> 38 #include <sys/vdev.h> 39 #include <sys/dmu_tx.h> 40 41 /* 42 * The zfs intent log (ZIL) saves transaction records of system calls 43 * that change the file system in memory with enough information 44 * to be able to replay them. These are stored in memory until 45 * either the DMU transaction group (txg) commits them to the stable pool 46 * and they can be discarded, or they are flushed to the stable log 47 * (also in the pool) due to a fsync, O_DSYNC or other synchronous 48 * requirement. In the event of a panic or power fail then those log 49 * records (transactions) are replayed. 50 * 51 * There is one ZIL per file system. Its on-disk (pool) format consists 52 * of 3 parts: 53 * 54 * - ZIL header 55 * - ZIL blocks 56 * - ZIL records 57 * 58 * A log record holds a system call transaction. Log blocks can 59 * hold many log records and the blocks are chained together. 60 * Each ZIL block contains a block pointer (blkptr_t) to the next 61 * ZIL block in the chain. The ZIL header points to the first 62 * block in the chain. Note there is not a fixed place in the pool 63 * to hold blocks. They are dynamically allocated and freed as 64 * needed from the blocks available. Figure X shows the ZIL structure: 65 */ 66 67 /* 68 * This global ZIL switch affects all pools 69 */ 70 int zil_disable = 0; /* disable intent logging */ 71 72 /* 73 * Tunable parameter for debugging or performance analysis. Setting 74 * zfs_nocacheflush will cause corruption on power loss if a volatile 75 * out-of-order write cache is enabled. 76 */ 77 boolean_t zfs_nocacheflush = B_FALSE; 78 79 static kmem_cache_t *zil_lwb_cache; 80 81 static int 82 zil_dva_compare(const void *x1, const void *x2) 83 { 84 const dva_t *dva1 = x1; 85 const dva_t *dva2 = x2; 86 87 if (DVA_GET_VDEV(dva1) < DVA_GET_VDEV(dva2)) 88 return (-1); 89 if (DVA_GET_VDEV(dva1) > DVA_GET_VDEV(dva2)) 90 return (1); 91 92 if (DVA_GET_OFFSET(dva1) < DVA_GET_OFFSET(dva2)) 93 return (-1); 94 if (DVA_GET_OFFSET(dva1) > DVA_GET_OFFSET(dva2)) 95 return (1); 96 97 return (0); 98 } 99 100 static void 101 zil_dva_tree_init(avl_tree_t *t) 102 { 103 avl_create(t, zil_dva_compare, sizeof (zil_dva_node_t), 104 offsetof(zil_dva_node_t, zn_node)); 105 } 106 107 static void 108 zil_dva_tree_fini(avl_tree_t *t) 109 { 110 zil_dva_node_t *zn; 111 void *cookie = NULL; 112 113 while ((zn = avl_destroy_nodes(t, &cookie)) != NULL) 114 kmem_free(zn, sizeof (zil_dva_node_t)); 115 116 avl_destroy(t); 117 } 118 119 static int 120 zil_dva_tree_add(avl_tree_t *t, dva_t *dva) 121 { 122 zil_dva_node_t *zn; 123 avl_index_t where; 124 125 if (avl_find(t, dva, &where) != NULL) 126 return (EEXIST); 127 128 zn = kmem_alloc(sizeof (zil_dva_node_t), KM_SLEEP); 129 zn->zn_dva = *dva; 130 avl_insert(t, zn, where); 131 132 return (0); 133 } 134 135 static zil_header_t * 136 zil_header_in_syncing_context(zilog_t *zilog) 137 { 138 return ((zil_header_t *)zilog->zl_header); 139 } 140 141 static void 142 zil_init_log_chain(zilog_t *zilog, blkptr_t *bp) 143 { 144 zio_cksum_t *zc = &bp->blk_cksum; 145 146 zc->zc_word[ZIL_ZC_GUID_0] = spa_get_random(-1ULL); 147 zc->zc_word[ZIL_ZC_GUID_1] = spa_get_random(-1ULL); 148 zc->zc_word[ZIL_ZC_OBJSET] = dmu_objset_id(zilog->zl_os); 149 zc->zc_word[ZIL_ZC_SEQ] = 1ULL; 150 } 151 152 /* 153 * Read a log block, make sure it's valid, and byteswap it if necessary. 154 */ 155 static int 156 zil_read_log_block(zilog_t *zilog, const blkptr_t *bp, arc_buf_t **abufpp) 157 { 158 blkptr_t blk = *bp; 159 zbookmark_t zb; 160 uint32_t aflags = ARC_WAIT; 161 int error; 162 163 zb.zb_objset = bp->blk_cksum.zc_word[ZIL_ZC_OBJSET]; 164 zb.zb_object = 0; 165 zb.zb_level = -1; 166 zb.zb_blkid = bp->blk_cksum.zc_word[ZIL_ZC_SEQ]; 167 168 *abufpp = NULL; 169 170 error = arc_read(NULL, zilog->zl_spa, &blk, byteswap_uint64_array, 171 arc_getbuf_func, abufpp, ZIO_PRIORITY_SYNC_READ, ZIO_FLAG_CANFAIL | 172 ZIO_FLAG_SPECULATIVE | ZIO_FLAG_SCRUB, &aflags, &zb); 173 174 if (error == 0) { 175 char *data = (*abufpp)->b_data; 176 uint64_t blksz = BP_GET_LSIZE(bp); 177 zil_trailer_t *ztp = (zil_trailer_t *)(data + blksz) - 1; 178 zio_cksum_t cksum = bp->blk_cksum; 179 180 /* 181 * Sequence numbers should be... sequential. The checksum 182 * verifier for the next block should be bp's checksum plus 1. 183 */ 184 cksum.zc_word[ZIL_ZC_SEQ]++; 185 186 if (bcmp(&cksum, &ztp->zit_next_blk.blk_cksum, sizeof (cksum))) 187 error = ESTALE; 188 else if (BP_IS_HOLE(&ztp->zit_next_blk)) 189 error = ENOENT; 190 else if (ztp->zit_nused > (blksz - sizeof (zil_trailer_t))) 191 error = EOVERFLOW; 192 193 if (error) { 194 VERIFY(arc_buf_remove_ref(*abufpp, abufpp) == 1); 195 *abufpp = NULL; 196 } 197 } 198 199 dprintf("error %d on %llu:%llu\n", error, zb.zb_objset, zb.zb_blkid); 200 201 return (error); 202 } 203 204 /* 205 * Parse the intent log, and call parse_func for each valid record within. 206 * Return the highest sequence number. 207 */ 208 uint64_t 209 zil_parse(zilog_t *zilog, zil_parse_blk_func_t *parse_blk_func, 210 zil_parse_lr_func_t *parse_lr_func, void *arg, uint64_t txg) 211 { 212 const zil_header_t *zh = zilog->zl_header; 213 uint64_t claim_seq = zh->zh_claim_seq; 214 uint64_t seq = 0; 215 uint64_t max_seq = 0; 216 blkptr_t blk = zh->zh_log; 217 arc_buf_t *abuf; 218 char *lrbuf, *lrp; 219 zil_trailer_t *ztp; 220 int reclen, error; 221 222 if (BP_IS_HOLE(&blk)) 223 return (max_seq); 224 225 /* 226 * Starting at the block pointed to by zh_log we read the log chain. 227 * For each block in the chain we strongly check that block to 228 * ensure its validity. We stop when an invalid block is found. 229 * For each block pointer in the chain we call parse_blk_func(). 230 * For each record in each valid block we call parse_lr_func(). 231 * If the log has been claimed, stop if we encounter a sequence 232 * number greater than the highest claimed sequence number. 233 */ 234 zil_dva_tree_init(&zilog->zl_dva_tree); 235 for (;;) { 236 seq = blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 237 238 if (claim_seq != 0 && seq > claim_seq) 239 break; 240 241 ASSERT(max_seq < seq); 242 max_seq = seq; 243 244 error = zil_read_log_block(zilog, &blk, &abuf); 245 246 if (parse_blk_func != NULL) 247 parse_blk_func(zilog, &blk, arg, txg); 248 249 if (error) 250 break; 251 252 lrbuf = abuf->b_data; 253 ztp = (zil_trailer_t *)(lrbuf + BP_GET_LSIZE(&blk)) - 1; 254 blk = ztp->zit_next_blk; 255 256 if (parse_lr_func == NULL) { 257 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 258 continue; 259 } 260 261 for (lrp = lrbuf; lrp < lrbuf + ztp->zit_nused; lrp += reclen) { 262 lr_t *lr = (lr_t *)lrp; 263 reclen = lr->lrc_reclen; 264 ASSERT3U(reclen, >=, sizeof (lr_t)); 265 parse_lr_func(zilog, lr, arg, txg); 266 } 267 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 268 } 269 zil_dva_tree_fini(&zilog->zl_dva_tree); 270 271 return (max_seq); 272 } 273 274 /* ARGSUSED */ 275 static void 276 zil_claim_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t first_txg) 277 { 278 spa_t *spa = zilog->zl_spa; 279 int err; 280 281 /* 282 * Claim log block if not already committed and not already claimed. 283 */ 284 if (bp->blk_birth >= first_txg && 285 zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp)) == 0) { 286 err = zio_wait(zio_claim(NULL, spa, first_txg, bp, NULL, NULL)); 287 ASSERT(err == 0); 288 } 289 } 290 291 static void 292 zil_claim_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t first_txg) 293 { 294 if (lrc->lrc_txtype == TX_WRITE) { 295 lr_write_t *lr = (lr_write_t *)lrc; 296 zil_claim_log_block(zilog, &lr->lr_blkptr, tx, first_txg); 297 } 298 } 299 300 /* ARGSUSED */ 301 static void 302 zil_free_log_block(zilog_t *zilog, blkptr_t *bp, void *tx, uint64_t claim_txg) 303 { 304 zio_free_blk(zilog->zl_spa, bp, dmu_tx_get_txg(tx)); 305 } 306 307 static void 308 zil_free_log_record(zilog_t *zilog, lr_t *lrc, void *tx, uint64_t claim_txg) 309 { 310 /* 311 * If we previously claimed it, we need to free it. 312 */ 313 if (claim_txg != 0 && lrc->lrc_txtype == TX_WRITE) { 314 lr_write_t *lr = (lr_write_t *)lrc; 315 blkptr_t *bp = &lr->lr_blkptr; 316 if (bp->blk_birth >= claim_txg && 317 !zil_dva_tree_add(&zilog->zl_dva_tree, BP_IDENTITY(bp))) { 318 (void) arc_free(NULL, zilog->zl_spa, 319 dmu_tx_get_txg(tx), bp, NULL, NULL, ARC_WAIT); 320 } 321 } 322 } 323 324 /* 325 * Create an on-disk intent log. 326 */ 327 static void 328 zil_create(zilog_t *zilog) 329 { 330 const zil_header_t *zh = zilog->zl_header; 331 lwb_t *lwb; 332 uint64_t txg = 0; 333 dmu_tx_t *tx = NULL; 334 blkptr_t blk; 335 int error = 0; 336 337 /* 338 * Wait for any previous destroy to complete. 339 */ 340 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 341 342 ASSERT(zh->zh_claim_txg == 0); 343 ASSERT(zh->zh_replay_seq == 0); 344 345 blk = zh->zh_log; 346 347 /* 348 * If we don't already have an initial log block, allocate one now. 349 */ 350 if (BP_IS_HOLE(&blk)) { 351 tx = dmu_tx_create(zilog->zl_os); 352 (void) dmu_tx_assign(tx, TXG_WAIT); 353 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 354 txg = dmu_tx_get_txg(tx); 355 356 error = zio_alloc_blk(zilog->zl_spa, ZIL_MIN_BLKSZ, &blk, 357 NULL, txg); 358 359 if (error == 0) 360 zil_init_log_chain(zilog, &blk); 361 } 362 363 /* 364 * Allocate a log write buffer (lwb) for the first log block. 365 */ 366 if (error == 0) { 367 lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 368 lwb->lwb_zilog = zilog; 369 lwb->lwb_blk = blk; 370 lwb->lwb_nused = 0; 371 lwb->lwb_sz = BP_GET_LSIZE(&lwb->lwb_blk); 372 lwb->lwb_buf = zio_buf_alloc(lwb->lwb_sz); 373 lwb->lwb_max_txg = txg; 374 lwb->lwb_zio = NULL; 375 376 mutex_enter(&zilog->zl_lock); 377 list_insert_tail(&zilog->zl_lwb_list, lwb); 378 mutex_exit(&zilog->zl_lock); 379 } 380 381 /* 382 * If we just allocated the first log block, commit our transaction 383 * and wait for zil_sync() to stuff the block poiner into zh_log. 384 * (zh is part of the MOS, so we cannot modify it in open context.) 385 */ 386 if (tx != NULL) { 387 dmu_tx_commit(tx); 388 txg_wait_synced(zilog->zl_dmu_pool, txg); 389 } 390 391 ASSERT(bcmp(&blk, &zh->zh_log, sizeof (blk)) == 0); 392 } 393 394 /* 395 * In one tx, free all log blocks and clear the log header. 396 * If keep_first is set, then we're replaying a log with no content. 397 * We want to keep the first block, however, so that the first 398 * synchronous transaction doesn't require a txg_wait_synced() 399 * in zil_create(). We don't need to txg_wait_synced() here either 400 * when keep_first is set, because both zil_create() and zil_destroy() 401 * will wait for any in-progress destroys to complete. 402 */ 403 void 404 zil_destroy(zilog_t *zilog, boolean_t keep_first) 405 { 406 const zil_header_t *zh = zilog->zl_header; 407 lwb_t *lwb; 408 dmu_tx_t *tx; 409 uint64_t txg; 410 411 /* 412 * Wait for any previous destroy to complete. 413 */ 414 txg_wait_synced(zilog->zl_dmu_pool, zilog->zl_destroy_txg); 415 416 if (BP_IS_HOLE(&zh->zh_log)) 417 return; 418 419 tx = dmu_tx_create(zilog->zl_os); 420 (void) dmu_tx_assign(tx, TXG_WAIT); 421 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 422 txg = dmu_tx_get_txg(tx); 423 424 mutex_enter(&zilog->zl_lock); 425 426 ASSERT3U(zilog->zl_destroy_txg, <, txg); 427 zilog->zl_destroy_txg = txg; 428 zilog->zl_keep_first = keep_first; 429 430 if (!list_is_empty(&zilog->zl_lwb_list)) { 431 ASSERT(zh->zh_claim_txg == 0); 432 ASSERT(!keep_first); 433 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 434 list_remove(&zilog->zl_lwb_list, lwb); 435 if (lwb->lwb_buf != NULL) 436 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 437 zio_free_blk(zilog->zl_spa, &lwb->lwb_blk, txg); 438 kmem_cache_free(zil_lwb_cache, lwb); 439 } 440 } else { 441 if (!keep_first) { 442 (void) zil_parse(zilog, zil_free_log_block, 443 zil_free_log_record, tx, zh->zh_claim_txg); 444 } 445 } 446 mutex_exit(&zilog->zl_lock); 447 448 dmu_tx_commit(tx); 449 450 if (keep_first) /* no need to wait in this case */ 451 return; 452 453 txg_wait_synced(zilog->zl_dmu_pool, txg); 454 ASSERT(BP_IS_HOLE(&zh->zh_log)); 455 } 456 457 int 458 zil_claim(char *osname, void *txarg) 459 { 460 dmu_tx_t *tx = txarg; 461 uint64_t first_txg = dmu_tx_get_txg(tx); 462 zilog_t *zilog; 463 zil_header_t *zh; 464 objset_t *os; 465 int error; 466 467 error = dmu_objset_open(osname, DMU_OST_ANY, DS_MODE_STANDARD, &os); 468 if (error) { 469 cmn_err(CE_WARN, "can't process intent log for %s", osname); 470 return (0); 471 } 472 473 zilog = dmu_objset_zil(os); 474 zh = zil_header_in_syncing_context(zilog); 475 476 /* 477 * Claim all log blocks if we haven't already done so, and remember 478 * the highest claimed sequence number. This ensures that if we can 479 * read only part of the log now (e.g. due to a missing device), 480 * but we can read the entire log later, we will not try to replay 481 * or destroy beyond the last block we successfully claimed. 482 */ 483 ASSERT3U(zh->zh_claim_txg, <=, first_txg); 484 if (zh->zh_claim_txg == 0 && !BP_IS_HOLE(&zh->zh_log)) { 485 zh->zh_claim_txg = first_txg; 486 zh->zh_claim_seq = zil_parse(zilog, zil_claim_log_block, 487 zil_claim_log_record, tx, first_txg); 488 dsl_dataset_dirty(dmu_objset_ds(os), tx); 489 } 490 491 ASSERT3U(first_txg, ==, (spa_last_synced_txg(zilog->zl_spa) + 1)); 492 dmu_objset_close(os); 493 return (0); 494 } 495 496 void 497 zil_add_vdev(zilog_t *zilog, uint64_t vdev) 498 { 499 zil_vdev_t *zv, *new; 500 uint64_t bmap_sz = sizeof (zilog->zl_vdev_bmap) << 3; 501 uchar_t *cp; 502 503 if (zfs_nocacheflush) 504 return; 505 506 if (vdev < bmap_sz) { 507 cp = zilog->zl_vdev_bmap + (vdev / 8); 508 atomic_or_8(cp, 1 << (vdev % 8)); 509 } else { 510 /* 511 * insert into ordered list 512 */ 513 mutex_enter(&zilog->zl_lock); 514 for (zv = list_head(&zilog->zl_vdev_list); zv != NULL; 515 zv = list_next(&zilog->zl_vdev_list, zv)) { 516 if (zv->vdev == vdev) { 517 /* duplicate found - just return */ 518 mutex_exit(&zilog->zl_lock); 519 return; 520 } 521 if (zv->vdev > vdev) { 522 /* insert before this entry */ 523 new = kmem_alloc(sizeof (zil_vdev_t), 524 KM_SLEEP); 525 new->vdev = vdev; 526 list_insert_before(&zilog->zl_vdev_list, 527 zv, new); 528 mutex_exit(&zilog->zl_lock); 529 return; 530 } 531 } 532 /* ran off end of list, insert at the end */ 533 ASSERT(zv == NULL); 534 new = kmem_alloc(sizeof (zil_vdev_t), KM_SLEEP); 535 new->vdev = vdev; 536 list_insert_tail(&zilog->zl_vdev_list, new); 537 mutex_exit(&zilog->zl_lock); 538 } 539 } 540 541 /* start an async flush of the write cache for this vdev */ 542 void 543 zil_flush_vdev(spa_t *spa, uint64_t vdev, zio_t **zio) 544 { 545 vdev_t *vd; 546 547 if (*zio == NULL) 548 *zio = zio_root(spa, NULL, NULL, ZIO_FLAG_CANFAIL); 549 550 vd = vdev_lookup_top(spa, vdev); 551 ASSERT(vd); 552 553 (void) zio_nowait(zio_ioctl(*zio, spa, vd, DKIOCFLUSHWRITECACHE, 554 NULL, NULL, ZIO_PRIORITY_NOW, 555 ZIO_FLAG_CANFAIL | ZIO_FLAG_DONT_RETRY)); 556 } 557 558 void 559 zil_flush_vdevs(zilog_t *zilog) 560 { 561 zil_vdev_t *zv; 562 zio_t *zio = NULL; 563 spa_t *spa = zilog->zl_spa; 564 uint64_t vdev; 565 uint8_t b; 566 int i, j; 567 568 ASSERT(zilog->zl_writer); 569 570 for (i = 0; i < sizeof (zilog->zl_vdev_bmap); i++) { 571 b = zilog->zl_vdev_bmap[i]; 572 if (b == 0) 573 continue; 574 for (j = 0; j < 8; j++) { 575 if (b & (1 << j)) { 576 vdev = (i << 3) + j; 577 zil_flush_vdev(spa, vdev, &zio); 578 } 579 } 580 zilog->zl_vdev_bmap[i] = 0; 581 } 582 583 while ((zv = list_head(&zilog->zl_vdev_list)) != NULL) { 584 zil_flush_vdev(spa, zv->vdev, &zio); 585 list_remove(&zilog->zl_vdev_list, zv); 586 kmem_free(zv, sizeof (zil_vdev_t)); 587 } 588 /* 589 * Wait for all the flushes to complete. Not all devices actually 590 * support the DKIOCFLUSHWRITECACHE ioctl, so it's OK if it fails. 591 */ 592 if (zio) 593 (void) zio_wait(zio); 594 } 595 596 /* 597 * Function called when a log block write completes 598 */ 599 static void 600 zil_lwb_write_done(zio_t *zio) 601 { 602 lwb_t *lwb = zio->io_private; 603 zilog_t *zilog = lwb->lwb_zilog; 604 605 /* 606 * Now that we've written this log block, we have a stable pointer 607 * to the next block in the chain, so it's OK to let the txg in 608 * which we allocated the next block sync. 609 */ 610 txg_rele_to_sync(&lwb->lwb_txgh); 611 612 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 613 mutex_enter(&zilog->zl_lock); 614 lwb->lwb_buf = NULL; 615 if (zio->io_error) { 616 zilog->zl_log_error = B_TRUE; 617 mutex_exit(&zilog->zl_lock); 618 return; 619 } 620 mutex_exit(&zilog->zl_lock); 621 } 622 623 /* 624 * Initialize the io for a log block. 625 * 626 * Note, we should not initialize the IO until we are about 627 * to use it, since zio_rewrite() does a spa_config_enter(). 628 */ 629 static void 630 zil_lwb_write_init(zilog_t *zilog, lwb_t *lwb) 631 { 632 zbookmark_t zb; 633 634 zb.zb_objset = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_OBJSET]; 635 zb.zb_object = 0; 636 zb.zb_level = -1; 637 zb.zb_blkid = lwb->lwb_blk.blk_cksum.zc_word[ZIL_ZC_SEQ]; 638 639 if (zilog->zl_root_zio == NULL) { 640 zilog->zl_root_zio = zio_root(zilog->zl_spa, NULL, NULL, 641 ZIO_FLAG_CANFAIL); 642 } 643 if (lwb->lwb_zio == NULL) { 644 lwb->lwb_zio = zio_rewrite(zilog->zl_root_zio, zilog->zl_spa, 645 ZIO_CHECKSUM_ZILOG, 0, &lwb->lwb_blk, lwb->lwb_buf, 646 lwb->lwb_sz, zil_lwb_write_done, lwb, 647 ZIO_PRIORITY_LOG_WRITE, ZIO_FLAG_MUSTSUCCEED, &zb); 648 } 649 } 650 651 /* 652 * Start a log block write and advance to the next log block. 653 * Calls are serialized. 654 */ 655 static lwb_t * 656 zil_lwb_write_start(zilog_t *zilog, lwb_t *lwb) 657 { 658 lwb_t *nlwb; 659 zil_trailer_t *ztp = (zil_trailer_t *)(lwb->lwb_buf + lwb->lwb_sz) - 1; 660 spa_t *spa = zilog->zl_spa; 661 blkptr_t *bp = &ztp->zit_next_blk; 662 uint64_t txg; 663 uint64_t zil_blksz; 664 int error; 665 666 ASSERT(lwb->lwb_nused <= ZIL_BLK_DATA_SZ(lwb)); 667 668 /* 669 * Allocate the next block and save its address in this block 670 * before writing it in order to establish the log chain. 671 * Note that if the allocation of nlwb synced before we wrote 672 * the block that points at it (lwb), we'd leak it if we crashed. 673 * Therefore, we don't do txg_rele_to_sync() until zil_lwb_write_done(). 674 */ 675 txg = txg_hold_open(zilog->zl_dmu_pool, &lwb->lwb_txgh); 676 txg_rele_to_quiesce(&lwb->lwb_txgh); 677 678 /* 679 * Pick a ZIL blocksize. We request a size that is the 680 * maximum of the previous used size, the current used size and 681 * the amount waiting in the queue. 682 */ 683 zil_blksz = MAX(zilog->zl_prev_used, 684 zilog->zl_cur_used + sizeof (*ztp)); 685 zil_blksz = MAX(zil_blksz, zilog->zl_itx_list_sz + sizeof (*ztp)); 686 zil_blksz = P2ROUNDUP_TYPED(zil_blksz, ZIL_MIN_BLKSZ, uint64_t); 687 if (zil_blksz > ZIL_MAX_BLKSZ) 688 zil_blksz = ZIL_MAX_BLKSZ; 689 690 BP_ZERO(bp); 691 /* pass the old blkptr in order to spread log blocks across devs */ 692 error = zio_alloc_blk(spa, zil_blksz, bp, &lwb->lwb_blk, txg); 693 if (error) { 694 dmu_tx_t *tx = dmu_tx_create_assigned(zilog->zl_dmu_pool, txg); 695 696 /* 697 * We dirty the dataset to ensure that zil_sync() will 698 * be called to remove this lwb from our zl_lwb_list. 699 * Failing to do so, may leave an lwb with a NULL lwb_buf 700 * hanging around on the zl_lwb_list. 701 */ 702 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 703 704 705 /* 706 * Since we've just experienced an allocation failure so we 707 * terminate the current lwb and send it on its way. 708 */ 709 ztp->zit_pad = 0; 710 ztp->zit_nused = lwb->lwb_nused; 711 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 712 zio_nowait(lwb->lwb_zio); 713 714 /* 715 * By returning NULL the caller will call tx_wait_synced() 716 */ 717 return (NULL); 718 } 719 720 ASSERT3U(bp->blk_birth, ==, txg); 721 ztp->zit_pad = 0; 722 ztp->zit_nused = lwb->lwb_nused; 723 ztp->zit_bt.zbt_cksum = lwb->lwb_blk.blk_cksum; 724 bp->blk_cksum = lwb->lwb_blk.blk_cksum; 725 bp->blk_cksum.zc_word[ZIL_ZC_SEQ]++; 726 727 /* 728 * Allocate a new log write buffer (lwb). 729 */ 730 nlwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); 731 732 nlwb->lwb_zilog = zilog; 733 nlwb->lwb_blk = *bp; 734 nlwb->lwb_nused = 0; 735 nlwb->lwb_sz = BP_GET_LSIZE(&nlwb->lwb_blk); 736 nlwb->lwb_buf = zio_buf_alloc(nlwb->lwb_sz); 737 nlwb->lwb_max_txg = txg; 738 nlwb->lwb_zio = NULL; 739 740 /* 741 * Put new lwb at the end of the log chain 742 */ 743 mutex_enter(&zilog->zl_lock); 744 list_insert_tail(&zilog->zl_lwb_list, nlwb); 745 mutex_exit(&zilog->zl_lock); 746 747 /* Record the vdev for later flushing */ 748 zil_add_vdev(zilog, DVA_GET_VDEV(BP_IDENTITY(&(lwb->lwb_blk)))); 749 750 /* 751 * kick off the write for the old log block 752 */ 753 dprintf_bp(&lwb->lwb_blk, "lwb %p txg %llu: ", lwb, txg); 754 ASSERT(lwb->lwb_zio); 755 zio_nowait(lwb->lwb_zio); 756 757 return (nlwb); 758 } 759 760 static lwb_t * 761 zil_lwb_commit(zilog_t *zilog, itx_t *itx, lwb_t *lwb) 762 { 763 lr_t *lrc = &itx->itx_lr; /* common log record */ 764 lr_write_t *lr = (lr_write_t *)lrc; 765 uint64_t txg = lrc->lrc_txg; 766 uint64_t reclen = lrc->lrc_reclen; 767 uint64_t dlen; 768 769 if (lwb == NULL) 770 return (NULL); 771 ASSERT(lwb->lwb_buf != NULL); 772 773 if (lrc->lrc_txtype == TX_WRITE && itx->itx_wr_state == WR_NEED_COPY) 774 dlen = P2ROUNDUP_TYPED( 775 lr->lr_length, sizeof (uint64_t), uint64_t); 776 else 777 dlen = 0; 778 779 zilog->zl_cur_used += (reclen + dlen); 780 781 zil_lwb_write_init(zilog, lwb); 782 783 /* 784 * If this record won't fit in the current log block, start a new one. 785 */ 786 if (lwb->lwb_nused + reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 787 lwb = zil_lwb_write_start(zilog, lwb); 788 if (lwb == NULL) 789 return (NULL); 790 zil_lwb_write_init(zilog, lwb); 791 ASSERT(lwb->lwb_nused == 0); 792 if (reclen + dlen > ZIL_BLK_DATA_SZ(lwb)) { 793 txg_wait_synced(zilog->zl_dmu_pool, txg); 794 return (lwb); 795 } 796 } 797 798 /* 799 * Update the lrc_seq, to be log record sequence number. See zil.h 800 * Then copy the record to the log buffer. 801 */ 802 lrc->lrc_seq = ++zilog->zl_lr_seq; /* we are single threaded */ 803 bcopy(lrc, lwb->lwb_buf + lwb->lwb_nused, reclen); 804 805 /* 806 * If it's a write, fetch the data or get its blkptr as appropriate. 807 */ 808 if (lrc->lrc_txtype == TX_WRITE) { 809 if (txg > spa_freeze_txg(zilog->zl_spa)) 810 txg_wait_synced(zilog->zl_dmu_pool, txg); 811 if (itx->itx_wr_state != WR_COPIED) { 812 char *dbuf; 813 int error; 814 815 /* alignment is guaranteed */ 816 lr = (lr_write_t *)(lwb->lwb_buf + lwb->lwb_nused); 817 if (dlen) { 818 ASSERT(itx->itx_wr_state == WR_NEED_COPY); 819 dbuf = lwb->lwb_buf + lwb->lwb_nused + reclen; 820 lr->lr_common.lrc_reclen += dlen; 821 } else { 822 ASSERT(itx->itx_wr_state == WR_INDIRECT); 823 dbuf = NULL; 824 } 825 error = zilog->zl_get_data( 826 itx->itx_private, lr, dbuf, lwb->lwb_zio); 827 if (error) { 828 ASSERT(error == ENOENT || error == EEXIST || 829 error == EALREADY); 830 return (lwb); 831 } 832 } 833 } 834 835 lwb->lwb_nused += reclen + dlen; 836 lwb->lwb_max_txg = MAX(lwb->lwb_max_txg, txg); 837 ASSERT3U(lwb->lwb_nused, <=, ZIL_BLK_DATA_SZ(lwb)); 838 ASSERT3U(P2PHASE(lwb->lwb_nused, sizeof (uint64_t)), ==, 0); 839 840 return (lwb); 841 } 842 843 itx_t * 844 zil_itx_create(int txtype, size_t lrsize) 845 { 846 itx_t *itx; 847 848 lrsize = P2ROUNDUP_TYPED(lrsize, sizeof (uint64_t), size_t); 849 850 itx = kmem_alloc(offsetof(itx_t, itx_lr) + lrsize, KM_SLEEP); 851 itx->itx_lr.lrc_txtype = txtype; 852 itx->itx_lr.lrc_reclen = lrsize; 853 itx->itx_lr.lrc_seq = 0; /* defensive */ 854 855 return (itx); 856 } 857 858 uint64_t 859 zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) 860 { 861 uint64_t seq; 862 863 ASSERT(itx->itx_lr.lrc_seq == 0); 864 865 mutex_enter(&zilog->zl_lock); 866 list_insert_tail(&zilog->zl_itx_list, itx); 867 zilog->zl_itx_list_sz += itx->itx_lr.lrc_reclen; 868 itx->itx_lr.lrc_txg = dmu_tx_get_txg(tx); 869 itx->itx_lr.lrc_seq = seq = ++zilog->zl_itx_seq; 870 mutex_exit(&zilog->zl_lock); 871 872 return (seq); 873 } 874 875 /* 876 * Free up all in-memory intent log transactions that have now been synced. 877 */ 878 static void 879 zil_itx_clean(zilog_t *zilog) 880 { 881 uint64_t synced_txg = spa_last_synced_txg(zilog->zl_spa); 882 uint64_t freeze_txg = spa_freeze_txg(zilog->zl_spa); 883 list_t clean_list; 884 itx_t *itx; 885 886 list_create(&clean_list, sizeof (itx_t), offsetof(itx_t, itx_node)); 887 888 mutex_enter(&zilog->zl_lock); 889 /* wait for a log writer to finish walking list */ 890 while (zilog->zl_writer) { 891 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 892 } 893 894 /* 895 * Move the sync'd log transactions to a separate list so we can call 896 * kmem_free without holding the zl_lock. 897 * 898 * There is no need to set zl_writer as we don't drop zl_lock here 899 */ 900 while ((itx = list_head(&zilog->zl_itx_list)) != NULL && 901 itx->itx_lr.lrc_txg <= MIN(synced_txg, freeze_txg)) { 902 list_remove(&zilog->zl_itx_list, itx); 903 zilog->zl_itx_list_sz -= itx->itx_lr.lrc_reclen; 904 list_insert_tail(&clean_list, itx); 905 } 906 cv_broadcast(&zilog->zl_cv_writer); 907 mutex_exit(&zilog->zl_lock); 908 909 /* destroy sync'd log transactions */ 910 while ((itx = list_head(&clean_list)) != NULL) { 911 list_remove(&clean_list, itx); 912 kmem_free(itx, offsetof(itx_t, itx_lr) 913 + itx->itx_lr.lrc_reclen); 914 } 915 list_destroy(&clean_list); 916 } 917 918 /* 919 * If there are any in-memory intent log transactions which have now been 920 * synced then start up a taskq to free them. 921 */ 922 void 923 zil_clean(zilog_t *zilog) 924 { 925 itx_t *itx; 926 927 mutex_enter(&zilog->zl_lock); 928 itx = list_head(&zilog->zl_itx_list); 929 if ((itx != NULL) && 930 (itx->itx_lr.lrc_txg <= spa_last_synced_txg(zilog->zl_spa))) { 931 (void) taskq_dispatch(zilog->zl_clean_taskq, 932 (void (*)(void *))zil_itx_clean, zilog, TQ_NOSLEEP); 933 } 934 mutex_exit(&zilog->zl_lock); 935 } 936 937 void 938 zil_commit_writer(zilog_t *zilog, uint64_t seq, uint64_t foid) 939 { 940 uint64_t txg; 941 uint64_t reclen; 942 uint64_t commit_seq = 0; 943 itx_t *itx, *itx_next = (itx_t *)-1; 944 lwb_t *lwb; 945 spa_t *spa; 946 947 zilog->zl_writer = B_TRUE; 948 zilog->zl_root_zio = NULL; 949 spa = zilog->zl_spa; 950 951 if (zilog->zl_suspend) { 952 lwb = NULL; 953 } else { 954 lwb = list_tail(&zilog->zl_lwb_list); 955 if (lwb == NULL) { 956 /* 957 * Return if there's nothing to flush before we 958 * dirty the fs by calling zil_create() 959 */ 960 if (list_is_empty(&zilog->zl_itx_list)) { 961 zilog->zl_writer = B_FALSE; 962 return; 963 } 964 mutex_exit(&zilog->zl_lock); 965 zil_create(zilog); 966 mutex_enter(&zilog->zl_lock); 967 lwb = list_tail(&zilog->zl_lwb_list); 968 } 969 } 970 971 /* Loop through in-memory log transactions filling log blocks. */ 972 DTRACE_PROBE1(zil__cw1, zilog_t *, zilog); 973 for (;;) { 974 /* 975 * Find the next itx to push: 976 * Push all transactions related to specified foid and all 977 * other transactions except TX_WRITE, TX_TRUNCATE, 978 * TX_SETATTR and TX_ACL for all other files. 979 */ 980 if (itx_next != (itx_t *)-1) 981 itx = itx_next; 982 else 983 itx = list_head(&zilog->zl_itx_list); 984 for (; itx != NULL; itx = list_next(&zilog->zl_itx_list, itx)) { 985 if (foid == 0) /* push all foids? */ 986 break; 987 if (itx->itx_sync) /* push all O_[D]SYNC */ 988 break; 989 switch (itx->itx_lr.lrc_txtype) { 990 case TX_SETATTR: 991 case TX_WRITE: 992 case TX_TRUNCATE: 993 case TX_ACL: 994 /* lr_foid is same offset for these records */ 995 if (((lr_write_t *)&itx->itx_lr)->lr_foid 996 != foid) { 997 continue; /* skip this record */ 998 } 999 } 1000 break; 1001 } 1002 if (itx == NULL) 1003 break; 1004 1005 reclen = itx->itx_lr.lrc_reclen; 1006 if ((itx->itx_lr.lrc_seq > seq) && 1007 ((lwb == NULL) || (lwb->lwb_nused == 0) || 1008 (lwb->lwb_nused + reclen > ZIL_BLK_DATA_SZ(lwb)))) { 1009 break; 1010 } 1011 1012 /* 1013 * Save the next pointer. Even though we soon drop 1014 * zl_lock all threads that may change the list 1015 * (another writer or zil_itx_clean) can't do so until 1016 * they have zl_writer. 1017 */ 1018 itx_next = list_next(&zilog->zl_itx_list, itx); 1019 list_remove(&zilog->zl_itx_list, itx); 1020 mutex_exit(&zilog->zl_lock); 1021 txg = itx->itx_lr.lrc_txg; 1022 ASSERT(txg); 1023 1024 if (txg > spa_last_synced_txg(spa) || 1025 txg > spa_freeze_txg(spa)) 1026 lwb = zil_lwb_commit(zilog, itx, lwb); 1027 kmem_free(itx, offsetof(itx_t, itx_lr) 1028 + itx->itx_lr.lrc_reclen); 1029 mutex_enter(&zilog->zl_lock); 1030 zilog->zl_itx_list_sz -= reclen; 1031 } 1032 DTRACE_PROBE1(zil__cw2, zilog_t *, zilog); 1033 /* determine commit sequence number */ 1034 itx = list_head(&zilog->zl_itx_list); 1035 if (itx) 1036 commit_seq = itx->itx_lr.lrc_seq; 1037 else 1038 commit_seq = zilog->zl_itx_seq; 1039 mutex_exit(&zilog->zl_lock); 1040 1041 /* write the last block out */ 1042 if (lwb != NULL && lwb->lwb_zio != NULL) 1043 lwb = zil_lwb_write_start(zilog, lwb); 1044 1045 zilog->zl_prev_used = zilog->zl_cur_used; 1046 zilog->zl_cur_used = 0; 1047 1048 /* 1049 * Wait if necessary for the log blocks to be on stable storage. 1050 */ 1051 if (zilog->zl_root_zio) { 1052 DTRACE_PROBE1(zil__cw3, zilog_t *, zilog); 1053 (void) zio_wait(zilog->zl_root_zio); 1054 DTRACE_PROBE1(zil__cw4, zilog_t *, zilog); 1055 if (!zfs_nocacheflush) 1056 zil_flush_vdevs(zilog); 1057 } 1058 1059 if (zilog->zl_log_error || lwb == NULL) { 1060 zilog->zl_log_error = 0; 1061 txg_wait_synced(zilog->zl_dmu_pool, 0); 1062 } 1063 1064 mutex_enter(&zilog->zl_lock); 1065 zilog->zl_writer = B_FALSE; 1066 1067 ASSERT3U(commit_seq, >=, zilog->zl_commit_seq); 1068 zilog->zl_commit_seq = commit_seq; 1069 } 1070 1071 /* 1072 * Push zfs transactions to stable storage up to the supplied sequence number. 1073 * If foid is 0 push out all transactions, otherwise push only those 1074 * for that file or might have been used to create that file. 1075 */ 1076 void 1077 zil_commit(zilog_t *zilog, uint64_t seq, uint64_t foid) 1078 { 1079 if (zilog == NULL || seq == 0) 1080 return; 1081 1082 mutex_enter(&zilog->zl_lock); 1083 1084 seq = MIN(seq, zilog->zl_itx_seq); /* cap seq at largest itx seq */ 1085 1086 while (zilog->zl_writer) { 1087 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1088 if (seq < zilog->zl_commit_seq) { 1089 mutex_exit(&zilog->zl_lock); 1090 return; 1091 } 1092 } 1093 zil_commit_writer(zilog, seq, foid); /* drops zl_lock */ 1094 /* wake up others waiting on the commit */ 1095 cv_broadcast(&zilog->zl_cv_writer); 1096 mutex_exit(&zilog->zl_lock); 1097 } 1098 1099 /* 1100 * Called in syncing context to free committed log blocks and update log header. 1101 */ 1102 void 1103 zil_sync(zilog_t *zilog, dmu_tx_t *tx) 1104 { 1105 zil_header_t *zh = zil_header_in_syncing_context(zilog); 1106 uint64_t txg = dmu_tx_get_txg(tx); 1107 spa_t *spa = zilog->zl_spa; 1108 lwb_t *lwb; 1109 1110 mutex_enter(&zilog->zl_lock); 1111 1112 ASSERT(zilog->zl_stop_sync == 0); 1113 1114 zh->zh_replay_seq = zilog->zl_replay_seq[txg & TXG_MASK]; 1115 1116 if (zilog->zl_destroy_txg == txg) { 1117 blkptr_t blk = zh->zh_log; 1118 1119 ASSERT(list_head(&zilog->zl_lwb_list) == NULL); 1120 ASSERT(spa_sync_pass(spa) == 1); 1121 1122 bzero(zh, sizeof (zil_header_t)); 1123 bzero(zilog->zl_replay_seq, sizeof (zilog->zl_replay_seq)); 1124 1125 if (zilog->zl_keep_first) { 1126 /* 1127 * If this block was part of log chain that couldn't 1128 * be claimed because a device was missing during 1129 * zil_claim(), but that device later returns, 1130 * then this block could erroneously appear valid. 1131 * To guard against this, assign a new GUID to the new 1132 * log chain so it doesn't matter what blk points to. 1133 */ 1134 zil_init_log_chain(zilog, &blk); 1135 zh->zh_log = blk; 1136 } 1137 } 1138 1139 for (;;) { 1140 lwb = list_head(&zilog->zl_lwb_list); 1141 if (lwb == NULL) { 1142 mutex_exit(&zilog->zl_lock); 1143 return; 1144 } 1145 zh->zh_log = lwb->lwb_blk; 1146 if (lwb->lwb_buf != NULL || lwb->lwb_max_txg > txg) 1147 break; 1148 list_remove(&zilog->zl_lwb_list, lwb); 1149 zio_free_blk(spa, &lwb->lwb_blk, txg); 1150 kmem_cache_free(zil_lwb_cache, lwb); 1151 1152 /* 1153 * If we don't have anything left in the lwb list then 1154 * we've had an allocation failure and we need to zero 1155 * out the zil_header blkptr so that we don't end 1156 * up freeing the same block twice. 1157 */ 1158 if (list_head(&zilog->zl_lwb_list) == NULL) 1159 BP_ZERO(&zh->zh_log); 1160 } 1161 mutex_exit(&zilog->zl_lock); 1162 } 1163 1164 void 1165 zil_init(void) 1166 { 1167 zil_lwb_cache = kmem_cache_create("zil_lwb_cache", 1168 sizeof (struct lwb), 0, NULL, NULL, NULL, NULL, NULL, 0); 1169 } 1170 1171 void 1172 zil_fini(void) 1173 { 1174 kmem_cache_destroy(zil_lwb_cache); 1175 } 1176 1177 zilog_t * 1178 zil_alloc(objset_t *os, zil_header_t *zh_phys) 1179 { 1180 zilog_t *zilog; 1181 1182 zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP); 1183 1184 zilog->zl_header = zh_phys; 1185 zilog->zl_os = os; 1186 zilog->zl_spa = dmu_objset_spa(os); 1187 zilog->zl_dmu_pool = dmu_objset_pool(os); 1188 zilog->zl_destroy_txg = TXG_INITIAL - 1; 1189 1190 mutex_init(&zilog->zl_lock, NULL, MUTEX_DEFAULT, NULL); 1191 1192 list_create(&zilog->zl_itx_list, sizeof (itx_t), 1193 offsetof(itx_t, itx_node)); 1194 1195 list_create(&zilog->zl_lwb_list, sizeof (lwb_t), 1196 offsetof(lwb_t, lwb_node)); 1197 1198 list_create(&zilog->zl_vdev_list, sizeof (zil_vdev_t), 1199 offsetof(zil_vdev_t, vdev_seq_node)); 1200 1201 return (zilog); 1202 } 1203 1204 void 1205 zil_free(zilog_t *zilog) 1206 { 1207 lwb_t *lwb; 1208 zil_vdev_t *zv; 1209 1210 zilog->zl_stop_sync = 1; 1211 1212 while ((lwb = list_head(&zilog->zl_lwb_list)) != NULL) { 1213 list_remove(&zilog->zl_lwb_list, lwb); 1214 if (lwb->lwb_buf != NULL) 1215 zio_buf_free(lwb->lwb_buf, lwb->lwb_sz); 1216 kmem_cache_free(zil_lwb_cache, lwb); 1217 } 1218 list_destroy(&zilog->zl_lwb_list); 1219 1220 while ((zv = list_head(&zilog->zl_vdev_list)) != NULL) { 1221 list_remove(&zilog->zl_vdev_list, zv); 1222 kmem_free(zv, sizeof (zil_vdev_t)); 1223 } 1224 list_destroy(&zilog->zl_vdev_list); 1225 1226 ASSERT(list_head(&zilog->zl_itx_list) == NULL); 1227 list_destroy(&zilog->zl_itx_list); 1228 mutex_destroy(&zilog->zl_lock); 1229 1230 kmem_free(zilog, sizeof (zilog_t)); 1231 } 1232 1233 /* 1234 * return true if the initial log block is not valid 1235 */ 1236 static int 1237 zil_empty(zilog_t *zilog) 1238 { 1239 const zil_header_t *zh = zilog->zl_header; 1240 arc_buf_t *abuf = NULL; 1241 1242 if (BP_IS_HOLE(&zh->zh_log)) 1243 return (1); 1244 1245 if (zil_read_log_block(zilog, &zh->zh_log, &abuf) != 0) 1246 return (1); 1247 1248 VERIFY(arc_buf_remove_ref(abuf, &abuf) == 1); 1249 return (0); 1250 } 1251 1252 /* 1253 * Open an intent log. 1254 */ 1255 zilog_t * 1256 zil_open(objset_t *os, zil_get_data_t *get_data) 1257 { 1258 zilog_t *zilog = dmu_objset_zil(os); 1259 1260 zilog->zl_get_data = get_data; 1261 zilog->zl_clean_taskq = taskq_create("zil_clean", 1, minclsyspri, 1262 2, 2, TASKQ_PREPOPULATE); 1263 1264 return (zilog); 1265 } 1266 1267 /* 1268 * Close an intent log. 1269 */ 1270 void 1271 zil_close(zilog_t *zilog) 1272 { 1273 /* 1274 * If the log isn't already committed, mark the objset dirty 1275 * (so zil_sync() will be called) and wait for that txg to sync. 1276 */ 1277 if (!zil_is_committed(zilog)) { 1278 uint64_t txg; 1279 dmu_tx_t *tx = dmu_tx_create(zilog->zl_os); 1280 (void) dmu_tx_assign(tx, TXG_WAIT); 1281 dsl_dataset_dirty(dmu_objset_ds(zilog->zl_os), tx); 1282 txg = dmu_tx_get_txg(tx); 1283 dmu_tx_commit(tx); 1284 txg_wait_synced(zilog->zl_dmu_pool, txg); 1285 } 1286 1287 taskq_destroy(zilog->zl_clean_taskq); 1288 zilog->zl_clean_taskq = NULL; 1289 zilog->zl_get_data = NULL; 1290 1291 zil_itx_clean(zilog); 1292 ASSERT(list_head(&zilog->zl_itx_list) == NULL); 1293 } 1294 1295 /* 1296 * Suspend an intent log. While in suspended mode, we still honor 1297 * synchronous semantics, but we rely on txg_wait_synced() to do it. 1298 * We suspend the log briefly when taking a snapshot so that the snapshot 1299 * contains all the data it's supposed to, and has an empty intent log. 1300 */ 1301 int 1302 zil_suspend(zilog_t *zilog) 1303 { 1304 const zil_header_t *zh = zilog->zl_header; 1305 1306 mutex_enter(&zilog->zl_lock); 1307 if (zh->zh_claim_txg != 0) { /* unplayed log */ 1308 mutex_exit(&zilog->zl_lock); 1309 return (EBUSY); 1310 } 1311 if (zilog->zl_suspend++ != 0) { 1312 /* 1313 * Someone else already began a suspend. 1314 * Just wait for them to finish. 1315 */ 1316 while (zilog->zl_suspending) 1317 cv_wait(&zilog->zl_cv_suspend, &zilog->zl_lock); 1318 ASSERT(BP_IS_HOLE(&zh->zh_log)); 1319 mutex_exit(&zilog->zl_lock); 1320 return (0); 1321 } 1322 zilog->zl_suspending = B_TRUE; 1323 mutex_exit(&zilog->zl_lock); 1324 1325 zil_commit(zilog, UINT64_MAX, 0); 1326 1327 /* 1328 * Wait for any in-flight log writes to complete. 1329 */ 1330 mutex_enter(&zilog->zl_lock); 1331 while (zilog->zl_writer) 1332 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1333 mutex_exit(&zilog->zl_lock); 1334 1335 zil_destroy(zilog, B_FALSE); 1336 1337 mutex_enter(&zilog->zl_lock); 1338 ASSERT(BP_IS_HOLE(&zh->zh_log)); 1339 zilog->zl_suspending = B_FALSE; 1340 cv_broadcast(&zilog->zl_cv_suspend); 1341 mutex_exit(&zilog->zl_lock); 1342 1343 return (0); 1344 } 1345 1346 void 1347 zil_resume(zilog_t *zilog) 1348 { 1349 mutex_enter(&zilog->zl_lock); 1350 ASSERT(zilog->zl_suspend != 0); 1351 zilog->zl_suspend--; 1352 mutex_exit(&zilog->zl_lock); 1353 } 1354 1355 typedef struct zil_replay_arg { 1356 objset_t *zr_os; 1357 zil_replay_func_t **zr_replay; 1358 void *zr_arg; 1359 uint64_t *zr_txgp; 1360 boolean_t zr_byteswap; 1361 char *zr_lrbuf; 1362 } zil_replay_arg_t; 1363 1364 static void 1365 zil_replay_log_record(zilog_t *zilog, lr_t *lr, void *zra, uint64_t claim_txg) 1366 { 1367 zil_replay_arg_t *zr = zra; 1368 const zil_header_t *zh = zilog->zl_header; 1369 uint64_t reclen = lr->lrc_reclen; 1370 uint64_t txtype = lr->lrc_txtype; 1371 char *name; 1372 int pass, error, sunk; 1373 1374 if (zilog->zl_stop_replay) 1375 return; 1376 1377 if (lr->lrc_txg < claim_txg) /* already committed */ 1378 return; 1379 1380 if (lr->lrc_seq <= zh->zh_replay_seq) /* already replayed */ 1381 return; 1382 1383 /* 1384 * Make a copy of the data so we can revise and extend it. 1385 */ 1386 bcopy(lr, zr->zr_lrbuf, reclen); 1387 1388 /* 1389 * The log block containing this lr may have been byteswapped 1390 * so that we can easily examine common fields like lrc_txtype. 1391 * However, the log is a mix of different data types, and only the 1392 * replay vectors know how to byteswap their records. Therefore, if 1393 * the lr was byteswapped, undo it before invoking the replay vector. 1394 */ 1395 if (zr->zr_byteswap) 1396 byteswap_uint64_array(zr->zr_lrbuf, reclen); 1397 1398 /* 1399 * If this is a TX_WRITE with a blkptr, suck in the data. 1400 */ 1401 if (txtype == TX_WRITE && reclen == sizeof (lr_write_t)) { 1402 lr_write_t *lrw = (lr_write_t *)lr; 1403 blkptr_t *wbp = &lrw->lr_blkptr; 1404 uint64_t wlen = lrw->lr_length; 1405 char *wbuf = zr->zr_lrbuf + reclen; 1406 1407 if (BP_IS_HOLE(wbp)) { /* compressed to a hole */ 1408 bzero(wbuf, wlen); 1409 } else { 1410 /* 1411 * A subsequent write may have overwritten this block, 1412 * in which case wbp may have been been freed and 1413 * reallocated, and our read of wbp may fail with a 1414 * checksum error. We can safely ignore this because 1415 * the later write will provide the correct data. 1416 */ 1417 zbookmark_t zb; 1418 1419 zb.zb_objset = dmu_objset_id(zilog->zl_os); 1420 zb.zb_object = lrw->lr_foid; 1421 zb.zb_level = -1; 1422 zb.zb_blkid = lrw->lr_offset / BP_GET_LSIZE(wbp); 1423 1424 (void) zio_wait(zio_read(NULL, zilog->zl_spa, 1425 wbp, wbuf, BP_GET_LSIZE(wbp), NULL, NULL, 1426 ZIO_PRIORITY_SYNC_READ, 1427 ZIO_FLAG_CANFAIL | ZIO_FLAG_SPECULATIVE, &zb)); 1428 (void) memmove(wbuf, wbuf + lrw->lr_blkoff, wlen); 1429 } 1430 } 1431 1432 /* 1433 * We must now do two things atomically: replay this log record, 1434 * and update the log header to reflect the fact that we did so. 1435 * We use the DMU's ability to assign into a specific txg to do this. 1436 */ 1437 for (pass = 1, sunk = B_FALSE; /* CONSTANTCONDITION */; pass++) { 1438 uint64_t replay_txg; 1439 dmu_tx_t *replay_tx; 1440 1441 replay_tx = dmu_tx_create(zr->zr_os); 1442 error = dmu_tx_assign(replay_tx, TXG_WAIT); 1443 if (error) { 1444 dmu_tx_abort(replay_tx); 1445 break; 1446 } 1447 1448 replay_txg = dmu_tx_get_txg(replay_tx); 1449 1450 if (txtype == 0 || txtype >= TX_MAX_TYPE) { 1451 error = EINVAL; 1452 } else { 1453 /* 1454 * On the first pass, arrange for the replay vector 1455 * to fail its dmu_tx_assign(). That's the only way 1456 * to ensure that those code paths remain well tested. 1457 */ 1458 *zr->zr_txgp = replay_txg - (pass == 1); 1459 error = zr->zr_replay[txtype](zr->zr_arg, zr->zr_lrbuf, 1460 zr->zr_byteswap); 1461 *zr->zr_txgp = TXG_NOWAIT; 1462 } 1463 1464 if (error == 0) { 1465 dsl_dataset_dirty(dmu_objset_ds(zr->zr_os), replay_tx); 1466 zilog->zl_replay_seq[replay_txg & TXG_MASK] = 1467 lr->lrc_seq; 1468 } 1469 1470 dmu_tx_commit(replay_tx); 1471 1472 if (!error) 1473 return; 1474 1475 /* 1476 * The DMU's dnode layer doesn't see removes until the txg 1477 * commits, so a subsequent claim can spuriously fail with 1478 * EEXIST. So if we receive any error other than ERESTART 1479 * we try syncing out any removes then retrying the 1480 * transaction. 1481 */ 1482 if (error != ERESTART && !sunk) { 1483 txg_wait_synced(spa_get_dsl(zilog->zl_spa), 0); 1484 sunk = B_TRUE; 1485 continue; /* retry */ 1486 } 1487 1488 if (error != ERESTART) 1489 break; 1490 1491 if (pass != 1) 1492 txg_wait_open(spa_get_dsl(zilog->zl_spa), 1493 replay_txg + 1); 1494 1495 dprintf("pass %d, retrying\n", pass); 1496 } 1497 1498 ASSERT(error && error != ERESTART); 1499 name = kmem_alloc(MAXNAMELEN, KM_SLEEP); 1500 dmu_objset_name(zr->zr_os, name); 1501 cmn_err(CE_WARN, "ZFS replay transaction error %d, " 1502 "dataset %s, seq 0x%llx, txtype %llu\n", 1503 error, name, (u_longlong_t)lr->lrc_seq, (u_longlong_t)txtype); 1504 zilog->zl_stop_replay = 1; 1505 kmem_free(name, MAXNAMELEN); 1506 } 1507 1508 /* ARGSUSED */ 1509 static void 1510 zil_incr_blks(zilog_t *zilog, blkptr_t *bp, void *arg, uint64_t claim_txg) 1511 { 1512 zilog->zl_replay_blks++; 1513 } 1514 1515 /* 1516 * If this dataset has a non-empty intent log, replay it and destroy it. 1517 */ 1518 void 1519 zil_replay(objset_t *os, void *arg, uint64_t *txgp, 1520 zil_replay_func_t *replay_func[TX_MAX_TYPE]) 1521 { 1522 zilog_t *zilog = dmu_objset_zil(os); 1523 const zil_header_t *zh = zilog->zl_header; 1524 zil_replay_arg_t zr; 1525 1526 if (zil_empty(zilog)) { 1527 zil_destroy(zilog, B_TRUE); 1528 return; 1529 } 1530 1531 zr.zr_os = os; 1532 zr.zr_replay = replay_func; 1533 zr.zr_arg = arg; 1534 zr.zr_txgp = txgp; 1535 zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log); 1536 zr.zr_lrbuf = kmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP); 1537 1538 /* 1539 * Wait for in-progress removes to sync before starting replay. 1540 */ 1541 txg_wait_synced(zilog->zl_dmu_pool, 0); 1542 1543 zilog->zl_stop_replay = 0; 1544 zilog->zl_replay_time = lbolt; 1545 ASSERT(zilog->zl_replay_blks == 0); 1546 (void) zil_parse(zilog, zil_incr_blks, zil_replay_log_record, &zr, 1547 zh->zh_claim_txg); 1548 kmem_free(zr.zr_lrbuf, 2 * SPA_MAXBLOCKSIZE); 1549 1550 zil_destroy(zilog, B_FALSE); 1551 } 1552 1553 /* 1554 * Report whether all transactions are committed 1555 */ 1556 int 1557 zil_is_committed(zilog_t *zilog) 1558 { 1559 lwb_t *lwb; 1560 int ret; 1561 1562 mutex_enter(&zilog->zl_lock); 1563 while (zilog->zl_writer) 1564 cv_wait(&zilog->zl_cv_writer, &zilog->zl_lock); 1565 1566 /* recent unpushed intent log transactions? */ 1567 if (!list_is_empty(&zilog->zl_itx_list)) { 1568 ret = B_FALSE; 1569 goto out; 1570 } 1571 1572 /* intent log never used? */ 1573 lwb = list_head(&zilog->zl_lwb_list); 1574 if (lwb == NULL) { 1575 ret = B_TRUE; 1576 goto out; 1577 } 1578 1579 /* 1580 * more than 1 log buffer means zil_sync() hasn't yet freed 1581 * entries after a txg has committed 1582 */ 1583 if (list_next(&zilog->zl_lwb_list, lwb)) { 1584 ret = B_FALSE; 1585 goto out; 1586 } 1587 1588 ASSERT(zil_empty(zilog)); 1589 ret = B_TRUE; 1590 out: 1591 cv_broadcast(&zilog->zl_cv_writer); 1592 mutex_exit(&zilog->zl_lock); 1593 return (ret); 1594 } 1595