1 // SPDX-License-Identifier: GPL-2.0 2 3 #include <linux/bitops.h> 4 #include <linux/slab.h> 5 #include <linux/blkdev.h> 6 #include <linux/sched/mm.h> 7 #include <linux/atomic.h> 8 #include <linux/vmalloc.h> 9 #include "ctree.h" 10 #include "volumes.h" 11 #include "zoned.h" 12 #include "disk-io.h" 13 #include "block-group.h" 14 #include "dev-replace.h" 15 #include "space-info.h" 16 #include "fs.h" 17 #include "accessors.h" 18 #include "bio.h" 19 #include "transaction.h" 20 #include "sysfs.h" 21 22 /* Maximum number of zones to report per blkdev_report_zones() call */ 23 #define BTRFS_REPORT_NR_ZONES 4096 24 /* Invalid allocation pointer value for missing devices */ 25 #define WP_MISSING_DEV ((u64)-1) 26 /* Pseudo write pointer value for conventional zone */ 27 #define WP_CONVENTIONAL ((u64)-2) 28 29 /* 30 * Location of the first zone of superblock logging zone pairs. 31 * 32 * - primary superblock: 0B (zone 0) 33 * - first copy: 512G (zone starting at that offset) 34 * - second copy: 4T (zone starting at that offset) 35 */ 36 #define BTRFS_SB_LOG_PRIMARY_OFFSET (0ULL) 37 #define BTRFS_SB_LOG_FIRST_OFFSET (512ULL * SZ_1G) 38 #define BTRFS_SB_LOG_SECOND_OFFSET (4096ULL * SZ_1G) 39 40 #define BTRFS_SB_LOG_FIRST_SHIFT const_ilog2(BTRFS_SB_LOG_FIRST_OFFSET) 41 #define BTRFS_SB_LOG_SECOND_SHIFT const_ilog2(BTRFS_SB_LOG_SECOND_OFFSET) 42 43 /* Number of superblock log zones */ 44 #define BTRFS_NR_SB_LOG_ZONES 2 45 46 /* Default number of max active zones when the device has no limits. */ 47 #define BTRFS_DEFAULT_MAX_ACTIVE_ZONES 128 48 49 /* 50 * Minimum of active zones we need: 51 * 52 * - BTRFS_SUPER_MIRROR_MAX zones for superblock mirrors 53 * - 3 zones to ensure at least one zone per SYSTEM, META and DATA block group 54 * - 1 zone for tree-log dedicated block group 55 * - 1 zone for relocation 56 */ 57 #define BTRFS_MIN_ACTIVE_ZONES (BTRFS_SUPER_MIRROR_MAX + 5) 58 59 /* 60 * Minimum / maximum supported zone size. Currently, SMR disks have a zone 61 * size of 256MiB, and we are expecting ZNS drives to be in the 1-4GiB range. 62 * We do not expect the zone size to become larger than 8GiB or smaller than 63 * 4MiB in the near future. 64 */ 65 #define BTRFS_MAX_ZONE_SIZE SZ_8G 66 #define BTRFS_MIN_ZONE_SIZE SZ_4M 67 68 #define SUPER_INFO_SECTORS ((u64)BTRFS_SUPER_INFO_SIZE >> SECTOR_SHIFT) 69 70 static void wait_eb_writebacks(struct btrfs_block_group *block_group); 71 static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written); 72 73 static inline bool sb_zone_is_full(const struct blk_zone *zone) 74 { 75 return (zone->cond == BLK_ZONE_COND_FULL) || 76 (zone->wp + SUPER_INFO_SECTORS > zone->start + zone->capacity); 77 } 78 79 static int copy_zone_info_cb(struct blk_zone *zone, unsigned int idx, void *data) 80 { 81 struct blk_zone *zones = data; 82 83 memcpy(&zones[idx], zone, sizeof(*zone)); 84 85 return 0; 86 } 87 88 static int sb_write_pointer(struct block_device *bdev, struct blk_zone *zones, 89 u64 *wp_ret) 90 { 91 bool empty[BTRFS_NR_SB_LOG_ZONES]; 92 bool full[BTRFS_NR_SB_LOG_ZONES]; 93 sector_t sector; 94 95 for (int i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { 96 ASSERT(zones[i].type != BLK_ZONE_TYPE_CONVENTIONAL); 97 empty[i] = (zones[i].cond == BLK_ZONE_COND_EMPTY); 98 full[i] = sb_zone_is_full(&zones[i]); 99 } 100 101 /* 102 * Possible states of log buffer zones 103 * 104 * Empty[0] In use[0] Full[0] 105 * Empty[1] * 0 1 106 * In use[1] x x 1 107 * Full[1] 0 0 C 108 * 109 * Log position: 110 * *: Special case, no superblock is written 111 * 0: Use write pointer of zones[0] 112 * 1: Use write pointer of zones[1] 113 * C: Compare super blocks from zones[0] and zones[1], use the latest 114 * one determined by generation 115 * x: Invalid state 116 */ 117 118 if (empty[0] && empty[1]) { 119 /* Special case to distinguish no superblock to read */ 120 *wp_ret = zones[0].start << SECTOR_SHIFT; 121 return -ENOENT; 122 } else if (full[0] && full[1]) { 123 /* Compare two super blocks */ 124 struct address_space *mapping = bdev->bd_mapping; 125 struct page *page[BTRFS_NR_SB_LOG_ZONES]; 126 struct btrfs_super_block *super[BTRFS_NR_SB_LOG_ZONES]; 127 128 for (int i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { 129 u64 zone_end = (zones[i].start + zones[i].capacity) << SECTOR_SHIFT; 130 u64 bytenr = ALIGN_DOWN(zone_end, BTRFS_SUPER_INFO_SIZE) - 131 BTRFS_SUPER_INFO_SIZE; 132 133 page[i] = read_cache_page_gfp(mapping, 134 bytenr >> PAGE_SHIFT, GFP_NOFS); 135 if (IS_ERR(page[i])) { 136 if (i == 1) 137 btrfs_release_disk_super(super[0]); 138 return PTR_ERR(page[i]); 139 } 140 super[i] = page_address(page[i]); 141 } 142 143 if (btrfs_super_generation(super[0]) > 144 btrfs_super_generation(super[1])) 145 sector = zones[1].start; 146 else 147 sector = zones[0].start; 148 149 for (int i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) 150 btrfs_release_disk_super(super[i]); 151 } else if (!full[0] && (empty[1] || full[1])) { 152 sector = zones[0].wp; 153 } else if (full[0]) { 154 sector = zones[1].wp; 155 } else { 156 return -EUCLEAN; 157 } 158 *wp_ret = sector << SECTOR_SHIFT; 159 return 0; 160 } 161 162 /* 163 * Get the first zone number of the superblock mirror 164 */ 165 static inline u32 sb_zone_number(int shift, int mirror) 166 { 167 u64 zone = U64_MAX; 168 169 ASSERT(mirror < BTRFS_SUPER_MIRROR_MAX); 170 switch (mirror) { 171 case 0: zone = 0; break; 172 case 1: zone = 1ULL << (BTRFS_SB_LOG_FIRST_SHIFT - shift); break; 173 case 2: zone = 1ULL << (BTRFS_SB_LOG_SECOND_SHIFT - shift); break; 174 } 175 176 ASSERT(zone <= U32_MAX); 177 178 return (u32)zone; 179 } 180 181 static inline sector_t zone_start_sector(u32 zone_number, 182 struct block_device *bdev) 183 { 184 return (sector_t)zone_number << ilog2(bdev_zone_sectors(bdev)); 185 } 186 187 static inline u64 zone_start_physical(u32 zone_number, 188 struct btrfs_zoned_device_info *zone_info) 189 { 190 return (u64)zone_number << zone_info->zone_size_shift; 191 } 192 193 /* 194 * Emulate blkdev_report_zones() for a non-zoned device. It slices up the block 195 * device into static sized chunks and fake a conventional zone on each of 196 * them. 197 */ 198 static int emulate_report_zones(struct btrfs_device *device, u64 pos, 199 struct blk_zone *zones, unsigned int nr_zones) 200 { 201 const sector_t zone_sectors = device->fs_info->zone_size >> SECTOR_SHIFT; 202 sector_t bdev_size = bdev_nr_sectors(device->bdev); 203 unsigned int i; 204 205 pos >>= SECTOR_SHIFT; 206 for (i = 0; i < nr_zones; i++) { 207 zones[i].start = i * zone_sectors + pos; 208 zones[i].len = zone_sectors; 209 zones[i].capacity = zone_sectors; 210 zones[i].wp = zones[i].start + zone_sectors; 211 zones[i].type = BLK_ZONE_TYPE_CONVENTIONAL; 212 zones[i].cond = BLK_ZONE_COND_NOT_WP; 213 214 if (zones[i].wp >= bdev_size) { 215 i++; 216 break; 217 } 218 } 219 220 return i; 221 } 222 223 static int btrfs_get_dev_zones(struct btrfs_device *device, u64 pos, 224 struct blk_zone *zones, unsigned int *nr_zones) 225 { 226 struct btrfs_zoned_device_info *zinfo = device->zone_info; 227 int ret; 228 229 if (!*nr_zones) 230 return 0; 231 232 if (!bdev_is_zoned(device->bdev)) { 233 ret = emulate_report_zones(device, pos, zones, *nr_zones); 234 *nr_zones = ret; 235 return 0; 236 } 237 238 /* Check cache */ 239 if (zinfo->zone_cache) { 240 unsigned int i; 241 u32 zno; 242 243 ASSERT(IS_ALIGNED(pos, zinfo->zone_size)); 244 zno = pos >> zinfo->zone_size_shift; 245 /* 246 * We cannot report zones beyond the zone end. So, it is OK to 247 * cap *nr_zones to at the end. 248 */ 249 *nr_zones = min_t(u32, *nr_zones, zinfo->nr_zones - zno); 250 251 for (i = 0; i < *nr_zones; i++) { 252 struct blk_zone *zone_info; 253 254 zone_info = &zinfo->zone_cache[zno + i]; 255 if (!zone_info->len) 256 break; 257 } 258 259 if (i == *nr_zones) { 260 /* Cache hit on all the zones */ 261 memcpy(zones, zinfo->zone_cache + zno, 262 sizeof(*zinfo->zone_cache) * *nr_zones); 263 return 0; 264 } 265 } 266 267 ret = blkdev_report_zones(device->bdev, pos >> SECTOR_SHIFT, *nr_zones, 268 copy_zone_info_cb, zones); 269 if (ret < 0) { 270 btrfs_err(device->fs_info, 271 "zoned: failed to read zone %llu on %s (devid %llu)", 272 pos, rcu_dereference(device->name), 273 device->devid); 274 return ret; 275 } 276 *nr_zones = ret; 277 if (!ret) 278 return -EIO; 279 280 /* Populate cache */ 281 if (zinfo->zone_cache) { 282 u32 zno = pos >> zinfo->zone_size_shift; 283 284 memcpy(zinfo->zone_cache + zno, zones, 285 sizeof(*zinfo->zone_cache) * *nr_zones); 286 } 287 288 return 0; 289 } 290 291 /* The emulated zone size is determined from the size of device extent */ 292 static int calculate_emulated_zone_size(struct btrfs_fs_info *fs_info) 293 { 294 BTRFS_PATH_AUTO_FREE(path); 295 struct btrfs_root *root = fs_info->dev_root; 296 struct btrfs_key key; 297 struct extent_buffer *leaf; 298 struct btrfs_dev_extent *dext; 299 int ret = 0; 300 301 key.objectid = 1; 302 key.type = BTRFS_DEV_EXTENT_KEY; 303 key.offset = 0; 304 305 path = btrfs_alloc_path(); 306 if (!path) 307 return -ENOMEM; 308 309 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 310 if (ret < 0) 311 return ret; 312 313 if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) { 314 ret = btrfs_next_leaf(root, path); 315 if (ret < 0) 316 return ret; 317 /* No dev extents at all? Not good */ 318 if (ret > 0) 319 return -EUCLEAN; 320 } 321 322 leaf = path->nodes[0]; 323 dext = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_extent); 324 fs_info->zone_size = btrfs_dev_extent_length(leaf, dext); 325 return 0; 326 } 327 328 int btrfs_get_dev_zone_info_all_devices(struct btrfs_fs_info *fs_info) 329 { 330 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 331 struct btrfs_device *device; 332 int ret = 0; 333 334 /* fs_info->zone_size might not set yet. Use the incomapt flag here. */ 335 if (!btrfs_fs_incompat(fs_info, ZONED)) 336 return 0; 337 338 mutex_lock(&fs_devices->device_list_mutex); 339 list_for_each_entry(device, &fs_devices->devices, dev_list) { 340 /* We can skip reading of zone info for missing devices */ 341 if (!device->bdev) 342 continue; 343 344 ret = btrfs_get_dev_zone_info(device, true); 345 if (ret) 346 break; 347 } 348 mutex_unlock(&fs_devices->device_list_mutex); 349 350 return ret; 351 } 352 353 int btrfs_get_dev_zone_info(struct btrfs_device *device, bool populate_cache) 354 { 355 struct btrfs_fs_info *fs_info = device->fs_info; 356 struct btrfs_zoned_device_info *zone_info = NULL; 357 struct block_device *bdev = device->bdev; 358 unsigned int max_active_zones; 359 unsigned int nactive; 360 sector_t nr_sectors; 361 sector_t sector = 0; 362 struct blk_zone *zones = NULL; 363 unsigned int i, nreported = 0, nr_zones; 364 sector_t zone_sectors; 365 char *model, *emulated; 366 int ret; 367 368 /* 369 * Cannot use btrfs_is_zoned here, since fs_info::zone_size might not 370 * yet be set. 371 */ 372 if (!btrfs_fs_incompat(fs_info, ZONED)) 373 return 0; 374 375 if (device->zone_info) 376 return 0; 377 378 zone_info = kzalloc(sizeof(*zone_info), GFP_KERNEL); 379 if (!zone_info) 380 return -ENOMEM; 381 382 device->zone_info = zone_info; 383 384 if (!bdev_is_zoned(bdev)) { 385 if (!fs_info->zone_size) { 386 ret = calculate_emulated_zone_size(fs_info); 387 if (ret) 388 goto out; 389 } 390 391 ASSERT(fs_info->zone_size); 392 zone_sectors = fs_info->zone_size >> SECTOR_SHIFT; 393 } else { 394 zone_sectors = bdev_zone_sectors(bdev); 395 } 396 397 ASSERT(is_power_of_two_u64(zone_sectors)); 398 zone_info->zone_size = zone_sectors << SECTOR_SHIFT; 399 400 /* We reject devices with a zone size larger than 8GB */ 401 if (zone_info->zone_size > BTRFS_MAX_ZONE_SIZE) { 402 btrfs_err(fs_info, 403 "zoned: %s: zone size %llu larger than supported maximum %llu", 404 rcu_dereference(device->name), 405 zone_info->zone_size, BTRFS_MAX_ZONE_SIZE); 406 ret = -EINVAL; 407 goto out; 408 } else if (zone_info->zone_size < BTRFS_MIN_ZONE_SIZE) { 409 btrfs_err(fs_info, 410 "zoned: %s: zone size %llu smaller than supported minimum %u", 411 rcu_dereference(device->name), 412 zone_info->zone_size, BTRFS_MIN_ZONE_SIZE); 413 ret = -EINVAL; 414 goto out; 415 } 416 417 nr_sectors = bdev_nr_sectors(bdev); 418 zone_info->zone_size_shift = ilog2(zone_info->zone_size); 419 zone_info->nr_zones = nr_sectors >> ilog2(zone_sectors); 420 if (!IS_ALIGNED(nr_sectors, zone_sectors)) 421 zone_info->nr_zones++; 422 423 max_active_zones = min_not_zero(bdev_max_active_zones(bdev), 424 bdev_max_open_zones(bdev)); 425 if (!max_active_zones && zone_info->nr_zones > BTRFS_DEFAULT_MAX_ACTIVE_ZONES) 426 max_active_zones = BTRFS_DEFAULT_MAX_ACTIVE_ZONES; 427 if (max_active_zones && max_active_zones < BTRFS_MIN_ACTIVE_ZONES) { 428 btrfs_err(fs_info, 429 "zoned: %s: max active zones %u is too small, need at least %u active zones", 430 rcu_dereference(device->name), max_active_zones, 431 BTRFS_MIN_ACTIVE_ZONES); 432 ret = -EINVAL; 433 goto out; 434 } 435 zone_info->max_active_zones = max_active_zones; 436 437 zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 438 if (!zone_info->seq_zones) { 439 ret = -ENOMEM; 440 goto out; 441 } 442 443 zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 444 if (!zone_info->empty_zones) { 445 ret = -ENOMEM; 446 goto out; 447 } 448 449 zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 450 if (!zone_info->active_zones) { 451 ret = -ENOMEM; 452 goto out; 453 } 454 455 zones = kvcalloc(BTRFS_REPORT_NR_ZONES, sizeof(struct blk_zone), GFP_KERNEL); 456 if (!zones) { 457 ret = -ENOMEM; 458 goto out; 459 } 460 461 /* 462 * Enable zone cache only for a zoned device. On a non-zoned device, we 463 * fill the zone info with emulated CONVENTIONAL zones, so no need to 464 * use the cache. 465 */ 466 if (populate_cache && bdev_is_zoned(device->bdev)) { 467 zone_info->zone_cache = vcalloc(zone_info->nr_zones, 468 sizeof(struct blk_zone)); 469 if (!zone_info->zone_cache) { 470 btrfs_err(device->fs_info, 471 "zoned: failed to allocate zone cache for %s", 472 rcu_dereference(device->name)); 473 ret = -ENOMEM; 474 goto out; 475 } 476 } 477 478 /* Get zones type */ 479 nactive = 0; 480 while (sector < nr_sectors) { 481 nr_zones = BTRFS_REPORT_NR_ZONES; 482 ret = btrfs_get_dev_zones(device, sector << SECTOR_SHIFT, zones, 483 &nr_zones); 484 if (ret) 485 goto out; 486 487 for (i = 0; i < nr_zones; i++) { 488 if (zones[i].type == BLK_ZONE_TYPE_SEQWRITE_REQ) 489 __set_bit(nreported, zone_info->seq_zones); 490 switch (zones[i].cond) { 491 case BLK_ZONE_COND_EMPTY: 492 __set_bit(nreported, zone_info->empty_zones); 493 break; 494 case BLK_ZONE_COND_IMP_OPEN: 495 case BLK_ZONE_COND_EXP_OPEN: 496 case BLK_ZONE_COND_CLOSED: 497 __set_bit(nreported, zone_info->active_zones); 498 nactive++; 499 break; 500 } 501 nreported++; 502 } 503 sector = zones[nr_zones - 1].start + zones[nr_zones - 1].len; 504 } 505 506 if (nreported != zone_info->nr_zones) { 507 btrfs_err(device->fs_info, 508 "inconsistent number of zones on %s (%u/%u)", 509 rcu_dereference(device->name), nreported, 510 zone_info->nr_zones); 511 ret = -EIO; 512 goto out; 513 } 514 515 if (max_active_zones) { 516 if (nactive > max_active_zones) { 517 btrfs_err(device->fs_info, 518 "zoned: %u active zones on %s exceeds max_active_zones %u", 519 nactive, rcu_dereference(device->name), 520 max_active_zones); 521 ret = -EIO; 522 goto out; 523 } 524 atomic_set(&zone_info->active_zones_left, 525 max_active_zones - nactive); 526 set_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags); 527 } 528 529 /* Validate superblock log */ 530 nr_zones = BTRFS_NR_SB_LOG_ZONES; 531 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 532 u32 sb_zone; 533 u64 sb_wp; 534 int sb_pos = BTRFS_NR_SB_LOG_ZONES * i; 535 536 sb_zone = sb_zone_number(zone_info->zone_size_shift, i); 537 if (sb_zone + 1 >= zone_info->nr_zones) 538 continue; 539 540 ret = btrfs_get_dev_zones(device, 541 zone_start_physical(sb_zone, zone_info), 542 &zone_info->sb_zones[sb_pos], 543 &nr_zones); 544 if (ret) 545 goto out; 546 547 if (nr_zones != BTRFS_NR_SB_LOG_ZONES) { 548 btrfs_err(device->fs_info, 549 "zoned: failed to read super block log zone info at devid %llu zone %u", 550 device->devid, sb_zone); 551 ret = -EUCLEAN; 552 goto out; 553 } 554 555 /* 556 * If zones[0] is conventional, always use the beginning of the 557 * zone to record superblock. No need to validate in that case. 558 */ 559 if (zone_info->sb_zones[BTRFS_NR_SB_LOG_ZONES * i].type == 560 BLK_ZONE_TYPE_CONVENTIONAL) 561 continue; 562 563 ret = sb_write_pointer(device->bdev, 564 &zone_info->sb_zones[sb_pos], &sb_wp); 565 if (ret != -ENOENT && ret) { 566 btrfs_err(device->fs_info, 567 "zoned: super block log zone corrupted devid %llu zone %u", 568 device->devid, sb_zone); 569 ret = -EUCLEAN; 570 goto out; 571 } 572 } 573 574 575 kvfree(zones); 576 577 if (bdev_is_zoned(bdev)) { 578 model = "host-managed zoned"; 579 emulated = ""; 580 } else { 581 model = "regular"; 582 emulated = "emulated "; 583 } 584 585 btrfs_info(fs_info, 586 "%s block device %s, %u %szones of %llu bytes", 587 model, rcu_dereference(device->name), zone_info->nr_zones, 588 emulated, zone_info->zone_size); 589 590 return 0; 591 592 out: 593 kvfree(zones); 594 btrfs_destroy_dev_zone_info(device); 595 return ret; 596 } 597 598 void btrfs_destroy_dev_zone_info(struct btrfs_device *device) 599 { 600 struct btrfs_zoned_device_info *zone_info = device->zone_info; 601 602 if (!zone_info) 603 return; 604 605 bitmap_free(zone_info->active_zones); 606 bitmap_free(zone_info->seq_zones); 607 bitmap_free(zone_info->empty_zones); 608 vfree(zone_info->zone_cache); 609 kfree(zone_info); 610 device->zone_info = NULL; 611 } 612 613 struct btrfs_zoned_device_info *btrfs_clone_dev_zone_info(struct btrfs_device *orig_dev) 614 { 615 struct btrfs_zoned_device_info *zone_info; 616 617 zone_info = kmemdup(orig_dev->zone_info, sizeof(*zone_info), GFP_KERNEL); 618 if (!zone_info) 619 return NULL; 620 621 zone_info->seq_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 622 if (!zone_info->seq_zones) 623 goto out; 624 625 bitmap_copy(zone_info->seq_zones, orig_dev->zone_info->seq_zones, 626 zone_info->nr_zones); 627 628 zone_info->empty_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 629 if (!zone_info->empty_zones) 630 goto out; 631 632 bitmap_copy(zone_info->empty_zones, orig_dev->zone_info->empty_zones, 633 zone_info->nr_zones); 634 635 zone_info->active_zones = bitmap_zalloc(zone_info->nr_zones, GFP_KERNEL); 636 if (!zone_info->active_zones) 637 goto out; 638 639 bitmap_copy(zone_info->active_zones, orig_dev->zone_info->active_zones, 640 zone_info->nr_zones); 641 zone_info->zone_cache = NULL; 642 643 return zone_info; 644 645 out: 646 bitmap_free(zone_info->seq_zones); 647 bitmap_free(zone_info->empty_zones); 648 bitmap_free(zone_info->active_zones); 649 kfree(zone_info); 650 return NULL; 651 } 652 653 static int btrfs_get_dev_zone(struct btrfs_device *device, u64 pos, struct blk_zone *zone) 654 { 655 unsigned int nr_zones = 1; 656 int ret; 657 658 ret = btrfs_get_dev_zones(device, pos, zone, &nr_zones); 659 if (ret != 0 || !nr_zones) 660 return ret ? ret : -EIO; 661 662 return 0; 663 } 664 665 static int btrfs_check_for_zoned_device(struct btrfs_fs_info *fs_info) 666 { 667 struct btrfs_device *device; 668 669 list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { 670 if (device->bdev && bdev_is_zoned(device->bdev)) { 671 btrfs_err(fs_info, 672 "zoned: mode not enabled but zoned device found: %pg", 673 device->bdev); 674 return -EINVAL; 675 } 676 } 677 678 return 0; 679 } 680 681 int btrfs_check_zoned_mode(struct btrfs_fs_info *fs_info) 682 { 683 struct queue_limits *lim = &fs_info->limits; 684 struct btrfs_device *device; 685 u64 zone_size = 0; 686 int ret; 687 688 /* 689 * Host-Managed devices can't be used without the ZONED flag. With the 690 * ZONED all devices can be used, using zone emulation if required. 691 */ 692 if (!btrfs_fs_incompat(fs_info, ZONED)) 693 return btrfs_check_for_zoned_device(fs_info); 694 695 blk_set_stacking_limits(lim); 696 697 list_for_each_entry(device, &fs_info->fs_devices->devices, dev_list) { 698 struct btrfs_zoned_device_info *zone_info = device->zone_info; 699 700 if (!device->bdev) 701 continue; 702 703 if (!zone_size) { 704 zone_size = zone_info->zone_size; 705 } else if (zone_info->zone_size != zone_size) { 706 btrfs_err(fs_info, 707 "zoned: unequal block device zone sizes: have %llu found %llu", 708 zone_info->zone_size, zone_size); 709 return -EINVAL; 710 } 711 712 /* 713 * With the zoned emulation, we can have non-zoned device on the 714 * zoned mode. In this case, we don't have a valid max zone 715 * append size. 716 */ 717 if (bdev_is_zoned(device->bdev)) 718 blk_stack_limits(lim, bdev_limits(device->bdev), 0); 719 } 720 721 ret = blk_validate_limits(lim); 722 if (ret) { 723 btrfs_err(fs_info, "zoned: failed to validate queue limits"); 724 return ret; 725 } 726 727 /* 728 * stripe_size is always aligned to BTRFS_STRIPE_LEN in 729 * btrfs_create_chunk(). Since we want stripe_len == zone_size, 730 * check the alignment here. 731 */ 732 if (!IS_ALIGNED(zone_size, BTRFS_STRIPE_LEN)) { 733 btrfs_err(fs_info, 734 "zoned: zone size %llu not aligned to stripe %u", 735 zone_size, BTRFS_STRIPE_LEN); 736 return -EINVAL; 737 } 738 739 if (btrfs_fs_incompat(fs_info, MIXED_GROUPS)) { 740 btrfs_err(fs_info, "zoned: mixed block groups not supported"); 741 return -EINVAL; 742 } 743 744 fs_info->zone_size = zone_size; 745 /* 746 * Also limit max_zone_append_size by max_segments * PAGE_SIZE. 747 * Technically, we can have multiple pages per segment. But, since 748 * we add the pages one by one to a bio, and cannot increase the 749 * metadata reservation even if it increases the number of extents, it 750 * is safe to stick with the limit. 751 */ 752 fs_info->max_zone_append_size = ALIGN_DOWN( 753 min3((u64)lim->max_zone_append_sectors << SECTOR_SHIFT, 754 (u64)lim->max_sectors << SECTOR_SHIFT, 755 (u64)lim->max_segments << PAGE_SHIFT), 756 fs_info->sectorsize); 757 fs_info->fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_ZONED; 758 759 fs_info->max_extent_size = min_not_zero(fs_info->max_extent_size, 760 fs_info->max_zone_append_size); 761 762 /* 763 * Check mount options here, because we might change fs_info->zoned 764 * from fs_info->zone_size. 765 */ 766 ret = btrfs_check_mountopts_zoned(fs_info, &fs_info->mount_opt); 767 if (ret) 768 return ret; 769 770 btrfs_info(fs_info, "zoned mode enabled with zone size %llu", zone_size); 771 return 0; 772 } 773 774 int btrfs_check_mountopts_zoned(const struct btrfs_fs_info *info, 775 unsigned long long *mount_opt) 776 { 777 if (!btrfs_is_zoned(info)) 778 return 0; 779 780 /* 781 * Space cache writing is not COWed. Disable that to avoid write errors 782 * in sequential zones. 783 */ 784 if (btrfs_raw_test_opt(*mount_opt, SPACE_CACHE)) { 785 btrfs_err(info, "zoned: space cache v1 is not supported"); 786 return -EINVAL; 787 } 788 789 if (btrfs_raw_test_opt(*mount_opt, NODATACOW)) { 790 btrfs_err(info, "zoned: NODATACOW not supported"); 791 return -EINVAL; 792 } 793 794 if (btrfs_raw_test_opt(*mount_opt, DISCARD_ASYNC)) { 795 btrfs_info(info, 796 "zoned: async discard ignored and disabled for zoned mode"); 797 btrfs_clear_opt(*mount_opt, DISCARD_ASYNC); 798 } 799 800 return 0; 801 } 802 803 static int sb_log_location(struct block_device *bdev, struct blk_zone *zones, 804 int rw, u64 *bytenr_ret) 805 { 806 u64 wp; 807 int ret; 808 809 if (zones[0].type == BLK_ZONE_TYPE_CONVENTIONAL) { 810 *bytenr_ret = zones[0].start << SECTOR_SHIFT; 811 return 0; 812 } 813 814 ret = sb_write_pointer(bdev, zones, &wp); 815 if (ret != -ENOENT && ret < 0) 816 return ret; 817 818 if (rw == WRITE) { 819 struct blk_zone *reset = NULL; 820 821 if (wp == zones[0].start << SECTOR_SHIFT) 822 reset = &zones[0]; 823 else if (wp == zones[1].start << SECTOR_SHIFT) 824 reset = &zones[1]; 825 826 if (reset && reset->cond != BLK_ZONE_COND_EMPTY) { 827 unsigned int nofs_flags; 828 829 ASSERT(sb_zone_is_full(reset)); 830 831 nofs_flags = memalloc_nofs_save(); 832 ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 833 reset->start, reset->len); 834 memalloc_nofs_restore(nofs_flags); 835 if (ret) 836 return ret; 837 838 reset->cond = BLK_ZONE_COND_EMPTY; 839 reset->wp = reset->start; 840 } 841 } else if (ret != -ENOENT) { 842 /* 843 * For READ, we want the previous one. Move write pointer to 844 * the end of a zone, if it is at the head of a zone. 845 */ 846 u64 zone_end = 0; 847 848 if (wp == zones[0].start << SECTOR_SHIFT) 849 zone_end = zones[1].start + zones[1].capacity; 850 else if (wp == zones[1].start << SECTOR_SHIFT) 851 zone_end = zones[0].start + zones[0].capacity; 852 if (zone_end) 853 wp = ALIGN_DOWN(zone_end << SECTOR_SHIFT, 854 BTRFS_SUPER_INFO_SIZE); 855 856 wp -= BTRFS_SUPER_INFO_SIZE; 857 } 858 859 *bytenr_ret = wp; 860 return 0; 861 862 } 863 864 int btrfs_sb_log_location_bdev(struct block_device *bdev, int mirror, int rw, 865 u64 *bytenr_ret) 866 { 867 struct blk_zone zones[BTRFS_NR_SB_LOG_ZONES]; 868 sector_t zone_sectors; 869 u32 sb_zone; 870 int ret; 871 u8 zone_sectors_shift; 872 sector_t nr_sectors; 873 u32 nr_zones; 874 875 if (!bdev_is_zoned(bdev)) { 876 *bytenr_ret = btrfs_sb_offset(mirror); 877 return 0; 878 } 879 880 ASSERT(rw == READ || rw == WRITE); 881 882 zone_sectors = bdev_zone_sectors(bdev); 883 if (!is_power_of_2(zone_sectors)) 884 return -EINVAL; 885 zone_sectors_shift = ilog2(zone_sectors); 886 nr_sectors = bdev_nr_sectors(bdev); 887 nr_zones = nr_sectors >> zone_sectors_shift; 888 889 sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror); 890 if (sb_zone + 1 >= nr_zones) 891 return -ENOENT; 892 893 ret = blkdev_report_zones(bdev, zone_start_sector(sb_zone, bdev), 894 BTRFS_NR_SB_LOG_ZONES, copy_zone_info_cb, 895 zones); 896 if (ret < 0) 897 return ret; 898 if (ret != BTRFS_NR_SB_LOG_ZONES) 899 return -EIO; 900 901 return sb_log_location(bdev, zones, rw, bytenr_ret); 902 } 903 904 int btrfs_sb_log_location(struct btrfs_device *device, int mirror, int rw, 905 u64 *bytenr_ret) 906 { 907 struct btrfs_zoned_device_info *zinfo = device->zone_info; 908 u32 zone_num; 909 910 /* 911 * For a zoned filesystem on a non-zoned block device, use the same 912 * super block locations as regular filesystem. Doing so, the super 913 * block can always be retrieved and the zoned flag of the volume 914 * detected from the super block information. 915 */ 916 if (!bdev_is_zoned(device->bdev)) { 917 *bytenr_ret = btrfs_sb_offset(mirror); 918 return 0; 919 } 920 921 zone_num = sb_zone_number(zinfo->zone_size_shift, mirror); 922 if (zone_num + 1 >= zinfo->nr_zones) 923 return -ENOENT; 924 925 return sb_log_location(device->bdev, 926 &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror], 927 rw, bytenr_ret); 928 } 929 930 static inline bool is_sb_log_zone(struct btrfs_zoned_device_info *zinfo, 931 int mirror) 932 { 933 u32 zone_num; 934 935 if (!zinfo) 936 return false; 937 938 zone_num = sb_zone_number(zinfo->zone_size_shift, mirror); 939 if (zone_num + 1 >= zinfo->nr_zones) 940 return false; 941 942 if (!test_bit(zone_num, zinfo->seq_zones)) 943 return false; 944 945 return true; 946 } 947 948 int btrfs_advance_sb_log(struct btrfs_device *device, int mirror) 949 { 950 struct btrfs_zoned_device_info *zinfo = device->zone_info; 951 struct blk_zone *zone; 952 int i; 953 954 if (!is_sb_log_zone(zinfo, mirror)) 955 return 0; 956 957 zone = &zinfo->sb_zones[BTRFS_NR_SB_LOG_ZONES * mirror]; 958 for (i = 0; i < BTRFS_NR_SB_LOG_ZONES; i++) { 959 /* Advance the next zone */ 960 if (zone->cond == BLK_ZONE_COND_FULL) { 961 zone++; 962 continue; 963 } 964 965 if (zone->cond == BLK_ZONE_COND_EMPTY) 966 zone->cond = BLK_ZONE_COND_IMP_OPEN; 967 968 zone->wp += SUPER_INFO_SECTORS; 969 970 if (sb_zone_is_full(zone)) { 971 /* 972 * No room left to write new superblock. Since 973 * superblock is written with REQ_SYNC, it is safe to 974 * finish the zone now. 975 * 976 * If the write pointer is exactly at the capacity, 977 * explicit ZONE_FINISH is not necessary. 978 */ 979 if (zone->wp != zone->start + zone->capacity) { 980 unsigned int nofs_flags; 981 int ret; 982 983 nofs_flags = memalloc_nofs_save(); 984 ret = blkdev_zone_mgmt(device->bdev, 985 REQ_OP_ZONE_FINISH, zone->start, 986 zone->len); 987 memalloc_nofs_restore(nofs_flags); 988 if (ret) 989 return ret; 990 } 991 992 zone->wp = zone->start + zone->len; 993 zone->cond = BLK_ZONE_COND_FULL; 994 } 995 return 0; 996 } 997 998 /* All the zones are FULL. Should not reach here. */ 999 DEBUG_WARN("unexpected state, all zones full"); 1000 return -EIO; 1001 } 1002 1003 int btrfs_reset_sb_log_zones(struct block_device *bdev, int mirror) 1004 { 1005 unsigned int nofs_flags; 1006 sector_t zone_sectors; 1007 sector_t nr_sectors; 1008 u8 zone_sectors_shift; 1009 u32 sb_zone; 1010 u32 nr_zones; 1011 int ret; 1012 1013 zone_sectors = bdev_zone_sectors(bdev); 1014 zone_sectors_shift = ilog2(zone_sectors); 1015 nr_sectors = bdev_nr_sectors(bdev); 1016 nr_zones = nr_sectors >> zone_sectors_shift; 1017 1018 sb_zone = sb_zone_number(zone_sectors_shift + SECTOR_SHIFT, mirror); 1019 if (sb_zone + 1 >= nr_zones) 1020 return -ENOENT; 1021 1022 nofs_flags = memalloc_nofs_save(); 1023 ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET, 1024 zone_start_sector(sb_zone, bdev), 1025 zone_sectors * BTRFS_NR_SB_LOG_ZONES); 1026 memalloc_nofs_restore(nofs_flags); 1027 return ret; 1028 } 1029 1030 /* 1031 * Find allocatable zones within a given region. 1032 * 1033 * @device: the device to allocate a region on 1034 * @hole_start: the position of the hole to allocate the region 1035 * @num_bytes: size of wanted region 1036 * @hole_end: the end of the hole 1037 * @return: position of allocatable zones 1038 * 1039 * Allocatable region should not contain any superblock locations. 1040 */ 1041 u64 btrfs_find_allocatable_zones(struct btrfs_device *device, u64 hole_start, 1042 u64 hole_end, u64 num_bytes) 1043 { 1044 struct btrfs_zoned_device_info *zinfo = device->zone_info; 1045 const u8 shift = zinfo->zone_size_shift; 1046 u64 nzones = num_bytes >> shift; 1047 u64 pos = hole_start; 1048 u64 begin, end; 1049 bool have_sb; 1050 int i; 1051 1052 ASSERT(IS_ALIGNED(hole_start, zinfo->zone_size)); 1053 ASSERT(IS_ALIGNED(num_bytes, zinfo->zone_size)); 1054 1055 while (pos < hole_end) { 1056 begin = pos >> shift; 1057 end = begin + nzones; 1058 1059 if (end > zinfo->nr_zones) 1060 return hole_end; 1061 1062 /* Check if zones in the region are all empty */ 1063 if (btrfs_dev_is_sequential(device, pos) && 1064 !bitmap_test_range_all_set(zinfo->empty_zones, begin, nzones)) { 1065 pos += zinfo->zone_size; 1066 continue; 1067 } 1068 1069 have_sb = false; 1070 for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) { 1071 u32 sb_zone; 1072 u64 sb_pos; 1073 1074 sb_zone = sb_zone_number(shift, i); 1075 if (!(end <= sb_zone || 1076 sb_zone + BTRFS_NR_SB_LOG_ZONES <= begin)) { 1077 have_sb = true; 1078 pos = zone_start_physical( 1079 sb_zone + BTRFS_NR_SB_LOG_ZONES, zinfo); 1080 break; 1081 } 1082 1083 /* We also need to exclude regular superblock positions */ 1084 sb_pos = btrfs_sb_offset(i); 1085 if (!(pos + num_bytes <= sb_pos || 1086 sb_pos + BTRFS_SUPER_INFO_SIZE <= pos)) { 1087 have_sb = true; 1088 pos = ALIGN(sb_pos + BTRFS_SUPER_INFO_SIZE, 1089 zinfo->zone_size); 1090 break; 1091 } 1092 } 1093 if (!have_sb) 1094 break; 1095 } 1096 1097 return pos; 1098 } 1099 1100 static bool btrfs_dev_set_active_zone(struct btrfs_device *device, u64 pos) 1101 { 1102 struct btrfs_zoned_device_info *zone_info = device->zone_info; 1103 unsigned int zno = (pos >> zone_info->zone_size_shift); 1104 1105 /* We can use any number of zones */ 1106 if (zone_info->max_active_zones == 0) 1107 return true; 1108 1109 if (!test_bit(zno, zone_info->active_zones)) { 1110 /* Active zone left? */ 1111 if (atomic_dec_if_positive(&zone_info->active_zones_left) < 0) 1112 return false; 1113 if (test_and_set_bit(zno, zone_info->active_zones)) { 1114 /* Someone already set the bit */ 1115 atomic_inc(&zone_info->active_zones_left); 1116 } 1117 } 1118 1119 return true; 1120 } 1121 1122 static void btrfs_dev_clear_active_zone(struct btrfs_device *device, u64 pos) 1123 { 1124 struct btrfs_zoned_device_info *zone_info = device->zone_info; 1125 unsigned int zno = (pos >> zone_info->zone_size_shift); 1126 1127 /* We can use any number of zones */ 1128 if (zone_info->max_active_zones == 0) 1129 return; 1130 1131 if (test_and_clear_bit(zno, zone_info->active_zones)) 1132 atomic_inc(&zone_info->active_zones_left); 1133 } 1134 1135 int btrfs_reset_device_zone(struct btrfs_device *device, u64 physical, 1136 u64 length, u64 *bytes) 1137 { 1138 unsigned int nofs_flags; 1139 int ret; 1140 1141 *bytes = 0; 1142 nofs_flags = memalloc_nofs_save(); 1143 ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_RESET, 1144 physical >> SECTOR_SHIFT, length >> SECTOR_SHIFT); 1145 memalloc_nofs_restore(nofs_flags); 1146 if (ret) 1147 return ret; 1148 1149 *bytes = length; 1150 while (length) { 1151 btrfs_dev_set_zone_empty(device, physical); 1152 btrfs_dev_clear_active_zone(device, physical); 1153 physical += device->zone_info->zone_size; 1154 length -= device->zone_info->zone_size; 1155 } 1156 1157 return 0; 1158 } 1159 1160 int btrfs_ensure_empty_zones(struct btrfs_device *device, u64 start, u64 size) 1161 { 1162 struct btrfs_zoned_device_info *zinfo = device->zone_info; 1163 const u8 shift = zinfo->zone_size_shift; 1164 unsigned long begin = start >> shift; 1165 unsigned long nbits = size >> shift; 1166 u64 pos; 1167 int ret; 1168 1169 ASSERT(IS_ALIGNED(start, zinfo->zone_size)); 1170 ASSERT(IS_ALIGNED(size, zinfo->zone_size)); 1171 1172 if (begin + nbits > zinfo->nr_zones) 1173 return -ERANGE; 1174 1175 /* All the zones are conventional */ 1176 if (bitmap_test_range_all_zero(zinfo->seq_zones, begin, nbits)) 1177 return 0; 1178 1179 /* All the zones are sequential and empty */ 1180 if (bitmap_test_range_all_set(zinfo->seq_zones, begin, nbits) && 1181 bitmap_test_range_all_set(zinfo->empty_zones, begin, nbits)) 1182 return 0; 1183 1184 for (pos = start; pos < start + size; pos += zinfo->zone_size) { 1185 u64 reset_bytes; 1186 1187 if (!btrfs_dev_is_sequential(device, pos) || 1188 btrfs_dev_is_empty_zone(device, pos)) 1189 continue; 1190 1191 /* Free regions should be empty */ 1192 btrfs_warn( 1193 device->fs_info, 1194 "zoned: resetting device %s (devid %llu) zone %llu for allocation", 1195 rcu_dereference(device->name), device->devid, pos >> shift); 1196 WARN_ON_ONCE(1); 1197 1198 ret = btrfs_reset_device_zone(device, pos, zinfo->zone_size, 1199 &reset_bytes); 1200 if (ret) 1201 return ret; 1202 } 1203 1204 return 0; 1205 } 1206 1207 /* 1208 * Calculate an allocation pointer from the extent allocation information 1209 * for a block group consist of conventional zones. It is pointed to the 1210 * end of the highest addressed extent in the block group as an allocation 1211 * offset. 1212 */ 1213 static int calculate_alloc_pointer(struct btrfs_block_group *cache, 1214 u64 *offset_ret, bool new) 1215 { 1216 struct btrfs_fs_info *fs_info = cache->fs_info; 1217 struct btrfs_root *root; 1218 BTRFS_PATH_AUTO_FREE(path); 1219 struct btrfs_key key; 1220 struct btrfs_key found_key; 1221 int ret; 1222 u64 length; 1223 1224 /* 1225 * Avoid tree lookups for a new block group, there's no use for it. 1226 * It must always be 0. 1227 * 1228 * Also, we have a lock chain of extent buffer lock -> chunk mutex. 1229 * For new a block group, this function is called from 1230 * btrfs_make_block_group() which is already taking the chunk mutex. 1231 * Thus, we cannot call calculate_alloc_pointer() which takes extent 1232 * buffer locks to avoid deadlock. 1233 */ 1234 if (new) { 1235 *offset_ret = 0; 1236 return 0; 1237 } 1238 1239 path = btrfs_alloc_path(); 1240 if (!path) 1241 return -ENOMEM; 1242 1243 key.objectid = cache->start + cache->length; 1244 key.type = 0; 1245 key.offset = 0; 1246 1247 root = btrfs_extent_root(fs_info, key.objectid); 1248 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0); 1249 /* We should not find the exact match */ 1250 if (!ret) 1251 ret = -EUCLEAN; 1252 if (ret < 0) 1253 return ret; 1254 1255 ret = btrfs_previous_extent_item(root, path, cache->start); 1256 if (ret) { 1257 if (ret == 1) { 1258 ret = 0; 1259 *offset_ret = 0; 1260 } 1261 return ret; 1262 } 1263 1264 btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]); 1265 1266 if (found_key.type == BTRFS_EXTENT_ITEM_KEY) 1267 length = found_key.offset; 1268 else 1269 length = fs_info->nodesize; 1270 1271 if (!(found_key.objectid >= cache->start && 1272 found_key.objectid + length <= cache->start + cache->length)) { 1273 return -EUCLEAN; 1274 } 1275 *offset_ret = found_key.objectid + length - cache->start; 1276 return 0; 1277 } 1278 1279 struct zone_info { 1280 u64 physical; 1281 u64 capacity; 1282 u64 alloc_offset; 1283 }; 1284 1285 static int btrfs_load_zone_info(struct btrfs_fs_info *fs_info, int zone_idx, 1286 struct zone_info *info, unsigned long *active, 1287 struct btrfs_chunk_map *map, bool new) 1288 { 1289 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 1290 struct btrfs_device *device; 1291 int dev_replace_is_ongoing = 0; 1292 unsigned int nofs_flag; 1293 struct blk_zone zone; 1294 int ret; 1295 1296 info->physical = map->stripes[zone_idx].physical; 1297 1298 down_read(&dev_replace->rwsem); 1299 device = map->stripes[zone_idx].dev; 1300 1301 if (!device->bdev) { 1302 up_read(&dev_replace->rwsem); 1303 info->alloc_offset = WP_MISSING_DEV; 1304 return 0; 1305 } 1306 1307 /* Consider a zone as active if we can allow any number of active zones. */ 1308 if (!device->zone_info->max_active_zones) 1309 __set_bit(zone_idx, active); 1310 1311 if (!btrfs_dev_is_sequential(device, info->physical)) { 1312 up_read(&dev_replace->rwsem); 1313 info->alloc_offset = WP_CONVENTIONAL; 1314 return 0; 1315 } 1316 1317 ASSERT(!new || btrfs_dev_is_empty_zone(device, info->physical)); 1318 1319 /* This zone will be used for allocation, so mark this zone non-empty. */ 1320 btrfs_dev_clear_zone_empty(device, info->physical); 1321 1322 dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace); 1323 if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) 1324 btrfs_dev_clear_zone_empty(dev_replace->tgtdev, info->physical); 1325 1326 /* 1327 * The group is mapped to a sequential zone. Get the zone write pointer 1328 * to determine the allocation offset within the zone. 1329 */ 1330 WARN_ON(!IS_ALIGNED(info->physical, fs_info->zone_size)); 1331 1332 if (new) { 1333 sector_t capacity; 1334 1335 capacity = bdev_zone_capacity(device->bdev, info->physical >> SECTOR_SHIFT); 1336 up_read(&dev_replace->rwsem); 1337 info->alloc_offset = 0; 1338 info->capacity = capacity << SECTOR_SHIFT; 1339 1340 return 0; 1341 } 1342 1343 nofs_flag = memalloc_nofs_save(); 1344 ret = btrfs_get_dev_zone(device, info->physical, &zone); 1345 memalloc_nofs_restore(nofs_flag); 1346 if (ret) { 1347 up_read(&dev_replace->rwsem); 1348 if (ret != -EIO && ret != -EOPNOTSUPP) 1349 return ret; 1350 info->alloc_offset = WP_MISSING_DEV; 1351 return 0; 1352 } 1353 1354 if (zone.type == BLK_ZONE_TYPE_CONVENTIONAL) { 1355 btrfs_err(fs_info, 1356 "zoned: unexpected conventional zone %llu on device %s (devid %llu)", 1357 zone.start << SECTOR_SHIFT, rcu_dereference(device->name), 1358 device->devid); 1359 up_read(&dev_replace->rwsem); 1360 return -EIO; 1361 } 1362 1363 info->capacity = (zone.capacity << SECTOR_SHIFT); 1364 1365 switch (zone.cond) { 1366 case BLK_ZONE_COND_OFFLINE: 1367 case BLK_ZONE_COND_READONLY: 1368 btrfs_err(fs_info, 1369 "zoned: offline/readonly zone %llu on device %s (devid %llu)", 1370 (info->physical >> device->zone_info->zone_size_shift), 1371 rcu_dereference(device->name), device->devid); 1372 info->alloc_offset = WP_MISSING_DEV; 1373 break; 1374 case BLK_ZONE_COND_EMPTY: 1375 info->alloc_offset = 0; 1376 break; 1377 case BLK_ZONE_COND_FULL: 1378 info->alloc_offset = info->capacity; 1379 break; 1380 default: 1381 /* Partially used zone. */ 1382 info->alloc_offset = ((zone.wp - zone.start) << SECTOR_SHIFT); 1383 __set_bit(zone_idx, active); 1384 break; 1385 } 1386 1387 up_read(&dev_replace->rwsem); 1388 1389 return 0; 1390 } 1391 1392 static int btrfs_load_block_group_single(struct btrfs_block_group *bg, 1393 struct zone_info *info, 1394 unsigned long *active) 1395 { 1396 if (info->alloc_offset == WP_MISSING_DEV) { 1397 btrfs_err(bg->fs_info, 1398 "zoned: cannot recover write pointer for zone %llu", 1399 info->physical); 1400 return -EIO; 1401 } 1402 1403 bg->alloc_offset = info->alloc_offset; 1404 bg->zone_capacity = info->capacity; 1405 if (test_bit(0, active)) 1406 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); 1407 return 0; 1408 } 1409 1410 static int btrfs_load_block_group_dup(struct btrfs_block_group *bg, 1411 struct btrfs_chunk_map *map, 1412 struct zone_info *zone_info, 1413 unsigned long *active, 1414 u64 last_alloc) 1415 { 1416 struct btrfs_fs_info *fs_info = bg->fs_info; 1417 1418 if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { 1419 btrfs_err(fs_info, "zoned: data DUP profile needs raid-stripe-tree"); 1420 return -EINVAL; 1421 } 1422 1423 bg->zone_capacity = min_not_zero(zone_info[0].capacity, zone_info[1].capacity); 1424 1425 if (zone_info[0].alloc_offset == WP_MISSING_DEV) { 1426 btrfs_err(bg->fs_info, 1427 "zoned: cannot recover write pointer for zone %llu", 1428 zone_info[0].physical); 1429 return -EIO; 1430 } 1431 if (zone_info[1].alloc_offset == WP_MISSING_DEV) { 1432 btrfs_err(bg->fs_info, 1433 "zoned: cannot recover write pointer for zone %llu", 1434 zone_info[1].physical); 1435 return -EIO; 1436 } 1437 1438 if (zone_info[0].alloc_offset == WP_CONVENTIONAL) 1439 zone_info[0].alloc_offset = last_alloc; 1440 1441 if (zone_info[1].alloc_offset == WP_CONVENTIONAL) 1442 zone_info[1].alloc_offset = last_alloc; 1443 1444 if (zone_info[0].alloc_offset != zone_info[1].alloc_offset) { 1445 btrfs_err(bg->fs_info, 1446 "zoned: write pointer offset mismatch of zones in DUP profile"); 1447 return -EIO; 1448 } 1449 1450 if (test_bit(0, active) != test_bit(1, active)) { 1451 if (!btrfs_zone_activate(bg)) 1452 return -EIO; 1453 } else if (test_bit(0, active)) { 1454 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); 1455 } 1456 1457 bg->alloc_offset = zone_info[0].alloc_offset; 1458 return 0; 1459 } 1460 1461 static int btrfs_load_block_group_raid1(struct btrfs_block_group *bg, 1462 struct btrfs_chunk_map *map, 1463 struct zone_info *zone_info, 1464 unsigned long *active, 1465 u64 last_alloc) 1466 { 1467 struct btrfs_fs_info *fs_info = bg->fs_info; 1468 int i; 1469 1470 if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { 1471 btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", 1472 btrfs_bg_type_to_raid_name(map->type)); 1473 return -EINVAL; 1474 } 1475 1476 /* In case a device is missing we have a cap of 0, so don't use it. */ 1477 bg->zone_capacity = min_not_zero(zone_info[0].capacity, zone_info[1].capacity); 1478 1479 for (i = 0; i < map->num_stripes; i++) { 1480 if (zone_info[i].alloc_offset == WP_MISSING_DEV) 1481 continue; 1482 1483 if (zone_info[i].alloc_offset == WP_CONVENTIONAL) 1484 zone_info[i].alloc_offset = last_alloc; 1485 1486 if ((zone_info[0].alloc_offset != zone_info[i].alloc_offset) && 1487 !btrfs_test_opt(fs_info, DEGRADED)) { 1488 btrfs_err(fs_info, 1489 "zoned: write pointer offset mismatch of zones in %s profile", 1490 btrfs_bg_type_to_raid_name(map->type)); 1491 return -EIO; 1492 } 1493 if (test_bit(0, active) != test_bit(i, active)) { 1494 if (!btrfs_test_opt(fs_info, DEGRADED) && 1495 !btrfs_zone_activate(bg)) { 1496 return -EIO; 1497 } 1498 } else { 1499 if (test_bit(0, active)) 1500 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); 1501 } 1502 } 1503 1504 if (zone_info[0].alloc_offset != WP_MISSING_DEV) 1505 bg->alloc_offset = zone_info[0].alloc_offset; 1506 else 1507 bg->alloc_offset = zone_info[i - 1].alloc_offset; 1508 1509 return 0; 1510 } 1511 1512 static int btrfs_load_block_group_raid0(struct btrfs_block_group *bg, 1513 struct btrfs_chunk_map *map, 1514 struct zone_info *zone_info, 1515 unsigned long *active, 1516 u64 last_alloc) 1517 { 1518 struct btrfs_fs_info *fs_info = bg->fs_info; 1519 1520 if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { 1521 btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", 1522 btrfs_bg_type_to_raid_name(map->type)); 1523 return -EINVAL; 1524 } 1525 1526 for (int i = 0; i < map->num_stripes; i++) { 1527 if (zone_info[i].alloc_offset == WP_MISSING_DEV) 1528 continue; 1529 1530 if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { 1531 u64 stripe_nr, full_stripe_nr; 1532 u64 stripe_offset; 1533 int stripe_index; 1534 1535 stripe_nr = div64_u64(last_alloc, map->stripe_size); 1536 stripe_offset = stripe_nr * map->stripe_size; 1537 full_stripe_nr = div_u64(stripe_nr, map->num_stripes); 1538 div_u64_rem(stripe_nr, map->num_stripes, &stripe_index); 1539 1540 zone_info[i].alloc_offset = 1541 full_stripe_nr * map->stripe_size; 1542 1543 if (stripe_index > i) 1544 zone_info[i].alloc_offset += map->stripe_size; 1545 else if (stripe_index == i) 1546 zone_info[i].alloc_offset += 1547 (last_alloc - stripe_offset); 1548 } 1549 1550 if (test_bit(0, active) != test_bit(i, active)) { 1551 if (!btrfs_zone_activate(bg)) 1552 return -EIO; 1553 } else { 1554 if (test_bit(0, active)) 1555 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); 1556 } 1557 bg->zone_capacity += zone_info[i].capacity; 1558 bg->alloc_offset += zone_info[i].alloc_offset; 1559 } 1560 1561 return 0; 1562 } 1563 1564 static int btrfs_load_block_group_raid10(struct btrfs_block_group *bg, 1565 struct btrfs_chunk_map *map, 1566 struct zone_info *zone_info, 1567 unsigned long *active, 1568 u64 last_alloc) 1569 { 1570 struct btrfs_fs_info *fs_info = bg->fs_info; 1571 1572 if ((map->type & BTRFS_BLOCK_GROUP_DATA) && !fs_info->stripe_root) { 1573 btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", 1574 btrfs_bg_type_to_raid_name(map->type)); 1575 return -EINVAL; 1576 } 1577 1578 for (int i = 0; i < map->num_stripes; i++) { 1579 if (zone_info[i].alloc_offset == WP_MISSING_DEV) 1580 continue; 1581 1582 if (test_bit(0, active) != test_bit(i, active)) { 1583 if (!btrfs_zone_activate(bg)) 1584 return -EIO; 1585 } else { 1586 if (test_bit(0, active)) 1587 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &bg->runtime_flags); 1588 } 1589 1590 if (zone_info[i].alloc_offset == WP_CONVENTIONAL) { 1591 u64 stripe_nr, full_stripe_nr; 1592 u64 stripe_offset; 1593 int stripe_index; 1594 1595 stripe_nr = div64_u64(last_alloc, map->stripe_size); 1596 stripe_offset = stripe_nr * map->stripe_size; 1597 full_stripe_nr = div_u64(stripe_nr, 1598 map->num_stripes / map->sub_stripes); 1599 div_u64_rem(stripe_nr, 1600 (map->num_stripes / map->sub_stripes), 1601 &stripe_index); 1602 1603 zone_info[i].alloc_offset = 1604 full_stripe_nr * map->stripe_size; 1605 1606 if (stripe_index > (i / map->sub_stripes)) 1607 zone_info[i].alloc_offset += map->stripe_size; 1608 else if (stripe_index == (i / map->sub_stripes)) 1609 zone_info[i].alloc_offset += 1610 (last_alloc - stripe_offset); 1611 } 1612 1613 if ((i % map->sub_stripes) == 0) { 1614 bg->zone_capacity += zone_info[i].capacity; 1615 bg->alloc_offset += zone_info[i].alloc_offset; 1616 } 1617 } 1618 1619 return 0; 1620 } 1621 1622 int btrfs_load_block_group_zone_info(struct btrfs_block_group *cache, bool new) 1623 { 1624 struct btrfs_fs_info *fs_info = cache->fs_info; 1625 struct btrfs_chunk_map *map; 1626 u64 logical = cache->start; 1627 u64 length = cache->length; 1628 struct zone_info *zone_info = NULL; 1629 int ret; 1630 int i; 1631 unsigned long *active = NULL; 1632 u64 last_alloc = 0; 1633 u32 num_sequential = 0, num_conventional = 0; 1634 u64 profile; 1635 1636 if (!btrfs_is_zoned(fs_info)) 1637 return 0; 1638 1639 /* Sanity check */ 1640 if (!IS_ALIGNED(length, fs_info->zone_size)) { 1641 btrfs_err(fs_info, 1642 "zoned: block group %llu len %llu unaligned to zone size %llu", 1643 logical, length, fs_info->zone_size); 1644 return -EIO; 1645 } 1646 1647 map = btrfs_find_chunk_map(fs_info, logical, length); 1648 if (!map) 1649 return -EINVAL; 1650 1651 cache->physical_map = map; 1652 1653 zone_info = kcalloc(map->num_stripes, sizeof(*zone_info), GFP_NOFS); 1654 if (!zone_info) { 1655 ret = -ENOMEM; 1656 goto out; 1657 } 1658 1659 active = bitmap_zalloc(map->num_stripes, GFP_NOFS); 1660 if (!active) { 1661 ret = -ENOMEM; 1662 goto out; 1663 } 1664 1665 for (i = 0; i < map->num_stripes; i++) { 1666 ret = btrfs_load_zone_info(fs_info, i, &zone_info[i], active, map, new); 1667 if (ret) 1668 goto out; 1669 1670 if (zone_info[i].alloc_offset == WP_CONVENTIONAL) 1671 num_conventional++; 1672 else 1673 num_sequential++; 1674 } 1675 1676 if (num_sequential > 0) 1677 set_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags); 1678 1679 if (num_conventional > 0) { 1680 /* Zone capacity is always zone size in emulation */ 1681 cache->zone_capacity = cache->length; 1682 ret = calculate_alloc_pointer(cache, &last_alloc, new); 1683 if (ret) { 1684 btrfs_err(fs_info, 1685 "zoned: failed to determine allocation offset of bg %llu", 1686 cache->start); 1687 goto out; 1688 } else if (map->num_stripes == num_conventional) { 1689 cache->alloc_offset = last_alloc; 1690 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags); 1691 goto out; 1692 } 1693 } 1694 1695 profile = map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK; 1696 switch (profile) { 1697 case 0: /* single */ 1698 ret = btrfs_load_block_group_single(cache, &zone_info[0], active); 1699 break; 1700 case BTRFS_BLOCK_GROUP_DUP: 1701 ret = btrfs_load_block_group_dup(cache, map, zone_info, active, 1702 last_alloc); 1703 break; 1704 case BTRFS_BLOCK_GROUP_RAID1: 1705 case BTRFS_BLOCK_GROUP_RAID1C3: 1706 case BTRFS_BLOCK_GROUP_RAID1C4: 1707 ret = btrfs_load_block_group_raid1(cache, map, zone_info, 1708 active, last_alloc); 1709 break; 1710 case BTRFS_BLOCK_GROUP_RAID0: 1711 ret = btrfs_load_block_group_raid0(cache, map, zone_info, 1712 active, last_alloc); 1713 break; 1714 case BTRFS_BLOCK_GROUP_RAID10: 1715 ret = btrfs_load_block_group_raid10(cache, map, zone_info, 1716 active, last_alloc); 1717 break; 1718 case BTRFS_BLOCK_GROUP_RAID5: 1719 case BTRFS_BLOCK_GROUP_RAID6: 1720 default: 1721 btrfs_err(fs_info, "zoned: profile %s not yet supported", 1722 btrfs_bg_type_to_raid_name(map->type)); 1723 ret = -EINVAL; 1724 goto out; 1725 } 1726 1727 if (ret == -EIO && profile != 0 && profile != BTRFS_BLOCK_GROUP_RAID0 && 1728 profile != BTRFS_BLOCK_GROUP_RAID10) { 1729 /* 1730 * Detected broken write pointer. Make this block group 1731 * unallocatable by setting the allocation pointer at the end of 1732 * allocatable region. Relocating this block group will fix the 1733 * mismatch. 1734 * 1735 * Currently, we cannot handle RAID0 or RAID10 case like this 1736 * because we don't have a proper zone_capacity value. But, 1737 * reading from this block group won't work anyway by a missing 1738 * stripe. 1739 */ 1740 cache->alloc_offset = cache->zone_capacity; 1741 } 1742 1743 out: 1744 /* Reject non SINGLE data profiles without RST */ 1745 if ((map->type & BTRFS_BLOCK_GROUP_DATA) && 1746 (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) && 1747 !fs_info->stripe_root) { 1748 btrfs_err(fs_info, "zoned: data %s needs raid-stripe-tree", 1749 btrfs_bg_type_to_raid_name(map->type)); 1750 return -EINVAL; 1751 } 1752 1753 if (cache->alloc_offset > cache->zone_capacity) { 1754 btrfs_err(fs_info, 1755 "zoned: invalid write pointer %llu (larger than zone capacity %llu) in block group %llu", 1756 cache->alloc_offset, cache->zone_capacity, 1757 cache->start); 1758 ret = -EIO; 1759 } 1760 1761 /* An extent is allocated after the write pointer */ 1762 if (!ret && num_conventional && last_alloc > cache->alloc_offset) { 1763 btrfs_err(fs_info, 1764 "zoned: got wrong write pointer in BG %llu: %llu > %llu", 1765 logical, last_alloc, cache->alloc_offset); 1766 ret = -EIO; 1767 } 1768 1769 if (!ret) { 1770 cache->meta_write_pointer = cache->alloc_offset + cache->start; 1771 if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &cache->runtime_flags)) { 1772 btrfs_get_block_group(cache); 1773 spin_lock(&fs_info->zone_active_bgs_lock); 1774 list_add_tail(&cache->active_bg_list, 1775 &fs_info->zone_active_bgs); 1776 spin_unlock(&fs_info->zone_active_bgs_lock); 1777 } 1778 } else { 1779 btrfs_free_chunk_map(cache->physical_map); 1780 cache->physical_map = NULL; 1781 } 1782 bitmap_free(active); 1783 kfree(zone_info); 1784 1785 return ret; 1786 } 1787 1788 void btrfs_calc_zone_unusable(struct btrfs_block_group *cache) 1789 { 1790 u64 unusable, free; 1791 1792 if (!btrfs_is_zoned(cache->fs_info)) 1793 return; 1794 1795 WARN_ON(cache->bytes_super != 0); 1796 unusable = (cache->alloc_offset - cache->used) + 1797 (cache->length - cache->zone_capacity); 1798 free = cache->zone_capacity - cache->alloc_offset; 1799 1800 /* We only need ->free_space in ALLOC_SEQ block groups */ 1801 cache->cached = BTRFS_CACHE_FINISHED; 1802 cache->free_space_ctl->free_space = free; 1803 cache->zone_unusable = unusable; 1804 } 1805 1806 bool btrfs_use_zone_append(struct btrfs_bio *bbio) 1807 { 1808 u64 start = (bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT); 1809 struct btrfs_inode *inode = bbio->inode; 1810 struct btrfs_fs_info *fs_info = bbio->fs_info; 1811 struct btrfs_block_group *cache; 1812 bool ret = false; 1813 1814 if (!btrfs_is_zoned(fs_info)) 1815 return false; 1816 1817 if (!inode || !is_data_inode(inode)) 1818 return false; 1819 1820 if (btrfs_op(&bbio->bio) != BTRFS_MAP_WRITE) 1821 return false; 1822 1823 /* 1824 * Using REQ_OP_ZONE_APPEND for relocation can break assumptions on the 1825 * extent layout the relocation code has. 1826 * Furthermore we have set aside own block-group from which only the 1827 * relocation "process" can allocate and make sure only one process at a 1828 * time can add pages to an extent that gets relocated, so it's safe to 1829 * use regular REQ_OP_WRITE for this special case. 1830 */ 1831 if (btrfs_is_data_reloc_root(inode->root)) 1832 return false; 1833 1834 cache = btrfs_lookup_block_group(fs_info, start); 1835 ASSERT(cache); 1836 if (!cache) 1837 return false; 1838 1839 ret = !!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &cache->runtime_flags); 1840 btrfs_put_block_group(cache); 1841 1842 return ret; 1843 } 1844 1845 void btrfs_record_physical_zoned(struct btrfs_bio *bbio) 1846 { 1847 const u64 physical = bbio->bio.bi_iter.bi_sector << SECTOR_SHIFT; 1848 struct btrfs_ordered_sum *sum = bbio->sums; 1849 1850 if (physical < bbio->orig_physical) 1851 sum->logical -= bbio->orig_physical - physical; 1852 else 1853 sum->logical += physical - bbio->orig_physical; 1854 } 1855 1856 static void btrfs_rewrite_logical_zoned(struct btrfs_ordered_extent *ordered, 1857 u64 logical) 1858 { 1859 struct extent_map_tree *em_tree = &ordered->inode->extent_tree; 1860 struct extent_map *em; 1861 1862 ordered->disk_bytenr = logical; 1863 1864 write_lock(&em_tree->lock); 1865 em = btrfs_search_extent_mapping(em_tree, ordered->file_offset, 1866 ordered->num_bytes); 1867 /* The em should be a new COW extent, thus it should not have an offset. */ 1868 ASSERT(em->offset == 0); 1869 em->disk_bytenr = logical; 1870 btrfs_free_extent_map(em); 1871 write_unlock(&em_tree->lock); 1872 } 1873 1874 static bool btrfs_zoned_split_ordered(struct btrfs_ordered_extent *ordered, 1875 u64 logical, u64 len) 1876 { 1877 struct btrfs_ordered_extent *new; 1878 1879 if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags) && 1880 btrfs_split_extent_map(ordered->inode, ordered->file_offset, 1881 ordered->num_bytes, len, logical)) 1882 return false; 1883 1884 new = btrfs_split_ordered_extent(ordered, len); 1885 if (IS_ERR(new)) 1886 return false; 1887 new->disk_bytenr = logical; 1888 btrfs_finish_one_ordered(new); 1889 return true; 1890 } 1891 1892 void btrfs_finish_ordered_zoned(struct btrfs_ordered_extent *ordered) 1893 { 1894 struct btrfs_inode *inode = ordered->inode; 1895 struct btrfs_fs_info *fs_info = inode->root->fs_info; 1896 struct btrfs_ordered_sum *sum; 1897 u64 logical, len; 1898 1899 /* 1900 * Write to pre-allocated region is for the data relocation, and so 1901 * it should use WRITE operation. No split/rewrite are necessary. 1902 */ 1903 if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered->flags)) 1904 return; 1905 1906 ASSERT(!list_empty(&ordered->list)); 1907 /* The ordered->list can be empty in the above pre-alloc case. */ 1908 sum = list_first_entry(&ordered->list, struct btrfs_ordered_sum, list); 1909 logical = sum->logical; 1910 len = sum->len; 1911 1912 while (len < ordered->disk_num_bytes) { 1913 sum = list_next_entry(sum, list); 1914 if (sum->logical == logical + len) { 1915 len += sum->len; 1916 continue; 1917 } 1918 if (!btrfs_zoned_split_ordered(ordered, logical, len)) { 1919 set_bit(BTRFS_ORDERED_IOERR, &ordered->flags); 1920 btrfs_err(fs_info, "failed to split ordered extent"); 1921 goto out; 1922 } 1923 logical = sum->logical; 1924 len = sum->len; 1925 } 1926 1927 if (ordered->disk_bytenr != logical) 1928 btrfs_rewrite_logical_zoned(ordered, logical); 1929 1930 out: 1931 /* 1932 * If we end up here for nodatasum I/O, the btrfs_ordered_sum structures 1933 * were allocated by btrfs_alloc_dummy_sum only to record the logical 1934 * addresses and don't contain actual checksums. We thus must free them 1935 * here so that we don't attempt to log the csums later. 1936 */ 1937 if ((inode->flags & BTRFS_INODE_NODATASUM) || 1938 test_bit(BTRFS_FS_STATE_NO_DATA_CSUMS, &fs_info->fs_state)) { 1939 while ((sum = list_first_entry_or_null(&ordered->list, 1940 typeof(*sum), list))) { 1941 list_del(&sum->list); 1942 kfree(sum); 1943 } 1944 } 1945 } 1946 1947 static bool check_bg_is_active(struct btrfs_eb_write_context *ctx, 1948 struct btrfs_block_group **active_bg) 1949 { 1950 const struct writeback_control *wbc = ctx->wbc; 1951 struct btrfs_block_group *block_group = ctx->zoned_bg; 1952 struct btrfs_fs_info *fs_info = block_group->fs_info; 1953 1954 if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) 1955 return true; 1956 1957 if (fs_info->treelog_bg == block_group->start) { 1958 if (!btrfs_zone_activate(block_group)) { 1959 int ret_fin = btrfs_zone_finish_one_bg(fs_info); 1960 1961 if (ret_fin != 1 || !btrfs_zone_activate(block_group)) 1962 return false; 1963 } 1964 } else if (*active_bg != block_group) { 1965 struct btrfs_block_group *tgt = *active_bg; 1966 1967 /* zoned_meta_io_lock protects fs_info->active_{meta,system}_bg. */ 1968 lockdep_assert_held(&fs_info->zoned_meta_io_lock); 1969 1970 if (tgt) { 1971 /* 1972 * If there is an unsent IO left in the allocated area, 1973 * we cannot wait for them as it may cause a deadlock. 1974 */ 1975 if (tgt->meta_write_pointer < tgt->start + tgt->alloc_offset) { 1976 if (wbc->sync_mode == WB_SYNC_NONE || 1977 (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync)) 1978 return false; 1979 } 1980 1981 /* Pivot active metadata/system block group. */ 1982 btrfs_zoned_meta_io_unlock(fs_info); 1983 wait_eb_writebacks(tgt); 1984 do_zone_finish(tgt, true); 1985 btrfs_zoned_meta_io_lock(fs_info); 1986 if (*active_bg == tgt) { 1987 btrfs_put_block_group(tgt); 1988 *active_bg = NULL; 1989 } 1990 } 1991 if (!btrfs_zone_activate(block_group)) 1992 return false; 1993 if (*active_bg != block_group) { 1994 ASSERT(*active_bg == NULL); 1995 *active_bg = block_group; 1996 btrfs_get_block_group(block_group); 1997 } 1998 } 1999 2000 return true; 2001 } 2002 2003 /* 2004 * Check if @ctx->eb is aligned to the write pointer. 2005 * 2006 * Return: 2007 * 0: @ctx->eb is at the write pointer. You can write it. 2008 * -EAGAIN: There is a hole. The caller should handle the case. 2009 * -EBUSY: There is a hole, but the caller can just bail out. 2010 */ 2011 int btrfs_check_meta_write_pointer(struct btrfs_fs_info *fs_info, 2012 struct btrfs_eb_write_context *ctx) 2013 { 2014 const struct writeback_control *wbc = ctx->wbc; 2015 const struct extent_buffer *eb = ctx->eb; 2016 struct btrfs_block_group *block_group = ctx->zoned_bg; 2017 2018 if (!btrfs_is_zoned(fs_info)) 2019 return 0; 2020 2021 if (block_group) { 2022 if (block_group->start > eb->start || 2023 block_group->start + block_group->length <= eb->start) { 2024 btrfs_put_block_group(block_group); 2025 block_group = NULL; 2026 ctx->zoned_bg = NULL; 2027 } 2028 } 2029 2030 if (!block_group) { 2031 block_group = btrfs_lookup_block_group(fs_info, eb->start); 2032 if (!block_group) 2033 return 0; 2034 ctx->zoned_bg = block_group; 2035 } 2036 2037 if (block_group->meta_write_pointer == eb->start) { 2038 struct btrfs_block_group **tgt; 2039 2040 if (!test_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags)) 2041 return 0; 2042 2043 if (block_group->flags & BTRFS_BLOCK_GROUP_SYSTEM) 2044 tgt = &fs_info->active_system_bg; 2045 else 2046 tgt = &fs_info->active_meta_bg; 2047 if (check_bg_is_active(ctx, tgt)) 2048 return 0; 2049 } 2050 2051 /* 2052 * Since we may release fs_info->zoned_meta_io_lock, someone can already 2053 * start writing this eb. In that case, we can just bail out. 2054 */ 2055 if (block_group->meta_write_pointer > eb->start) 2056 return -EBUSY; 2057 2058 /* If for_sync, this hole will be filled with transaction commit. */ 2059 if (wbc->sync_mode == WB_SYNC_ALL && !wbc->for_sync) 2060 return -EAGAIN; 2061 return -EBUSY; 2062 } 2063 2064 int btrfs_zoned_issue_zeroout(struct btrfs_device *device, u64 physical, u64 length) 2065 { 2066 if (!btrfs_dev_is_sequential(device, physical)) 2067 return -EOPNOTSUPP; 2068 2069 return blkdev_issue_zeroout(device->bdev, physical >> SECTOR_SHIFT, 2070 length >> SECTOR_SHIFT, GFP_NOFS, 0); 2071 } 2072 2073 static int read_zone_info(struct btrfs_fs_info *fs_info, u64 logical, 2074 struct blk_zone *zone) 2075 { 2076 struct btrfs_io_context *bioc = NULL; 2077 u64 mapped_length = PAGE_SIZE; 2078 unsigned int nofs_flag; 2079 int nmirrors; 2080 int i, ret; 2081 2082 ret = btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical, 2083 &mapped_length, &bioc, NULL, NULL); 2084 if (ret || !bioc || mapped_length < PAGE_SIZE) { 2085 ret = -EIO; 2086 goto out_put_bioc; 2087 } 2088 2089 if (bioc->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) { 2090 ret = -EINVAL; 2091 goto out_put_bioc; 2092 } 2093 2094 nofs_flag = memalloc_nofs_save(); 2095 nmirrors = (int)bioc->num_stripes; 2096 for (i = 0; i < nmirrors; i++) { 2097 u64 physical = bioc->stripes[i].physical; 2098 struct btrfs_device *dev = bioc->stripes[i].dev; 2099 2100 /* Missing device */ 2101 if (!dev->bdev) 2102 continue; 2103 2104 ret = btrfs_get_dev_zone(dev, physical, zone); 2105 /* Failing device */ 2106 if (ret == -EIO || ret == -EOPNOTSUPP) 2107 continue; 2108 break; 2109 } 2110 memalloc_nofs_restore(nofs_flag); 2111 out_put_bioc: 2112 btrfs_put_bioc(bioc); 2113 return ret; 2114 } 2115 2116 /* 2117 * Synchronize write pointer in a zone at @physical_start on @tgt_dev, by 2118 * filling zeros between @physical_pos to a write pointer of dev-replace 2119 * source device. 2120 */ 2121 int btrfs_sync_zone_write_pointer(struct btrfs_device *tgt_dev, u64 logical, 2122 u64 physical_start, u64 physical_pos) 2123 { 2124 struct btrfs_fs_info *fs_info = tgt_dev->fs_info; 2125 struct blk_zone zone; 2126 u64 length; 2127 u64 wp; 2128 int ret; 2129 2130 if (!btrfs_dev_is_sequential(tgt_dev, physical_pos)) 2131 return 0; 2132 2133 ret = read_zone_info(fs_info, logical, &zone); 2134 if (ret) 2135 return ret; 2136 2137 wp = physical_start + ((zone.wp - zone.start) << SECTOR_SHIFT); 2138 2139 if (physical_pos == wp) 2140 return 0; 2141 2142 if (physical_pos > wp) 2143 return -EUCLEAN; 2144 2145 length = wp - physical_pos; 2146 return btrfs_zoned_issue_zeroout(tgt_dev, physical_pos, length); 2147 } 2148 2149 /* 2150 * Activate block group and underlying device zones 2151 * 2152 * @block_group: the block group to activate 2153 * 2154 * Return: true on success, false otherwise 2155 */ 2156 bool btrfs_zone_activate(struct btrfs_block_group *block_group) 2157 { 2158 struct btrfs_fs_info *fs_info = block_group->fs_info; 2159 struct btrfs_chunk_map *map; 2160 struct btrfs_device *device; 2161 u64 physical; 2162 const bool is_data = (block_group->flags & BTRFS_BLOCK_GROUP_DATA); 2163 bool ret; 2164 int i; 2165 2166 if (!btrfs_is_zoned(block_group->fs_info)) 2167 return true; 2168 2169 map = block_group->physical_map; 2170 2171 spin_lock(&fs_info->zone_active_bgs_lock); 2172 spin_lock(&block_group->lock); 2173 if (test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { 2174 ret = true; 2175 goto out_unlock; 2176 } 2177 2178 if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) { 2179 /* The caller should check if the block group is full. */ 2180 if (WARN_ON_ONCE(btrfs_zoned_bg_is_full(block_group))) { 2181 ret = false; 2182 goto out_unlock; 2183 } 2184 } else { 2185 /* Since it is already written, it should have been active. */ 2186 WARN_ON_ONCE(block_group->meta_write_pointer != block_group->start); 2187 } 2188 2189 for (i = 0; i < map->num_stripes; i++) { 2190 struct btrfs_zoned_device_info *zinfo; 2191 int reserved = 0; 2192 2193 device = map->stripes[i].dev; 2194 physical = map->stripes[i].physical; 2195 zinfo = device->zone_info; 2196 2197 if (!device->bdev) 2198 continue; 2199 2200 if (zinfo->max_active_zones == 0) 2201 continue; 2202 2203 if (is_data) 2204 reserved = zinfo->reserved_active_zones; 2205 /* 2206 * For the data block group, leave active zones for one 2207 * metadata block group and one system block group. 2208 */ 2209 if (atomic_read(&zinfo->active_zones_left) <= reserved) { 2210 ret = false; 2211 goto out_unlock; 2212 } 2213 2214 if (!btrfs_dev_set_active_zone(device, physical)) { 2215 /* Cannot activate the zone */ 2216 ret = false; 2217 goto out_unlock; 2218 } 2219 if (!is_data) 2220 zinfo->reserved_active_zones--; 2221 } 2222 2223 /* Successfully activated all the zones */ 2224 set_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); 2225 spin_unlock(&block_group->lock); 2226 2227 /* For the active block group list */ 2228 btrfs_get_block_group(block_group); 2229 list_add_tail(&block_group->active_bg_list, &fs_info->zone_active_bgs); 2230 spin_unlock(&fs_info->zone_active_bgs_lock); 2231 2232 return true; 2233 2234 out_unlock: 2235 spin_unlock(&block_group->lock); 2236 spin_unlock(&fs_info->zone_active_bgs_lock); 2237 return ret; 2238 } 2239 2240 static void wait_eb_writebacks(struct btrfs_block_group *block_group) 2241 { 2242 struct btrfs_fs_info *fs_info = block_group->fs_info; 2243 const u64 end = block_group->start + block_group->length; 2244 struct extent_buffer *eb; 2245 unsigned long index, start = (block_group->start >> fs_info->nodesize_bits); 2246 2247 rcu_read_lock(); 2248 xa_for_each_start(&fs_info->buffer_tree, index, eb, start) { 2249 if (eb->start < block_group->start) 2250 continue; 2251 if (eb->start >= end) 2252 break; 2253 rcu_read_unlock(); 2254 wait_on_extent_buffer_writeback(eb); 2255 rcu_read_lock(); 2256 } 2257 rcu_read_unlock(); 2258 } 2259 2260 static int call_zone_finish(struct btrfs_block_group *block_group, 2261 struct btrfs_io_stripe *stripe) 2262 { 2263 struct btrfs_device *device = stripe->dev; 2264 const u64 physical = stripe->physical; 2265 struct btrfs_zoned_device_info *zinfo = device->zone_info; 2266 int ret; 2267 2268 if (!device->bdev) 2269 return 0; 2270 2271 if (zinfo->max_active_zones == 0) 2272 return 0; 2273 2274 if (btrfs_dev_is_sequential(device, physical)) { 2275 unsigned int nofs_flags; 2276 2277 nofs_flags = memalloc_nofs_save(); 2278 ret = blkdev_zone_mgmt(device->bdev, REQ_OP_ZONE_FINISH, 2279 physical >> SECTOR_SHIFT, 2280 zinfo->zone_size >> SECTOR_SHIFT); 2281 memalloc_nofs_restore(nofs_flags); 2282 2283 if (ret) 2284 return ret; 2285 } 2286 2287 if (!(block_group->flags & BTRFS_BLOCK_GROUP_DATA)) 2288 zinfo->reserved_active_zones++; 2289 btrfs_dev_clear_active_zone(device, physical); 2290 2291 return 0; 2292 } 2293 2294 static int do_zone_finish(struct btrfs_block_group *block_group, bool fully_written) 2295 { 2296 struct btrfs_fs_info *fs_info = block_group->fs_info; 2297 struct btrfs_chunk_map *map; 2298 const bool is_metadata = (block_group->flags & 2299 (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)); 2300 struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace; 2301 int ret = 0; 2302 int i; 2303 2304 spin_lock(&block_group->lock); 2305 if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags)) { 2306 spin_unlock(&block_group->lock); 2307 return 0; 2308 } 2309 2310 /* Check if we have unwritten allocated space */ 2311 if (is_metadata && 2312 block_group->start + block_group->alloc_offset > block_group->meta_write_pointer) { 2313 spin_unlock(&block_group->lock); 2314 return -EAGAIN; 2315 } 2316 2317 /* 2318 * If we are sure that the block group is full (= no more room left for 2319 * new allocation) and the IO for the last usable block is completed, we 2320 * don't need to wait for the other IOs. This holds because we ensure 2321 * the sequential IO submissions using the ZONE_APPEND command for data 2322 * and block_group->meta_write_pointer for metadata. 2323 */ 2324 if (!fully_written) { 2325 if (test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { 2326 spin_unlock(&block_group->lock); 2327 return -EAGAIN; 2328 } 2329 spin_unlock(&block_group->lock); 2330 2331 ret = btrfs_inc_block_group_ro(block_group, false); 2332 if (ret) 2333 return ret; 2334 2335 /* Ensure all writes in this block group finish */ 2336 btrfs_wait_block_group_reservations(block_group); 2337 /* No need to wait for NOCOW writers. Zoned mode does not allow that */ 2338 btrfs_wait_ordered_roots(fs_info, U64_MAX, block_group); 2339 /* Wait for extent buffers to be written. */ 2340 if (is_metadata) 2341 wait_eb_writebacks(block_group); 2342 2343 spin_lock(&block_group->lock); 2344 2345 /* 2346 * Bail out if someone already deactivated the block group, or 2347 * allocated space is left in the block group. 2348 */ 2349 if (!test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, 2350 &block_group->runtime_flags)) { 2351 spin_unlock(&block_group->lock); 2352 btrfs_dec_block_group_ro(block_group); 2353 return 0; 2354 } 2355 2356 if (block_group->reserved || 2357 test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, 2358 &block_group->runtime_flags)) { 2359 spin_unlock(&block_group->lock); 2360 btrfs_dec_block_group_ro(block_group); 2361 return -EAGAIN; 2362 } 2363 } 2364 2365 clear_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, &block_group->runtime_flags); 2366 block_group->alloc_offset = block_group->zone_capacity; 2367 if (block_group->flags & (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM)) 2368 block_group->meta_write_pointer = block_group->start + 2369 block_group->zone_capacity; 2370 block_group->free_space_ctl->free_space = 0; 2371 btrfs_clear_treelog_bg(block_group); 2372 btrfs_clear_data_reloc_bg(block_group); 2373 spin_unlock(&block_group->lock); 2374 2375 down_read(&dev_replace->rwsem); 2376 map = block_group->physical_map; 2377 for (i = 0; i < map->num_stripes; i++) { 2378 2379 ret = call_zone_finish(block_group, &map->stripes[i]); 2380 if (ret) { 2381 up_read(&dev_replace->rwsem); 2382 return ret; 2383 } 2384 } 2385 up_read(&dev_replace->rwsem); 2386 2387 if (!fully_written) 2388 btrfs_dec_block_group_ro(block_group); 2389 2390 spin_lock(&fs_info->zone_active_bgs_lock); 2391 ASSERT(!list_empty(&block_group->active_bg_list)); 2392 list_del_init(&block_group->active_bg_list); 2393 spin_unlock(&fs_info->zone_active_bgs_lock); 2394 2395 /* For active_bg_list */ 2396 btrfs_put_block_group(block_group); 2397 2398 clear_and_wake_up_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); 2399 2400 return 0; 2401 } 2402 2403 int btrfs_zone_finish(struct btrfs_block_group *block_group) 2404 { 2405 if (!btrfs_is_zoned(block_group->fs_info)) 2406 return 0; 2407 2408 return do_zone_finish(block_group, false); 2409 } 2410 2411 bool btrfs_can_activate_zone(struct btrfs_fs_devices *fs_devices, u64 flags) 2412 { 2413 struct btrfs_fs_info *fs_info = fs_devices->fs_info; 2414 struct btrfs_device *device; 2415 bool ret = false; 2416 2417 if (!btrfs_is_zoned(fs_info)) 2418 return true; 2419 2420 if (test_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags)) 2421 return false; 2422 2423 /* Check if there is a device with active zones left */ 2424 mutex_lock(&fs_info->chunk_mutex); 2425 spin_lock(&fs_info->zone_active_bgs_lock); 2426 list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) { 2427 struct btrfs_zoned_device_info *zinfo = device->zone_info; 2428 int reserved = 0; 2429 2430 if (!device->bdev) 2431 continue; 2432 2433 if (!zinfo->max_active_zones) { 2434 ret = true; 2435 break; 2436 } 2437 2438 if (flags & BTRFS_BLOCK_GROUP_DATA) 2439 reserved = zinfo->reserved_active_zones; 2440 2441 switch (flags & BTRFS_BLOCK_GROUP_PROFILE_MASK) { 2442 case 0: /* single */ 2443 ret = (atomic_read(&zinfo->active_zones_left) >= (1 + reserved)); 2444 break; 2445 case BTRFS_BLOCK_GROUP_DUP: 2446 ret = (atomic_read(&zinfo->active_zones_left) >= (2 + reserved)); 2447 break; 2448 } 2449 if (ret) 2450 break; 2451 } 2452 spin_unlock(&fs_info->zone_active_bgs_lock); 2453 mutex_unlock(&fs_info->chunk_mutex); 2454 2455 if (!ret) 2456 set_bit(BTRFS_FS_NEED_ZONE_FINISH, &fs_info->flags); 2457 2458 return ret; 2459 } 2460 2461 void btrfs_zone_finish_endio(struct btrfs_fs_info *fs_info, u64 logical, u64 length) 2462 { 2463 struct btrfs_block_group *block_group; 2464 u64 min_alloc_bytes; 2465 2466 if (!btrfs_is_zoned(fs_info)) 2467 return; 2468 2469 block_group = btrfs_lookup_block_group(fs_info, logical); 2470 ASSERT(block_group); 2471 2472 /* No MIXED_BG on zoned btrfs. */ 2473 if (block_group->flags & BTRFS_BLOCK_GROUP_DATA) 2474 min_alloc_bytes = fs_info->sectorsize; 2475 else 2476 min_alloc_bytes = fs_info->nodesize; 2477 2478 /* Bail out if we can allocate more data from this block group. */ 2479 if (logical + length + min_alloc_bytes <= 2480 block_group->start + block_group->zone_capacity) 2481 goto out; 2482 2483 do_zone_finish(block_group, true); 2484 2485 out: 2486 btrfs_put_block_group(block_group); 2487 } 2488 2489 static void btrfs_zone_finish_endio_workfn(struct work_struct *work) 2490 { 2491 struct btrfs_block_group *bg = 2492 container_of(work, struct btrfs_block_group, zone_finish_work); 2493 2494 wait_on_extent_buffer_writeback(bg->last_eb); 2495 free_extent_buffer(bg->last_eb); 2496 btrfs_zone_finish_endio(bg->fs_info, bg->start, bg->length); 2497 btrfs_put_block_group(bg); 2498 } 2499 2500 void btrfs_schedule_zone_finish_bg(struct btrfs_block_group *bg, 2501 struct extent_buffer *eb) 2502 { 2503 if (!test_bit(BLOCK_GROUP_FLAG_SEQUENTIAL_ZONE, &bg->runtime_flags) || 2504 eb->start + eb->len * 2 <= bg->start + bg->zone_capacity) 2505 return; 2506 2507 if (WARN_ON(bg->zone_finish_work.func == btrfs_zone_finish_endio_workfn)) { 2508 btrfs_err(bg->fs_info, "double scheduling of bg %llu zone finishing", 2509 bg->start); 2510 return; 2511 } 2512 2513 /* For the work */ 2514 btrfs_get_block_group(bg); 2515 refcount_inc(&eb->refs); 2516 bg->last_eb = eb; 2517 INIT_WORK(&bg->zone_finish_work, btrfs_zone_finish_endio_workfn); 2518 queue_work(system_unbound_wq, &bg->zone_finish_work); 2519 } 2520 2521 void btrfs_clear_data_reloc_bg(struct btrfs_block_group *bg) 2522 { 2523 struct btrfs_fs_info *fs_info = bg->fs_info; 2524 2525 spin_lock(&fs_info->relocation_bg_lock); 2526 if (fs_info->data_reloc_bg == bg->start) 2527 fs_info->data_reloc_bg = 0; 2528 spin_unlock(&fs_info->relocation_bg_lock); 2529 } 2530 2531 void btrfs_zoned_reserve_data_reloc_bg(struct btrfs_fs_info *fs_info) 2532 { 2533 struct btrfs_space_info *data_sinfo = fs_info->data_sinfo; 2534 struct btrfs_space_info *space_info = data_sinfo; 2535 struct btrfs_trans_handle *trans; 2536 struct btrfs_block_group *bg; 2537 struct list_head *bg_list; 2538 u64 alloc_flags; 2539 bool first = true; 2540 bool did_chunk_alloc = false; 2541 int index; 2542 int ret; 2543 2544 if (!btrfs_is_zoned(fs_info)) 2545 return; 2546 2547 if (fs_info->data_reloc_bg) 2548 return; 2549 2550 if (sb_rdonly(fs_info->sb)) 2551 return; 2552 2553 alloc_flags = btrfs_get_alloc_profile(fs_info, space_info->flags); 2554 index = btrfs_bg_flags_to_raid_index(alloc_flags); 2555 2556 /* Scan the data space_info to find empty block groups. Take the second one. */ 2557 again: 2558 bg_list = &space_info->block_groups[index]; 2559 list_for_each_entry(bg, bg_list, list) { 2560 if (bg->alloc_offset != 0) 2561 continue; 2562 2563 if (first) { 2564 first = false; 2565 continue; 2566 } 2567 2568 if (space_info == data_sinfo) { 2569 /* Migrate the block group to the data relocation space_info. */ 2570 struct btrfs_space_info *reloc_sinfo = data_sinfo->sub_group[0]; 2571 int factor; 2572 2573 ASSERT(reloc_sinfo->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC); 2574 factor = btrfs_bg_type_to_factor(bg->flags); 2575 2576 down_write(&space_info->groups_sem); 2577 list_del_init(&bg->list); 2578 /* We can assume this as we choose the second empty one. */ 2579 ASSERT(!list_empty(&space_info->block_groups[index])); 2580 up_write(&space_info->groups_sem); 2581 2582 spin_lock(&space_info->lock); 2583 space_info->total_bytes -= bg->length; 2584 space_info->disk_total -= bg->length * factor; 2585 /* There is no allocation ever happened. */ 2586 ASSERT(bg->used == 0); 2587 ASSERT(bg->zone_unusable == 0); 2588 /* No super block in a block group on the zoned setup. */ 2589 ASSERT(bg->bytes_super == 0); 2590 spin_unlock(&space_info->lock); 2591 2592 bg->space_info = reloc_sinfo; 2593 if (reloc_sinfo->block_group_kobjs[index] == NULL) 2594 btrfs_sysfs_add_block_group_type(bg); 2595 2596 btrfs_add_bg_to_space_info(fs_info, bg); 2597 } 2598 2599 fs_info->data_reloc_bg = bg->start; 2600 set_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &bg->runtime_flags); 2601 btrfs_zone_activate(bg); 2602 2603 return; 2604 } 2605 2606 if (did_chunk_alloc) 2607 return; 2608 2609 trans = btrfs_join_transaction(fs_info->tree_root); 2610 if (IS_ERR(trans)) 2611 return; 2612 2613 /* Allocate new BG in the data relocation space_info. */ 2614 space_info = data_sinfo->sub_group[0]; 2615 ASSERT(space_info->subgroup_id == BTRFS_SUB_GROUP_DATA_RELOC); 2616 ret = btrfs_chunk_alloc(trans, space_info, alloc_flags, CHUNK_ALLOC_FORCE); 2617 btrfs_end_transaction(trans); 2618 if (ret == 1) { 2619 /* 2620 * We allocated a new block group in the data relocation space_info. We 2621 * can take that one. 2622 */ 2623 first = false; 2624 did_chunk_alloc = true; 2625 goto again; 2626 } 2627 } 2628 2629 void btrfs_free_zone_cache(struct btrfs_fs_info *fs_info) 2630 { 2631 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 2632 struct btrfs_device *device; 2633 2634 if (!btrfs_is_zoned(fs_info)) 2635 return; 2636 2637 mutex_lock(&fs_devices->device_list_mutex); 2638 list_for_each_entry(device, &fs_devices->devices, dev_list) { 2639 if (device->zone_info) { 2640 vfree(device->zone_info->zone_cache); 2641 device->zone_info->zone_cache = NULL; 2642 } 2643 } 2644 mutex_unlock(&fs_devices->device_list_mutex); 2645 } 2646 2647 bool btrfs_zoned_should_reclaim(const struct btrfs_fs_info *fs_info) 2648 { 2649 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 2650 struct btrfs_device *device; 2651 u64 total = btrfs_super_total_bytes(fs_info->super_copy); 2652 u64 used = 0; 2653 u64 factor; 2654 2655 ASSERT(btrfs_is_zoned(fs_info)); 2656 2657 if (fs_info->bg_reclaim_threshold == 0) 2658 return false; 2659 2660 mutex_lock(&fs_devices->device_list_mutex); 2661 list_for_each_entry(device, &fs_devices->devices, dev_list) { 2662 if (!device->bdev) 2663 continue; 2664 2665 used += device->bytes_used; 2666 } 2667 mutex_unlock(&fs_devices->device_list_mutex); 2668 2669 factor = div64_u64(used * 100, total); 2670 return factor >= fs_info->bg_reclaim_threshold; 2671 } 2672 2673 void btrfs_zoned_release_data_reloc_bg(struct btrfs_fs_info *fs_info, u64 logical, 2674 u64 length) 2675 { 2676 struct btrfs_block_group *block_group; 2677 2678 if (!btrfs_is_zoned(fs_info)) 2679 return; 2680 2681 block_group = btrfs_lookup_block_group(fs_info, logical); 2682 /* It should be called on a previous data relocation block group. */ 2683 ASSERT(block_group && (block_group->flags & BTRFS_BLOCK_GROUP_DATA)); 2684 2685 spin_lock(&block_group->lock); 2686 if (!test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) 2687 goto out; 2688 2689 /* All relocation extents are written. */ 2690 if (block_group->start + block_group->alloc_offset == logical + length) { 2691 /* 2692 * Now, release this block group for further allocations and 2693 * zone finish. 2694 */ 2695 clear_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, 2696 &block_group->runtime_flags); 2697 } 2698 2699 out: 2700 spin_unlock(&block_group->lock); 2701 btrfs_put_block_group(block_group); 2702 } 2703 2704 int btrfs_zone_finish_one_bg(struct btrfs_fs_info *fs_info) 2705 { 2706 struct btrfs_block_group *block_group; 2707 struct btrfs_block_group *min_bg = NULL; 2708 u64 min_avail = U64_MAX; 2709 int ret; 2710 2711 spin_lock(&fs_info->zone_active_bgs_lock); 2712 list_for_each_entry(block_group, &fs_info->zone_active_bgs, 2713 active_bg_list) { 2714 u64 avail; 2715 2716 spin_lock(&block_group->lock); 2717 if (block_group->reserved || block_group->alloc_offset == 0 || 2718 !(block_group->flags & BTRFS_BLOCK_GROUP_DATA) || 2719 test_bit(BLOCK_GROUP_FLAG_ZONED_DATA_RELOC, &block_group->runtime_flags)) { 2720 spin_unlock(&block_group->lock); 2721 continue; 2722 } 2723 2724 avail = block_group->zone_capacity - block_group->alloc_offset; 2725 if (min_avail > avail) { 2726 if (min_bg) 2727 btrfs_put_block_group(min_bg); 2728 min_bg = block_group; 2729 min_avail = avail; 2730 btrfs_get_block_group(min_bg); 2731 } 2732 spin_unlock(&block_group->lock); 2733 } 2734 spin_unlock(&fs_info->zone_active_bgs_lock); 2735 2736 if (!min_bg) 2737 return 0; 2738 2739 ret = btrfs_zone_finish(min_bg); 2740 btrfs_put_block_group(min_bg); 2741 2742 return ret < 0 ? ret : 1; 2743 } 2744 2745 int btrfs_zoned_activate_one_bg(struct btrfs_fs_info *fs_info, 2746 struct btrfs_space_info *space_info, 2747 bool do_finish) 2748 { 2749 struct btrfs_block_group *bg; 2750 int index; 2751 2752 if (!btrfs_is_zoned(fs_info) || (space_info->flags & BTRFS_BLOCK_GROUP_DATA)) 2753 return 0; 2754 2755 for (;;) { 2756 int ret; 2757 bool need_finish = false; 2758 2759 down_read(&space_info->groups_sem); 2760 for (index = 0; index < BTRFS_NR_RAID_TYPES; index++) { 2761 list_for_each_entry(bg, &space_info->block_groups[index], 2762 list) { 2763 if (!spin_trylock(&bg->lock)) 2764 continue; 2765 if (btrfs_zoned_bg_is_full(bg) || 2766 test_bit(BLOCK_GROUP_FLAG_ZONE_IS_ACTIVE, 2767 &bg->runtime_flags)) { 2768 spin_unlock(&bg->lock); 2769 continue; 2770 } 2771 spin_unlock(&bg->lock); 2772 2773 if (btrfs_zone_activate(bg)) { 2774 up_read(&space_info->groups_sem); 2775 return 1; 2776 } 2777 2778 need_finish = true; 2779 } 2780 } 2781 up_read(&space_info->groups_sem); 2782 2783 if (!do_finish || !need_finish) 2784 break; 2785 2786 ret = btrfs_zone_finish_one_bg(fs_info); 2787 if (ret == 0) 2788 break; 2789 if (ret < 0) 2790 return ret; 2791 } 2792 2793 return 0; 2794 } 2795 2796 /* 2797 * Reserve zones for one metadata block group, one tree-log block group, and one 2798 * system block group. 2799 */ 2800 void btrfs_check_active_zone_reservation(struct btrfs_fs_info *fs_info) 2801 { 2802 struct btrfs_fs_devices *fs_devices = fs_info->fs_devices; 2803 struct btrfs_block_group *block_group; 2804 struct btrfs_device *device; 2805 /* Reserve zones for normal SINGLE metadata and tree-log block group. */ 2806 unsigned int metadata_reserve = 2; 2807 /* Reserve a zone for SINGLE system block group. */ 2808 unsigned int system_reserve = 1; 2809 2810 if (!test_bit(BTRFS_FS_ACTIVE_ZONE_TRACKING, &fs_info->flags)) 2811 return; 2812 2813 /* 2814 * This function is called from the mount context. So, there is no 2815 * parallel process touching the bits. No need for read_seqretry(). 2816 */ 2817 if (fs_info->avail_metadata_alloc_bits & BTRFS_BLOCK_GROUP_DUP) 2818 metadata_reserve = 4; 2819 if (fs_info->avail_system_alloc_bits & BTRFS_BLOCK_GROUP_DUP) 2820 system_reserve = 2; 2821 2822 /* Apply the reservation on all the devices. */ 2823 mutex_lock(&fs_devices->device_list_mutex); 2824 list_for_each_entry(device, &fs_devices->devices, dev_list) { 2825 if (!device->bdev) 2826 continue; 2827 2828 device->zone_info->reserved_active_zones = 2829 metadata_reserve + system_reserve; 2830 } 2831 mutex_unlock(&fs_devices->device_list_mutex); 2832 2833 /* Release reservation for currently active block groups. */ 2834 spin_lock(&fs_info->zone_active_bgs_lock); 2835 list_for_each_entry(block_group, &fs_info->zone_active_bgs, active_bg_list) { 2836 struct btrfs_chunk_map *map = block_group->physical_map; 2837 2838 if (!(block_group->flags & 2839 (BTRFS_BLOCK_GROUP_METADATA | BTRFS_BLOCK_GROUP_SYSTEM))) 2840 continue; 2841 2842 for (int i = 0; i < map->num_stripes; i++) 2843 map->stripes[i].dev->zone_info->reserved_active_zones--; 2844 } 2845 spin_unlock(&fs_info->zone_active_bgs_lock); 2846 } 2847 2848 /* 2849 * Reset the zones of unused block groups from @space_info->bytes_zone_unusable. 2850 * 2851 * @space_info: the space to work on 2852 * @num_bytes: targeting reclaim bytes 2853 * 2854 * This one resets the zones of a block group, so we can reuse the region 2855 * without removing the block group. On the other hand, btrfs_delete_unused_bgs() 2856 * just removes a block group and frees up the underlying zones. So, we still 2857 * need to allocate a new block group to reuse the zones. 2858 * 2859 * Resetting is faster than deleting/recreating a block group. It is similar 2860 * to freeing the logical space on the regular mode. However, we cannot change 2861 * the block group's profile with this operation. 2862 */ 2863 int btrfs_reset_unused_block_groups(struct btrfs_space_info *space_info, u64 num_bytes) 2864 { 2865 struct btrfs_fs_info *fs_info = space_info->fs_info; 2866 const sector_t zone_size_sectors = fs_info->zone_size >> SECTOR_SHIFT; 2867 2868 if (!btrfs_is_zoned(fs_info)) 2869 return 0; 2870 2871 while (num_bytes > 0) { 2872 struct btrfs_chunk_map *map; 2873 struct btrfs_block_group *bg = NULL; 2874 bool found = false; 2875 u64 reclaimed = 0; 2876 2877 /* 2878 * Here, we choose a fully zone_unusable block group. It's 2879 * technically possible to reset a partly zone_unusable block 2880 * group, which still has some free space left. However, 2881 * handling that needs to cope with the allocation side, which 2882 * makes the logic more complex. So, let's handle the easy case 2883 * for now. 2884 */ 2885 spin_lock(&fs_info->unused_bgs_lock); 2886 list_for_each_entry(bg, &fs_info->unused_bgs, bg_list) { 2887 if ((bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) != space_info->flags) 2888 continue; 2889 2890 /* 2891 * Use trylock to avoid locking order violation. In 2892 * btrfs_reclaim_bgs_work(), the lock order is 2893 * &bg->lock -> &fs_info->unused_bgs_lock. We skip a 2894 * block group if we cannot take its lock. 2895 */ 2896 if (!spin_trylock(&bg->lock)) 2897 continue; 2898 if (btrfs_is_block_group_used(bg) || bg->zone_unusable < bg->length) { 2899 spin_unlock(&bg->lock); 2900 continue; 2901 } 2902 spin_unlock(&bg->lock); 2903 found = true; 2904 break; 2905 } 2906 if (!found) { 2907 spin_unlock(&fs_info->unused_bgs_lock); 2908 return 0; 2909 } 2910 2911 list_del_init(&bg->bg_list); 2912 btrfs_put_block_group(bg); 2913 spin_unlock(&fs_info->unused_bgs_lock); 2914 2915 /* 2916 * Since the block group is fully zone_unusable and we cannot 2917 * allocate from this block group anymore, we don't need to set 2918 * this block group read-only. 2919 */ 2920 2921 down_read(&fs_info->dev_replace.rwsem); 2922 map = bg->physical_map; 2923 for (int i = 0; i < map->num_stripes; i++) { 2924 struct btrfs_io_stripe *stripe = &map->stripes[i]; 2925 unsigned int nofs_flags; 2926 int ret; 2927 2928 nofs_flags = memalloc_nofs_save(); 2929 ret = blkdev_zone_mgmt(stripe->dev->bdev, REQ_OP_ZONE_RESET, 2930 stripe->physical >> SECTOR_SHIFT, 2931 zone_size_sectors); 2932 memalloc_nofs_restore(nofs_flags); 2933 2934 if (ret) { 2935 up_read(&fs_info->dev_replace.rwsem); 2936 return ret; 2937 } 2938 } 2939 up_read(&fs_info->dev_replace.rwsem); 2940 2941 spin_lock(&space_info->lock); 2942 spin_lock(&bg->lock); 2943 ASSERT(!btrfs_is_block_group_used(bg)); 2944 if (bg->ro) { 2945 spin_unlock(&bg->lock); 2946 spin_unlock(&space_info->lock); 2947 continue; 2948 } 2949 2950 reclaimed = bg->alloc_offset; 2951 bg->zone_unusable = bg->length - bg->zone_capacity; 2952 bg->alloc_offset = 0; 2953 /* 2954 * This holds because we currently reset fully used then freed 2955 * block group. 2956 */ 2957 ASSERT(reclaimed == bg->zone_capacity); 2958 bg->free_space_ctl->free_space += reclaimed; 2959 space_info->bytes_zone_unusable -= reclaimed; 2960 spin_unlock(&bg->lock); 2961 btrfs_return_free_space(space_info, reclaimed); 2962 spin_unlock(&space_info->lock); 2963 2964 if (num_bytes <= reclaimed) 2965 break; 2966 num_bytes -= reclaimed; 2967 } 2968 2969 return 0; 2970 } 2971